diff --git a/api/apps/chunk_app.py b/api/apps/chunk_app.py index e6ceb66e69..2387dd5d01 100644 --- a/api/apps/chunk_app.py +++ b/api/apps/chunk_app.py @@ -498,6 +498,15 @@ async def retrieval_test(): _question += await keyword_extraction(chat_mdl, _question) labels = label_question(_question, [kb]) + highlight_val = req.get("highlight", None) + if highlight_val is None: + highlight = False + elif isinstance(highlight_val, bool): + highlight = highlight_val + elif isinstance(highlight_val, str): + highlight = highlight_val.lower() in ("true", "1", "yes", "on") + else: + highlight = bool(highlight_val) ranks = await settings.retriever.retrieval( _question, embd_mdl, @@ -510,6 +519,7 @@ async def retrieval_test(): doc_ids=local_doc_ids, top=top, rerank_mdl=rerank_mdl, + highlight=highlight, rank_feature=labels ) diff --git a/rag/nlp/search.py b/rag/nlp/search.py index 7ad19fe7c4..5b1e88ad9c 100644 --- a/rag/nlp/search.py +++ b/rag/nlp/search.py @@ -654,6 +654,7 @@ class Dealer: chunk = self.dataStore.get(cid, idx_nms[0], kb_ids) if not chunk: continue + has_highlights = any(ck.get("highlight") for ck in chunks) d = { "chunk_id": cid, "content_ltks": chunk["content_ltks"], @@ -670,6 +671,8 @@ class Dealer: "positions": chunk.get("position_int", []), "doc_type_kwd": chunk.get("doc_type_kwd", "") } + if has_highlights: + d["highlight"] = chunk["content_with_weight"] for k in chunk.keys(): if k[-4:] == "_vec": d["vector"] = chunk[k] @@ -702,6 +705,7 @@ class Dealer: vector_size = 1024 for id, cks in mom_chunks.items(): chunk = self.dataStore.get(id, idx_nms[0], [ck["kb_id"] for ck in cks]) + child_highlights = [ck["highlight"] for ck in cks if ck.get("highlight")] d = { "chunk_id": id, "content_ltks": " ".join([ck["content_ltks"] for ck in cks]), @@ -718,6 +722,8 @@ class Dealer: "positions": chunk.get("position_int", []), "doc_type_kwd": chunk.get("doc_type_kwd", "") } + if child_highlights: + d["highlight"] = " ".join(child_highlights) for k in cks[0].keys(): if k[-4:] == "_vec": d["vector"] = cks[0][k] diff --git a/test/testcases/test_web_api/test_chunk_app/test_retrieval_chunks.py b/test/testcases/test_web_api/test_chunk_app/test_retrieval_chunks.py index 14857210f4..0296bd25f8 100644 --- a/test/testcases/test_web_api/test_chunk_app/test_retrieval_chunks.py +++ b/test/testcases/test_web_api/test_chunk_app/test_retrieval_chunks.py @@ -265,8 +265,8 @@ class TestChunksRetrieval: @pytest.mark.parametrize( "payload, expected_code, expected_highlight, expected_message", [ - pytest.param({"highlight": True}, 0, True, "", marks=pytest.mark.skip(reason="highlight not functionnal")), - pytest.param({"highlight": "True"}, 0, True, "", marks=pytest.mark.skip(reason="highlight not functionnal")), + ({"highlight": True}, 0, True, ""), + ({"highlight": "True"}, 0, True, ""), ({"highlight": False}, 0, False, ""), ({"highlight": "False"}, 0, False, ""), ({"highlight": None}, 0, False, "") diff --git a/web/src/hooks/use-knowledge-request.ts b/web/src/hooks/use-knowledge-request.ts index 715707e59c..59e12cad57 100644 --- a/web/src/hooks/use-knowledge-request.ts +++ b/web/src/hooks/use-knowledge-request.ts @@ -625,6 +625,7 @@ export const useTestChunkRetrieval = (): ResponsePostType & { const { data } = await kbService.retrievalTest({ ...values, kb_id: values.kb_id ?? knowledgeBaseId, + highlight: true, page, size: pageSize, }); @@ -669,6 +670,7 @@ export const useTestChunkAllRetrieval = (): ResponsePostType & { const { data } = await kbService.retrievalTest({ ...values, kb_id: values.kb_id ?? knowledgeBaseId, + highlight: true, doc_ids: [], page, size: pageSize, diff --git a/web/src/less/mixins.less b/web/src/less/mixins.less index c77eac838c..a58268a3a5 100644 --- a/web/src/less/mixins.less +++ b/web/src/less/mixins.less @@ -5,7 +5,7 @@ .chunkText() { em { - color: var(--accent-primary); + color: rgb(var(--accent-primary)); font-style: normal; } table { diff --git a/web/src/pages/dataset/testing/testing-result.tsx b/web/src/pages/dataset/testing/testing-result.tsx index 5dceefee05..fdf7d83a1c 100644 --- a/web/src/pages/dataset/testing/testing-result.tsx +++ b/web/src/pages/dataset/testing/testing-result.tsx @@ -1,5 +1,6 @@ import { EmptyType } from '@/components/empty/constant'; import Empty from '@/components/empty/empty'; +import HighLightMarkdown from '@/components/highlight-markdown'; import { FilterButton } from '@/components/list-filter-bar'; import { FilterPopover } from '@/components/list-filter-bar/filter-popover'; import { FilterCollection } from '@/components/list-filter-bar/interface'; @@ -91,7 +92,11 @@ export function TestingResult({
-

{x.content_with_weight}

+
+ + {x.highlight || x.content_with_weight} + +
))} diff --git a/web/src/pages/next-search/search-view.tsx b/web/src/pages/next-search/search-view.tsx index 388d78605e..c1a85b4f28 100644 --- a/web/src/pages/next-search/search-view.tsx +++ b/web/src/pages/next-search/search-view.tsx @@ -201,7 +201,7 @@ export default function SearchingView({ > )} - {chunk.content_with_weight} + {chunk.highlight || chunk.content_with_weight}