From 57b907cd1ef1d7deea40680ed45d2b970b417de2 Mon Sep 17 00:00:00 2001 From: Wang Qi Date: Mon, 17 Aug 2026 16:19:05 +0800 Subject: [PATCH] Refactor: do not include compliation chunks in retrieval testing (#18364) Retrieval Testing: do not include compilation result. Search: include compilation result. --- api/apps/restful_apis/dataset_api.py | 4 ++-- api/apps/services/dataset_api_service.py | 1 + api/utils/validation_utils.py | 2 ++ internal/engine/elasticsearch/chunk.go | 12 ++++++++++++ internal/handler/dataset_search_test.go | 5 ++++- internal/service/dataset/search.go | 3 +++ internal/service/dataset/search_test.go | 5 +++++ internal/service/dataset_types.go | 3 +++ internal/service/nlp/retrieval.go | 3 +++ internal/service/nlp/retrieval_test.go | 13 +++++++++++-- web/src/hooks/use-knowledge-request.ts | 1 + 11 files changed, 47 insertions(+), 5 deletions(-) diff --git a/api/apps/restful_apis/dataset_api.py b/api/apps/restful_apis/dataset_api.py index 4cc0f969c8..341f01b683 100644 --- a/api/apps/restful_apis/dataset_api.py +++ b/api/apps/restful_apis/dataset_api.py @@ -515,7 +515,7 @@ async def search_datasets(tenant_id): POST /api/v1/datasets/search JSON body: {"dataset_ids": list[str] (required), "question": str (required), "doc_ids": list[str], "top_k": int, "page": int, "size": int, "similarity_threshold": float, "vector_similarity_weight": float, "use_kg": bool, - "cross_languages": list[str], "keyword": bool, "meta_data_filter": dict} + "cross_languages": list[str], "keyword": bool, "meta_data_filter": dict, "include_knowledge_compilation": bool (default true)} Success: {"code": 0, "data": {"chunks": [...], "total": int, "labels": [...]}} Errors: ARGUMENT_ERROR (101) for invalid payload; DATA_ERROR (102) for access denied or internal errors. """ @@ -538,7 +538,7 @@ async def search(tenant_id, dataset_id): POST /api/v1/datasets//search JSON body: {"question": str (required), "doc_ids": list[str], "top_k": int, "page": int, "size": int, "similarity_threshold": float, "vector_similarity_weight": float, "use_kg": bool, - "cross_languages": list[str], "keyword": bool, "meta_data_filter": dict} + "cross_languages": list[str], "keyword": bool, "meta_data_filter": dict, "include_knowledge_compilation": bool (default true)} Success: {"code": 0, "data": {"chunks": [...], "total": int, "labels": [...]}} Errors: ARGUMENT_ERROR (101) for invalid payload; DATA_ERROR (102) for access denied or internal errors. """ diff --git a/api/apps/services/dataset_api_service.py b/api/apps/services/dataset_api_service.py index a5cb969f41..f71b973bc3 100644 --- a/api/apps/services/dataset_api_service.py +++ b/api/apps/services/dataset_api_service.py @@ -1530,6 +1530,7 @@ async def search_datasets(tenant_id: str, req: dict): rerank_mdl=rerank_mdl, rank_feature=labels, trace_id=search_id, + must_not=None if req.get("include_knowledge_compilation", True) else {"exists": "compile_kwd"}, ) if use_kg: diff --git a/api/utils/validation_utils.py b/api/utils/validation_utils.py index ddb455762c..054c6f19c5 100644 --- a/api/utils/validation_utils.py +++ b/api/utils/validation_utils.py @@ -992,6 +992,7 @@ class SearchDatasetReq(BaseModel): rerank_id: Annotated[str | None, Field(default=None)] tenant_rerank_id: Annotated[str | None, Field(default=None)] meta_data_filter: Annotated[dict | None, Field(default=None)] + include_knowledge_compilation: Annotated[bool, Field(default=True)] class SearchDatasetsReq(BaseModel): @@ -1014,6 +1015,7 @@ class SearchDatasetsReq(BaseModel): rerank_id: Annotated[str | None, Field(default=None)] tenant_rerank_id: Annotated[str | None, Field(default=None)] meta_data_filter: Annotated[dict | None, Field(default=None)] + include_knowledge_compilation: Annotated[bool, Field(default=True)] class BaseListReq(BaseModel): diff --git a/internal/engine/elasticsearch/chunk.go b/internal/engine/elasticsearch/chunk.go index ae98fa1371..9255d1f29c 100644 --- a/internal/engine/elasticsearch/chunk.go +++ b/internal/engine/elasticsearch/chunk.go @@ -1679,6 +1679,7 @@ func memoryMessageStatusBool(value interface{}) bool { // message indexes use memory_id plus message-specific storage fields. func buildBoolQueryFromCondition(filter map[string]interface{}, kbIDs []string, isSkillIndex, isMemoryIndex bool) map[string]interface{} { var mustClauses []interface{} + var mustNotClauses []interface{} var filterClauses []interface{} var shouldClauses []interface{} @@ -1753,6 +1754,14 @@ func buildBoolQueryFromCondition(filter map[string]interface{}, kbIDs []string, } continue } + if k == "must_not" { + if condition, ok := v.(map[string]interface{}); ok { + if field, ok := condition["exists"].(string); ok && field != "" { + mustNotClauses = append(mustNotClauses, map[string]interface{}{"exists": map[string]interface{}{"field": field}}) + } + } + continue + } if k == "id" { if v == nil || v == "" { continue @@ -1818,6 +1827,9 @@ func buildBoolQueryFromCondition(filter map[string]interface{}, kbIDs []string, if len(mustClauses) > 0 { boolQuery["must"] = mustClauses } + if len(mustNotClauses) > 0 { + boolQuery["must_not"] = mustNotClauses + } if len(filterClauses) > 0 { boolQuery["filter"] = filterClauses } diff --git a/internal/handler/dataset_search_test.go b/internal/handler/dataset_search_test.go index 9eaf94773c..2163da217e 100644 --- a/internal/handler/dataset_search_test.go +++ b/internal/handler/dataset_search_test.go @@ -157,7 +157,7 @@ func TestDatasetsHandlerSearchDatasetsSuccess(t *testing.T) { h := &DatasetsHandler{searchDatasetsService: fake} rec := httptest.NewRecorder() - req := httptest.NewRequest(http.MethodPost, "/api/v1/datasets/search", strings.NewReader(`{"question":" hello ","dataset_ids":["ds-1"],"top_k":7}`)) + req := httptest.NewRequest(http.MethodPost, "/api/v1/datasets/search", strings.NewReader(`{"question":" hello ","dataset_ids":["ds-1"],"top_k":7,"include_knowledge_compilation":false}`)) req.Header.Set("Content-Type", "application/json") c, _ := gin.CreateTestContext(rec) c.Request = req @@ -171,6 +171,9 @@ func TestDatasetsHandlerSearchDatasetsSuccess(t *testing.T) { if fake.userID != "user-1" || fake.req == nil || fake.req.Question != "hello" || len(fake.req.DatasetIDs) != 1 || fake.req.DatasetIDs[0] != "ds-1" { t.Fatalf("call args userID=%q req=%#v", fake.userID, fake.req) } + if fake.req.IncludeCompiledChunks == nil || *fake.req.IncludeCompiledChunks { + t.Fatalf("include_knowledge_compilation=%v want false", fake.req.IncludeCompiledChunks) + } body := decodeSearchResponse(t, rec) if body["code"] != float64(common.CodeSuccess) { t.Fatalf("code=%v want=%d body=%s", body["code"], common.CodeSuccess, rec.Body.String()) diff --git a/internal/service/dataset/search.go b/internal/service/dataset/search.go index 4394af649e..5549365d5a 100644 --- a/internal/service/dataset/search.go +++ b/internal/service/dataset/search.go @@ -270,6 +270,9 @@ func (d *DatasetService) SearchDatasets(ctx context.Context, req *service.Search RankFeature: &labels, EmbeddingModel: embeddingModel, } + if req.IncludeCompiledChunks != nil && !*req.IncludeCompiledChunks { + retrievalReq.Filter = map[string]interface{}{"must_not": map[string]interface{}{"exists": "compile_kwd"}} + } retrievalResult, err := nlp.NewRetrievalService(d.docEngine, d.documentDAO).Retrieval(ctx, retrievalReq) if err != nil { diff --git a/internal/service/dataset/search_test.go b/internal/service/dataset/search_test.go index 1b90d535d6..b898964d6a 100644 --- a/internal/service/dataset/search_test.go +++ b/internal/service/dataset/search_test.go @@ -16,6 +16,7 @@ func TestSearchDatasetRequestToSearchDatasetsRequest(t *testing.T) { vectorSimilarityWeight := 0.8 searchID := "search-1" rerankID := "rerank-1" + includeKnowledgeCompilation := false req := &service.SearchDatasetRequest{ Question: "hello world", Page: &page, @@ -30,6 +31,7 @@ func TestSearchDatasetRequestToSearchDatasetsRequest(t *testing.T) { Keyword: &keyword, SimilarityThreshold: &similarityThreshold, VectorSimilarityWeight: &vectorSimilarityWeight, + IncludeCompiledChunks: &includeKnowledgeCompilation, } converted := req.ToSearchDatasetsRequest("dataset-1") @@ -51,4 +53,7 @@ func TestSearchDatasetRequestToSearchDatasetsRequest(t *testing.T) { if converted.SimilarityThreshold != req.SimilarityThreshold || converted.VectorSimilarityWeight != req.VectorSimilarityWeight { t.Fatalf("converted request did not preserve thresholds: %#v", converted) } + if converted.IncludeCompiledChunks != req.IncludeCompiledChunks { + t.Fatalf("converted request did not preserve include_knowledge_compilation: %#v", converted) + } } diff --git a/internal/service/dataset_types.go b/internal/service/dataset_types.go index 0864d66aa1..6d61253eba 100644 --- a/internal/service/dataset_types.go +++ b/internal/service/dataset_types.go @@ -51,6 +51,7 @@ type SearchDatasetsRequest struct { Keyword *bool `json:"keyword,omitempty"` SimilarityThreshold *float64 `json:"similarity_threshold,omitempty"` VectorSimilarityWeight *float64 `json:"vector_similarity_weight,omitempty"` + IncludeCompiledChunks *bool `json:"include_knowledge_compilation,omitempty"` ForceRefresh bool `json:"force_refresh"` } @@ -77,6 +78,7 @@ type SearchDatasetRequest struct { Keyword *bool `json:"keyword,omitempty"` SimilarityThreshold *float64 `json:"similarity_threshold,omitempty"` VectorSimilarityWeight *float64 `json:"vector_similarity_weight,omitempty"` + IncludeCompiledChunks *bool `json:"include_knowledge_compilation,omitempty"` } // ToSearchDatasetsRequest converts a single-dataset search request into the multi-dataset form. @@ -99,6 +101,7 @@ func (req *SearchDatasetRequest) ToSearchDatasetsRequest(datasetID string) *Sear Keyword: req.Keyword, SimilarityThreshold: req.SimilarityThreshold, VectorSimilarityWeight: req.VectorSimilarityWeight, + IncludeCompiledChunks: req.IncludeCompiledChunks, } } diff --git a/internal/service/nlp/retrieval.go b/internal/service/nlp/retrieval.go index 50ae32467f..63b5a21653 100644 --- a/internal/service/nlp/retrieval.go +++ b/internal/service/nlp/retrieval.go @@ -61,6 +61,7 @@ type RetrievalRequest struct { EmbeddingModel *models.EmbeddingModel Aggs *bool Highlight *bool + Filter map[string]interface{} } // RetrievalResult result from retrieval search @@ -145,6 +146,7 @@ func (s *RetrievalService) Retrieval(ctx context.Context, req *RetrievalRequest) Top: *req.Top, RankFeature: *req.RankFeature, EmbeddingModel: req.EmbeddingModel, + Filter: req.Filter, } searchResult, err := s.Search(ctx, searchReq) if err != nil { @@ -434,6 +436,7 @@ func (s *RetrievalService) countThresholdValidMatches(ctx context.Context, req * Top: *req.Top, RankFeature: *req.RankFeature, EmbeddingModel: req.EmbeddingModel, + Filter: req.Filter, } searchResult, err := s.Search(ctx, searchReq) if err != nil { diff --git a/internal/service/nlp/retrieval_test.go b/internal/service/nlp/retrieval_test.go index 531616ecb0..1cb7f782d1 100644 --- a/internal/service/nlp/retrieval_test.go +++ b/internal/service/nlp/retrieval_test.go @@ -41,6 +41,7 @@ func TestRetrievalTotalCountsThresholdValidMatchesBeyondRerankWindow(t *testing. SimilarityThreshold: &threshold, VectorSimilarityWeight: &vectorWeight, Aggs: &aggs, + Filter: map[string]interface{}{"must_not": map[string]interface{}{"exists": "compile_kwd"}}, }) if err != nil { t.Fatalf("Retrieval failed: %v", err) @@ -54,15 +55,23 @@ func TestRetrievalTotalCountsThresholdValidMatchesBeyondRerankWindow(t *testing. if len(engine.searchLimits) != 2 || engine.searchLimits[0] != 70 || engine.searchLimits[1] != 100 { t.Fatalf("search limits = %v, want [70 100]", engine.searchLimits) } + for _, filters := range engine.searchFilters { + mustNot, ok := filters["must_not"].(map[string]interface{}) + if !ok || mustNot["exists"] != "compile_kwd" { + t.Fatalf("must_not filter = %#v", filters["must_not"]) + } + } } type retrievalCountEngine struct { - rows []map[string]interface{} - searchLimits []int + rows []map[string]interface{} + searchLimits []int + searchFilters []map[string]interface{} } func (e *retrievalCountEngine) Search(_ context.Context, req *types.SearchRequest) (*types.SearchResult, error) { e.searchLimits = append(e.searchLimits, req.Limit) + e.searchFilters = append(e.searchFilters, req.Filter) offset := req.Offset if offset > len(e.rows) { offset = len(e.rows) diff --git a/web/src/hooks/use-knowledge-request.ts b/web/src/hooks/use-knowledge-request.ts index e35c38236a..1302179ea2 100644 --- a/web/src/hooks/use-knowledge-request.ts +++ b/web/src/hooks/use-knowledge-request.ts @@ -136,6 +136,7 @@ export const useTestRetrieval = () => { page: 1, doc_ids: filterValue.doc_ids, highlight: true, + include_knowledge_compilation: false, }; }, [filterValue, knowledgeBaseId, values]);