zlei6/ragflow
1
0
Fork 0
mirror of https://github.com/infiniflow/ragflow.git synced 2026-08-19 06:48:25 +08:00
Code Issues Packages Projects Releases Wiki Activity

Refactor: do not include compliation chunks in retrieval testing (#18364)

Browse Source 
Retrieval Testing: do not include compilation result.
Search: include compilation result.
This commit is contained in:
Wang Qi
2026-08-17 16:19:05 +08:00
committed by GitHub
parent 3e38b6cacb
commit 57b907cd1e
11 changed files with 47 additions and 5 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
api/apps/restful_apis/dataset_api.py
View File

@@ -515,7 +515,7 @@ async def search_datasets(tenant_id):
POST /api/v1/datasets/search
JSON body: {"dataset_ids": list[str] (required), "question": str (required), "doc_ids": list[str], "top_k": int, "page": int, "size": int,
"similarity_threshold": float, "vector_similarity_weight": float, "use_kg": bool,
"cross_languages": list[str], "keyword": bool, "meta_data_filter": dict}
"cross_languages": list[str], "keyword": bool, "meta_data_filter": dict, "include_knowledge_compilation": bool (default true)}
Success: {"code": 0, "data": {"chunks": [...], "total": int, "labels": [...]}}
Errors: ARGUMENT_ERROR (101) for invalid payload; DATA_ERROR (102) for access denied or internal errors.
"""
@@ -538,7 +538,7 @@ async def search(tenant_id, dataset_id):
POST /api/v1/datasets/<dataset_id>/search
JSON body: {"question": str (required), "doc_ids": list[str], "top_k": int, "page": int, "size": int,
"similarity_threshold": float, "vector_similarity_weight": float, "use_kg": bool,
"cross_languages": list[str], "keyword": bool, "meta_data_filter": dict}
"cross_languages": list[str], "keyword": bool, "meta_data_filter": dict, "include_knowledge_compilation": bool (default true)}
Success: {"code": 0, "data": {"chunks": [...], "total": int, "labels": [...]}}
Errors: ARGUMENT_ERROR (101) for invalid payload; DATA_ERROR (102) for access denied or internal errors.
"""

1
api/apps/services/dataset_api_service.py
View File

@@ -1530,6 +1530,7 @@ async def search_datasets(tenant_id: str, req: dict):
rerank_mdl=rerank_mdl,
rank_feature=labels,
trace_id=search_id,
must_not=None if req.get("include_knowledge_compilation", True) else {"exists": "compile_kwd"},
)
if use_kg:

2
api/utils/validation_utils.py
View File

@@ -992,6 +992,7 @@ class SearchDatasetReq(BaseModel):
rerank_id: Annotated[str | None, Field(default=None)]
tenant_rerank_id: Annotated[str | None, Field(default=None)]
meta_data_filter: Annotated[dict | None, Field(default=None)]
include_knowledge_compilation: Annotated[bool, Field(default=True)]
class SearchDatasetsReq(BaseModel):
@@ -1014,6 +1015,7 @@ class SearchDatasetsReq(BaseModel):
rerank_id: Annotated[str | None, Field(default=None)]
tenant_rerank_id: Annotated[str | None, Field(default=None)]
meta_data_filter: Annotated[dict | None, Field(default=None)]
include_knowledge_compilation: Annotated[bool, Field(default=True)]
class BaseListReq(BaseModel):

12
internal/engine/elasticsearch/chunk.go
View File

@@ -1679,6 +1679,7 @@ func memoryMessageStatusBool(value interface{}) bool {
// message indexes use memory_id plus message-specific storage fields.
func buildBoolQueryFromCondition(filter map[string]interface{}, kbIDs []string, isSkillIndex, isMemoryIndex bool) map[string]interface{} {
var mustClauses []interface{}
var mustNotClauses []interface{}
var filterClauses []interface{}
var shouldClauses []interface{}
@@ -1753,6 +1754,14 @@ func buildBoolQueryFromCondition(filter map[string]interface{}, kbIDs []string,
}
continue
}
if k == "must_not" {
if condition, ok := v.(map[string]interface{}); ok {
if field, ok := condition["exists"].(string); ok && field != "" {
mustNotClauses = append(mustNotClauses, map[string]interface{}{"exists": map[string]interface{}{"field": field}})
}
}
continue
}
if k == "id" {
if v == nil || v == "" {
continue
@@ -1818,6 +1827,9 @@ func buildBoolQueryFromCondition(filter map[string]interface{}, kbIDs []string,
if len(mustClauses) > 0 {
boolQuery["must"] = mustClauses
}
if len(mustNotClauses) > 0 {
boolQuery["must_not"] = mustNotClauses
}
if len(filterClauses) > 0 {
boolQuery["filter"] = filterClauses
}

5
internal/handler/dataset_search_test.go
View File

@@ -157,7 +157,7 @@ func TestDatasetsHandlerSearchDatasetsSuccess(t *testing.T) {
h := &DatasetsHandler{searchDatasetsService: fake}
rec := httptest.NewRecorder()
req := httptest.NewRequest(http.MethodPost, "/api/v1/datasets/search", strings.NewReader(`{"question":" hello ","dataset_ids":["ds-1"],"top_k":7}`))
req := httptest.NewRequest(http.MethodPost, "/api/v1/datasets/search", strings.NewReader(`{"question":" hello ","dataset_ids":["ds-1"],"top_k":7,"include_knowledge_compilation":false}`))
req.Header.Set("Content-Type", "application/json")
c, _ := gin.CreateTestContext(rec)
c.Request = req
@@ -171,6 +171,9 @@ func TestDatasetsHandlerSearchDatasetsSuccess(t *testing.T) {
if fake.userID != "user-1" || fake.req == nil || fake.req.Question != "hello" || len(fake.req.DatasetIDs) != 1 || fake.req.DatasetIDs[0] != "ds-1" {
t.Fatalf("call args userID=%q req=%#v", fake.userID, fake.req)
}
if fake.req.IncludeCompiledChunks == nil || *fake.req.IncludeCompiledChunks {
t.Fatalf("include_knowledge_compilation=%v want false", fake.req.IncludeCompiledChunks)
}
body := decodeSearchResponse(t, rec)
if body["code"] != float64(common.CodeSuccess) {
t.Fatalf("code=%v want=%d body=%s", body["code"], common.CodeSuccess, rec.Body.String())

3
internal/service/dataset/search.go
View File

@@ -270,6 +270,9 @@ func (d *DatasetService) SearchDatasets(ctx context.Context, req *service.Search
RankFeature: &labels,
EmbeddingModel: embeddingModel,
}
if req.IncludeCompiledChunks != nil && !*req.IncludeCompiledChunks {
retrievalReq.Filter = map[string]interface{}{"must_not": map[string]interface{}{"exists": "compile_kwd"}}
}
retrievalResult, err := nlp.NewRetrievalService(d.docEngine, d.documentDAO).Retrieval(ctx, retrievalReq)
if err != nil {

5
internal/service/dataset/search_test.go
View File

@@ -16,6 +16,7 @@ func TestSearchDatasetRequestToSearchDatasetsRequest(t *testing.T) {
vectorSimilarityWeight := 0.8
searchID := "search-1"
rerankID := "rerank-1"
includeKnowledgeCompilation := false
req := &service.SearchDatasetRequest{
Question: "hello world",
Page: &page,
@@ -30,6 +31,7 @@ func TestSearchDatasetRequestToSearchDatasetsRequest(t *testing.T) {
Keyword: &keyword,
SimilarityThreshold: &similarityThreshold,
VectorSimilarityWeight: &vectorSimilarityWeight,
IncludeCompiledChunks: &includeKnowledgeCompilation,
}
converted := req.ToSearchDatasetsRequest("dataset-1")
@@ -51,4 +53,7 @@ func TestSearchDatasetRequestToSearchDatasetsRequest(t *testing.T) {
if converted.SimilarityThreshold != req.SimilarityThreshold || converted.VectorSimilarityWeight != req.VectorSimilarityWeight {
t.Fatalf("converted request did not preserve thresholds: %#v", converted)
}
if converted.IncludeCompiledChunks != req.IncludeCompiledChunks {
t.Fatalf("converted request did not preserve include_knowledge_compilation: %#v", converted)
}
}

3
internal/service/dataset_types.go
View File

@@ -51,6 +51,7 @@ type SearchDatasetsRequest struct {
Keyword *bool `json:"keyword,omitempty"`
SimilarityThreshold *float64 `json:"similarity_threshold,omitempty"`
VectorSimilarityWeight *float64 `json:"vector_similarity_weight,omitempty"`
IncludeCompiledChunks *bool `json:"include_knowledge_compilation,omitempty"`
ForceRefresh bool `json:"force_refresh"`
}
@@ -77,6 +78,7 @@ type SearchDatasetRequest struct {
Keyword *bool `json:"keyword,omitempty"`
SimilarityThreshold *float64 `json:"similarity_threshold,omitempty"`
VectorSimilarityWeight *float64 `json:"vector_similarity_weight,omitempty"`
IncludeCompiledChunks *bool `json:"include_knowledge_compilation,omitempty"`
}
// ToSearchDatasetsRequest converts a single-dataset search request into the multi-dataset form.
@@ -99,6 +101,7 @@ func (req *SearchDatasetRequest) ToSearchDatasetsRequest(datasetID string) *Sear
Keyword: req.Keyword,
SimilarityThreshold: req.SimilarityThreshold,
VectorSimilarityWeight: req.VectorSimilarityWeight,
IncludeCompiledChunks: req.IncludeCompiledChunks,
}
}

3
internal/service/nlp/retrieval.go
View File

@@ -61,6 +61,7 @@ type RetrievalRequest struct {
EmbeddingModel *models.EmbeddingModel
Aggs *bool
Highlight *bool
Filter map[string]interface{}
}
// RetrievalResult result from retrieval search
@@ -145,6 +146,7 @@ func (s *RetrievalService) Retrieval(ctx context.Context, req *RetrievalRequest)
Top: *req.Top,
RankFeature: *req.RankFeature,
EmbeddingModel: req.EmbeddingModel,
Filter: req.Filter,
}
searchResult, err := s.Search(ctx, searchReq)
if err != nil {
@@ -434,6 +436,7 @@ func (s *RetrievalService) countThresholdValidMatches(ctx context.Context, req *
Top: *req.Top,
RankFeature: *req.RankFeature,
EmbeddingModel: req.EmbeddingModel,
Filter: req.Filter,
}
searchResult, err := s.Search(ctx, searchReq)
if err != nil {

13
internal/service/nlp/retrieval_test.go
View File

@@ -41,6 +41,7 @@ func TestRetrievalTotalCountsThresholdValidMatchesBeyondRerankWindow(t *testing.
SimilarityThreshold: &threshold,
VectorSimilarityWeight: &vectorWeight,
Aggs: &aggs,
Filter: map[string]interface{}{"must_not": map[string]interface{}{"exists": "compile_kwd"}},
})
if err != nil {
t.Fatalf("Retrieval failed: %v", err)
@@ -54,15 +55,23 @@ func TestRetrievalTotalCountsThresholdValidMatchesBeyondRerankWindow(t *testing.
if len(engine.searchLimits) != 2 || engine.searchLimits[0] != 70 || engine.searchLimits[1] != 100 {
t.Fatalf("search limits = %v, want [70 100]", engine.searchLimits)
}
for _, filters := range engine.searchFilters {
mustNot, ok := filters["must_not"].(map[string]interface{})
if !ok || mustNot["exists"] != "compile_kwd" {
t.Fatalf("must_not filter = %#v", filters["must_not"])
}
}
}
type retrievalCountEngine struct {
rows []map[string]interface{}
searchLimits []int
rows []map[string]interface{}
searchLimits []int
searchFilters []map[string]interface{}
}
func (e *retrievalCountEngine) Search(_ context.Context, req *types.SearchRequest) (*types.SearchResult, error) {
e.searchLimits = append(e.searchLimits, req.Limit)
e.searchFilters = append(e.searchFilters, req.Filter)
offset := req.Offset
if offset > len(e.rows) {
offset = len(e.rows)

1
web/src/hooks/use-knowledge-request.ts
View File

@@ -136,6 +136,7 @@ export const useTestRetrieval = () => {
page: 1,
doc_ids: filterValue.doc_ids,
highlight: true,
include_knowledge_compilation: false,
};
}, [filterValue, knowledgeBaseId, values]);