From 0108cdac67604aa53315a36821eee41329aa154f Mon Sep 17 00:00:00 2001 From: Haruko386 Date: Fri, 31 Jul 2026 16:55:34 +0800 Subject: [PATCH] fix: write wrong type of metadata (#17605) --- internal/engine/elasticsearch/metadata.go | 13 ++++- internal/handler/document.go | 3 -- .../service/document/document_metadata.go | 50 +++++++++++++++++++ .../document/document_metadata_test.go | 29 +++++++++++ internal/service/document/document_test.go | 19 ++++++- 5 files changed, 109 insertions(+), 5 deletions(-) diff --git a/internal/engine/elasticsearch/metadata.go b/internal/engine/elasticsearch/metadata.go index acc64d8dd6..f997e9f5e2 100644 --- a/internal/engine/elasticsearch/metadata.go +++ b/internal/engine/elasticsearch/metadata.go @@ -22,6 +22,7 @@ import ( "encoding/json" "fmt" "io" + "os" "strings" "github.com/elastic/go-elasticsearch/v8/esapi" @@ -30,6 +31,7 @@ import ( "ragflow/internal/common" "ragflow/internal/dao" "ragflow/internal/engine/types" + "ragflow/internal/utility" "go.uber.org/zap" ) @@ -47,9 +49,18 @@ func (e *elasticsearchEngine) CreateMetadataStore(ctx context.Context, tenantID return nil } - // Index will be created with mapping from index template (ragflow_doc_meta_mapping) + mappingPath, err := utility.FindConfFileInProject("doc_meta_es_mapping.json") + if err != nil { + return err + } + data, err := os.ReadFile(*mappingPath) + if err != nil { + return fmt.Errorf("failed to read mapping file %q: %w", *mappingPath, err) + } + req := esapi.IndicesCreateRequest{ Index: indexName, + Body: bytes.NewReader(data), } res, err := req.Do(ctx, e.client) if err != nil { diff --git a/internal/handler/document.go b/internal/handler/document.go index b1197dca5f..b641661902 100644 --- a/internal/handler/document.go +++ b/internal/handler/document.go @@ -577,9 +577,6 @@ func (h *DocumentHandler) ListDocuments(c *gin.Context) { return } - // Mirror Python: an explicit id/name filter that matches nothing is an - // ownership error rather than an empty page. Each filter's existence is - // checked independently of the other. if docID := c.Query("id"); docID != "" { idOpts := opts idOpts.DocIDs = []string{docID} diff --git a/internal/service/document/document_metadata.go b/internal/service/document/document_metadata.go index a749fd252c..ea5d500124 100644 --- a/internal/service/document/document_metadata.go +++ b/internal/service/document/document_metadata.go @@ -54,6 +54,7 @@ func (s *DocumentService) SetDocumentMetadata(ctx context.Context, docID string, return fmt.Errorf("failed to ensure metadata store: %w", err) } + meta = splitCombinedDocumentMetadataValues(meta) if err = s.docEngine.UpdateMetadata(ctx, docID, doc.KbID, meta, tenantID); err != nil { return fmt.Errorf("failed to update metadata: %w", err) } @@ -711,6 +712,55 @@ func cloneDocumentMetadata(meta map[string]interface{}) map[string]interface{} { return cloned } +func splitCombinedDocumentMetadataValues(meta map[string]any) map[string]any { + if len(meta) == 0 { + return meta + } + out := make(map[string]any, len(meta)) + for key, value := range meta { + switch typed := value.(type) { + case []interface{}: + out[key] = splitCombinedDocumentMetadataList(typed) + case []string: + items := make([]any, 0, len(typed)) + for _, item := range typed { + items = append(items, item) + } + out[key] = splitCombinedDocumentMetadataList(items) + default: + out[key] = value + } + } + return out +} + +var combinedDocumentMetadataValueSplitter = regexp.MustCompile(`[、,，;；|]+`) + +func splitCombinedDocumentMetadataList(items []any) []any { + out := make([]interface{}, 0, len(items)) + for _, item := range items { + text, ok := item.(string) + if !ok { + out = append(out, item) + continue + } + parts := combinedDocumentMetadataValueSplitter.Split(strings.TrimSpace(text), -1) + added := false + for _, part := range parts { + part = strings.TrimSpace(part) + if part == "" { + continue + } + out = append(out, part) + added = true + } + if !added { + out = append(out, item) + } + } + return dedupeDocumentMetadataList(out) +} + func cloneDocumentMetadataValue(v interface{}) interface{} { switch typed := v.(type) { case []interface{}: diff --git a/internal/service/document/document_metadata_test.go b/internal/service/document/document_metadata_test.go index 49013e6abe..1a903bbae4 100644 --- a/internal/service/document/document_metadata_test.go +++ b/internal/service/document/document_metadata_test.go @@ -51,6 +51,35 @@ func TestNormalizeMetadataListValue(t *testing.T) { } } +func TestSplitCombinedDocumentMetadataValues(t *testing.T) { + meta := map[string]interface{}{ + "character": []interface{}{"关羽、孙权", "张辽|赵云", "曹操"}, + "author": "alice,bob", + "year": 2026, + } + + got := splitCombinedDocumentMetadataValues(meta) + characters, ok := got["character"].([]interface{}) + if !ok { + t.Fatalf("character has unexpected type: %T", got["character"]) + } + want := []interface{}{"关羽", "孙权", "张辽", "赵云", "曹操"} + if len(characters) != len(want) { + t.Fatalf("character length = %d, want %d: %#v", len(characters), len(want), characters) + } + for i := range want { + if characters[i] != want[i] { + t.Fatalf("character[%d] = %#v, want %#v", i, characters[i], want[i]) + } + } + if got["author"] != "alice,bob" { + t.Fatalf("scalar author should be preserved, got %#v", got["author"]) + } + if got["year"] != 2026 { + t.Fatalf("year should be preserved, got %#v", got["year"]) + } +} + func TestFirstScalarMetadataValue(t *testing.T) { if v, ok := firstScalarMetadataValue([]interface{}{"a", "b"}); !ok || v != "a" { t.Errorf("should return first non-nil scalar: %v (ok=%v)", v, ok) diff --git a/internal/service/document/document_test.go b/internal/service/document/document_test.go index bfe4e3795d..29d02202d6 100644 --- a/internal/service/document/document_test.go +++ b/internal/service/document/document_test.go @@ -2029,7 +2029,11 @@ func TestSetDocumentMetadataMergesMetadataRow(t *testing.T) { svc.metadataSvc = service.NewMetadataServiceForTest(dao.NewKnowledgebaseDAO(), engine) ctx := t.Context() - if err := svc.SetDocumentMetadata(ctx, "doc-1", map[string]interface{}{"category": "tech", "year": 2026}); err != nil { + if err := svc.SetDocumentMetadata(ctx, "doc-1", map[string]interface{}{ + "category": "tech", + "year": 2026, + "character": []interface{}{"关羽、孙权", "张辽|赵云"}, + }); err != nil { t.Fatalf("SetDocumentMetadata failed: %v", err) } if got := engine.records["doc-1"]["author"]; got != "alice" { @@ -2041,6 +2045,19 @@ func TestSetDocumentMetadataMergesMetadataRow(t *testing.T) { if got := engine.records["doc-1"]["year"]; got != 2026 { t.Fatalf("year = %#v, want 2026", got) } + characters, ok := engine.records["doc-1"]["character"].([]interface{}) + if !ok { + t.Fatalf("character has unexpected type: %T", engine.records["doc-1"]["character"]) + } + wantCharacters := []interface{}{"关羽", "孙权", "张辽", "赵云"} + if len(characters) != len(wantCharacters) { + t.Fatalf("character length = %d, want %d: %#v", len(characters), len(wantCharacters), characters) + } + for i := range wantCharacters { + if characters[i] != wantCharacters[i] { + t.Fatalf("character[%d] = %#v, want %#v", i, characters[i], wantCharacters[i]) + } + } if got := engine.docKBs["doc-1"]; got != "kb-1" { t.Fatalf("kb_id = %q, want kb-1", got) }