From 0471fd5695436f3e9eaac09cdc8b81dea5ff17fa Mon Sep 17 00:00:00 2001 From: Haruko386 Date: Fri, 24 Jul 2026 19:33:13 +0800 Subject: [PATCH] fix: unable to add metadata in dataset (#17359) ### Summary As title --- internal/service/dataset/helpers.go | 21 ++++++ internal/service/dataset/update.go | 7 +- internal/service/dataset/update_test.go | 96 +++++++++++++++++++++++++ 3 files changed, 121 insertions(+), 3 deletions(-) diff --git a/internal/service/dataset/helpers.go b/internal/service/dataset/helpers.go index edbb4603f3..21b937ac83 100644 --- a/internal/service/dataset/helpers.go +++ b/internal/service/dataset/helpers.go @@ -7,6 +7,7 @@ import ( "strings" "ragflow/internal/dao" + "ragflow/internal/entity" pipelinepkg "ragflow/internal/ingestion/pipeline" "ragflow/internal/service" @@ -230,6 +231,26 @@ func datasetUpdateEmbeddingID(req service.UpdateDatasetRequest) (string, bool, e return embdID, true, nil } +func preserveDatasetParserConfigMetadata(next, existing entity.JSONMap, incoming map[string]interface{}) entity.JSONMap { + if next == nil { + next = entity.JSONMap{} + } + for _, key := range []string{"metadata", "built_in_metadata", "enable_metadata"} { + if incoming != nil { + if value, ok := incoming[key]; ok { + next[key] = value + continue + } + } + if existing != nil { + if value, ok := existing[key]; ok { + next[key] = value + } + } + } + return next +} + func normalizeDatasetUpdateExt(ext map[string]interface{}) map[string]interface{} { if ext == nil { return nil diff --git a/internal/service/dataset/update.go b/internal/service/dataset/update.go index 88340d00fd..f5e01f969d 100644 --- a/internal/service/dataset/update.go +++ b/internal/service/dataset/update.go @@ -241,7 +241,8 @@ func (d *DatasetService) UpdateDataset(ctx context.Context, datasetID, tenantID zap.String("parserID", effParserID), zap.Error(dslErr)) } if dslJSON != nil { - updates["parser_config"] = pipelinepkg.BuildParserConfig(dslJSON, map[string]interface{}(req.ParserConfig)) + parserConfig := pipelinepkg.BuildParserConfig(dslJSON, map[string]interface{}(req.ParserConfig)) + updates["parser_config"] = preserveDatasetParserConfigMetadata(parserConfig, lockedKB.ParserConfig, req.ParserConfig) } } if pagerankRequested { @@ -260,7 +261,7 @@ func (d *DatasetService) UpdateDataset(ctx context.Context, datasetID, tenantID common.Warn("failed to resolve component params defaults on parser_id switch", zap.String("parserID", parserID), zap.Error(cpErr)) } else if resolved != nil { - updates["parser_config"] = resolved + updates["parser_config"] = preserveDatasetParserConfigMetadata(resolved, lockedKB.ParserConfig, req.ParserConfig) } } } @@ -280,7 +281,7 @@ func (d *DatasetService) UpdateDataset(ctx context.Context, datasetID, tenantID common.Warn("failed to resolve component params defaults on pipeline change", zap.String("parserID", cfgParserID), zap.Error(cpErr)) } else if cpDefaults != nil { - updates["parser_config"] = cpDefaults + updates["parser_config"] = preserveDatasetParserConfigMetadata(cpDefaults, lockedKB.ParserConfig, req.ParserConfig) } } if len(updates) > 0 { diff --git a/internal/service/dataset/update_test.go b/internal/service/dataset/update_test.go index fe1a0a1b08..4efb9443f7 100644 --- a/internal/service/dataset/update_test.go +++ b/internal/service/dataset/update_test.go @@ -18,6 +18,7 @@ package dataset import ( "encoding/json" + "reflect" "strings" "testing" @@ -1043,6 +1044,101 @@ func TestUpdateDataset_AcceptsValidComponentParams_Builtin(t *testing.T) { } } +func TestUpdateDataset_PreservesIncomingMetadataWhenCleaningParserConfig(t *testing.T) { + db := setupDatasetUpdateTestDB(t) + pushServiceDB(t, db) + insertDatasetUpdateKB(t, "kb-1", "tenant-1", "Original") + + incomingMetadata := []interface{}{map[string]interface{}{ + "key": "author", + "type": "string", + "description": "Author", + }} + incomingBuiltInMetadata := []interface{}{map[string]interface{}{ + "key": "document_name", + "type": "string", + }} + parserConfig := map[string]interface{}{ + "Parser:HipSignsRhyme": map[string]interface{}{ + "pdf": map[string]interface{}{"parse_method": "deepdoc"}, + }, + "metadata": incomingMetadata, + "built_in_metadata": incomingBuiltInMetadata, + "enable_metadata": true, + } + + _, code, err := testDatasetUpdateService(t).UpdateDataset(t.Context(), "kb-1", "tenant-1", service.UpdateDatasetRequest{ + ParserConfig: parserConfig, + }) + if err != nil || code != common.CodeSuccess { + t.Fatalf("UpdateDataset failed: code=%d err=%v", code, err) + } + + persisted, err := dao.NewKnowledgebaseDAO().GetByID("kb-1") + if err != nil { + t.Fatalf("get updated kb: %v", err) + } + if !reflect.DeepEqual(persisted.ParserConfig["metadata"], incomingMetadata) { + t.Fatalf("metadata was not preserved: %#v", persisted.ParserConfig["metadata"]) + } + if !reflect.DeepEqual(persisted.ParserConfig["built_in_metadata"], incomingBuiltInMetadata) { + t.Fatalf("built_in_metadata was not preserved: %#v", persisted.ParserConfig["built_in_metadata"]) + } + if persisted.ParserConfig["enable_metadata"] != true { + t.Fatalf("enable_metadata was not preserved: %#v", persisted.ParserConfig["enable_metadata"]) + } + if _, ok := persisted.ParserConfig["Parser:HipSignsRhyme"].(map[string]interface{}); !ok { + t.Fatalf("component parser_config missing: %#v", persisted.ParserConfig) + } +} + +func TestUpdateDataset_PreservesExistingMetadataWhenParserConfigOmitsIt(t *testing.T) { + db := setupDatasetUpdateTestDB(t) + pushServiceDB(t, db) + insertDatasetUpdateKB(t, "kb-1", "tenant-1", "Original") + + existingMetadata := []interface{}{map[string]interface{}{ + "key": "category", + "type": "string", + }} + existingBuiltInMetadata := []interface{}{map[string]interface{}{ + "key": "document_name", + "type": "string", + }} + if err := dao.DB.Model(&entity.Knowledgebase{}).Where("id = ?", "kb-1").Update("parser_config", entity.JSONMap{ + "metadata": existingMetadata, + "built_in_metadata": existingBuiltInMetadata, + "enable_metadata": true, + }).Error; err != nil { + t.Fatalf("seed parser_config: %v", err) + } + + _, code, err := testDatasetUpdateService(t).UpdateDataset(t.Context(), "kb-1", "tenant-1", service.UpdateDatasetRequest{ + ParserConfig: map[string]interface{}{ + "Parser:HipSignsRhyme": map[string]interface{}{ + "pdf": map[string]interface{}{"parse_method": "deepdoc"}, + }, + }, + }) + if err != nil || code != common.CodeSuccess { + t.Fatalf("UpdateDataset failed: code=%d err=%v", code, err) + } + + persisted, err := dao.NewKnowledgebaseDAO().GetByID("kb-1") + if err != nil { + t.Fatalf("get updated kb: %v", err) + } + if !reflect.DeepEqual(persisted.ParserConfig["metadata"], existingMetadata) { + t.Fatalf("existing metadata was not preserved: %#v", persisted.ParserConfig["metadata"]) + } + if !reflect.DeepEqual(persisted.ParserConfig["built_in_metadata"], existingBuiltInMetadata) { + t.Fatalf("existing built_in_metadata was not preserved: %#v", persisted.ParserConfig["built_in_metadata"]) + } + if persisted.ParserConfig["enable_metadata"] != true { + t.Fatalf("existing enable_metadata was not preserved: %#v", persisted.ParserConfig["enable_metadata"]) + } +} + func TestUpdateDataset_StripsCanvasUnknownParam(t *testing.T) { db := setupDatasetUpdateTestDB(t) pushServiceDB(t, db)