fix: unable to add metadata in dataset (#17359)

### Summary

As title
This commit is contained in:
Haruko386
2026-07-24 19:33:13 +08:00
committed by GitHub
parent 601742e000
commit 0471fd5695
3 changed files with 121 additions and 3 deletions

View File

@@ -7,6 +7,7 @@ import (
"strings"
"ragflow/internal/dao"
"ragflow/internal/entity"
pipelinepkg "ragflow/internal/ingestion/pipeline"
"ragflow/internal/service"
@@ -230,6 +231,26 @@ func datasetUpdateEmbeddingID(req service.UpdateDatasetRequest) (string, bool, e
return embdID, true, nil
}
func preserveDatasetParserConfigMetadata(next, existing entity.JSONMap, incoming map[string]interface{}) entity.JSONMap {
if next == nil {
next = entity.JSONMap{}
}
for _, key := range []string{"metadata", "built_in_metadata", "enable_metadata"} {
if incoming != nil {
if value, ok := incoming[key]; ok {
next[key] = value
continue
}
}
if existing != nil {
if value, ok := existing[key]; ok {
next[key] = value
}
}
}
return next
}
func normalizeDatasetUpdateExt(ext map[string]interface{}) map[string]interface{} {
if ext == nil {
return nil

View File

@@ -241,7 +241,8 @@ func (d *DatasetService) UpdateDataset(ctx context.Context, datasetID, tenantID
zap.String("parserID", effParserID), zap.Error(dslErr))
}
if dslJSON != nil {
updates["parser_config"] = pipelinepkg.BuildParserConfig(dslJSON, map[string]interface{}(req.ParserConfig))
parserConfig := pipelinepkg.BuildParserConfig(dslJSON, map[string]interface{}(req.ParserConfig))
updates["parser_config"] = preserveDatasetParserConfigMetadata(parserConfig, lockedKB.ParserConfig, req.ParserConfig)
}
}
if pagerankRequested {
@@ -260,7 +261,7 @@ func (d *DatasetService) UpdateDataset(ctx context.Context, datasetID, tenantID
common.Warn("failed to resolve component params defaults on parser_id switch",
zap.String("parserID", parserID), zap.Error(cpErr))
} else if resolved != nil {
updates["parser_config"] = resolved
updates["parser_config"] = preserveDatasetParserConfigMetadata(resolved, lockedKB.ParserConfig, req.ParserConfig)
}
}
}
@@ -280,7 +281,7 @@ func (d *DatasetService) UpdateDataset(ctx context.Context, datasetID, tenantID
common.Warn("failed to resolve component params defaults on pipeline change",
zap.String("parserID", cfgParserID), zap.Error(cpErr))
} else if cpDefaults != nil {
updates["parser_config"] = cpDefaults
updates["parser_config"] = preserveDatasetParserConfigMetadata(cpDefaults, lockedKB.ParserConfig, req.ParserConfig)
}
}
if len(updates) > 0 {

View File

@@ -18,6 +18,7 @@ package dataset
import (
"encoding/json"
"reflect"
"strings"
"testing"
@@ -1043,6 +1044,101 @@ func TestUpdateDataset_AcceptsValidComponentParams_Builtin(t *testing.T) {
}
}
func TestUpdateDataset_PreservesIncomingMetadataWhenCleaningParserConfig(t *testing.T) {
db := setupDatasetUpdateTestDB(t)
pushServiceDB(t, db)
insertDatasetUpdateKB(t, "kb-1", "tenant-1", "Original")
incomingMetadata := []interface{}{map[string]interface{}{
"key": "author",
"type": "string",
"description": "Author",
}}
incomingBuiltInMetadata := []interface{}{map[string]interface{}{
"key": "document_name",
"type": "string",
}}
parserConfig := map[string]interface{}{
"Parser:HipSignsRhyme": map[string]interface{}{
"pdf": map[string]interface{}{"parse_method": "deepdoc"},
},
"metadata": incomingMetadata,
"built_in_metadata": incomingBuiltInMetadata,
"enable_metadata": true,
}
_, code, err := testDatasetUpdateService(t).UpdateDataset(t.Context(), "kb-1", "tenant-1", service.UpdateDatasetRequest{
ParserConfig: parserConfig,
})
if err != nil || code != common.CodeSuccess {
t.Fatalf("UpdateDataset failed: code=%d err=%v", code, err)
}
persisted, err := dao.NewKnowledgebaseDAO().GetByID("kb-1")
if err != nil {
t.Fatalf("get updated kb: %v", err)
}
if !reflect.DeepEqual(persisted.ParserConfig["metadata"], incomingMetadata) {
t.Fatalf("metadata was not preserved: %#v", persisted.ParserConfig["metadata"])
}
if !reflect.DeepEqual(persisted.ParserConfig["built_in_metadata"], incomingBuiltInMetadata) {
t.Fatalf("built_in_metadata was not preserved: %#v", persisted.ParserConfig["built_in_metadata"])
}
if persisted.ParserConfig["enable_metadata"] != true {
t.Fatalf("enable_metadata was not preserved: %#v", persisted.ParserConfig["enable_metadata"])
}
if _, ok := persisted.ParserConfig["Parser:HipSignsRhyme"].(map[string]interface{}); !ok {
t.Fatalf("component parser_config missing: %#v", persisted.ParserConfig)
}
}
func TestUpdateDataset_PreservesExistingMetadataWhenParserConfigOmitsIt(t *testing.T) {
db := setupDatasetUpdateTestDB(t)
pushServiceDB(t, db)
insertDatasetUpdateKB(t, "kb-1", "tenant-1", "Original")
existingMetadata := []interface{}{map[string]interface{}{
"key": "category",
"type": "string",
}}
existingBuiltInMetadata := []interface{}{map[string]interface{}{
"key": "document_name",
"type": "string",
}}
if err := dao.DB.Model(&entity.Knowledgebase{}).Where("id = ?", "kb-1").Update("parser_config", entity.JSONMap{
"metadata": existingMetadata,
"built_in_metadata": existingBuiltInMetadata,
"enable_metadata": true,
}).Error; err != nil {
t.Fatalf("seed parser_config: %v", err)
}
_, code, err := testDatasetUpdateService(t).UpdateDataset(t.Context(), "kb-1", "tenant-1", service.UpdateDatasetRequest{
ParserConfig: map[string]interface{}{
"Parser:HipSignsRhyme": map[string]interface{}{
"pdf": map[string]interface{}{"parse_method": "deepdoc"},
},
},
})
if err != nil || code != common.CodeSuccess {
t.Fatalf("UpdateDataset failed: code=%d err=%v", code, err)
}
persisted, err := dao.NewKnowledgebaseDAO().GetByID("kb-1")
if err != nil {
t.Fatalf("get updated kb: %v", err)
}
if !reflect.DeepEqual(persisted.ParserConfig["metadata"], existingMetadata) {
t.Fatalf("existing metadata was not preserved: %#v", persisted.ParserConfig["metadata"])
}
if !reflect.DeepEqual(persisted.ParserConfig["built_in_metadata"], existingBuiltInMetadata) {
t.Fatalf("existing built_in_metadata was not preserved: %#v", persisted.ParserConfig["built_in_metadata"])
}
if persisted.ParserConfig["enable_metadata"] != true {
t.Fatalf("existing enable_metadata was not preserved: %#v", persisted.ParserConfig["enable_metadata"])
}
}
func TestUpdateDataset_StripsCanvasUnknownParam(t *testing.T) {
db := setupDatasetUpdateTestDB(t)
pushServiceDB(t, db)