mirror of
https://github.com/infiniflow/ragflow.git
synced 2026-07-25 09:53:29 +08:00
@@ -7,6 +7,7 @@ import (
|
||||
"strings"
|
||||
|
||||
"ragflow/internal/dao"
|
||||
"ragflow/internal/entity"
|
||||
pipelinepkg "ragflow/internal/ingestion/pipeline"
|
||||
"ragflow/internal/service"
|
||||
|
||||
@@ -230,6 +231,26 @@ func datasetUpdateEmbeddingID(req service.UpdateDatasetRequest) (string, bool, e
|
||||
return embdID, true, nil
|
||||
}
|
||||
|
||||
func preserveDatasetParserConfigMetadata(next, existing entity.JSONMap, incoming map[string]interface{}) entity.JSONMap {
|
||||
if next == nil {
|
||||
next = entity.JSONMap{}
|
||||
}
|
||||
for _, key := range []string{"metadata", "built_in_metadata", "enable_metadata"} {
|
||||
if incoming != nil {
|
||||
if value, ok := incoming[key]; ok {
|
||||
next[key] = value
|
||||
continue
|
||||
}
|
||||
}
|
||||
if existing != nil {
|
||||
if value, ok := existing[key]; ok {
|
||||
next[key] = value
|
||||
}
|
||||
}
|
||||
}
|
||||
return next
|
||||
}
|
||||
|
||||
func normalizeDatasetUpdateExt(ext map[string]interface{}) map[string]interface{} {
|
||||
if ext == nil {
|
||||
return nil
|
||||
|
||||
@@ -241,7 +241,8 @@ func (d *DatasetService) UpdateDataset(ctx context.Context, datasetID, tenantID
|
||||
zap.String("parserID", effParserID), zap.Error(dslErr))
|
||||
}
|
||||
if dslJSON != nil {
|
||||
updates["parser_config"] = pipelinepkg.BuildParserConfig(dslJSON, map[string]interface{}(req.ParserConfig))
|
||||
parserConfig := pipelinepkg.BuildParserConfig(dslJSON, map[string]interface{}(req.ParserConfig))
|
||||
updates["parser_config"] = preserveDatasetParserConfigMetadata(parserConfig, lockedKB.ParserConfig, req.ParserConfig)
|
||||
}
|
||||
}
|
||||
if pagerankRequested {
|
||||
@@ -260,7 +261,7 @@ func (d *DatasetService) UpdateDataset(ctx context.Context, datasetID, tenantID
|
||||
common.Warn("failed to resolve component params defaults on parser_id switch",
|
||||
zap.String("parserID", parserID), zap.Error(cpErr))
|
||||
} else if resolved != nil {
|
||||
updates["parser_config"] = resolved
|
||||
updates["parser_config"] = preserveDatasetParserConfigMetadata(resolved, lockedKB.ParserConfig, req.ParserConfig)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -280,7 +281,7 @@ func (d *DatasetService) UpdateDataset(ctx context.Context, datasetID, tenantID
|
||||
common.Warn("failed to resolve component params defaults on pipeline change",
|
||||
zap.String("parserID", cfgParserID), zap.Error(cpErr))
|
||||
} else if cpDefaults != nil {
|
||||
updates["parser_config"] = cpDefaults
|
||||
updates["parser_config"] = preserveDatasetParserConfigMetadata(cpDefaults, lockedKB.ParserConfig, req.ParserConfig)
|
||||
}
|
||||
}
|
||||
if len(updates) > 0 {
|
||||
|
||||
@@ -18,6 +18,7 @@ package dataset
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"reflect"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
@@ -1043,6 +1044,101 @@ func TestUpdateDataset_AcceptsValidComponentParams_Builtin(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestUpdateDataset_PreservesIncomingMetadataWhenCleaningParserConfig(t *testing.T) {
|
||||
db := setupDatasetUpdateTestDB(t)
|
||||
pushServiceDB(t, db)
|
||||
insertDatasetUpdateKB(t, "kb-1", "tenant-1", "Original")
|
||||
|
||||
incomingMetadata := []interface{}{map[string]interface{}{
|
||||
"key": "author",
|
||||
"type": "string",
|
||||
"description": "Author",
|
||||
}}
|
||||
incomingBuiltInMetadata := []interface{}{map[string]interface{}{
|
||||
"key": "document_name",
|
||||
"type": "string",
|
||||
}}
|
||||
parserConfig := map[string]interface{}{
|
||||
"Parser:HipSignsRhyme": map[string]interface{}{
|
||||
"pdf": map[string]interface{}{"parse_method": "deepdoc"},
|
||||
},
|
||||
"metadata": incomingMetadata,
|
||||
"built_in_metadata": incomingBuiltInMetadata,
|
||||
"enable_metadata": true,
|
||||
}
|
||||
|
||||
_, code, err := testDatasetUpdateService(t).UpdateDataset(t.Context(), "kb-1", "tenant-1", service.UpdateDatasetRequest{
|
||||
ParserConfig: parserConfig,
|
||||
})
|
||||
if err != nil || code != common.CodeSuccess {
|
||||
t.Fatalf("UpdateDataset failed: code=%d err=%v", code, err)
|
||||
}
|
||||
|
||||
persisted, err := dao.NewKnowledgebaseDAO().GetByID("kb-1")
|
||||
if err != nil {
|
||||
t.Fatalf("get updated kb: %v", err)
|
||||
}
|
||||
if !reflect.DeepEqual(persisted.ParserConfig["metadata"], incomingMetadata) {
|
||||
t.Fatalf("metadata was not preserved: %#v", persisted.ParserConfig["metadata"])
|
||||
}
|
||||
if !reflect.DeepEqual(persisted.ParserConfig["built_in_metadata"], incomingBuiltInMetadata) {
|
||||
t.Fatalf("built_in_metadata was not preserved: %#v", persisted.ParserConfig["built_in_metadata"])
|
||||
}
|
||||
if persisted.ParserConfig["enable_metadata"] != true {
|
||||
t.Fatalf("enable_metadata was not preserved: %#v", persisted.ParserConfig["enable_metadata"])
|
||||
}
|
||||
if _, ok := persisted.ParserConfig["Parser:HipSignsRhyme"].(map[string]interface{}); !ok {
|
||||
t.Fatalf("component parser_config missing: %#v", persisted.ParserConfig)
|
||||
}
|
||||
}
|
||||
|
||||
func TestUpdateDataset_PreservesExistingMetadataWhenParserConfigOmitsIt(t *testing.T) {
|
||||
db := setupDatasetUpdateTestDB(t)
|
||||
pushServiceDB(t, db)
|
||||
insertDatasetUpdateKB(t, "kb-1", "tenant-1", "Original")
|
||||
|
||||
existingMetadata := []interface{}{map[string]interface{}{
|
||||
"key": "category",
|
||||
"type": "string",
|
||||
}}
|
||||
existingBuiltInMetadata := []interface{}{map[string]interface{}{
|
||||
"key": "document_name",
|
||||
"type": "string",
|
||||
}}
|
||||
if err := dao.DB.Model(&entity.Knowledgebase{}).Where("id = ?", "kb-1").Update("parser_config", entity.JSONMap{
|
||||
"metadata": existingMetadata,
|
||||
"built_in_metadata": existingBuiltInMetadata,
|
||||
"enable_metadata": true,
|
||||
}).Error; err != nil {
|
||||
t.Fatalf("seed parser_config: %v", err)
|
||||
}
|
||||
|
||||
_, code, err := testDatasetUpdateService(t).UpdateDataset(t.Context(), "kb-1", "tenant-1", service.UpdateDatasetRequest{
|
||||
ParserConfig: map[string]interface{}{
|
||||
"Parser:HipSignsRhyme": map[string]interface{}{
|
||||
"pdf": map[string]interface{}{"parse_method": "deepdoc"},
|
||||
},
|
||||
},
|
||||
})
|
||||
if err != nil || code != common.CodeSuccess {
|
||||
t.Fatalf("UpdateDataset failed: code=%d err=%v", code, err)
|
||||
}
|
||||
|
||||
persisted, err := dao.NewKnowledgebaseDAO().GetByID("kb-1")
|
||||
if err != nil {
|
||||
t.Fatalf("get updated kb: %v", err)
|
||||
}
|
||||
if !reflect.DeepEqual(persisted.ParserConfig["metadata"], existingMetadata) {
|
||||
t.Fatalf("existing metadata was not preserved: %#v", persisted.ParserConfig["metadata"])
|
||||
}
|
||||
if !reflect.DeepEqual(persisted.ParserConfig["built_in_metadata"], existingBuiltInMetadata) {
|
||||
t.Fatalf("existing built_in_metadata was not preserved: %#v", persisted.ParserConfig["built_in_metadata"])
|
||||
}
|
||||
if persisted.ParserConfig["enable_metadata"] != true {
|
||||
t.Fatalf("existing enable_metadata was not preserved: %#v", persisted.ParserConfig["enable_metadata"])
|
||||
}
|
||||
}
|
||||
|
||||
func TestUpdateDataset_StripsCanvasUnknownParam(t *testing.T) {
|
||||
db := setupDatasetUpdateTestDB(t)
|
||||
pushServiceDB(t, db)
|
||||
|
||||
Reference in New Issue
Block a user