zlei6/ragflow
1
0
Fork 0
mirror of https://github.com/infiniflow/ragflow.git synced 2026-07-18 21:57:27 +08:00
Code Issues Packages Projects Releases Wiki Activity

fix: JSONMap scan in dataset index chunking config (#16489)

Browse Source 
### Summary

As title

This PR fixes dataset index task creation failing with unsupported data
type: entity.JSONMap when loading document chunking config.

#### issues:
```
2026/06/30 15:19:40 /home/infiniflow/Documents/development/ragflow/internal/dao/document.go:162 
[error] unsupported data type: ragflow/internal/entity.JSONMap
```

#### Changes:
+ Adds the missing GORM type:longtext tag to ParserConfig in
DocumentDAO.GetChunkingConfig.
+ Adds a DAO regression test covering GetChunkingConfig joins across
document, knowledgebase, and tenant while scanning parser_config.
This commit is contained in:
Haruko386
2026-07-02 12:06:53 +08:00
committed by GitHub
parent d6b1c5937b
commit b4825166a7
2 changed files with 61 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
internal/dao/document.go
View File

@@ -130,7 +130,7 @@ func (dao *DocumentDAO) GetChunkingConfig(docID string) (map[string]interface{},
ID string `gorm:"column:id"`
KbID string `gorm:"column:kb_id"`
ParserID string `gorm:"column:parser_id"`
ParserConfig entity.JSONMap `gorm:"column:parser_config"`
ParserConfig entity.JSONMap `gorm:"column:parser_config;type:longtext"`
Size int64 `gorm:"column:size"`
ContentHash *string `gorm:"column:content_hash"`
Language *string `gorm:"column:language"`

60
internal/dao/document_test.go
View File

@@ -35,6 +35,8 @@ func setupDocumentTestDB(t *testing.T) *gorm.DB {
}
if err := db.AutoMigrate(
&entity.Document{},
&entity.Knowledgebase{},
&entity.Tenant{},
); err != nil {
t.Fatalf("failed to migrate: %v", err)
}
@@ -164,4 +166,62 @@ func TestDocumentGetByDocumentIDAndDatasetIDUsesKBID(t *testing.T) {
}
}
func TestDocumentGetChunkingConfigScansParserConfig(t *testing.T) {
db := setupDocumentTestDB(t)
pushDocDB(t, db)
if err := db.Create(&entity.Tenant{
ID: "tenant1",
LLMID: "llm1",
EmbdID: "embd1",
ASRID: "asr1",
Img2TxtID: "img2txt1",
RerankID: "rerank1",
ParserIDs: "naive",
}).Error; err != nil {
t.Fatalf("create tenant: %v", err)
}
if err := db.Create(&entity.Knowledgebase{
ID: "kb1",
TenantID: "tenant1",
Name: "Dataset 1",
Language: sp("English"),
EmbdID: "kb-embd1",
Permission: "me",
CreatedBy: "user1",
ParserID: "naive",
ParserConfig: entity.JSONMap{},
}).Error; err != nil {
t.Fatalf("create knowledgebase: %v", err)
}
if err := db.Create(&entity.Document{
ID: "doc1",
KbID: "kb1",
ParserID: "naive",
ParserConfig: entity.JSONMap{"chunk_token_num": float64(128), "delimiter": "\\n"},
SourceType: "local",
Type: "doc",
CreatedBy: "user1",
Size: 42,
Suffix: ".txt",
}).Error; err != nil {
t.Fatalf("create document: %v", err)
}
config, err := NewDocumentDAO().GetChunkingConfig("doc1")
if err != nil {
t.Fatalf("GetChunkingConfig failed: %v", err)
}
parserConfig, ok := config["parser_config"].(entity.JSONMap)
if !ok {
t.Fatalf("parser_config type = %T, want entity.JSONMap", config["parser_config"])
}
if parserConfig["chunk_token_num"] != float64(128) || parserConfig["delimiter"] != "\\n" {
t.Fatalf("unexpected parser_config: %#v", parserConfig)
}
if config["tenant_id"] != "tenant1" || config["embd_id"] != "kb-embd1" {
t.Fatalf("unexpected joined config: %#v", config)
}
}
func sp(s string) *string { return &s }