mirror of
https://github.com/infiniflow/ragflow.git
synced 2026-08-05 23:24:05 +08:00
refactor(ingestion/task): emit kb_id as a single string in ProcessChunksForPipeline (#17802)
## Summary
- `ProcessChunksForPipeline` now sets `kb_id` to a plain string instead
of `[]string{kbID}`, removing an index-physical array shape from the
ingestion domain.
- Stored documents are byte-identical: Elasticsearch overrides `kb_id`
with `datasetID` on write, and Infinity's `transformChunkFields` already
accepts a plain string.
- Infinity is intentionally left unchanged — `service/chunk` paths still
feed `kb_id` as `[]string`, and Infinity handles both forms. The
`dataset` artifact merge (`dataset_artifact_service.go`) is out of scope
for this step.
- Unit assertion updated to expect a string.
## Scope / non-goals
This is the smallest first step (T1) of the index-schema leak cleanup
tracked in #17371. It does **not** move the other leaks (`docnm_kwd`,
`create_timestamp_flt`, position ints) to the engine boundary — those
are later steps behind a read-back golden test.
## Test plan
- `go test ./internal/ingestion/task/indexdoc/...` passes.
- The two `task` "Real" integration tests fail identically on a clean
tree (environment lacks real embedding/parsing); they are pre-existing,
unrelated to this change.
🤖 Generated with [CodeBuddy Code](https://cnb.cool/codebuddy)
This commit is contained in:
@@ -77,7 +77,7 @@ func ProcessChunksForPipeline(
|
||||
|
||||
for _, ck := range chunks {
|
||||
ck["doc_id"] = docID
|
||||
ck["kb_id"] = []string{kbID}
|
||||
ck["kb_id"] = kbID
|
||||
ck["docnm_kwd"] = docName
|
||||
ck["create_time"] = timeStr
|
||||
ck["create_timestamp_flt"] = timestamp
|
||||
|
||||
@@ -53,12 +53,8 @@ func TestProcessChunksForPipeline_SetsDocIDAndKBID(t *testing.T) {
|
||||
if chunks[0]["doc_id"] != "doc-1" {
|
||||
t.Errorf("doc_id = %q, want \"doc-1\"", chunks[0]["doc_id"])
|
||||
}
|
||||
if kbIDs, ok := chunks[0]["kb_id"].([]string); ok {
|
||||
if len(kbIDs) != 1 || kbIDs[0] != "kb-1" {
|
||||
t.Errorf("kb_id = %v, want [\"kb-1\"]", chunks[0]["kb_id"])
|
||||
}
|
||||
} else {
|
||||
t.Errorf("kb_id should be []string, got %T", chunks[0]["kb_id"])
|
||||
if kbID, ok := chunks[0]["kb_id"].(string); !ok || kbID != "kb-1" {
|
||||
t.Errorf("kb_id = %v, want \"kb-1\" (string)", chunks[0]["kb_id"])
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -32,7 +32,7 @@ import (
|
||||
"ragflow/internal/engine/infinity"
|
||||
indexdoc "ragflow/internal/ingestion/task/indexdoc"
|
||||
"ragflow/internal/ingestion/testutil"
|
||||
"ragflow/internal/server"
|
||||
"ragflow/internal/server/config"
|
||||
"ragflow/internal/service"
|
||||
)
|
||||
|
||||
@@ -67,7 +67,7 @@ func setupTestDocEngine(t *testing.T, engineType engine.EngineType, tenantID, da
|
||||
esPassword = "infini_rag_flow"
|
||||
}
|
||||
|
||||
cfg := &server.ElasticsearchConfig{
|
||||
cfg := config.ElasticsearchConfig{
|
||||
Hosts: esHost,
|
||||
Username: esUser,
|
||||
Password: esPassword,
|
||||
@@ -91,7 +91,7 @@ func setupTestDocEngine(t *testing.T, engineType engine.EngineType, tenantID, da
|
||||
return nil, func() {}
|
||||
}
|
||||
|
||||
cfg := &server.InfinityConfig{
|
||||
cfg := config.InfinityConfig{
|
||||
URI: infURI,
|
||||
DBName: "ragflow_e2e_test",
|
||||
PostgresPort: 5432,
|
||||
|
||||
@@ -720,8 +720,8 @@ func taskS3SafeBucketName(s string) string {
|
||||
}
|
||||
|
||||
// taskChunkFieldEqualsStr compares a chunk field to a plain string, tolerating
|
||||
// the slice form used internally (e.g. kb_id is []string{kbID} and survives a
|
||||
// JSON round-trip as []any{kbID}).
|
||||
// either form a producer may emit: a plain string (e.g. kb_id is "kb-1" after
|
||||
// T1) or a single-element slice that survives a JSON round-trip as []any{"kb-1"}.
|
||||
func taskChunkFieldEqualsStr(v any, want string) bool {
|
||||
switch val := v.(type) {
|
||||
case string:
|
||||
|
||||
Reference in New Issue
Block a user