mirror of
https://github.com/infiniflow/ragflow.git
synced 2026-08-05 15:20:30 +08:00
b59d6e8ba1
## Summary Extract the pipeline-output → search-engine index document mapping helpers out of the `task` package into a dedicated, dependency-light leaf package `internal/ingestion/task/indexdoc`. These functions are pure transforms (they only depend on `common`/`utility`) and are not task-orchestration concerns: - `NormalizeChunks`, `DeepCopyChunks` (was unexported `deepCopyChunks`), `toChunkMaps` → `indexdoc/normalize.go` - `ProcessChunksForPipeline`, `RenameTextToContentWithWeight`, `GetEmbeddingTokenConsumption`, `cleanupConsumedChunkFields`, `mergeChunkMetadata`, `processChunkPositions`, `AggregateTableDocMetadata`, `resolveTableColumnConfig` → `indexdoc/process.go` - `AddPositions` → `indexdoc/position.go` - `EmbeddingTokenConsumptionKey` constant → `indexdoc/constants.go` (task/constants.go keeps only `GRAPH_RAPTOR_FAKE_DOC_ID`) Call sites in `pipeline_executor.go` and `golden_compare.go` now reference the `indexdoc` package; package-task tests qualify the moved symbols. ## Why The `task` package had grown into a "orchestration + pure mapping + debug" mix. Splitting the pure mapping helpers into a leaf package sharpens package boundaries, removes a misleading top-level `ingestion/chunk` candidate (there are already `parser/chunk` and `service/chunk`), and lets the golden tool / future reuse pull in the mapping logic without dragging in `task`'s `dao`/`engine`/`service` dependency graph (Go subpackage import does not pull in the parent). ## Test plan - `build.sh --test ./internal/ingestion/task/...` — **green** (task 4.7s, indexdoc 0.007s), matching the pre-change baseline. - `gofmt` clean; `build.sh` builds both `ragflow-cli` and `ragflow_server` successfully. - Integration/E2E tiers are delegated to CI (need real MySQL/MinIO/ES services). Note: `pipeline_e2e_test.go` has a **pre-existing** compile error (`server.ElasticsearchConfig` / `server.InfinityConfig` are now defined under `internal/server/config/`, not re-exported by `internal/server`). This is unrelated to this change — the diff to that file is only the added `indexdoc` import and the qualified `EmbeddingTokenConsumptionKey` reference.
102 lines
3.4 KiB
Go
102 lines
3.4 KiB
Go
package indexdoc
|
||
|
||
import (
|
||
"testing"
|
||
)
|
||
|
||
// =============================================================================
|
||
// AddPositions
|
||
// Canonical format: [pageNum, left, right, top, bottom] × N
|
||
// Contract: input page numbers are ALREADY 1-indexed (the 0→1 conversion
|
||
// happens once, at the parser boundary in normalizePDFPageNumber). This
|
||
// function is a passthrough — it must NOT add +1, otherwise callers that
|
||
// already feed 1-indexed values (the PDF path after normalization) get a
|
||
// double-incremented page number.
|
||
// =============================================================================
|
||
|
||
func TestAddPositions_Basic(t *testing.T) {
|
||
chunk := map[string]any{}
|
||
// [pn=1 (first page, 1-indexed), left=100, right=50, top=200, bottom=150]
|
||
positions := []float64{1, 100, 50, 200, 150}
|
||
AddPositions(chunk, positions)
|
||
|
||
pageNum, ok := chunk["page_num_int"].([]int)
|
||
if !ok || len(pageNum) != 1 || pageNum[0] != 1 {
|
||
t.Errorf("page_num_int = %v, want [1]", pageNum)
|
||
}
|
||
top, ok := chunk["top_int"].([]int)
|
||
if !ok || len(top) != 1 || top[0] != 200 {
|
||
t.Errorf("top_int = %v, want [200]", top)
|
||
}
|
||
position, ok := chunk["position_int"].([][]int)
|
||
if !ok || len(position) != 1 {
|
||
t.Fatalf("position_int = %v, want [[1 100 50 200 150]]", position)
|
||
}
|
||
if position[0][0] != 1 || position[0][1] != 100 || position[0][2] != 50 || position[0][3] != 200 || position[0][4] != 150 {
|
||
t.Errorf("position_int[0] = %v, want [1 100 50 200 150]", position[0])
|
||
}
|
||
}
|
||
|
||
func TestAddPositions_MultiplePositions(t *testing.T) {
|
||
chunk := map[string]any{}
|
||
positions := []float64{
|
||
1, 100, 50, 200, 150, // pn=1, left=100, right=50, top=200, bottom=150
|
||
2, 200, 60, 300, 250, // pn=2, left=200, right=60, top=300, bottom=250
|
||
}
|
||
AddPositions(chunk, positions)
|
||
|
||
pageNum := chunk["page_num_int"].([]int)
|
||
if len(pageNum) != 2 || pageNum[0] != 1 || pageNum[1] != 2 {
|
||
t.Errorf("page_num_int = %v, want [1 2]", pageNum)
|
||
}
|
||
top := chunk["top_int"].([]int)
|
||
if len(top) != 2 || top[0] != 200 || top[1] != 300 {
|
||
t.Errorf("top_int = %v, want [200 300]", top)
|
||
}
|
||
position := chunk["position_int"].([][]int)
|
||
if len(position) != 2 {
|
||
t.Fatalf("position_int len = %d, want 2", len(position))
|
||
}
|
||
}
|
||
|
||
func TestAddPositions_NilPositions(t *testing.T) {
|
||
chunk := map[string]any{}
|
||
AddPositions(chunk, nil)
|
||
if _, exists := chunk["page_num_int"]; exists {
|
||
t.Error("page_num_int should not be set for nil positions")
|
||
}
|
||
}
|
||
|
||
func TestAddPositions_EmptyPositions(t *testing.T) {
|
||
chunk := map[string]any{}
|
||
AddPositions(chunk, []float64{})
|
||
if _, exists := chunk["page_num_int"]; exists {
|
||
t.Error("page_num_int should not be set for empty positions")
|
||
}
|
||
}
|
||
|
||
func TestAddPositions_PartialPositions(t *testing.T) {
|
||
chunk := map[string]any{}
|
||
positions := []float64{1, 100} // only 2 elements, not a complete position
|
||
AddPositions(chunk, positions)
|
||
if _, exists := chunk["page_num_int"]; exists {
|
||
t.Error("page_num_int should not be set for partial positions")
|
||
}
|
||
}
|
||
|
||
func TestAddPositions_PassthroughNoOffset(t *testing.T) {
|
||
// page numbers are 1-indexed on entry; AddPositions must not add +1.
|
||
chunk := map[string]any{}
|
||
positions := []float64{6, 100, 50, 200, 150} // pn=6, 1-indexed
|
||
AddPositions(chunk, positions)
|
||
|
||
pageNum := chunk["page_num_int"].([]int)
|
||
if pageNum[0] != 6 {
|
||
t.Errorf("page_num_int = %d, want 6 (passthrough, no +1)", pageNum[0])
|
||
}
|
||
position := chunk["position_int"].([][]int)
|
||
if position[0][0] != 6 {
|
||
t.Errorf("position_int[0][0] = %d, want 6 (passthrough, no +1)", position[0][0])
|
||
}
|
||
}
|