diff --git a/internal/ingestion/component/tokenizer.go b/internal/ingestion/component/tokenizer.go index ba91cdaf98..13b70e3159 100644 --- a/internal/ingestion/component/tokenizer.go +++ b/internal/ingestion/component/tokenizer.go @@ -323,13 +323,13 @@ func (c *TokenizerComponent) Invoke(ctx context.Context, db *gorm.DB, inputs map normalizeChunkTextFallback(chunks) - // Diff Tokenizer-8: chunk_order_int must be set on all paths - // (Python sets it unconditionally). tokenizeChunks also sets it - // for the full_text path; this covers the embedding-only path. + // chunk_order_int is the position of the chunk in the (post-filter) reading + // sequence. It is set unconditionally on every surviving chunk so that all + // retrievable chunks carry a stable reading-order index on every path, not + // just chunks+full_text. Because it enumerates the slice after filtering, + // the values are contiguous and 0-based within Go. for i := range chunks { - if chunks[i].ChunkOrderInt == nil { - chunks[i].ChunkOrderInt = intPtr(i) - } + chunks[i].ChunkOrderInt = intPtr(i) } language := globals.GlobalOrInput(ctx, inputs, "lang", "English") @@ -559,13 +559,15 @@ func chunksFromTokenizerUpstream(in schema.TokenizerFromUpstream) []schema.Chunk default: raw = cloneChunkDocs(in.JSONResult) } - // Discard zero-value ChunkDocs (no text, no content_with_weight, no - // image, no summary) so they don't produce phantom embeddings - // downstream. Python's pipeline filters none-chunks before the - // tokenizer. + // Keep only chunks that have retrievable content: a chunk is dropped only + // when both text and content_with_weight are empty. The ContentWithWeight + // guard preserves the Parser path, whose blocks carry content_with_weight + // without text; normalizeChunkTextFallback backfills text afterwards, so + // this guard is required to avoid dropping legitimate Parser blocks before + // the backfill runs. filtered := raw[:0] for _, ck := range raw { - if isPhantomChunk(ck) { + if ck.Text == "" && ck.ContentWithWeight == "" { continue } filtered = append(filtered, ck) @@ -573,14 +575,6 @@ func chunksFromTokenizerUpstream(in schema.TokenizerFromUpstream) []schema.Chunk return filtered } -// isPhantomChunk returns true when a ChunkDoc has no usable content for -// downstream tokenization or embedding. A chunk carrying only a Summary -// (no Text/Image/ContentWithWeight) is kept — tokenizeChunks tokenizes the -// Summary in that case. Mirrors Python's none-chunk filtering. -func isPhantomChunk(ck schema.ChunkDoc) bool { - return ck.Text == "" && ck.Image == "" && ck.ContentWithWeight == "" && ck.Summary == "" -} - func textPayloadToChunks(payload *string) []schema.ChunkDoc { if payload == nil || strings.TrimSpace(*payload) == "" { return []schema.ChunkDoc{} @@ -660,7 +654,6 @@ func tokenizeChunks(chunks []schema.ChunkDoc, titleStem string, language string) tok := tokenizer.New(language) for i := range chunks { ck := &chunks[i] - ck.ChunkOrderInt = intPtr(i) titleTk, err := tok.Tokenize(titleStem) if err != nil { return fmt.Errorf("tokenizer: title tokenize: %w", err) diff --git a/internal/ingestion/component/tokenizer_unit_test.go b/internal/ingestion/component/tokenizer_unit_test.go index 5c60d96cb7..2454ea79f8 100644 --- a/internal/ingestion/component/tokenizer_unit_test.go +++ b/internal/ingestion/component/tokenizer_unit_test.go @@ -361,30 +361,6 @@ func TestChunkDocsToMaps_PreservesPDFPositions(t *testing.T) { } } -// TestIsPhantomChunk verifies that zero-value ChunkDocs (no Text, no -// Image, no ContentWithWeight, no Summary) are identified as phantom, -// while any one of those fields being present keeps the chunk. -func TestIsPhantomChunk(t *testing.T) { - if !isPhantomChunk(schema.ChunkDoc{}) { - t.Error("empty ChunkDoc must be phantom") - } - if !isPhantomChunk(schema.ChunkDoc{Text: ""}) { - t.Error("ChunkDoc with empty Text only must be phantom") - } - if isPhantomChunk(schema.ChunkDoc{Text: "hello"}) { - t.Error("ChunkDoc with Text must not be phantom") - } - if isPhantomChunk(schema.ChunkDoc{Image: "data:image/png;base64,abc"}) { - t.Error("ChunkDoc with Image must not be phantom") - } - if isPhantomChunk(schema.ChunkDoc{ContentWithWeight: "weight"}) { - t.Error("ChunkDoc with ContentWithWeight must not be phantom") - } - if isPhantomChunk(schema.ChunkDoc{Summary: "a summary"}) { - t.Error("ChunkDoc with Summary must not be phantom") - } -} - // TestTruncateForEmbedding_SmallMaxTokens covers Tokenizer Diff-14. For any // positive maxTokens, truncateForEmbedding keeps the first maxTokens tokens // (non-empty) and the result is strictly shorter than the input. @@ -500,18 +476,23 @@ func TestChunkOrderInt_EmbeddingOnly(t *testing.T) { } } -// TestChunksFromTokenizerUpstream_FiltersPhantomChunks covers Tokenizer -// Omission-2 at the pipeline level: when upstream input contains a -// zero-value ChunkDoc (no Text, no Image, no ContentWithWeight), it must -// be silently dropped from the output before tokenization and embedding. -// This mirrors Python's `if not text and not d.get("image"): continue` -// in tokenizer.py:80-82. +// TestChunksFromTokenizerUpstream_FiltersPhantomChunks covers A5 at the +// pipeline level: a chunk is dropped only when BOTH text and +// content_with_weight are empty. Blocks that carry only image / summary / +// questions / nothing are dropped (they produce no retrievable content), +// while a content_with_weight-only block is kept (Parser path; normalize +// backfills text). Deliberate deviation from Python's "filter None only" +// rule — user value (retrievable content) wins. func TestChunksFromTokenizerUpstream_FiltersPhantomChunks(t *testing.T) { - // JSON path: three items, the middle one is a phantom. + // JSON path: 6 items. Only the text block and the content_with_weight + // block survive; the other four are dropped by the A5 gate. items := []map[string]any{ - {"text": "valid chunk", "doc_type_kwd": "text"}, - {}, // phantom: no text, no image, no content_with_weight - {"text": "another valid", "doc_type_kwd": "text"}, + {"text": "valid chunk", "doc_type_kwd": "text"}, // keep + {"image": "data:image/png;base64,abc"}, // drop: no text, no content_with_weight + {"summary": "a summary"}, // drop + {"questions": "q1"}, // drop + {"content_with_weight": "weighted"}, // keep (text backfilled by normalize) + {}, // drop: empty } // Use embedding-only mode to avoid CGo tokenizer dependency. stub := newStubEmbedder(3) @@ -531,16 +512,142 @@ func TestChunksFromTokenizerUpstream_FiltersPhantomChunks(t *testing.T) { t.Fatalf("Invoke: %v", err) } chunks := out["chunks"].([]map[string]any) - // Must drop the phantom — only 2 valid chunks remain. + // A5 keeps only the text block and the content_with_weight block. if len(chunks) != 2 { - t.Fatalf("want 2 chunks (phantom filtered), got %d", len(chunks)) + t.Fatalf("want 2 chunks (4 phantoms filtered), got %d: %#v", len(chunks), chunks) } - // Verify the surviving chunks are the valid ones. + // Surviving chunks, in upstream order. if chunks[0]["text"] != "valid chunk" { t.Errorf("chunk 0 text = %q, want %q", chunks[0]["text"], "valid chunk") } - if chunks[1]["text"] != "another valid" { - t.Errorf("chunk 1 text = %q, want %q", chunks[1]["text"], "another valid") + if chunks[1]["text"] != "weighted" { + t.Errorf("chunk 1 text = %q, want %q (content_with_weight backfilled to text)", chunks[1]["text"], "weighted") + } +} + +// TestChunkOrderInt_AllPathsUnconditional is the A6 regression gate: every +// surviving chunk must carry chunk_order_int, equal to its index in the +// (post-filter) reading sequence — on every output_format and every +// search_method combination. Go sets it unconditionally (deviation from +// Python's chunks+full_text-only rule) so every retrieval path can sort by +// reading order. +func TestChunkOrderInt_AllPathsUnconditional(t *testing.T) { + // identity tokenizer: Tokenize returns input unchanged — no CGo pool. + tokenizer.SetEngineType("infinity") + defer tokenizer.SetEngineType("") + + stub := newStubEmbedder(8) + textChunks := []map[string]any{{"text": "a"}, {"text": "b"}, {"text": "c"}} + jsonChunks := []map[string]any{ + {"text": "a", "doc_type_kwd": "text"}, + {"text": "b", "doc_type_kwd": "text"}, + {"text": "c", "doc_type_kwd": "text"}, + } + cases := []struct { + name string + searchMethod []string + inputs map[string]any + wantChunks int + }{ + {"chunks/full_text", []string{"full_text"}, + map[string]any{"output_format": "chunks", "chunks": textChunks}, 3}, + {"chunks/embedding", []string{"embedding"}, + map[string]any{"output_format": "chunks", "chunks": textChunks}, 3}, + {"chunks/full_text+embedding", []string{"full_text", "embedding"}, + map[string]any{"output_format": "chunks", "chunks": textChunks}, 3}, + {"json/full_text", []string{"full_text"}, + map[string]any{"output_format": "json", "json": jsonChunks}, 3}, + {"markdown/full_text", []string{"full_text"}, + map[string]any{"output_format": "markdown", "markdown": "# H\n\nbody one\n\nbody two"}, 1}, + {"text/full_text", []string{"full_text"}, + map[string]any{"output_format": "text", "text": "body text"}, 1}, + {"html/full_text", []string{"full_text"}, + map[string]any{"output_format": "html", "html": "

body text

"}, 1}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + comp, err := NewTokenizerComponentWithResolver( + map[string]any{"search_method": tc.searchMethod, "fields": []string{"text"}}, + func(ctx context.Context, _, _, _ string) (Embedder, error) { return stub, nil }, + ) + if err != nil { + t.Fatalf("NewTokenizerComponentWithResolver: %v", err) + } + out, err := comp.Invoke(context.Background(), nil, tc.inputs) + if err != nil { + t.Fatalf("Invoke: %v", err) + } + chunks := out["chunks"].([]map[string]any) + if len(chunks) != tc.wantChunks { + t.Fatalf("want %d chunks, got %d: %#v", tc.wantChunks, len(chunks), chunks) + } + for i, ck := range chunks { + coi, ok := ck["chunk_order_int"] + if !ok { + t.Errorf("chunk %d: chunk_order_int missing", i) + continue + } + got, ok := coi.(float64) + if !ok { + t.Errorf("chunk %d: chunk_order_int type %T, want number", i, coi) + continue + } + if int(got) != i { + t.Errorf("chunk %d: chunk_order_int = %v, want %d", i, got, i) + } + } + }) + } +} + +// TestChunkOrderInt_A5FilterKeepsSequenceContiguous is the A5+A6 joint gate: +// when an upstream text-empty chunk is dropped by the A5 gate, the surviving +// text chunks still receive contiguous 0-based chunk_order_int values (Go +// indexes the post-filter slice, so no gap is left by the dropped chunk). +func TestChunkOrderInt_A5FilterKeepsSequenceContiguous(t *testing.T) { + tokenizer.SetEngineType("infinity") + defer tokenizer.SetEngineType("") + + stub := newStubEmbedder(8) + comp, err := NewTokenizerComponentWithResolver( + map[string]any{"search_method": []string{"full_text", "embedding"}, "fields": []string{"text"}}, + func(ctx context.Context, _, _, _ string) (Embedder, error) { return stub, nil }, + ) + if err != nil { + t.Fatalf("NewTokenizerComponentWithResolver: %v", err) + } + // Middle chunk has empty text and no content_with_weight -> dropped by A5. + items := []map[string]any{ + {"text": "first"}, + {"questions": "orphan question"}, // dropped: no retrievable content + {"text": "second"}, + } + out, err := comp.Invoke(context.Background(), nil, map[string]any{ + "name": "doc.pdf", + "output_format": "chunks", + "chunks": items, + }) + if err != nil { + t.Fatalf("Invoke: %v", err) + } + chunks := out["chunks"].([]map[string]any) + if len(chunks) != 2 { + t.Fatalf("want 2 surviving chunks, got %d: %#v", len(chunks), chunks) + } + for i, ck := range chunks { + coi, ok := ck["chunk_order_int"] + if !ok { + t.Errorf("chunk %d: chunk_order_int missing", i) + continue + } + got, ok := coi.(float64) + if !ok { + t.Errorf("chunk %d: chunk_order_int type %T, want number", i, coi) + continue + } + if int(got) != i { + t.Errorf("chunk %d: chunk_order_int = %v, want %d (contiguous after A5 drop)", i, got, i) + } } }