diff --git a/internal/ingestion/component/chunker/common.go b/internal/ingestion/component/chunker/common.go index 7633d03425..6ecf24db45 100644 --- a/internal/ingestion/component/chunker/common.go +++ b/internal/ingestion/component/chunker/common.go @@ -126,16 +126,18 @@ func compileDelimPattern(delims []string) *regexp.Regexp { return regexp.MustCompile(strings.Join(custom, "|")) } -// splitKeepingDelim is the Go equivalent of python's -// `re.split((pattern), text, flags=re.DOTALL)` with the matched -// delimiter preserved (alternation keeps the original delimiter text -// in the output stream so the rebuilding at token_chunker.py:88-93 -// stays lossy-free). +// splitKeepingDelim mirrors Python token_chunker._split_text_by_pattern +// (token_chunker.py:79-94): re.split with a captured delimiter group yields +// [text, delim, text, delim, ...]; each delimiter is glued to the END of the +// preceding text segment, so it never surfaces as a standalone chunk. A +// delimiter with no preceding text (a leading delimiter or one adjacent to +// another) is dropped together with the empty segment, matching Python's +// `if not chunk: continue`. func splitKeepingDelim(text string, pattern *regexp.Regexp) []string { if pattern == nil { return []string{text} } - idxs := pattern.FindAllStringSubmatchIndex(text, -1) + idxs := pattern.FindAllStringIndex(text, -1) if len(idxs) == 0 { return []string{text} } @@ -143,10 +145,11 @@ func splitKeepingDelim(text string, pattern *regexp.Regexp) []string { cursor := 0 for _, idx := range idxs { start, end := idx[0], idx[1] - if start > cursor { - out = append(out, text[cursor:start]) + if start == cursor { + cursor = end + continue } - out = append(out, text[start:end]) + out = append(out, text[cursor:end]) cursor = end } if cursor < len(text) { diff --git a/internal/ingestion/component/chunker/token_test.go b/internal/ingestion/component/chunker/token_test.go index e6fe116231..f37d3a4484 100644 --- a/internal/ingestion/component/chunker/token_test.go +++ b/internal/ingestion/component/chunker/token_test.go @@ -99,6 +99,43 @@ func TestTokenChunker_InvokeDelimMode_BasicChunking(t *testing.T) { } } +// TestTokenChunker_DelimNeverStandaloneChunk is the regression test for +// the "666" bug: the delimiter must be glued to the end of the preceding +// segment (Python _split_text_by_pattern), never emitted as its own chunk. +func TestTokenChunker_DelimNeverStandaloneChunk(t *testing.T) { + c, err := NewTokenChunker(map[string]any{ + "delimiter_mode": "delimiter", + "delimiters": []string{"`666`"}, + }) + if err != nil { + t.Fatalf("NewTokenChunker: %v", err) + } + out, err := c.Invoke(context.Background(), map[string]any{ + "name": "doc.txt", + "output_format": "text", + "text": "alpha section\n666\nbeta section", + }) + if err != nil { + t.Fatalf("Invoke: %v", err) + } + chunks, _ := out["chunks"].([]map[string]any) + if len(chunks) != 2 { + t.Fatalf("chunks = %d, want 2: %v", len(chunks), chunks) + } + for i, ck := range chunks { + text, _ := ck["text"].(string) + if text == "666" || text == "\n666" || text == "666\n" { + t.Errorf("chunk[%d] is the bare delimiter %q", i, text) + } + } + if got, want := chunks[0]["text"], "alpha section\n666"; got != want { + t.Errorf("chunk[0] text = %q, want %q", got, want) + } + if got, want := chunks[1]["text"], "\nbeta section"; got != want { + t.Errorf("chunk[1] text = %q, want %q", got, want) + } +} + // TestTokenChunker_InvokeTokenSize_FallbackToMerge covers the // "no delimiter hit" branch — the chunker should fall back to // token-size merge and emit >=1 chunk.