fix(chunker): glue delimiter to preceding segment instead of emitting it as a standalone chunk (#17168)

internal/ingestion/component/chunker/common.go

@@ -126,16 +126,18 @@ func compileDelimPattern(delims []string) *regexp.Regexp {
 	return regexp.MustCompile(strings.Join(custom, "|"))
 }
 
-// splitKeepingDelim is the Go equivalent of python's
-// `re.split((pattern), text, flags=re.DOTALL)` with the matched
-// delimiter preserved (alternation keeps the original delimiter text
-// in the output stream so the rebuilding at token_chunker.py:88-93
-// stays lossy-free).
+// splitKeepingDelim mirrors Python token_chunker._split_text_by_pattern
+// (token_chunker.py:79-94): re.split with a captured delimiter group yields
+// [text, delim, text, delim, ...]; each delimiter is glued to the END of the
+// preceding text segment, so it never surfaces as a standalone chunk. A
+// delimiter with no preceding text (a leading delimiter or one adjacent to
+// another) is dropped together with the empty segment, matching Python's
+// `if not chunk: continue`.
 func splitKeepingDelim(text string, pattern *regexp.Regexp) []string {
 	if pattern == nil {
 		return []string{text}
 	}
-	idxs := pattern.FindAllStringSubmatchIndex(text, -1)
+	idxs := pattern.FindAllStringIndex(text, -1)
 	if len(idxs) == 0 {
 		return []string{text}
 	}
@@ -143,10 +145,11 @@ func splitKeepingDelim(text string, pattern *regexp.Regexp) []string {
 	cursor := 0
 	for _, idx := range idxs {
 		start, end := idx[0], idx[1]
-		if start > cursor {
-			out = append(out, text[cursor:start])
+		if start == cursor {
+			cursor = end
+			continue
 		}
-		out = append(out, text[start:end])
+		out = append(out, text[cursor:end])
 		cursor = end
 	}
 	if cursor < len(text) {

internal/ingestion/component/chunker/token_test.go

@@ -99,6 +99,43 @@ func TestTokenChunker_InvokeDelimMode_BasicChunking(t *testing.T) {
 	}
 }
 
+// TestTokenChunker_DelimNeverStandaloneChunk is the regression test for
+// the "666" bug: the delimiter must be glued to the end of the preceding
+// segment (Python _split_text_by_pattern), never emitted as its own chunk.
+func TestTokenChunker_DelimNeverStandaloneChunk(t *testing.T) {
+	c, err := NewTokenChunker(map[string]any{
+		"delimiter_mode": "delimiter",
+		"delimiters":     []string{"`666`"},
+	})
+	if err != nil {
+		t.Fatalf("NewTokenChunker: %v", err)
+	}
+	out, err := c.Invoke(context.Background(), map[string]any{
+		"name":          "doc.txt",
+		"output_format": "text",
+		"text":          "alpha section\n666\nbeta section",
+	})
+	if err != nil {
+		t.Fatalf("Invoke: %v", err)
+	}
+	chunks, _ := out["chunks"].([]map[string]any)
+	if len(chunks) != 2 {
+		t.Fatalf("chunks = %d, want 2: %v", len(chunks), chunks)
+	}
+	for i, ck := range chunks {
+		text, _ := ck["text"].(string)
+		if text == "666" || text == "\n666" || text == "666\n" {
+			t.Errorf("chunk[%d] is the bare delimiter %q", i, text)
+		}
+	}
+	if got, want := chunks[0]["text"], "alpha section\n666"; got != want {
+		t.Errorf("chunk[0] text = %q, want %q", got, want)
+	}
+	if got, want := chunks[1]["text"], "\nbeta section"; got != want {
+		t.Errorf("chunk[1] text = %q, want %q", got, want)
+	}
+}
+
 // TestTokenChunker_InvokeTokenSize_FallbackToMerge covers the
 // "no delimiter hit" branch — the chunker should fall back to
 // token-size merge and emit >=1 chunk.