diff --git a/internal/ingestion/component/chunker/token.go b/internal/ingestion/component/chunker/token.go index 81dc412c84..10fa83510a 100644 --- a/internal/ingestion/component/chunker/token.go +++ b/internal/ingestion/component/chunker/token.go @@ -1134,32 +1134,14 @@ func applyChildrenDelimText(docs []schema.ChunkDoc, pattern *regexp.Regexp) []sc // compileChildrenPattern is the children_delimiters version of // compileDelimPattern. Returns nil when no delimiters exist. +// compileChildrenPattern builds the children-split regex from a +// `children_delimiters` list. Every non-empty entry is active (including bare +// ones), and backtick-wrapped entries contribute their inner content — see +// chunk.CompileDelimiterPatternList. Delegating keeps children splitting +// consistent with the main delimiter list (backtick stripping + rune-descending +// order) instead of re-implementing a divergent copy. func compileChildrenPattern(delims []string) *regexp.Regexp { - if len(delims) == 0 { - return nil - } - escaped := make([]string, 0, len(delims)) - for _, d := range delims { - if d == "" { - continue - } - escaped = append(escaped, regexp.QuoteMeta(d)) - } - if len(escaped) == 0 { - return nil - } - sortSlice(escaped) - return regexp.MustCompile(strings.Join(escaped, "|")) -} - -// sortSlice sorts in place by descending length (longest pattern -// first, mirroring python's `sorted(set, key=len, reverse=True)`). -func sortSlice(in []string) { - for i := 1; i < len(in); i++ { - for j := i; j > 0 && len(in[j-1]) < len(in[j]); j-- { - in[j-1], in[j] = in[j], in[j-1] - } - } + return chunk.CompileDelimiterPatternList(delims, true) } // stringFromInputs returns the string value at the first matching key diff --git a/internal/ingestion/component/chunker/token_children_delim_test.go b/internal/ingestion/component/chunker/token_children_delim_test.go index 8337b1b0e9..b97b89f36f 100644 --- a/internal/ingestion/component/chunker/token_children_delim_test.go +++ b/internal/ingestion/component/chunker/token_children_delim_test.go @@ -2,6 +2,7 @@ package chunker import ( "context" + "strings" "testing" ) @@ -47,6 +48,60 @@ func TestTokenChunker_ChildrenDelimiterDroppedJSON(t *testing.T) { // children_delimiters split also DROPS the delimiter (applyChildrenDelim / // applyChildrenDelimText mirror _split_text_by_pattern), keeping the parent // segment in "mom". +// TestTokenChunker_ChildrenDelimiterBacktickStripped asserts that a +// backtick-wrapped children_delimiter contributes its INNER content as the +// split pattern (not the literal wrapped token), and the matched delimiter is +// dropped from each child — consistent with the main delimiter list behavior. +func TestTokenChunker_ChildrenDelimiterBacktickStripped(t *testing.T) { + c, err := NewTokenChunker(map[string]any{ + "delimiter_mode": "delimiter", + "delimiters": []string{"\n"}, + "children_delimiters": []string{"`###`"}, + }) + if err != nil { + t.Fatalf("NewTokenChunker: %v", err) + } + out, err := c.Invoke(context.Background(), nil, map[string]any{ + "name": "doc.txt", + "output_format": "text", + "text": "sec one###sec two###sec three", + }) + if err != nil { + t.Fatalf("Invoke: %v", err) + } + chunks, _ := out["chunks"].([]map[string]any) + want := []string{"sec one", "sec two", "sec three"} + if len(chunks) != len(want) { + t.Fatalf("chunk count: want %d got %d (%v)", len(want), len(chunks), chunkTexts(chunks)) + } + for i, w := range want { + got := chunks[i]["text"].(string) + if got != w { + t.Errorf("chunk[%d] text: want %q got %q", i, w, got) + } + } + // The literal backtick token `###` must never match as a whole string. + c2, _ := NewTokenChunker(map[string]any{ + "delimiter_mode": "delimiter", + "delimiters": []string{"\n"}, + "children_delimiters": []string{"`###`"}, + }) + out2, err := c2.Invoke(context.Background(), nil, map[string]any{ + "name": "doc.txt", + "output_format": "text", + "text": "a `###` b", + }) + if err != nil { + t.Fatalf("Invoke: %v", err) + } + chunks2, _ := out2["chunks"].([]map[string]any) + for _, ck := range chunks2 { + if strings.Contains(ck["text"].(string), "###") { + t.Errorf("child text kept literal backtick-wrapped token: %q", ck["text"].(string)) + } + } +} + func TestTokenChunker_ChildrenDelimiterDroppedText(t *testing.T) { c, err := NewTokenChunker(map[string]any{ "delimiter_mode": "delimiter", diff --git a/internal/parser/chunk/delim.go b/internal/parser/chunk/delim.go index aebadbd60a..1b897ab5ea 100644 --- a/internal/parser/chunk/delim.go +++ b/internal/parser/chunk/delim.go @@ -135,16 +135,28 @@ func CompileDelimiterPattern(delimiters []string) *regexp.Regexp { return regexp.MustCompile(strings.Join(escaped, "|")) } -// CompileDelimiterListPattern compiles a TokenChunker-style []string delimiter -// list. Entries wrapped in backticks contribute their inner content as a -// split pattern (Python token_chunker / historical Go compileDelimPattern). -// Bare list entries are ignored for the active pattern — they are only used -// by merge paths when no custom pattern exists. +// CompileDelimiterPatternList builds an alternation regex from a +// TokenChunker-style []string delimiter list, with a shared policy for how +// backtick-wrapped and bare entries are treated. Every non-empty entry is +// regexp.QuoteMeta'd and the result is sorted longest-first (stable by rune +// count) so a longer delimiter always wins over a shorter prefix inside it. // -// Prefer ParseDelimiterField for the single-string parser_config.delimiter -// field. This helper exists for the dataflow list API. -func CompileDelimiterListPattern(delims []string) *regexp.Regexp { - var custom []string +// - Backtick-wrapped entries (“ `abc` “) contribute their INNER content as +// the split pattern; the backticks are stripped. This is the Python +// token_chunker / historical Go compileDelimPattern convention. +// - Bare entries (e.g. ". ") behave according to keepBare: +// keepBare=false (the main `delimiters` list): a bare entry is IGNORED for +// the active pattern — it only influences merge paths when no custom +// pattern exists. +// keepBare=true (the `children_delimiters` list): a bare entry is KEPT +// active, because every non-empty children delimiter splits — including +// bare ones. +// +// Returns nil when no active entry remains. Prefer ParseDelimiterField for the +// single-string parser_config.delimiter field; this helper exists for the +// dataflow list API. +func CompileDelimiterPatternList(delims []string, keepBare bool) *regexp.Regexp { + var out []string for _, d := range delims { if d == "" { continue @@ -154,22 +166,38 @@ func CompileDelimiterListPattern(delims []string) *regexp.Regexp { if inner == "" { continue } - custom = append(custom, inner) + out = append(out, inner) + continue } + if !keepBare { + continue + } + out = append(out, d) } - if len(custom) == 0 { + if len(out) == 0 { return nil } - sort.SliceStable(custom, func(i, j int) bool { - return utf8.RuneCountInString(custom[i]) > utf8.RuneCountInString(custom[j]) + sort.SliceStable(out, func(i, j int) bool { + return utf8.RuneCountInString(out[i]) > utf8.RuneCountInString(out[j]) }) - escaped := make([]string, 0, len(custom)) - for _, d := range custom { + escaped := make([]string, 0, len(out)) + for _, d := range out { escaped = append(escaped, regexp.QuoteMeta(d)) } return regexp.MustCompile(strings.Join(escaped, "|")) } +// CompileDelimiterListPattern compiles a TokenChunker-style []string delimiter +// list. It is CompileDelimiterPatternList with keepBare=false: backtick entries +// contribute their inner content and bare entries are ignored for the active +// pattern (they only steer merge paths when no custom pattern exists). +// +// Prefer ParseDelimiterField for the single-string parser_config.delimiter +// field. This helper exists for the dataflow list API. +func CompileDelimiterListPattern(delims []string) *regexp.Regexp { + return CompileDelimiterPatternList(delims, false) +} + // HasCustomDelimiterList reports whether any entry in a TokenChunker-style // delimiter list uses backtick syntax. func HasCustomDelimiterList(delims []string) bool { diff --git a/internal/parser/chunk/delim_test.go b/internal/parser/chunk/delim_test.go index 7a1034200b..4c5a65fb43 100644 --- a/internal/parser/chunk/delim_test.go +++ b/internal/parser/chunk/delim_test.go @@ -175,6 +175,44 @@ func TestCompileDelimiterListPattern(t *testing.T) { } } +func TestCompileDelimiterPatternListKeepBare(t *testing.T) { + // keepBare=true (children_delimiters): bare entries stay active and + // backtick entries contribute their inner content, all QuoteMeta'd and + // sorted longest-first by rune count. + pat := CompileDelimiterPatternList([]string{". ", "`###`", "#"}, true) + if pat == nil { + t.Fatal("expected a pattern from mixed bare + wrapped entries") + } + want := `###|\. |#` + if got := pat.String(); got != want { + t.Errorf("pattern = %q, want %q", got, want) + } + // Backtick is stripped: splitting on "###" matches the inner content, not + // the literal wrapped token. + if got := pat.FindString("a###b"); got != "###" { + t.Errorf("FindString(a###b) = %q, want %q", got, "###") + } + // Bare ". " stays active and is matched as the meta-escaped ". ". + if got := pat.FindString("alpha. beta"); got != ". " { + t.Errorf("FindString(alpha. beta) = %q, want %q", got, ". ") + } +} + +func TestCompileDelimiterPatternListKeepBareFalse(t *testing.T) { + // keepBare=false must equal CompileDelimiterListPattern: bare entries + // ignored, only wrapped inner content participates. + if CompileDelimiterPatternList([]string{"\n", "!"}, false) != nil { + t.Fatal("bare entries should be ignored when keepBare=false") + } + pat := CompileDelimiterPatternList([]string{"`##`", "`#`"}, false) + if pat == nil { + t.Fatal("expected pattern from wrapped entries") + } + if got := pat.FindString("###"); got != "##" { + t.Errorf("got %q want ##", got) + } +} + func TestHasCustomDelimiterList(t *testing.T) { if HasCustomDelimiterList([]string{"\n", "!"}) { t.Fatal("bare list should be false")