fix(go): unify children delimiter pattern with backtick-strip + rune order (#17926)

`compileChildrenPattern` re-implemented the delimiter-list compile
inline with two divergences from the shared
`CompileDelimiterListPattern`:

- It never stripped backticks, so a backtick-wrapped
`children_delimiter` like `` `###` `` matched the **literal wrapped
token** rather than the inner `###`.
- It sorted by **byte length** instead of rune count (`sortSlice`), so
multi-byte delimiters could be ordered incorrectly and a longer
delimiter could fail to win over a shorter prefix.

internal/ingestion/component/chunker/token.go

@@ -1134,32 +1134,14 @@ func applyChildrenDelimText(docs []schema.ChunkDoc, pattern *regexp.Regexp) []sc
 
 // compileChildrenPattern is the children_delimiters version of
 // compileDelimPattern. Returns nil when no delimiters exist.
+// compileChildrenPattern builds the children-split regex from a
+// `children_delimiters` list. Every non-empty entry is active (including bare
+// ones), and backtick-wrapped entries contribute their inner content — see
+// chunk.CompileDelimiterPatternList. Delegating keeps children splitting
+// consistent with the main delimiter list (backtick stripping + rune-descending
+// order) instead of re-implementing a divergent copy.
 func compileChildrenPattern(delims []string) *regexp.Regexp {
-	if len(delims) == 0 {
-		return nil
-	}
-	escaped := make([]string, 0, len(delims))
-	for _, d := range delims {
-		if d == "" {
-			continue
-		}
-		escaped = append(escaped, regexp.QuoteMeta(d))
-	}
-	if len(escaped) == 0 {
-		return nil
-	}
-	sortSlice(escaped)
-	return regexp.MustCompile(strings.Join(escaped, "|"))
-}
-
-// sortSlice sorts in place by descending length (longest pattern
-// first, mirroring python's `sorted(set, key=len, reverse=True)`).
-func sortSlice(in []string) {
-	for i := 1; i < len(in); i++ {
-		for j := i; j > 0 && len(in[j-1]) < len(in[j]); j-- {
-			in[j-1], in[j] = in[j], in[j-1]
-		}
-	}
+	return chunk.CompileDelimiterPatternList(delims, true)
 }
 
 // stringFromInputs returns the string value at the first matching key

internal/ingestion/component/chunker/token_children_delim_test.go

@@ -2,6 +2,7 @@ package chunker
 
 import (
 	"context"
+	"strings"
 	"testing"
 )
 
@@ -47,6 +48,60 @@ func TestTokenChunker_ChildrenDelimiterDroppedJSON(t *testing.T) {
 // children_delimiters split also DROPS the delimiter (applyChildrenDelim /
 // applyChildrenDelimText mirror _split_text_by_pattern), keeping the parent
 // segment in "mom".
+// TestTokenChunker_ChildrenDelimiterBacktickStripped asserts that a
+// backtick-wrapped children_delimiter contributes its INNER content as the
+// split pattern (not the literal wrapped token), and the matched delimiter is
+// dropped from each child — consistent with the main delimiter list behavior.
+func TestTokenChunker_ChildrenDelimiterBacktickStripped(t *testing.T) {
+	c, err := NewTokenChunker(map[string]any{
+		"delimiter_mode":      "delimiter",
+		"delimiters":          []string{"\n"},
+		"children_delimiters": []string{"`###`"},
+	})
+	if err != nil {
+		t.Fatalf("NewTokenChunker: %v", err)
+	}
+	out, err := c.Invoke(context.Background(), nil, map[string]any{
+		"name":          "doc.txt",
+		"output_format": "text",
+		"text":          "sec one###sec two###sec three",
+	})
+	if err != nil {
+		t.Fatalf("Invoke: %v", err)
+	}
+	chunks, _ := out["chunks"].([]map[string]any)
+	want := []string{"sec one", "sec two", "sec three"}
+	if len(chunks) != len(want) {
+		t.Fatalf("chunk count: want %d got %d (%v)", len(want), len(chunks), chunkTexts(chunks))
+	}
+	for i, w := range want {
+		got := chunks[i]["text"].(string)
+		if got != w {
+			t.Errorf("chunk[%d] text: want %q got %q", i, w, got)
+		}
+	}
+	// The literal backtick token `###` must never match as a whole string.
+	c2, _ := NewTokenChunker(map[string]any{
+		"delimiter_mode":      "delimiter",
+		"delimiters":          []string{"\n"},
+		"children_delimiters": []string{"`###`"},
+	})
+	out2, err := c2.Invoke(context.Background(), nil, map[string]any{
+		"name":          "doc.txt",
+		"output_format": "text",
+		"text":          "a `###` b",
+	})
+	if err != nil {
+		t.Fatalf("Invoke: %v", err)
+	}
+	chunks2, _ := out2["chunks"].([]map[string]any)
+	for _, ck := range chunks2 {
+		if strings.Contains(ck["text"].(string), "###") {
+			t.Errorf("child text kept literal backtick-wrapped token: %q", ck["text"].(string))
+		}
+	}
+}
+
 func TestTokenChunker_ChildrenDelimiterDroppedText(t *testing.T) {
 	c, err := NewTokenChunker(map[string]any{
 		"delimiter_mode":      "delimiter",

internal/parser/chunk/delim.go

@@ -135,16 +135,28 @@ func CompileDelimiterPattern(delimiters []string) *regexp.Regexp {
 	return regexp.MustCompile(strings.Join(escaped, "|"))
 }
 
-// CompileDelimiterListPattern compiles a TokenChunker-style []string delimiter
-// list. Entries wrapped in backticks contribute their inner content as a
-// split pattern (Python token_chunker / historical Go compileDelimPattern).
-// Bare list entries are ignored for the active pattern — they are only used
-// by merge paths when no custom pattern exists.
+// CompileDelimiterPatternList builds an alternation regex from a
+// TokenChunker-style []string delimiter list, with a shared policy for how
+// backtick-wrapped and bare entries are treated. Every non-empty entry is
+// regexp.QuoteMeta'd and the result is sorted longest-first (stable by rune
+// count) so a longer delimiter always wins over a shorter prefix inside it.
 //
-// Prefer ParseDelimiterField for the single-string parser_config.delimiter
-// field. This helper exists for the dataflow list API.
-func CompileDelimiterListPattern(delims []string) *regexp.Regexp {
-	var custom []string
+//   - Backtick-wrapped entries (“ `abc` “) contribute their INNER content as
+//     the split pattern; the backticks are stripped. This is the Python
+//     token_chunker / historical Go compileDelimPattern convention.
+//   - Bare entries (e.g. ". ") behave according to keepBare:
+//     keepBare=false (the main `delimiters` list): a bare entry is IGNORED for
+//     the active pattern — it only influences merge paths when no custom
+//     pattern exists.
+//     keepBare=true (the `children_delimiters` list): a bare entry is KEPT
+//     active, because every non-empty children delimiter splits — including
+//     bare ones.
+//
+// Returns nil when no active entry remains. Prefer ParseDelimiterField for the
+// single-string parser_config.delimiter field; this helper exists for the
+// dataflow list API.
+func CompileDelimiterPatternList(delims []string, keepBare bool) *regexp.Regexp {
+	var out []string
 	for _, d := range delims {
 		if d == "" {
 			continue
@@ -154,22 +166,38 @@ func CompileDelimiterListPattern(delims []string) *regexp.Regexp {
 			if inner == "" {
 				continue
 			}
-			custom = append(custom, inner)
+			out = append(out, inner)
+			continue
 		}
+		if !keepBare {
+			continue
+		}
+		out = append(out, d)
 	}
-	if len(custom) == 0 {
+	if len(out) == 0 {
 		return nil
 	}
-	sort.SliceStable(custom, func(i, j int) bool {
-		return utf8.RuneCountInString(custom[i]) > utf8.RuneCountInString(custom[j])
+	sort.SliceStable(out, func(i, j int) bool {
+		return utf8.RuneCountInString(out[i]) > utf8.RuneCountInString(out[j])
 	})
-	escaped := make([]string, 0, len(custom))
-	for _, d := range custom {
+	escaped := make([]string, 0, len(out))
+	for _, d := range out {
 		escaped = append(escaped, regexp.QuoteMeta(d))
 	}
 	return regexp.MustCompile(strings.Join(escaped, "|"))
 }
 
+// CompileDelimiterListPattern compiles a TokenChunker-style []string delimiter
+// list. It is CompileDelimiterPatternList with keepBare=false: backtick entries
+// contribute their inner content and bare entries are ignored for the active
+// pattern (they only steer merge paths when no custom pattern exists).
+//
+// Prefer ParseDelimiterField for the single-string parser_config.delimiter
+// field. This helper exists for the dataflow list API.
+func CompileDelimiterListPattern(delims []string) *regexp.Regexp {
+	return CompileDelimiterPatternList(delims, false)
+}
+
 // HasCustomDelimiterList reports whether any entry in a TokenChunker-style
 // delimiter list uses backtick syntax.
 func HasCustomDelimiterList(delims []string) bool {

internal/parser/chunk/delim_test.go

@@ -175,6 +175,44 @@ func TestCompileDelimiterListPattern(t *testing.T) {
 	}
 }
 
+func TestCompileDelimiterPatternListKeepBare(t *testing.T) {
+	// keepBare=true (children_delimiters): bare entries stay active and
+	// backtick entries contribute their inner content, all QuoteMeta'd and
+	// sorted longest-first by rune count.
+	pat := CompileDelimiterPatternList([]string{". ", "`###`", "#"}, true)
+	if pat == nil {
+		t.Fatal("expected a pattern from mixed bare + wrapped entries")
+	}
+	want := `###|\. |#`
+	if got := pat.String(); got != want {
+		t.Errorf("pattern = %q, want %q", got, want)
+	}
+	// Backtick is stripped: splitting on "###" matches the inner content, not
+	// the literal wrapped token.
+	if got := pat.FindString("a###b"); got != "###" {
+		t.Errorf("FindString(a###b) = %q, want %q", got, "###")
+	}
+	// Bare ". " stays active and is matched as the meta-escaped ". ".
+	if got := pat.FindString("alpha. beta"); got != ". " {
+		t.Errorf("FindString(alpha. beta) = %q, want %q", got, ". ")
+	}
+}
+
+func TestCompileDelimiterPatternListKeepBareFalse(t *testing.T) {
+	// keepBare=false must equal CompileDelimiterListPattern: bare entries
+	// ignored, only wrapped inner content participates.
+	if CompileDelimiterPatternList([]string{"\n", "!"}, false) != nil {
+		t.Fatal("bare entries should be ignored when keepBare=false")
+	}
+	pat := CompileDelimiterPatternList([]string{"`##`", "`#`"}, false)
+	if pat == nil {
+		t.Fatal("expected pattern from wrapped entries")
+	}
+	if got := pat.FindString("###"); got != "##" {
+		t.Errorf("got %q want ##", got)
+	}
+}
+
 func TestHasCustomDelimiterList(t *testing.T) {
 	if HasCustomDelimiterList([]string{"\n", "!"}) {
 		t.Fatal("bare list should be false")