refactor(nlp): collapse 6 delimiter-parser implementations into one canonical helper (#17383) (#17387)

## Summary Six sites used to read the same `parser_config.delimiter` field with divergent grammars: - `rag.nlp.get_delimiters` (PDF/DOCX/HTML/EPUB/JSON/CSV/XLSX/email/book) - `rag.nlp.naive_merge` (custom-delimiter branch) - `rag.nlp.naive_merge_with_images` - `rag.nlp._build_cks` - `deepdoc.parser.txt_parser.parser_txt` (.txt, code) - `deepdoc.parser.markdown_parser.MarkdownElementExtractor.get_delimiters` The six implementations disagreed on bare-vs-wrapped chars, dedupe, sort order, CRLF normalization, and `re.I` (#17384). The shipped default `` `\n!?;。；！？` `` was a no-op for `.md` because the markdown path only matched backtick-wrapped tokens. ## Changes - **new:** `rag/nlp/delim.py` with `parse_delimiter_field` and `compile_delimiter_pattern`. Single source of truth. CRLF normalization at the top; longest-first stable sort; insertion-ordered dedupe; no `re.I`. - **refactor:** all six call sites delegate to the helper. - `rag/nlp/__init__.py::get_delimiters` becomes a thin shim. - `deepdoc/parser/txt_parser.py::parser_txt` drops the `[encode/decode/unicode_escape]` round-trip. - `deepdoc/parser/markdown_parser.py::get_delimiters` honors bare chars (fixes [1]). - **tests:** `test/unit_test/rag/test_delim.py` (85 tests) — helper, acceptance table, frontend parity, static guard against re-inlining. - **tests:** `test/unit_test/rag/test_delimiter_case_sensitive.py` (from #17386) updated to retarget the static check at the new helper + AST-based broader scan. ## Acceptance criteria - All six sites produce the same regex pattern for the same input. - Shipped default keeps working for `.txt` / `.pdf` / `.docx`. - Shipped default for `.md` now splits (was a silent no-op). - Tooltip example `` `\n##;` `` produces three effective delimiters regardless of file type. - Bare whitespace inputs split on every occurrence. - Backtick-wrapped whitespace splits only on the exact N-char sequence. - CRLF-line-ending documents split identically to LF-line-ending documents. - 123 tests pass (85 new + 38 existing). ## Rebase protocol As #17385 and #17386 evolve, this branch will be rebased on top. The only overlap between this PR's diff and the other two is `test_delimiter_case_sensitive.py`, where #17383 modifies the static check to point at the new helper location. --------- Co-authored-by: kiloconnect[bot] <240665456+kiloconnect[bot]@users.noreply.github.com>
2026-08-03 06:17:29 +08:00 · 2026-08-02 14:37:14 +05:30
parent 01d667296d
commit d4ceeee4ed
14 changed files with 1551 additions and 321 deletions
--- a/internal/ingestion/component/chunker/common.go
+++ b/internal/ingestion/component/chunker/common.go
@@ -19,11 +19,11 @@ package chunker
 import (
 	"fmt"
 	"regexp"
-	"sort"
 	"strings"

 	"ragflow/internal/agent/runtime"
 	"ragflow/internal/ingestion/component/schema"
+	"ragflow/internal/parser/chunk"
 	"ragflow/internal/tokenizer"
 )

@@ -76,87 +76,14 @@ func stringListFromAny(in []any) []string {
 // regex / split helpers
 // ---------------------------------------------------------------------------

-// backtickDelimRE extracts backtick-wrapped delimiter tokens.
-// Case-sensitive on purpose (mirrors Python after #17384 / PR #17386):
-// never add (?i) here — delimiter matching must preserve letter casing.
-var backtickDelimRE = regexp.MustCompile("`([^`]+)`")
-
-// escapeAndSort QuoteMeta-escapes tokens and sorts longest-first.
-// Empty tokens are dropped. When dedup is true, exact duplicate tokens
-// are collapsed (first occurrence wins). Matching stays case-sensitive.
-func escapeAndSort(tokens []string, dedup bool) []string {
-	out := make([]string, 0, len(tokens))
-	var seen map[string]struct{}
-	if dedup {
-		seen = make(map[string]struct{}, len(tokens))
-	}
-	for _, tok := range tokens {
-		if tok == "" {
-			continue
-		}
-		if dedup {
-			if _, ok := seen[tok]; ok {
-				continue
-			}
-			seen[tok] = struct{}{}
-		}
-		out = append(out, regexp.QuoteMeta(tok))
-	}
-	sort.SliceStable(out, func(i, j int) bool { return len(out[i]) > len(out[j]) })
-	return out
-}
-
-// getDelimiters ports rag.nlp.get_delimiters (rag/nlp/__init__.py).
-//
-// It walks a single delimiter string, pulling out backtick-wrapped tokens
-// and every bare character outside those spans, then returns a
-// length-sorted, QuoteMeta-escaped alternation suitable for regexp.Split.
-// Matching is case-sensitive: "a" does not match "A", and "`end`" does not
-// match "End" / "END".
-func getDelimiters(delimiters string) string {
-	var dels []string
-	s := 0
-	for _, m := range backtickDelimRE.FindAllStringSubmatchIndex(delimiters, -1) {
-		// m = [fullStart, fullEnd, g1Start, g1End]
-		f, t := m[0], m[1]
-		dels = append(dels, delimiters[m[2]:m[3]])
-		for _, r := range delimiters[s:f] {
-			dels = append(dels, string(r))
-		}
-		s = t
-	}
-	if s < len(delimiters) {
-		for _, r := range delimiters[s:] {
-			dels = append(dels, string(r))
-		}
-	}
-	// Python get_delimiters does not dedup; preserve that behavior.
-	return strings.Join(escapeAndSort(dels, false), "|")
-}
-
-// compileDelimPattern builds an alternation from backtick-wrapped tokens
-// across delimiter entries (mirrors Python _compile_delimiter_pattern).
-// Each entry is scanned independently so token boundaries cannot span
-// adjacent slice elements. Extraction is case-sensitive — see backtickDelimRE.
-//
-// Plain (non-backtick) delimiters are not compiled here; callers that
-// need bare-char splitting use getDelimiters (naive_merge path).
+// compileDelimPattern compiles a TokenChunker-style []string delimiter list.
+// Only backtick-wrapped entries produce an active pattern (Python
+// token_chunker / rag/nlp/delim list helper). Plain entries are ignored here
+// and used by mergeByTokenSize for sentence-level splitting when no active
+// pattern exists. Canonical single-string parser_config.delimiter parsing
+// lives in ragflow/internal/parser/chunk (ParseDelimiterField).
 func compileDelimPattern(delims []string) *regexp.Regexp {
-	var tokens []string
-	for _, d := range delims {
-		if d == "" {
-			continue
-		}
-		for _, m := range backtickDelimRE.FindAllStringSubmatch(d, -1) {
-			tokens = append(tokens, m[1])
-		}
-	}
-	// Python _compile_delimiter_pattern dedups via set(...).
-	custom := escapeAndSort(tokens, true)
-	if len(custom) == 0 {
-		return nil
-	}
-	return regexp.MustCompile(strings.Join(custom, "|"))
+	return chunk.CompileDelimiterListPattern(delims)
 }

 // splitKeepingDelim mirrors Python token_chunker._split_text_by_pattern
--- a/internal/ingestion/component/chunker/delimiter_case_sensitive_test.go
+++ b/internal/ingestion/component/chunker/delimiter_case_sensitive_test.go
@@ -32,8 +32,18 @@ import (
 	"regexp"
 	"strings"
 	"testing"
+
+	"ragflow/internal/parser/chunk"
 )

+func getDelimiters(delimiters string) string {
+	p := chunk.CompileDelimiterPattern(chunk.ParseDelimiterField(delimiters))
+	if p == nil {
+		return ""
+	}
+	return p.String()
+}
+
 func TestGetDelimiters_BareCharAReturnsLiteralPattern(t *testing.T) {
 	// Bare-char delimiter "a" must produce the pattern "a", not "a|A".
 	if got, want := getDelimiters("a"), "a"; got != want {
@@ -219,15 +229,20 @@ func TestTokenChunker_BacktickASplitsOnlyAtLowercase(t *testing.T) {
 	}
 }

-func TestBacktickDelimRE_HasNoIgnoreCaseFlag(t *testing.T) {
-	// Structural guard: the shared backtick extractor must stay case-sensitive.
-	src := backtickDelimRE.String()
-	if strings.Contains(src, "(?i)") || strings.HasPrefix(src, "(?i)") {
-		t.Fatalf("backtickDelimRE must not use (?i); got %q", src)
+func TestBacktickDelimiterIsCaseSensitive(t *testing.T) {
+	// Extraction and compiled pattern must both preserve letter casing.
+	parsed := chunk.ParseDelimiterField("`End`")
+	if len(parsed) != 1 || parsed[0] != "End" {
+		t.Fatalf("ParseDelimiterField(`End`) = %#v, want [End]", parsed)
 	}
-	// Sanity: the pattern still extracts backtick contents.
-	m := backtickDelimRE.FindStringSubmatch("`End`")
-	if len(m) < 2 || m[1] != "End" {
-		t.Fatalf("backtickDelimRE(`End`) = %#v, want group1=End", m)
+	p := chunk.CompileDelimiterPattern(parsed)
+	if p == nil {
+		t.Fatal("CompileDelimiterPattern returned nil")
+	}
+	if !p.MatchString("End") || p.MatchString("end") || p.MatchString("END") {
+		t.Fatalf("pattern %q is not case-sensitive", p.String())
+	}
+	if strings.Contains(p.String(), "(?i)") {
+		t.Fatalf("compiled pattern must not carry (?i); got %q", p.String())
 	}
 }
--- a/internal/ingestion/component/chunker/token.go
+++ b/internal/ingestion/component/chunker/token.go
@@ -23,10 +23,11 @@
 //     mirroring Python's normalize_overlapped_percent), table_context_size ≥ 0,
 //     image_context_size ≥ 0. enum/range checks live in param.Check.
 //
-//   - DELIMITER PARSING mirrors python `_compile_delimiter_pattern`:
-//     entries wrapped in backticks (e.g. "`\\n\\n`") are treated as
-//     regex split points; plain strings are regex-escaped and joined
-//     into the same alternation. Empty entries are filtered.
+//   - DELIMITER PARSING for the TokenChunker list API mirrors Python
+//     token_chunker: only entries wrapped in backticks (e.g. "`\\n\\n`")
+//     produce an active split pattern. Plain list entries are not
+//     compiled into the pattern. Single-string parser_config.delimiter
+//     parsing lives in ragflow/internal/parser/chunk (ParseDelimiterField).
 //
 //   - CHILDREN DELIMITERS (the secondary split) is implemented via the
 //     shared splitKeepingDelim helper; emitted chunks carry the parent
@@ -63,12 +64,13 @@ import (
 	"strings"
 	"sync"

+	"gorm.io/gorm"
 	"ragflow/internal/agent/runtime"
 	deepdoctype "ragflow/internal/deepdoc/parser/type"
 	"ragflow/internal/ingestion/component/globals"
 	"ragflow/internal/ingestion/component/schema"

-	"gorm.io/gorm"
+	"ragflow/internal/parser/chunk"
 )

 const ComponentNameTokenChunker = "TokenChunker"
@@ -1075,14 +1077,9 @@ func hasActiveDelimiter(p *regexp.Regexp) bool {

 // hasCustomDelim reports whether any delimiter uses backtick syntax
 // (`pattern`). Python's naive_merge skips token-size merging when
-// custom delimiters are present (naive_merge:1194-1213).
+// custom delimiters are present. Delegates to the canonical helper.
 func hasCustomDelim(delims []string) bool {
-	for _, d := range delims {
-		if strings.HasPrefix(d, "`") && strings.HasSuffix(d, "`") && len(d) >= 2 {
-			return true
-		}
-	}
-	return false
+	return chunk.HasCustomDelimiterList(delims)
 }

 // applyChildrenDelim mirrors token_chunker.py:325-334.
--- a/internal/parser/chunk/delim.go
+++ b/internal/parser/chunk/delim.go
@@ -0,0 +1,194 @@
+//
+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+package chunk
+
+import (
+	"regexp"
+	"sort"
+	"strings"
+	"unicode/utf8"
+)
+
+// Canonical parser for the parser_config.delimiter field.
+//
+// Mirrors Python rag/nlp/delim.py (#17383). A delimiter field is a string
+// with the grammar:
+//
+//	delimiter_field  := token*
+//	token            := backtick_wrapped | bare_char
+//	backtick_wrapped := "`" bare_char+ "`"
+//	bare_char        := any single Unicode character except "`"
+//
+// Semantics:
+//  1. Characters between matching backticks form one multi-character delimiter.
+//  2. Any character outside backticks is its own single-character delimiter.
+//  3. Results are deduplicated and sorted longest-first (stable for equal length).
+//  4. CRLF and standalone CR are normalized to LF before parsing.
+//  5. Matching is case-sensitive.
+
+// backtickWrappedRE matches a backtick-wrapped multi-character token.
+// Case-sensitive on purpose (see #17384).
+var backtickWrappedRE = regexp.MustCompile("`([^`]+)`")
+
+// NormalizeTextNewlines converts CRLF and standalone CR to LF.
+func NormalizeTextNewlines(text string) string {
+	if text == "" {
+		return text
+	}
+	text = strings.ReplaceAll(text, "\r\n", "\n")
+	return strings.ReplaceAll(text, "\r", "\n")
+}
+
+// HasWrappedDelimiter reports whether the delimiter field contains at least
+// one backtick-wrapped token. Used to decide the historical "custom delimiter"
+// mode that bypasses chunk_token_num. Separate from whether any delimiter is
+// present after parsing (bare single-character delimiters still split).
+func HasWrappedDelimiter(s string) bool {
+	if s == "" {
+		return false
+	}
+	return backtickWrappedRE.MatchString(s)
+}
+
+// ParseDelimiterField parses the delimiter field into delimiter strings.
+//
+// Returns nil for an empty field. Whitespace characters are valid
+// single-character delimiters. Output is sorted longest-first and
+// deduplicated while preserving first-occurrence order for equal-length
+// items (the sort is stable). CRLF/CR inside the field are normalized to LF.
+func ParseDelimiterField(s string) []string {
+	if s == "" {
+		return nil
+	}
+	normalized := NormalizeTextNewlines(s)
+
+	var delimiters []string
+	seen := make(map[string]struct{})
+	cursor := 0
+	for _, loc := range backtickWrappedRE.FindAllStringSubmatchIndex(normalized, -1) {
+		// loc: [fullStart, fullEnd, groupStart, groupEnd]
+		start, end := loc[0], loc[1]
+		for _, ch := range runesOf(normalized[cursor:start]) {
+			if _, ok := seen[ch]; ok {
+				continue
+			}
+			seen[ch] = struct{}{}
+			delimiters = append(delimiters, ch)
+		}
+		token := normalized[loc[2]:loc[3]]
+		if token != "" {
+			if _, ok := seen[token]; !ok {
+				seen[token] = struct{}{}
+				delimiters = append(delimiters, token)
+			}
+		}
+		cursor = end
+	}
+	for _, ch := range runesOf(normalized[cursor:]) {
+		if _, ok := seen[ch]; ok {
+			continue
+		}
+		seen[ch] = struct{}{}
+		delimiters = append(delimiters, ch)
+	}
+
+	sort.SliceStable(delimiters, func(i, j int) bool {
+		return utf8.RuneCountInString(delimiters[i]) > utf8.RuneCountInString(delimiters[j])
+	})
+	return delimiters
+}
+
+// CompileDelimiterPattern builds an alternation regex from delimiter strings.
+//
+// Each delimiter is regexp.QuoteMeta'd so whitespace and metacharacters match
+// literally. Returns nil when delimiters is empty or yields no non-empty
+// entries. The pattern is intended for use with a capturing split (so
+// delimiters appear in the split output) or FindAllStringIndex.
+func CompileDelimiterPattern(delimiters []string) *regexp.Regexp {
+	if len(delimiters) == 0 {
+		return nil
+	}
+	escaped := make([]string, 0, len(delimiters))
+	for _, d := range delimiters {
+		if d == "" {
+			continue
+		}
+		escaped = append(escaped, regexp.QuoteMeta(d))
+	}
+	if len(escaped) == 0 {
+		return nil
+	}
+	return regexp.MustCompile(strings.Join(escaped, "|"))
+}
+
+// CompileDelimiterListPattern compiles a TokenChunker-style []string delimiter
+// list. Entries wrapped in backticks contribute their inner content as a
+// split pattern (Python token_chunker / historical Go compileDelimPattern).
+// Bare list entries are ignored for the active pattern — they are only used
+// by merge paths when no custom pattern exists.
+//
+// Prefer ParseDelimiterField for the single-string parser_config.delimiter
+// field. This helper exists for the dataflow list API.
+func CompileDelimiterListPattern(delims []string) *regexp.Regexp {
+	var custom []string
+	for _, d := range delims {
+		if d == "" {
+			continue
+		}
+		if strings.HasPrefix(d, "`") && strings.HasSuffix(d, "`") && len(d) >= 2 {
+			inner := d[1 : len(d)-1]
+			if inner == "" {
+				continue
+			}
+			custom = append(custom, inner)
+		}
+	}
+	if len(custom) == 0 {
+		return nil
+	}
+	sort.SliceStable(custom, func(i, j int) bool {
+		return utf8.RuneCountInString(custom[i]) > utf8.RuneCountInString(custom[j])
+	})
+	escaped := make([]string, 0, len(custom))
+	for _, d := range custom {
+		escaped = append(escaped, regexp.QuoteMeta(d))
+	}
+	return regexp.MustCompile(strings.Join(escaped, "|"))
+}
+
+// HasCustomDelimiterList reports whether any entry in a TokenChunker-style
+// delimiter list uses backtick syntax.
+func HasCustomDelimiterList(delims []string) bool {
+	for _, d := range delims {
+		if strings.HasPrefix(d, "`") && strings.HasSuffix(d, "`") && len(d) >= 2 {
+			return true
+		}
+	}
+	return false
+}
+
+// runesOf splits s into individual Unicode characters (as strings).
+func runesOf(s string) []string {
+	if s == "" {
+		return nil
+	}
+	out := make([]string, 0, utf8.RuneCountInString(s))
+	for _, r := range s {
+		out = append(out, string(r))
+	}
+	return out
+}
--- a/internal/parser/chunk/delim_test.go
+++ b/internal/parser/chunk/delim_test.go
@@ -0,0 +1,185 @@
+//
+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+package chunk
+
+import (
+	"reflect"
+	"regexp"
+	"testing"
+)
+
+func TestNormalizeTextNewlines(t *testing.T) {
+	tests := []struct {
+		in, want string
+	}{
+		{"", ""},
+		{"a\nb", "a\nb"},
+		{"a\r\nb", "a\nb"},
+		{"a\rb", "a\nb"},
+		{"a\r\nb\rc", "a\nb\nc"},
+	}
+	for _, tc := range tests {
+		if got := NormalizeTextNewlines(tc.in); got != tc.want {
+			t.Errorf("NormalizeTextNewlines(%q) = %q, want %q", tc.in, got, tc.want)
+		}
+	}
+}
+
+func TestHasWrappedDelimiter(t *testing.T) {
+	if HasWrappedDelimiter("") {
+		t.Fatal("empty should be false")
+	}
+	if HasWrappedDelimiter("!;") {
+		t.Fatal("bare chars should be false")
+	}
+	if !HasWrappedDelimiter("`##`") {
+		t.Fatal("wrapped should be true")
+	}
+	if !HasWrappedDelimiter("\n`##`;") {
+		t.Fatal("mixed should be true")
+	}
+	if !HasWrappedDelimiter("`;`") {
+		t.Fatal("wrapped single char should be true")
+	}
+}
+
+func TestParseDelimiterField(t *testing.T) {
+	tests := []struct {
+		name  string
+		field string
+		want  []string
+	}{
+		{"empty", "", nil},
+		{"single bare", "!", []string{"!"}},
+		{"bare pair", "!?", []string{"!", "?"}},
+		{"space", " ", []string{" "}},
+		{"newline", "\n", []string{"\n"}},
+		{"crlf field", "\r\n", []string{"\n"}},
+		{"bare cr", "\r", []string{"\n"}},
+		{"wrapped end", "`end`", []string{"end"}},
+		{"wrapped hierarchy", "`###``##``#`", []string{"###", "##", "#"}},
+		{"tooltip example", "\n`##`;", []string{"##", "\n", ";"}},
+		{"dedupe", "`a`a`a`", []string{"a"}},
+		{"wrapped double newline", "`\n\n`", []string{"\n\n"}},
+		{"crlf wrapped", "`\r\n`", []string{"\n"}},
+		{"shipped default", "\n!?;。；！？", []string{"\n", "!", "?", ";", "。", "；", "！", "？"}},
+		{"chinese", "。；", []string{"。", "；"}},
+		{"mixed order equal length", "`##`#\n", []string{"##", "#", "\n"}},
+		{"empty backticks bare", "``", []string{"`"}},
+		{"double space bare dedupe", "  ", []string{" "}},
+		{"wrapped double space", "`  `", []string{"  "}},
+	}
+	for _, tc := range tests {
+		t.Run(tc.name, func(t *testing.T) {
+			got := ParseDelimiterField(tc.field)
+			if !reflect.DeepEqual(got, tc.want) {
+				t.Errorf("ParseDelimiterField(%q) = %#v, want %#v", tc.field, got, tc.want)
+			}
+		})
+	}
+}
+
+func TestCompileDelimiterPattern(t *testing.T) {
+	if CompileDelimiterPattern(nil) != nil {
+		t.Fatal("empty list should yield nil")
+	}
+	if CompileDelimiterPattern([]string{}) != nil {
+		t.Fatal("empty slice should yield nil")
+	}
+
+	pat := CompileDelimiterPattern([]string{"##", "#"})
+	if pat == nil {
+		t.Fatal("expected pattern")
+	}
+	if got := pat.FindString("###"); got != "##" {
+		t.Errorf("longest match: got %q, want ##", got)
+	}
+
+	// Metacharacters match literally.
+	for _, ch := range []string{".", "(", "?", "+"} {
+		p := CompileDelimiterPattern([]string{ch})
+		if p == nil || !p.MatchString(ch) {
+			t.Errorf("pattern for %q should match itself", ch)
+		}
+		if ch != "." && p.MatchString("z") {
+			t.Errorf("pattern for %q should not match z", ch)
+		}
+	}
+}
+
+func TestCompileDelimiterPatternShippedDefault(t *testing.T) {
+	pat := CompileDelimiterPattern(ParseDelimiterField("\n!?;。；！？"))
+	if pat == nil {
+		t.Fatal("expected pattern")
+	}
+	for _, ch := range []string{"\n", "!", "?", ";", "。", "；", "！", "？"} {
+		if !pat.MatchString(ch) {
+			t.Errorf("default pattern must match %q", ch)
+		}
+	}
+	if pat.MatchString("a") {
+		t.Error("default pattern must not match a")
+	}
+}
+
+func TestCompileDelimiterPatternCaseSensitive(t *testing.T) {
+	pat := CompileDelimiterPattern(ParseDelimiterField("a"))
+	parts := regexp.MustCompile("("+pat.String()+")").Split("AaBb", -1)
+	// Split removes matches; "A" + "Bb" with "a" consumed.
+	if len(parts) < 2 {
+		t.Fatalf("parts=%v", parts)
+	}
+	// Only lowercase a splits.
+	re := regexp.MustCompile("(" + pat.String() + ")")
+	got := re.Split("AaBb", -1)
+	want := []string{"A", "Bb"}
+	// re.Split in Go does not keep delimiters; with capturing group behavior
+	// differs — just check case sensitivity via FindAll.
+	matches := pat.FindAllString("AaBb", -1)
+	if !reflect.DeepEqual(matches, []string{"a"}) {
+		t.Errorf("matches=%v want [a]; split parts=%v want-ish %v", matches, got, want)
+	}
+}
+
+func TestCompileDelimiterListPattern(t *testing.T) {
+	// Bare list entries are ignored (TokenChunker list API).
+	if CompileDelimiterListPattern([]string{"\n", "!"}) != nil {
+		t.Fatal("bare list entries should not produce a pattern")
+	}
+	pat := CompileDelimiterListPattern([]string{"`##`", "`#`"})
+	if pat == nil {
+		t.Fatal("expected pattern from wrapped list entries")
+	}
+	if got := pat.FindString("###"); got != "##" {
+		t.Errorf("got %q want ##", got)
+	}
+
+	// Verify unescaped rune length sorting order: "a.b" (3 runes) vs "ab" (2 runes with meta chars)
+	patMeta := CompileDelimiterListPattern([]string{"`a`", "`a.b`"})
+	if patMeta.String() != `a\.b|a` {
+		t.Errorf("got pattern %q, want %q", patMeta.String(), `a\.b|a`)
+	}
+}
+
+func TestHasCustomDelimiterList(t *testing.T) {
+	if HasCustomDelimiterList([]string{"\n", "!"}) {
+		t.Fatal("bare list should be false")
+	}
+	if !HasCustomDelimiterList([]string{"\n", "`##`"}) {
+		t.Fatal("wrapped list entry should be true")
+	}
+}