refactor(nlp): collapse 6 delimiter-parser implementations into one canonical helper (#17383) (#17387)

## Summary Six sites used to read the same `parser_config.delimiter` field with divergent grammars: - `rag.nlp.get_delimiters` (PDF/DOCX/HTML/EPUB/JSON/CSV/XLSX/email/book) - `rag.nlp.naive_merge` (custom-delimiter branch) - `rag.nlp.naive_merge_with_images` - `rag.nlp._build_cks` - `deepdoc.parser.txt_parser.parser_txt` (.txt, code) - `deepdoc.parser.markdown_parser.MarkdownElementExtractor.get_delimiters` The six implementations disagreed on bare-vs-wrapped chars, dedupe, sort order, CRLF normalization, and `re.I` (#17384). The shipped default `` `\n!?;。；！？` `` was a no-op for `.md` because the markdown path only matched backtick-wrapped tokens. ## Changes - **new:** `rag/nlp/delim.py` with `parse_delimiter_field` and `compile_delimiter_pattern`. Single source of truth. CRLF normalization at the top; longest-first stable sort; insertion-ordered dedupe; no `re.I`. - **refactor:** all six call sites delegate to the helper. - `rag/nlp/__init__.py::get_delimiters` becomes a thin shim. - `deepdoc/parser/txt_parser.py::parser_txt` drops the `[encode/decode/unicode_escape]` round-trip. - `deepdoc/parser/markdown_parser.py::get_delimiters` honors bare chars (fixes [1]). - **tests:** `test/unit_test/rag/test_delim.py` (85 tests) — helper, acceptance table, frontend parity, static guard against re-inlining. - **tests:** `test/unit_test/rag/test_delimiter_case_sensitive.py` (from #17386) updated to retarget the static check at the new helper + AST-based broader scan. ## Acceptance criteria - All six sites produce the same regex pattern for the same input. - Shipped default keeps working for `.txt` / `.pdf` / `.docx`. - Shipped default for `.md` now splits (was a silent no-op). - Tooltip example `` `\n##;` `` produces three effective delimiters regardless of file type. - Bare whitespace inputs split on every occurrence. - Backtick-wrapped whitespace splits only on the exact N-char sequence. - CRLF-line-ending documents split identically to LF-line-ending documents. - 123 tests pass (85 new + 38 existing). ## Rebase protocol As #17385 and #17386 evolve, this branch will be rebased on top. The only overlap between this PR's diff and the other two is `test_delimiter_case_sensitive.py`, where #17383 modifies the static check to point at the new helper location. --------- Co-authored-by: kiloconnect[bot] <240665456+kiloconnect[bot]@users.noreply.github.com>
2026-08-04 23:00:30 +08:00 · 2026-08-02 14:37:14 +05:30
parent 01d667296d
commit d4ceeee4ed
14 changed files with 1551 additions and 321 deletions
--- a/internal/parser/chunk/delim.go
+++ b/internal/parser/chunk/delim.go
@@ -0,0 +1,194 @@
+//
+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+package chunk
+
+import (
+	"regexp"
+	"sort"
+	"strings"
+	"unicode/utf8"
+)
+
+// Canonical parser for the parser_config.delimiter field.
+//
+// Mirrors Python rag/nlp/delim.py (#17383). A delimiter field is a string
+// with the grammar:
+//
+//	delimiter_field  := token*
+//	token            := backtick_wrapped | bare_char
+//	backtick_wrapped := "`" bare_char+ "`"
+//	bare_char        := any single Unicode character except "`"
+//
+// Semantics:
+//  1. Characters between matching backticks form one multi-character delimiter.
+//  2. Any character outside backticks is its own single-character delimiter.
+//  3. Results are deduplicated and sorted longest-first (stable for equal length).
+//  4. CRLF and standalone CR are normalized to LF before parsing.
+//  5. Matching is case-sensitive.
+
+// backtickWrappedRE matches a backtick-wrapped multi-character token.
+// Case-sensitive on purpose (see #17384).
+var backtickWrappedRE = regexp.MustCompile("`([^`]+)`")
+
+// NormalizeTextNewlines converts CRLF and standalone CR to LF.
+func NormalizeTextNewlines(text string) string {
+	if text == "" {
+		return text
+	}
+	text = strings.ReplaceAll(text, "\r\n", "\n")
+	return strings.ReplaceAll(text, "\r", "\n")
+}
+
+// HasWrappedDelimiter reports whether the delimiter field contains at least
+// one backtick-wrapped token. Used to decide the historical "custom delimiter"
+// mode that bypasses chunk_token_num. Separate from whether any delimiter is
+// present after parsing (bare single-character delimiters still split).
+func HasWrappedDelimiter(s string) bool {
+	if s == "" {
+		return false
+	}
+	return backtickWrappedRE.MatchString(s)
+}
+
+// ParseDelimiterField parses the delimiter field into delimiter strings.
+//
+// Returns nil for an empty field. Whitespace characters are valid
+// single-character delimiters. Output is sorted longest-first and
+// deduplicated while preserving first-occurrence order for equal-length
+// items (the sort is stable). CRLF/CR inside the field are normalized to LF.
+func ParseDelimiterField(s string) []string {
+	if s == "" {
+		return nil
+	}
+	normalized := NormalizeTextNewlines(s)
+
+	var delimiters []string
+	seen := make(map[string]struct{})
+	cursor := 0
+	for _, loc := range backtickWrappedRE.FindAllStringSubmatchIndex(normalized, -1) {
+		// loc: [fullStart, fullEnd, groupStart, groupEnd]
+		start, end := loc[0], loc[1]
+		for _, ch := range runesOf(normalized[cursor:start]) {
+			if _, ok := seen[ch]; ok {
+				continue
+			}
+			seen[ch] = struct{}{}
+			delimiters = append(delimiters, ch)
+		}
+		token := normalized[loc[2]:loc[3]]
+		if token != "" {
+			if _, ok := seen[token]; !ok {
+				seen[token] = struct{}{}
+				delimiters = append(delimiters, token)
+			}
+		}
+		cursor = end
+	}
+	for _, ch := range runesOf(normalized[cursor:]) {
+		if _, ok := seen[ch]; ok {
+			continue
+		}
+		seen[ch] = struct{}{}
+		delimiters = append(delimiters, ch)
+	}
+
+	sort.SliceStable(delimiters, func(i, j int) bool {
+		return utf8.RuneCountInString(delimiters[i]) > utf8.RuneCountInString(delimiters[j])
+	})
+	return delimiters
+}
+
+// CompileDelimiterPattern builds an alternation regex from delimiter strings.
+//
+// Each delimiter is regexp.QuoteMeta'd so whitespace and metacharacters match
+// literally. Returns nil when delimiters is empty or yields no non-empty
+// entries. The pattern is intended for use with a capturing split (so
+// delimiters appear in the split output) or FindAllStringIndex.
+func CompileDelimiterPattern(delimiters []string) *regexp.Regexp {
+	if len(delimiters) == 0 {
+		return nil
+	}
+	escaped := make([]string, 0, len(delimiters))
+	for _, d := range delimiters {
+		if d == "" {
+			continue
+		}
+		escaped = append(escaped, regexp.QuoteMeta(d))
+	}
+	if len(escaped) == 0 {
+		return nil
+	}
+	return regexp.MustCompile(strings.Join(escaped, "|"))
+}
+
+// CompileDelimiterListPattern compiles a TokenChunker-style []string delimiter
+// list. Entries wrapped in backticks contribute their inner content as a
+// split pattern (Python token_chunker / historical Go compileDelimPattern).
+// Bare list entries are ignored for the active pattern — they are only used
+// by merge paths when no custom pattern exists.
+//
+// Prefer ParseDelimiterField for the single-string parser_config.delimiter
+// field. This helper exists for the dataflow list API.
+func CompileDelimiterListPattern(delims []string) *regexp.Regexp {
+	var custom []string
+	for _, d := range delims {
+		if d == "" {
+			continue
+		}
+		if strings.HasPrefix(d, "`") && strings.HasSuffix(d, "`") && len(d) >= 2 {
+			inner := d[1 : len(d)-1]
+			if inner == "" {
+				continue
+			}
+			custom = append(custom, inner)
+		}
+	}
+	if len(custom) == 0 {
+		return nil
+	}
+	sort.SliceStable(custom, func(i, j int) bool {
+		return utf8.RuneCountInString(custom[i]) > utf8.RuneCountInString(custom[j])
+	})
+	escaped := make([]string, 0, len(custom))
+	for _, d := range custom {
+		escaped = append(escaped, regexp.QuoteMeta(d))
+	}
+	return regexp.MustCompile(strings.Join(escaped, "|"))
+}
+
+// HasCustomDelimiterList reports whether any entry in a TokenChunker-style
+// delimiter list uses backtick syntax.
+func HasCustomDelimiterList(delims []string) bool {
+	for _, d := range delims {
+		if strings.HasPrefix(d, "`") && strings.HasSuffix(d, "`") && len(d) >= 2 {
+			return true
+		}
+	}
+	return false
+}
+
+// runesOf splits s into individual Unicode characters (as strings).
+func runesOf(s string) []string {
+	if s == "" {
+		return nil
+	}
+	out := make([]string, 0, utf8.RuneCountInString(s))
+	for _, r := range s {
+		out = append(out, string(r))
+	}
+	return out
+}
--- a/internal/parser/chunk/delim_test.go
+++ b/internal/parser/chunk/delim_test.go
@@ -0,0 +1,185 @@
+//
+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+package chunk
+
+import (
+	"reflect"
+	"regexp"
+	"testing"
+)
+
+func TestNormalizeTextNewlines(t *testing.T) {
+	tests := []struct {
+		in, want string
+	}{
+		{"", ""},
+		{"a\nb", "a\nb"},
+		{"a\r\nb", "a\nb"},
+		{"a\rb", "a\nb"},
+		{"a\r\nb\rc", "a\nb\nc"},
+	}
+	for _, tc := range tests {
+		if got := NormalizeTextNewlines(tc.in); got != tc.want {
+			t.Errorf("NormalizeTextNewlines(%q) = %q, want %q", tc.in, got, tc.want)
+		}
+	}
+}
+
+func TestHasWrappedDelimiter(t *testing.T) {
+	if HasWrappedDelimiter("") {
+		t.Fatal("empty should be false")
+	}
+	if HasWrappedDelimiter("!;") {
+		t.Fatal("bare chars should be false")
+	}
+	if !HasWrappedDelimiter("`##`") {
+		t.Fatal("wrapped should be true")
+	}
+	if !HasWrappedDelimiter("\n`##`;") {
+		t.Fatal("mixed should be true")
+	}
+	if !HasWrappedDelimiter("`;`") {
+		t.Fatal("wrapped single char should be true")
+	}
+}
+
+func TestParseDelimiterField(t *testing.T) {
+	tests := []struct {
+		name  string
+		field string
+		want  []string
+	}{
+		{"empty", "", nil},
+		{"single bare", "!", []string{"!"}},
+		{"bare pair", "!?", []string{"!", "?"}},
+		{"space", " ", []string{" "}},
+		{"newline", "\n", []string{"\n"}},
+		{"crlf field", "\r\n", []string{"\n"}},
+		{"bare cr", "\r", []string{"\n"}},
+		{"wrapped end", "`end`", []string{"end"}},
+		{"wrapped hierarchy", "`###``##``#`", []string{"###", "##", "#"}},
+		{"tooltip example", "\n`##`;", []string{"##", "\n", ";"}},
+		{"dedupe", "`a`a`a`", []string{"a"}},
+		{"wrapped double newline", "`\n\n`", []string{"\n\n"}},
+		{"crlf wrapped", "`\r\n`", []string{"\n"}},
+		{"shipped default", "\n!?;。；！？", []string{"\n", "!", "?", ";", "。", "；", "！", "？"}},
+		{"chinese", "。；", []string{"。", "；"}},
+		{"mixed order equal length", "`##`#\n", []string{"##", "#", "\n"}},
+		{"empty backticks bare", "``", []string{"`"}},
+		{"double space bare dedupe", "  ", []string{" "}},
+		{"wrapped double space", "`  `", []string{"  "}},
+	}
+	for _, tc := range tests {
+		t.Run(tc.name, func(t *testing.T) {
+			got := ParseDelimiterField(tc.field)
+			if !reflect.DeepEqual(got, tc.want) {
+				t.Errorf("ParseDelimiterField(%q) = %#v, want %#v", tc.field, got, tc.want)
+			}
+		})
+	}
+}
+
+func TestCompileDelimiterPattern(t *testing.T) {
+	if CompileDelimiterPattern(nil) != nil {
+		t.Fatal("empty list should yield nil")
+	}
+	if CompileDelimiterPattern([]string{}) != nil {
+		t.Fatal("empty slice should yield nil")
+	}
+
+	pat := CompileDelimiterPattern([]string{"##", "#"})
+	if pat == nil {
+		t.Fatal("expected pattern")
+	}
+	if got := pat.FindString("###"); got != "##" {
+		t.Errorf("longest match: got %q, want ##", got)
+	}
+
+	// Metacharacters match literally.
+	for _, ch := range []string{".", "(", "?", "+"} {
+		p := CompileDelimiterPattern([]string{ch})
+		if p == nil || !p.MatchString(ch) {
+			t.Errorf("pattern for %q should match itself", ch)
+		}
+		if ch != "." && p.MatchString("z") {
+			t.Errorf("pattern for %q should not match z", ch)
+		}
+	}
+}
+
+func TestCompileDelimiterPatternShippedDefault(t *testing.T) {
+	pat := CompileDelimiterPattern(ParseDelimiterField("\n!?;。；！？"))
+	if pat == nil {
+		t.Fatal("expected pattern")
+	}
+	for _, ch := range []string{"\n", "!", "?", ";", "。", "；", "！", "？"} {
+		if !pat.MatchString(ch) {
+			t.Errorf("default pattern must match %q", ch)
+		}
+	}
+	if pat.MatchString("a") {
+		t.Error("default pattern must not match a")
+	}
+}
+
+func TestCompileDelimiterPatternCaseSensitive(t *testing.T) {
+	pat := CompileDelimiterPattern(ParseDelimiterField("a"))
+	parts := regexp.MustCompile("("+pat.String()+")").Split("AaBb", -1)
+	// Split removes matches; "A" + "Bb" with "a" consumed.
+	if len(parts) < 2 {
+		t.Fatalf("parts=%v", parts)
+	}
+	// Only lowercase a splits.
+	re := regexp.MustCompile("(" + pat.String() + ")")
+	got := re.Split("AaBb", -1)
+	want := []string{"A", "Bb"}
+	// re.Split in Go does not keep delimiters; with capturing group behavior
+	// differs — just check case sensitivity via FindAll.
+	matches := pat.FindAllString("AaBb", -1)
+	if !reflect.DeepEqual(matches, []string{"a"}) {
+		t.Errorf("matches=%v want [a]; split parts=%v want-ish %v", matches, got, want)
+	}
+}
+
+func TestCompileDelimiterListPattern(t *testing.T) {
+	// Bare list entries are ignored (TokenChunker list API).
+	if CompileDelimiterListPattern([]string{"\n", "!"}) != nil {
+		t.Fatal("bare list entries should not produce a pattern")
+	}
+	pat := CompileDelimiterListPattern([]string{"`##`", "`#`"})
+	if pat == nil {
+		t.Fatal("expected pattern from wrapped list entries")
+	}
+	if got := pat.FindString("###"); got != "##" {
+		t.Errorf("got %q want ##", got)
+	}
+
+	// Verify unescaped rune length sorting order: "a.b" (3 runes) vs "ab" (2 runes with meta chars)
+	patMeta := CompileDelimiterListPattern([]string{"`a`", "`a.b`"})
+	if patMeta.String() != `a\.b|a` {
+		t.Errorf("got pattern %q, want %q", patMeta.String(), `a\.b|a`)
+	}
+}
+
+func TestHasCustomDelimiterList(t *testing.T) {
+	if HasCustomDelimiterList([]string{"\n", "!"}) {
+		t.Fatal("bare list should be false")
+	}
+	if !HasCustomDelimiterList([]string{"\n", "`##`"}) {
+		t.Fatal("wrapped list entry should be true")
+	}
+}