mirror of
https://github.com/infiniflow/ragflow.git
synced 2026-08-01 21:37:33 +08:00
fix(nlp): drop dead re.I from delimiter finditer calls (#17386)
Closes #17384. ## Summary Drops a dead `re.I` flag from two outlier delimiter-parsing sites and adds regression tests so the inconsistency can't creep back. ## What's wrong Two of the six delimiter-parsing implementations pass `re.I` to `re.finditer`: - `rag/nlp/__init__.py::get_delimiters` (line 1633) - `deepdoc/parser/txt_parser.py::parser_txt` (line 51) The other four implementations correctly omit `re.I`: - `rag/nlp/__init__.py::naive_merge` custom-delimiter path (line 1195) - `rag/nlp/__init__.py::naive_merge_with_images` custom-delimiter path (line 1269) - `rag/nlp/__init__.py::_build_cks` (line 1389) - `rag/flow/chunker/token_chunker.py` (line 73) ## Why this matters (and why it doesn't break anything) The flag is **dead code** today. Verified empirically with a Python REPL: ```python >>> import re >>> for m in re.finditer(r"`([^`]+)`", "`end`", re.I): ... print(repr(m.group(1))) 'end' # plain string, no flag attached >>> re.split("(a)", "Class A is a Sample") ['Cl', 'a', '', 's', ' A i', 's', ' a Sample'] # Case-sensitive: only lowercase 'a' splits. Uppercase 'A' is preserved. ``` `re.I` does not propagate from `re.finditer` to `m.group(1)` or to downstream `re.split` / `re.match` calls (which all omit `re.I`). So the actual splitting behavior has always been case-sensitive — removing the flag is a **defensive cleanup**, not a behavioral fix. So why bother? 1. **Consistency** — the two sites were the only outliers in a six-way implementation cluster. The three sibling sites in `rag/nlp/__init__.py` already omit `re.I`, which strongly suggests the flag was accidental. 2. **Future-proofing** — a refactor could easily propagate the flag to a downstream `re.split` call where it *would* change behavior. The tests added here pin the case-sensitive semantics so that regression fails loudly. 3. **Reader clarity** — the flag is misleading. Anyone reading `re.finditer(..., re.I)` reasonably assumes case-insensitive matching, then has to trace all downstream calls to discover it's a no-op. ## Changes - `rag/nlp/__init__.py` — drop `re.I` from `get_delimiters` (line 1633). - `deepdoc/parser/txt_parser.py` — drop `re.I` from `parser_txt` (line 51). - `test/unit_test/rag/test_delimiter_case_sensitive.py` — new test file with: - 4 behavioral tests on `get_delimiters` (pattern output + `re.split` round-trip). - 3 end-to-end tests through `naive_merge` (bare-char + backtick-wrapped, both cases). - 2 parametrized static checks that `re.I` / `re.IGNORECASE` is not present at either of the two `re.finditer` sites. ## Testing ``` $ pytest test/unit_test/rag/test_delimiter_case_sensitive.py -v ============================= 9 passed in 0.19s ============================== ``` All tests pass on the patched code. Before the patch, the 2 static checks fail with a clear assertion message (the 7 behavioral tests pass either way, confirming `re.I` was dead code). ## Related - #17384 — the issue this PR closes. Note the issue's reproduction code (`re.split(..., flags=re.I)`) doesn't actually match what the production code does — the production `re.split` calls all omit `re.I`, which is why current behavior is already case-sensitive. The fix here is still valuable as a defensive cleanup + test coverage, but it's not a behavioral fix per se. - #17383 — broader parser consolidation (six implementations → one). The fix here is independent and small enough to land first. - #17385 — sibling UX PR (tooltip + live preview). Files are disjoint (`web/src/**` vs `rag/nlp/**` + `deepdoc/parser/**`), so no interaction. --------- Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Co-authored-by: kiloconnect[bot] <240665456+kiloconnect[bot]@users.noreply.github.com>
This commit is contained in:
S
@@ -76,28 +76,86 @@ func stringListFromAny(in []any) []string {
|
||||
// regex / split helpers
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
// compileDelimPattern joins all delimiter entries into a single
|
||||
// alternation. Entries wrapped in backticks are treated as regex
|
||||
// literals and regex-escaped; plain strings are simply regex-escaped.
|
||||
// Longer patterns win (matches python `sorted(set, key=len, reverse=True)`).
|
||||
// Mirrors Python _compile_delimiter_pattern: only backtick-wrapped delimiters
|
||||
// produce an active pattern. Plain delimiters are not compiled — they are only
|
||||
// used by naive_merge / mergeByTokenSize for sentence-level splitting when no
|
||||
// active pattern exists.
|
||||
// backtickDelimRE extracts backtick-wrapped delimiter tokens.
|
||||
// Case-sensitive on purpose (mirrors Python after #17384 / PR #17386):
|
||||
// never add (?i) here — delimiter matching must preserve letter casing.
|
||||
var backtickDelimRE = regexp.MustCompile("`([^`]+)`")
|
||||
|
||||
// escapeAndSort QuoteMeta-escapes tokens and sorts longest-first.
|
||||
// Empty tokens are dropped. When dedup is true, exact duplicate tokens
|
||||
// are collapsed (first occurrence wins). Matching stays case-sensitive.
|
||||
func escapeAndSort(tokens []string, dedup bool) []string {
|
||||
out := make([]string, 0, len(tokens))
|
||||
var seen map[string]struct{}
|
||||
if dedup {
|
||||
seen = make(map[string]struct{}, len(tokens))
|
||||
}
|
||||
for _, tok := range tokens {
|
||||
if tok == "" {
|
||||
continue
|
||||
}
|
||||
if dedup {
|
||||
if _, ok := seen[tok]; ok {
|
||||
continue
|
||||
}
|
||||
seen[tok] = struct{}{}
|
||||
}
|
||||
out = append(out, regexp.QuoteMeta(tok))
|
||||
}
|
||||
sort.SliceStable(out, func(i, j int) bool { return len(out[i]) > len(out[j]) })
|
||||
return out
|
||||
}
|
||||
|
||||
// getDelimiters ports rag.nlp.get_delimiters (rag/nlp/__init__.py).
|
||||
//
|
||||
// It walks a single delimiter string, pulling out backtick-wrapped tokens
|
||||
// and every bare character outside those spans, then returns a
|
||||
// length-sorted, QuoteMeta-escaped alternation suitable for regexp.Split.
|
||||
// Matching is case-sensitive: "a" does not match "A", and "`end`" does not
|
||||
// match "End" / "END".
|
||||
func getDelimiters(delimiters string) string {
|
||||
var dels []string
|
||||
s := 0
|
||||
for _, m := range backtickDelimRE.FindAllStringSubmatchIndex(delimiters, -1) {
|
||||
// m = [fullStart, fullEnd, g1Start, g1End]
|
||||
f, t := m[0], m[1]
|
||||
dels = append(dels, delimiters[m[2]:m[3]])
|
||||
for _, r := range delimiters[s:f] {
|
||||
dels = append(dels, string(r))
|
||||
}
|
||||
s = t
|
||||
}
|
||||
if s < len(delimiters) {
|
||||
for _, r := range delimiters[s:] {
|
||||
dels = append(dels, string(r))
|
||||
}
|
||||
}
|
||||
// Python get_delimiters does not dedup; preserve that behavior.
|
||||
return strings.Join(escapeAndSort(dels, false), "|")
|
||||
}
|
||||
|
||||
// compileDelimPattern builds an alternation from backtick-wrapped tokens
|
||||
// across delimiter entries (mirrors Python _compile_delimiter_pattern).
|
||||
// Each entry is scanned independently so token boundaries cannot span
|
||||
// adjacent slice elements. Extraction is case-sensitive — see backtickDelimRE.
|
||||
//
|
||||
// Plain (non-backtick) delimiters are not compiled here; callers that
|
||||
// need bare-char splitting use getDelimiters (naive_merge path).
|
||||
func compileDelimPattern(delims []string) *regexp.Regexp {
|
||||
var custom []string
|
||||
var tokens []string
|
||||
for _, d := range delims {
|
||||
if d == "" {
|
||||
continue
|
||||
}
|
||||
if strings.HasPrefix(d, "`") && strings.HasSuffix(d, "`") && len(d) >= 2 {
|
||||
custom = append(custom, regexp.QuoteMeta(d[1:len(d)-1]))
|
||||
for _, m := range backtickDelimRE.FindAllStringSubmatch(d, -1) {
|
||||
tokens = append(tokens, m[1])
|
||||
}
|
||||
}
|
||||
// Python _compile_delimiter_pattern dedups via set(...).
|
||||
custom := escapeAndSort(tokens, true)
|
||||
if len(custom) == 0 {
|
||||
return nil
|
||||
}
|
||||
sort.SliceStable(custom, func(i, j int) bool { return len(custom[i]) > len(custom[j]) })
|
||||
return regexp.MustCompile(strings.Join(custom, "|"))
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,233 @@
|
||||
//
|
||||
// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
//
|
||||
|
||||
// Regression tests for case-sensitive delimiter parsing (#17384 / PR #17386).
|
||||
//
|
||||
// Locks Go parity with the Python fix that dropped re.I from get_delimiters
|
||||
// and parser_txt. Delimiter matching must preserve letter casing:
|
||||
//
|
||||
// - bare "a" matches only "a", not "A"
|
||||
// - backtick-wrapped "`end`" matches only "end", not "End"/"END"/etc.
|
||||
//
|
||||
// Go's regexp package is case-sensitive by default; these tests guard against
|
||||
// a future (?i) flag or case-folding change creeping into the delimiter path.
|
||||
|
||||
package chunker
|
||||
|
||||
import (
|
||||
"context"
|
||||
"regexp"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestGetDelimiters_BareCharAReturnsLiteralPattern(t *testing.T) {
|
||||
// Bare-char delimiter "a" must produce the pattern "a", not "a|A".
|
||||
if got, want := getDelimiters("a"), "a"; got != want {
|
||||
t.Fatalf("getDelimiters(%q) = %q, want %q", "a", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestGetDelimiters_BareCharAUpperReturnsLiteralPattern(t *testing.T) {
|
||||
if got, want := getDelimiters("A"), "A"; got != want {
|
||||
t.Fatalf("getDelimiters(%q) = %q, want %q", "A", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestGetDelimiters_BacktickEndReturnsExactToken(t *testing.T) {
|
||||
// Backtick-wrapped delimiter must preserve the captured group verbatim.
|
||||
if got, want := getDelimiters("`end`"), "end"; got != want {
|
||||
t.Fatalf("getDelimiters(%q) = %q, want %q", "`end`", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestGetDelimiters_PatternSplitsCaseSensitively(t *testing.T) {
|
||||
// The pattern returned by getDelimiters must split case-sensitively.
|
||||
pat := getDelimiters("a")
|
||||
re := regexp.MustCompile("(" + pat + ")")
|
||||
// Only the lowercase 'a' splits; uppercase 'A' is preserved intact.
|
||||
got := splitKeepingCapture("AaBb", re)
|
||||
want := []string{"A", "a", "Bb"}
|
||||
if len(got) != len(want) {
|
||||
t.Fatalf("split = %#v, want %#v", got, want)
|
||||
}
|
||||
for i := range want {
|
||||
if got[i] != want[i] {
|
||||
t.Fatalf("split[%d] = %q, want %q (full=%#v)", i, got[i], want[i], got)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// splitKeepingCapture mirrors Python re.split("("+pat+")", text):
|
||||
// returns [text, delim, text, delim, ...] including empty leading/trailing.
|
||||
func splitKeepingCapture(text string, re *regexp.Regexp) []string {
|
||||
idxs := re.FindAllStringIndex(text, -1)
|
||||
if len(idxs) == 0 {
|
||||
return []string{text}
|
||||
}
|
||||
var out []string
|
||||
cursor := 0
|
||||
for _, idx := range idxs {
|
||||
start, end := idx[0], idx[1]
|
||||
out = append(out, text[cursor:start])
|
||||
out = append(out, text[start:end])
|
||||
cursor = end
|
||||
}
|
||||
out = append(out, text[cursor:])
|
||||
return out
|
||||
}
|
||||
|
||||
func TestCompileDelimPattern_BacktickEndIsCaseSensitive(t *testing.T) {
|
||||
p := compileDelimPattern([]string{"`end`"})
|
||||
if p == nil {
|
||||
t.Fatal("compileDelimPattern(`end`) returned nil")
|
||||
}
|
||||
if p.MatchString("End") || p.MatchString("END") || p.MatchString("eNd") {
|
||||
t.Fatalf("pattern %q must not match case variants of end", p.String())
|
||||
}
|
||||
if !p.MatchString("end") {
|
||||
t.Fatalf("pattern %q must match exact lowercase end", p.String())
|
||||
}
|
||||
// Pattern string itself must not carry a case-insensitive flag.
|
||||
if strings.HasPrefix(p.String(), "(?i)") || strings.Contains(p.String(), "(?i)") {
|
||||
t.Fatalf("compileDelimPattern must not emit (?i); got %q", p.String())
|
||||
}
|
||||
}
|
||||
|
||||
func TestCompileDelimPattern_BareCharIsNotActivePattern(t *testing.T) {
|
||||
// Plain delimiters are not compiled — only backtick-wrapped tokens are.
|
||||
if p := compileDelimPattern([]string{"a"}); p != nil {
|
||||
t.Fatalf("compileDelimPattern([a]) = %v, want nil", p)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCompileDelimPattern_ExtractsPerEntryIndependently(t *testing.T) {
|
||||
// Adjacent entries must not form a cross-boundary backtick pair.
|
||||
// Concatenating "`aa" + "`bb`" would invent an "aa`bb" token; per-entry
|
||||
// extraction only sees the complete "`bb`" pair in the second entry.
|
||||
p := compileDelimPattern([]string{"`aa", "`bb`"})
|
||||
if p == nil {
|
||||
t.Fatal("expected pattern from second entry `bb`")
|
||||
}
|
||||
if got := p.String(); got != "bb" {
|
||||
t.Fatalf("pattern = %q, want %q (no cross-entry token)", got, "bb")
|
||||
}
|
||||
if p.MatchString("aa") {
|
||||
t.Fatalf("must not match incomplete first-entry token; pattern=%q", p.String())
|
||||
}
|
||||
if !p.MatchString("bb") {
|
||||
t.Fatalf("must match token from second entry; pattern=%q", p.String())
|
||||
}
|
||||
|
||||
// Multiple well-formed entries still combine.
|
||||
p2 := compileDelimPattern([]string{"`end`", "`foo`"})
|
||||
if p2 == nil {
|
||||
t.Fatal("expected combined pattern")
|
||||
}
|
||||
if !p2.MatchString("end") || !p2.MatchString("foo") {
|
||||
t.Fatalf("combined pattern %q must match both tokens", p2.String())
|
||||
}
|
||||
}
|
||||
|
||||
func TestTokenChunker_BacktickEndSplitsOnlyAtLowercase(t *testing.T) {
|
||||
// End-to-end: delimiter-mode with "`end`" must split only at lowercase
|
||||
// "end", leaving "End" / "END" intact inside the following segment.
|
||||
c, err := NewTokenChunker(map[string]any{
|
||||
"delimiter_mode": "delimiter",
|
||||
"delimiters": []string{"`end`"},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("NewTokenChunker: %v", err)
|
||||
}
|
||||
out, err := c.Invoke(context.Background(), nil, map[string]any{
|
||||
"name": "doc.txt",
|
||||
"output_format": "text",
|
||||
"text": "the end and End and END come",
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("Invoke: %v", err)
|
||||
}
|
||||
chunks, _ := out["chunks"].([]map[string]any)
|
||||
got := make([]string, 0, len(chunks))
|
||||
for _, ck := range chunks {
|
||||
text, _ := ck["text"].(string)
|
||||
text = strings.TrimSpace(text)
|
||||
if text != "" {
|
||||
got = append(got, text)
|
||||
}
|
||||
}
|
||||
// splitKeepingDelim glues the matched delimiter onto the preceding
|
||||
// segment: "the end" | " and End and END come"
|
||||
want := []string{"the end", "and End and END come"}
|
||||
if len(got) != len(want) {
|
||||
t.Fatalf("chunks = %#v, want %#v", got, want)
|
||||
}
|
||||
for i := range want {
|
||||
if got[i] != want[i] {
|
||||
t.Fatalf("chunk[%d] = %q, want %q (full=%#v)", i, got[i], want[i], got)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestTokenChunker_BacktickASplitsOnlyAtLowercase(t *testing.T) {
|
||||
c, err := NewTokenChunker(map[string]any{
|
||||
"delimiter_mode": "delimiter",
|
||||
"delimiters": []string{"`a`"},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("NewTokenChunker: %v", err)
|
||||
}
|
||||
out, err := c.Invoke(context.Background(), nil, map[string]any{
|
||||
"name": "doc.txt",
|
||||
"output_format": "text",
|
||||
"text": "BaAb",
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("Invoke: %v", err)
|
||||
}
|
||||
chunks, _ := out["chunks"].([]map[string]any)
|
||||
got := make([]string, 0, len(chunks))
|
||||
for _, ck := range chunks {
|
||||
text, _ := ck["text"].(string)
|
||||
text = strings.TrimSpace(text)
|
||||
if text != "" {
|
||||
got = append(got, text)
|
||||
}
|
||||
}
|
||||
// "B" + "a" glued → "Ba"; remainder "Ab"
|
||||
want := []string{"Ba", "Ab"}
|
||||
if len(got) != len(want) {
|
||||
t.Fatalf("chunks = %#v, want %#v", got, want)
|
||||
}
|
||||
for i := range want {
|
||||
if got[i] != want[i] {
|
||||
t.Fatalf("chunk[%d] = %q, want %q (full=%#v)", i, got[i], want[i], got)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestBacktickDelimRE_HasNoIgnoreCaseFlag(t *testing.T) {
|
||||
// Structural guard: the shared backtick extractor must stay case-sensitive.
|
||||
src := backtickDelimRE.String()
|
||||
if strings.Contains(src, "(?i)") || strings.HasPrefix(src, "(?i)") {
|
||||
t.Fatalf("backtickDelimRE must not use (?i); got %q", src)
|
||||
}
|
||||
// Sanity: the pattern still extracts backtick contents.
|
||||
m := backtickDelimRE.FindStringSubmatch("`End`")
|
||||
if len(m) < 2 || m[1] != "End" {
|
||||
t.Fatalf("backtickDelimRE(`End`) = %#v, want group1=End", m)
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user