mirror of
https://github.com/infiniflow/ragflow.git
synced 2026-08-14 12:47:14 +08:00
refactor(parser): reuse shared DefaultTextCodeDelimiter; add adjacent-delimiter test
- text_parser.go now uses the shared DefaultTextCodeDelimiter from delimiter.go instead of a private duplicate that could drift silently. - Fix the golden-regeneration comment: no generator script is committed; the baseline is reproducible from the golden's meta block. - Add TestTextParser_AdjacentDelimiters pinning the documented adjacent- delimiter behavior (Go drops the standalone second delimiter) so a future silent change in splitCapturingDelims is caught.
This commit is contained in:
@@ -335,9 +335,11 @@ func TestTextParser_ParseWithResult_DefaultDelimiter(t *testing.T) {
|
||||
// comparison normalizes both (delimiters stripped, whitespace collapsed) and
|
||||
// joins on whitespace, reconciling the boundary difference.
|
||||
//
|
||||
// Regenerate the baseline with:
|
||||
//
|
||||
// .venv/bin/python internal/parser/parser/testdata/gen_textcode_golden.py
|
||||
// No generator script is committed. The baseline is reproducible from the
|
||||
// golden's meta block alone (see textcode.python.golden.json: generator,
|
||||
// sample, delimiter, keep_delimiters, chunk_token_num): call the python flow
|
||||
// _code on the sample with keep_delimiters=True and the default delimiter set,
|
||||
// then project each merged section to {"text": section[0], "doc_type_kwd": "text"}.
|
||||
func TestTextParser_AlignmentGolden(t *testing.T) {
|
||||
ctx := t.Context()
|
||||
p := NewTextParser()
|
||||
@@ -361,3 +363,46 @@ func TestTextParser_AlignmentGolden(t *testing.T) {
|
||||
t.Fatalf("text&code parser not aligned with Python golden:%s", diff)
|
||||
}
|
||||
}
|
||||
|
||||
// TestTextParser_AdjacentDelimiters pins Go's behavior on adjacent delimiters,
|
||||
// which intentionally diverges from Python's deepdoc and is only reconciled by
|
||||
// the alignment test's delimiter-strip normalization. Python keeps each trailing
|
||||
// delimiter as its own segment ("a!!b" → ["a!", "!", "b"]), while Go merges the
|
||||
// run so the standalone second delimiter is dropped ("a!!b" → ["a!", "b"]). The
|
||||
// normalized comparison still treats both as equal, so without this test a future
|
||||
// silent change to splitCapturingDelims would go unnoticed.
|
||||
func TestTextParser_AdjacentDelimiters(t *testing.T) {
|
||||
ctx := t.Context()
|
||||
p := NewTextParser()
|
||||
|
||||
// Two adjacent sentence delimiters: Go merges them into the preceding
|
||||
// segment and drops the standalone second delimiter.
|
||||
src := []byte("a!!b")
|
||||
res := p.ParseWithResult(ctx, "doc.txt", src)
|
||||
if res.Err != nil {
|
||||
t.Fatalf("ParseWithResult: %v", res.Err)
|
||||
}
|
||||
want := []string{"a!", "b"}
|
||||
if len(res.JSON) != len(want) {
|
||||
t.Fatalf("adjacent delimiters: JSON len = %d, want %d: %#v", len(res.JSON), len(want), res.JSON)
|
||||
}
|
||||
for i, w := range want {
|
||||
if got := res.JSON[i]["text"]; got != w {
|
||||
t.Errorf("adjacent delimiters: JSON[%d].text = %v, want %v", i, got, w)
|
||||
}
|
||||
}
|
||||
|
||||
// Delimiters separated by text each attach to their own segment (no merge
|
||||
// across the gap).
|
||||
src = []byte("x?y!z")
|
||||
res = p.ParseWithResult(ctx, "doc.txt", src)
|
||||
want = []string{"x?", "y!", "z"}
|
||||
if len(res.JSON) != len(want) {
|
||||
t.Fatalf("mixed delimiters: JSON len = %d, want %d: %#v", len(res.JSON), len(want), res.JSON)
|
||||
}
|
||||
for i, w := range want {
|
||||
if got := res.JSON[i]["text"]; got != w {
|
||||
t.Errorf("mixed delimiters: JSON[%d].text = %v, want %v", i, got, w)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -135,20 +135,18 @@ func decodeRune(p []byte) (rune, int) {
|
||||
return 0xFFFD, 1
|
||||
}
|
||||
|
||||
// defaultTextDelimiter is the flow parser's default delimiter set for the
|
||||
// text&code family (rag/flow/parser/parser.py:_code → deepdoc TxtParser default
|
||||
// "\n!?;。;!?"). The Parser component has no user-facing delimiter config
|
||||
// entry (see PARSER_ALIGNMENT_HANDOFF.md §3.3), so this hard-coded default is
|
||||
// exactly what the python flow always splits on.
|
||||
const defaultTextDelimiter = "\n!?;。;!?"
|
||||
|
||||
// defaultTextDelimiterPattern is the regexp alternation of the default
|
||||
// delimiter set, each rune re.escape'd to mirror
|
||||
// rag/nlp/delim.compile_delimiter_pattern. Go's regexp.Split drops captured
|
||||
// delimiters, so splitCapturingDelims walks the match indexes manually to
|
||||
// reproduce python's re.split(r"(%s)" % pattern, txt) interleaving.
|
||||
// defaultTextDelimiterPattern is the regexp alternation of the flow parser's
|
||||
// default delimiter set DefaultTextCodeDelimiter (rag/flow/parser/parser.py:_code
|
||||
// → deepdoc TxtParser default "\n!?;。;!?"), each rune re.escape'd to mirror
|
||||
// rag/nlp/delim.compile_delimiter_pattern. The Parser component has no user-facing
|
||||
// delimiter config entry (see PARSER_ALIGNMENT_HANDOFF.md §3.3), so this default
|
||||
// is exactly what the python flow always splits on. The shared DefaultTextCodeDelimiter
|
||||
// const lives in delimiter.go so production and the alignment tests use one source
|
||||
// of truth. Go's regexp.Split drops captured delimiters, so splitCapturingDelims
|
||||
// walks the match indexes manually to reproduce python's re.split(r"(%s)" % pattern, txt)
|
||||
// interleaving.
|
||||
var (
|
||||
defaultTextDelimiterPattern = buildDelimiterPattern(defaultTextDelimiter)
|
||||
defaultTextDelimiterPattern = buildDelimiterPattern(DefaultTextCodeDelimiter)
|
||||
textDelimiterSplitRe = regexp.MustCompile(defaultTextDelimiterPattern)
|
||||
textDelimiterExactRe = regexp.MustCompile("^(?:" + defaultTextDelimiterPattern + ")$")
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user