refactor(parser): reuse shared DefaultTextCodeDelimiter; add adjacent-delimiter test

- text_parser.go now uses the shared DefaultTextCodeDelimiter from
  delimiter.go instead of a private duplicate that could drift silently.
- Fix the golden-regeneration comment: no generator script is committed;
  the baseline is reproducible from the golden's meta block.
- Add TestTextParser_AdjacentDelimiters pinning the documented adjacent-
  delimiter behavior (Go drops the standalone second delimiter) so a
  future silent change in splitCapturingDelims is caught.
This commit is contained in:
xugangqiang
2026-08-11 16:46:20 +08:00
parent e229370e5d
commit f8688b17e9
2 changed files with 59 additions and 16 deletions

View File

@@ -335,9 +335,11 @@ func TestTextParser_ParseWithResult_DefaultDelimiter(t *testing.T) {
// comparison normalizes both (delimiters stripped, whitespace collapsed) and
// joins on whitespace, reconciling the boundary difference.
//
// Regenerate the baseline with:
//
// .venv/bin/python internal/parser/parser/testdata/gen_textcode_golden.py
// No generator script is committed. The baseline is reproducible from the
// golden's meta block alone (see textcode.python.golden.json: generator,
// sample, delimiter, keep_delimiters, chunk_token_num): call the python flow
// _code on the sample with keep_delimiters=True and the default delimiter set,
// then project each merged section to {"text": section[0], "doc_type_kwd": "text"}.
func TestTextParser_AlignmentGolden(t *testing.T) {
ctx := t.Context()
p := NewTextParser()
@@ -361,3 +363,46 @@ func TestTextParser_AlignmentGolden(t *testing.T) {
t.Fatalf("text&code parser not aligned with Python golden:%s", diff)
}
}
// TestTextParser_AdjacentDelimiters pins Go's behavior on adjacent delimiters,
// which intentionally diverges from Python's deepdoc and is only reconciled by
// the alignment test's delimiter-strip normalization. Python keeps each trailing
// delimiter as its own segment ("a!!b" → ["a!", "!", "b"]), while Go merges the
// run so the standalone second delimiter is dropped ("a!!b" → ["a!", "b"]). The
// normalized comparison still treats both as equal, so without this test a future
// silent change to splitCapturingDelims would go unnoticed.
func TestTextParser_AdjacentDelimiters(t *testing.T) {
ctx := t.Context()
p := NewTextParser()
// Two adjacent sentence delimiters: Go merges them into the preceding
// segment and drops the standalone second delimiter.
src := []byte("a!!b")
res := p.ParseWithResult(ctx, "doc.txt", src)
if res.Err != nil {
t.Fatalf("ParseWithResult: %v", res.Err)
}
want := []string{"a!", "b"}
if len(res.JSON) != len(want) {
t.Fatalf("adjacent delimiters: JSON len = %d, want %d: %#v", len(res.JSON), len(want), res.JSON)
}
for i, w := range want {
if got := res.JSON[i]["text"]; got != w {
t.Errorf("adjacent delimiters: JSON[%d].text = %v, want %v", i, got, w)
}
}
// Delimiters separated by text each attach to their own segment (no merge
// across the gap).
src = []byte("x?y!z")
res = p.ParseWithResult(ctx, "doc.txt", src)
want = []string{"x?", "y!", "z"}
if len(res.JSON) != len(want) {
t.Fatalf("mixed delimiters: JSON len = %d, want %d: %#v", len(res.JSON), len(want), res.JSON)
}
for i, w := range want {
if got := res.JSON[i]["text"]; got != w {
t.Errorf("mixed delimiters: JSON[%d].text = %v, want %v", i, got, w)
}
}
}

View File

@@ -135,20 +135,18 @@ func decodeRune(p []byte) (rune, int) {
return 0xFFFD, 1
}
// defaultTextDelimiter is the flow parser's default delimiter set for the
// text&code family (rag/flow/parser/parser.py:_code → deepdoc TxtParser default
// "\n!?;。;!?"). The Parser component has no user-facing delimiter config
// entry (see PARSER_ALIGNMENT_HANDOFF.md §3.3), so this hard-coded default is
// exactly what the python flow always splits on.
const defaultTextDelimiter = "\n!?;。;!?"
// defaultTextDelimiterPattern is the regexp alternation of the default
// delimiter set, each rune re.escape'd to mirror
// rag/nlp/delim.compile_delimiter_pattern. Go's regexp.Split drops captured
// delimiters, so splitCapturingDelims walks the match indexes manually to
// reproduce python's re.split(r"(%s)" % pattern, txt) interleaving.
// defaultTextDelimiterPattern is the regexp alternation of the flow parser's
// default delimiter set DefaultTextCodeDelimiter (rag/flow/parser/parser.py:_code
// → deepdoc TxtParser default "\n!?;。;!?"), each rune re.escape'd to mirror
// rag/nlp/delim.compile_delimiter_pattern. The Parser component has no user-facing
// delimiter config entry (see PARSER_ALIGNMENT_HANDOFF.md §3.3), so this default
// is exactly what the python flow always splits on. The shared DefaultTextCodeDelimiter
// const lives in delimiter.go so production and the alignment tests use one source
// of truth. Go's regexp.Split drops captured delimiters, so splitCapturingDelims
// walks the match indexes manually to reproduce python's re.split(r"(%s)" % pattern, txt)
// interleaving.
var (
defaultTextDelimiterPattern = buildDelimiterPattern(defaultTextDelimiter)
defaultTextDelimiterPattern = buildDelimiterPattern(DefaultTextCodeDelimiter)
textDelimiterSplitRe = regexp.MustCompile(defaultTextDelimiterPattern)
textDelimiterExactRe = regexp.MustCompile("^(?:" + defaultTextDelimiterPattern + ")$")
)