From f8688b17e9221444941c7e6d53f66faccd756f16 Mon Sep 17 00:00:00 2001 From: xugangqiang Date: Tue, 11 Aug 2026 16:46:20 +0800 Subject: [PATCH] refactor(parser): reuse shared DefaultTextCodeDelimiter; add adjacent-delimiter test - text_parser.go now uses the shared DefaultTextCodeDelimiter from delimiter.go instead of a private duplicate that could drift silently. - Fix the golden-regeneration comment: no generator script is committed; the baseline is reproducible from the golden's meta block. - Add TestTextParser_AdjacentDelimiters pinning the documented adjacent- delimiter behavior (Go drops the standalone second delimiter) so a future silent change in splitCapturingDelims is caught. --- .../parser/parser/parse_with_result_test.go | 51 +++++++++++++++++-- internal/parser/parser/text_parser.go | 24 ++++----- 2 files changed, 59 insertions(+), 16 deletions(-) diff --git a/internal/parser/parser/parse_with_result_test.go b/internal/parser/parser/parse_with_result_test.go index ca2c03269e..23c77685d9 100644 --- a/internal/parser/parser/parse_with_result_test.go +++ b/internal/parser/parser/parse_with_result_test.go @@ -335,9 +335,11 @@ func TestTextParser_ParseWithResult_DefaultDelimiter(t *testing.T) { // comparison normalizes both (delimiters stripped, whitespace collapsed) and // joins on whitespace, reconciling the boundary difference. // -// Regenerate the baseline with: -// -// .venv/bin/python internal/parser/parser/testdata/gen_textcode_golden.py +// No generator script is committed. The baseline is reproducible from the +// golden's meta block alone (see textcode.python.golden.json: generator, +// sample, delimiter, keep_delimiters, chunk_token_num): call the python flow +// _code on the sample with keep_delimiters=True and the default delimiter set, +// then project each merged section to {"text": section[0], "doc_type_kwd": "text"}. func TestTextParser_AlignmentGolden(t *testing.T) { ctx := t.Context() p := NewTextParser() @@ -361,3 +363,46 @@ func TestTextParser_AlignmentGolden(t *testing.T) { t.Fatalf("text&code parser not aligned with Python golden:%s", diff) } } + +// TestTextParser_AdjacentDelimiters pins Go's behavior on adjacent delimiters, +// which intentionally diverges from Python's deepdoc and is only reconciled by +// the alignment test's delimiter-strip normalization. Python keeps each trailing +// delimiter as its own segment ("a!!b" → ["a!", "!", "b"]), while Go merges the +// run so the standalone second delimiter is dropped ("a!!b" → ["a!", "b"]). The +// normalized comparison still treats both as equal, so without this test a future +// silent change to splitCapturingDelims would go unnoticed. +func TestTextParser_AdjacentDelimiters(t *testing.T) { + ctx := t.Context() + p := NewTextParser() + + // Two adjacent sentence delimiters: Go merges them into the preceding + // segment and drops the standalone second delimiter. + src := []byte("a!!b") + res := p.ParseWithResult(ctx, "doc.txt", src) + if res.Err != nil { + t.Fatalf("ParseWithResult: %v", res.Err) + } + want := []string{"a!", "b"} + if len(res.JSON) != len(want) { + t.Fatalf("adjacent delimiters: JSON len = %d, want %d: %#v", len(res.JSON), len(want), res.JSON) + } + for i, w := range want { + if got := res.JSON[i]["text"]; got != w { + t.Errorf("adjacent delimiters: JSON[%d].text = %v, want %v", i, got, w) + } + } + + // Delimiters separated by text each attach to their own segment (no merge + // across the gap). + src = []byte("x?y!z") + res = p.ParseWithResult(ctx, "doc.txt", src) + want = []string{"x?", "y!", "z"} + if len(res.JSON) != len(want) { + t.Fatalf("mixed delimiters: JSON len = %d, want %d: %#v", len(res.JSON), len(want), res.JSON) + } + for i, w := range want { + if got := res.JSON[i]["text"]; got != w { + t.Errorf("mixed delimiters: JSON[%d].text = %v, want %v", i, got, w) + } + } +} diff --git a/internal/parser/parser/text_parser.go b/internal/parser/parser/text_parser.go index 063220c2ab..033a02a883 100644 --- a/internal/parser/parser/text_parser.go +++ b/internal/parser/parser/text_parser.go @@ -135,20 +135,18 @@ func decodeRune(p []byte) (rune, int) { return 0xFFFD, 1 } -// defaultTextDelimiter is the flow parser's default delimiter set for the -// text&code family (rag/flow/parser/parser.py:_code → deepdoc TxtParser default -// "\n!?;。；！？"). The Parser component has no user-facing delimiter config -// entry (see PARSER_ALIGNMENT_HANDOFF.md §3.3), so this hard-coded default is -// exactly what the python flow always splits on. -const defaultTextDelimiter = "\n!?;。；！？" - -// defaultTextDelimiterPattern is the regexp alternation of the default -// delimiter set, each rune re.escape'd to mirror -// rag/nlp/delim.compile_delimiter_pattern. Go's regexp.Split drops captured -// delimiters, so splitCapturingDelims walks the match indexes manually to -// reproduce python's re.split(r"(%s)" % pattern, txt) interleaving. +// defaultTextDelimiterPattern is the regexp alternation of the flow parser's +// default delimiter set DefaultTextCodeDelimiter (rag/flow/parser/parser.py:_code +// → deepdoc TxtParser default "\n!?;。；！？"), each rune re.escape'd to mirror +// rag/nlp/delim.compile_delimiter_pattern. The Parser component has no user-facing +// delimiter config entry (see PARSER_ALIGNMENT_HANDOFF.md §3.3), so this default +// is exactly what the python flow always splits on. The shared DefaultTextCodeDelimiter +// const lives in delimiter.go so production and the alignment tests use one source +// of truth. Go's regexp.Split drops captured delimiters, so splitCapturingDelims +// walks the match indexes manually to reproduce python's re.split(r"(%s)" % pattern, txt) +// interleaving. var ( - defaultTextDelimiterPattern = buildDelimiterPattern(defaultTextDelimiter) + defaultTextDelimiterPattern = buildDelimiterPattern(DefaultTextCodeDelimiter) textDelimiterSplitRe = regexp.MustCompile(defaultTextDelimiterPattern) textDelimiterExactRe = regexp.MustCompile("^(?:" + defaultTextDelimiterPattern + ")$") )