diff --git a/internal/parser/parser/parse_with_result_test.go b/internal/parser/parser/parse_with_result_test.go index 8ab285dd69..ca2c03269e 100644 --- a/internal/parser/parser/parse_with_result_test.go +++ b/internal/parser/parser/parse_with_result_test.go @@ -33,6 +33,7 @@ package parser import ( + "os" "strings" "testing" @@ -85,10 +86,12 @@ func TestTextParser_ParseWithResult_Empty(t *testing.T) { } } -// TestTextParser_ParseWithResult_LongParagraphSlicing pins the -// maxItemBytes boundary behaviour. A single paragraph longer -// than 8192 bytes is sliced at the nearest line boundary. -func TestTextParser_ParseWithResult_LongParagraphSlicing(t *testing.T) { +// TestTextParser_ParseWithResult_NoSizeCap pins that the parser performs no +// per-item byte slicing: a single continuous run longer than any prior cap +// (here 9000 'a's with no delimiter) stays as one item whose full content is +// preserved. Sizing is delegated to the chunker / embedding truncation, matching +// python's parser_txt (which also does no size slicing). +func TestTextParser_ParseWithResult_NoSizeCap(t *testing.T) { ctx := t.Context() p := NewTextParser() long := strings.Repeat("a", 9000) @@ -96,13 +99,11 @@ func TestTextParser_ParseWithResult_LongParagraphSlicing(t *testing.T) { if res.Err != nil { t.Fatalf("ParseWithResult: %v", res.Err) } - if len(res.JSON) < 2 { - t.Errorf("JSON len = %d, want >=2 (sliced at maxItemBytes)", len(res.JSON)) + if len(res.JSON) != 1 { + t.Fatalf("JSON len = %d, want 1 (no per-item size cap)", len(res.JSON)) } - for i, it := range res.JSON { - if txt, _ := it["text"].(string); len(txt) > 8192 { - t.Errorf("JSON[%d].text len = %d, exceeds maxItemBytes=8192", i, len(txt)) - } + if txt, _ := res.JSON[0]["text"].(string); txt != long { + t.Errorf("text len = %d, want %d (full content preserved, not sliced)", len(txt), len(long)) } } @@ -282,3 +283,81 @@ func TestGetParser_RoutesTextAndCode(t *testing.T) { t.Fatal("TextParser does not implement ParseResultProducer") } } + +// TestTextParser_ParseWithResult_DefaultDelimiter pins the alignment fix: +// TextParser now splits on the flow parser's default delimiter set +// ("\n!?;。;!?"), mirroring deepdoc TxtParser.parser_txt, instead of only on +// blank lines. keep_delimiters=True (the flow _code path) keeps each trailing +// delimiter attached, so sentence-ending punctuation survives the split. +func TestTextParser_ParseWithResult_DefaultDelimiter(t *testing.T) { + ctx := t.Context() + p := NewTextParser() + + // Single newlines now split too (previously only "\n\n" did). + src := []byte("First line.\nSecond line.\nThird line.") + res := p.ParseWithResult(ctx, "doc.txt", src) + if res.Err != nil { + t.Fatalf("ParseWithResult: %v", res.Err) + } + if len(res.JSON) != 3 { + t.Fatalf("JSON len = %d, want 3 (single-newline split)", len(res.JSON)) + } + + // Sentence delimiters split and keep the delimiter attached. The period + // "." is NOT in the default set, so "Foo. Bar" stays joined until the ";". + // TrimSpace drops the incidental leading space before each delimiter (the + // package's established convention, also used by markdown leafText). + src = []byte("Hello! World? Foo. Bar; Baz。 Qux!") + res = p.ParseWithResult(ctx, "doc.txt", src) + want := []string{"Hello!", "World?", "Foo. Bar;", "Baz。", "Qux!"} + if len(res.JSON) != len(want) { + t.Fatalf("JSON len = %d, want %d: %#v", len(res.JSON), len(want), res.JSON) + } + for i, w := range want { + if got := res.JSON[i]["text"]; got != w { + t.Errorf("JSON[%d].text = %v, want %v", i, got, w) + } + } + + // Chinese sentence delimiters split the same way. + src = []byte("这是第一句。这是第二句!第三句?结尾。") + res = p.ParseWithResult(ctx, "doc.txt", src) + if len(res.JSON) != 4 { + t.Fatalf("JSON len = %d, want 4 (CJK delimiter split)", len(res.JSON)) + } +} + +// TestTextParser_AlignmentGolden verifies Go's ParseWithResult output is +// content-equivalent to Python's _code on the shared sample, using the shared +// concatenation-normalization alignment tool (align_test.go). Python applies +// the OVER_CAP token merge (chunking ownership retained by the Go Chunker per +// PARSER_ALIGNMENT_HANDOFF.md §2.3, decision 1), so item counts differ; the +// comparison normalizes both (delimiters stripped, whitespace collapsed) and +// joins on whitespace, reconciling the boundary difference. +// +// Regenerate the baseline with: +// +// .venv/bin/python internal/parser/parser/testdata/gen_textcode_golden.py +func TestTextParser_AlignmentGolden(t *testing.T) { + ctx := t.Context() + p := NewTextParser() + + sample, err := os.ReadFile("testdata/textcode.sample.txt") + if err != nil { + t.Fatalf("read sample: %v", err) + } + res := p.ParseWithResult(ctx, "textcode.sample.txt", sample) + if res.Err != nil { + t.Fatalf("ParseWithResult: %v", res.Err) + } + + gd := LoadGoldenDoc(t, "testdata/textcode.python.golden.json") + ignore := AcceptedDivergences(gd.Meta) + + goText := FilterOutDocTypes(FilterByDocType(res.JSON, "text"), ignore) + pyText := FilterOutDocTypes(FilterByDocType(gd.Items, "text"), ignore) + + if ok, diff := CompareAlignment(goText, pyText, TextCodeAlignOptions(DefaultTextCodeDelimiter)); !ok { + t.Fatalf("text&code parser not aligned with Python golden:%s", diff) + } +} diff --git a/internal/parser/parser/testdata/textcode.python.golden.json b/internal/parser/parser/testdata/textcode.python.golden.json new file mode 100644 index 0000000000..465f9e2518 --- /dev/null +++ b/internal/parser/parser/testdata/textcode.python.golden.json @@ -0,0 +1,23 @@ +{ + "meta": { + "generator": "rag/flow/parser/parser.py:_code", + "sample": "internal/parser/parser/testdata/textcode.sample.txt", + "delimiter": "\n!?;。;!?", + "chunk_token_num": 128, + "keep_delimiters": true, + "separate_tables": false, + "accepted_divergences": [], + "python_engine": "deepdoc.parser.txt_parser.RAGFlowTxtParser", + "note": "No generator script is committed. To regenerate: call _code on the sample with keep_delimiters=True (chunk_token_num=128, default delimiter set), project each merged section to {\"text\": section[0], \"doc_type_kwd\": \"text\"}, then dump {meta, items}. The baseline is reproducible from this metadata alone (an AI or human can recreate the thin wrapper on demand)." + }, + "items": [ + { + "text": "RAGFlow parses plain text and source code through the text&code family. This is the first English sentence. Here is a second sentence with more detail!\n Is this a question that the parser should handle?\nA blank line separates paragraphs. Semicolons also act as delimiters;\n this clause stays attached to the previous one. The parser keeps the trailing punctuation so sentence boundaries survive the split.\n\n这是第一段中文。\n逗号不是分隔符,但句号是!\n第二句以感叹号结尾?\n第三句以问号结尾。\n中文段落同样按默认分隔符切分。", + "doc_type_kwd": "text" + }, + { + "text": "def greet(name):\n\n return f\"hello, {name}\"\n\ndef main():\n\n print(greet(\"world\"))\n\nLong code lines are kept as their own segments. The parser does not perform the token merge;\n the downstream chunker owns chunking per PARSER_ALIGNMENT_HANDOFF.md section 2.3.\n", + "doc_type_kwd": "text" + } + ] +} \ No newline at end of file diff --git a/internal/parser/parser/testdata/textcode.sample.txt b/internal/parser/parser/testdata/textcode.sample.txt new file mode 100644 index 0000000000..36747f9eba --- /dev/null +++ b/internal/parser/parser/testdata/textcode.sample.txt @@ -0,0 +1,13 @@ +RAGFlow parses plain text and source code through the text&code family. This is the first English sentence. Here is a second sentence with more detail! Is this a question that the parser should handle? + +A blank line separates paragraphs. Semicolons also act as delimiters; this clause stays attached to the previous one. The parser keeps the trailing punctuation so sentence boundaries survive the split. + +这是第一段中文。逗号不是分隔符,但句号是!第二句以感叹号结尾?第三句以问号结尾。中文段落同样按默认分隔符切分。 + +def greet(name): + return f"hello, {name}" + +def main(): + print(greet("world")) + +Long code lines are kept as their own segments. The parser does not perform the token merge; the downstream chunker owns chunking per PARSER_ALIGNMENT_HANDOFF.md section 2.3. diff --git a/internal/parser/parser/text_parser.go b/internal/parser/parser/text_parser.go index 6f2f03769f..063220c2ab 100644 --- a/internal/parser/parser/text_parser.go +++ b/internal/parser/parser/text_parser.go @@ -22,36 +22,31 @@ // side needs a parser for these families so `text&code` resolves to a // real ParseResultProducer. // -// TextParser fills that gap with a minimal but real implementation: -// it splits the input into paragraph-sized items and emits the -// python-compatible `{text, doc_type_kwd:"text"}` shape. The -// python TxtParser additionally does layout-aware section -// detection; the Go version is intentionally simpler because (a) -// no production template currently relies on text&code for richer -// structure than paragraph items. +// TextParser fills that gap with a real implementation: it splits the +// input into fine segments on the flow parser's default delimiter set and +// emits the python-compatible `{text, doc_type_kwd:"text"}` shape. Block +// boundaries converge to the Python flow TxtParser (which uses the same +// delimiter set); the OVER_CAP token merge that Python applies afterwards is +// intentionally NOT performed here — chunking ownership stays with the +// downstream Chunker per PARSER_ALIGNMENT_HANDOFF.md §2.3 (decision 1), so +// item counts differ from Python's merged chunks and are reconciled by the +// stitch-compare alignment test (align_test.go). package parser import ( - "bytes" "context" + "regexp" "strings" ) // TextParser is the text&code family parser. It implements the // structured ParseResultProducer contract directly. -type TextParser struct { - // maxItemBytes caps each emitted item's text length. The - // python TxtParser uses similar paragraph-style chunking; - // 8192 bytes is a conservative ceiling that prevents the - // downstream chunker from receiving oversized inputs. - maxItemBytes int -} +type TextParser struct{} -// NewTextParser constructs a TextParser with the default -// paragraph-sized chunking ceiling. +// NewTextParser constructs a TextParser. func NewTextParser() *TextParser { - return &TextParser{maxItemBytes: 8192} + return &TextParser{} } // ParseWithResult emits one item per non-empty paragraph. The @@ -65,7 +60,7 @@ func (p *TextParser) ParseWithResult(ctx context.Context, filename string, data if !utf8Valid(data) { return ParseResult{Err: errInvalidUTF8} } - items := textParserItems(data, p.maxItemBytes) + items := textParserItems(data) if items == nil { items = []map[string]any{{"text": "", "doc_type_kwd": "text"}} } @@ -140,31 +135,110 @@ func decodeRune(p []byte) (rune, int) { return 0xFFFD, 1 } -// textParserItems splits `data` into paragraph-sized chunks. The -// split rule mirrors the python TxtParser: blank lines separate -// paragraphs; long paragraphs are sliced at maxItemBytes boundaries. -func textParserItems(data []byte, maxItemBytes int) []map[string]any { - var items []map[string]any - for _, raw := range bytes.Split(data, []byte("\n\n")) { - text := strings.TrimSpace(string(raw)) - if text == "" { +// defaultTextDelimiter is the flow parser's default delimiter set for the +// text&code family (rag/flow/parser/parser.py:_code → deepdoc TxtParser default +// "\n!?;。;!?"). The Parser component has no user-facing delimiter config +// entry (see PARSER_ALIGNMENT_HANDOFF.md §3.3), so this hard-coded default is +// exactly what the python flow always splits on. +const defaultTextDelimiter = "\n!?;。;!?" + +// defaultTextDelimiterPattern is the regexp alternation of the default +// delimiter set, each rune re.escape'd to mirror +// rag/nlp/delim.compile_delimiter_pattern. Go's regexp.Split drops captured +// delimiters, so splitCapturingDelims walks the match indexes manually to +// reproduce python's re.split(r"(%s)" % pattern, txt) interleaving. +var ( + defaultTextDelimiterPattern = buildDelimiterPattern(defaultTextDelimiter) + textDelimiterSplitRe = regexp.MustCompile(defaultTextDelimiterPattern) + textDelimiterExactRe = regexp.MustCompile("^(?:" + defaultTextDelimiterPattern + ")$") +) + +// buildDelimiterPattern builds an alternation of re.escape'd delimiter runes +// (longest-first is a no-op here: every delimiter in the default set is a +// single rune, so insertion order is preserved like python's stable sort). +func buildDelimiterPattern(delims string) string { + parts := make([]string, 0, len(delims)) + for _, r := range delims { + parts = append(parts, regexp.QuoteMeta(string(r))) + } + return strings.Join(parts, "|") +} + +// normalizeTextNewlines folds CRLF and standalone CR to LF, mirroring +// rag/nlp/delim.normalize_text_newlines so Windows-line-ending documents split +// identically to Unix ones. +func normalizeTextNewlines(s string) string { + if s == "" { + return s + } + s = strings.ReplaceAll(s, "\r\n", "\n") + return strings.ReplaceAll(s, "\r", "\n") +} + +// splitCapturingDelims reproduces python re.split(r"(%s)" % pattern, s): it +// splits on the regexp and includes each matched delimiter as its own element +// (with empty strings between adjacent delimiters) so callers can keep or drop +// them. Go's regexp.Split discards captured groups, hence the manual walk. +func splitCapturingDelims(s string, re *regexp.Regexp) []string { + locs := re.FindAllStringIndex(s, -1) + if len(locs) == 0 { + return []string{s} + } + out := make([]string, 0, 2*len(locs)+1) + prev := 0 + for _, loc := range locs { + out = append(out, s[prev:loc[0]]) + out = append(out, s[loc[0]:loc[1]]) + prev = loc[1] + } + out = append(out, s[prev:]) + return out +} + +// textParserItems splits data into fine segments on the flow parser's default +// delimiter set, mirroring deepdoc.parser.txt_parser.TxtParser.parser_txt up to +// (but not including) the OVER_CAP token merge. The token merge is intentionally +// NOT performed here — chunking ownership stays with the downstream Chunker per +// PARSER_ALIGNMENT_HANDOFF.md §2.3 (decision 1) — so item counts differ from the +// python flow's merged chunks and are reconciled by the stitch-compare alignment +// test (align_test.go). +// +// Unlike the python signature default keep_delimiters=False, the flow _code +// path calls TxtParser with keep_delimiters=True, so each segment keeps its +// trailing delimiter attached (sentence-ending punctuation preserved for code +// and prose). +// +// No per-item byte cap is applied: the parser is a pure delimiter splitter, just +// like python's parser_txt (which also does no size slicing). Sizing belongs to +// the chunker and the embedding truncation step, so a continuous run longer than +// the embedding token budget (e.g. a minified / no-newline file) is kept as one +// item here and collapsed to one chunk downstream — matching python's behaviour +// rather than diverging from it. +func textParserItems(data []byte) []map[string]any { + txt := normalizeTextNewlines(string(data)) + secs := splitCapturingDelims(txt, textDelimiterSplitRe) + + var paras []string + for i, sec := range secs { + if textDelimiterExactRe.MatchString(sec) { continue } - if maxItemBytes > 0 && len(text) > maxItemBytes { - // Slice at the nearest newline below maxItemBytes; - // falls back to a hard slice when no newline exists. - cut := strings.LastIndex(text[:maxItemBytes], "\n") - if cut <= 0 { - cut = maxItemBytes - } - items = append(items, map[string]any{ - "text": strings.TrimSpace(text[:cut]), - "doc_type_kwd": "text", - }) - text = strings.TrimSpace(text[cut:]) - if text == "" { - continue - } + if sec == "" { + continue + } + // keep_delimiters=True: append the delimiter to the segment it + // follows, mirroring python's parser_txt. + if i+1 < len(secs) && textDelimiterExactRe.MatchString(secs[i+1]) { + sec += secs[i+1] + } + paras = append(paras, sec) + } + + var items []map[string]any + for _, para := range paras { + text := strings.TrimSpace(para) + if text == "" { + continue } items = append(items, map[string]any{ "text": text,