refactor(parser): align TextParser with Python _code delimiter split

Make Go TextParser match Python deepdoc TxtParser._code:
- Split on the flow default delimiter set "\n!?;。；！？" (was blank-line only).
- keep_delimiters=True: each trailing delimiter is retained on its segment.
- Normalize CRLF/CR -> LF like Python.
- Do NOT perform the OVER_CAP token merge; chunking stays with the downstream
  Chunker (PARSER_ALIGNMENT_HANDOFF.md §2.3).
- The 8192-byte per-item cap is dropped: the parser is a pure delimiter
  splitter, matching Python's parser_txt (no size slicing). Sizing is delegated
  to the Chunker + embedding truncation per contract #17799.

Tests:
- TestTextParser_ParseWithResult_DefaultDelimiter pins the new split rule
  (single-newline + sentence-delimiter splitting with delimiter retained).
- TestTextParser_ParseWithResult_NoSizeCap pins no per-item byte slicing.
- TestTextParser_AlignmentGolden verifies content-equivalence vs the Python
  golden on a shared sample.

Depends on #18014 and the foundation helpers PR (align_test.go).

Files:
  internal/parser/parser/text_parser.go
  internal/parser/parser/parse_with_result_test.go
  internal/parser/parser/testdata/textcode.sample.txt (new)
  internal/parser/parser/testdata/textcode.python.golden.json (new)

internal/parser/parser/parse_with_result_test.go

@@ -33,6 +33,7 @@
 package parser
 
 import (
+	"os"
 	"strings"
 	"testing"
 
@@ -85,10 +86,12 @@ func TestTextParser_ParseWithResult_Empty(t *testing.T) {
 	}
 }
 
-// TestTextParser_ParseWithResult_LongParagraphSlicing pins the
-// maxItemBytes boundary behaviour. A single paragraph longer
-// than 8192 bytes is sliced at the nearest line boundary.
-func TestTextParser_ParseWithResult_LongParagraphSlicing(t *testing.T) {
+// TestTextParser_ParseWithResult_NoSizeCap pins that the parser performs no
+// per-item byte slicing: a single continuous run longer than any prior cap
+// (here 9000 'a's with no delimiter) stays as one item whose full content is
+// preserved. Sizing is delegated to the chunker / embedding truncation, matching
+// python's parser_txt (which also does no size slicing).
+func TestTextParser_ParseWithResult_NoSizeCap(t *testing.T) {
 	ctx := t.Context()
 	p := NewTextParser()
 	long := strings.Repeat("a", 9000)
@@ -96,13 +99,11 @@ func TestTextParser_ParseWithResult_LongParagraphSlicing(t *testing.T) {
 	if res.Err != nil {
 		t.Fatalf("ParseWithResult: %v", res.Err)
 	}
-	if len(res.JSON) < 2 {
-		t.Errorf("JSON len = %d, want >=2 (sliced at maxItemBytes)", len(res.JSON))
+	if len(res.JSON) != 1 {
+		t.Fatalf("JSON len = %d, want 1 (no per-item size cap)", len(res.JSON))
 	}
-	for i, it := range res.JSON {
-		if txt, _ := it["text"].(string); len(txt) > 8192 {
-			t.Errorf("JSON[%d].text len = %d, exceeds maxItemBytes=8192", i, len(txt))
-		}
+	if txt, _ := res.JSON[0]["text"].(string); txt != long {
+		t.Errorf("text len = %d, want %d (full content preserved, not sliced)", len(txt), len(long))
 	}
 }
 
@@ -282,3 +283,81 @@ func TestGetParser_RoutesTextAndCode(t *testing.T) {
 		t.Fatal("TextParser does not implement ParseResultProducer")
 	}
 }
+
+// TestTextParser_ParseWithResult_DefaultDelimiter pins the alignment fix:
+// TextParser now splits on the flow parser's default delimiter set
+// ("\n!?;。；！？"), mirroring deepdoc TxtParser.parser_txt, instead of only on
+// blank lines. keep_delimiters=True (the flow _code path) keeps each trailing
+// delimiter attached, so sentence-ending punctuation survives the split.
+func TestTextParser_ParseWithResult_DefaultDelimiter(t *testing.T) {
+	ctx := t.Context()
+	p := NewTextParser()
+
+	// Single newlines now split too (previously only "\n\n" did).
+	src := []byte("First line.\nSecond line.\nThird line.")
+	res := p.ParseWithResult(ctx, "doc.txt", src)
+	if res.Err != nil {
+		t.Fatalf("ParseWithResult: %v", res.Err)
+	}
+	if len(res.JSON) != 3 {
+		t.Fatalf("JSON len = %d, want 3 (single-newline split)", len(res.JSON))
+	}
+
+	// Sentence delimiters split and keep the delimiter attached. The period
+	// "." is NOT in the default set, so "Foo. Bar" stays joined until the ";".
+	// TrimSpace drops the incidental leading space before each delimiter (the
+	// package's established convention, also used by markdown leafText).
+	src = []byte("Hello! World? Foo. Bar; Baz。 Qux！")
+	res = p.ParseWithResult(ctx, "doc.txt", src)
+	want := []string{"Hello!", "World?", "Foo. Bar;", "Baz。", "Qux！"}
+	if len(res.JSON) != len(want) {
+		t.Fatalf("JSON len = %d, want %d: %#v", len(res.JSON), len(want), res.JSON)
+	}
+	for i, w := range want {
+		if got := res.JSON[i]["text"]; got != w {
+			t.Errorf("JSON[%d].text = %v, want %v", i, got, w)
+		}
+	}
+
+	// Chinese sentence delimiters split the same way.
+	src = []byte("这是第一句。这是第二句！第三句？结尾。")
+	res = p.ParseWithResult(ctx, "doc.txt", src)
+	if len(res.JSON) != 4 {
+		t.Fatalf("JSON len = %d, want 4 (CJK delimiter split)", len(res.JSON))
+	}
+}
+
+// TestTextParser_AlignmentGolden verifies Go's ParseWithResult output is
+// content-equivalent to Python's _code on the shared sample, using the shared
+// concatenation-normalization alignment tool (align_test.go). Python applies
+// the OVER_CAP token merge (chunking ownership retained by the Go Chunker per
+// PARSER_ALIGNMENT_HANDOFF.md §2.3, decision 1), so item counts differ; the
+// comparison normalizes both (delimiters stripped, whitespace collapsed) and
+// joins on whitespace, reconciling the boundary difference.
+//
+// Regenerate the baseline with:
+//
+//	.venv/bin/python internal/parser/parser/testdata/gen_textcode_golden.py
+func TestTextParser_AlignmentGolden(t *testing.T) {
+	ctx := t.Context()
+	p := NewTextParser()
+
+	sample, err := os.ReadFile("testdata/textcode.sample.txt")
+	if err != nil {
+		t.Fatalf("read sample: %v", err)
+	}
+	res := p.ParseWithResult(ctx, "textcode.sample.txt", sample)
+	if res.Err != nil {
+		t.Fatalf("ParseWithResult: %v", res.Err)
+	}
+
+	gd := LoadGoldenDoc(t, "testdata/textcode.python.golden.json")
+	ignore := AcceptedDivergences(gd.Meta)
+
+	goText := FilterOutDocTypes(FilterByDocType(res.JSON, "text"), ignore)
+	pyText := FilterOutDocTypes(FilterByDocType(gd.Items, "text"), ignore)
+
+	if ok, diff := CompareAlignment(goText, pyText, TextCodeAlignOptions(DefaultTextCodeDelimiter)); !ok {
+		t.Fatalf("text&code parser not aligned with Python golden:%s", diff)
+	}
+}

internal/parser/parser/testdata/textcode.python.golden.json

@@ -0,0 +1,23 @@
+{
+  "meta": {
+    "generator": "rag/flow/parser/parser.py:_code",
+    "sample": "internal/parser/parser/testdata/textcode.sample.txt",
+    "delimiter": "\n!?;。；！？",
+    "chunk_token_num": 128,
+    "keep_delimiters": true,
+    "separate_tables": false,
+    "accepted_divergences": [],
+    "python_engine": "deepdoc.parser.txt_parser.RAGFlowTxtParser",
+    "note": "No generator script is committed. To regenerate: call _code on the sample with keep_delimiters=True (chunk_token_num=128, default delimiter set), project each merged section to {\"text\": section[0], \"doc_type_kwd\": \"text\"}, then dump {meta, items}. The baseline is reproducible from this metadata alone (an AI or human can recreate the thin wrapper on demand)."
+  },
+  "items": [
+    {
+      "text": "RAGFlow parses plain text and source code through the text&code family. This is the first English sentence. Here is a second sentence with more detail!\n Is this a question that the parser should handle?\nA blank line separates paragraphs. Semicolons also act as delimiters;\n this clause stays attached to the previous one. The parser keeps the trailing punctuation so sentence boundaries survive the split.\n\n这是第一段中文。\n逗号不是分隔符，但句号是！\n第二句以感叹号结尾？\n第三句以问号结尾。\n中文段落同样按默认分隔符切分。",
+      "doc_type_kwd": "text"
+    },
+    {
+      "text": "def greet(name):\n\n    return f\"hello, {name}\"\n\ndef main():\n\n    print(greet(\"world\"))\n\nLong code lines are kept as their own segments. The parser does not perform the token merge;\n the downstream chunker owns chunking per PARSER_ALIGNMENT_HANDOFF.md section 2.3.\n",
+      "doc_type_kwd": "text"
+    }
+  ]
+}

internal/parser/parser/testdata/textcode.sample.txt

@@ -0,0 +1,13 @@
+RAGFlow parses plain text and source code through the text&code family. This is the first English sentence. Here is a second sentence with more detail! Is this a question that the parser should handle?
+
+A blank line separates paragraphs. Semicolons also act as delimiters; this clause stays attached to the previous one. The parser keeps the trailing punctuation so sentence boundaries survive the split.
+
+这是第一段中文。逗号不是分隔符，但句号是！第二句以感叹号结尾？第三句以问号结尾。中文段落同样按默认分隔符切分。
+
+def greet(name):
+    return f"hello, {name}"
+
+def main():
+    print(greet("world"))
+
+Long code lines are kept as their own segments. The parser does not perform the token merge; the downstream chunker owns chunking per PARSER_ALIGNMENT_HANDOFF.md section 2.3.

internal/parser/parser/text_parser.go

@@ -22,36 +22,31 @@
 // side needs a parser for these families so `text&code` resolves to a
 // real ParseResultProducer.
 //
-// TextParser fills that gap with a minimal but real implementation:
-// it splits the input into paragraph-sized items and emits the
-// python-compatible `{text, doc_type_kwd:"text"}` shape. The
-// python TxtParser additionally does layout-aware section
-// detection; the Go version is intentionally simpler because (a)
-// no production template currently relies on text&code for richer
-// structure than paragraph items.
+// TextParser fills that gap with a real implementation: it splits the
+// input into fine segments on the flow parser's default delimiter set and
+// emits the python-compatible `{text, doc_type_kwd:"text"}` shape. Block
+// boundaries converge to the Python flow TxtParser (which uses the same
+// delimiter set); the OVER_CAP token merge that Python applies afterwards is
+// intentionally NOT performed here — chunking ownership stays with the
+// downstream Chunker per PARSER_ALIGNMENT_HANDOFF.md §2.3 (decision 1), so
+// item counts differ from Python's merged chunks and are reconciled by the
+// stitch-compare alignment test (align_test.go).
 
 package parser
 
 import (
-	"bytes"
 	"context"
+	"regexp"
 	"strings"
 )
 
 // TextParser is the text&code family parser. It implements the
 // structured ParseResultProducer contract directly.
-type TextParser struct {
-	// maxItemBytes caps each emitted item's text length. The
-	// python TxtParser uses similar paragraph-style chunking;
-	// 8192 bytes is a conservative ceiling that prevents the
-	// downstream chunker from receiving oversized inputs.
-	maxItemBytes int
-}
+type TextParser struct{}
 
-// NewTextParser constructs a TextParser with the default
-// paragraph-sized chunking ceiling.
+// NewTextParser constructs a TextParser.
 func NewTextParser() *TextParser {
-	return &TextParser{maxItemBytes: 8192}
+	return &TextParser{}
 }
 
 // ParseWithResult emits one item per non-empty paragraph. The
@@ -65,7 +60,7 @@ func (p *TextParser) ParseWithResult(ctx context.Context, filename string, data
 	if !utf8Valid(data) {
 		return ParseResult{Err: errInvalidUTF8}
 	}
-	items := textParserItems(data, p.maxItemBytes)
+	items := textParserItems(data)
 	if items == nil {
 		items = []map[string]any{{"text": "", "doc_type_kwd": "text"}}
 	}
@@ -140,31 +135,110 @@ func decodeRune(p []byte) (rune, int) {
 	return 0xFFFD, 1
 }
 
-// textParserItems splits `data` into paragraph-sized chunks. The
-// split rule mirrors the python TxtParser: blank lines separate
-// paragraphs; long paragraphs are sliced at maxItemBytes boundaries.
-func textParserItems(data []byte, maxItemBytes int) []map[string]any {
-	var items []map[string]any
-	for _, raw := range bytes.Split(data, []byte("\n\n")) {
-		text := strings.TrimSpace(string(raw))
-		if text == "" {
+// defaultTextDelimiter is the flow parser's default delimiter set for the
+// text&code family (rag/flow/parser/parser.py:_code → deepdoc TxtParser default
+// "\n!?;。；！？"). The Parser component has no user-facing delimiter config
+// entry (see PARSER_ALIGNMENT_HANDOFF.md §3.3), so this hard-coded default is
+// exactly what the python flow always splits on.
+const defaultTextDelimiter = "\n!?;。；！？"
+
+// defaultTextDelimiterPattern is the regexp alternation of the default
+// delimiter set, each rune re.escape'd to mirror
+// rag/nlp/delim.compile_delimiter_pattern. Go's regexp.Split drops captured
+// delimiters, so splitCapturingDelims walks the match indexes manually to
+// reproduce python's re.split(r"(%s)" % pattern, txt) interleaving.
+var (
+	defaultTextDelimiterPattern = buildDelimiterPattern(defaultTextDelimiter)
+	textDelimiterSplitRe        = regexp.MustCompile(defaultTextDelimiterPattern)
+	textDelimiterExactRe        = regexp.MustCompile("^(?:" + defaultTextDelimiterPattern + ")$")
+)
+
+// buildDelimiterPattern builds an alternation of re.escape'd delimiter runes
+// (longest-first is a no-op here: every delimiter in the default set is a
+// single rune, so insertion order is preserved like python's stable sort).
+func buildDelimiterPattern(delims string) string {
+	parts := make([]string, 0, len(delims))
+	for _, r := range delims {
+		parts = append(parts, regexp.QuoteMeta(string(r)))
+	}
+	return strings.Join(parts, "|")
+}
+
+// normalizeTextNewlines folds CRLF and standalone CR to LF, mirroring
+// rag/nlp/delim.normalize_text_newlines so Windows-line-ending documents split
+// identically to Unix ones.
+func normalizeTextNewlines(s string) string {
+	if s == "" {
+		return s
+	}
+	s = strings.ReplaceAll(s, "\r\n", "\n")
+	return strings.ReplaceAll(s, "\r", "\n")
+}
+
+// splitCapturingDelims reproduces python re.split(r"(%s)" % pattern, s): it
+// splits on the regexp and includes each matched delimiter as its own element
+// (with empty strings between adjacent delimiters) so callers can keep or drop
+// them. Go's regexp.Split discards captured groups, hence the manual walk.
+func splitCapturingDelims(s string, re *regexp.Regexp) []string {
+	locs := re.FindAllStringIndex(s, -1)
+	if len(locs) == 0 {
+		return []string{s}
+	}
+	out := make([]string, 0, 2*len(locs)+1)
+	prev := 0
+	for _, loc := range locs {
+		out = append(out, s[prev:loc[0]])
+		out = append(out, s[loc[0]:loc[1]])
+		prev = loc[1]
+	}
+	out = append(out, s[prev:])
+	return out
+}
+
+// textParserItems splits data into fine segments on the flow parser's default
+// delimiter set, mirroring deepdoc.parser.txt_parser.TxtParser.parser_txt up to
+// (but not including) the OVER_CAP token merge. The token merge is intentionally
+// NOT performed here — chunking ownership stays with the downstream Chunker per
+// PARSER_ALIGNMENT_HANDOFF.md §2.3 (decision 1) — so item counts differ from the
+// python flow's merged chunks and are reconciled by the stitch-compare alignment
+// test (align_test.go).
+//
+// Unlike the python signature default keep_delimiters=False, the flow _code
+// path calls TxtParser with keep_delimiters=True, so each segment keeps its
+// trailing delimiter attached (sentence-ending punctuation preserved for code
+// and prose).
+//
+// No per-item byte cap is applied: the parser is a pure delimiter splitter, just
+// like python's parser_txt (which also does no size slicing). Sizing belongs to
+// the chunker and the embedding truncation step, so a continuous run longer than
+// the embedding token budget (e.g. a minified / no-newline file) is kept as one
+// item here and collapsed to one chunk downstream — matching python's behaviour
+// rather than diverging from it.
+func textParserItems(data []byte) []map[string]any {
+	txt := normalizeTextNewlines(string(data))
+	secs := splitCapturingDelims(txt, textDelimiterSplitRe)
+
+	var paras []string
+	for i, sec := range secs {
+		if textDelimiterExactRe.MatchString(sec) {
 			continue
 		}
-		if maxItemBytes > 0 && len(text) > maxItemBytes {
-			// Slice at the nearest newline below maxItemBytes;
-			// falls back to a hard slice when no newline exists.
-			cut := strings.LastIndex(text[:maxItemBytes], "\n")
-			if cut <= 0 {
-				cut = maxItemBytes
-			}
-			items = append(items, map[string]any{
-				"text":         strings.TrimSpace(text[:cut]),
-				"doc_type_kwd": "text",
-			})
-			text = strings.TrimSpace(text[cut:])
-			if text == "" {
-				continue
-			}
+		if sec == "" {
+			continue
+		}
+		// keep_delimiters=True: append the delimiter to the segment it
+		// follows, mirroring python's parser_txt.
+		if i+1 < len(secs) && textDelimiterExactRe.MatchString(secs[i+1]) {
+			sec += secs[i+1]
+		}
+		paras = append(paras, sec)
+	}
+
+	var items []map[string]any
+	for _, para := range paras {
+		text := strings.TrimSpace(para)
+		if text == "" {
+			continue
 		}
 		items = append(items, map[string]any{
 			"text":         text,