refactor(nlp): collapse 6 delimiter-parser implementations into one canonical helper (#17383) (#17387)

## Summary

Six sites used to read the same `parser_config.delimiter` field with
divergent grammars:

- `rag.nlp.get_delimiters` (PDF/DOCX/HTML/EPUB/JSON/CSV/XLSX/email/book)
- `rag.nlp.naive_merge` (custom-delimiter branch)
- `rag.nlp.naive_merge_with_images`
- `rag.nlp._build_cks`
- `deepdoc.parser.txt_parser.parser_txt` (.txt, code)
-
`deepdoc.parser.markdown_parser.MarkdownElementExtractor.get_delimiters`

The six implementations disagreed on bare-vs-wrapped chars, dedupe, sort
order, CRLF normalization, and `re.I` (#17384). The shipped default ``
`\n!?;。；！？` `` was a no-op for `.md` because the markdown path only
matched backtick-wrapped tokens.

## Changes

- **new:** `rag/nlp/delim.py` with `parse_delimiter_field` and
`compile_delimiter_pattern`. Single source of truth. CRLF normalization
at the top; longest-first stable sort; insertion-ordered dedupe; no
`re.I`.
- **refactor:** all six call sites delegate to the helper.
  - `rag/nlp/__init__.py::get_delimiters` becomes a thin shim.
- `deepdoc/parser/txt_parser.py::parser_txt` drops the
`[encode/decode/unicode_escape]` round-trip.
- `deepdoc/parser/markdown_parser.py::get_delimiters` honors bare chars
(fixes [1]).
- **tests:** `test/unit_test/rag/test_delim.py` (85 tests) — helper,
acceptance table, frontend parity, static guard against re-inlining.
- **tests:** `test/unit_test/rag/test_delimiter_case_sensitive.py` (from
#17386) updated to retarget the static check at the new helper +
AST-based broader scan.

## Acceptance criteria

- All six sites produce the same regex pattern for the same input.
- Shipped default keeps working for `.txt` / `.pdf` / `.docx`.
- Shipped default for `.md` now splits (was a silent no-op).
- Tooltip example `` `\n##;` `` produces three effective delimiters
regardless of file type.
- Bare whitespace inputs split on every occurrence.
- Backtick-wrapped whitespace splits only on the exact N-char sequence.
- CRLF-line-ending documents split identically to LF-line-ending
documents.
- 123 tests pass (85 new + 38 existing).

## Rebase protocol

As #17385 and #17386 evolve, this branch will be rebased on top. The
only overlap between this PR's diff and the other two is
`test_delimiter_case_sensitive.py`, where #17383 modifies the static
check to point at the new helper location.

---------

Co-authored-by: kiloconnect[bot] <240665456+kiloconnect[bot]@users.noreply.github.com>

deepdoc/parser/markdown_parser.py

@@ -20,6 +20,12 @@ import re
 
 from markdown import markdown
 
+from rag.nlp.delim import (
+    compile_delimiter_pattern,
+    normalize_text_newlines,
+    parse_delimiter_field,
+)
+
 logger = logging.getLogger(__name__)
 
 
@@ -153,13 +159,19 @@ class RAGFlowMarkdownParser:
 
 class MarkdownElementExtractor:
     def __init__(self, markdown_content):
-        self.markdown_content = markdown_content
-        self.lines = markdown_content.split("\n")
+        # Normalize CRLF/CR so compiled delimiter patterns (which use LF)
+        # match Windows-line-ending source the same as Unix source.
+        self.markdown_content = normalize_text_newlines(markdown_content)
+        self.lines = self.markdown_content.split("\n")
 
     def get_delimiters(self, delimiters):
-        toks = re.findall(r"`([^`]+)`", delimiters)
-        toks = sorted(set(toks), key=lambda x: -len(x))
-        return "|".join(re.escape(t) for t in toks if t)
+        # Delegate to the canonical parser (#17383). The previous
+        # implementation matched only backtick-wrapped tokens and dropped
+        # bare characters, which silently made the shipped default
+        # delimiter field a no-op for the markdown path. The helper honors
+        # both bare chars and wrapped tokens, so the same field produces
+        # the same splits in every file type.
+        return compile_delimiter_pattern(parse_delimiter_field(delimiters))
 
     def _get_fence_marker(self, line):
         match = re.match(r"^[ \t]{0,3}(?P<fence>`{3,}|~{3,})(?:.*)$", line)

deepdoc/parser/txt_parser.py

@@ -20,6 +20,11 @@ import re
 from common.token_utils import num_tokens_from_string
 from deepdoc.parser.utils import get_text
 from rag.nlp import _split_oversized_unit
+from rag.nlp.delim import (
+    compile_delimiter_pattern,
+    normalize_text_newlines,
+    parse_delimiter_field,
+)
 
 
 class RAGFlowTxtParser:
@@ -33,10 +38,9 @@ class RAGFlowTxtParser:
             raise TypeError("txt type should be str!")
         cks = [""]
         tk_nums = [0]
-        delimiter = delimiter.encode("utf-8").decode("unicode_escape").encode("latin1").decode("utf-8")
 
         def add_chunk(t):
-            nonlocal cks, tk_nums, delimiter
+            nonlocal cks, tk_nums
             tnum = num_tokens_from_string(t)
 
             if cks[-1] == "":
@@ -54,21 +58,17 @@ class RAGFlowTxtParser:
             cks.append(t)
             tk_nums.append(tnum)
 
-        dels = []
-        s = 0
-        for m in re.finditer(r"`([^`]+)`", delimiter):
-            f, m_t = m.span()
-            dels.append(m.group(1))
-            dels.extend(list(delimiter[s:f]))
-            s = m_t
-        if s < len(delimiter):
-            dels.extend(list(delimiter[s:]))
-        dels = [re.escape(d) for d in dels if d]
-        dels = [d for d in dels if d]
-        dels = "|".join(dels)
-        secs = re.split(r"(%s)" % dels, txt)
+        txt = normalize_text_newlines(txt)
+        parsed_dels = parse_delimiter_field(delimiter)
+        dels = compile_delimiter_pattern(parsed_dels)
+        logging.debug(
+            "RAGFlowTxtParser.parser_txt: delimiter_count=%d, splitting=%s",
+            len(parsed_dels),
+            bool(dels),
+        )
+        secs = re.split(r"(%s)" % dels, txt) if dels else [txt]
         for sec in secs:
-            if re.match(f"^{dels}$", sec):
+            if dels and re.match(f"^{dels}$", sec):
                 continue
             if not sec:
                 continue