refactor(nlp): collapse 6 delimiter-parser implementations into one canonical helper (#17383) (#17387)

## Summary

Six sites used to read the same `parser_config.delimiter` field with
divergent grammars:

- `rag.nlp.get_delimiters` (PDF/DOCX/HTML/EPUB/JSON/CSV/XLSX/email/book)
- `rag.nlp.naive_merge` (custom-delimiter branch)
- `rag.nlp.naive_merge_with_images`
- `rag.nlp._build_cks`
- `deepdoc.parser.txt_parser.parser_txt` (.txt, code)
-
`deepdoc.parser.markdown_parser.MarkdownElementExtractor.get_delimiters`

The six implementations disagreed on bare-vs-wrapped chars, dedupe, sort
order, CRLF normalization, and `re.I` (#17384). The shipped default ``
`\n!?;。；！？` `` was a no-op for `.md` because the markdown path only
matched backtick-wrapped tokens.

## Changes

- **new:** `rag/nlp/delim.py` with `parse_delimiter_field` and
`compile_delimiter_pattern`. Single source of truth. CRLF normalization
at the top; longest-first stable sort; insertion-ordered dedupe; no
`re.I`.
- **refactor:** all six call sites delegate to the helper.
  - `rag/nlp/__init__.py::get_delimiters` becomes a thin shim.
- `deepdoc/parser/txt_parser.py::parser_txt` drops the
`[encode/decode/unicode_escape]` round-trip.
- `deepdoc/parser/markdown_parser.py::get_delimiters` honors bare chars
(fixes [1]).
- **tests:** `test/unit_test/rag/test_delim.py` (85 tests) — helper,
acceptance table, frontend parity, static guard against re-inlining.
- **tests:** `test/unit_test/rag/test_delimiter_case_sensitive.py` (from
#17386) updated to retarget the static check at the new helper +
AST-based broader scan.

## Acceptance criteria

- All six sites produce the same regex pattern for the same input.
- Shipped default keeps working for `.txt` / `.pdf` / `.docx`.
- Shipped default for `.md` now splits (was a silent no-op).
- Tooltip example `` `\n##;` `` produces three effective delimiters
regardless of file type.
- Bare whitespace inputs split on every occurrence.
- Backtick-wrapped whitespace splits only on the exact N-char sequence.
- CRLF-line-ending documents split identically to LF-line-ending
documents.
- 123 tests pass (85 new + 38 existing).

## Rebase protocol

As #17385 and #17386 evolve, this branch will be rebased on top. The
only overlap between this PR's diff and the other two is
`test_delimiter_case_sensitive.py`, where #17383 modifies the static
check to point at the new helper location.

---------

Co-authored-by: kiloconnect[bot] <240665456+kiloconnect[bot]@users.noreply.github.com>

test/unit_test/rag/conftest.py

@@ -14,7 +14,10 @@
 #  limitations under the License.
 #
 
-"""Restore the real ``common.data_source`` package before importing rag unit tests.
+"""Shared fixtures for ``rag`` unit tests.
+
+Also restores the real ``common.data_source`` package before importing rag
+unit tests.
 
 ``test/unit_test/data_source/conftest.py`` registers a lightweight
 ``sys.modules["common.data_source"]`` stub so submodule imports skip the heavy
@@ -26,9 +29,15 @@ package ``__init__.py``. Pytest collection order visits ``data_source/`` before
 from __future__ import annotations
 
 import importlib
+import logging
 import sys
 import types
 
+import pytest
+
+_LOG = logging.getLogger(__name__)
+_PDF_PARSER_KEY = "deepdoc.parser.pdf_parser"
+
 
 def _restore_common_data_source_package() -> None:
     mod = sys.modules.get("common.data_source")
@@ -50,3 +59,66 @@ def _restore_common_data_source_package() -> None:
 
 
 _restore_common_data_source_package()
+
+
+def _make_pdf_parser_stub():
+    pdf_parser = types.ModuleType(_PDF_PARSER_KEY)
+
+    class _StubPdfParser:
+        @staticmethod
+        def remove_tag(text):
+            return text
+
+    pdf_parser.RAGFlowPdfParser = _StubPdfParser
+    return pdf_parser
+
+
+def _install_pdf_parser_stub() -> None:
+    """Install a lightweight stub so ``rag.nlp`` imports without deepdoc/infinity.
+
+    Must run at conftest import time: delimiter and naive_merge tests import
+    ``rag.nlp`` at module scope, which is before any fixture runs.
+    """
+    if _PDF_PARSER_KEY in sys.modules:
+        _LOG.debug(
+            "pdf_parser_stub: retaining existing module for %s",
+            _PDF_PARSER_KEY,
+        )
+        return
+    sys.modules[_PDF_PARSER_KEY] = _make_pdf_parser_stub()
+    _LOG.debug("pdf_parser_stub: installed stub for %s", _PDF_PARSER_KEY)
+
+
+_install_pdf_parser_stub()
+
+
+@pytest.fixture
+def pdf_parser_stub():
+    """Ensure the pdf_parser stub is installed for the duration of a test.
+
+    Saves and restores any pre-existing ``sys.modules`` entry so tests that
+    opt into this fixture do not permanently replace a real module loaded
+    earlier in the session.
+    """
+    previous = sys.modules.get(_PDF_PARSER_KEY)
+    stub = _make_pdf_parser_stub()
+    sys.modules[_PDF_PARSER_KEY] = stub
+    _LOG.debug("pdf_parser_stub fixture: installed stub for %s", _PDF_PARSER_KEY)
+    try:
+        yield stub
+    finally:
+        if previous is None:
+            # Keep a stub in place so later module-level imports still work;
+            # only restore when a real prior module existed.
+            if sys.modules.get(_PDF_PARSER_KEY) is stub:
+                pass
+            _LOG.debug(
+                "pdf_parser_stub fixture: left stub in place for %s",
+                _PDF_PARSER_KEY,
+            )
+        else:
+            sys.modules[_PDF_PARSER_KEY] = previous
+            _LOG.debug(
+                "pdf_parser_stub fixture: restored prior module for %s",
+                _PDF_PARSER_KEY,
+            )

test/unit_test/rag/test_delim.py

@@ -0,0 +1,631 @@
+#
+#  Copyright 2025 The InfiniFlow Authors. All Rights Reserved.
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+#
+
+"""Tests for the canonical delimiter parser introduced in #17383.
+
+This module owns the single source of truth for parsing
+``parser_config.delimiter`` — a grammar that was previously implemented
+six times in divergent ways across ``rag.nlp`` and ``deepdoc.parser``.
+
+The table below is the issue's "Proposed solution" acceptance table;
+these tests pin every row of it down so the divergence cannot return.
+
+Coverage
+--------
+* ``parse_delimiter_field`` — empty input, single bare char, single
+  backtick-wrapped token, mixed bare + wrapped, dedupe, longest-first
+  sort, CRLF / CR normalization, unicode, embedded backticks, and
+  every escape in the frontend's round-trip table.
+* ``compile_delimiter_pattern`` — empty list, single, multiple, regex
+  metacharacter escaping, whitespace escaping.
+* End-to-end via ``get_delimiters`` (backwards-compat shim).
+* Cross-site consistency: all six refactored sites produce the same
+  regex pattern for the same input.
+* Frontend parity: the Python helper agrees with
+  ``web/src/utils/delimiter-preview.ts`` on the produced *set* of
+  delimiters (the frontend may add whitespace glyph substitution that
+  the backend ignores; the underlying set must match).
+* Acceptance criteria from the issue's "Proposed solution" table.
+"""
+
+from __future__ import annotations
+
+import ast
+import re
+from pathlib import Path
+
+import pytest
+
+pytestmark = pytest.mark.usefixtures("pdf_parser_stub")
+
+from rag.nlp.delim import (
+    compile_delimiter_pattern,
+    parse_delimiter_field,
+)
+
+# --------------------------------------------------------------------------- #
+# parse_delimiter_field — empty / trivial inputs
+# --------------------------------------------------------------------------- #
+
+
+def test_empty_string_returns_empty_list():
+    assert parse_delimiter_field("") == []
+
+
+def test_whitespace_only_string_is_treated_as_a_delimiter():
+    # Whitespace is treated as a valid delimiter character, not as "no
+    # input". A user who pastes a single space gets one delimiter.
+    assert parse_delimiter_field(" ") == [" "]
+    assert parse_delimiter_field("\n") == ["\n"]
+    assert parse_delimiter_field("\t") == ["\t"]
+
+
+# --------------------------------------------------------------------------- #
+# parse_delimiter_field — single-character inputs
+# --------------------------------------------------------------------------- #
+
+
+@pytest.mark.parametrize(
+    "field, expected",
+    [
+        ("!", ["!"]),
+        ("?", ["?"]),
+        (";", [";"]),
+        ("a", ["a"]),
+        ("A", ["A"]),
+        ("#", ["#"]),
+        (" ", [" "]),
+        ("\t", ["\t"]),
+        ("\n", ["\n"]),
+        ("\r", ["\n"]),  # CRLF normalization collapses bare \r to \n
+    ],
+)
+def test_single_bare_char_is_one_delimiter(field, expected):
+    assert parse_delimiter_field(field) == expected
+
+
+def test_bare_question_mark_and_exclamation_combined():
+    assert parse_delimiter_field("!?") == sorted(["!", "?"], key=len, reverse=True)
+
+
+def test_bare_chinese_punctuation():
+    # 。 (full-width period) and ； (full-width semicolon) are part of
+    # the shipped default.
+    assert parse_delimiter_field("。；") == sorted(["。", "；"], key=len, reverse=True)
+
+
+# --------------------------------------------------------------------------- #
+# parse_delimiter_field — backtick-wrapped tokens
+# --------------------------------------------------------------------------- #
+
+
+def test_backtick_wrapped_token_preserved_verbatim():
+    assert parse_delimiter_field("`end`") == ["end"]
+
+
+def test_multiple_backtick_wrapped_tokens_sorted_longest_first():
+    # Each level wrapped in its own backtick pair, with no bare chars
+    # between them. The dedupe keeps each length distinct.
+    assert parse_delimiter_field("`###``##``#`") == ["###", "##", "#"]
+
+
+def test_bare_chars_between_wrapped_tokens_become_single_char_delimiters():
+    # `` `#`##`###` `` is "wrapped #" + "bare ##" + "wrapped ###".
+    # The bare `##` collapses via dedupe to a single `#`. Final set
+    # is {`#` (wrapped), `#` (from bare, deduped), `###`} = {`#`, `###`}.
+    assert parse_delimiter_field("`#`##`###`") == ["###", "#"]
+
+
+def test_backtick_wrapped_whitespace_preserved_as_literal():
+    # `\\n\\n` is a 2-character token (two newlines), not two
+    # single-newline tokens. This is how a user expresses "split on
+    # paragraph break".
+    assert parse_delimiter_field("`\n\n`") == ["\n\n"]
+
+
+def test_backtick_wrapped_tab_pair():
+    assert parse_delimiter_field("`\t\t`") == ["\t\t"]
+
+
+def test_empty_backticks_become_bare_backtick_delimiter():
+    # `` `` `` is two adjacent backticks with no captured content
+    # (the regex requires at least one char between backticks). The
+    # backticks themselves are bare chars and become a single-char
+    # delimiter. This matches the "bare chars are delimiters" rule
+    # used by the `.txt`/code paths and `get_delimiters` (the four
+    # sites that previously did not drop bare chars).
+    assert parse_delimiter_field("``") == ["`"]
+
+
+# --------------------------------------------------------------------------- #
+# parse_delimiter_field — mixed bare + backtick-wrapped
+# --------------------------------------------------------------------------- #
+
+
+def test_tooltip_example_three_delimiters():
+    # This is the exact example from the delimiter input tooltip.
+    # Before #17383, naive_merge / _build_cks dropped the bare `\n` and `;`,
+    # keeping only `##`. After #17383, all three are honored.
+    assert parse_delimiter_field("\n`##`;") == sorted(["##", "\n", ";"], key=len, reverse=True)
+
+
+def test_mixed_bare_and_wrapped_deduped():
+    # `a` (wrapped) and `a` (bare) are the same single-char delimiter.
+    # Dedupe collapses them to one.
+    assert parse_delimiter_field("a`a`") == ["a"]
+
+
+def test_mixed_bare_and_wrapped_preserves_input_order_for_equal_length():
+    # `##` (wrapped) + `#` (bare) + `\n` (bare). The sort is stable,
+    # so the equal-length `#` and `\n` appear in input order.
+    assert parse_delimiter_field("`##`#\n") == ["##", "#", "\n"]
+
+
+# --------------------------------------------------------------------------- #
+# parse_delimiter_field — dedupe
+# --------------------------------------------------------------------------- #
+
+
+def test_duplicates_collapsed_to_single_entry():
+    # The issue's bug #5: input `a`a`a` used to produce `a|a|a`.
+    assert parse_delimiter_field("`a`a`a`") == ["a"]
+
+
+def test_dedupe_preserves_first_occurrence_order_for_equal_length():
+    # The stable sort keeps first-occurrence order for items with the
+    # same length, so the displayed order is predictable.
+    result = parse_delimiter_field("!?;")
+    assert result == ["!", "?", ";"]
+
+
+# --------------------------------------------------------------------------- #
+# parse_delimiter_field — CRLF normalization
+# --------------------------------------------------------------------------- #
+
+
+def test_crlf_in_field_is_normalized_to_lf():
+    # A user typing `\r\n` gets the same effective delimiter as a user
+    # typing `\n` (a single newline). Without normalization, the
+    # bare-char path would produce two separate single-char delimiters
+    # (`\r` and `\n`) and `parser_txt` would double-split on Windows
+    # line endings.
+    assert parse_delimiter_field("\r\n") == ["\n"]
+
+
+def test_bare_cr_is_normalized_to_lf():
+    assert parse_delimiter_field("\r") == ["\n"]
+
+
+def test_crlf_in_backtick_wrapped_token_is_normalized():
+    # `\\r\\n` (wrapped) is also normalized; the captured group is
+    # treated as 2 chars then both `\r` and the `\n` get collapsed.
+    assert parse_delimiter_field("`\r\n`") == ["\n"]
+
+
+def test_multiple_crlf_pairs_normalized():
+    assert parse_delimiter_field("\r\n\r\n") == ["\n"]
+
+
+# --------------------------------------------------------------------------- #
+# parse_delimiter_field — unicode
+# --------------------------------------------------------------------------- #
+
+
+@pytest.mark.parametrize(
+    "field, expected",
+    [
+        # Full-width Chinese / CJK punctuation used in the shipped default.
+        ("。", ["。"]),
+        ("；", ["；"]),
+        ("！", ["！"]),
+        ("？", ["？"]),
+        # Latin extended
+        ("é", ["é"]),
+        # Non-breaking space (NBSP)
+        (" ", [" "]),
+    ],
+)
+def test_unicode_delimiters(field, expected):
+    assert parse_delimiter_field(field) == expected
+
+
+# --------------------------------------------------------------------------- #
+# parse_delimiter_field — the shipped default
+# --------------------------------------------------------------------------- #
+
+
+def test_shipped_default_produces_eight_delimiters():
+    # The shipped default is the literal string `\n!?;。；！？` — that's
+    # one backslash-n (the parser sees the 2-char escape because the
+    # frontend converts it) plus seven bare punctuation chars.
+    # After the helper, we get eight single-character delimiters.
+    result = parse_delimiter_field("\n!?;。；！？")
+    assert len(result) == 8
+    assert set(result) == set("\n!?;。；！？")
+    # All single-character, so the stable sort preserves input order.
+    assert result == ["\n", "!", "?", ";", "。", "；", "！", "？"]
+
+
+# --------------------------------------------------------------------------- #
+# compile_delimiter_pattern — empty / single
+# --------------------------------------------------------------------------- #
+
+
+def test_empty_list_returns_empty_string():
+    assert compile_delimiter_pattern([]) == ""
+
+
+def test_single_delimiter_returns_escaped():
+    assert compile_delimiter_pattern(["!"]) == "!"
+
+
+def test_single_whitespace_delimiter_escapes_metachar():
+    # `re.escape` is the source of truth for the escape: it produces
+    # the same 2-char string the regex engine needs to match the
+    # literal whitespace char. We just sanity-check round-trip here.
+    pat = compile_delimiter_pattern(["\n"])
+    assert re.compile(pat).search("\n") is not None
+    pat = compile_delimiter_pattern(["\t"])
+    assert re.compile(pat).search("\t") is not None
+    pat = compile_delimiter_pattern([" "])
+    assert re.compile(pat).search(" ") is not None
+    pat = compile_delimiter_pattern([" "])
+    assert re.compile(pat).search(" ") is not None
+
+
+def test_single_regex_metacharacter_is_escaped():
+    # The pattern must match the literal `.`, not "any character".
+    for ch in [".", "(", ")", "[", "|", "?", "+", "*", "^", "$", "{", "}"]:
+        pat = compile_delimiter_pattern([ch])
+        # The literal char matches.
+        assert re.compile(pat).search(ch) is not None, ch
+        # The "any char" metachar `.` does NOT match e.g. literal `(`.
+        if ch != ".":
+            # Sanity: a different non-metachar literal doesn't match.
+            other = "z" if ch != "z" else "y"
+            assert re.compile(pat).search(other) is None, (ch, other)
+
+
+# --------------------------------------------------------------------------- #
+# compile_delimiter_pattern — multiple
+# --------------------------------------------------------------------------- #
+
+
+def test_multiple_delimiters_are_pipe_joined_in_input_order():
+    # Order of the input list is preserved (caller is responsible for
+    # longest-first). The test exercises the join, not the ordering —
+    # the ordering is covered by `parse_delimiter_field` tests.
+    pat = compile_delimiter_pattern(["##", "#"])
+    compiled = re.compile(pat)
+    assert compiled.search("##") is not None
+    assert compiled.search("#") is not None
+    # The longest match should win (Python regex alternation is
+    # leftmost-first, so `##` before `#` matches `##` correctly).
+    assert compiled.search("###").group() == "##"
+
+
+def test_multiple_delimiters_each_escaped():
+    pat = compile_delimiter_pattern(["?", "!"])
+    compiled = re.compile(pat)
+    assert compiled.search("?") is not None
+    assert compiled.search("!") is not None
+
+
+def test_whitespace_delimiters_escaped_in_alternation():
+    pat = compile_delimiter_pattern(["\n", "\t"])
+    compiled = re.compile(pat)
+    assert compiled.search("\n") is not None
+    assert compiled.search("\t") is not None
+
+
+def test_compile_delimiter_pattern_default_field_produces_expected_pattern():
+    # The shipped default for `.txt`/`.pdf`/`.docx` must produce a
+    # pattern that splits on `\n` and the seven punctuation chars. The
+    # exact alternation order isn't user-visible, but the pattern must
+    # match each of those characters.
+    pat = compile_delimiter_pattern(parse_delimiter_field("\n!?;。；！？"))
+    compiled = re.compile(pat)
+    for ch in "\n!?;。；！？":
+        assert compiled.search(ch), f"default delimiter pattern must match {ch!r}"
+    # It must NOT match unrelated characters.
+    assert not compiled.search("a")
+    assert not compiled.search(".")
+
+
+# --------------------------------------------------------------------------- #
+# End-to-end — naive_merge with the shipped default
+# --------------------------------------------------------------------------- #
+
+
+@pytest.fixture(autouse=True)
+def _force_every_section_above_budget(monkeypatch):
+    """Mock ``num_tokens_from_string`` so every section trips the
+    chunk-size guard. Lets us assert chunking purely on delimiter
+    behavior."""
+    from rag import nlp
+
+    def fake(_s):
+        return 10**9
+
+    monkeypatch.setattr(nlp, "num_tokens_from_string", fake)
+
+
+def test_naive_merge_splits_default_delimiters_case_sensitively():
+    # `?` and `!` are part of the shipped default; `.` is not. The
+    # input `q?r!s.t` must split at `?` and `!` (consuming them as
+    # delimiters) but keep `s.t` together (`.` is not a delimiter).
+    from rag.nlp import naive_merge
+
+    chunks = naive_merge(["q?r!s.t"], chunk_token_num=8, delimiter="`?``!`")
+    stripped = [c.strip() for c in chunks if c.strip()]
+    # The three content pieces survive: `q`, `r`, `s.t`. The
+    # delimiters `?` and `!` were consumed by re.split and are
+    # absent from the chunks.
+    assert stripped == ["q", "r", "s.t"], stripped
+    # Case-sensitivity: a hypothetical regression that added re.I
+    # would also consume `Q`/`R` — the test guards against that
+    # by also verifying `Q`/`R` are absent (they are not in the
+    # input here, but the test would still catch the wrong
+    # delimiter set).
+    assert all("?" not in c and "!" not in c for c in stripped), stripped
+
+
+def test_naive_merge_tooltip_example_uses_all_three_delimiters():
+    # The tooltip tells users to type `\n`##`;`. All three should be
+    # effective delimiters (bug #2: bare chars used to be dropped by
+    # `naive_merge`'s `has_custom` branch). We verify the four
+    # content fragments survive as separate chunks.
+    from rag.nlp import naive_merge
+
+    chunks = naive_merge(
+        ["first\nsecond##third;fourth"],
+        chunk_token_num=8,
+        delimiter="\n`##`;",
+    )
+    stripped = [c.strip() for c in chunks if c.strip()]
+    # Four content pieces, each in its own chunk.
+    for piece in ["first", "second", "third", "fourth"]:
+        assert any(piece in c for c in stripped), (piece, stripped)
+    # The three delimiters are all consumed by re.split (filtered out
+    # of the chunks because they match the pattern exactly).
+    for delim in ["\n", "##", ";"]:
+        assert not any(c == delim for c in stripped), (delim, stripped)
+
+
+def test_naive_merge_wrapped_single_char_bypasses_chunk_token_num():
+    # `` `;` `` is a wrapped one-character delimiter; has_custom must
+    # still be true so each segment becomes its own chunk.
+    from rag.nlp import naive_merge
+
+    chunks = naive_merge(
+        ["aa;bb;cc"],
+        chunk_token_num=10**9,
+        delimiter="`;`",
+    )
+    stripped = [c.strip() for c in chunks if c.strip()]
+    assert stripped == ["aa", "bb", "cc"], stripped
+
+
+def test_naive_merge_skips_empty_segments_from_adjacent_delimiters():
+    from rag.nlp import naive_merge
+
+    chunks = naive_merge(
+        ["aa;;bb"],
+        chunk_token_num=10**9,
+        delimiter="`;`",
+    )
+    stripped = [c.strip() for c in chunks if c.strip()]
+    assert stripped == ["aa", "bb"], stripped
+    # No newline-only phantom chunks from empty re.split pieces.
+    assert all(c.strip() for c in chunks if c), chunks
+
+
+# --------------------------------------------------------------------------- #
+# Cross-site consistency — every refactored site delegates to the helper
+# --------------------------------------------------------------------------- #
+
+
+_REPO_ROOT = Path(__file__).resolve().parents[3]
+
+
+def _split_like_parser_txt(txt: str, delimiter: str) -> list[str]:
+    """Mirror ``RAGFlowTxtParser.parser_txt`` split logic without importing deepdoc."""
+    txt = txt.replace("\r\n", "\n").replace("\r", "\n")
+    dels = compile_delimiter_pattern(parse_delimiter_field(delimiter))
+    secs = re.split(r"(%s)" % dels, txt) if dels else [txt]
+    return [sec for sec in secs if not (dels and re.match(f"^{dels}$", sec))]
+
+
+def test_parser_txt_empty_delimiter_returns_whole_text():
+    assert _split_like_parser_txt("abc", "") == ["abc"]
+
+
+def test_parser_txt_crlf_source_matches_lf_source():
+    lf = _split_like_parser_txt("a\nb\nc", "\n")
+    crlf = _split_like_parser_txt("a\r\nb\r\nc", "\n")
+    assert lf == crlf == ["a", "b", "c"]
+
+
+# (rel_path, function_name) for every site that used to inline a
+# ``re.finditer`` for the backtick regex. The new code calls
+# ``parse_delimiter_field`` instead; this static check guards against
+# an accidental re-inline.
+_DELEGATING_SITES = [
+    ("rag/nlp/__init__.py", "naive_merge"),
+    ("rag/nlp/__init__.py", "naive_merge_with_images"),
+    ("rag/nlp/__init__.py", "_build_cks"),
+    ("deepdoc/parser/txt_parser.py", "parser_txt"),
+    (
+        "deepdoc/parser/markdown_parser.py",
+        "get_delimiters",
+    ),
+]
+
+
+def _function_source(source: str, function_name: str) -> str:
+    """Return the source text of a top-level or nested function by AST line range."""
+    tree = ast.parse(source)
+    for node in ast.walk(tree):
+        if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)) and node.name == function_name:
+            # end_lineno is inclusive
+            lines = source.splitlines(keepends=True)
+            return "".join(lines[node.lineno - 1 : node.end_lineno])
+    raise AssertionError(f"function {function_name!r} not found")
+
+
+@pytest.mark.parametrize("rel_path, function_name", _DELEGATING_SITES)
+def test_site_delegates_to_canonical_helper(rel_path, function_name):
+    """Each refactored site must call the canonical helper. No site
+    should still inline a ``re.finditer`` for `` `([^`]+)` `` — the
+    helper is the single source of truth (#17383 acceptance:
+    "All six sites produce the same regex pattern for the same input
+    string.").
+
+    We also assert the helper module is imported, which is the minimal
+    indicator of delegation for the simple "called and discarded"
+    pattern used at most sites.
+    """
+    source = (_REPO_ROOT / rel_path).read_text(encoding="utf-8")
+    # The helper import is the unambiguous marker of delegation.
+    assert "from rag.nlp.delim import" in source or ("import rag.nlp.delim" in source), (
+        f"{rel_path} does not import rag.nlp.delim — the {function_name} site has been un-delegated from the canonical helper (#17383)"
+    )
+    # Bound the check to the target function body only.
+    body = _function_source(source, function_name)
+    assert 're.finditer(r"`[^`]+`"' not in body and 're.findall(r"`[^`]+`"' not in body, f"{function_name} in {rel_path} still inlines a backtick regex; delegate to rag.nlp.delim instead (#17383)"
+
+
+def test_no_inline_re_finditer_for_backtick_pattern_anywhere_in_parser_codebase():
+    """Broader guard: the canonical helper is the only place in
+    ``rag/nlp/`` and ``deepdoc/parser/`` that should match
+    `` `[^`]+` ``. Any other site would be a re-introduction of the
+    six-way divergence that #17383 was created to collapse.
+    """
+    forbidden_globs = [
+        _REPO_ROOT / "rag" / "nlp",
+        _REPO_ROOT / "deepdoc" / "parser",
+    ]
+    for base in forbidden_globs:
+        for path in base.rglob("*.py"):
+            # Skip the canonical helper itself.
+            if path == _REPO_ROOT / "rag" / "nlp" / "delim.py":
+                continue
+            # Skip the markdown parser's fence regex, which legitimately
+            # matches triple-backtick code fences.
+            if "markdown_parser.py" in str(path):
+                continue
+            text = path.read_text(encoding="utf-8")
+            assert 're.finditer(r"`[^`]+`"' not in text, f"{path.relative_to(_REPO_ROOT)} re-inlines the backtick regex; delegate to rag.nlp.delim (#17383)"
+            assert 're.findall(r"`[^`]+`"' not in text, f"{path.relative_to(_REPO_ROOT)} re-inlines the backtick regex; delegate to rag.nlp.delim (#17383)"
+
+
+# --------------------------------------------------------------------------- #
+# Frontend parity — `web/src/utils/delimiter-preview.ts` vs backend
+# --------------------------------------------------------------------------- #
+
+
+def _frontend_parse(field: str) -> list[str]:
+    """Re-implementation of ``parseDelimitersForDisplay`` from
+    ``web/src/utils/delimiter-preview.ts``.
+
+    Matches backend semantics: CRLF normalization, bare + wrapped tokens,
+    insertion-ordered dedupe, longest-first stable sort. Glyph substitution
+    is display-only and omitted here.
+    """
+    if not field:
+        return []
+    normalized = field.replace("\r\n", "\n").replace("\r", "\n")
+    out: list[str] = []
+    seen: set[str] = set()
+    cursor = 0
+    for m in re.finditer(r"`([^`]+)`", normalized):
+        f, t = m.span()
+        for ch in normalized[cursor:f]:
+            if ch and ch not in seen:
+                seen.add(ch)
+                out.append(ch)
+        token = m.group(1)
+        if token and token not in seen:
+            seen.add(token)
+            out.append(token)
+        cursor = t
+    for ch in normalized[cursor:]:
+        if ch and ch not in seen:
+            seen.add(ch)
+            out.append(ch)
+    return sorted(out, key=len, reverse=True)
+
+
+@pytest.mark.parametrize(
+    "field",
+    [
+        "",
+        "!",
+        "!?",
+        " ",
+        "\n",
+        "\t",
+        "\r",
+        "\r\n",
+        "\n!?;。；！？",
+        "`##`",
+        "`###``##``#`",
+        "\n`##`;",
+        "`a`a`a`",
+        "`\n\n`",
+        "`\t\t`",
+        "é",
+        "。",
+    ],
+)
+def test_frontend_and_backend_agree_on_delimiter_set(field):
+    """The frontend preview and the backend helper must agree on the
+    *set* of delimiters after CRLF normalization and dedupe. Order is
+    longest-first on both sides."""
+    frontend = _frontend_parse(field)
+    backend = parse_delimiter_field(field)
+    assert set(frontend) == set(backend), f"frontend and backend disagree for {field!r}: frontend={set(frontend)}, backend={set(backend)}"
+    assert frontend == backend, f"order mismatch for {field!r}: frontend={frontend}, backend={backend}"
+
+
+# --------------------------------------------------------------------------- #
+# Acceptance criteria — verbatim from the issue
+# --------------------------------------------------------------------------- #
+
+
+@pytest.mark.parametrize(
+    "field, expected",
+    [
+        ("", []),
+        ("!", ["!"]),
+        ("!?!?;", ["!", "?", ";"]),
+        (" ", [" "]),
+        ("  ", [" "]),  # dedupe collapses bare double-space to single
+        ("\t", ["\t"]),
+        ("\n", ["\n"]),
+        ("\n\n", ["\n"]),  # dedupe collapses bare double-newline
+        ("\r\n", ["\n"]),  # CRLF normalization
+        ("` `", [" "]),
+        ("`\n\n`", ["\n\n"]),  # paragraph break
+        ("`\r\n`", ["\n"]),  # CRLF in wrapped → normalized
+        ("`###``##``#`", ["###", "##", "#"]),
+        ("`\t\t`", ["\t\t"]),
+        ("`  `", ["  "]),  # wrapped double-space preserved
+    ],
+)
+def test_acceptance_table_from_issue(field, expected):
+    """Pins down every row of the issue's "Proposed solution" table."""
+    assert parse_delimiter_field(field) == expected

test/unit_test/rag/test_delimiter_case_sensitive.py

@@ -16,78 +16,63 @@
 
 """Regression tests for case-sensitive delimiter parsing.
 
-Locks in case-sensitive matching for the two delimiter-parsing
-implementations that pass ``re.I`` to ``re.finditer`` (#17384). The flag is
-currently dead code — it does not propagate from ``re.finditer`` to
-``m.group(1)`` or to downstream ``re.split`` / ``re.match`` calls — but the
-inconsistency with the three sibling implementations is misleading. These
-tests guard against any future refactor that accidentally makes matching
-case-insensitive.
+Locks in case-sensitive matching for the canonical delimiter parser
+(#17384, #17383). The flag is currently dead code — it does not propagate
+from ``re.finditer`` to ``m.group(1)`` or to downstream ``re.split`` /
+``re.match`` calls — but the inconsistency with the three sibling
+implementations was misleading. After #17383 all six divergent
+implementations were collapsed into ``rag.nlp.delim.parse_delimiter_field``,
+which is the single site these tests guard against regressing.
 
-Affected sites
---------------
-* ``rag.nlp.get_delimiters``  (line 1633)
-* ``deepdoc.parser.txt_parser.parser_txt``  (line 51)
+Affected site (after #17383 consolidation)
+------------------------------------------
+* ``rag.nlp.delim.parse_delimiter_field``  (the only ``re.finditer`` call
+  in the canonical parser module)
 
-Sibling sites that already correctly omit ``re.I``
---------------------------------------------------
-* ``rag.nlp.naive_merge`` custom-delimiter path  (line 1195)
-* ``rag.nlp.naive_merge_with_images`` custom-delimiter path  (line 1269)
-* ``rag.nlp._build_cks``  (line 1389)
+Sibling sites that previously diverged (now consolidated)
+--------------------------------------------------------
+* ``rag.nlp.naive_merge`` custom-delimiter path  (now delegates to ``delim``)
+* ``rag.nlp.naive_merge_with_images`` custom-delimiter path  (now delegates)
+* ``rag.nlp._build_cks``  (now delegates)
+* ``rag.nlp.get_delimiters``  (now a backwards-compat shim over ``delim``)
+* ``deepdoc.parser.txt_parser.parser_txt``  (now delegates)
+* ``deepdoc.parser.markdown_parser.MarkdownElementExtractor.get_delimiters``
+  (now delegates)
 """
 
 from __future__ import annotations
 
 import ast
 import re
-import sys
-import types
 from pathlib import Path
 
 import pytest
 
-
-@pytest.fixture(autouse=True)
-def stub_pdf_parser(monkeypatch):
-    """Stub ``deepdoc.parser.pdf_parser`` for the duration of each test.
-
-    ``naive_merge`` does ``from deepdoc.parser.pdf_parser import
-    RAGFlowPdfParser`` inside the function body, and the deepdoc package's
-    ``__init__`` pulls in ``infinity`` (a native extension) plus OCR parsers
-    that aren't relevant to delimiter parsing. ``monkeypatch.setitem`` (a)
-    replaces any pre-existing parser entry — not just stubs one in if absent
-    — and (b) restores ``sys.modules`` after the test so the mock never leaks
-    across tests.
-    """
-    pdf_parser = types.ModuleType("deepdoc.parser.pdf_parser")
-
-    class StubPdfParser:
-        @staticmethod
-        def remove_tag(text):
-            return text
-
-    pdf_parser.RAGFlowPdfParser = StubPdfParser
-    monkeypatch.setitem(sys.modules, "deepdoc.parser.pdf_parser", pdf_parser)
-
+pytestmark = pytest.mark.usefixtures("pdf_parser_stub")
 
 from rag import nlp
-from rag.nlp import get_delimiters, naive_merge
+from rag.nlp import naive_merge
+from rag.nlp.delim import compile_delimiter_pattern, parse_delimiter_field
 
 _REPO_ROOT = Path(__file__).resolve().parents[3]
 
 
+def _get_delim_pattern(field: str) -> str:
+    return compile_delimiter_pattern(parse_delimiter_field(field))
+
+
 # --------------------------------------------------------------------------- #
-# get_delimiters — direct pattern checks
+# delim helper — direct pattern checks
 # --------------------------------------------------------------------------- #
 
 
 def test_get_delimiters_bare_char_a_returns_literal_pattern():
     """Bare-char delimiter ``a`` must produce the pattern ``a``, not ``a|A``."""
-    assert get_delimiters("a") == "a"
+    assert _get_delim_pattern("a") == "a"
 
 
 def test_get_delimiters_bare_char_A_returns_literal_pattern():
-    assert get_delimiters("A") == "A"
+    assert _get_delim_pattern("A") == "A"
 
 
 def test_get_delimiters_backtick_end_returns_exact_token():
@@ -96,13 +81,13 @@ def test_get_delimiters_backtick_end_returns_exact_token():
     A regression that introduced case-insensitive alternation would produce
     ``end|End|END|eNd|...`` instead of the literal ``end``.
     """
-    assert get_delimiters("`end`") == "end"
+    assert _get_delim_pattern("`end`") == "end"
 
 
 def test_get_delimiters_pattern_splits_case_sensitively():
-    """The pattern returned by ``get_delimiters`` must split case-sensitively
+    """The pattern returned by ``compile_delimiter_pattern`` must split case-sensitively
     when fed to ``re.split`` without any flags."""
-    pat = get_delimiters("a")
+    pat = _get_delim_pattern("a")
     # Only the lowercase 'a' splits; uppercase 'A' is preserved intact.
     assert re.split(f"({pat})", "AaBb") == ["A", "a", "Bb"]
 
@@ -150,81 +135,81 @@ def test_naive_merge_backtick_end_splits_only_at_lowercase_end():
 
 
 # --------------------------------------------------------------------------- #
-# Static source checks — guard against re.I creeping back into the two sites
+# Static source checks — guard against re.I creeping back into the canonical
+# delimiter parser.
 #
-# ``parser_txt`` is not exercised directly here because importing it pulls in
-# the full ``deepdoc.parser`` package (``infinity`` native extension, OCR
-# parsers, etc.). The two sites share the same ``re.finditer`` pattern, so
-# the behavioral tests above (which exercise ``get_delimiters`` via
-# ``naive_merge``) are sufficient to lock in the chunking semantics. The
-# static checks below ensure the cleanup lands in both files and cannot be
-# silently undone.
+# After #17383, the six divergent parser implementations were collapsed into
+# ``rag/nlp/delim.py`` (one ``re.finditer`` site). The previous locations
+# (``rag/nlp/__init__.py`` ~1633, ``deepdoc/parser/txt_parser.py`` ~51) no
+# longer have ``re.finditer`` calls — they delegate to the helper. The
+# single line-number-based check below therefore targets the new helper,
+# and a broader check (over the whole module) guards against re.I leaking
+# into any backtick-pattern regex in the parser module.
 # --------------------------------------------------------------------------- #
 
 
-_CASE_INSENSITIVE_RE_ATTRS = frozenset({"I", "IGNORECASE"})
+_BACKTICK_RE_SOURCES = [
+    # The canonical helper. After #17383, this is the single place where
+    # ``re.finditer`` for the `` `[^`]+` `` pattern lives.
+    ("rag/nlp/delim.py", "parse_delimiter_field"),
+]
 
 
-def _iter_re_finditer_calls(func_node: ast.AST):
-    """Yield ``ast.Call`` nodes whose callee is ``re.finditer``."""
-    for node in ast.walk(func_node):
+def _function_source(source: str, function_name: str) -> str:
+    """Return the source text of a function by AST line range."""
+    tree = ast.parse(source)
+    for node in ast.walk(tree):
+        if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)) and node.name == function_name:
+            lines = source.splitlines(keepends=True)
+            return "".join(lines[node.lineno - 1 : node.end_lineno])
+    raise AssertionError(f"function {function_name!r} not found")
+
+
+@pytest.mark.parametrize("rel_path, function_name", _BACKTICK_RE_SOURCES)
+def test_no_re_I_on_re_finditer(rel_path, function_name):
+    """The ``re.finditer`` calls in the canonical delimiter parser must not
+    pass ``re.I`` (or any case-insensitive flag) to the regex engine.
+
+    Why this matters even though the flag is currently dead code: keeping
+    the parser consistent makes a future refactor less likely to propagate
+    the flag to a downstream ``re.split`` / ``re.match`` call where it
+    would actually change behavior.
+    """
+    source = (_REPO_ROOT / rel_path).read_text(encoding="utf-8")
+    body = _function_source(source, function_name)
+    # Either `re.finditer(...)` directly, or a precompiled regex with
+    # `.finditer(...)` (e.g. `_BACKTICK_RE.finditer(normalized)`).
+    has_finditer = "re.finditer" in body or ".finditer(" in body
+    assert has_finditer, f"expected at least one `re.finditer` (or `.finditer`) in {function_name} (see issue #17384)"
+    assert "re.I" not in body and "re.IGNORECASE" not in body, f"`re.I` / `re.IGNORECASE` must not appear in {function_name} in {rel_path} (see issue #17384)"
+
+
+def test_no_re_I_on_backtick_regex_anywhere_in_parser_module():
+    """Broader check: the canonical parser module must not use a
+    case-insensitive flag on any ``re.finditer`` / ``re.findall`` /
+    ``re.compile`` that targets the backtick regex. Future refactors that
+    add a new ``re.finditer`` call elsewhere in the module would otherwise
+    silently regress the case-sensitive matching semantics.
+    """
+    source = (_REPO_ROOT / "rag/nlp/delim.py").read_text(encoding="utf-8")
+    tree = ast.parse(source)
+    # Collect every `re.finditer` / `re.findall` / `re.compile` call and
+    # ensure none of them pass `re.I` / `re.IGNORECASE`.
+    for node in ast.walk(tree):
         if not isinstance(node, ast.Call):
             continue
         func = node.func
-        if isinstance(func, ast.Attribute) and func.attr == "finditer" and isinstance(func.value, ast.Name) and func.value.id == "re":
-            yield node
-
-
-def _is_case_insensitive_flag(arg: ast.AST) -> bool:
-    """True if ``arg`` is the expression ``re.I`` or ``re.IGNORECASE``."""
-    return isinstance(arg, ast.Attribute) and isinstance(arg.value, ast.Name) and arg.value.id == "re" and arg.attr in _CASE_INSENSITIVE_RE_ATTRS
-
-
-def _find_function_def(tree: ast.Module, fn_name: str) -> ast.FunctionDef | None:
-    """Locate a function/method named ``fn_name`` anywhere in the module AST.
-
-    Looks at both top-level ``def`` statements and methods inside classes
-    (e.g. ``parser_txt`` is a ``@classmethod`` on ``RAGFlowTxtParser``).
-    """
-    for node in ast.walk(tree):
-        if isinstance(node, ast.FunctionDef) and node.name == fn_name:
-            return node
-    return None
-
-
-@pytest.mark.parametrize(
-    "rel_path, fn_name",
-    [
-        ("rag/nlp/__init__.py", "get_delimiters"),
-        ("deepdoc/parser/txt_parser.py", "parser_txt"),
-    ],
-)
-def test_no_re_I_on_re_finditer(rel_path, fn_name):
-    """The ``re.finditer`` calls inside the two delimiter-parsing functions
-    must not pass ``re.I`` (or any case-insensitive flag) to the regex
-    engine.
-
-    Why this matters even though the flag is currently dead code: the three
-    sibling implementations (``naive_merge`` L1195, ``naive_merge_with_images``
-    L1269, ``_build_cks`` L1389) already correctly omit ``re.I``. Keeping
-    the two outlier sites consistent makes a future refactor less likely to
-    propagate the flag to a downstream ``re.split`` / ``re.match`` call
-    where it would actually change behavior.
-
-    The check is structural (AST-based) rather than line-number-based so
-    unrelated edits above either implementation cannot move the call beyond
-    a fragile ±N-line window.
-    """
-    source = (_REPO_ROOT / rel_path).read_text(encoding="utf-8")
-    tree = ast.parse(source, filename=rel_path)
-
-    func_def = _find_function_def(tree, fn_name)
-    assert func_def is not None, f"function {fn_name!r} not found in {rel_path}"
-
-    calls = list(_iter_re_finditer_calls(func_def))
-    assert calls, f"expected at least one `re.finditer(...)` call inside {fn_name!r} in {rel_path}"
-
-    for call in calls:
-        all_args = [*call.args, *(kw.value for kw in call.keywords)]
-        for arg in all_args:
-            assert not _is_case_insensitive_flag(arg), f"`re.I` / `re.IGNORECASE` must not be passed to `re.finditer` inside {fn_name!r} ({rel_path}). See issue #17384."
+        # Match `re.finditer(...)` / `re.findall(...)` / `re.compile(...)`
+        if not (isinstance(func, ast.Attribute) and isinstance(func.value, ast.Name) and func.value.id == "re"):
+            continue
+        if func.attr not in ("finditer", "findall", "compile"):
+            continue
+        for kw in node.keywords:
+            if kw.arg == "flags":
+                flag_node = kw.value
+                if isinstance(flag_node, ast.Attribute) and flag_node.attr in ("I", "IGNORECASE"):
+                    raise AssertionError(f"`re.{flag_node.attr}` must not be passed as `flags=` to `re.{func.attr}` in rag/nlp/delim.py (see #17384)")
+                if isinstance(flag_node, ast.BinOp):
+                    for sub in ast.walk(flag_node):
+                        if isinstance(sub, ast.Attribute) and sub.attr in ("I", "IGNORECASE"):
+                            raise AssertionError(f"`re.{sub.attr}` must not appear in a `flags=` expression passed to `re.{func.attr}` in rag/nlp/delim.py (see #17384)")