refactor(nlp): collapse 6 delimiter-parser implementations into one canonical helper (#17383) (#17387)

## Summary

Six sites used to read the same `parser_config.delimiter` field with
divergent grammars:

- `rag.nlp.get_delimiters` (PDF/DOCX/HTML/EPUB/JSON/CSV/XLSX/email/book)
- `rag.nlp.naive_merge` (custom-delimiter branch)
- `rag.nlp.naive_merge_with_images`
- `rag.nlp._build_cks`
- `deepdoc.parser.txt_parser.parser_txt` (.txt, code)
-
`deepdoc.parser.markdown_parser.MarkdownElementExtractor.get_delimiters`

The six implementations disagreed on bare-vs-wrapped chars, dedupe, sort
order, CRLF normalization, and `re.I` (#17384). The shipped default ``
`\n!?;。；！？` `` was a no-op for `.md` because the markdown path only
matched backtick-wrapped tokens.

## Changes

- **new:** `rag/nlp/delim.py` with `parse_delimiter_field` and
`compile_delimiter_pattern`. Single source of truth. CRLF normalization
at the top; longest-first stable sort; insertion-ordered dedupe; no
`re.I`.
- **refactor:** all six call sites delegate to the helper.
  - `rag/nlp/__init__.py::get_delimiters` becomes a thin shim.
- `deepdoc/parser/txt_parser.py::parser_txt` drops the
`[encode/decode/unicode_escape]` round-trip.
- `deepdoc/parser/markdown_parser.py::get_delimiters` honors bare chars
(fixes [1]).
- **tests:** `test/unit_test/rag/test_delim.py` (85 tests) — helper,
acceptance table, frontend parity, static guard against re-inlining.
- **tests:** `test/unit_test/rag/test_delimiter_case_sensitive.py` (from
#17386) updated to retarget the static check at the new helper +
AST-based broader scan.

## Acceptance criteria

- All six sites produce the same regex pattern for the same input.
- Shipped default keeps working for `.txt` / `.pdf` / `.docx`.
- Shipped default for `.md` now splits (was a silent no-op).
- Tooltip example `` `\n##;` `` produces three effective delimiters
regardless of file type.
- Bare whitespace inputs split on every occurrence.
- Backtick-wrapped whitespace splits only on the exact N-char sequence.
- CRLF-line-ending documents split identically to LF-line-ending
documents.
- 123 tests pass (85 new + 38 existing).

## Rebase protocol

As #17385 and #17386 evolve, this branch will be rebased on top. The
only overlap between this PR's diff and the other two is
`test_delimiter_case_sensitive.py`, where #17383 modifies the static
check to point at the new helper location.

---------

Co-authored-by: kiloconnect[bot] <240665456+kiloconnect[bot]@users.noreply.github.com>

web/src/utils/__tests__/delimiter-preview.test.ts

@@ -0,0 +1,50 @@
+import { parseDelimitersForDisplay } from '../delimiter-preview';
+
+describe('parseDelimitersForDisplay', () => {
+  it('returns empty for undefined or empty input', () => {
+    expect(parseDelimitersForDisplay(undefined)).toEqual([]);
+    expect(parseDelimitersForDisplay('')).toEqual([]);
+  });
+
+  it.each([
+    ['!', ['!']],
+    ['!?', ['!', '?']],
+    [' ', [' ']],
+    ['\n', ['\n']],
+    ['\t', ['\t']],
+    ['\r', ['\n']],
+    ['\r\n', ['\n']],
+    ['\n!?;。；！？', ['\n', '!', '?', ';', '。', '；', '！', '？']],
+    ['`##`', ['##']],
+    ['`###``##``#`', ['###', '##', '#']],
+    ['\n`##`;', ['##', '\n', ';']],
+    ['`a`a`a`', ['a']],
+    ['`\n\n`', ['\n\n']],
+    ['`\t\t`', ['\t\t']],
+    ['é', ['é']],
+    ['。', ['。']],
+  ])('parses %j to match backend set/order', (field, expected) => {
+    const got = parseDelimitersForDisplay(field).map((d) => d.raw);
+    expect(got).toEqual(expected);
+  });
+
+  it('sorts longest-first', () => {
+    const got = parseDelimitersForDisplay('`#``##``###`').map((d) => d.raw);
+    expect(got).toEqual(['###', '##', '#']);
+  });
+
+  it('normalizes CRLF before parsing', () => {
+    expect(parseDelimitersForDisplay('\r\n').map((d) => d.raw)).toEqual([
+      '\n',
+    ]);
+    expect(parseDelimitersForDisplay('`\r\n`').map((d) => d.raw)).toEqual([
+      '\n',
+    ]);
+  });
+
+  it('applies whitespace glyphs only for display', () => {
+    const [item] = parseDelimitersForDisplay('\n');
+    expect(item.raw).toBe('\n');
+    expect(item.display).toBe('↵');
+  });
+});

web/src/utils/delimiter-preview.ts

@@ -1,15 +1,9 @@
 /**
  * Parses a "Delimiter for text" field value into a list of delimiters for
- * display in the UI. The result is meant to be informative only — the
- * actual chunking behavior is determined by the backend.
- *
- * The parsing rule mirrors `rag/nlp/__init__.py::get_delimiters` in
- * spirit (backticks group characters into a multi-character delimiter;
- * anything outside backticks is itself a delimiter) but does **not**
- * exactly match any of the six divergent backend implementations, since
- * those implementations disagree on dedupe, sort order, the use of
- * `re.I`, and CRLF normalization. The helper is a *preview*, not a
- * contract. See #17383 for the backend consolidation.
+ * display in the UI. Mirrors the canonical backend parser in
+ * `rag/nlp/delim.py` (`parse_delimiter_field`): CRLF normalization, bare
+ * and backtick-wrapped tokens, insertion-ordered dedupe, and longest-first
+ * stable sort. Whitespace glyph substitution is display-only.
  */
 export interface ParsedDelimiter {
   /** The original characters the user entered. */
@@ -49,16 +43,9 @@ function toDisplay(raw: string): string {
 }
 
 /**
- * Parse the delimiter field into a deduplicated, display-ready list.
- *
- * Differences from the backend `get_delimiters`:
- *  - Deduplicates by raw value so distinct delimiters that happen to
- *    share a display glyph (e.g. a literal `\n` and a user-typed `↵`)
- *    are not silently merged. The backend does not always dedupe (see
- *    #17383).
- *  - Preserves left-to-right input order rather than sorting
- *    longest-first; visual order is more intuitive for users.
- *  - Does not regex-escape the values (irrelevant for display).
+ * Parse the delimiter field into a deduplicated, longest-first list that
+ * matches backend `parse_delimiter_field` semantics. Display glyphs are
+ * applied after parsing.
  *
  * Returns an empty array if the value is undefined or empty.
  */
@@ -67,28 +54,28 @@ export function parseDelimitersForDisplay(
 ): ParsedDelimiter[] {
   if (!value) return [];
 
+  // Match backend: \r\n → \n, then standalone \r → \n.
+  const normalizedValue = value.replaceAll('\r\n', '\n').replaceAll('\r', '\n');
+
   const result: ParsedDelimiter[] = [];
   const seen = new Set<string>();
 
   const push = (raw: string) => {
-    if (seen.has(raw)) return;
+    if (!raw || seen.has(raw)) return;
     seen.add(raw);
     result.push({ raw, display: toDisplay(raw) });
   };
 
   let cursor = 0;
-  for (const match of value.matchAll(/`([^`]+)`/g)) {
+  for (const match of normalizedValue.matchAll(/`([^`]+)`/g)) {
     const start = match.index!;
     const end = start + match[0].length;
-    // bare characters before this backtick-wrapped token
-    for (const ch of value.slice(cursor, start)) push(ch);
-    // the backtick-wrapped token
+    for (const ch of normalizedValue.slice(cursor, start)) push(ch);
     push(match[1]);
     cursor = end;
   }
-  // bare characters after the last token (or the whole string if no
-  // backticks were present)
-  for (const ch of value.slice(cursor)) push(ch);
+  for (const ch of normalizedValue.slice(cursor)) push(ch);
 
-  return result;
+  // Stable sort longest-first (matches backend parse_delimiter_field).
+  return result.sort((a, b) => b.raw.length - a.raw.length);
 }