From c48eb70e67f4edcfa85f34ebce3486f2fd62bc63 Mon Sep 17 00:00:00 2001 From: S <5842097+skbs-eng@users.noreply.github.com> Date: Mon, 27 Jul 2026 10:51:54 +0530 Subject: [PATCH] Improve "Delimiter for text" UX: clearer tooltip + live parsed-delimiter preview (#17385) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Summary Improve the UX of the **"Delimiter for text"** field on the dataset configuration page. The field is a single string with a backtick-based mini-syntax, but both the tooltip and the surrounding UI failed to surface what delimiters the backend would actually derive from a given value — leaving users to discover by trial-and-error that the same string produces different splits depending on file type (see #7436, #4704, #9680). --- .../components/children-delimiter-form.tsx | 2 + web/src/components/delimiter-form-field.tsx | 2 + web/src/components/delimiter-preview.tsx | 66 +++++++++++++ web/src/locales/en.ts | 8 +- web/src/utils/delimiter-preview.ts | 94 +++++++++++++++++++ 5 files changed, 170 insertions(+), 2 deletions(-) create mode 100644 web/src/components/delimiter-preview.tsx create mode 100644 web/src/utils/delimiter-preview.ts diff --git a/web/src/components/children-delimiter-form.tsx b/web/src/components/children-delimiter-form.tsx index 678be44e11..ba7ebbb279 100644 --- a/web/src/components/children-delimiter-form.tsx +++ b/web/src/components/children-delimiter-form.tsx @@ -2,6 +2,7 @@ import { cn } from '@/lib/utils'; import { forwardRef } from 'react'; import { useFormContext } from 'react-hook-form'; import { useTranslation } from 'react-i18next'; +import { DelimiterPreview } from './delimiter-preview'; import { FormControl, FormField, @@ -108,6 +109,7 @@ export function ChildrenDelimiterForm() { data-testid="ds-settings-parser-child-chunk-delimiter-input" /> +
diff --git a/web/src/components/delimiter-form-field.tsx b/web/src/components/delimiter-form-field.tsx index 40f97a2c81..3bc8c7649f 100644 --- a/web/src/components/delimiter-form-field.tsx +++ b/web/src/components/delimiter-form-field.tsx @@ -2,6 +2,7 @@ import { cn } from '@/lib/utils'; import { forwardRef } from 'react'; import { useFormContext } from 'react-hook-form'; import { useTranslation } from 'react-i18next'; +import { DelimiterPreview } from './delimiter-preview'; import { FormControl, FormField, @@ -74,6 +75,7 @@ export function DelimiterFormField() { +
diff --git a/web/src/components/delimiter-preview.tsx b/web/src/components/delimiter-preview.tsx new file mode 100644 index 0000000000..605c1282a9 --- /dev/null +++ b/web/src/components/delimiter-preview.tsx @@ -0,0 +1,66 @@ +import { useTranslation } from 'react-i18next'; + +import { Badge } from './ui/badge'; +import { parseDelimitersForDisplay } from '@/utils/delimiter-preview'; + +interface Props { + /** The current value of the delimiter field. */ + value: string | undefined; +} + +/** + * Renders a best-effort, informational preview of the delimiters + * derived from `value`. Pure display — does not affect the stored value + * or the chunking behavior. The preview may differ from what the + * backend actually parses (see #17383). + * + * Each delimiter is shown as a small monospace badge; whitespace + * characters (which would otherwise be invisible in the single-line + * input) are rendered with visible Unicode glyphs (`↵` for newline, + * `⇥` for tab, `␣` for space, etc.). + */ +export function DelimiterPreview({ value }: Props) { + const { t } = useTranslation(); + const parsed = parseDelimitersForDisplay(value); + + if (parsed.length === 0) { + return ( +

+ {t('knowledgeDetails.delimiterPreviewEmpty', { + defaultValue: 'No delimiters — text will be chunked by size only.', + })} +

+ ); + } + + return ( +
+ + {t('knowledgeDetails.delimiterPreviewLabel', { + defaultValue: 'Splits at:', + })} + + {parsed.map((d, i) => ( + + {d.display} + + ))} + + {t('knowledgeDetails.delimiterPreviewCount', { + count: parsed.length, + defaultValue: '({{count}})', + }).replace('{{count}}', String(parsed.length))} + +
+ ); +} diff --git a/web/src/locales/en.ts b/web/src/locales/en.ts index ad8ca660c8..1a04f2c3da 100644 --- a/web/src/locales/en.ts +++ b/web/src/locales/en.ts @@ -619,11 +619,15 @@ Example: A 1 KB message with 1024-dim embedding uses ~9 KB. The 5 MB default lim topKTip: `Used together with the Rerank model, this setting defines the number of text chunks to be sent to the specified reranking model.`, delimiter: `Delimiter for text`, delimiterTip: - 'A delimiter or separator can consist of one or multiple special characters. If it is multiple characters, ensure they are enclosed in backticks( ``). For example, if you configure your delimiters like this: \\n`##`;, then your texts will be separated at line breaks, double hash symbols (##), and semicolons.', + 'A delimiter string is parsed into a list of delimiters. Backticks (` `) are the delimiter-of-delimiters: any matching pair of backticks wraps the characters inside it into one multi-character delimiter; any character outside backticks is itself a delimiter. Delimiters are matched longest-first. Example: \\n`##`; parses to three delimiters — newline (\\n), double hash (##), semicolon (;) — and your text will be split at any of them. Single-char delimiters (e.g. !?; → !, ?, ;) need no backticks; multi-char ones (e.g. ##, END) must be backtick-wrapped.', enableChildrenDelimiter: 'Child chunk are used for retrieval', childrenDelimiter: 'Delimiter for text', childrenDelimiterTip: - 'A delimiter or separator can consist of one or multiple special characters. If it is multiple characters, ensure they are enclosed in backticks( ``). For example, if you configure your delimiters like this: \\n`##`;, then your texts will be separated at line breaks, double hash symbols (##), and semicolons.', + 'A delimiter string is parsed into a list of delimiters. Backticks (` `) are the delimiter-of-delimiters: any matching pair of backticks wraps the characters inside it into one multi-character delimiter; any character outside backticks is itself a delimiter. Delimiters are matched longest-first. Example: \\n`##`; parses to three delimiters — newline (\\n), double hash (##), semicolon (;) — and your text will be split at any of them. Single-char delimiters (e.g. !?; → !, ?, ;) need no backticks; multi-char ones (e.g. ##, END) must be backtick-wrapped.', + delimiterPreviewLabel: 'Splits at:', + delimiterPreviewEmpty: + 'No delimiters — text will be chunked by size only.', + delimiterPreviewCount: '({{count}})', html4excel: 'Excel to HTML', html4excelTip: `Use with the General chunking method. When disabled, spreadsheets (XLSX or XLS(Excel 97-2003)) in the dataset will be parsed into key-value pairs. When enabled, they will be parsed into HTML tables, splitting every 12 rows if the original table has more than 12 rows. See https://ragflow.io/docs/dev/enable_excel2html for details.`, diff --git a/web/src/utils/delimiter-preview.ts b/web/src/utils/delimiter-preview.ts new file mode 100644 index 0000000000..6c2df35a8b --- /dev/null +++ b/web/src/utils/delimiter-preview.ts @@ -0,0 +1,94 @@ +/** + * Parses a "Delimiter for text" field value into a list of delimiters for + * display in the UI. The result is meant to be informative only — the + * actual chunking behavior is determined by the backend. + * + * The parsing rule mirrors `rag/nlp/__init__.py::get_delimiters` in + * spirit (backticks group characters into a multi-character delimiter; + * anything outside backticks is itself a delimiter) but does **not** + * exactly match any of the six divergent backend implementations, since + * those implementations disagree on dedupe, sort order, the use of + * `re.I`, and CRLF normalization. The helper is a *preview*, not a + * contract. See #17383 for the backend consolidation. + */ +export interface ParsedDelimiter { + /** The original characters the user entered. */ + raw: string; + /** + * The same content with whitespace characters replaced by visible + * Unicode glyphs so that whitespace-only delimiters (which are + * invisible in a single-line input) still appear in the preview. + */ + display: string; +} + +/** + * Map of whitespace characters to visible glyphs. Only used for + * display; the value sent to the backend is the raw string. + */ +const WHITESPACE_GLYPHS: Record = { + '\n': '↵', // DOWNWARDS ARROW WITH CORNER LEFTWARDS + '\t': '⇥', // RIGHTWARDS ARROW TO BAR + '\r': '␍', // CARRIAGE RETURN SYMBOL + ' ': '␣', // OPEN BOX + '\f': '␌', // FORM FEED SYMBOL + '\v': '␋', // VERTICAL TAB SYMBOL +}; + +/** + * Render a raw delimiter string for display by substituting visible + * Unicode glyphs for each whitespace character. Non-whitespace + * characters pass through unchanged. + */ +function toDisplay(raw: string): string { + let out = ''; + for (const ch of raw) { + out += ch in WHITESPACE_GLYPHS ? WHITESPACE_GLYPHS[ch] : ch; + } + return out; +} + +/** + * Parse the delimiter field into a deduplicated, display-ready list. + * + * Differences from the backend `get_delimiters`: + * - Deduplicates by raw value so distinct delimiters that happen to + * share a display glyph (e.g. a literal `\n` and a user-typed `↵`) + * are not silently merged. The backend does not always dedupe (see + * #17383). + * - Preserves left-to-right input order rather than sorting + * longest-first; visual order is more intuitive for users. + * - Does not regex-escape the values (irrelevant for display). + * + * Returns an empty array if the value is undefined or empty. + */ +export function parseDelimitersForDisplay( + value: string | undefined, +): ParsedDelimiter[] { + if (!value) return []; + + const result: ParsedDelimiter[] = []; + const seen = new Set(); + + const push = (raw: string) => { + if (seen.has(raw)) return; + seen.add(raw); + result.push({ raw, display: toDisplay(raw) }); + }; + + let cursor = 0; + for (const match of value.matchAll(/`([^`]+)`/g)) { + const start = match.index!; + const end = start + match[0].length; + // bare characters before this backtick-wrapped token + for (const ch of value.slice(cursor, start)) push(ch); + // the backtick-wrapped token + push(match[1]); + cursor = end; + } + // bare characters after the last token (or the whole string if no + // backticks were present) + for (const ch of value.slice(cursor)) push(ch); + + return result; +}