fix(chunker): honor bare (non-backtick) delimiters in TokenChunker (#17723) (#18182)

Go's `TokenChunker` previously ignored **bare (non-backtick)
delimiters** such as `::`, `.`, or `；`. `compileDelimPattern` compiled
them with `keepBare=false`, so a bare delimiter produced a `nil` pattern
and the payload was routed to the single-section merge that never split
on it — diverging from Python's `naive_merge`, which splits on bare
delimiters and then merges by token size.

internal/ingestion/component/chunker/bare_delimiter_test.go

@@ -0,0 +1,239 @@
+package chunker
+
+import (
+	"context"
+	"strings"
+	"testing"
+)
+
+// bare_delimiter_test covers the fix for #17723: bare (non-backtick)
+// delimiters must now be honored by TokenChunker — splitting the payload into
+// paragraphs that are then merged by token size — instead of being silently
+// ignored (the previous keepBare=false contract).
+//
+// Python contract (rag/nlp/__init__.py:1406-1415, naive_merge default path):
+// the payload is split on the delimiter, the delimiter text is DROPPED, and
+// each kept paragraph is rebuilt with a leading "\n" (its original surrounding
+// whitespace is preserved). Paragraphs are thus joined by "\n", never by the
+// raw delimiter. One chunk per segment (no token merge) happens ONLY for a
+// backtick-wrapped CUSTOM delimiter.
+//
+// These tests target BOTH the text/markdown/html path (invokeTextPayload) and
+// the JSON path (invokeJSONPayload -> chunkFromItem + global merge), and lock
+// the custom(backtick) vs bare distinction.
+
+// newBareChunker builds a TokenChunker for the given delimiter list and a small
+// token budget so merges are observable.
+func newBareChunker(t *testing.T, delims []string, tokenSize float64) *TokenChunkerComponent {
+	t.Helper()
+	c, err := NewTokenChunker(map[string]any{
+		"delimiter_mode":   "delimiter",
+		"delimiters":       delims,
+		"chunk_token_size": tokenSize,
+	})
+	if err != nil {
+		t.Fatalf("NewTokenChunker: %v", err)
+	}
+	return c.(*TokenChunkerComponent)
+}
+
+func invokeText(t *testing.T, c *TokenChunkerComponent, text string) []map[string]any {
+	t.Helper()
+	out, err := c.Invoke(context.Background(), nil, map[string]any{
+		"name":          "doc.txt",
+		"output_format": "text",
+		"text":          text,
+	})
+	if err != nil {
+		t.Fatalf("Invoke: %v", err)
+	}
+	if msg, ok := out["_ERROR"].(string); ok && msg != "" {
+		t.Fatalf("Go returned _ERROR: %s", msg)
+	}
+	chunks, _ := out["chunks"].([]map[string]any)
+	return chunks
+}
+
+func joinedText(chunks []map[string]any) string {
+	var b strings.Builder
+	for _, ck := range chunks {
+		b.WriteString(ck["text"].(string))
+	}
+	return b.String()
+}
+
+// TestBareDelimiterSplitsAndTokenMergesText asserts the core fix: a bare
+// multi-char delimiter ("::") now splits the text into paragraphs, the
+// delimiter text is dropped, and the paragraphs are rejoined with "\n"
+// (matching Python's "\n"+sub_sec reconstruction) — not with the raw
+// delimiter. The paragraphs are then merged by token size.
+func TestBareDelimiterSplitsAndTokenMergesText(t *testing.T) {
+	c := newBareChunker(t, []string{"::"}, 1024)
+	const text = "alpha::beta::gamma::delta"
+	chunks := invokeText(t, c, text)
+	if len(chunks) == 0 {
+		t.Fatal("expected at least one chunk")
+	}
+	// The delimiter must never survive inside a chunk.
+	for _, ck := range chunks {
+		if strings.Contains(ck["text"].(string), "::") {
+			t.Errorf("bare delimiter leaked into chunk: %q", ck["text"].(string))
+		}
+	}
+	// Paragraphs are rejoined with "\n" (each paragraph keeps a leading "\n"),
+	// so the joined content equals the source with "::" replaced by "\n".
+	const want = "alpha\nbeta\ngamma\ndelta"
+	if got := joinedText(chunks); got != want {
+		t.Errorf("joined text: want %q got %q (chunks=%v)", want, got, chunkTexts(chunks))
+	}
+}
+
+// TestBareSingleCharDelimiterSplitsText covers the classic case: a single
+// ASCII char delimiter (".") splits sentences, the delimiter is dropped, and
+// paragraphs are rejoined with "\n".
+func TestBareSingleCharDelimiterSplitsText(t *testing.T) {
+	c := newBareChunker(t, []string{"."}, 1024)
+	chunks := invokeText(t, c, "first.second.third")
+	const want = "first\nsecond\nthird"
+	if got := joinedText(chunks); got != want {
+		t.Errorf("joined text: want %q got %q (chunks=%v)", want, got, chunkTexts(chunks))
+	}
+	for _, ck := range chunks {
+		if strings.Contains(ck["text"].(string), ".") {
+			t.Errorf("bare delimiter leaked into chunk: %q", ck["text"].(string))
+		}
+	}
+}
+
+// TestBareCJKBoundarySplitsText covers a CJK punctuation delimiter.
+func TestBareCJKBoundarySplitsText(t *testing.T) {
+	c := newBareChunker(t, []string{"；"}, 1024)
+	chunks := invokeText(t, c, "第一部分；第二部分；第三部分")
+	const want = "第一部分\n第二部分\n第三部分"
+	if got := joinedText(chunks); got != want {
+		t.Errorf("joined text: want %q got %q (chunks=%v)", want, got, chunkTexts(chunks))
+	}
+}
+
+// TestBareDelimiterSplitsJSON asserts the JSON path also honors bare
+// delimiters (chunkFromItem now receives a non-nil pattern and the global
+// merge applies because there is no custom delimiter).
+func TestBareDelimiterSplitsJSON(t *testing.T) {
+	c := newBareChunker(t, []string{"。"}, 1024)
+	out, err := c.Invoke(context.Background(), nil, map[string]any{
+		"name":          "doc.json",
+		"output_format": "json",
+		"json": []map[string]any{
+			{"text": "第一句内容。第二句内容。第三句内容。", "doc_type_kwd": "text"},
+		},
+	})
+	if err != nil {
+		t.Fatalf("Invoke: %v", err)
+	}
+	chunks, _ := out["chunks"].([]map[string]any)
+	const want = "第一句内容\n第二句内容\n第三句内容"
+	if got := joinedText(chunks); got != want {
+		t.Errorf("joined text: want %q got %q (chunks=%v)", want, got, chunkTexts(chunks))
+	}
+}
+
+// TestCustomDelimiterStillOneChunkPerSegment locks the distinction: a
+// backtick-wrapped delimiter must yield one chunk per segment with NO token
+// merge (and the delimiter is dropped with no "\n" rejoin), even with a tight
+// budget.
+func TestCustomDelimiterStillOneChunkPerSegment(t *testing.T) {
+	c, err := NewTokenChunker(map[string]any{
+		"delimiter_mode":   "delimiter",
+		"delimiters":       []string{"`::`"},
+		"chunk_token_size": float64(4),
+	})
+	if err != nil {
+		t.Fatalf("NewTokenChunker: %v", err)
+	}
+	chunks := invokeText(t, c.(*TokenChunkerComponent), "alpha::beta::gamma::delta")
+	want := []string{"alpha", "beta", "gamma", "delta"}
+	if len(chunks) != len(want) {
+		t.Fatalf("chunk count: want %d got %d (%v)", len(want), len(chunks), chunkTexts(chunks))
+	}
+	for i, w := range want {
+		if got := chunks[i]["text"].(string); got != w {
+			t.Errorf("chunk[%d] text: want %q got %q", i, w, got)
+		}
+	}
+}
+
+// TestBareDelimiterEmptyText asserts empty input yields no chunks (no panic).
+func TestBareDelimiterEmptyText(t *testing.T) {
+	c := newBareChunker(t, []string{"::"}, 8)
+	chunks := invokeText(t, c, "")
+	if len(chunks) != 0 {
+		t.Errorf("empty text should produce no chunks, got %d", len(chunks))
+	}
+}
+
+// TestBareDelimiterDropsEmptySegment asserts that a genuinely empty segment
+// produced by a doubled delimiter ("alpha||beta") is dropped, so the joined
+// text has no empty paragraph (mirrors Python's `if not sub_sec: continue`).
+// (Whitespace-only segments are NOT dropped — Python keeps them — so this test
+// deliberately uses an empty, not whitespace, segment.)
+func TestBareDelimiterDropsEmptySegment(t *testing.T) {
+	c := newBareChunker(t, []string{"|"}, 1024)
+	chunks := invokeText(t, c, "alpha||beta")
+	const want = "alpha\nbeta"
+	if got := joinedText(chunks); got != want {
+		t.Errorf("joined text: want %q got %q (chunks=%v)", want, got, chunkTexts(chunks))
+	}
+}
+
+// TestBareDelimiterCRLFNormalized asserts CRLF/CR are normalized to LF before
+// splitting (mirrors Python naive_merge text normalization). The "\n" left
+// behind by the dropped delimiter is preserved between paragraphs.
+func TestBareDelimiterCRLFNormalized(t *testing.T) {
+	c := newBareChunker(t, []string{"."}, 1024)
+	chunks := invokeText(t, c, "line one.\r\nline two.\rline three.")
+	const want = "line one\n\nline two\n\nline three"
+	if got := joinedText(chunks); got != want {
+		t.Errorf("joined text: want %q got %q (chunks=%v)", want, got, chunkTexts(chunks))
+	}
+}
+
+// TestBareDelimiterNoDelimiterFallback asserts that with no delimiter at all
+// the classic single-section merge still applies (regression guard for the
+// fallback path): the bare-delimiter machinery is bypassed entirely.
+func TestBareDelimiterNoDelimiterFallback(t *testing.T) {
+	c, err := NewTokenChunker(map[string]any{
+		"delimiter_mode":   "delimiter",
+		"delimiters":       []string{},
+		"chunk_token_size": float64(8),
+	})
+	if err != nil {
+		t.Fatalf("NewTokenChunker: %v", err)
+	}
+	chunks := invokeText(t, c.(*TokenChunkerComponent), "alpha beta gamma delta epsilon zeta eta theta")
+	if len(chunks) == 0 {
+		t.Fatalf("expected at least one chunk")
+	}
+	// No delimiter configured, so the whole text is one section merged by
+	// token size; its content must be preserved in order.
+	if got := joinedText(chunks); !strings.Contains(got, "alpha beta gamma") {
+		t.Errorf("no-delimiter fallback lost content: %q", got)
+	}
+}
+
+// TestBareDelimiterTokenMergeCoalesces asserts that short paragraphs separated
+// by a bare delimiter are merged up to the token budget (not one chunk per
+// paragraph when below budget), while still keeping the "\n" paragraph
+// boundary — exactly mirroring Python's merge.
+func TestBareDelimiterTokenMergeCoalesces(t *testing.T) {
+	c := newBareChunker(t, []string{"。"}, 1024)
+	chunks := invokeText(t, c, "短句一。短句二。短句三。")
+	// All three short sentences fit under the large budget, so they coalesce
+	// into a single merged chunk whose text keeps the "\n" paragraph joins.
+	if len(chunks) != 1 {
+		t.Fatalf("expected single merged chunk, got %d (%v)", len(chunks), chunkTexts(chunks))
+	}
+	const want = "短句一\n短句二\n短句三"
+	if got := chunks[0]["text"].(string); got != want {
+		t.Errorf("merged text: want %q got %q", want, got)
+	}
+}

internal/ingestion/component/chunker/common.go

@@ -77,13 +77,15 @@ func stringListFromAny(in []any) []string {
 // ---------------------------------------------------------------------------
 
 // compileDelimPattern compiles a TokenChunker-style []string delimiter list.
-// Only backtick-wrapped entries produce an active pattern (Python
-// token_chunker / rag/nlp/delim list helper); plain (non-backtick) entries
-// are ignored here. mergeByTokenSize uses its own hardcoded sentenceDelimiter,
-// not this list. Canonical single-string parser_config.delimiter parsing
-// lives in ragflow/internal/parser/chunk (ParseDelimiterField).
+// Every non-empty entry is active, including bare (non-backtick) delimiters,
+// mirroring Python naive_merge / rag/nlp/delim where bare single-character
+// delimiters still split. Backtick-wrapped entries contribute their inner
+// content. invokeTextPayload decides whether an active delimiter yields one
+// chunk per segment (custom/backtick, no merge) or splits into paragraphs that
+// are merged by token size (bare). Canonical single-string parser_config.delimiter
+// parsing lives in ragflow/internal/parser/chunk (ParseDelimiterField).
 func compileDelimPattern(delims []string) *regexp.Regexp {
-	return chunk.CompileDelimiterListPattern(delims)
+	return chunk.CompileDelimiterPatternList(delims, true)
 }
 
 // splitDroppingDelim mirrors Python's _split_text_by_pattern

internal/ingestion/component/chunker/delimiter_case_sensitive_test.go

@@ -119,30 +119,43 @@ func TestCompileDelimPattern_BacktickEndIsCaseSensitive(t *testing.T) {
 	}
 }
 
-func TestCompileDelimPattern_BareCharIsNotActivePattern(t *testing.T) {
-	// Plain delimiters are not compiled — only backtick-wrapped tokens are.
-	if p := compileDelimPattern([]string{"a"}); p != nil {
-		t.Fatalf("compileDelimPattern([a]) = %v, want nil", p)
+func TestCompileDelimPattern_BareCharIsActivePattern(t *testing.T) {
+	// Plain (bare) delimiters ARE compiled into an active pattern (#17723
+	// fix): they split the payload into paragraphs that are then merged by
+	// token size. Only when no entry yields a pattern does compileDelimPattern
+	// return nil.
+	if p := compileDelimPattern([]string{"a"}); p == nil {
+		t.Fatalf("compileDelimPattern([a]) = nil, want active pattern")
+	}
+	// Empty / all-empty entries still yield nil.
+	if p := compileDelimPattern([]string{""}); p != nil {
+		t.Fatalf("compileDelimPattern([\"\"]) = %v, want nil", p)
 	}
 }
 
 func TestCompileDelimPattern_ExtractsPerEntryIndependently(t *testing.T) {
-	// Adjacent entries must not form a cross-boundary backtick pair.
-	// Concatenating "`aa" + "`bb`" would invent an "aa`bb" token; per-entry
-	// extraction only sees the complete "`bb`" pair in the second entry.
+	// Adjacent entries must not form a cross-boundary backtick pair:
+	// "`aa" + "`bb`" must NOT invent an "aa`bb" token. Per-entry extraction
+	// compiles "`aa" as its literal content (keepBare only promotes entries
+	// with NO backticks at all) and "`bb`" as the inner "bb"; the two are
+	// alternated, never concatenated into a single "aa`bb" delimiter.
 	p := compileDelimPattern([]string{"`aa", "`bb`"})
 	if p == nil {
-		t.Fatal("expected pattern from second entry `bb`")
-	}
-	if got := p.String(); got != "bb" {
-		t.Fatalf("pattern = %q, want %q (no cross-entry token)", got, "bb")
-	}
-	if p.MatchString("aa") {
-		t.Fatalf("must not match incomplete first-entry token; pattern=%q", p.String())
+		t.Fatal("expected pattern from entries")
 	}
+	// The complete backtick entry "`bb`" is active.
 	if !p.MatchString("bb") {
 		t.Fatalf("must match token from second entry; pattern=%q", p.String())
 	}
+	// An incomplete backtick entry "`aa" is compiled as its literal content,
+	// not promoted to a bare "aa", so a bare "aa" must not match.
+	if p.MatchString("aa") {
+		t.Fatalf("incomplete backtick entry must not match a bare token; pattern=%q", p.String())
+	}
+	// No cross-entry "aa`bb" delimiter was invented.
+	if strings.Contains(p.String(), "aa`bb") {
+		t.Fatalf("cross-entry backtick token invented; pattern=%q", p.String())
+	}
 
 	// Multiple well-formed entries still combine.
 	p2 := compileDelimPattern([]string{"`end`", "`foo`"})

internal/ingestion/component/chunker/token.go

@@ -316,10 +316,29 @@ func (c *TokenChunkerComponent) invokeTextPayload(_ context.Context, text string
 		return emptyOutputs()
 	}
 
+	// No active delimiter at all: single-section merge that only re-splits
+	// oversized sections on sentence boundaries.
 	if !hasActiveDelimiter(delimPattern) {
-		return c.mergeByTokenSize(text, childrenPattern)
+		return c.mergeByTokenSize(text, nil, childrenPattern)
 	}
 
+	// Custom (backtick-wrapped) delimiter: one chunk per segment, no token
+	// merge — mirrors Python naive_merge's has_custom branch
+	// (token_chunker.py:1194-1213).
+	if hasCustomDelim(c.param.Delimiters) {
+		return c.chunkPerSegment(text, delimPattern, childrenPattern)
+	}
+
+	// Bare delimiter: split into paragraphs (delimiter dropped), then merge
+	// by token size — mirrors Python naive_merge's default branch. This is
+	// the #17723 fix: bare delimiters were previously ignored entirely.
+	return c.mergeByTokenSize(text, delimPattern, childrenPattern)
+}
+
+// chunkPerSegment splits text on a custom (backtick) delimiter and emits one
+// chunk per segment with no token-size merge. Mirrors Python naive_merge's
+// has_custom branch (token_chunker.py:1194-1213).
+func (c *TokenChunkerComponent) chunkPerSegment(text string, delimPattern, childrenPattern *regexp.Regexp) map[string]any {
 	parts := splitDroppingDelim(text, delimPattern)
 	cleaned := make([]string, 0, len(parts))
 	for _, p := range parts {
@@ -341,12 +360,6 @@ func (c *TokenChunkerComponent) invokeTextPayload(_ context.Context, text string
 		textDocs = append(textDocs, schema.ChunkDoc{Text: s, DocType: "text", CKType: "text"})
 	}
 	docs := applyChildrenDelimText(textDocs, childrenPattern)
-
-	// Python's naive_merge: a custom (backtick) delimiter yields one chunk
-	// per segment and no token-size merge (naive_merge:1194-1213). A
-	// non-custom active delimiter cannot reach here — delimPattern is
-	// non-nil only when a backtick delimiter exists, so the split-then-
-	// merge branch was unreachable and has been removed.
 	return chunkOutputs(docs)
 }
 
@@ -390,7 +403,7 @@ func computeOverlapPrefix(prevText string, overlappedPct float64) (string, int)
 // unconditional overlap prefix), matching Python naive_merge / token_chunker.
 // When overlap>0 the previous chunk's tail is always prepended, so a chunk may
 // exceed chunk_token_size by up to the overlap amount.
-func (c *TokenChunkerComponent) mergeByTokenSize(text string, childrenPattern *regexp.Regexp) map[string]any {
+func (c *TokenChunkerComponent) mergeByTokenSize(text string, delimPattern, childrenPattern *regexp.Regexp) map[string]any {
 	target := c.param.ChunkTokenSize
 	overlapPct := c.param.OverlappedPercent
 	// Clamp to [0,100] so the merge math below never produces a
@@ -408,48 +421,76 @@ func (c *TokenChunkerComponent) mergeByTokenSize(text string, childrenPattern *r
 	// naive_merge runs text.replace("\r\n", "\n").replace("\r", "\n"),
 	// then treats the input string as one section.
 	text = strings.ReplaceAll(strings.ReplaceAll(text, "\r\n", "\n"), "\r", "\n")
-	sections := []string{text}
-	if len(sections) == 0 {
-		return emptyOutputs()
-	}
 
-	// Build merge units from the (single) section. Each unit is a paragraph
-	// (split on sentence delimiters when the section exceeds target); its
-	// token count is precomputed so mergeUnits can keep a running sum.
+	// Build the merge units.
+	//
+	// When a bare (non-custom) delimiter is active we split the text into
+	// paragraphs (the delimiter is DROPPED, mirroring Python naive_merge) and
+	// use each paragraph VERBATIM as a unit. This preserves the original
+	// inter-paragraph whitespace, so the merged text matches the source (and
+	// Python naive_merge); an oversized paragraph also stands alone (#17799)
+	// instead of being re-split — both matching Python.
+	//
+	// Otherwise (no active delimiter) the whole text is a single section and
+	// oversized sections are re-split on production sentence delimiters, which
+	// injects "\n" to mirror Python's sentence-boundary handling. This is the
+	// historical behavior and must stay byte-for-byte identical.
+	useDelimSplit := hasActiveDelimiter(delimPattern) && !hasCustomDelim(c.param.Delimiters)
 	var units []schema.ChunkDoc
-	for _, sec := range sections {
-		sec = strings.TrimSpace(sec)
-		if sec == "" {
-			continue
-		}
-		t := "\n" + sec
-		tk := tokenizeStr(t)
-		if tk <= target {
-			units = append(units, schema.ChunkDoc{Text: t, TKNums: intPtr(tk), CKType: "text"})
-			continue
-		}
-		// Oversized section: split on production sentence delimiters into
-		// units. An oversize unit (still exceeds the budget) is kept whole —
-		// no atom-split, matching Python naive_merge. Per the unified
-		// algorithm an over-budget unit STANDS ALONE (#17799): never merged
-		// into the previous chunk.
-		parts := sentenceDelimiter.Split(sec, -1)
-		hadPart := false
-		for _, part := range parts {
-			// Keep the raw split fragment, including any inter-line trailing
-			// whitespace (Python's naive_merge builds each unit from
-			// "\n" + sub_sec with its trailing space, never TrimSpaced). Only
-			// genuinely empty fragments are skipped.
-			if part == "" {
+	if useDelimSplit {
+		// Mirror Python naive_merge's default path (rag/nlp/__init__.py:1406-1415):
+		// split on the delimiter, DROP the delimiter text, and prepend "\n" to
+		// each kept paragraph. The sub-sec keeps its original surrounding
+		// whitespace (e.g. the trailing space before the delimiter); only the
+		// delimiter itself is removed. No sentence re-split is performed here —
+		// an oversize paragraph stands alone as its own chunk, matching Python.
+		for _, sub := range splitDroppingDelim(text, delimPattern) {
+			if sub == "" {
 				continue
 			}
-			hadPart = true
-			seg := "\n" + part
-			units = append(units, schema.ChunkDoc{Text: seg, TKNums: intPtr(tokenizeStr(seg)), CKType: "text"})
-		}
-		if !hadPart {
+			t := "\n" + sub
 			units = append(units, schema.ChunkDoc{Text: t, TKNums: intPtr(tokenizeStr(t)), CKType: "text"})
 		}
+	} else {
+		sections := []string{text}
+		for _, sec := range sections {
+			sec = strings.TrimSpace(sec)
+			if sec == "" {
+				continue
+			}
+			t := "\n" + sec
+			tk := tokenizeStr(t)
+			if tk <= target {
+				units = append(units, schema.ChunkDoc{Text: t, TKNums: intPtr(tk), CKType: "text"})
+				continue
+			}
+			// Oversized section: split on production sentence delimiters into
+			// units. An oversize unit (still exceeds the budget) is kept whole
+			// — no atom-split, matching Python naive_merge. Per the unified
+			// algorithm an over-budget unit STANDS ALONE (#17799): never merged
+			// into the previous chunk.
+			parts := sentenceDelimiter.Split(sec, -1)
+			hadPart := false
+			for _, part := range parts {
+				// Keep the raw split fragment, including any inter-line
+				// trailing whitespace (Python's naive_merge builds each unit
+				// from "\n" + sub_sec with its trailing space, never TrimSpaced).
+				// Only genuinely empty fragments are skipped.
+				if part == "" {
+					continue
+				}
+				hadPart = true
+				seg := "\n" + part
+				units = append(units, schema.ChunkDoc{Text: seg, TKNums: intPtr(tokenizeStr(seg)), CKType: "text"})
+			}
+			if !hadPart {
+				units = append(units, schema.ChunkDoc{Text: t, TKNums: intPtr(tokenizeStr(t)), CKType: "text"})
+			}
+		}
+	}
+
+	if len(units) == 0 {
+		return emptyOutputs()
 	}
 
 	// Merge with the unified core (scaled overlap threshold + unconditional

internal/ingestion/component/chunker/token_delim_contract_test.go

@@ -6,13 +6,12 @@ import (
 	"testing"
 )
 
-// TestTokenChunker_BareDelimiterIgnored locks T1: a bare (non-backtick)
-// delimiter entry is IGNORED by CompileDelimiterListPattern, so setting
-// delimiter_mode + a bare delimiter is effectively a no-op — the text is
-// merged by token_size and the bare token survives inside a chunk rather
-// than acting as a split point. Regression guard for the "bare entries are
-// ignored" contract.
-func TestTokenChunker_BareDelimiterIgnored(t *testing.T) {
+// TestTokenChunker_BareDelimiterHonored locks the #17723 fix: a bare
+// (non-backtick) delimiter entry is now honored by TokenChunker. The payload
+// is split on the delimiter into paragraphs (the delimiter is DROPPED, matching
+// Python naive_merge), and those paragraphs are then merged by token_size.
+// Regression guard for the "bare entries are active" contract.
+func TestTokenChunker_BareDelimiterHonored(t *testing.T) {
 	c, err := NewTokenChunker(map[string]any{
 		"delimiter_mode":   "delimiter",
 		"delimiters":       []string{"::"},
@@ -38,10 +37,17 @@ func TestTokenChunker_BareDelimiterIgnored(t *testing.T) {
 	for _, ck := range chunks {
 		joined.WriteString(ck["text"].(string))
 	}
-	// No content dropped and the bare "::" is preserved (just chunked by
-	// token size, not split on "::").
-	if joined.String() != text {
-		t.Errorf("bare delimiter not ignored: joined=%q want %q (chunks=%v)", joined.String(), text, chunkTexts(chunks))
+	// No content dropped, and the bare "::" is split away (not preserved inside
+	// a chunk). Python's naive_merge rebuilds each paragraph with a leading
+	// "\n", so the joined text equals the source with "::" replaced by "\n".
+	const want = "alpha\nbeta\ngamma\ndelta"
+	if joined.String() != want {
+		t.Errorf("bare delimiter not honored: joined=%q want %q (chunks=%v)", joined.String(), want, chunkTexts(chunks))
+	}
+	for _, ck := range chunks {
+		if strings.Contains(ck["text"].(string), "::") {
+			t.Errorf("bare delimiter leaked into chunk: %q", ck["text"].(string))
+		}
 	}
 }
 

internal/ingestion/component/chunker/token_strict_cap_test.go

@@ -132,7 +132,7 @@ func TestMergeByTokenSize_TextPathStrictCap(t *testing.T) {
 		t.Fatalf("NewTokenChunker: %v", err)
 	}
 	tc := comp.(*TokenChunkerComponent)
-	out := tc.mergeByTokenSize(b.String(), nil)
+	out := tc.mergeByTokenSize(b.String(), nil, nil)
 	chunks, _ := out["chunks"].([]map[string]any)
 	if len(chunks) < 2 {
 		t.Fatalf("want multiple chunks, got %d", len(chunks))
@@ -165,7 +165,7 @@ func TestMergeByTokenSize_OversizedUnitStaysWhole(t *testing.T) {
 	if err != nil {
 		t.Fatalf("NewTokenChunker: %v", err)
 	}
-	out := comp.(*TokenChunkerComponent).mergeByTokenSize(long, nil)
+	out := comp.(*TokenChunkerComponent).mergeByTokenSize(long, nil, nil)
 	chunks, _ := out["chunks"].([]map[string]any)
 	if len(chunks) != 1 {
 		t.Fatalf("over-budget unit must stay whole, got %d chunk(s)", len(chunks))
@@ -203,7 +203,7 @@ func TestMergeByTokenSize_UnderCapNoOverflow(t *testing.T) {
 		if err != nil {
 			t.Fatalf("NewTokenChunker: %v", err)
 		}
-		out := comp.(*TokenChunkerComponent).mergeByTokenSize(b.String(), nil)
+		out := comp.(*TokenChunkerComponent).mergeByTokenSize(b.String(), nil, nil)
 		chunks, _ := out["chunks"].([]map[string]any)
 		return chunks
 	}

internal/ingestion/component/chunker/token_test.go

@@ -607,7 +607,7 @@ func TestMergeByTokenSize_CRLFNormalization(t *testing.T) {
 		c := &TokenChunkerComponent{}
 		c.param.ChunkTokenSize = 128
 		c.param.OverlappedPercent = 0
-		out := c.mergeByTokenSize(text, nil)
+		out := c.mergeByTokenSize(text, nil, nil)
 		raw, ok := out["chunks"].([]map[string]any)
 		if !ok {
 			t.Fatalf("mergeByTokenSize output missing chunks: %v", out)
@@ -652,7 +652,7 @@ func TestMergeByTokenSize_PreservesBlankLines(t *testing.T) {
 	c := &TokenChunkerComponent{}
 	c.param.ChunkTokenSize = 128
 	c.param.OverlappedPercent = 0
-	out := c.mergeByTokenSize("A\n\n\nB", nil)
+	out := c.mergeByTokenSize("A\n\n\nB", nil, nil)
 	raw, ok := out["chunks"].([]map[string]any)
 	if !ok {
 		t.Fatalf("mergeByTokenSize output missing chunks: %v", out)
@@ -681,7 +681,7 @@ func TestMergeByTokenSize_OversizeDropsDelimiters(t *testing.T) {
 	c.param.ChunkTokenSize = 5
 	c.param.OverlappedPercent = 0
 	text := "第一句。第二句。第三句。第四句。第五句。"
-	out := c.mergeByTokenSize(text, nil)
+	out := c.mergeByTokenSize(text, nil, nil)
 	raw, ok := out["chunks"].([]map[string]any)
 	if !ok {
 		t.Fatalf("mergeByTokenSize output missing chunks: %v", out)
@@ -712,7 +712,7 @@ func TestMergeByTokenSize_OversizeDropsBlankLines(t *testing.T) {
 	// dropped, mirroring Python, so the blank line must not survive.
 	block := strings.Repeat("知识库检索增强生成技术", 7) // 70 chars
 	text := block + "\n\n" + block
-	out := c.mergeByTokenSize(text, nil)
+	out := c.mergeByTokenSize(text, nil, nil)
 	raw, ok := out["chunks"].([]map[string]any)
 	if !ok {
 		t.Fatalf("mergeByTokenSize output missing chunks: %v", out)