diff --git a/internal/ingestion/component/chunker/bare_delimiter_test.go b/internal/ingestion/component/chunker/bare_delimiter_test.go new file mode 100644 index 0000000000..0f4fc9a682 --- /dev/null +++ b/internal/ingestion/component/chunker/bare_delimiter_test.go @@ -0,0 +1,239 @@ +package chunker + +import ( + "context" + "strings" + "testing" +) + +// bare_delimiter_test covers the fix for #17723: bare (non-backtick) +// delimiters must now be honored by TokenChunker — splitting the payload into +// paragraphs that are then merged by token size — instead of being silently +// ignored (the previous keepBare=false contract). +// +// Python contract (rag/nlp/__init__.py:1406-1415, naive_merge default path): +// the payload is split on the delimiter, the delimiter text is DROPPED, and +// each kept paragraph is rebuilt with a leading "\n" (its original surrounding +// whitespace is preserved). Paragraphs are thus joined by "\n", never by the +// raw delimiter. One chunk per segment (no token merge) happens ONLY for a +// backtick-wrapped CUSTOM delimiter. +// +// These tests target BOTH the text/markdown/html path (invokeTextPayload) and +// the JSON path (invokeJSONPayload -> chunkFromItem + global merge), and lock +// the custom(backtick) vs bare distinction. + +// newBareChunker builds a TokenChunker for the given delimiter list and a small +// token budget so merges are observable. +func newBareChunker(t *testing.T, delims []string, tokenSize float64) *TokenChunkerComponent { + t.Helper() + c, err := NewTokenChunker(map[string]any{ + "delimiter_mode": "delimiter", + "delimiters": delims, + "chunk_token_size": tokenSize, + }) + if err != nil { + t.Fatalf("NewTokenChunker: %v", err) + } + return c.(*TokenChunkerComponent) +} + +func invokeText(t *testing.T, c *TokenChunkerComponent, text string) []map[string]any { + t.Helper() + out, err := c.Invoke(context.Background(), nil, map[string]any{ + "name": "doc.txt", + "output_format": "text", + "text": text, + }) + if err != nil { + t.Fatalf("Invoke: %v", err) + } + if msg, ok := out["_ERROR"].(string); ok && msg != "" { + t.Fatalf("Go returned _ERROR: %s", msg) + } + chunks, _ := out["chunks"].([]map[string]any) + return chunks +} + +func joinedText(chunks []map[string]any) string { + var b strings.Builder + for _, ck := range chunks { + b.WriteString(ck["text"].(string)) + } + return b.String() +} + +// TestBareDelimiterSplitsAndTokenMergesText asserts the core fix: a bare +// multi-char delimiter ("::") now splits the text into paragraphs, the +// delimiter text is dropped, and the paragraphs are rejoined with "\n" +// (matching Python's "\n"+sub_sec reconstruction) — not with the raw +// delimiter. The paragraphs are then merged by token size. +func TestBareDelimiterSplitsAndTokenMergesText(t *testing.T) { + c := newBareChunker(t, []string{"::"}, 1024) + const text = "alpha::beta::gamma::delta" + chunks := invokeText(t, c, text) + if len(chunks) == 0 { + t.Fatal("expected at least one chunk") + } + // The delimiter must never survive inside a chunk. + for _, ck := range chunks { + if strings.Contains(ck["text"].(string), "::") { + t.Errorf("bare delimiter leaked into chunk: %q", ck["text"].(string)) + } + } + // Paragraphs are rejoined with "\n" (each paragraph keeps a leading "\n"), + // so the joined content equals the source with "::" replaced by "\n". + const want = "alpha\nbeta\ngamma\ndelta" + if got := joinedText(chunks); got != want { + t.Errorf("joined text: want %q got %q (chunks=%v)", want, got, chunkTexts(chunks)) + } +} + +// TestBareSingleCharDelimiterSplitsText covers the classic case: a single +// ASCII char delimiter (".") splits sentences, the delimiter is dropped, and +// paragraphs are rejoined with "\n". +func TestBareSingleCharDelimiterSplitsText(t *testing.T) { + c := newBareChunker(t, []string{"."}, 1024) + chunks := invokeText(t, c, "first.second.third") + const want = "first\nsecond\nthird" + if got := joinedText(chunks); got != want { + t.Errorf("joined text: want %q got %q (chunks=%v)", want, got, chunkTexts(chunks)) + } + for _, ck := range chunks { + if strings.Contains(ck["text"].(string), ".") { + t.Errorf("bare delimiter leaked into chunk: %q", ck["text"].(string)) + } + } +} + +// TestBareCJKBoundarySplitsText covers a CJK punctuation delimiter. +func TestBareCJKBoundarySplitsText(t *testing.T) { + c := newBareChunker(t, []string{"；"}, 1024) + chunks := invokeText(t, c, "第一部分；第二部分；第三部分") + const want = "第一部分\n第二部分\n第三部分" + if got := joinedText(chunks); got != want { + t.Errorf("joined text: want %q got %q (chunks=%v)", want, got, chunkTexts(chunks)) + } +} + +// TestBareDelimiterSplitsJSON asserts the JSON path also honors bare +// delimiters (chunkFromItem now receives a non-nil pattern and the global +// merge applies because there is no custom delimiter). +func TestBareDelimiterSplitsJSON(t *testing.T) { + c := newBareChunker(t, []string{"。"}, 1024) + out, err := c.Invoke(context.Background(), nil, map[string]any{ + "name": "doc.json", + "output_format": "json", + "json": []map[string]any{ + {"text": "第一句内容。第二句内容。第三句内容。", "doc_type_kwd": "text"}, + }, + }) + if err != nil { + t.Fatalf("Invoke: %v", err) + } + chunks, _ := out["chunks"].([]map[string]any) + const want = "第一句内容\n第二句内容\n第三句内容" + if got := joinedText(chunks); got != want { + t.Errorf("joined text: want %q got %q (chunks=%v)", want, got, chunkTexts(chunks)) + } +} + +// TestCustomDelimiterStillOneChunkPerSegment locks the distinction: a +// backtick-wrapped delimiter must yield one chunk per segment with NO token +// merge (and the delimiter is dropped with no "\n" rejoin), even with a tight +// budget. +func TestCustomDelimiterStillOneChunkPerSegment(t *testing.T) { + c, err := NewTokenChunker(map[string]any{ + "delimiter_mode": "delimiter", + "delimiters": []string{"`::`"}, + "chunk_token_size": float64(4), + }) + if err != nil { + t.Fatalf("NewTokenChunker: %v", err) + } + chunks := invokeText(t, c.(*TokenChunkerComponent), "alpha::beta::gamma::delta") + want := []string{"alpha", "beta", "gamma", "delta"} + if len(chunks) != len(want) { + t.Fatalf("chunk count: want %d got %d (%v)", len(want), len(chunks), chunkTexts(chunks)) + } + for i, w := range want { + if got := chunks[i]["text"].(string); got != w { + t.Errorf("chunk[%d] text: want %q got %q", i, w, got) + } + } +} + +// TestBareDelimiterEmptyText asserts empty input yields no chunks (no panic). +func TestBareDelimiterEmptyText(t *testing.T) { + c := newBareChunker(t, []string{"::"}, 8) + chunks := invokeText(t, c, "") + if len(chunks) != 0 { + t.Errorf("empty text should produce no chunks, got %d", len(chunks)) + } +} + +// TestBareDelimiterDropsEmptySegment asserts that a genuinely empty segment +// produced by a doubled delimiter ("alpha||beta") is dropped, so the joined +// text has no empty paragraph (mirrors Python's `if not sub_sec: continue`). +// (Whitespace-only segments are NOT dropped — Python keeps them — so this test +// deliberately uses an empty, not whitespace, segment.) +func TestBareDelimiterDropsEmptySegment(t *testing.T) { + c := newBareChunker(t, []string{"|"}, 1024) + chunks := invokeText(t, c, "alpha||beta") + const want = "alpha\nbeta" + if got := joinedText(chunks); got != want { + t.Errorf("joined text: want %q got %q (chunks=%v)", want, got, chunkTexts(chunks)) + } +} + +// TestBareDelimiterCRLFNormalized asserts CRLF/CR are normalized to LF before +// splitting (mirrors Python naive_merge text normalization). The "\n" left +// behind by the dropped delimiter is preserved between paragraphs. +func TestBareDelimiterCRLFNormalized(t *testing.T) { + c := newBareChunker(t, []string{"."}, 1024) + chunks := invokeText(t, c, "line one.\r\nline two.\rline three.") + const want = "line one\n\nline two\n\nline three" + if got := joinedText(chunks); got != want { + t.Errorf("joined text: want %q got %q (chunks=%v)", want, got, chunkTexts(chunks)) + } +} + +// TestBareDelimiterNoDelimiterFallback asserts that with no delimiter at all +// the classic single-section merge still applies (regression guard for the +// fallback path): the bare-delimiter machinery is bypassed entirely. +func TestBareDelimiterNoDelimiterFallback(t *testing.T) { + c, err := NewTokenChunker(map[string]any{ + "delimiter_mode": "delimiter", + "delimiters": []string{}, + "chunk_token_size": float64(8), + }) + if err != nil { + t.Fatalf("NewTokenChunker: %v", err) + } + chunks := invokeText(t, c.(*TokenChunkerComponent), "alpha beta gamma delta epsilon zeta eta theta") + if len(chunks) == 0 { + t.Fatalf("expected at least one chunk") + } + // No delimiter configured, so the whole text is one section merged by + // token size; its content must be preserved in order. + if got := joinedText(chunks); !strings.Contains(got, "alpha beta gamma") { + t.Errorf("no-delimiter fallback lost content: %q", got) + } +} + +// TestBareDelimiterTokenMergeCoalesces asserts that short paragraphs separated +// by a bare delimiter are merged up to the token budget (not one chunk per +// paragraph when below budget), while still keeping the "\n" paragraph +// boundary — exactly mirroring Python's merge. +func TestBareDelimiterTokenMergeCoalesces(t *testing.T) { + c := newBareChunker(t, []string{"。"}, 1024) + chunks := invokeText(t, c, "短句一。短句二。短句三。") + // All three short sentences fit under the large budget, so they coalesce + // into a single merged chunk whose text keeps the "\n" paragraph joins. + if len(chunks) != 1 { + t.Fatalf("expected single merged chunk, got %d (%v)", len(chunks), chunkTexts(chunks)) + } + const want = "短句一\n短句二\n短句三" + if got := chunks[0]["text"].(string); got != want { + t.Errorf("merged text: want %q got %q", want, got) + } +} diff --git a/internal/ingestion/component/chunker/common.go b/internal/ingestion/component/chunker/common.go index da6699e6fc..92c32f5679 100644 --- a/internal/ingestion/component/chunker/common.go +++ b/internal/ingestion/component/chunker/common.go @@ -77,13 +77,15 @@ func stringListFromAny(in []any) []string { // --------------------------------------------------------------------------- // compileDelimPattern compiles a TokenChunker-style []string delimiter list. -// Only backtick-wrapped entries produce an active pattern (Python -// token_chunker / rag/nlp/delim list helper); plain (non-backtick) entries -// are ignored here. mergeByTokenSize uses its own hardcoded sentenceDelimiter, -// not this list. Canonical single-string parser_config.delimiter parsing -// lives in ragflow/internal/parser/chunk (ParseDelimiterField). +// Every non-empty entry is active, including bare (non-backtick) delimiters, +// mirroring Python naive_merge / rag/nlp/delim where bare single-character +// delimiters still split. Backtick-wrapped entries contribute their inner +// content. invokeTextPayload decides whether an active delimiter yields one +// chunk per segment (custom/backtick, no merge) or splits into paragraphs that +// are merged by token size (bare). Canonical single-string parser_config.delimiter +// parsing lives in ragflow/internal/parser/chunk (ParseDelimiterField). func compileDelimPattern(delims []string) *regexp.Regexp { - return chunk.CompileDelimiterListPattern(delims) + return chunk.CompileDelimiterPatternList(delims, true) } // splitDroppingDelim mirrors Python's _split_text_by_pattern diff --git a/internal/ingestion/component/chunker/delimiter_case_sensitive_test.go b/internal/ingestion/component/chunker/delimiter_case_sensitive_test.go index 613ab0e62b..63598d838f 100644 --- a/internal/ingestion/component/chunker/delimiter_case_sensitive_test.go +++ b/internal/ingestion/component/chunker/delimiter_case_sensitive_test.go @@ -119,30 +119,43 @@ func TestCompileDelimPattern_BacktickEndIsCaseSensitive(t *testing.T) { } } -func TestCompileDelimPattern_BareCharIsNotActivePattern(t *testing.T) { - // Plain delimiters are not compiled — only backtick-wrapped tokens are. - if p := compileDelimPattern([]string{"a"}); p != nil { - t.Fatalf("compileDelimPattern([a]) = %v, want nil", p) +func TestCompileDelimPattern_BareCharIsActivePattern(t *testing.T) { + // Plain (bare) delimiters ARE compiled into an active pattern (#17723 + // fix): they split the payload into paragraphs that are then merged by + // token size. Only when no entry yields a pattern does compileDelimPattern + // return nil. + if p := compileDelimPattern([]string{"a"}); p == nil { + t.Fatalf("compileDelimPattern([a]) = nil, want active pattern") + } + // Empty / all-empty entries still yield nil. + if p := compileDelimPattern([]string{""}); p != nil { + t.Fatalf("compileDelimPattern([\"\"]) = %v, want nil", p) } } func TestCompileDelimPattern_ExtractsPerEntryIndependently(t *testing.T) { - // Adjacent entries must not form a cross-boundary backtick pair. - // Concatenating "`aa" + "`bb`" would invent an "aa`bb" token; per-entry - // extraction only sees the complete "`bb`" pair in the second entry. + // Adjacent entries must not form a cross-boundary backtick pair: + // "`aa" + "`bb`" must NOT invent an "aa`bb" token. Per-entry extraction + // compiles "`aa" as its literal content (keepBare only promotes entries + // with NO backticks at all) and "`bb`" as the inner "bb"; the two are + // alternated, never concatenated into a single "aa`bb" delimiter. p := compileDelimPattern([]string{"`aa", "`bb`"}) if p == nil { - t.Fatal("expected pattern from second entry `bb`") - } - if got := p.String(); got != "bb" { - t.Fatalf("pattern = %q, want %q (no cross-entry token)", got, "bb") - } - if p.MatchString("aa") { - t.Fatalf("must not match incomplete first-entry token; pattern=%q", p.String()) + t.Fatal("expected pattern from entries") } + // The complete backtick entry "`bb`" is active. if !p.MatchString("bb") { t.Fatalf("must match token from second entry; pattern=%q", p.String()) } + // An incomplete backtick entry "`aa" is compiled as its literal content, + // not promoted to a bare "aa", so a bare "aa" must not match. + if p.MatchString("aa") { + t.Fatalf("incomplete backtick entry must not match a bare token; pattern=%q", p.String()) + } + // No cross-entry "aa`bb" delimiter was invented. + if strings.Contains(p.String(), "aa`bb") { + t.Fatalf("cross-entry backtick token invented; pattern=%q", p.String()) + } // Multiple well-formed entries still combine. p2 := compileDelimPattern([]string{"`end`", "`foo`"}) diff --git a/internal/ingestion/component/chunker/token.go b/internal/ingestion/component/chunker/token.go index de85340dae..d54a105ed0 100644 --- a/internal/ingestion/component/chunker/token.go +++ b/internal/ingestion/component/chunker/token.go @@ -316,10 +316,29 @@ func (c *TokenChunkerComponent) invokeTextPayload(_ context.Context, text string return emptyOutputs() } + // No active delimiter at all: single-section merge that only re-splits + // oversized sections on sentence boundaries. if !hasActiveDelimiter(delimPattern) { - return c.mergeByTokenSize(text, childrenPattern) + return c.mergeByTokenSize(text, nil, childrenPattern) } + // Custom (backtick-wrapped) delimiter: one chunk per segment, no token + // merge — mirrors Python naive_merge's has_custom branch + // (token_chunker.py:1194-1213). + if hasCustomDelim(c.param.Delimiters) { + return c.chunkPerSegment(text, delimPattern, childrenPattern) + } + + // Bare delimiter: split into paragraphs (delimiter dropped), then merge + // by token size — mirrors Python naive_merge's default branch. This is + // the #17723 fix: bare delimiters were previously ignored entirely. + return c.mergeByTokenSize(text, delimPattern, childrenPattern) +} + +// chunkPerSegment splits text on a custom (backtick) delimiter and emits one +// chunk per segment with no token-size merge. Mirrors Python naive_merge's +// has_custom branch (token_chunker.py:1194-1213). +func (c *TokenChunkerComponent) chunkPerSegment(text string, delimPattern, childrenPattern *regexp.Regexp) map[string]any { parts := splitDroppingDelim(text, delimPattern) cleaned := make([]string, 0, len(parts)) for _, p := range parts { @@ -341,12 +360,6 @@ func (c *TokenChunkerComponent) invokeTextPayload(_ context.Context, text string textDocs = append(textDocs, schema.ChunkDoc{Text: s, DocType: "text", CKType: "text"}) } docs := applyChildrenDelimText(textDocs, childrenPattern) - - // Python's naive_merge: a custom (backtick) delimiter yields one chunk - // per segment and no token-size merge (naive_merge:1194-1213). A - // non-custom active delimiter cannot reach here — delimPattern is - // non-nil only when a backtick delimiter exists, so the split-then- - // merge branch was unreachable and has been removed. return chunkOutputs(docs) } @@ -390,7 +403,7 @@ func computeOverlapPrefix(prevText string, overlappedPct float64) (string, int) // unconditional overlap prefix), matching Python naive_merge / token_chunker. // When overlap>0 the previous chunk's tail is always prepended, so a chunk may // exceed chunk_token_size by up to the overlap amount. -func (c *TokenChunkerComponent) mergeByTokenSize(text string, childrenPattern *regexp.Regexp) map[string]any { +func (c *TokenChunkerComponent) mergeByTokenSize(text string, delimPattern, childrenPattern *regexp.Regexp) map[string]any { target := c.param.ChunkTokenSize overlapPct := c.param.OverlappedPercent // Clamp to [0,100] so the merge math below never produces a @@ -408,48 +421,76 @@ func (c *TokenChunkerComponent) mergeByTokenSize(text string, childrenPattern *r // naive_merge runs text.replace("\r\n", "\n").replace("\r", "\n"), // then treats the input string as one section. text = strings.ReplaceAll(strings.ReplaceAll(text, "\r\n", "\n"), "\r", "\n") - sections := []string{text} - if len(sections) == 0 { - return emptyOutputs() - } - // Build merge units from the (single) section. Each unit is a paragraph - // (split on sentence delimiters when the section exceeds target); its - // token count is precomputed so mergeUnits can keep a running sum. + // Build the merge units. + // + // When a bare (non-custom) delimiter is active we split the text into + // paragraphs (the delimiter is DROPPED, mirroring Python naive_merge) and + // use each paragraph VERBATIM as a unit. This preserves the original + // inter-paragraph whitespace, so the merged text matches the source (and + // Python naive_merge); an oversized paragraph also stands alone (#17799) + // instead of being re-split — both matching Python. + // + // Otherwise (no active delimiter) the whole text is a single section and + // oversized sections are re-split on production sentence delimiters, which + // injects "\n" to mirror Python's sentence-boundary handling. This is the + // historical behavior and must stay byte-for-byte identical. + useDelimSplit := hasActiveDelimiter(delimPattern) && !hasCustomDelim(c.param.Delimiters) var units []schema.ChunkDoc - for _, sec := range sections { - sec = strings.TrimSpace(sec) - if sec == "" { - continue - } - t := "\n" + sec - tk := tokenizeStr(t) - if tk <= target { - units = append(units, schema.ChunkDoc{Text: t, TKNums: intPtr(tk), CKType: "text"}) - continue - } - // Oversized section: split on production sentence delimiters into - // units. An oversize unit (still exceeds the budget) is kept whole — - // no atom-split, matching Python naive_merge. Per the unified - // algorithm an over-budget unit STANDS ALONE (#17799): never merged - // into the previous chunk. - parts := sentenceDelimiter.Split(sec, -1) - hadPart := false - for _, part := range parts { - // Keep the raw split fragment, including any inter-line trailing - // whitespace (Python's naive_merge builds each unit from - // "\n" + sub_sec with its trailing space, never TrimSpaced). Only - // genuinely empty fragments are skipped. - if part == "" { + if useDelimSplit { + // Mirror Python naive_merge's default path (rag/nlp/__init__.py:1406-1415): + // split on the delimiter, DROP the delimiter text, and prepend "\n" to + // each kept paragraph. The sub-sec keeps its original surrounding + // whitespace (e.g. the trailing space before the delimiter); only the + // delimiter itself is removed. No sentence re-split is performed here — + // an oversize paragraph stands alone as its own chunk, matching Python. + for _, sub := range splitDroppingDelim(text, delimPattern) { + if sub == "" { continue } - hadPart = true - seg := "\n" + part - units = append(units, schema.ChunkDoc{Text: seg, TKNums: intPtr(tokenizeStr(seg)), CKType: "text"}) - } - if !hadPart { + t := "\n" + sub units = append(units, schema.ChunkDoc{Text: t, TKNums: intPtr(tokenizeStr(t)), CKType: "text"}) } + } else { + sections := []string{text} + for _, sec := range sections { + sec = strings.TrimSpace(sec) + if sec == "" { + continue + } + t := "\n" + sec + tk := tokenizeStr(t) + if tk <= target { + units = append(units, schema.ChunkDoc{Text: t, TKNums: intPtr(tk), CKType: "text"}) + continue + } + // Oversized section: split on production sentence delimiters into + // units. An oversize unit (still exceeds the budget) is kept whole + // — no atom-split, matching Python naive_merge. Per the unified + // algorithm an over-budget unit STANDS ALONE (#17799): never merged + // into the previous chunk. + parts := sentenceDelimiter.Split(sec, -1) + hadPart := false + for _, part := range parts { + // Keep the raw split fragment, including any inter-line + // trailing whitespace (Python's naive_merge builds each unit + // from "\n" + sub_sec with its trailing space, never TrimSpaced). + // Only genuinely empty fragments are skipped. + if part == "" { + continue + } + hadPart = true + seg := "\n" + part + units = append(units, schema.ChunkDoc{Text: seg, TKNums: intPtr(tokenizeStr(seg)), CKType: "text"}) + } + if !hadPart { + units = append(units, schema.ChunkDoc{Text: t, TKNums: intPtr(tokenizeStr(t)), CKType: "text"}) + } + } + } + + if len(units) == 0 { + return emptyOutputs() } // Merge with the unified core (scaled overlap threshold + unconditional diff --git a/internal/ingestion/component/chunker/token_delim_contract_test.go b/internal/ingestion/component/chunker/token_delim_contract_test.go index 5b55ddf326..859d969608 100644 --- a/internal/ingestion/component/chunker/token_delim_contract_test.go +++ b/internal/ingestion/component/chunker/token_delim_contract_test.go @@ -6,13 +6,12 @@ import ( "testing" ) -// TestTokenChunker_BareDelimiterIgnored locks T1: a bare (non-backtick) -// delimiter entry is IGNORED by CompileDelimiterListPattern, so setting -// delimiter_mode + a bare delimiter is effectively a no-op — the text is -// merged by token_size and the bare token survives inside a chunk rather -// than acting as a split point. Regression guard for the "bare entries are -// ignored" contract. -func TestTokenChunker_BareDelimiterIgnored(t *testing.T) { +// TestTokenChunker_BareDelimiterHonored locks the #17723 fix: a bare +// (non-backtick) delimiter entry is now honored by TokenChunker. The payload +// is split on the delimiter into paragraphs (the delimiter is DROPPED, matching +// Python naive_merge), and those paragraphs are then merged by token_size. +// Regression guard for the "bare entries are active" contract. +func TestTokenChunker_BareDelimiterHonored(t *testing.T) { c, err := NewTokenChunker(map[string]any{ "delimiter_mode": "delimiter", "delimiters": []string{"::"}, @@ -38,10 +37,17 @@ func TestTokenChunker_BareDelimiterIgnored(t *testing.T) { for _, ck := range chunks { joined.WriteString(ck["text"].(string)) } - // No content dropped and the bare "::" is preserved (just chunked by - // token size, not split on "::"). - if joined.String() != text { - t.Errorf("bare delimiter not ignored: joined=%q want %q (chunks=%v)", joined.String(), text, chunkTexts(chunks)) + // No content dropped, and the bare "::" is split away (not preserved inside + // a chunk). Python's naive_merge rebuilds each paragraph with a leading + // "\n", so the joined text equals the source with "::" replaced by "\n". + const want = "alpha\nbeta\ngamma\ndelta" + if joined.String() != want { + t.Errorf("bare delimiter not honored: joined=%q want %q (chunks=%v)", joined.String(), want, chunkTexts(chunks)) + } + for _, ck := range chunks { + if strings.Contains(ck["text"].(string), "::") { + t.Errorf("bare delimiter leaked into chunk: %q", ck["text"].(string)) + } } } diff --git a/internal/ingestion/component/chunker/token_strict_cap_test.go b/internal/ingestion/component/chunker/token_strict_cap_test.go index f9926009d8..320a3c9f53 100644 --- a/internal/ingestion/component/chunker/token_strict_cap_test.go +++ b/internal/ingestion/component/chunker/token_strict_cap_test.go @@ -132,7 +132,7 @@ func TestMergeByTokenSize_TextPathStrictCap(t *testing.T) { t.Fatalf("NewTokenChunker: %v", err) } tc := comp.(*TokenChunkerComponent) - out := tc.mergeByTokenSize(b.String(), nil) + out := tc.mergeByTokenSize(b.String(), nil, nil) chunks, _ := out["chunks"].([]map[string]any) if len(chunks) < 2 { t.Fatalf("want multiple chunks, got %d", len(chunks)) @@ -165,7 +165,7 @@ func TestMergeByTokenSize_OversizedUnitStaysWhole(t *testing.T) { if err != nil { t.Fatalf("NewTokenChunker: %v", err) } - out := comp.(*TokenChunkerComponent).mergeByTokenSize(long, nil) + out := comp.(*TokenChunkerComponent).mergeByTokenSize(long, nil, nil) chunks, _ := out["chunks"].([]map[string]any) if len(chunks) != 1 { t.Fatalf("over-budget unit must stay whole, got %d chunk(s)", len(chunks)) @@ -203,7 +203,7 @@ func TestMergeByTokenSize_UnderCapNoOverflow(t *testing.T) { if err != nil { t.Fatalf("NewTokenChunker: %v", err) } - out := comp.(*TokenChunkerComponent).mergeByTokenSize(b.String(), nil) + out := comp.(*TokenChunkerComponent).mergeByTokenSize(b.String(), nil, nil) chunks, _ := out["chunks"].([]map[string]any) return chunks } diff --git a/internal/ingestion/component/chunker/token_test.go b/internal/ingestion/component/chunker/token_test.go index 98ab935adb..4d2ac8e206 100644 --- a/internal/ingestion/component/chunker/token_test.go +++ b/internal/ingestion/component/chunker/token_test.go @@ -607,7 +607,7 @@ func TestMergeByTokenSize_CRLFNormalization(t *testing.T) { c := &TokenChunkerComponent{} c.param.ChunkTokenSize = 128 c.param.OverlappedPercent = 0 - out := c.mergeByTokenSize(text, nil) + out := c.mergeByTokenSize(text, nil, nil) raw, ok := out["chunks"].([]map[string]any) if !ok { t.Fatalf("mergeByTokenSize output missing chunks: %v", out) @@ -652,7 +652,7 @@ func TestMergeByTokenSize_PreservesBlankLines(t *testing.T) { c := &TokenChunkerComponent{} c.param.ChunkTokenSize = 128 c.param.OverlappedPercent = 0 - out := c.mergeByTokenSize("A\n\n\nB", nil) + out := c.mergeByTokenSize("A\n\n\nB", nil, nil) raw, ok := out["chunks"].([]map[string]any) if !ok { t.Fatalf("mergeByTokenSize output missing chunks: %v", out) @@ -681,7 +681,7 @@ func TestMergeByTokenSize_OversizeDropsDelimiters(t *testing.T) { c.param.ChunkTokenSize = 5 c.param.OverlappedPercent = 0 text := "第一句。第二句。第三句。第四句。第五句。" - out := c.mergeByTokenSize(text, nil) + out := c.mergeByTokenSize(text, nil, nil) raw, ok := out["chunks"].([]map[string]any) if !ok { t.Fatalf("mergeByTokenSize output missing chunks: %v", out) @@ -712,7 +712,7 @@ func TestMergeByTokenSize_OversizeDropsBlankLines(t *testing.T) { // dropped, mirroring Python, so the blank line must not survive. block := strings.Repeat("知识库检索增强生成技术", 7) // 70 chars text := block + "\n\n" + block - out := c.mergeByTokenSize(text, nil) + out := c.mergeByTokenSize(text, nil, nil) raw, ok := out["chunks"].([]map[string]any) if !ok { t.Fatalf("mergeByTokenSize output missing chunks: %v", out)