From 1aa4e3c1f31899a6204545a4290e49f4b87d864b Mon Sep 17 00:00:00 2001 From: Jack Date: Fri, 7 Aug 2026 16:11:42 +0800 Subject: [PATCH] refactor(chunker): converge delimiter_mode to {delimiter, one}, drop token_size (#17979) Converge `TokenChunker.delimiter_mode` from three values (`token_size`, `delimiter`, `one`) to two (`delimiter`, `one`). The unified `delimiter` mode now carries the old `token_size` semantics: when no active (backtick) delimiter is present, text/JSON chunks are merged up to `chunk_token_size`; when a backtick delimiter is present, the text is split by it and not merged. `one` continues to be handled by the separate `OneChunker`. --- .../advanced_ingestion_pipeline.json | 4 +- agent/templates/chunk_summary.json | 4 +- agent/templates/compiler.json | 4 +- internal/ingestion/component/chunker/token.go | 2 +- .../chunker/token_strict_cap_test.go | 8 +- .../component/chunker/token_tag_test.go | 2 +- .../ingestion/component/chunker/token_test.go | 20 ++-- .../ingestion/component/schema/chunker.go | 6 +- .../ingestion/component/schema/schema_test.go | 4 +- .../template/ingestion_pipeline_email.json | 4 +- .../template/ingestion_pipeline_general.json | 4 +- ...ingestion_pipeline_knowledge_compiler.json | 2 +- .../task/pipeline_executor_defaults_test.go | 6 +- rag/flow/chunker/token_chunker.py | 88 +++++++++--------- rag/flow/tests/test_token_chunker.py | 92 +++++++++++++++++-- 15 files changed, 167 insertions(+), 83 deletions(-) diff --git a/agent/templates/advanced_ingestion_pipeline.json b/agent/templates/advanced_ingestion_pipeline.json index ecb8fd30fa..b709b741e6 100644 --- a/agent/templates/advanced_ingestion_pipeline.json +++ b/agent/templates/advanced_ingestion_pipeline.json @@ -336,7 +336,7 @@ "params": { "children_delimiters": [], "chunk_token_size": 512, - "delimiter_mode": "token_size", + "delimiter_mode": "delimiter", "delimiters": [], "image_context_size": 0, "outputs": { @@ -576,7 +576,7 @@ "form": { "children_delimiters": [], "chunk_token_size": 512, - "delimiter_mode": "token_size", + "delimiter_mode": "delimiter", "delimiters": [ { "value": "\n" diff --git a/agent/templates/chunk_summary.json b/agent/templates/chunk_summary.json index 980ba62b5b..2f76945db8 100644 --- a/agent/templates/chunk_summary.json +++ b/agent/templates/chunk_summary.json @@ -219,7 +219,7 @@ "params": { "children_delimiters": [], "chunk_token_size": 512, - "delimiter_mode": "token_size", + "delimiter_mode": "delimiter", "delimiters": [], "image_context_size": 0, "outputs": { @@ -429,7 +429,7 @@ "data": { "form": { "chunk_token_size": 512, - "delimiter_mode": "token_size", + "delimiter_mode": "delimiter", "delimiters": [ { "value": "\n" diff --git a/agent/templates/compiler.json b/agent/templates/compiler.json index 437524b766..3895f903eb 100644 --- a/agent/templates/compiler.json +++ b/agent/templates/compiler.json @@ -192,7 +192,7 @@ "params": { "children_delimiters": [], "chunk_token_size": 512, - "delimiter_mode": "token_size", + "delimiter_mode": "delimiter", "delimiters": [], "image_context_size": 0, "outputs": { @@ -374,7 +374,7 @@ "form": { "children_delimiters": [], "chunk_token_size": 512, - "delimiter_mode": "token_size", + "delimiter_mode": "delimiter", "delimiters": [ { "value": "\n" diff --git a/internal/ingestion/component/chunker/token.go b/internal/ingestion/component/chunker/token.go index 93781081a1..8e5d14fa17 100644 --- a/internal/ingestion/component/chunker/token.go +++ b/internal/ingestion/component/chunker/token.go @@ -16,7 +16,7 @@ // SCOPE (honest) for token.go: // -// - WHITELIST: delimiter_mode ∈ {"token_size","delimiter"} (the +// - WHITELIST: delimiter_mode ∈ {"delimiter"} (the // single-chunk "one" behaviour moved to OneChunker in one.go). // chunk_token_size > 0, overlapped_percent accepts a [0,1) fraction or a // [0,90] percentage (normalized to [0,90] by normalizeOverlappedPercent, diff --git a/internal/ingestion/component/chunker/token_strict_cap_test.go b/internal/ingestion/component/chunker/token_strict_cap_test.go index eaf68acb74..9e2a339c03 100644 --- a/internal/ingestion/component/chunker/token_strict_cap_test.go +++ b/internal/ingestion/component/chunker/token_strict_cap_test.go @@ -125,7 +125,7 @@ func TestMergeByTokenSize_TextPathStrictCap(t *testing.T) { b.WriteString("\n\n") } comp, err := NewTokenChunker(map[string]any{ - "delimiter_mode": "token_size", + "delimiter_mode": "delimiter", "chunk_token_size": budget, }) if err != nil { @@ -159,7 +159,7 @@ func TestMergeByTokenSize_OversizedUnitStaysWhole(t *testing.T) { // run is one unit that still exceeds the budget and must stay whole. long := strings.TrimSpace(strings.Repeat("word ", 100)) comp, err := NewTokenChunker(map[string]any{ - "delimiter_mode": "token_size", + "delimiter_mode": "delimiter", "chunk_token_size": budget, }) if err != nil { @@ -196,7 +196,7 @@ func TestMergeByTokenSize_UnderCapNoOverflow(t *testing.T) { run := func(underCap bool) []map[string]any { comp, err := NewTokenChunker(map[string]any{ - "delimiter_mode": "token_size", + "delimiter_mode": "delimiter", "chunk_token_size": budget, "under_cap": underCap, }) @@ -250,7 +250,7 @@ func TestInvokeTextPayload_StrictCapEndToEnd(t *testing.T) { b.WriteByte('\n') } comp, err := NewTokenChunker(map[string]any{ - "delimiter_mode": "token_size", + "delimiter_mode": "delimiter", "chunk_token_size": budget, }) if err != nil { diff --git a/internal/ingestion/component/chunker/token_tag_test.go b/internal/ingestion/component/chunker/token_tag_test.go index 97ba086a21..bca95db810 100644 --- a/internal/ingestion/component/chunker/token_tag_test.go +++ b/internal/ingestion/component/chunker/token_tag_test.go @@ -25,7 +25,7 @@ import ( // the body text still carries coordinate markers. func TestTokenChunker_TextPath_StripsParserTags(t *testing.T) { c, err := NewTokenChunker(map[string]any{ - "delimiter_mode": "token_size", + "delimiter_mode": "delimiter", "chunk_token_size": 1000, "delimiters": []string{"\n"}, }) diff --git a/internal/ingestion/component/chunker/token_test.go b/internal/ingestion/component/chunker/token_test.go index 8da9c5d474..98ab935adb 100644 --- a/internal/ingestion/component/chunker/token_test.go +++ b/internal/ingestion/component/chunker/token_test.go @@ -145,7 +145,7 @@ func TestTokenChunker_DelimNeverStandaloneChunk(t *testing.T) { // token-size merge and emit >=1 chunk. func TestTokenChunker_InvokeTokenSize_FallbackToMerge(t *testing.T) { c, err := NewTokenChunker(map[string]any{ - "delimiter_mode": "token_size", + "delimiter_mode": "delimiter", "chunk_token_size": 50, "delimiters": []string{"`\n\n`"}, }) @@ -365,9 +365,9 @@ func TestTokenChunker_NewRejectsBadParam(t *testing.T) { }{ {"bad delimiter_mode", map[string]any{"delimiter_mode": "nope"}}, {"one delimiter_mode (use OneChunker)", map[string]any{"delimiter_mode": "one"}}, - {"zero chunk_token_size", map[string]any{"delimiter_mode": "token_size", "chunk_token_size": 0}}, - {"negative chunk_token_size", map[string]any{"delimiter_mode": "token_size", "chunk_token_size": -5}}, - {"negative table_context_size", map[string]any{"delimiter_mode": "token_size", "chunk_token_size": 50, "table_context_size": -1}}, + {"zero chunk_token_size", map[string]any{"delimiter_mode": "delimiter", "chunk_token_size": 0}}, + {"negative chunk_token_size", map[string]any{"delimiter_mode": "delimiter", "chunk_token_size": -5}}, + {"negative table_context_size", map[string]any{"delimiter_mode": "delimiter", "chunk_token_size": 50, "table_context_size": -1}}, } for _, tc := range cases { t.Run(tc.name, func(t *testing.T) { @@ -380,14 +380,14 @@ func TestTokenChunker_NewRejectsBadParam(t *testing.T) { // TestTokenChunker_NewAcceptsDefaults ensures the no-config // constructor returns a usable component with a working default -// delimiter_mode = "token_size". +// delimiter_mode = "delimiter". func TestTokenChunker_NewAcceptsDefaults(t *testing.T) { c, err := NewTokenChunker(nil) if err != nil { t.Fatalf("NewTokenChunker(nil): %v", err) } - if got := c.(*TokenChunkerComponent).param.DelimiterMode; got != "token_size" { - t.Errorf("default delimiter_mode = %q, want token_size", got) + if got := c.(*TokenChunkerComponent).param.DelimiterMode; got != "delimiter" { + t.Errorf("default delimiter_mode = %q, want delimiter", got) } } @@ -439,7 +439,7 @@ func TestTokenChunker_NewAcceptsPythonOverlappedRange(t *testing.T) { // percentages, including out-of-range inputs that Python clamps). for _, pct := range []float64{0, 0.1, 0.5, 15, 30, 50, 90, 95, -5} { conf := map[string]any{ - "delimiter_mode": "token_size", + "delimiter_mode": "delimiter", "chunk_token_size": 100, "overlapped_percent": pct, } @@ -535,7 +535,7 @@ func TestTokenChunker_NormalizesOverlappedPercent(t *testing.T) { for _, tc := range cases { t.Run(tc.name, func(t *testing.T) { c, err := NewTokenChunker(map[string]any{ - "delimiter_mode": "token_size", + "delimiter_mode": "delimiter", "chunk_token_size": 100, "overlapped_percent": tc.in, }) @@ -575,7 +575,7 @@ func TestTokenChunkerParam_ValidateOverlappedRange(t *testing.T) { for _, tc := range cases { t.Run(tc.name, func(t *testing.T) { p := schema.TokenChunkerParam{ - DelimiterMode: "token_size", + DelimiterMode: "delimiter", ChunkTokenSize: 100, OverlappedPercent: tc.in, } diff --git a/internal/ingestion/component/schema/chunker.go b/internal/ingestion/component/schema/chunker.go index ad9b9a421c..e920950de1 100644 --- a/internal/ingestion/component/schema/chunker.go +++ b/internal/ingestion/component/schema/chunker.go @@ -173,7 +173,7 @@ type ChunkerOutputs struct { type TokenChunkerParam struct { // DelimiterMode selects the chunking strategy. - // Allowed values: "token_size", "delimiter". + // Allowed value: "delimiter". // The single-chunk "one" behavior is provided by the separate // OneChunker component. DelimiterMode string `json:"delimiter_mode"` @@ -246,7 +246,7 @@ func (p TokenChunkerParam) MergeStrategy() MergeStrategy { // Defaults returns the Python default TokenChunkerParam. func (TokenChunkerParam) Defaults() TokenChunkerParam { return TokenChunkerParam{ - DelimiterMode: "token_size", + DelimiterMode: "delimiter", ChunkTokenSize: 512, Delimiters: []string{"\n"}, OverlappedPercent: 0, @@ -357,7 +357,7 @@ func (p *TokenChunkerParam) Validate() error { p.OverlappedPercent = math.Round(v * 100) } switch p.DelimiterMode { - case "token_size", "delimiter": + case "delimiter": default: return errInvalidValue{Field: "delimiter_mode", Value: p.DelimiterMode} } diff --git a/internal/ingestion/component/schema/schema_test.go b/internal/ingestion/component/schema/schema_test.go index 517513f951..d9559b0e66 100644 --- a/internal/ingestion/component/schema/schema_test.go +++ b/internal/ingestion/component/schema/schema_test.go @@ -278,8 +278,8 @@ func TestChunkerOutputsJSONRoundTrip(t *testing.T) { func TestTokenChunkerParamDefaults(t *testing.T) { p := TokenChunkerParam{}.Defaults() - if p.DelimiterMode != "token_size" { - t.Errorf("default delimiter_mode = %q, want token_size", p.DelimiterMode) + if p.DelimiterMode != "delimiter" { + t.Errorf("default delimiter_mode = %q, want delimiter", p.DelimiterMode) } if p.ChunkTokenSize != 512 { t.Errorf("default chunk_token_size = %d, want 512", p.ChunkTokenSize) diff --git a/internal/ingestion/pipeline/template/ingestion_pipeline_email.json b/internal/ingestion/pipeline/template/ingestion_pipeline_email.json index 805a5df922..09c58c291e 100644 --- a/internal/ingestion/pipeline/template/ingestion_pipeline_email.json +++ b/internal/ingestion/pipeline/template/ingestion_pipeline_email.json @@ -83,7 +83,7 @@ "params": { "children_delimiters": [], "chunk_token_size": 512, - "delimiter_mode": "token_size", + "delimiter_mode": "delimiter", "delimiters": [ "\n", "!", @@ -265,7 +265,7 @@ "form": { "children_delimiters": [], "chunk_token_size": 512, - "delimiter_mode": "token_size", + "delimiter_mode": "delimiter", "delimiters": [ { "value": "\n" diff --git a/internal/ingestion/pipeline/template/ingestion_pipeline_general.json b/internal/ingestion/pipeline/template/ingestion_pipeline_general.json index ff4a3165ba..22d0d6fe7e 100644 --- a/internal/ingestion/pipeline/template/ingestion_pipeline_general.json +++ b/internal/ingestion/pipeline/template/ingestion_pipeline_general.json @@ -161,7 +161,7 @@ "params": { "children_delimiters": [], "chunk_token_size": 512, - "delimiter_mode": "token_size", + "delimiter_mode": "delimiter", "delimiters": [ "\n", "!", @@ -391,7 +391,7 @@ "form": { "children_delimiters": [], "chunk_token_size": 512, - "delimiter_mode": "token_size", + "delimiter_mode": "delimiter", "delimiters": [ { "value": "\n" diff --git a/internal/ingestion/pipeline/template/ingestion_pipeline_knowledge_compiler.json b/internal/ingestion/pipeline/template/ingestion_pipeline_knowledge_compiler.json index 364bc66b09..db0ab19d38 100644 --- a/internal/ingestion/pipeline/template/ingestion_pipeline_knowledge_compiler.json +++ b/internal/ingestion/pipeline/template/ingestion_pipeline_knowledge_compiler.json @@ -157,7 +157,7 @@ "params": { "children_delimiters": [], "chunk_token_size": 512, - "delimiter_mode": "token_size", + "delimiter_mode": "delimiter", "delimiters": [ "\n", "!", diff --git a/internal/ingestion/task/pipeline_executor_defaults_test.go b/internal/ingestion/task/pipeline_executor_defaults_test.go index b1eca5024f..fc483af648 100644 --- a/internal/ingestion/task/pipeline_executor_defaults_test.go +++ b/internal/ingestion/task/pipeline_executor_defaults_test.go @@ -40,8 +40,8 @@ import ( var builtinComponentParamsGolden = map[string]string{ "audio": "{\"File\": {}, \"Parser:SongsFillAir\": {\"audio\": {\"output_format\": \"text\", \"preprocess\": [\"main_content\"], \"suffix\": [\"aac\", \"aiff\", \"ape\", \"au\", \"da\", \"flac\", \"midi\", \"mp3\", \"ogg\", \"oggvorbis\", \"realaudio\", \"vqf\", \"wav\", \"wave\", \"wma\"]}}, \"TokenChunker:BlueSkiesLaugh\": {}, \"Tokenizer:KindEyesWatch\": {\"fields\": \"text\", \"filename_embd_weight\": 0.1, \"search_method\": [\"embedding\", \"full_text\"]}, \"Extractor:AutoExtractDefault\": {\"field_name\": \"\", \"auto_keywords\": 0, \"auto_questions\": 0, \"llm_id\": \"\", \"auto_tags\": 0, \"enable_metadata\": 0, \"metadata\": [], \"tag_file_id\": \"\"}}", "book": "{\"File\": {}, \"Parser:HipSignsRhyme\": {\"doc\": {\"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"doc\"]}, \"docx\": {\"flatten_media_to_text\": false, \"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"docx\"], \"vlm\": {}}, \"html\": {\"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"htm\", \"html\"]}, \"pdf\": {\"flatten_media_to_text\": false, \"output_format\": \"json\", \"parse_method\": \"DeepDOC\", \"preprocess\": [\"main_content\"], \"remove_toc\": true, \"suffix\": [\"pdf\"], \"vlm\": {}}, \"text&code\": {\"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"txt\"]}}, \"TitleChunker:GrumpyGarlicsBake\": {\"hierarchy\": 5, \"include_heading_content\": true, \"levels\": [[\"^#[^#]\", \"^##[^#]\", \"^###[^#]\", \"^####[^#]\"], [\"第[零一二三四五六七八九十百0-9]+(分?编|部分)\", \"第[零一二三四五六七八九十百0-9]+章\", \"第[零一二三四五六七八九十百0-9]+节\", \"第[零一二三四五六七八九十百0-9]+条\", \"[\\\\(（][零一二三四五六七八九十百]+[\\\\)）]\"], [\"第[0-9]+章\", \"第[0-9]+节\", \"[0-9]{1,2}[\\\\. 、]\", \"[0-9]{1,2}\\\\.[0-9]{1,2}($|[^a-zA-Z/%~.-])\", \"[0-9]{1,2}\\\\.[0-9]{1,2}\\\\.[0-9]{1,2}\"], [\"第[零一二三四五六七八九十百0-9]+章\", \"第[零一二三四五六七八九十百0-9]+节\", \"[零一二三四五六七八九十百]+[ 、]\", \"[\\\\(（][零一二三四五六七八九十百]+[\\\\)）]\", \"[\\\\(（][0-9]{,2}[\\\\)）]\"], [\"PART (ONE|TWO|THREE|FOUR|FIVE|SIX|SEVEN|EIGHT|NINE|TEN)\", \"Chapter (I+V?|VI*|XI|IX|X)\", \"Section [0-9]+\", \"Article [0-9]+\"]], \"method\": \"hierarchy\"}, \"Tokenizer:HotDonutsRing\": {\"fields\": \"text\", \"filename_embd_weight\": 0.1, \"search_method\": [\"embedding\", \"full_text\"]}, \"Extractor:AutoExtractDefault\": {\"field_name\": \"\", \"auto_keywords\": 0, \"auto_questions\": 0, \"llm_id\": \"\", \"auto_tags\": 0, \"enable_metadata\": 0, \"metadata\": [], \"tag_file_id\": \"\"}}", - "email": "{\"File\": {}, \"Parser:BirdsFlutterHigh\": {\"email\": {\"fields\": [\"from\", \"to\", \"cc\", \"bcc\", \"date\", \"subject\", \"body\", \"attachments\"], \"output_format\": \"text\", \"preprocess\": [\"main_content\"], \"suffix\": [\"eml\"]}}, \"TokenChunker:WarmBreadSmells\": {\"children_delimiters\": [], \"chunk_token_size\": 512, \"delimiter_mode\": \"token_size\", \"delimiters\": [\"\\n\", \"!\", \"?\", \"。\", \"；\", \"！\", \"？\"], \"image_context_size\": 0, \"overlapped_percent\": 0, \"table_context_size\": 0}, \"Tokenizer:NiceWordsSpoken\": {\"fields\": \"text\", \"filename_embd_weight\": 0.1, \"search_method\": [\"embedding\", \"full_text\"]}, \"Extractor:AutoExtractDefault\": {\"field_name\": \"\", \"auto_keywords\": 0, \"auto_questions\": 0, \"llm_id\": \"\", \"auto_tags\": 0, \"enable_metadata\": 0, \"metadata\": [], \"tag_file_id\": \"\"}}", - "general": "{\"File\": {}, \"Parser:HipSignsRhyme\": {\"doc\": {\"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"doc\"]}, \"docx\": {\"flatten_media_to_text\": false, \"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"docx\"], \"vlm\": {}}, \"html\": {\"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"htm\", \"html\"]}, \"markdown\": {\"flatten_media_to_text\": false, \"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"md\", \"markdown\", \"mdx\"], \"vlm\": {}}, \"pdf\": {\"flatten_media_to_text\": false, \"output_format\": \"json\", \"pages\": [[1, 100000]], \"parse_method\": \"DeepDOC\", \"preprocess\": [\"main_content\"], \"suffix\": [\"pdf\"], \"vlm\": {}}, \"spreadsheet\": {\"flatten_media_to_text\": false, \"output_format\": \"html\", \"parse_method\": \"DeepDOC\", \"preprocess\": [\"main_content\"], \"suffix\": [\"xls\", \"xlsx\", \"csv\"], \"vlm\": {}}, \"text&code\": {\"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"txt\", \"py\", \"js\", \"java\", \"c\", \"cpp\", \"h\", \"php\", \"go\", \"ts\", \"sh\", \"cs\", \"kt\", \"sql\"]}}, \"TokenChunker:SixApplesFall\": {\"children_delimiters\": [], \"chunk_token_size\": 512, \"delimiter_mode\": \"token_size\", \"delimiters\": [\"\\n\", \"!\", \"?\", \"。\", \"；\", \"！\", \"？\"], \"image_context_size\": 0, \"overlapped_percent\": 0, \"table_context_size\": 0}, \"Tokenizer:LegalReadersDecide\": {\"fields\": \"text\", \"filename_embd_weight\": 0.1, \"search_method\": [\"embedding\", \"full_text\"]}, \"Extractor:AutoExtractDefault\": {\"field_name\": \"\", \"auto_keywords\": 0, \"auto_questions\": 0, \"llm_id\": \"\", \"auto_tags\": 0, \"enable_metadata\": 0, \"metadata\": [], \"tag_file_id\": \"\"}}", + "email": "{\"File\": {}, \"Parser:BirdsFlutterHigh\": {\"email\": {\"fields\": [\"from\", \"to\", \"cc\", \"bcc\", \"date\", \"subject\", \"body\", \"attachments\"], \"output_format\": \"text\", \"preprocess\": [\"main_content\"], \"suffix\": [\"eml\"]}}, \"TokenChunker:WarmBreadSmells\": {\"children_delimiters\": [], \"chunk_token_size\": 512, \"delimiter_mode\": \"delimiter\", \"delimiters\": [\"\\n\", \"!\", \"?\", \"。\", \"；\", \"！\", \"？\"], \"image_context_size\": 0, \"overlapped_percent\": 0, \"table_context_size\": 0}, \"Tokenizer:NiceWordsSpoken\": {\"fields\": \"text\", \"filename_embd_weight\": 0.1, \"search_method\": [\"embedding\", \"full_text\"]}, \"Extractor:AutoExtractDefault\": {\"field_name\": \"\", \"auto_keywords\": 0, \"auto_questions\": 0, \"llm_id\": \"\", \"auto_tags\": 0, \"enable_metadata\": 0, \"metadata\": [], \"tag_file_id\": \"\"}}", + "general": "{\"File\": {}, \"Parser:HipSignsRhyme\": {\"doc\": {\"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"doc\"]}, \"docx\": {\"flatten_media_to_text\": false, \"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"docx\"], \"vlm\": {}}, \"html\": {\"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"htm\", \"html\"]}, \"markdown\": {\"flatten_media_to_text\": false, \"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"md\", \"markdown\", \"mdx\"], \"vlm\": {}}, \"pdf\": {\"flatten_media_to_text\": false, \"output_format\": \"json\", \"pages\": [[1, 100000]], \"parse_method\": \"DeepDOC\", \"preprocess\": [\"main_content\"], \"suffix\": [\"pdf\"], \"vlm\": {}}, \"spreadsheet\": {\"flatten_media_to_text\": false, \"output_format\": \"html\", \"parse_method\": \"DeepDOC\", \"preprocess\": [\"main_content\"], \"suffix\": [\"xls\", \"xlsx\", \"csv\"], \"vlm\": {}}, \"text&code\": {\"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"txt\", \"py\", \"js\", \"java\", \"c\", \"cpp\", \"h\", \"php\", \"go\", \"ts\", \"sh\", \"cs\", \"kt\", \"sql\"]}}, \"TokenChunker:SixApplesFall\": {\"children_delimiters\": [], \"chunk_token_size\": 512, \"delimiter_mode\": \"delimiter\", \"delimiters\": [\"\\n\", \"!\", \"?\", \"。\", \"；\", \"！\", \"？\"], \"image_context_size\": 0, \"overlapped_percent\": 0, \"table_context_size\": 0}, \"Tokenizer:LegalReadersDecide\": {\"fields\": \"text\", \"filename_embd_weight\": 0.1, \"search_method\": [\"embedding\", \"full_text\"]}, \"Extractor:AutoExtractDefault\": {\"field_name\": \"\", \"auto_keywords\": 0, \"auto_questions\": 0, \"llm_id\": \"\", \"auto_tags\": 0, \"enable_metadata\": 0, \"metadata\": [], \"tag_file_id\": \"\"}}", "laws": "{\"File\": {}, \"Parser:HipSignsRhyme\": {\"doc\": {\"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"doc\"]}, \"docx\": {\"flatten_media_to_text\": false, \"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"docx\"], \"vlm\": {}}, \"html\": {\"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"htm\", \"html\"]}, \"markdown\": {\"flatten_media_to_text\": false, \"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"md\", \"markdown\", \"mdx\"], \"vlm\": {}}, \"pdf\": {\"flatten_media_to_text\": false, \"output_format\": \"json\", \"parse_method\": \"DeepDOC\", \"preprocess\": [\"main_content\"], \"remove_toc\": true, \"suffix\": [\"pdf\"], \"vlm\": {}}, \"text&code\": {\"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"txt\"]}}, \"TitleChunker:SpicyKeysKick\": {\"hierarchy\": 2, \"include_heading_content\": false, \"levels\": [[\"^#[^#]\", \"^##[^#]\", \"^###[^#]\", \"^####[^#]\"], [\"第[零一二三四五六七八九十百0-9]+(分?编|部分)\", \"第[零一二三四五六七八九十百0-9]+章\", \"第[零一二三四五六七八九十百0-9]+节\", \"第[零一二三四五六七八九十百0-9]+条\", \"[\\\\(（][零一二三四五六七八九十百]+[\\\\)）]\"], [\"第[0-9]+章\", \"第[0-9]+节\", \"[0-9]{1,2}[\\\\. 、]\", \"[0-9]{1,2}\\\\.[0-9]{1,2}($|[^a-zA-Z/%~.-])\", \"[0-9]{1,2}\\\\.[0-9]{1,2}\\\\.[0-9]{1,2}\"], [\"第[零一二三四五六七八九十百0-9]+章\", \"第[零一二三四五六七八九十百0-9]+节\", \"[零一二三四五六七八九十百]+[ 、]\", \"[\\\\(（][零一二三四五六七八九十百]+[\\\\)）]\", \"[\\\\(（][0-9]{,2}[\\\\)）]\"], [\"PART (ONE|TWO|THREE|FOUR|FIVE|SIX|SEVEN|EIGHT|NINE|TEN)\", \"Chapter (I+V?|VI*|XI|IX|X)\", \"Section [0-9]+\", \"Article [0-9]+\"]], \"method\": \"hierarchy\"}, \"Tokenizer:PublicJobsTake\": {\"fields\": \"text\", \"filename_embd_weight\": 0.1, \"search_method\": [\"embedding\", \"full_text\"]}, \"Extractor:AutoExtractDefault\": {\"field_name\": \"\", \"auto_keywords\": 0, \"auto_questions\": 0, \"llm_id\": \"\", \"auto_tags\": 0, \"enable_metadata\": 0, \"metadata\": [], \"tag_file_id\": \"\"}}", "manual": "{\"File\": {}, \"Parser:HipSignsRhyme\": {\"doc\": {\"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"doc\"]}, \"docx\": {\"flatten_media_to_text\": false, \"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"docx\"], \"vlm\": {}}, \"pdf\": {\"flatten_media_to_text\": false, \"output_format\": \"json\", \"parse_method\": \"DeepDOC\", \"preprocess\": [\"main_content\"], \"suffix\": [\"pdf\"], \"vlm\": {}}}, \"TitleChunker:NineInsectsFind\": {\"hierarchy\": 0, \"include_heading_content\": false, \"levels\": [[\"^#[^#]\", \"^##[^#]\", \"^###[^#]\", \"^####[^#]\"], [\"第[零一二三四五六七八九十百0-9]+(分?编|部分)\", \"第[零一二三四五六七八九十百0-9]+章\", \"第[零一二三四五六七八九十百0-9]+节\", \"第[零一二三四五六七八九十百0-9]+条\", \"[\\\\(（][零一二三四五六七八九十百]+[\\\\)）]\"], [\"第[0-9]+章\", \"第[0-9]+节\", \"[0-9]{1,2}[\\\\. 、]\", \"[0-9]{1,2}\\\\.[0-9]{1,2}($|[^a-zA-Z/%~.-])\", \"[0-9]{1,2}\\\\.[0-9]{1,2}\\\\.[0-9]{1,2}\"], [\"第[零一二三四五六七八九十百0-9]+章\", \"第[零一二三四五六七八九十百0-9]+节\", \"[零一二三四五六七八九十百]+[ 、]\", \"[\\\\(（][零一二三四五六七八九十百]+[\\\\)）]\", \"[\\\\(（][0-9]{,2}[\\\\)）]\"], [\"PART (ONE|TWO|THREE|FOUR|FIVE|SIX|SEVEN|EIGHT|NINE|TEN)\", \"Chapter (I+V?|VI*|XI|IX|X)\", \"Section [0-9]+\", \"Article [0-9]+\"]], \"method\": \"group\"}, \"Tokenizer:FunnyBalloonsGrin\": {\"fields\": \"text\", \"filename_embd_weight\": 0.1, \"search_method\": [\"embedding\", \"full_text\"]}, \"Extractor:AutoExtractDefault\": {\"field_name\": \"\", \"auto_keywords\": 0, \"auto_questions\": 0, \"llm_id\": \"\", \"auto_tags\": 0, \"enable_metadata\": 0, \"metadata\": [], \"tag_file_id\": \"\"}}", "one": "{\"File\": {}, \"Parser:HipSignsRhyme\": {\"doc\": {\"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"doc\"]}, \"docx\": {\"flatten_media_to_text\": false, \"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"docx\"], \"vlm\": {}}, \"html\": {\"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"htm\", \"html\"]}, \"markdown\": {\"flatten_media_to_text\": false, \"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"md\", \"markdown\", \"mdx\"], \"vlm\": {}}, \"pdf\": {\"flatten_media_to_text\": false, \"output_format\": \"json\", \"parse_method\": \"DeepDOC\", \"preprocess\": [\"main_content\"], \"suffix\": [\"pdf\"], \"vlm\": {}}, \"spreadsheet\": {\"flatten_media_to_text\": false, \"output_format\": \"html\", \"parse_method\": \"DeepDOC\", \"preprocess\": [\"main_content\"], \"suffix\": [\"xls\", \"xlsx\"], \"vlm\": {}}, \"text&code\": {\"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"txt\"]}}, \"OneChunker:DryDrinksVisit\": {}, \"Tokenizer:FrankWeeksListen\": {\"fields\": \"text\", \"filename_embd_weight\": 0.1, \"search_method\": [\"embedding\", \"full_text\"]}, \"Extractor:AutoExtractDefault\": {\"field_name\": \"\", \"auto_keywords\": 0, \"auto_questions\": 0, \"llm_id\": \"\", \"auto_tags\": 0, \"enable_metadata\": 0, \"metadata\": [], \"tag_file_id\": \"\"}}", @@ -51,7 +51,7 @@ var builtinComponentParamsGolden = map[string]string{ "qa": "{\"File\": {}, \"Parser:HipSignsRhyme\": {\"docx\": {\"flatten_media_to_text\": false, \"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"docx\"], \"vlm\": {}}, \"markdown\": {\"flatten_media_to_text\": false, \"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"md\", \"markdown\", \"mdx\"], \"vlm\": {}}, \"pdf\": {\"flatten_media_to_text\": false, \"output_format\": \"json\", \"parse_method\": \"DeepDOC\", \"preprocess\": [\"main_content\"], \"suffix\": [\"pdf\"], \"vlm\": {}}, \"spreadsheet\": {\"flatten_media_to_text\": false, \"output_format\": \"html\", \"parse_method\": \"DeepDOC\", \"preprocess\": [\"main_content\"], \"suffix\": [\"xls\", \"xlsx\", \"csv\"], \"vlm\": {}}, \"text&code\": {\"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"txt\"]}}, \"Tokenizer:ColdCloudsDream\": {\"fields\": \"text\", \"filename_embd_weight\": 0.1, \"search_method\": [\"embedding\", \"full_text\"]}, \"QAChunker:TidyCloudsThink\": {}}", "resume": "{\"Extractor:ThreeDrinksAct\": {\"field_name\": \"metadata\", \"frequencyPenaltyEnabled\": true, \"frequency_penalty\": 0.7, \"llm_id\": \"THUDM/GLM-4.1V-9B-Thinking@SILICONFLOW\", \"maxTokensEnabled\": false, \"max_tokens\": 256, \"presencePenaltyEnabled\": true, \"presence_penalty\": 0.4, \"prompts\": [{\"content\": \"Content: {TitleChunker:FlatMiceFix@chunks}\", \"role\": \"user\"}], \"sys_prompt\": \"Act as a precise resume metadata extractor. Extract stable, chunk-supported metadata from the provided resume content.\\n\\nRules:\\n1. Use only information explicitly stated in the content. Do not infer, guess, normalize, or add missing facts.\\n2. The input may be only one chunk of a resume. Extract only what this content directly supports.\\n3. Use only these field names:\\ncandidate_name, gender, phone, email, city, location, nationality, linkedin, github, website, highest_degree, degree_levels, school_names, majors, graduation_years, work_experience_years, current_job_title, job_titles, company_names, job_experience, industries, target_job_titles, target_locations, employment_types, skills, certificates, awards, summary_tags\\n4. Ignore detailed responsibilities, project descriptions, achievement narratives, self-evaluation, and other low-value local details.\\n5. Keep values in the same language as the source text whenever possible.\\n6. Remove duplicates and keep only concise, high-value metadata.\\n7. Return only fields that are explicitly supported by the content. Do not return empty or unsupported fields.\\n\\nField guidance:\\n- highest_degree: highest explicit degree level mentioned\\n- degree_levels: all explicit degree levels mentioned\\n- school_names: explicit school, college, or university names\\n- majors: explicit fields of study\\n- graduation_years: explicit graduation years only\\n- work_experience_years: only if explicitly stated\\n- current_job_title: only if explicitly current or most recent\\n- job_titles: explicit role titles\\n- company_names: explicit employer names\\n- job_experience: concise structured work entries explicitly supported by the content, preferably including title, company, and time information when available\\n- industries: explicit industry names only\\n- target_job_titles: explicit desired roles only\\n- target_locations: explicit desired work locations only\\n- skills: concise, core, search-useful skills explicitly mentioned\\n- certificates: explicit certificate names only\\n- awards: explicit award names only\\n- summary_tags: short, high-value tags strictly supported by the content\\n\\nReturn only the extracted metadata. Do not output explanatory text.\", \"temperature\": 0.1, \"temperatureEnabled\": true, \"tenant_llm_id\": 29, \"topPEnabled\": true, \"top_p\": 0.3, \"auto_keywords\": 0, \"auto_questions\": 0, \"auto_tags\": 0, \"enable_metadata\": 0, \"metadata\": [], \"tag_file_id\": \"\"}, \"File\": {}, \"Parser:HipSignsRhyme\": {\"docx\": {\"flatten_media_to_text\": true, \"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"docx\"], \"vlm\": {}}, \"pdf\": {\"flatten_media_to_text\": true, \"output_format\": \"json\", \"parse_method\": \"DeepDOC\", \"preprocess\": [\"main_content\"], \"suffix\": [\"pdf\"], \"vlm\": {}}, \"text&code\": {\"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"txt\"]}}, \"TitleChunker:FlatMiceFix\": {\"hierarchy\": 1, \"include_heading_content\": false, \"levels\": [[\"^\\\\s*(?i:(?:\\\\d+[\\\\.\\\\)]\\\\s*)?(?:EDUCATION|ACADEMIC\\\\s*BACKGROUND|ACADEMIC\\\\s*HISTORY|EDUCATIONAL\\\\s*BACKGROUND|RELEVANT\\\\s*COURSEWORK|COURSEWORK|EXPERIENCE|WORK\\\\s*EXPERIENCE|PROFESSIONAL\\\\s*EXPERIENCE|RELEVANT\\\\s*EXPERIENCE|EMPLOYMENT\\\\s*HISTORY|CAREER\\\\s*HISTORY|INTERNSHIP\\\\s*EXPERIENCE|PROJECTS|PROJECT\\\\s*EXPERIENCE|ACADEMIC\\\\s*PROJECTS|PROFESSIONAL\\\\s*PROJECTS|SKILLS|TECHNICAL\\\\s*SKILLS|CORE\\\\s*COMPETENCIES|COMPETENCIES|QUALIFICATIONS|SUMMARY\\\\s*OF\\\\s*QUALIFICATIONS|CERTIFICATIONS|LICENSES|CERTIFICATES|AWARDS|HONORS|HONOURS|ACHIEVEMENTS|PUBLICATIONS|RESEARCH|RESEARCH\\\\s*EXPERIENCE|LEADERSHIP|LEADERSHIP\\\\s*EXPERIENCE|ACTIVITIES|EXTRACURRICULAR\\\\s*ACTIVITIES|ACTIVITIES\\\\s*(?:&|AND)\\\\s*SKILLS|INVOLVEMENT|CAMPUS\\\\s*INVOLVEMENT|VOLUNTEER\\\\s*EXPERIENCE|VOLUNTEERING|COMMUNITY\\\\s*SERVICE|LANGUAGES|INTERESTS|HOBBIES|PROFILE|PROFESSIONAL\\\\s*PROFILE|SUMMARY|PROFESSIONAL\\\\s*SUMMARY|CAREER\\\\s*SUMMARY|OBJECTIVE|CAREER\\\\s*OBJECTIVE|PERSONAL\\\\s*INFORMATION|CONTACT\\\\s*INFORMATION|ADDITIONAL\\\\s*INFORMATION|TRAINING))\\\\s*[:：]?\\\\s*$\"], [\"^\\\\s*(?:\\\\d+[\\\\.、\\\\)]\\\\s*)?(?:教育背景|教育经历|学历背景|学术背景|技术背景|工作经历|工作经验|实习经历|项目经历|项目经验|科研经历|研究经历|校园经历|实践经历|专业经历|职业经历|技能|专业技能|技能特长|核心技能|技术栈|个人技能|工作技能|职业技能|技能与评价|技能与自我评价|工作技能与自我评价|职业技能与自我评价|证书|资格证书|职业资格|资质证书|获奖情况|获奖经历|荣誉|荣誉奖项|奖项|科研成果|论文发表|发表论文|领导经历|学生工作|校园活动|社团经历|活动经历|志愿经历|志愿服务|社会实践|语言能力|语言|自我评价|个人评价|自我总结|个人总结|个人优势|个人简介|个人信息|基本信息|联系方式|求职意向|应聘意向|职业目标|求职目标|兴趣爱好|兴趣特长|培训经历|其他信息|附加信息)\\\\s*[:：]?\\\\s*$\"]], \"method\": \"hierarchy\"}, \"Tokenizer:KindHandsWin\": {\"fields\": \"text\", \"filename_embd_weight\": 0.1, \"search_method\": [\"embedding\", \"full_text\"]}}", "table": "{\"File\": {}, \"Parser:HipSignsRhyme\": {\"spreadsheet\": {\"flatten_media_to_text\": false, \"output_format\": \"html\", \"parse_method\": \"DeepDOC\", \"preprocess\": [\"main_content\"], \"suffix\": [\"xls\", \"xlsx\", \"csv\"], \"vlm\": {}, \"column_mode\": \"auto\", \"column_roles\": {}, \"column_names\": []}, \"text&code\": {\"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"txt\"]}}, \"TableChunker:FastFoxesJump\": {}, \"Tokenizer:DeepLakesShine\": {\"fields\": \"text\", \"filename_embd_weight\": 0.1, \"search_method\": [\"embedding\", \"full_text\"]}}", - "knowledge_compiler": "{\"File\": {}, \"Compiler:KnownSwiftLions\": {\"language\": \"English\", \"llm_id\": \"\", \"variant\": \"structure\"}, \"Parser:HipSignsRhyme\": {\"setups\": {\"doc\": {\"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"doc\"]}, \"docx\": {\"flatten_media_to_text\": false, \"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"docx\"], \"vlm\": {}}, \"html\": {\"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"htm\", \"html\"]}, \"markdown\": {\"flatten_media_to_text\": false, \"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"md\", \"markdown\", \"mdx\"], \"vlm\": {}}, \"pdf\": {\"flatten_media_to_text\": false, \"output_format\": \"json\", \"parse_method\": \"DeepDOC\", \"preprocess\": [\"main_content\"], \"suffix\": [\"pdf\"], \"vlm\": {}}, \"spreadsheet\": {\"flatten_media_to_text\": false, \"output_format\": \"html\", \"parse_method\": \"DeepDOC\", \"preprocess\": [\"main_content\"], \"suffix\": [\"xls\", \"xlsx\", \"csv\"], \"vlm\": {}}, \"text&code\": {\"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"txt\", \"py\", \"js\", \"java\", \"c\", \"cpp\", \"h\", \"php\", \"go\", \"ts\", \"sh\", \"cs\", \"kt\", \"sql\"]}}}, \"TokenChunker:SixApplesFall\": {\"children_delimiters\": [], \"chunk_token_size\": 512, \"delimiter_mode\": \"token_size\", \"delimiters\": [\"\\n\", \"!\", \"?\", \"。\", \"；\", \"！\", \"？\"], \"image_context_size\": 0, \"overlapped_percent\": 0, \"table_context_size\": 0}}", + "knowledge_compiler": "{\"File\": {}, \"Compiler:KnownSwiftLions\": {\"language\": \"English\", \"llm_id\": \"\", \"variant\": \"structure\"}, \"Parser:HipSignsRhyme\": {\"setups\": {\"doc\": {\"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"doc\"]}, \"docx\": {\"flatten_media_to_text\": false, \"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"docx\"], \"vlm\": {}}, \"html\": {\"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"htm\", \"html\"]}, \"markdown\": {\"flatten_media_to_text\": false, \"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"md\", \"markdown\", \"mdx\"], \"vlm\": {}}, \"pdf\": {\"flatten_media_to_text\": false, \"output_format\": \"json\", \"parse_method\": \"DeepDOC\", \"preprocess\": [\"main_content\"], \"suffix\": [\"pdf\"], \"vlm\": {}}, \"spreadsheet\": {\"flatten_media_to_text\": false, \"output_format\": \"html\", \"parse_method\": \"DeepDOC\", \"preprocess\": [\"main_content\"], \"suffix\": [\"xls\", \"xlsx\", \"csv\"], \"vlm\": {}}, \"text&code\": {\"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"txt\", \"py\", \"js\", \"java\", \"c\", \"cpp\", \"h\", \"php\", \"go\", \"ts\", \"sh\", \"cs\", \"kt\", \"sql\"]}}}, \"TokenChunker:SixApplesFall\": {\"children_delimiters\": [], \"chunk_token_size\": 512, \"delimiter_mode\": \"delimiter\", \"delimiters\": [\"\\n\", \"!\", \"?\", \"。\", \"；\", \"！\", \"？\"], \"image_context_size\": 0, \"overlapped_percent\": 0, \"table_context_size\": 0}}", } // Per-template test methods. Each resolves default component params from a diff --git a/rag/flow/chunker/token_chunker.py b/rag/flow/chunker/token_chunker.py index a9a9ff7a95..0fce2c18c8 100644 --- a/rag/flow/chunker/token_chunker.py +++ b/rag/flow/chunker/token_chunker.py @@ -32,7 +32,7 @@ from rag.nlp import naive_merge class TokenChunkerParam(ProcessParamBase): def __init__(self): super().__init__() - self.delimiter_mode = "token_size" + self.delimiter_mode = "delimiter" self.chunk_token_size = 512 self.delimiters = ["\n"] self.overlapped_percent = 0 @@ -41,7 +41,13 @@ class TokenChunkerParam(ProcessParamBase): self.image_context_size = 0 def check(self): - self.check_valid_value(self.delimiter_mode, "Delimiter mode abnormal.", ["token_size", "delimiter", "one"]) + # Backward-compat: "token_size" was removed but is behaviorally identical + # to "delimiter" at runtime (both route through the same code path), so + # accept and coerce it instead of rejecting legacy configs / pre-fix + # frontends. Only genuinely unknown values are rejected. + if self.delimiter_mode == "token_size": + self.delimiter_mode = "delimiter" + self.check_valid_value(self.delimiter_mode, "Delimiter mode abnormal.", ["delimiter", "one"]) if self.delimiters is None: self.delimiters = [] elif isinstance(self.delimiters, str): @@ -316,9 +322,7 @@ class TokenChunker(ProcessBase): self.set_output("chunks", [{"text": payload}] if payload.strip() else []) self.callback(1, "Done.") return - if self._param.delimiter_mode == "delimiter": - cks = _split_text_by_pattern(payload, delimiter_pattern) - elif delimiter_pattern: + if delimiter_pattern: cks = _split_text_by_pattern(payload, delimiter_pattern) else: cks = naive_merge( @@ -358,8 +362,14 @@ class TokenChunker(ProcessBase): self.callback(1, "Done.") return - if self._param.delimiter_mode == "delimiter": - text_chunks = _build_json_chunks(json_result, "") + # Both branches start from per-item chunks (no pre-split by the + # delimiter pattern). The delimiter branch splits the buffered text + # stream while preserving per-segment PDF positions; the no-delimiter + # branch merges adjacent text items to chunk_token_size (the removed + # "token_size" behaviour, and a parity match with the Go JSON path). + text_chunks = _build_json_chunks(json_result, "") + + if delimiter_pattern: chunks = [] text_buffer = [] text_buffer_pos = [] @@ -372,9 +382,9 @@ class TokenChunker(ProcessBase): # The delimiter is then applied to the combined text; a segment may # span across item boundaries (the "\n" glue is not itself a # delimiter), so each segment carries only the PDF positions of the - # buffered item(s) whose text contributed to it -- never the union of - # every item (which previously leaked page-N coordinates into - # page-M chunks and made all segments share one preview image). + # item(s) that contributed to it -- never the union of every item + # (which previously leaked page-N coordinates into page-M chunks and + # made all segments share one preview image). parts = [] item_ranges = [] # (start, end) of each buffered item in combined_text offset = 0 @@ -387,27 +397,24 @@ class TokenChunker(ProcessBase): offset += 1 combined_text = "".join(parts[:-1]) # drop the trailing glue - if delimiter_pattern: - raw = re.split(r"(%s)" % delimiter_pattern, combined_text, flags=re.DOTALL) - segments = [] # (text, start, end) within combined_text - pos = 0 - for i in range(0, len(raw), 2): - seg = raw[i] - seg_start = pos - seg_end = pos + len(seg) - if seg: - segments.append((seg, seg_start, seg_end)) - pos = seg_end - if i + 1 < len(raw): - pos += len(raw[i + 1]) - else: - segments = [(combined_text, 0, len(combined_text))] + raw = re.split(r"(%s)" % delimiter_pattern, combined_text, flags=re.DOTALL) + segments = [] # (text, start, end) within combined_text + pos = 0 + for i in range(0, len(raw), 2): + seg = raw[i] + seg_start = pos + seg_end = pos + len(seg) + if seg: + segments.append((seg, seg_start, seg_end)) + pos = seg_end + if i + 1 < len(raw): + pos += len(raw[i + 1]) for text, seg_start, seg_end in segments: if not text.strip(): continue seg_pos = [] - for (istart, iend), item_pos in zip(item_ranges, text_buffer_pos): + for (istart, iend), item_pos in zip(item_ranges, text_buffer_pos, strict=True): # A segment overlaps an item when their character ranges # intersect; collect that item's coordinates. if seg_start < iend and istart < seg_end: @@ -436,22 +443,19 @@ class TokenChunker(ProcessBase): if custom_pattern: chunks = _split_chunk_docs_by_children(chunks, custom_pattern) _attach_context_to_media_chunks(chunks, self._param.table_context_size, self._param.image_context_size) - await restore_pdf_text_previews(chunks, from_upstream, self._canvas) - self.set_output("chunks", _finalize_json_chunks(chunks)) - self.callback(1, "Done.") - return - - # Structured JSON input is normalized first, then optionally enriched with - # media context, and finally merged only when delimiter splitting is inactive. - chunks = _build_json_chunks(json_result, delimiter_pattern) - _attach_context_to_media_chunks(chunks, self._param.table_context_size, self._param.image_context_size) - if self._param.delimiter_mode == "token_size" and not delimiter_pattern: - chunks = _merge_text_chunks_by_token_size(chunks, self._param.chunk_token_size, overlapped_percent) - - if custom_pattern: - chunks = _split_chunk_docs_by_children(chunks, custom_pattern) + else: + # No active delimiter: merge adjacent text items to chunk_token_size. + # This runs on the per-item chunks (NOT a single concatenated chunk), + # so the token cap is actually enforced -- matching the previous + # "token_size" mode and the Go JSON path. Media chunks break the merge. + # Media context is attached on the per-item chunks before merging, as + # the removed "token_size" branch did, to preserve context windows. + _attach_context_to_media_chunks(text_chunks, self._param.table_context_size, self._param.image_context_size) + chunks = _merge_text_chunks_by_token_size(text_chunks, self._param.chunk_token_size, overlapped_percent) + if custom_pattern: + chunks = _split_chunk_docs_by_children(chunks, custom_pattern) await restore_pdf_text_previews(chunks, from_upstream, self._canvas) - cks = _finalize_json_chunks(chunks) - self.set_output("chunks", cks) + self.set_output("chunks", _finalize_json_chunks(chunks)) self.callback(1, "Done.") + return diff --git a/rag/flow/tests/test_token_chunker.py b/rag/flow/tests/test_token_chunker.py index 8866b53de2..a33732a9c0 100644 --- a/rag/flow/tests/test_token_chunker.py +++ b/rag/flow/tests/test_token_chunker.py @@ -55,6 +55,19 @@ def _load_token_chunker_with_stubs(): def __init__(self): pass + def check_valid_value(self, value, msg, allowed): + if value not in allowed: + raise ValueError(msg) + + def check_positive_integer(self, value, msg): + pass + + def check_decimal_float(self, value, msg): + pass + + def check_nonnegative_number(self, value, msg): + pass + class ProcessBase: def __init__(self, _pipeline, _id, param): self._pipeline = _pipeline @@ -277,6 +290,73 @@ def test_json_delimiter_mode_pdf_positions_retained(): assert chunks[0].get("pdf_positions") == [[1, 0, 10, 0, 5], [2, 0, 20, 0, 8]] +def test_token_size_mode_normalized_to_delimiter(): + # Backward-compat: the removed "token_size" value must still be accepted by + # check() and coerced to "delimiter" (runtime behavior is identical), so + # legacy configs / pre-fix frontends don't get rejected. Unknown values are + # still rejected. + with _load_token_chunker_with_stubs() as token_chunker_module: + param = token_chunker_module.TokenChunkerParam() + param.delimiter_mode = "token_size" + param.check() + assert param.delimiter_mode == "delimiter" + + bad = token_chunker_module.TokenChunkerParam() + bad.delimiter_mode = "nope" + try: + bad.check() + raise AssertionError("expected check() to reject unknown delimiter_mode") + except Exception: + pass + + +def test_json_no_delimiter_mode_merges_to_token_cap(): + # Regression for #17979: with no active (backtick) delimiter, the JSON path + # must merge per-item text chunks up to chunk_token_size -- mirroring the old + # "token_size" mode and the Go JSON path. Concatenating every item into a + # single chunk before the merge would defeat the cap and emit one oversized + # chunk. The per-token stub (num_tokens_from_string -> 1) makes the cap easy + # to exceed: 12 one-token items under a cap of 5 must yield several chunks. + for _module, chunker in _build_json_chunker({"delimiter_mode": "delimiter", "delimiters": [], "chunk_token_size": 5}): + kwargs = { + "name": "token_chunker", + "output_format": "json", + "json_result": [{"text": f"item{i}"} for i in range(12)], + } + asyncio.run(chunker._invoke(**kwargs)) + chunks = chunker._outputs["chunks"] + # 12 one-token items under a cap of 5 must NOT collapse into one chunk. + assert len(chunks) > 1, f"cap not enforced: 12 items collapsed to {len(chunks)} chunk(s)" + # Each merged chunk holds at most one overflow unit past the cap (<= 6 + # items); the final output drops tk_nums, so count item markers instead. + for c in chunks: + assert c["text"].count("item") <= 6, f"chunk exceeds cap: {c['text'].count('item')} items" + # No text lost: all 12 items must survive, joined by the "\n" glue. + joined = "\n".join(c["text"] for c in chunks) + for i in range(12): + assert f"item{i}" in joined, f"item{i} dropped from output" + + +def test_json_no_delimiter_mode_media_breaks_merge(): + # A non-text (media) chunk interleaved between text items must stay as its + # own chunk and reset the merge, so text before/after it are sized separately. + for _module, chunker in _build_json_chunker({"delimiter_mode": "delimiter", "delimiters": [], "chunk_token_size": 5}): + kwargs = { + "name": "token_chunker", + "output_format": "json", + "json_result": [{"text": f"t{i}", "doc_type_kwd": "text"} for i in range(6)] + + [{"text": "IMG", "doc_type_kwd": "image", "img_id": "im1"}] + + [{"text": f"u{i}", "doc_type_kwd": "text"} for i in range(12)], + } + asyncio.run(chunker._invoke(**kwargs)) + chunks = chunker._outputs["chunks"] + doc_types = [c["doc_type_kwd"] for c in chunks] + # The media chunk is preserved and breaks the text merge. + assert doc_types.count("image") == 1, doc_types + # Several text chunks on each side of the media boundary. + assert doc_types.count("text") > 2, doc_types + + def test_json_delimiter_mode_pdf_positions_per_segment_not_broadcast(): # Regression for #3 (PDF coordinate leak): when consecutive text items from # different pages are buffered and then split by a custom delimiter, each @@ -346,11 +426,11 @@ def test_json_delimiter_mode_consecutive_delimiter_keeps_boundary(): assert all("##" not in t for t in texts) -def test_text_delimiter_mode_token_size_zero_or_one_no_atom_split(): - # token_size=0/1 must not atom-split delimiter segments into 1-token chunks; - # the delimiter path produces delimiter-boundary chunks regardless of cap. - for module, chunker in _build_json_chunker({"delimiter_mode": "token_size", "delimiters": ["`|`"]}): - # text path: delimiter_mode is token_size but a custom delimiter is +def test_text_delimiter_mode_zero_or_one_no_atom_split(): + # chunk_token_size=0/1 must not atom-split delimiter segments into 1-token + # chunks; the delimiter path produces delimiter-boundary chunks regardless of cap. + for module, chunker in _build_json_chunker({"delimiter_mode": "delimiter", "delimiters": ["`|`"]}): + # text path: delimiter_mode is delimiter but a custom delimiter is # present, so the delimiter branch (_split_text_by_pattern) is used. kwargs = { "name": "token_chunker", @@ -362,4 +442,4 @@ def test_text_delimiter_mode_token_size_zero_or_one_no_atom_split(): asyncio.run(chunker._invoke(**kwargs)) chunks = chunker._outputs["chunks"] texts = [c["text"] for c in chunks] - assert texts == ["aaa", "bbb", "ccc"], f"token_size={chunk_token_size} atom-split a delimiter segment: {texts}" + assert texts == ["aaa", "bbb", "ccc"], f"chunk_token_size={chunk_token_size} atom-split a delimiter segment: {texts}"