fix: route TokenChunker delimiter_mode "one" to OneChunker in Go ingestion (#18121)

This commit is contained in:
euvre
2026-08-14 01:55:32 -07:00
committed by GitHub
parent ae256bcf59
commit fe963e69ba
2 changed files with 20 additions and 0 deletions

View File

@@ -36,6 +36,12 @@ import (
func newChunkerByName(name string, params map[string]any) (runtime.Component, error) {
switch name {
case ComponentNameTokenChunker:
// The DSL contract (shared by the web UI and the Python runtime)
// expresses single-chunk mode as TokenChunker delimiter_mode "one";
// in Go that behaviour lives in the OneChunker component.
if mode, _ := params["delimiter_mode"].(string); mode == "one" {
return NewOneChunker(params)
}
return NewTokenChunker(params)
case ComponentNameTitleChunker:
return NewTitleChunker(params)

View File

@@ -39,6 +39,20 @@ func oneChunksOf(t *testing.T, inputs map[string]any) []map[string]any {
return chunks
}
// TestNewChunkerByName_TokenChunkerOneMode pins the DSL-contract
// translation: a TokenChunker component whose delimiter_mode is "one"
// (what the web UI and the Python runtime emit) must build a
// OneChunker, not fail schema validation.
func TestNewChunkerByName_TokenChunkerOneMode(t *testing.T) {
comp, err := newChunkerByName(ComponentNameTokenChunker, map[string]any{"delimiter_mode": "one"})
if err != nil {
t.Fatalf("newChunkerByName: %v", err)
}
if _, ok := comp.(*OneChunkerComponent); !ok {
t.Fatalf("component type = %T, want *OneChunkerComponent", comp)
}
}
// TestOneChunker_Text emits exactly one chunk for a text payload,
// faithful to rag/app/one.py (whole file = one chunk).
func TestOneChunker_Text(t *testing.T) {