diff --git a/internal/ingestion/component/chunker/common.go b/internal/ingestion/component/chunker/common.go index c0d57170bb..230df7fbf4 100644 --- a/internal/ingestion/component/chunker/common.go +++ b/internal/ingestion/component/chunker/common.go @@ -36,6 +36,12 @@ import ( func newChunkerByName(name string, params map[string]any) (runtime.Component, error) { switch name { case ComponentNameTokenChunker: + // The DSL contract (shared by the web UI and the Python runtime) + // expresses single-chunk mode as TokenChunker delimiter_mode "one"; + // in Go that behaviour lives in the OneChunker component. + if mode, _ := params["delimiter_mode"].(string); mode == "one" { + return NewOneChunker(params) + } return NewTokenChunker(params) case ComponentNameTitleChunker: return NewTitleChunker(params) diff --git a/internal/ingestion/component/chunker/one_test.go b/internal/ingestion/component/chunker/one_test.go index 5fda780c96..f96adf3db5 100644 --- a/internal/ingestion/component/chunker/one_test.go +++ b/internal/ingestion/component/chunker/one_test.go @@ -39,6 +39,20 @@ func oneChunksOf(t *testing.T, inputs map[string]any) []map[string]any { return chunks } +// TestNewChunkerByName_TokenChunkerOneMode pins the DSL-contract +// translation: a TokenChunker component whose delimiter_mode is "one" +// (what the web UI and the Python runtime emit) must build a +// OneChunker, not fail schema validation. +func TestNewChunkerByName_TokenChunkerOneMode(t *testing.T) { + comp, err := newChunkerByName(ComponentNameTokenChunker, map[string]any{"delimiter_mode": "one"}) + if err != nil { + t.Fatalf("newChunkerByName: %v", err) + } + if _, ok := comp.(*OneChunkerComponent); !ok { + t.Fatalf("component type = %T, want *OneChunkerComponent", comp) + } +} + // TestOneChunker_Text emits exactly one chunk for a text payload, // faithful to rag/app/one.py (whole file = one chunk). func TestOneChunker_Text(t *testing.T) {