mirror of
https://github.com/infiniflow/ragflow.git
synced 2026-08-15 13:14:28 +08:00
fix: route TokenChunker delimiter_mode "one" to OneChunker in Go ingestion (#18121)
This commit is contained in:
@@ -36,6 +36,12 @@ import (
|
||||
func newChunkerByName(name string, params map[string]any) (runtime.Component, error) {
|
||||
switch name {
|
||||
case ComponentNameTokenChunker:
|
||||
// The DSL contract (shared by the web UI and the Python runtime)
|
||||
// expresses single-chunk mode as TokenChunker delimiter_mode "one";
|
||||
// in Go that behaviour lives in the OneChunker component.
|
||||
if mode, _ := params["delimiter_mode"].(string); mode == "one" {
|
||||
return NewOneChunker(params)
|
||||
}
|
||||
return NewTokenChunker(params)
|
||||
case ComponentNameTitleChunker:
|
||||
return NewTitleChunker(params)
|
||||
|
||||
@@ -39,6 +39,20 @@ func oneChunksOf(t *testing.T, inputs map[string]any) []map[string]any {
|
||||
return chunks
|
||||
}
|
||||
|
||||
// TestNewChunkerByName_TokenChunkerOneMode pins the DSL-contract
|
||||
// translation: a TokenChunker component whose delimiter_mode is "one"
|
||||
// (what the web UI and the Python runtime emit) must build a
|
||||
// OneChunker, not fail schema validation.
|
||||
func TestNewChunkerByName_TokenChunkerOneMode(t *testing.T) {
|
||||
comp, err := newChunkerByName(ComponentNameTokenChunker, map[string]any{"delimiter_mode": "one"})
|
||||
if err != nil {
|
||||
t.Fatalf("newChunkerByName: %v", err)
|
||||
}
|
||||
if _, ok := comp.(*OneChunkerComponent); !ok {
|
||||
t.Fatalf("component type = %T, want *OneChunkerComponent", comp)
|
||||
}
|
||||
}
|
||||
|
||||
// TestOneChunker_Text emits exactly one chunk for a text payload,
|
||||
// faithful to rag/app/one.py (whole file = one chunk).
|
||||
func TestOneChunker_Text(t *testing.T) {
|
||||
|
||||
Reference in New Issue
Block a user