diff --git a/internal/parser/chunk/split.go b/internal/parser/chunk/split.go index 7c2152f9e..04c36c123 100644 --- a/internal/parser/chunk/split.go +++ b/internal/parser/chunk/split.go @@ -23,9 +23,17 @@ import ( ) type SplitOperator struct { - strategy string + strategy string + boundaries []string + keepSeparators bool + chunkSize int + overlap int } +// defaultLengthChunkSize is the window size (in runes) used by the "length" +// strategy when no positive chunk_size is configured. +const defaultLengthChunkSize = 256 + func NewSplitOperator(config map[string]interface{}) (*SplitOperator, error) { op := &SplitOperator{} @@ -35,9 +43,49 @@ func NewSplitOperator(config map[string]interface{}) (*SplitOperator, error) { } } + if params, ok := config["params"].(map[string]interface{}); ok { + if b, ok := params["boundaries"]; ok { + if boundStrs, ok := b.([]interface{}); ok { + for _, bs := range boundStrs { + if s, ok := bs.(string); ok { + op.boundaries = append(op.boundaries, s) + } + } + } + } + if ks, ok := params["keep_separators"]; ok { + if b, ok := ks.(bool); ok { + op.keepSeparators = b + } + } + // chunk_size / overlap drive the "length" strategy. JSON numbers + // decode as float64, so accept that alongside the integer types. + if cs, ok := params["chunk_size"]; ok { + op.chunkSize = toInt(cs) + } + if ov, ok := params["overlap"]; ok { + op.overlap = toInt(ov) + } + } + return op, nil } +// toInt coerces a DSL numeric value (float64 from JSON, or a native integer) +// to int. Any other type yields 0. +func toInt(v interface{}) int { + switch n := v.(type) { + case float64: + return int(n) + case int: + return n + case int64: + return int(n) + default: + return 0 + } +} + func (o *SplitOperator) Prepare(ctx *ChunkContext) error { return nil } @@ -56,6 +104,8 @@ func (o *SplitOperator) Execute(ctx *ChunkContext) error { ctx.SplitChunks = o.splitByChar(text) case "paragraph": ctx.SplitChunks = o.splitByParagraph(text) + case "length": + ctx.SplitChunks = o.splitByLength(text) default: ctx.SplitChunks = o.splitSentences(text) } @@ -71,6 +121,13 @@ func (o *SplitOperator) String() string { var buf strings.Builder buf.WriteString("split:\n") fmt.Fprintf(&buf, " strategy: %q\n", o.strategy) + fmt.Fprintf(&buf, " boundaries:\n") + for _, r := range o.boundaries { + fmt.Fprintf(&buf, " - %q\n", r) + } + fmt.Fprintf(&buf, " keep_separators: %t\n", o.keepSeparators) + fmt.Fprintf(&buf, " chunk_size: %d\n", o.chunkSize) + fmt.Fprintf(&buf, " overlap: %d\n", o.overlap) return buf.String() } @@ -145,6 +202,60 @@ func (o *SplitOperator) splitByChar(text string) []ChunkData { return chunks } +// splitByLength splits text into fixed-size, rune-aware windows of chunkSize +// runes, carrying overlap runes from the end of each window into the start of +// the next. This is the canonical fixed-size-with-overlap chunking used by RAG +// pipelines to bound chunk length while preserving cross-boundary context. +// +// chunkSize defaults to defaultLengthChunkSize when not positive. overlap is +// clamped to [0, chunkSize-1] so the window always advances by at least one +// rune and the function terminates. Sizing is by rune count (not bytes), so +// multi-byte (e.g. CJK) text is windowed by character. +func (o *SplitOperator) splitByLength(text string) []ChunkData { + chunkSize := o.chunkSize + if chunkSize <= 0 { + chunkSize = defaultLengthChunkSize + } + + overlap := o.overlap + if overlap < 0 { + overlap = 0 + } + if overlap >= chunkSize { + overlap = chunkSize - 1 + } + step := chunkSize - overlap + + runes := []rune(text) + if len(runes) == 0 { + return nil + } + + var chunks []ChunkData + for start := 0; start < len(runes); start += step { + end := start + chunkSize + if end > len(runes) { + end = len(runes) + } + + content := string(runes[start:end]) + chunks = append(chunks, ChunkData{ + Content: content, + Size: end - start, + Index: len(chunks), + Metadata: map[string]interface{}{ + "language": DetectLanguage(content), + }, + }) + + if end == len(runes) { + break + } + } + + return chunks +} + func (o *SplitOperator) splitByParagraph(text string) []ChunkData { paragraphs := strings.Split(text, "\n") chunks := make([]ChunkData, 0, len(paragraphs)) diff --git a/internal/parser/chunk/split_test.go b/internal/parser/chunk/split_test.go new file mode 100644 index 000000000..c19494964 --- /dev/null +++ b/internal/parser/chunk/split_test.go @@ -0,0 +1,163 @@ +// +// Copyright 2026 The InfiniFlow Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +package chunk + +import ( + "strings" + "testing" + "unicode/utf8" +) + +// runLength builds a SplitOperator for the "length" strategy from a DSL-style +// config (numbers as float64, as JSON decoding produces) and runs it. +func runLength(t *testing.T, text string, chunkSize, overlap float64) []ChunkData { + t.Helper() + op, err := NewSplitOperator(map[string]interface{}{ + "strategy": "length", + "params": map[string]interface{}{ + "chunk_size": chunkSize, + "overlap": overlap, + }, + }) + if err != nil { + t.Fatalf("NewSplitOperator returned error: %v", err) + } + ctx := &ChunkContext{TextAfterPreprocess: text} + if err := op.Execute(ctx); err != nil { + t.Fatalf("Execute returned error: %v", err) + } + return ctx.SplitChunks +} + +func TestSplitByLengthBasicWindowing(t *testing.T) { + chunks := runLength(t, "abcdefghij", 4, 0) // 10 runes, size 4, no overlap + + want := []string{"abcd", "efgh", "ij"} + if len(chunks) != len(want) { + t.Fatalf("got %d chunks, want %d: %#v", len(chunks), len(want), chunks) + } + for i, w := range want { + if chunks[i].Content != w { + t.Errorf("chunk %d = %q, want %q", i, chunks[i].Content, w) + } + if chunks[i].Index != i { + t.Errorf("chunk %d Index = %d, want %d", i, chunks[i].Index, i) + } + if chunks[i].Size != utf8.RuneCountInString(w) { + t.Errorf("chunk %d Size = %d, want %d", i, chunks[i].Size, utf8.RuneCountInString(w)) + } + } +} + +func TestSplitByLengthWithOverlap(t *testing.T) { + chunks := runLength(t, "abcdefg", 4, 2) // size 4, overlap 2 -> step 2 + + want := []string{"abcd", "cdef", "efg"} + got := make([]string, len(chunks)) + for i, c := range chunks { + got[i] = c.Content + } + if strings.Join(got, "|") != strings.Join(want, "|") { + t.Fatalf("got %v, want %v", got, want) + } +} + +func TestSplitByLengthOverlapClampedTerminates(t *testing.T) { + // overlap >= chunk_size must be clamped so the window still advances and + // the call terminates instead of looping forever. + chunks := runLength(t, "abcde", 3, 9) + if len(chunks) == 0 { + t.Fatal("expected chunks, got none") + } + // step = chunkSize-(chunkSize-1) = 1, so windows advance one rune at a time. + if chunks[0].Content != "abc" { + t.Errorf("first chunk = %q, want %q", chunks[0].Content, "abc") + } + last := chunks[len(chunks)-1] + if !strings.HasSuffix("abcde", last.Content) { + t.Errorf("last chunk %q is not a suffix of the input", last.Content) + } +} + +func TestSplitByLengthDefaultSize(t *testing.T) { + // chunk_size <= 0 falls back to the default window; short text => one chunk. + chunks := runLength(t, "hello world", 0, 0) + if len(chunks) != 1 { + t.Fatalf("got %d chunks, want 1", len(chunks)) + } + if chunks[0].Content != "hello world" { + t.Errorf("content = %q, want %q", chunks[0].Content, "hello world") + } +} + +func TestSplitByLengthRuneAware(t *testing.T) { + // Multi-byte runes must be windowed by rune, never split mid-rune. + text := "你好世界朋友" // 6 CJK runes, 3 bytes each + chunks := runLength(t, text, 2, 0) + + want := []string{"你好", "世界", "朋友"} + if len(chunks) != len(want) { + t.Fatalf("got %d chunks, want %d", len(chunks), len(want)) + } + for i, w := range want { + if chunks[i].Content != w { + t.Errorf("chunk %d = %q, want %q", i, chunks[i].Content, w) + } + if !utf8.ValidString(chunks[i].Content) { + t.Errorf("chunk %d %q is not valid UTF-8 (rune was split)", i, chunks[i].Content) + } + } +} + +func TestSplitByLengthNoOverlapReconstructs(t *testing.T) { + // With no overlap, concatenating the chunks must reproduce the input exactly. + text := "The quick brown fox jumps over the lazy dog." + chunks := runLength(t, text, 7, 0) + + var sb strings.Builder + for _, c := range chunks { + sb.WriteString(c.Content) + } + if sb.String() != text { + t.Errorf("reconstructed %q, want %q", sb.String(), text) + } +} + +func TestSplitByLengthEmptyInput(t *testing.T) { + if chunks := runLength(t, "", 4, 0); len(chunks) != 0 { + t.Errorf("empty input produced %d chunks, want 0", len(chunks)) + } +} + +func TestNewSplitOperatorParsesLengthParams(t *testing.T) { + op, err := NewSplitOperator(map[string]interface{}{ + "strategy": "length", + "params": map[string]interface{}{ + "chunk_size": float64(128), + "overlap": float64(16), + }, + }) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if op.chunkSize != 128 { + t.Errorf("chunkSize = %d, want 128", op.chunkSize) + } + if op.overlap != 16 { + t.Errorf("overlap = %d, want 16", op.overlap) + } +}