feat(go): add length split strategy with overlap to chunk pipeline (#16047)

### What problem does this PR solve?

The Go ingestion chunk pipeline's `SplitOperator`
(`internal/ingestion/chunk/split.go`) supported only `sentence`, `char`,
and `paragraph` strategies, but not **fixed-size (length) chunking with
overlap** — the canonical RAG strategy for bounding chunk length while
preserving cross-boundary context.

This adds a `length` strategy alongside the existing ones, configurable
via DSL `params`:

- `chunk_size` — target window size in **runes** (rune-aware:
multi-byte/CJK text is windowed by character, never split mid-rune).
- `overlap` — runes carried from the end of each window into the next.

The window advances by `chunk_size - overlap`. `chunk_size` falls back
to a default (256) when unset/non-positive, and `overlap` is clamped to
`[0, chunk_size-1]` so the window always advances and the operation
terminates. Implementation follows the existing
`splitByChar`/`splitByParagraph` pattern and reuses `DetectLanguage` for
chunk metadata.

It also adds `split_test.go` — the first unit tests for the `chunk`
package — covering basic windowing, overlap, overlap
clamping/termination, rune-awareness (CJK), default sizing, no-overlap
reconstruction, empty input, and DSL param parsing.

Validation: `gofmt` clean, `go vet ./internal/ingestion/chunk/` clean,
`go build` ok, `go test ./internal/ingestion/chunk/` — all tests pass.

Closes #16046

### Type of change

- [x] New Feature (non-breaking change which adds functionality)

Co-authored-by: bittoby <218712309+bittoby@users.noreply.github.com>

internal/parser/chunk/split.go

@@ -23,9 +23,17 @@ import (
 )
 
 type SplitOperator struct {
-	strategy string
+	strategy       string
+	boundaries     []string
+	keepSeparators bool
+	chunkSize      int
+	overlap        int
 }
 
+// defaultLengthChunkSize is the window size (in runes) used by the "length"
+// strategy when no positive chunk_size is configured.
+const defaultLengthChunkSize = 256
+
 func NewSplitOperator(config map[string]interface{}) (*SplitOperator, error) {
 	op := &SplitOperator{}
 
@@ -35,9 +43,49 @@ func NewSplitOperator(config map[string]interface{}) (*SplitOperator, error) {
 		}
 	}
 
+	if params, ok := config["params"].(map[string]interface{}); ok {
+		if b, ok := params["boundaries"]; ok {
+			if boundStrs, ok := b.([]interface{}); ok {
+				for _, bs := range boundStrs {
+					if s, ok := bs.(string); ok {
+						op.boundaries = append(op.boundaries, s)
+					}
+				}
+			}
+		}
+		if ks, ok := params["keep_separators"]; ok {
+			if b, ok := ks.(bool); ok {
+				op.keepSeparators = b
+			}
+		}
+		// chunk_size / overlap drive the "length" strategy. JSON numbers
+		// decode as float64, so accept that alongside the integer types.
+		if cs, ok := params["chunk_size"]; ok {
+			op.chunkSize = toInt(cs)
+		}
+		if ov, ok := params["overlap"]; ok {
+			op.overlap = toInt(ov)
+		}
+	}
+
 	return op, nil
 }
 
+// toInt coerces a DSL numeric value (float64 from JSON, or a native integer)
+// to int. Any other type yields 0.
+func toInt(v interface{}) int {
+	switch n := v.(type) {
+	case float64:
+		return int(n)
+	case int:
+		return n
+	case int64:
+		return int(n)
+	default:
+		return 0
+	}
+}
+
 func (o *SplitOperator) Prepare(ctx *ChunkContext) error {
 	return nil
 }
@@ -56,6 +104,8 @@ func (o *SplitOperator) Execute(ctx *ChunkContext) error {
 		ctx.SplitChunks = o.splitByChar(text)
 	case "paragraph":
 		ctx.SplitChunks = o.splitByParagraph(text)
+	case "length":
+		ctx.SplitChunks = o.splitByLength(text)
 	default:
 		ctx.SplitChunks = o.splitSentences(text)
 	}
@@ -71,6 +121,13 @@ func (o *SplitOperator) String() string {
 	var buf strings.Builder
 	buf.WriteString("split:\n")
 	fmt.Fprintf(&buf, "  strategy: %q\n", o.strategy)
+	fmt.Fprintf(&buf, "  boundaries:\n")
+	for _, r := range o.boundaries {
+		fmt.Fprintf(&buf, "    - %q\n", r)
+	}
+	fmt.Fprintf(&buf, "  keep_separators: %t\n", o.keepSeparators)
+	fmt.Fprintf(&buf, "  chunk_size: %d\n", o.chunkSize)
+	fmt.Fprintf(&buf, "  overlap: %d\n", o.overlap)
 	return buf.String()
 }
 
@@ -145,6 +202,60 @@ func (o *SplitOperator) splitByChar(text string) []ChunkData {
 	return chunks
 }
 
+// splitByLength splits text into fixed-size, rune-aware windows of chunkSize
+// runes, carrying overlap runes from the end of each window into the start of
+// the next. This is the canonical fixed-size-with-overlap chunking used by RAG
+// pipelines to bound chunk length while preserving cross-boundary context.
+//
+// chunkSize defaults to defaultLengthChunkSize when not positive. overlap is
+// clamped to [0, chunkSize-1] so the window always advances by at least one
+// rune and the function terminates. Sizing is by rune count (not bytes), so
+// multi-byte (e.g. CJK) text is windowed by character.
+func (o *SplitOperator) splitByLength(text string) []ChunkData {
+	chunkSize := o.chunkSize
+	if chunkSize <= 0 {
+		chunkSize = defaultLengthChunkSize
+	}
+
+	overlap := o.overlap
+	if overlap < 0 {
+		overlap = 0
+	}
+	if overlap >= chunkSize {
+		overlap = chunkSize - 1
+	}
+	step := chunkSize - overlap
+
+	runes := []rune(text)
+	if len(runes) == 0 {
+		return nil
+	}
+
+	var chunks []ChunkData
+	for start := 0; start < len(runes); start += step {
+		end := start + chunkSize
+		if end > len(runes) {
+			end = len(runes)
+		}
+
+		content := string(runes[start:end])
+		chunks = append(chunks, ChunkData{
+			Content: content,
+			Size:    end - start,
+			Index:   len(chunks),
+			Metadata: map[string]interface{}{
+				"language": DetectLanguage(content),
+			},
+		})
+
+		if end == len(runes) {
+			break
+		}
+	}
+
+	return chunks
+}
+
 func (o *SplitOperator) splitByParagraph(text string) []ChunkData {
 	paragraphs := strings.Split(text, "\n")
 	chunks := make([]ChunkData, 0, len(paragraphs))

internal/parser/chunk/split_test.go

@@ -0,0 +1,163 @@
+//
+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+package chunk
+
+import (
+	"strings"
+	"testing"
+	"unicode/utf8"
+)
+
+// runLength builds a SplitOperator for the "length" strategy from a DSL-style
+// config (numbers as float64, as JSON decoding produces) and runs it.
+func runLength(t *testing.T, text string, chunkSize, overlap float64) []ChunkData {
+	t.Helper()
+	op, err := NewSplitOperator(map[string]interface{}{
+		"strategy": "length",
+		"params": map[string]interface{}{
+			"chunk_size": chunkSize,
+			"overlap":    overlap,
+		},
+	})
+	if err != nil {
+		t.Fatalf("NewSplitOperator returned error: %v", err)
+	}
+	ctx := &ChunkContext{TextAfterPreprocess: text}
+	if err := op.Execute(ctx); err != nil {
+		t.Fatalf("Execute returned error: %v", err)
+	}
+	return ctx.SplitChunks
+}
+
+func TestSplitByLengthBasicWindowing(t *testing.T) {
+	chunks := runLength(t, "abcdefghij", 4, 0) // 10 runes, size 4, no overlap
+
+	want := []string{"abcd", "efgh", "ij"}
+	if len(chunks) != len(want) {
+		t.Fatalf("got %d chunks, want %d: %#v", len(chunks), len(want), chunks)
+	}
+	for i, w := range want {
+		if chunks[i].Content != w {
+			t.Errorf("chunk %d = %q, want %q", i, chunks[i].Content, w)
+		}
+		if chunks[i].Index != i {
+			t.Errorf("chunk %d Index = %d, want %d", i, chunks[i].Index, i)
+		}
+		if chunks[i].Size != utf8.RuneCountInString(w) {
+			t.Errorf("chunk %d Size = %d, want %d", i, chunks[i].Size, utf8.RuneCountInString(w))
+		}
+	}
+}
+
+func TestSplitByLengthWithOverlap(t *testing.T) {
+	chunks := runLength(t, "abcdefg", 4, 2) // size 4, overlap 2 -> step 2
+
+	want := []string{"abcd", "cdef", "efg"}
+	got := make([]string, len(chunks))
+	for i, c := range chunks {
+		got[i] = c.Content
+	}
+	if strings.Join(got, "|") != strings.Join(want, "|") {
+		t.Fatalf("got %v, want %v", got, want)
+	}
+}
+
+func TestSplitByLengthOverlapClampedTerminates(t *testing.T) {
+	// overlap >= chunk_size must be clamped so the window still advances and
+	// the call terminates instead of looping forever.
+	chunks := runLength(t, "abcde", 3, 9)
+	if len(chunks) == 0 {
+		t.Fatal("expected chunks, got none")
+	}
+	// step = chunkSize-(chunkSize-1) = 1, so windows advance one rune at a time.
+	if chunks[0].Content != "abc" {
+		t.Errorf("first chunk = %q, want %q", chunks[0].Content, "abc")
+	}
+	last := chunks[len(chunks)-1]
+	if !strings.HasSuffix("abcde", last.Content) {
+		t.Errorf("last chunk %q is not a suffix of the input", last.Content)
+	}
+}
+
+func TestSplitByLengthDefaultSize(t *testing.T) {
+	// chunk_size <= 0 falls back to the default window; short text => one chunk.
+	chunks := runLength(t, "hello world", 0, 0)
+	if len(chunks) != 1 {
+		t.Fatalf("got %d chunks, want 1", len(chunks))
+	}
+	if chunks[0].Content != "hello world" {
+		t.Errorf("content = %q, want %q", chunks[0].Content, "hello world")
+	}
+}
+
+func TestSplitByLengthRuneAware(t *testing.T) {
+	// Multi-byte runes must be windowed by rune, never split mid-rune.
+	text := "你好世界朋友" // 6 CJK runes, 3 bytes each
+	chunks := runLength(t, text, 2, 0)
+
+	want := []string{"你好", "世界", "朋友"}
+	if len(chunks) != len(want) {
+		t.Fatalf("got %d chunks, want %d", len(chunks), len(want))
+	}
+	for i, w := range want {
+		if chunks[i].Content != w {
+			t.Errorf("chunk %d = %q, want %q", i, chunks[i].Content, w)
+		}
+		if !utf8.ValidString(chunks[i].Content) {
+			t.Errorf("chunk %d %q is not valid UTF-8 (rune was split)", i, chunks[i].Content)
+		}
+	}
+}
+
+func TestSplitByLengthNoOverlapReconstructs(t *testing.T) {
+	// With no overlap, concatenating the chunks must reproduce the input exactly.
+	text := "The quick brown fox jumps over the lazy dog."
+	chunks := runLength(t, text, 7, 0)
+
+	var sb strings.Builder
+	for _, c := range chunks {
+		sb.WriteString(c.Content)
+	}
+	if sb.String() != text {
+		t.Errorf("reconstructed %q, want %q", sb.String(), text)
+	}
+}
+
+func TestSplitByLengthEmptyInput(t *testing.T) {
+	if chunks := runLength(t, "", 4, 0); len(chunks) != 0 {
+		t.Errorf("empty input produced %d chunks, want 0", len(chunks))
+	}
+}
+
+func TestNewSplitOperatorParsesLengthParams(t *testing.T) {
+	op, err := NewSplitOperator(map[string]interface{}{
+		"strategy": "length",
+		"params": map[string]interface{}{
+			"chunk_size": float64(128),
+			"overlap":    float64(16),
+		},
+	})
+	if err != nil {
+		t.Fatalf("unexpected error: %v", err)
+	}
+	if op.chunkSize != 128 {
+		t.Errorf("chunkSize = %d, want 128", op.chunkSize)
+	}
+	if op.overlap != 16 {
+		t.Errorf("overlap = %d, want 16", op.overlap)
+	}
+}