Align Go ingestion boundaries with Python (#16647)

Moves doc_id blob resolution into Parser, tightens chunker/tokenizer to
Python output_format semantics, updates extractor list handling, and
fixes real-template integration tests.

internal/parser/chunk/chenk_type.go

@@ -0,0 +1,55 @@
+//
+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+package chunk
+
+// Operator defines the interface for all chunking pipeline stages.
+type Operator interface {
+	// Prepare configures the operator from a DSL stage config map.
+	Prepare(ctx *ChunkContext) error
+	// Execute runs the operator on the shared context.
+	Execute(ctx *ChunkContext) error
+	// Finish performs any cleanup.
+	Finish(ctx *ChunkContext) error
+
+	String() string
+}
+
+// ChunkData represents a single chunk produced by the pipeline.
+type ChunkData struct {
+	Content  string                 `json:"content"`
+	Size     int                    `json:"size"`
+	Index    int                    `json:"index,omitempty"`
+	Metadata map[string]interface{} `json:"metadata,omitempty"`
+}
+
+func (c *ChunkData) GetContent() string {
+	if c == nil {
+		return ""
+	}
+	return c.Content
+}
+
+// ChunkContext flows through the pipeline, carrying text and chunks.
+type ChunkContext struct {
+	Origin string // raw text
+
+	TextAfterPreprocess string // text after preprocess operator
+
+	SplitChunks []ChunkData // chunks after split operator
+
+	ResultChunks []ChunkData // final or intermediate chunks
+}

internal/parser/chunk/execute.go

@@ -0,0 +1,95 @@
+//
+//  Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
+//
+//  Licensed under the Apache License, Version 2.0 (the "License");
+//  you may not use this file except in compliance with the License.
+//  You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+//  Unless required by applicable law or agreed to in writing, software
+//  distributed under the License is distributed on an "AS IS" BASIS,
+//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+//  See the License for the specific language governing permissions and
+//  limitations under the License.
+//
+
+// Internal chunk execution entrypoint used by production callers.
+package chunk
+
+import "fmt"
+
+// Run executes the internal chunk steps against `text` using typed
+// options. The sequence is preprocess -> split -> postprocess.
+func Run(text string, opts ChunkOptions) (*ChunkContext, error) {
+	if err := opts.validate(); err != nil {
+		return nil, err
+	}
+
+	ctx := &ChunkContext{
+		Origin:              text,
+		TextAfterPreprocess: text,
+	}
+
+	var stages []Operator
+	var stageNames []string
+
+	if opts.NormalizeNewlines || opts.StripWhitespace || opts.RemoveEmptyLines {
+		pre, err := NewPreprocessOperator(map[string]interface{}{
+			"normalize_newlines": opts.NormalizeNewlines,
+			"strip_whitespace":   opts.StripWhitespace,
+			"remove_empty_lines": opts.RemoveEmptyLines,
+		})
+		if err != nil {
+			return nil, fmt.Errorf("chunk: build preprocess: %w", err)
+		}
+		stages = append(stages, pre)
+		stageNames = append(stageNames, "preprocess")
+	}
+
+	split, err := NewSplitOperator(map[string]interface{}{
+		"strategy": opts.SplitStrategy,
+	})
+	if err != nil {
+		return nil, fmt.Errorf("chunk: build split: %w", err)
+	}
+	stages = append(stages, split)
+	stageNames = append(stageNames, "split")
+
+	postCfg := map[string]interface{}{}
+	if opts.MergeTargetSize > 0 {
+		postCfg["merge"] = map[string]interface{}{
+			"target_size": float64(opts.MergeTargetSize),
+			"strategy":    "greedy",
+		}
+	}
+	if opts.FilterMinLength > 0 {
+		postCfg["filter"] = map[string]interface{}{
+			"min_length": float64(opts.FilterMinLength),
+		}
+	}
+	if len(postCfg) > 0 {
+		post, err := NewPostprocessOperator(postCfg)
+		if err != nil {
+			return nil, fmt.Errorf("chunk: build postprocess: %w", err)
+		}
+		stages = append(stages, post)
+		stageNames = append(stageNames, "postprocess")
+	}
+
+	for i, op := range stages {
+		if err := op.Prepare(ctx); err != nil {
+			return ctx, fmt.Errorf("%s: prepare: %w", stageNames[i], err)
+		}
+		if err := op.Execute(ctx); err != nil {
+			return ctx, fmt.Errorf("%s: execute: %w", stageNames[i], err)
+		}
+		if err := op.Finish(ctx); err != nil {
+			return ctx, fmt.Errorf("%s: finish: %w", stageNames[i], err)
+		}
+	}
+	if len(ctx.ResultChunks) == 0 && len(ctx.SplitChunks) > 0 {
+		ctx.ResultChunks = ctx.SplitChunks
+	}
+	return ctx, nil
+}

internal/parser/chunk/execute_test.go

@@ -0,0 +1,53 @@
+//
+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+package chunk
+
+import "testing"
+
+func TestRun_SentenceSplitHandlesASCIIBoundaries(t *testing.T) {
+	ctx, err := Run("One. Two! Three? Four;", ChunkOptions{SplitStrategy: "sentence"})
+	if err != nil {
+		t.Fatalf("Run: %v", err)
+	}
+	if len(ctx.ResultChunks) != 4 {
+		t.Fatalf("got %d chunks, want 4", len(ctx.ResultChunks))
+	}
+	want := []string{"One", "Two", "Three", "Four"}
+	for i, chunk := range ctx.ResultChunks {
+		if chunk.Content != want[i] {
+			t.Errorf("chunk[%d] = %q, want %q", i, chunk.Content, want[i])
+		}
+	}
+}
+
+func TestRun_PostprocessHonorsTypedNumericOptions(t *testing.T) {
+	ctx, err := Run("a\nbb\nccc", ChunkOptions{
+		SplitStrategy:    "paragraph",
+		MergeTargetSize:  100,
+		FilterMinLength:  2,
+		RemoveEmptyLines: true,
+	})
+	if err != nil {
+		t.Fatalf("Run: %v", err)
+	}
+	if len(ctx.ResultChunks) != 1 {
+		t.Fatalf("got %d chunks, want 1", len(ctx.ResultChunks))
+	}
+	if ctx.ResultChunks[0].Content != "a bb ccc" {
+		t.Errorf("merged content = %q, want %q", ctx.ResultChunks[0].Content, "a bb ccc")
+	}
+}

internal/parser/chunk/helpers.go

@@ -0,0 +1,43 @@
+//
+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+package chunk
+
+import "unicode"
+
+// DetectLanguage returns a best-effort language code based on the
+// proportion of CJK characters.
+func DetectLanguage(text string) string {
+	cjk := 0
+	total := 0
+	for _, r := range text {
+		if unicode.Is(unicode.Han, r) {
+			cjk++
+		}
+		if unicode.IsLetter(r) {
+			total++
+		}
+	}
+	if total > 0 && float64(cjk)/float64(total) > 0.3 {
+		return "zh"
+	}
+	return "en"
+}
+
+// RuneCount returns the number of runes in text.
+func RuneCount(text string) int {
+	return len([]rune(text))
+}

internal/parser/chunk/options.go

@@ -0,0 +1,59 @@
+//
+//  Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
+//
+//  Licensed under the Apache License, Version 2.0 (the "License");
+//  you may not use this file except in compliance with the License.
+//  You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+//  Unless required by applicable law or agreed to in writing, software
+//  distributed under the License is distributed on an "AS IS" BASIS,
+//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+//  See the License for the specific language governing permissions and
+//  limitations under the License.
+//
+
+// Typed options for the internal chunk execution path.
+
+package chunk
+
+import (
+	"fmt"
+)
+
+// ChunkOptions is the typed configuration for production callers.
+// It intentionally models only the option subset used by the current
+// production call sites.
+type ChunkOptions struct {
+	// Preprocess flags. Any combination of the three is honoured; all
+	// false means "no preprocess stage" (the engine skips the stage
+	// rather than running an identity preprocess).
+	NormalizeNewlines bool
+	StripWhitespace   bool
+	RemoveEmptyLines  bool
+
+	// Split configuration. Callers must select a strategy; an unset
+	// strategy degrades to "sentence" inside the operator.
+	SplitStrategy string
+
+	// Postprocess configuration. Zero values mean "do not run that
+	// step"; non-zero values enable it.
+	MergeTargetSize int
+
+	// FilterMinLength > 0 drops chunks shorter than that (rune count).
+	FilterMinLength int
+}
+
+// validate ensures the typed option set is internally consistent.
+// The check is cheap; an option set that fails validation will not
+// produce meaningful results at run time.
+func (o ChunkOptions) validate() error {
+	if o.MergeTargetSize < 0 {
+		return fmt.Errorf("chunk: MergeTargetSize must be >= 0 (got %d)", o.MergeTargetSize)
+	}
+	if o.FilterMinLength < 0 {
+		return fmt.Errorf("chunk: FilterMinLength must be >= 0 (got %d)", o.FilterMinLength)
+	}
+	return nil
+}

internal/parser/chunk/postprocess.go

@@ -0,0 +1,200 @@
+//
+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+package chunk
+
+import (
+	"fmt"
+	"strings"
+)
+
+type mergeConfig struct {
+	TargetSize int `json:"target_size"`
+}
+
+type filterConfig struct {
+	MinLength int `json:"min_length"`
+}
+
+// ---------------------------------------------------------------------------
+// PostprocessOperator
+// ---------------------------------------------------------------------------
+
+type PostprocessOperator struct {
+	merge  *mergeConfig
+	filter *filterConfig
+}
+
+func NewPostprocessOperator(config map[string]interface{}) (*PostprocessOperator, error) {
+	op := &PostprocessOperator{}
+
+	// Merge
+	if m, ok := config["merge"].(map[string]interface{}); ok {
+		op.merge = &mergeConfig{}
+		if ts, ok := m["target_size"].(float64); ok {
+			op.merge.TargetSize = int(ts)
+		} else {
+			op.merge.TargetSize = 500
+		}
+	}
+
+	// Filter
+	if f, ok := config["filter"].(map[string]interface{}); ok {
+		op.filter = &filterConfig{}
+		if v, ok := f["min_length"].(float64); ok {
+			op.filter.MinLength = int(v)
+		}
+	}
+
+	return op, nil
+}
+
+func (o *PostprocessOperator) Prepare(chunkCtx *ChunkContext) error {
+
+	return nil
+}
+
+func (o *PostprocessOperator) Execute(chunkCtx *ChunkContext) error {
+	chunks := chunkCtx.SplitChunks
+	if len(chunks) == 0 {
+		return nil
+	}
+
+	// 1. Merge
+	if o.merge != nil {
+		chunks = o.mergeChunks(chunks)
+	}
+
+	// 2. Filter
+	if o.filter != nil {
+		chunks = o.filterChunks(chunks)
+	}
+
+	// Re-index
+	for i := range chunks {
+		chunks[i].Index = i
+		chunks[i].Size = len(chunks[i].GetContent())
+	}
+
+	chunkCtx.ResultChunks = chunks
+	return nil
+}
+
+func (o *PostprocessOperator) Finish(chunkCtx *ChunkContext) error {
+	return nil
+}
+
+func (o *PostprocessOperator) String() string {
+	var buf strings.Builder
+	buf.WriteString("postprocess:\n")
+
+	if o.merge != nil {
+		fmt.Fprintf(&buf, "  merge:\n")
+		fmt.Fprintf(&buf, "    target_size: %d\n", o.merge.TargetSize)
+	}
+
+	if o.filter != nil {
+		fmt.Fprintf(&buf, "  filter:\n")
+		fmt.Fprintf(&buf, "    min_length: %d\n", o.filter.MinLength)
+	}
+
+	return buf.String()
+}
+
+// mergeChunks greedily merges small chunks into larger ones up to target_size.
+func (o *PostprocessOperator) mergeChunks(chunks []ChunkData) []ChunkData {
+	target := o.merge.TargetSize
+	if target <= 0 {
+		target = 500
+	}
+
+	var merged []ChunkData
+	var buf strings.Builder
+	var bufMeta map[string]interface{}
+	firstIndex := 0
+
+	for i, c := range chunks {
+		// If this single chunk already exceeds target, flush first then add
+		if len([]rune(c.Content)) >= target {
+			if buf.Len() > 0 {
+				merged = append(merged, ChunkData{
+					Content:  buf.String(),
+					Index:    firstIndex,
+					Metadata: bufMeta,
+				})
+				buf.Reset()
+				bufMeta = nil
+			}
+			merged = append(merged, c)
+			firstIndex = i + 1
+			continue
+		}
+
+		if buf.Len() == 0 {
+			buf.WriteString(c.Content)
+			bufMeta = c.Metadata
+			firstIndex = c.Index
+		} else {
+			nextLen := len([]rune(c.Content))
+			// If adding this chunk would exceed target, flush current and start new
+			if buf.Len()+nextLen+1 > target {
+				merged = append(merged, ChunkData{
+					Content:  buf.String(),
+					Index:    firstIndex,
+					Metadata: bufMeta,
+				})
+				buf.Reset()
+				buf.WriteString(c.Content)
+				bufMeta = c.Metadata
+				firstIndex = c.Index
+			} else {
+				buf.WriteString(" ")
+				buf.WriteString(c.Content)
+				// Merge metadata (last wins for overlapping keys)
+				if c.Metadata != nil && bufMeta == nil {
+					bufMeta = make(map[string]interface{})
+				}
+				for k, v := range c.Metadata {
+					bufMeta[k] = v
+				}
+			}
+		}
+	}
+
+	// Flush remaining
+	if buf.Len() > 0 {
+		merged = append(merged, ChunkData{
+			Content:  buf.String(),
+			Index:    firstIndex,
+			Metadata: bufMeta,
+		})
+	}
+
+	return merged
+}
+
+// filterChunks removes chunks outside the length bounds.
+func (o *PostprocessOperator) filterChunks(chunks []ChunkData) []ChunkData {
+	filtered := make([]ChunkData, 0, len(chunks))
+	for _, c := range chunks {
+		l := len([]rune(c.Content))
+		if o.filter.MinLength > 0 && l < o.filter.MinLength {
+			continue
+		}
+		filtered = append(filtered, c)
+	}
+	return filtered
+}

internal/parser/chunk/preprocess.go

@@ -0,0 +1,133 @@
+//
+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+package chunk
+
+import (
+	"fmt"
+	"regexp"
+	"strings"
+)
+
+type PreprocessOperator struct {
+	normalizeNewlines    bool
+	stripWhitespace      bool
+	removeEmptyLines     bool
+	softLineBreakMerging bool
+}
+
+func NewPreprocessOperator(config map[string]interface{}) (*PreprocessOperator, error) {
+	operator := &PreprocessOperator{}
+	if v, ok := config["normalize_newlines"]; ok {
+		operator.normalizeNewlines, ok = v.(bool)
+		if !ok {
+			return nil, fmt.Errorf("preprocess: normalize_newlines must be bool")
+		}
+	}
+	if v, ok := config["strip_whitespace"]; ok {
+		operator.stripWhitespace, ok = v.(bool)
+		if !ok {
+			return nil, fmt.Errorf("preprocess: strip_whitespace must be bool")
+		}
+	}
+	if v, ok := config["remove_empty_lines"]; ok {
+		operator.removeEmptyLines, ok = v.(bool)
+		if !ok {
+			return nil, fmt.Errorf("preprocess: remove_empty_lines must be bool")
+		}
+	}
+	if v, ok := config["soft_line_break_merging"]; ok {
+		operator.softLineBreakMerging, ok = v.(bool)
+		if !ok {
+			return nil, fmt.Errorf("preprocess: soft_line_break_merging must be bool")
+		}
+	}
+	return operator, nil
+}
+
+func (o *PreprocessOperator) Prepare(chunkCtx *ChunkContext) error {
+	return nil
+}
+
+func (o *PreprocessOperator) Execute(chunkCtx *ChunkContext) error {
+	text := chunkCtx.Origin
+
+	if o.normalizeNewlines {
+		// \r\n → \n, \r → \n
+		text = strings.ReplaceAll(text, "\r\n", "\n")
+		text = strings.ReplaceAll(text, "\r", "\n")
+		// Collapse multiple \n into one
+		re := regexp.MustCompile(`\n{2,}`)
+		text = re.ReplaceAllString(text, "\n")
+	}
+
+	if o.stripWhitespace {
+		// Trim leading/trailing whitespace on each line
+		lines := strings.Split(text, "\n")
+		for i, line := range lines {
+			lines[i] = strings.TrimSpace(line)
+		}
+		text = strings.Join(lines, "\n")
+	}
+
+	if o.removeEmptyLines {
+		lines := strings.Split(text, "\n")
+		filtered := make([]string, 0, len(lines))
+		for _, line := range lines {
+			if strings.TrimSpace(line) != "" {
+				filtered = append(filtered, line)
+			}
+		}
+		text = strings.Join(filtered, "\n")
+	}
+
+	if o.softLineBreakMerging {
+		lines := strings.Split(text, "\n")
+		var merged []string
+		var current strings.Builder
+
+		sentenceEnd := regexp.MustCompile(`[.!?][\s]*$`)
+
+		for i, line := range lines {
+			if current.Len() > 0 {
+				current.WriteString(" ")
+			}
+			current.WriteString(line)
+
+			if i == len(lines)-1 || sentenceEnd.MatchString(line) {
+				merged = append(merged, current.String())
+				current.Reset()
+			}
+		}
+		text = strings.Join(merged, "\n")
+	}
+
+	chunkCtx.TextAfterPreprocess = text
+	return nil
+}
+
+func (o *PreprocessOperator) Finish(chunkCtx *ChunkContext) error {
+	return nil
+}
+
+func (o *PreprocessOperator) String() string {
+	var buf strings.Builder
+	buf.WriteString("preprocess:\n")
+	fmt.Fprintf(&buf, "  normalize_newlines: %t\n", o.normalizeNewlines)
+	fmt.Fprintf(&buf, "  strip_whitespace: %t\n", o.stripWhitespace)
+	fmt.Fprintf(&buf, "  remove_empty_lines: %t\n", o.removeEmptyLines)
+	return buf.String()
+}

internal/parser/chunk/split.go

@@ -0,0 +1,163 @@
+//
+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+package chunk
+
+import (
+	"fmt"
+	"strings"
+	"unicode/utf8"
+)
+
+type SplitOperator struct {
+	strategy string
+}
+
+func NewSplitOperator(config map[string]interface{}) (*SplitOperator, error) {
+	op := &SplitOperator{}
+
+	if v, ok := config["strategy"]; ok {
+		if s, ok := v.(string); ok {
+			op.strategy = s
+		}
+	}
+
+	return op, nil
+}
+
+func (o *SplitOperator) Prepare(ctx *ChunkContext) error {
+	return nil
+}
+
+func (o *SplitOperator) Execute(ctx *ChunkContext) error {
+	text := ctx.TextAfterPreprocess
+
+	if o.strategy == "" {
+		o.strategy = "sentence"
+	}
+
+	switch o.strategy {
+	case "sentence":
+		ctx.SplitChunks = o.splitSentences(text)
+	case "char":
+		ctx.SplitChunks = o.splitByChar(text)
+	case "paragraph":
+		ctx.SplitChunks = o.splitByParagraph(text)
+	default:
+		ctx.SplitChunks = o.splitSentences(text)
+	}
+
+	return nil
+}
+
+func (o *SplitOperator) Finish(ctx *ChunkContext) error {
+	return nil
+}
+
+func (o *SplitOperator) String() string {
+	var buf strings.Builder
+	buf.WriteString("split:\n")
+	fmt.Fprintf(&buf, "  strategy: %q\n", o.strategy)
+	return buf.String()
+}
+
+var sentenceBoundaries = []string{"。", "！", "？", ".", "!", "?", ";", "\n"}
+
+// splitSentences splits text at the built-in sentence boundaries.
+func (o *SplitOperator) splitSentences(text string) []ChunkData {
+	var chunks []ChunkData
+	var buf strings.Builder
+	i := 0
+
+	for i < len(text) {
+		// Try to match any boundary at current position (first match wins)
+		matchedBound := ""
+		for _, bound := range sentenceBoundaries {
+			if bound != "" && i+len(bound) <= len(text) && text[i:i+len(bound)] == bound {
+				matchedBound = bound
+				break
+			}
+		}
+
+		if matchedBound != "" {
+			if buf.Len() > 0 {
+				content := strings.TrimSpace(buf.String())
+				if content != "" {
+					chunks = append(chunks, ChunkData{
+						Content: content,
+						Index:   len(chunks),
+						Metadata: map[string]interface{}{
+							"language": DetectLanguage(content),
+						},
+					})
+				}
+				buf.Reset()
+			}
+			i += len(matchedBound)
+		} else {
+			r, size := utf8.DecodeRuneInString(text[i:])
+			buf.WriteRune(r)
+			i += size
+		}
+	}
+
+	// flush remaining text
+	if buf.Len() > 0 {
+		content := strings.TrimSpace(buf.String())
+		if content != "" {
+			chunks = append(chunks, ChunkData{
+				Content: content,
+				Index:   len(chunks),
+				Metadata: map[string]interface{}{
+					"language": DetectLanguage(content),
+				},
+			})
+		}
+	}
+
+	return chunks
+}
+
+func (o *SplitOperator) splitByChar(text string) []ChunkData {
+	var chunks []ChunkData
+	for len(text) > 0 {
+		r, size := utf8.DecodeRuneInString(text)
+		chunks = append(chunks, ChunkData{
+			Content:  string(r),
+			Index:    len(chunks),
+			Metadata: map[string]interface{}{"language": DetectLanguage(text)},
+		})
+		text = text[size:]
+	}
+	return chunks
+}
+
+func (o *SplitOperator) splitByParagraph(text string) []ChunkData {
+	paragraphs := strings.Split(text, "\n")
+	chunks := make([]ChunkData, 0, len(paragraphs))
+	for i, p := range paragraphs {
+		trimmed := strings.TrimSpace(p)
+		if trimmed == "" {
+			continue
+		}
+		chunks = append(chunks, ChunkData{
+			Content:  trimmed,
+			Index:    i,
+			Metadata: map[string]interface{}{"language": DetectLanguage(trimmed)},
+		})
+	}
+	return chunks
+}