ragflow/internal/ingestion/chunk_engine.go

//
// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//

package ingestion

import (
	"encoding/json"
	"fmt"
	"strings"

	"ragflow/internal/ingestion/chunk"
)

/*
 DSL reference — see comment block above for the full JSON structure.

 Pipeline stages:
   1. "preprocess"  → chunk.PreprocessOperator
   2. "split"       → chunk.SplitOperator
   3. "postprocess" → chunk.PostprocessOperator
*/

// ChunkPlan holds the ordered pipeline operators.
type ChunkPlan struct {
	Operators   []chunk.Operator
	Version     string
	Description string
	Name        string
}

// ChunkEngine parses DSL JSON into a plan and executes it.
type ChunkEngine struct{}

func NewChunkEngine() *ChunkEngine {
	return &ChunkEngine{}
}

// ---------------------------------------------------------------------------
// Compile  — compile DSL JSON into an ordered operator list
// ---------------------------------------------------------------------------

func (e *ChunkEngine) Compile(dsl string) (*ChunkPlan, error) {
	var parsed map[string]interface{}
	if err := json.Unmarshal([]byte(dsl), &parsed); err != nil {
		return nil, fmt.Errorf("compile DSL: %w", err)
	}

	plan := &ChunkPlan{}

	pipelineRaw, ok := parsed["pipeline"].([]interface{})
	if !ok || len(pipelineRaw) == 0 {
		return plan, nil
	}

	plan.Name, ok = parsed["name"].(string)
	if !ok {
		plan.Name = "No name"
	}
	plan.Description, ok = parsed["description"].(string)
	if !ok {
		plan.Description = "No description"
	}
	plan.Version, ok = parsed["version"].(string)
	if !ok {
		plan.Version = "1.0"
	}

	for i, operatorRaw := range pipelineRaw {
		var operator map[string]interface{}
		operator, ok = operatorRaw.(map[string]interface{})
		if !ok {
			return nil, fmt.Errorf("pipeline[%d]: expected object", i)
		}

		var op chunk.Operator
		var err error
		operatorName, _ := operator["operator"].(string)
		switch operatorName {
		case "preprocess":
			op, err = chunk.NewPreprocessOperator(operator)
			if err != nil {
				return nil, fmt.Errorf("create preprocess operator[%d]: %w", i, err)
			}
		case "split":
			op, err = chunk.NewSplitOperator(operator)
			if err != nil {
				return nil, fmt.Errorf("create split operator[%d]: %w", i, err)
			}
		case "postprocess":
			op, err = chunk.NewPostprocessOperator(operator)
			if err != nil {
				return nil, fmt.Errorf("create postprocess operator[%d]: %w", i, err)
			}
		default:
			return nil, fmt.Errorf("pipeline[%d]: unknown operator %s", i, operatorName)
		}
		delete(operator, "operator")

		plan.Operators = append(plan.Operators, op)
	}

	return plan, nil
}

// ---------------------------------------------------------------------------
// Execute — run the pipeline operators on input text
// ---------------------------------------------------------------------------

func (e *ChunkEngine) Execute(plan *ChunkPlan, text string) (*chunk.ChunkContext, error) {
	chunkContext := &chunk.ChunkContext{Origin: text}

	for i, op := range plan.Operators {
		if err := op.Prepare(chunkContext); err != nil {
			return nil, fmt.Errorf("re-prepare operator[%d]: %w", i, err)
		}

		if err := op.Execute(chunkContext); err != nil {
			return nil, fmt.Errorf("execute operator[%d]: %w", i, err)
		}

		if err := op.Finish(chunkContext); err != nil {
			return nil, fmt.Errorf("finish operator[%d]: %w", i, err)
		}
	}

	return chunkContext, nil
}

// ---------------------------------------------------------------------------
// Explain — describe the plan in human-readable form
// ---------------------------------------------------------------------------

func (e *ChunkEngine) Explain(plan *ChunkPlan) (string, error) {
	var buf strings.Builder
	buf.WriteString("Chunk Pipeline Plan:\n")
	for i, op := range plan.Operators {
		buf.WriteString(fmt.Sprintf("  [%d] %s\n", i, op.String()))
	}
	return buf.String(), nil
}