ragflow/internal/ingestion/task/indexdoc/normalize.go

//
//  Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
//
//  Licensed under the Apache License, Version 2.0 (the "License");
//  you may not use this file except in compliance with the License.
//  You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
//  Unless required by applicable law or agreed to in writing, software
//  distributed under the License is distributed on an "AS IS" BASIS,
//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
//  See the License for the specific language governing permissions and
//  limitations under the License.
//

package indexdoc

import (
	"fmt"

	"ragflow/internal/common"
)

// NormalizeChunks converts pipeline output into a uniform []map[string]any slice.
// Mirrors Python: DataflowService._normalize_chunks()
func NormalizeChunks(output map[string]any) []map[string]any {
	if output == nil {
		return nil
	}

	if chunks, ok := output["chunks"].([]map[string]any); ok {
		return DeepCopyChunks(chunks)
	}
	if chunks, ok := toChunkMaps(output["chunks"]); ok {
		return DeepCopyChunks(chunks)
	}
	if json, ok := output["json"].([]map[string]any); ok {
		return DeepCopyChunks(json)
	}
	if json, ok := toChunkMaps(output["json"]); ok {
		return DeepCopyChunks(json)
	}
	if md, ok := output["markdown"].(string); ok && md != "" {
		return []map[string]any{{"text": md}}
	}
	if txt, ok := output["text"].(string); ok && txt != "" {
		return []map[string]any{{"text": txt}}
	}
	if html, ok := output["html"].(string); ok && html != "" {
		return []map[string]any{{"text": html}}
	}
	return nil
}

func toChunkMaps(v any) ([]map[string]any, bool) {
	items, ok := v.([]any)
	if !ok {
		return nil, false
	}
	out := make([]map[string]any, 0, len(items))
	for _, item := range items {
		m, ok := item.(map[string]any)
		if !ok {
			return nil, false
		}
		out = append(out, m)
	}
	return out, true
}

// DeepCopyChunks returns a copy of the chunk slice and each chunk map.
// The chunk maps themselves are freshly allocated, and the value types that
// the pipeline actually emits — []float64 (vectors), []int, and []string — are
// element-wise copied so callers cannot mutate the originals through them.
// Other value types (nested maps, [][]float64 positions, etc.) are shared by
// reference, not recursively deep-copied; positions are later flattened and
// copied independently by processChunkPositions. It is therefore NOT a full
// recursive deep copy (unlike Python's copy.deepcopy), only the copy needed
// for the post-processing pass over pipeline output.
func DeepCopyChunks(chunks []map[string]any) []map[string]any {
	if chunks == nil {
		return nil
	}
	out := make([]map[string]any, len(chunks))
	for i, c := range chunks {
		cp := make(map[string]any, len(c))
		for k, v := range c {
			switch val := v.(type) {
			case []float64:
				vec := make([]float64, len(val))
				copy(vec, val)
				cp[k] = vec
			case []int:
				sl := make([]int, len(val))
				copy(sl, val)
				cp[k] = sl
			case []string:
				sl := make([]string, len(val))
				copy(sl, val)
				cp[k] = sl
			default:
				cp[k] = v
			}
		}
		out[i] = cp
	}
	return out
}

// PrepareTextsForPipelineEmbedding extracts texts for embedding from chunks.
// Priority: questions > summary > text.
// Mirrors Python: EmbeddingUtils.prepare_texts_for_dataflow_embedding()
func PrepareTextsForPipelineEmbedding(chunks []map[string]any) []string {
	if chunks == nil {
		return nil
	}
	texts := make([]string, 0, len(chunks))
	for _, chunk := range chunks {
		text, _ := chunk["questions"].(string)
		if text == "" {
			text, _ = chunk["summary"].(string)
		}
		if text == "" {
			chunkText, err := GetChunkTextString(chunk)
			if err != nil {
				common.Error("chunk[text] is not string", err)
			} else {
				text = chunkText
			}
		}
		if text != "" {
			texts = append(texts, text)
		}
	}
	return texts
}

// GetChunkTextString returns chunk["text"] when it is a string.
// Missing text is allowed and returns empty string. A present-but-non-string
// value is an upstream contract violation and returns an error; the caller
// decides whether to fail the task or skip the chunk.
func GetChunkTextString(chunk map[string]any) (string, error) {
	val, exists := chunk["text"]
	if !exists || val == nil {
		return "", nil
	}
	text, ok := val.(string)
	if ok {
		return text, nil
	}
	return "", fmt.Errorf("invalid chunk text type %T, expected string", val)
}