ragflow/internal/common/parser_config.go

package common

import "strings"

// InjectExtractorLLMID finds all Extractor component entries (keys prefixed
// with "extractor:" or "extractor_") in parserConfig and sets their llm_id
// to the given value. Returns whether any entry was updated.
func InjectExtractorLLMID(parserConfig map[string]interface{}, llmID string) bool {
	if parserConfig == nil || llmID == "" {
		return false
	}
	updated := false
	for cid, raw := range parserConfig {
		compMap, ok := raw.(map[string]interface{})
		if !ok {
			continue
		}
		cidLower := strings.ToLower(cid)
		if strings.HasPrefix(cidLower, "extractor:") || strings.HasPrefix(cidLower, "extractor_") {
			if current, ok := compMap["llm_id"].(string); !ok || current == "" {
				compMap["llm_id"] = llmID
				updated = true
			}
		}
	}
	return updated
}

// InjectExtractorEnableMetadata enables auto-metadata (enable_metadata) extraction
// on every Extractor node when the dataset has enable_metadata on and a
// non-empty field set (metadata and/or built_in_metadata). The dataset-level
// enable_metadata flag is authoritative (mirrors Python task_executor.py:519,
// which reads parser_config directly and never consults a per-node flag): a
// shipped DSL that defaults enable_metadata to 0 is still turned on. Only a
// node the user already turned ON (enable_metadata truthy) is left untouched,
// so an explicit per-node enablement keeps its own config. The field schema is
// taken from parserConfig["metadata"] and parserConfig["built_in_metadata"]
// (combined). Returns whether any entry was updated.
func InjectExtractorEnableMetadata(parserConfig map[string]interface{}) bool {
	if parserConfig == nil {
		return false
	}
	if !isTruthy(parserConfig["enable_metadata"]) {
		return false
	}
	fields := metadataFieldDefs(parserConfig)
	if len(fields) == 0 {
		return false
	}
	updated := false
	for cid, raw := range parserConfig {
		compMap, ok := raw.(map[string]interface{})
		if !ok {
			continue
		}
		cidLower := strings.ToLower(cid)
		if !strings.HasPrefix(cidLower, "extractor:") && !strings.HasPrefix(cidLower, "extractor_") {
			continue
		}
		// The dataset-level enable_metadata flag is authoritative (mirrors
		// Python task_executor.py:519, which reads parser_config directly and
		// never consults a per-node flag). Only a node the user already turned
		// ON (truthy) is left alone; a shipped DSL that defaults the field to
		// 0 must still be enabled by the dataset flag, otherwise auto-metadata
		// could never turn on for any of the built-in pipelines.
		if isTruthy(compMap["enable_metadata"]) {
			continue
		}
		compMap["enable_metadata"] = 1
		compMap["metadata"] = fields
		updated = true
	}
	return updated
}

// metadataFieldDefs combines parserConfig["metadata"] and
// parserConfig["built_in_metadata"] into the field list injected as
// metadata (each entry keeps key/type/description/enum, mirroring the
// stored shape from dataset/helpers.go normalizeMetadataConfigFields).
//
// It returns []any (i.e. []interface{}) rather than []map[string]interface{}
// because the injected value is handed to NewExtractorComponent, which reads
// params["metadata"].([]any). A []map[string]interface{} value would fail that
// type assertion (Go slice types are not covariant) and the field schema would
// be silently dropped, so auto-metadata never reached ExtractorParam.Metadata.
func metadataFieldDefs(parserConfig map[string]interface{}) []any {
	var out []any
	for _, key := range []string{"metadata", "built_in_metadata"} {
		raw, ok := parserConfig[key].([]interface{})
		if !ok {
			continue
		}
		for _, item := range raw {
			m, ok := item.(map[string]interface{})
			if !ok {
				continue
			}
			k, _ := m["key"].(string)
			if strings.TrimSpace(k) == "" {
				continue
			}
			out = append(out, m)
		}
	}
	return out
}

// isTruthy reports whether a parserConfig flag (e.g. enable_metadata) is on.
// It tolerates bool, numeric >0 and the strings "true"/"1" so storage
// representation differences don't silently disable the feature.
func isTruthy(v interface{}) bool {
	switch t := v.(type) {
	case bool:
		return t
	case string:
		return t == "true" || t == "1" || t == "True" || t == "TRUE"
	case float64:
		return t > 0
	case int:
		return t > 0
	case int64:
		return t > 0
	}
	return false
}

// deepCopyMap duplicates a JSON-like map so later merges do not mutate shared defaults.
func deepCopyMap(source map[string]interface{}) map[string]interface{} {
	if source == nil {
		return nil
	}

	cloned := make(map[string]interface{}, len(source))
	for key, value := range source {
		cloned[key] = deepCopyValue(value)
	}
	return cloned
}

// deepCopyValue recursively copies nested maps and slices inside parser_config values.
func deepCopyValue(value interface{}) interface{} {
	switch typedValue := value.(type) {
	case map[string]interface{}:
		return deepCopyMap(typedValue)
	case []interface{}:
		cloned := make([]interface{}, len(typedValue))
		for idx, item := range typedValue {
			cloned[idx] = deepCopyValue(item)
		}
		return cloned
	default:
		return typedValue
	}
}

// DeepMergeMaps applies override onto base while preserving nested defaults such as raptor/graphrag.
func DeepMergeMaps(base, override map[string]interface{}) map[string]interface{} {
	merged := deepCopyMap(base)
	if merged == nil {
		merged = make(map[string]interface{})
	}
	if override == nil {
		return merged
	}

	for key, value := range override {
		overrideMap, overrideIsMap := value.(map[string]interface{})
		existingMap, existingIsMap := merged[key].(map[string]interface{})
		if overrideIsMap && existingIsMap {
			merged[key] = DeepMergeMaps(existingMap, overrideMap)
			continue
		}
		merged[key] = deepCopyValue(value)
	}
	return merged
}

// GetParserConfig builds the final parser_config stored on a dataset:
// base defaults -> chunk-method defaults -> caller overrides.
func GetParserConfig(parserID string, parserConfig map[string]interface{}) map[string]interface{} {
	baseDefaults := map[string]interface{}{
		"table_context_size": 0,
		"image_context_size": 0,
	}

	defaultConfigs := map[string]map[string]interface{}{
		"naive": {
			"layout_recognize": "DeepDOC",
			"chunk_token_num":  512,
			"delimiter":        "\n",
			"auto_keywords":    0,
			"auto_questions":   0,
			"html4excel":       false,
			"topn_tags":        3,
		},
		"qa":           nil,
		"resume":       nil,
		"manual":       nil,
		"paper":        nil,
		"book":         nil,
		"laws":         nil,
		"presentation": nil,
	}

	merged := DeepMergeMaps(baseDefaults, defaultConfigs[parserID])
	return DeepMergeMaps(merged, parserConfig)
}

func ExtractPipelineDefaults(dsl map[string]interface{}) map[string]interface{} {
	if dsl == nil {
		return nil
	}
	if inner, ok := dsl["dsl"].(map[string]interface{}); ok {
		dsl = inner
	}
	components, _ := dsl["components"].(map[string]interface{})
	if components == nil {
		return nil
	}

	result := make(map[string]interface{})
	hasAny := false
	for cid, compVal := range components {
		compMap, ok := compVal.(map[string]interface{})
		if !ok {
			continue
		}
		obj, _ := compMap["obj"].(map[string]interface{})
		if obj == nil {
			continue
		}
		name, _ := obj["component_name"].(string)
		if name == "" || name == "File" {
			continue
		}
		params, _ := obj["params"].(map[string]interface{})
		if params == nil {
			continue
		}
		copy_ := deepCopyMap(params)
		delete(copy_, "outputs")
		result[cid] = copy_
		hasAny = true
	}
	if !hasAny {
		return nil
	}
	return result
}