ragflow/internal/agent/harness/sufficiency.go

//
//  Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
//
//  Licensed under the Apache License, Version 2.0 (the "License");
//  you may not use this file except in compliance with the License.
//  You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
//  Unless required by applicable law or agreed to in writing, software
//  distributed under the License is distributed on an "AS IS" BASIS,
//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
//  See the License for the specific language governing permissions and
//  limitations under the License.
//

package harness

import (
	"fmt"
	"regexp"
	"strings"
)

// Sufficiency scoring (code-only, mirrors Python sufficiency.py): cross-check an
// agent result against the evidence chunks, fuse agent confidence + cross-check
// pass rate, then route to a 5-way verdict.

var reNumber = regexp.MustCompile(`\d+\.?\d*`)
var reEntities = regexp.MustCompile(`\b[A-Z][a-z]+(?:\s+[A-Z][a-z]+)*\b`)

// extractNumbers returns numeric values found in text.
func extractNumbers(text string) []string {
	return reNumber.FindAllString(text, -1)
}

// extractNamedEntities returns capitalized multi-word sequences.
func extractNamedEntities(text string) []string {
	seen := map[string]bool{}
	var out []string
	for _, e := range reEntities.FindAllString(text, -1) {
		if !seen[e] {
			seen[e] = true
			out = append(out, e)
		}
	}
	return out
}

// CrossCheckClaim performs a code-level cross-check of an agent result against
// the accumulated evidence chunks (number matching + entity presence).
func CrossCheckClaim(agent *AgentResult, allChunks map[int]map[string]interface{}) ClaimCrossCheckResult {
	if agent == nil {
		return ClaimCrossCheckResult{ClaimID: "", CrossCheckPassed: false, Mismatches: []string{"nil agent result"}}
	}
	if !agent.IsVerified {
		return ClaimCrossCheckResult{ClaimID: agent.ClaimID, CrossCheckPassed: false, Mismatches: []string{"agent self-reported as unverified"}}
	}
	numbers := extractNumbers(agent.Report)
	entities := extractNamedEntities(agent.Report)

	var matches, mismatches []string
	for _, eid := range agent.EvidenceIDs {
		chunk, ok := allChunks[eid]
		if !ok {
			mismatches = append(mismatches, fmt.Sprintf("evidence_id=%d: chunk not found", eid))
			continue
		}
		text := ""
		if c, ok := chunk["content_with_weight"].(string); ok {
			text = strings.ToLower(c)
		} else if c, ok := chunk["content"].(string); ok {
			text = strings.ToLower(c)
		}
		for _, num := range numbers {
			if strings.Contains(text, num) {
				matches = append(matches, fmt.Sprintf("number %s found in chunk %d", num, eid))
			} else {
				mismatches = append(mismatches, fmt.Sprintf("number %s not found in chunk %d", num, eid))
			}
		}
		for _, ent := range entities {
			if strings.Contains(text, strings.ToLower(ent)) {
				matches = append(matches, fmt.Sprintf("entity '%s' found in chunk %d", ent, eid))
			} else {
				mismatches = append(mismatches, fmt.Sprintf("entity '%s' not found in chunk %d", ent, eid))
			}
		}
	}
	// HasEvidence is true when at least one evidence id resolved to a chunk with
	// content. A claim with no resolvable evidence is never considered verified.
	hasEvidence := len(matches)+len(mismatches) > 0
	total := len(matches) + len(mismatches)
	crossScore := 0.0
	if total > 0 {
		crossScore = float64(len(matches)) / float64(total)
	}
	// Entity presence now contributes to matches too, so it can raise the score;
	// use a float comparison to avoid integer-division truncation bias.
	crossPassed := hasEvidence && float64(len(mismatches)) < float64(len(matches))/2.0
	return ClaimCrossCheckResult{
		ClaimID: agent.ClaimID, CrossCheckPassed: crossPassed, CrossCheckScore: crossScore,
		EvidenceMatches: matches, Mismatches: mismatches, HasEvidence: hasEvidence,
	}
}

// ComputeFusionScore fuses agent confidence + cross-check pass rate into a
// SufficiencyVerdict for the given mode.
func ComputeFusionScore(agentResults []AgentResult, crossResults []ClaimCrossCheckResult, mode ExecutionStrategy) SufficiencyVerdict {
	verified := 0
	for _, r := range agentResults {
		if r.IsVerified {
			verified++
		}
	}
	agentScore := 0.0
	if len(agentResults) > 0 {
		agentScore = float64(verified) / float64(len(agentResults))
	}
	passed := 0
	for _, r := range crossResults {
		if r.CrossCheckPassed {
			passed++
		}
	}
	crossScore := 0.0
	if len(crossResults) > 0 {
		crossScore = float64(passed) / float64(len(crossResults))
	}

	fusionScore := agentScore
	if crossScore > fusionScore {
		fusionScore = crossScore // low/medium default: max
	}
	switch mode.Label {
	case "ultra":
		fusionScore = min(agentScore, crossScore)
	case "high":
		fusionScore = (agentScore + crossScore) / 2
	}

	hasConflicts := false
	for _, r := range crossResults {
		if len(r.Mismatches) > 0 {
			hasConflicts = true
			break
		}
	}

	// Empty-evidence guard: if no claim examined any evidence chunk, the answer
	// cannot be grounded at all — this is UNANSWERABLE, not merely incomplete.
	anyEvidence := false
	for _, r := range crossResults {
		if r.HasEvidence {
			anyEvidence = true
			break
		}
	}

	status := "INSUFFICIENT"
	switch {
	case !anyEvidence:
		status = "UNANSWERABLE"
	case hasConflicts && fusionScore < mode.PartialThreshold:
		status = "CONFLICTING"
	case fusionScore >= mode.SufficiencyThreshold:
		status = "SUFFICIENT"
	case fusionScore >= mode.PartialThreshold:
		status = "USEFUL_BUT_INCOMPLETE"
	case func() bool {
		for _, r := range crossResults {
			if r.CrossCheckPassed {
				return true
			}
		}
		return false
	}():
		status = "INSUFFICIENT"
	default:
		status = "UNANSWERABLE"
	}

	var missing []string
	for _, r := range crossResults {
		if !r.CrossCheckPassed || !r.HasEvidence {
			missing = append(missing, r.ClaimID)
		}
	}

	assessments := make([]map[string]interface{}, 0, len(crossResults))
	for _, r := range crossResults {
		assessments = append(assessments, map[string]interface{}{
			"claim_id": r.ClaimID, "is_verified": r.CrossCheckPassed && r.HasEvidence, "score": r.CrossCheckScore,
			"mismatches": r.Mismatches, "has_evidence": r.HasEvidence,
		})
	}

	return SufficiencyVerdict{
		Status: status, Score: fusionScore, AgentScore: agentScore, CrossScore: crossScore,
		ClaimAssessments: assessments, HasConflicts: hasConflicts, MissingClaims: missing,
		Feedback: buildFeedback(missing, crossResults), OverallReason: fmt.Sprintf("%s score=%.2f missing=%v", status, fusionScore, missing),
	}
}

func buildFeedback(missing []string, results []ClaimCrossCheckResult) string {
	if len(missing) == 0 {
		return "all claims verified"
	}
	var hints []string
	for _, r := range results {
		if !r.CrossCheckPassed {
			hints = append(hints, fmt.Sprintf("claim %s: %d mismatch(es)", r.ClaimID, len(r.Mismatches)))
		}
	}
	return "missing: " + strings.Join(hints, "; ")
}

// RouteSufficiencyVerdict returns (action, shouldContinue) from the verdict.
func RouteSufficiencyVerdict(v SufficiencyVerdict, modeLabel string, cycle, maxCycles int) (string, bool) {
	mode, _ := GetMode(modeLabel)
	if mode.Label == "" {
		mode = THINKING_MODES["medium"]
	}
	switch v.Status {
	case "SUFFICIENT":
		return "ANSWER", false
	case "USEFUL_BUT_INCOMPLETE":
		if mode.RequiresSelectiveGen {
			return "ANSWER_PARTIAL", false
		}
		return "CONTINUE", false
	case "INSUFFICIENT":
		if cycle >= int(float64(maxCycles)*0.8) {
			return "ANSWER_PARTIAL", false
		}
		return "CONTINUE", true
	case "CONFLICTING":
		if mode.AllowsReplan && cycle < int(float64(maxCycles)*0.5) {
			return "REPLAN", true
		}
		return "ANSWER_PARTIAL", false
	case "UNANSWERABLE":
		if mode.FallbackToDirectLLM {
			return "FALLBACK_LLM", false
		}
		return "ABSTAIN", false
	default:
		return "CONTINUE", true
	}
}