boshu2__agentops/cli/internal/commands/eval/module.go

// Package eval owns Cobra presentation for the eval command family.
package eval

import (
	"context"
	"errors"
	"fmt"
	"strings"
	"time"

	"github.com/spf13/cobra"

	"github.com/boshu2/agentops/cli/internal/clicontract"
	aoeval "github.com/boshu2/agentops/cli/internal/eval"
	"github.com/boshu2/agentops/cli/internal/evalsubstrate"
	scenarioapp "github.com/boshu2/agentops/cli/internal/scenario"
)

type CoreUseCases interface {
	Run(context.Context, aoeval.CoreRunRequest) (aoeval.CoreRunResult, error)
	Compare(context.Context, aoeval.CoreCompareRequest) (aoeval.CoreCompareResult, error)
	PromoteBaseline(context.Context, aoeval.CoreBaselineRequest) (*aoeval.RunRecord, error)
	AuditBaseline(context.Context, aoeval.CoreBaselineAuditRequest) (*aoeval.BaselineAuditReport, error)
	Scorecard(context.Context, aoeval.CoreScorecardRequest) (*aoeval.Scorecard, error)
	Coverage(context.Context, aoeval.CoreCoverageRequest) (*aoeval.CoverageReport, error)
}

type CleanupUseCases interface {
	Execute(context.Context, aoeval.CleanupRequest) (aoeval.CleanupReport, error)
}

type TaskUseCases interface {
	Add(context.Context, aoeval.TaskAddRequest) (aoeval.TaskAddResult, error)
	List(context.Context) (aoeval.TaskListResult, error)
	Show(context.Context, string) (*evalsubstrate.Task, error)
	Run(context.Context, aoeval.TaskRunRequest) (aoeval.TaskRunResult, error)
}

type SuiteUseCases interface {
	Verdict(context.Context, aoeval.SuiteVerdictRequest) (aoeval.SuiteVerdictResult, error)
	NRequired(context.Context, aoeval.SuiteNRequiredRequest) (aoeval.SuiteNRequiredResult, error)
}
type OutcomesUseCases interface {
	Compile(context.Context, string) (evalsubstrate.Rubric, error)
	Ingest(context.Context, aoeval.OutcomesIngestRequest) (aoeval.OutcomesIngestResult, error)
}
type ScenarioUseCases interface {
	Add(context.Context, aoeval.ScenarioAddRequest) (*scenarioapp.CreateResult, error)
	Init(context.Context) (string, error)
	List(context.Context, string) (aoeval.ScenarioListResult, error)
	Validate(context.Context) (aoeval.ScenarioValidationResult, error)
	Evaluate(context.Context, aoeval.ScenarioEvaluateRequest) (*aoeval.ScenarioEvaluateReport, error)
}
type ScenarioABUseCases interface {
	Run(context.Context, aoeval.ScenarioABRequest) (aoeval.ScenarioABResult, error)
	Moat(context.Context, aoeval.ScenarioMoatRequest) (aoeval.MoatClaimResult, error)
}

type UseCases struct {
	Core       CoreUseCases
	Cleanup    CleanupUseCases
	Task       TaskUseCases
	Suite      SuiteUseCases
	Outcomes   OutcomesUseCases
	Scenario   ScenarioUseCases
	ScenarioAB ScenarioABUseCases
	Aliases    aoeval.AliasUseCases
	Bench      aoeval.BenchUseCases
}

type Module struct {
	useCases UseCases
	host     clicontract.HostOptions
}

func NewModule(useCases UseCases, host clicontract.HostOptions) Module {
	return Module{useCases: useCases, host: host}
}

func (Module) Contract() clicontract.CommandContract {
	return clicontract.CommandContract{
		ID: "ao.eval",
		Profiles: clicontract.ProfileDefault | clicontract.ProfileFlywheel |
			clicontract.ProfileLegacy | clicontract.ProfileCombined,
		Args:        clicontract.ArgsPolicy{Name: "no-args", Validate: cobra.NoArgs},
		Output:      clicontract.OutputNone,
		Effects:     clicontract.EffectFilesystem | clicontract.EffectProcess | clicontract.EffectEnvironment | clicontract.EffectClock,
		ExitClasses: map[int]clicontract.ExitClass{0: clicontract.ExitSuccess, 1: clicontract.ExitFailure},
	}
}

// RunContract declares the real behavior of the `ao eval run <suite.json>`
// leaf. It takes exactly one positional suite path, emits a human text summary
// (JSON under -o json), reads the suite and writes a run record while executing
// the deterministic runtime (filesystem, process, environment, clock), and
// exits 0 on success or 1 on failure.
func (Module) RunContract() clicontract.CommandContract {
	return clicontract.CommandContract{
		ID: "ao.eval.run",
		Profiles: clicontract.ProfileDefault | clicontract.ProfileFlywheel |
			clicontract.ProfileLegacy | clicontract.ProfileCombined,
		Args:        clicontract.ArgsPolicy{Name: "exact", Validate: cobra.ExactArgs(1)},
		Output:      clicontract.OutputText,
		Effects:     clicontract.EffectFilesystem | clicontract.EffectProcess | clicontract.EffectEnvironment | clicontract.EffectClock,
		ExitClasses: map[int]clicontract.ExitClass{0: clicontract.ExitSuccess, 1: clicontract.ExitFailure},
	}
}

type runOptions struct {
	output, runID, runtime, baseline, baselineMode     string
	contextMode, contextOffDir, contextOnDir, deltaOut string
}
type compareOptions struct {
	output                     string
	maxAggregate, maxDimension float64
}
type baselineOptions struct{ output, promotedBy, rationale string }
type baselineAuditOptions struct{ root, baselineDir string }
type scorecardOptions struct {
	output, kind string
	maxCategory  float64
}
type coverageOptions struct {
	root                                         string
	domains, evidenceKinds, dimensions, runtimes []string
}

func (module Module) Command() *cobra.Command {
	command := &cobra.Command{
		Use: "eval", Short: "Run deterministic local evaluation suites",
		Long: `Run deterministic AgentOps evaluation suites and compare run records.

The eval surface intentionally supports only offline deterministic runs in this
release. Live Claude and Codex adapters are evaluated by a later runtime tier.`,
	}
	command.GroupID = "workflow"
	command.AddCommand(module.runCommand(), module.compareCommand(), module.baselineCommand(), module.baselineAuditCommand(), module.scorecardCommand(), module.coverageCommand())
	if module.useCases.Cleanup != nil {
		command.AddCommand(module.cleanupCommand())
	}
	if module.useCases.Task != nil {
		command.AddCommand(module.taskCommand())
	}
	if module.useCases.Suite != nil {
		command.AddCommand(module.suiteCommand())
	}
	if module.useCases.Outcomes != nil {
		command.AddCommand(module.outcomesCommand())
	}
	if module.useCases.Scenario != nil {
		command.AddCommand(module.scenarioCommand())
	}
	if module.useCases.ScenarioAB != nil {
		command.AddCommand(module.scenarioABCommand(), module.scenarioMoatCommand())
	}
	if module.useCases.Aliases != nil {
		command.AddCommand(module.sessionOutcomeCommand(), module.chaosCommand())
	}
	if module.useCases.Bench != nil {
		command.AddCommand(module.benchCommand())
	}
	return command
}

func (module Module) outputMode() string {
	if module.host.OutputMode == nil {
		return ""
	}
	return module.host.OutputMode()
}

func (module Module) runCommand() *cobra.Command {
	options := runOptions{baselineMode: string(aoeval.BaselineModeSkillOn), contextMode: string(aoeval.ContextModeNone)}
	command := &cobra.Command{
		Use:   "run <suite.json>",
		Short: "Run a deterministic eval suite",
		Long: "Run a deterministic eval suite.\n\n" +
			"The suite file must conform to the JSON Schema at schemas/eval-suite.v1.schema.json.\n" +
			"See evals/agentops-core for working example suites (e.g. evals/agentops-core/rpi-behavior.json).\n\n" +
			"Example:\n  ao eval run evals/agentops-core/rpi-behavior.json",
		Args: cobra.ExactArgs(1),
	}
	flags := command.Flags()
	flags.StringVar(&options.output, "out", "", "write eval run record to path")
	flags.StringVar(&options.runID, "run-id", "", "stable run id to use in the run record")
	flags.StringVar(&options.runtime, "runtime", "", "runtime override (static, mock, shell, claude, codex)")
	flags.StringVar(&options.baseline, "baseline", "", "compare the run against a baseline run record")
	flags.StringVar(&options.baselineMode, "baseline-mode", string(aoeval.BaselineModeSkillOn), "skill-on | skill-off | both — runs the suite once with skills loaded, once with hooks suppressed, or both for a delta scorecard")
	flags.StringVar(&options.contextMode, "context-mode", string(aoeval.ContextModeNone), "none | ab — run context-off/context-on legs over isolated AO_AGENTS_DIR roots")
	flags.StringVar(&options.contextOffDir, "context-off-agents-dir", "", "AO_AGENTS_DIR root for the context-off leg (defaults to suite fixtures)")
	flags.StringVar(&options.contextOnDir, "context-on-agents-dir", "", "AO_AGENTS_DIR root for the context-on leg (defaults to suite fixtures)")
	flags.StringVar(&options.deltaOut, "delta-out", "", "write delta scorecard JSON to path (with --baseline-mode=both or --context-mode=ab)")
	_ = command.RegisterFlagCompletionFunc("runtime", staticCompletion("static", "mock", "shell", "claude", "codex"))
	_ = command.RegisterFlagCompletionFunc("baseline-mode", staticCompletion(aoeval.AllBaselineModes()...))
	_ = command.RegisterFlagCompletionFunc("context-mode", staticCompletion(aoeval.AllContextModes()...))
	command.RunE = func(command *cobra.Command, args []string) error {
		result, err := module.useCases.Core.Run(command.Context(), aoeval.CoreRunRequest{SuitePath: args[0], RunID: options.runID, Runtime: options.runtime, OutputPath: options.output, BaselinePath: options.baseline, BaselineMode: options.baselineMode, ContextMode: options.contextMode, ContextOffDir: options.contextOffDir, ContextOnDir: options.contextOnDir, DeltaOut: options.deltaOut})
		if err != nil {
			return annotateSuiteParseError(err)
		}
		return module.renderRun(command, result)
	}
	return command
}

func (module Module) compareCommand() *cobra.Command {
	var options compareOptions
	command := &cobra.Command{Use: "compare <candidate-run.json> <baseline-run.json>", Short: "Compare an eval run against a baseline", Args: cobra.ExactArgs(2)}
	command.Flags().StringVar(&options.output, "out", "", "write compared eval run record to path")
	command.Flags().Float64Var(&options.maxAggregate, "max-aggregate-regression", 0, "allowed aggregate regression before verdict becomes regression")
	command.Flags().Float64Var(&options.maxDimension, "max-dimension-regression", 0, "allowed per-dimension regression before verdict becomes regression")
	command.RunE = func(command *cobra.Command, args []string) error {
		result, err := module.useCases.Core.Compare(command.Context(), aoeval.CoreCompareRequest{CandidatePath: args[0], BaselinePath: args[1], OutputPath: options.output, MaxAggregateRegression: options.maxAggregate, MaxDimensionRegression: options.maxDimension})
		if err != nil {
			return err
		}
		if module.outputMode() == "yaml" {
			return clicontract.WriteYAML(command.OutOrStdout(), result.Candidate)
		}
		if module.outputMode() == "json" {
			return clicontract.WriteJSON(command.OutOrStdout(), result.Candidate)
		}
		delta := 0.0
		if result.Candidate.BaselineComparison != nil {
			delta = result.Candidate.BaselineComparison.AggregateDelta
		}
		fmt.Fprintf(command.OutOrStdout(), "Eval compare %s vs %s: %s (aggregate delta %.4f)\n", result.Candidate.RunID, result.Baseline.RunID, result.Candidate.Verdict, delta)
		if result.OutputPath != "" {
			fmt.Fprintf(command.OutOrStdout(), "Comparison record: %s\n", result.OutputPath)
		}
		return nil
	}
	return command
}

func (module Module) baselineCommand() *cobra.Command {
	var options baselineOptions
	command := &cobra.Command{Use: "baseline <run.json>", Short: "Promote an eval run record as a baseline", Args: cobra.ExactArgs(1)}
	command.Flags().StringVar(&options.output, "out", "", "write promoted baseline run record to path")
	command.Flags().StringVar(&options.promotedBy, "promoted-by", "", "identity promoting the baseline")
	command.Flags().StringVar(&options.rationale, "rationale", "", "rationale for promoting the baseline")
	command.RunE = func(command *cobra.Command, args []string) error {
		result, err := module.useCases.Core.PromoteBaseline(command.Context(), aoeval.CoreBaselineRequest{RunPath: args[0], OutputPath: options.output, PromotedBy: options.promotedBy, Rationale: options.rationale})
		if err != nil {
			return err
		}
		if module.outputMode() == "yaml" {
			return clicontract.WriteYAML(command.OutOrStdout(), result)
		}
		if module.outputMode() == "json" {
			return clicontract.WriteJSON(command.OutOrStdout(), result)
		}
		path := ""
		if result.Baseline != nil {
			path = result.Baseline.BaselinePath
		}
		fmt.Fprintf(command.OutOrStdout(), "Eval baseline promoted: %s\n", path)
		return nil
	}
	return command
}

func (module Module) baselineAuditCommand() *cobra.Command {
	options := baselineAuditOptions{root: "evals/agentops-core", baselineDir: ".agents/evals/baselines"}
	command := &cobra.Command{Use: "baseline-audit [suite.json ...]", Short: "Audit eval suite baseline policy against promoted baselines", Args: cobra.ArbitraryArgs}
	command.Flags().StringVar(&options.root, "root", options.root, "suite root to scan when no suite paths are provided")
	command.Flags().StringVar(&options.baselineDir, "baseline-dir", options.baselineDir, "promoted baseline directory")
	command.RunE = func(command *cobra.Command, args []string) error {
		report, err := module.useCases.Core.AuditBaseline(command.Context(), aoeval.CoreBaselineAuditRequest{SuitePaths: args, Root: options.root, BaselineDir: options.baselineDir})
		if err != nil {
			return err
		}
		if module.outputMode() == "yaml" {
			return clicontract.WriteYAML(command.OutOrStdout(), report)
		}
		if module.outputMode() == "json" {
			return clicontract.WriteJSON(command.OutOrStdout(), report)
		}
		fmt.Fprintf(command.OutOrStdout(), "Eval baseline audit: %d suites, %d baselines, %d policy mismatches\n", report.SuiteCount, report.BaselineCount, report.PolicyMismatchCount)
		if len(report.MissingCompareBaselines) > 0 {
			fmt.Fprintf(command.OutOrStdout(), "Missing compare baselines: %d\n", len(report.MissingCompareBaselines))
		}
		if len(report.UnexpectedBaselinesForNone) > 0 {
			fmt.Fprintf(command.OutOrStdout(), "Unexpected baselines for none policy: %d\n", len(report.UnexpectedBaselinesForNone))
		}
		if len(report.OrphanBaselines) > 0 {
			fmt.Fprintf(command.OutOrStdout(), "Orphan baselines: %d\n", len(report.OrphanBaselines))
		}
		if len(report.StaleSuiteHashes) > 0 {
			fmt.Fprintf(command.OutOrStdout(), "Stale suite hashes: %d\n", len(report.StaleSuiteHashes))
		}
		return nil
	}
	return command
}

func (module Module) scorecardCommand() *cobra.Command {
	options := scorecardOptions{kind: string(aoeval.ScorecardKindRPI)}
	command := &cobra.Command{Use: "scorecard <candidate-run.json> [baseline-run.json]", Short: "Build an eval scorecard from run records", Args: cobra.RangeArgs(1, 2)}
	command.Flags().StringVar(&options.output, "out", "", "write scorecard JSON to path")
	command.Flags().StringVar(&options.kind, "kind", options.kind, "scorecard kind (rpi, skill-change)")
	command.Flags().Float64Var(&options.maxCategory, "max-category-regression", 0, "allowed per-category regression before verdict becomes regression")
	_ = command.RegisterFlagCompletionFunc("kind", staticCompletion(string(aoeval.ScorecardKindRPI), string(aoeval.ScorecardKindSkillChange)))
	command.RunE = func(command *cobra.Command, args []string) error {
		baselinePath := ""
		if len(args) == 2 {
			baselinePath = args[1]
		}
		card, err := module.useCases.Core.Scorecard(command.Context(), aoeval.CoreScorecardRequest{CandidatePath: args[0], BaselinePath: baselinePath, OutputPath: options.output, Kind: options.kind, MaxCategoryRegression: options.maxCategory})
		if err != nil {
			return err
		}
		if module.outputMode() == "yaml" {
			return clicontract.WriteYAML(command.OutOrStdout(), card)
		}
		if module.outputMode() == "json" {
			return clicontract.WriteJSON(command.OutOrStdout(), card)
		}
		fmt.Fprintf(command.OutOrStdout(), "Eval scorecard %s: %s (%s, categories %d)\n", card.CandidateRunID, card.Verdict, card.Kind, len(card.Categories))
		if options.output != "" {
			fmt.Fprintf(command.OutOrStdout(), "Scorecard: %s\n", options.output)
		}
		return nil
	}
	return command
}

func (module Module) coverageCommand() *cobra.Command {
	options := coverageOptions{root: "evals/agentops-core", domains: append([]string(nil), aoeval.DefaultCoverageDomains...), dimensions: append([]string(nil), aoeval.DefaultCoverageDimensions...), runtimes: append([]string(nil), aoeval.DefaultCoverageRuntimes...)}
	command := &cobra.Command{Use: "coverage [suite.json ...]", Short: "Summarize eval suite coverage", Args: cobra.ArbitraryArgs}
	command.Flags().StringVar(&options.root, "root", options.root, "suite root to scan when no suite paths are provided")
	command.Flags().StringArrayVar(&options.domains, "require-domain", options.domains, "required product domain for missing-domain reporting")
	command.Flags().StringArrayVar(&options.evidenceKinds, "require-evidence-kind", nil, "required evidence kind for missing-evidence-kind reporting")
	command.Flags().StringArrayVar(&options.dimensions, "require-dimension", options.dimensions, "required score dimension for missing-dimension reporting")
	command.Flags().StringArrayVar(&options.runtimes, "require-runtime", options.runtimes, "required deterministic runtime for missing-runtime reporting")
	command.RunE = func(command *cobra.Command, args []string) error {
		report, err := module.useCases.Core.Coverage(command.Context(), aoeval.CoreCoverageRequest{SuitePaths: args, Root: options.root, RequiredDomains: options.domains, RequiredEvidenceKinds: options.evidenceKinds, RequiredDimensions: options.dimensions, RequiredRuntimes: options.runtimes})
		if err != nil {
			return err
		}
		if module.outputMode() == "yaml" {
			return clicontract.WriteYAML(command.OutOrStdout(), report)
		}
		if module.outputMode() == "json" {
			return clicontract.WriteJSON(command.OutOrStdout(), report)
		}
		fmt.Fprintf(command.OutOrStdout(), "Eval coverage: %d suites, %d cases, %d critical cases\n", report.SuiteCount, report.CaseCount, report.CriticalCaseCount)
		renderCoverage(command, "domains", report.MissingRequiredDomains, report.RequiredDomains)
		renderCoverage(command, "evidence kinds", report.MissingRequiredEvidenceKinds, report.RequiredEvidenceKinds)
		renderCoverage(command, "dimensions", report.MissingRequiredDimensions, report.RequiredDimensions)
		renderCoverage(command, "runtimes", report.MissingRequiredRuntimes, report.RequiredRuntimes)
		return nil
	}
	return command
}

func (module Module) cleanupCommand() *cobra.Command {
	options := aoeval.CleanupRequest{TmpAgeSeconds: 60}
	command := &cobra.Command{
		Use: "cleanup", Short: "Recover stale Runs (state transitions, --delete, --tmp-files)",
		Long: `Per SCHEMA.md §4 cleanup state-transition rule (rc2):

  Stale pending  (no running transition within 60s)
                                       -> aborted (retraction_reason=never_started)
  Stale running  (no heartbeat within 5min OR Inspect process not alive)
                                       -> failed   (retraction_reason=orphaned_process)

After transitions:
  --delete       Remove Run dirs whose status is failed OR aborted (NEVER retracted).
  --tmp-files    Sweep orphan manifest.json.tmp left from rename-step crashes.
  --dry-run      Print what would be done; no mutations.

The cleanup procedure honors the §4 atomic-write contract on every state
transition (temp + fsync + rename + fsync-parent-dir). Retracted Runs are
never auto-removed — retraction is an audit trail.`,
	}
	command.Flags().BoolVar(&options.Delete, "delete", false, "Remove Run directories whose status is failed or aborted (never retracted)")
	command.Flags().BoolVar(&options.TmpFiles, "tmp-files", false, "Sweep orphan *.tmp files older than --tmp-age")
	command.Flags().Int64Var(&options.TmpAgeSeconds, "tmp-age", 60, "Minimum tmp-file age in seconds before sweep (0 = sweep all)")
	command.Flags().BoolVar(&options.DryRun, "dry-run", false, "Preview without mutations")
	_ = command.Flags().MarkHidden("dry-run")
	command.RunE = func(command *cobra.Command, _ []string) error {
		report, err := module.useCases.Cleanup.Execute(command.Context(), options)
		if err != nil {
			return err
		}
		if module.outputMode() == "yaml" {
			return clicontract.WriteYAML(command.OutOrStdout(), report)
		}
		if module.outputMode() == "json" {
			return clicontract.WriteJSON(command.OutOrStdout(), report)
		}
		fmt.Fprintf(command.OutOrStdout(), "Eval cleanup:\n  transitions->aborted: %d\n  transitions->failed:  %d\n  runs deleted:         %d\n  tmp files swept:      %d\n", report.TransitionsAborted, report.TransitionsFailed, report.RunsDeleted, report.TmpFilesSwept)
		verbose := module.host.Verbose != nil && module.host.Verbose()
		if len(report.Touched) > 0 && verbose {
			fmt.Fprintln(command.OutOrStdout(), "Touched:")
			for _, touched := range report.Touched {
				fmt.Fprintf(command.OutOrStdout(), "  %s\n", touched)
			}
		}
		return nil
	}
	return command
}

func (module Module) taskCommand() *cobra.Command {
	command := &cobra.Command{Use: "task", Short: "Manage evaluation Tasks (add, list, show, run)", Long: `Operate on the §3 Task primitive of the eval substrate.

Tasks live under $AGENTOPS_EVALS_ROOT/tasks/<id>/task.yaml and define the
input/output contract a Run will be evaluated against.`}
	command.AddCommand(module.taskAddCommand(), module.taskListCommand(), module.taskShowCommand(), module.taskRunCommand())
	return command
}

func (module Module) taskAddCommand() *cobra.Command {
	command := &cobra.Command{Use: "add <task.yaml>", Short: "Register a Task by copying its yaml + samples into the substrate", Args: cobra.ExactArgs(1)}
	command.RunE = func(command *cobra.Command, args []string) error {
		result, err := module.useCases.Task.Add(command.Context(), aoeval.TaskAddRequest{SourcePath: args[0]})
		if err != nil {
			return err
		}
		fmt.Fprintf(command.OutOrStdout(), "Task registered: %s\n  path:        %s\n  min_n_samples: %d\n", result.Task.ID, result.Destination, result.Task.Stats.MinNSamples)
		return nil
	}
	return command
}

func (module Module) taskListCommand() *cobra.Command {
	command := &cobra.Command{Use: "list", Short: "List registered Task ids", Args: cobra.NoArgs}
	command.RunE = func(command *cobra.Command, _ []string) error {
		result, err := module.useCases.Task.List(command.Context())
		if err != nil {
			return err
		}
		ids := make([]string, 0, len(result.Tasks))
		for _, summary := range result.Tasks {
			ids = append(ids, summary.ID)
		}
		if module.outputMode() == "yaml" {
			return clicontract.WriteYAML(command.OutOrStdout(), map[string]any{"tasks": ids, "root": result.Root})
		}
		if module.outputMode() == "json" {
			return clicontract.WriteJSON(command.OutOrStdout(), map[string]any{"tasks": ids, "root": result.Root})
		}
		if len(result.Tasks) == 0 {
			fmt.Fprintln(command.OutOrStdout(), "No tasks registered")
			return nil
		}
		fmt.Fprintf(command.OutOrStdout(), "Tasks under %s:\n", result.Root)
		for _, summary := range result.Tasks {
			if summary.Error != "" {
				fmt.Fprintf(command.OutOrStdout(), "  %s\t<unreadable: %s>\n", summary.ID, summary.Error)
				continue
			}
			task := summary.Task
			fmt.Fprintf(command.OutOrStdout(), "  %s\tschema_version=%d min_n_samples=%d metric=%s\n", task.ID, task.SchemaVersion, task.Stats.MinNSamples, task.Stats.Metric)
		}
		return nil
	}
	return command
}

func (module Module) taskShowCommand() *cobra.Command {
	command := &cobra.Command{Use: "show <task-id>", Short: "Print a registered Task summary", Args: cobra.ExactArgs(1)}
	command.RunE = func(command *cobra.Command, args []string) error {
		task, err := module.useCases.Task.Show(command.Context(), args[0])
		if err != nil {
			return err
		}
		if module.outputMode() == "yaml" {
			return clicontract.WriteYAML(command.OutOrStdout(), task)
		}
		if module.outputMode() == "json" {
			return clicontract.WriteJSON(command.OutOrStdout(), task)
		}
		fmt.Fprintf(command.OutOrStdout(), "Task: %s\n", task.ID)
		fmt.Fprintf(command.OutOrStdout(), "  schema_version: %d\n  domain:         %s\n  description:    %s\n  harness_ref:    %s\n", task.SchemaVersion, task.Domain, task.Description, task.HarnessRef)
		fmt.Fprintf(command.OutOrStdout(), "  stats.metric:        %s\n  stats.paired:        %v\n  stats.min_n_samples: %d\n  stats.decision_rule: kind=%s confidence=%.2f\n", task.Stats.Metric, task.Stats.Paired, task.Stats.MinNSamples, task.Stats.DecisionRule.Kind, task.Stats.DecisionRule.Confidence)
		return nil
	}
	return command
}

func (module Module) taskRunCommand() *cobra.Command {
	options := aoeval.TaskRunRequest{InspectVersion: "0.3.216"}
	command := &cobra.Command{Use: "run <task-id>", Short: "Open a new Run manifest for <task-id>; refuses on gate failure", Args: cobra.ExactArgs(1)}
	flags := command.Flags()
	flags.StringVar(&options.SuiteRef, "suite", "", "Suite id or path to suite.yaml (required)")
	flags.StringVar(&options.RigID, "rig-id", "", "Rig identifier stamped into the Run manifest")
	flags.StringVar(&options.Seeds, "seeds", "", "Comma-separated seeds (>=3, per §4)")
	flags.StringVar(&options.HarnessRef, "harness", "", "Harness id (recorded into manifest)")
	flags.StringVar(&options.HarnessDir, "harness-dir", "", "Path to harness source dir for snapshot + gate #8")
	flags.StringVar(&options.ModelSpecID, "model-spec", "", "ModelSpec id already present in the eval substrate's model-specs store")
	flags.StringVar(&options.GroundTruthRef, "ground-truth", "", "Ground-truth row id (head of supersession chain)")
	flags.StringVar(&options.SampleSplit, "sample-split", "", "Sample split (dev|holdout); default from suite")
	flags.IntVar(&options.NSamples, "n-samples", 0, "Override Suite.n_samples")
	flags.StringVar(&options.InspectVersion, "inspect-version", options.InspectVersion, "Inspect AI version stamped into manifest")
	flags.StringVar(&options.InspectCommand, "inspect-command", "", "Inspect command recorded into the Run manifest (not executed yet)")
	flags.BoolVar(&options.CrossSpec, "cross-spec", false, "Allow ModelSpec drift (gate #4)")
	flags.BoolVar(&options.AllowWeak, "allow-weak-labels", false, "Allow runs against confidence=weak ground-truth rows (gate #7)")
	flags.BoolVar(&options.QuickSession, "quick", false, "Mark Run as quick_session=true (excluded from --vs auto-baseline pool)")
	flags.BoolVar(&options.DryRun, "dry-run", false, "Run gates and exit without writing a Run manifest")
	_ = flags.MarkHidden("dry-run")
	command.RunE = func(command *cobra.Command, args []string) error {
		options.TaskID = args[0]
		result, err := module.useCases.Task.Run(command.Context(), options)
		if err != nil {
			var gateFailure *aoeval.TaskGateError
			if errors.As(err, &gateFailure) {
				fmt.Fprintln(command.ErrOrStderr(), gateFailure.Message)
			}
			return err
		}
		if result.DryRun {
			fmt.Fprintln(command.OutOrStdout(), "Dry run: gates passed; no Run manifest written.")
			return nil
		}
		if module.outputMode() == "yaml" {
			return clicontract.WriteYAML(command.OutOrStdout(), result.Manifest)
		}
		if module.outputMode() == "json" {
			return clicontract.WriteJSON(command.OutOrStdout(), result.Manifest)
		}
		manifest := result.Manifest
		fmt.Fprintf(command.OutOrStdout(), "Run opened: %s\n  status:    %s\n  manifest:  %s\n  rig_id:    %s\n  task_ref:  %s\n  suite_ref: %s\n", manifest.ID, manifest.Status, result.Path, manifest.RigID, manifest.TaskRef, manifest.SuiteRef)
		return nil
	}
	return command
}

func (module Module) suiteCommand() *cobra.Command {
	command := &cobra.Command{Use: "suite", Short: "Suite-level operations (verdict, n-required)", Long: "Suite-level operations against the §6.5 statistical contract."}
	command.AddCommand(module.suiteVerdictCommand(), module.suiteNRequiredCommand())
	return command
}

func (module Module) suiteVerdictCommand() *cobra.Command {
	options := aoeval.SuiteVerdictRequest{BootstrapSamples: 10000}
	command := &cobra.Command{Use: "verdict <suite-id> --arms a,b --inputs <bootstrap-inputs.json>", Short: "Compute the §6.5 paired cluster-bootstrap verdict", Args: cobra.ExactArgs(1)}
	command.Flags().StringVar(&options.Arms, "arms", "", "Comma-separated arm ids (default: from suite varied_axis)")
	command.Flags().StringVar(&options.InputsPath, "inputs", "", "Path to canonical bootstrap-inputs JSON (REQUIRED)")
	command.Flags().Float64Var(&options.MDE, "mde", 0, "Minimum detectable effect (used for inconclusive_high_variance)")
	command.Flags().IntVar(&options.BootstrapSamples, "B", 10000, "Bootstrap resamples")
	command.Flags().IntVar(&options.NRequired, "n-required", 0, "Override n_required (default: derived from suite power block)")
	command.RunE = func(command *cobra.Command, args []string) error {
		options.SuiteID = args[0]
		result, err := module.useCases.Suite.Verdict(command.Context(), options)
		if err != nil {
			return err
		}
		if module.outputMode() == "yaml" {
			return clicontract.JSONToYAML(command.OutOrStdout(), result.Raw)
		}
		if module.outputMode() == "json" {
			_, err = command.OutOrStdout().Write(append(append([]byte(nil), result.Raw...), '\n'))
			return err
		}
		value := result.Values
		fmt.Fprintf(command.OutOrStdout(), "Suite verdict: %v\n  delta_point: %v\n  ci: [%v, %v]\n  n_clusters: %v\n  n_required: %v\n  bootstrap_seed: %v\n  bootstrap_inputs_hash: %v\n", value["verdict"], value["delta_point"], value["ci_low"], value["ci_high"], value["n_clusters"], value["n_required"], value["bootstrap_seed"], value["bootstrap_inputs_hash"])
		return nil
	}
	return command
}

func (module Module) suiteNRequiredCommand() *cobra.Command {
	options := aoeval.SuiteNRequiredRequest{BaselineRate: .5, MDE: .05, Alpha: .05, Power: .80, Paired: true}
	command := &cobra.Command{Use: "n-required", Short: "Compute power-derived n_required (gate #6 input on Day 3+)"}
	command.Flags().Float64Var(&options.BaselineRate, "baseline-rate", .5, "Baseline rate (binomial worst-case fallback)")
	command.Flags().Float64Var(&options.MDE, "mde", .05, "Minimum detectable effect")
	command.Flags().Float64Var(&options.Alpha, "alpha", .05, "Type-I error rate")
	command.Flags().Float64Var(&options.Power, "power", .80, "Statistical power (1-beta)")
	command.Flags().BoolVar(&options.Paired, "paired", true, "Paired comparison")
	command.RunE = func(command *cobra.Command, _ []string) error {
		result, err := module.useCases.Suite.NRequired(command.Context(), options)
		if err != nil {
			return err
		}
		if module.outputMode() == "yaml" {
			return clicontract.JSONToYAML(command.OutOrStdout(), result.Raw)
		}
		if module.outputMode() == "json" {
			_, err = command.OutOrStdout().Write(append(append([]byte(nil), result.Raw...), '\n'))
			return err
		}
		fmt.Fprintf(command.OutOrStdout(), "n_required: %d  (baseline_rate=%g MDE=%g alpha=%g power=%g paired=%v)\n", result.NRequired, options.BaselineRate, options.MDE, options.Alpha, options.Power, options.Paired)
		return nil
	}
	return command
}

func (module Module) outcomesCommand() *cobra.Command {
	command := &cobra.Command{Use: "outcomes", Short: "Project the locked eval substrate into Outcomes grading payloads (holdout-safe)", Long: "Outcomes is a derived projection of the locked eval substrate (SCHEMA.md), never an alternate authority."}
	command.AddCommand(module.outcomesCompileCommand(), module.outcomesIngestCommand())
	return command
}

func (module Module) outcomesCompileCommand() *cobra.Command {
	command := &cobra.Command{Use: "compile <input.json>", Short: "Compile a holdout-safe Outcomes rubric payload from a locked Task + criteria", Args: cobra.ExactArgs(1)}
	command.RunE = func(command *cobra.Command, args []string) error {
		rubric, err := module.useCases.Outcomes.Compile(command.Context(), args[0])
		if err != nil {
			return err
		}
		return clicontract.WriteJSON(command.OutOrStdout(), rubric)
	}
	return command
}

func (module Module) outcomesIngestCommand() *cobra.Command {
	var options aoeval.OutcomesIngestRequest
	command := &cobra.Command{Use: "ingest <score.json>", Short: "Ingest an Outcomes score payload into the one council verdict record", Args: cobra.ExactArgs(1)}
	command.Flags().StringVar(&options.ExpectedJudgeHash, "expect-judge-hash", "", "refuse the ingest if the score's judge_content_hash does not match this value (gate #2 rubric-drift parity)")
	command.Flags().StringVar(&options.BurnLedgerPath, "burn-ledger", "", "path to a JSON HoldoutBurnLedger; when set, a holdout-split score registers a burn and is REFUSED if the (suite,gt) quota is exhausted (gate #3 runtime enforcement), persisted across invocations")
	command.Flags().StringVar(&options.ManifestDir, "manifest-out", "", "also write an eval-run.v1 manifest to <dir>/<run-id>/manifest.json so the verdict pipeline feeds the Knowledge Flywheel (closes the Outcomes→Flywheel loop)")
	command.Flags().StringVar(&options.RunID, "run-id", "", "run id for the --manifest-out manifest; defaults to the score's run_id, then source_task_id (sanitized to the eval-run.v1 pattern)")
	command.RunE = func(command *cobra.Command, args []string) error {
		options.ScorePath = args[0]
		result, err := module.useCases.Outcomes.Ingest(command.Context(), options)
		if err != nil {
			return err
		}
		if result.Warning != "" {
			fmt.Fprintln(command.ErrOrStderr(), result.Warning)
		}
		// ingest emits the verdict as structured output unconditionally (its
		// default is JSON, not a human table); honor -o yaml when asked.
		emit := clicontract.WriteJSON
		if module.outputMode() == "yaml" {
			emit = clicontract.WriteYAML
		}
		if err := emit(command.OutOrStdout(), result.Verdict); err != nil {
			return err
		}
		if result.ManifestPath != "" {
			fmt.Fprintf(command.ErrOrStderr(), "wrote eval-run manifest: %s\n", result.ManifestPath)
		}
		return nil
	}
	return command
}

func (module Module) scenarioCommand() *cobra.Command {
	command := &cobra.Command{Use: "scenario", Short: "Manage holdout scenarios for behavioral validation", Long: "Create, list, validate, and evaluate holdout scenarios stored in .agents/holdout/."}
	command.AddCommand(module.scenarioInitCommand(), module.scenarioAddCommand(), module.scenarioListCommand(), module.scenarioValidateCommand(), module.scenarioEvaluateCommand())
	return command
}

func (module Module) scenarioInitCommand() *cobra.Command {
	command := &cobra.Command{Use: "init", Short: "Initialize .agents/holdout/ directory for scenario storage"}
	command.RunE = func(command *cobra.Command, _ []string) error {
		path, err := module.useCases.Scenario.Init(command.Context())
		if err == nil {
			fmt.Fprintf(command.OutOrStdout(), "Initialized holdout directory at %s\n", path)
		}
		return err
	}
	return command
}

func (module Module) scenarioAddCommand() *cobra.Command {
	options := aoeval.ScenarioAddRequest{Threshold: .8, Status: "draft", Source: "human"}
	command := &cobra.Command{Use: "add <goal>", Short: "Author a holdout scenario from a goal description", Args: cobra.ExactArgs(1)}
	command.Flags().StringVar(&options.Narrative, "narrative", "", "Narrative description (default: inferred from goal)")
	command.Flags().StringVar(&options.ExpectedOutcome, "expected-outcome", "", "Expected observable outcome (default: inferred from goal)")
	command.Flags().Float64Var(&options.Threshold, "threshold", .8, "Satisfaction threshold in [0,1]")
	command.Flags().StringVar(&options.Status, "status", "draft", "Scenario status (active, draft, retired)")
	command.Flags().StringVar(&options.Source, "source", "human", "Scenario source (human, agent, prod-telemetry)")
	_ = command.RegisterFlagCompletionFunc("status", staticCompletion("active", "draft", "retired"))
	_ = command.RegisterFlagCompletionFunc("source", staticCompletion("human", "agent", "prod-telemetry"))
	command.RunE = func(command *cobra.Command, args []string) error {
		options.Goal = args[0]
		result, err := module.useCases.Scenario.Add(command.Context(), options)
		if err != nil {
			return err
		}
		if module.outputMode() == "yaml" {
			return clicontract.WriteYAML(command.OutOrStdout(), result.Scenario)
		}
		if module.outputMode() == "json" {
			return clicontract.WriteJSON(command.OutOrStdout(), result.Scenario)
		}
		fmt.Fprintf(command.OutOrStdout(), "Created scenario %s at %s\n", result.Scenario.ID, result.Path)
		return nil
	}
	return command
}

func (module Module) scenarioListCommand() *cobra.Command {
	var status string
	command := &cobra.Command{Use: "list", Short: "List holdout scenarios"}
	command.Flags().StringVar(&status, "status", "", "Filter by status (active, draft, retired)")
	command.RunE = func(command *cobra.Command, _ []string) error {
		result, err := module.useCases.Scenario.List(command.Context(), status)
		if err != nil {
			return err
		}
		// stdout is always exactly one JSON document (`[]` for both empty
		// states) so `ao eval scenario list | jq` never breaks; any human hint
		// goes to stderr. Exit 0.
		if result.MissingDirectory {
			fmt.Fprintln(command.ErrOrStderr(), "No holdout directory found. Run 'ao eval scenario init' first.")
			return clicontract.WriteJSON(command.OutOrStdout(), []aoeval.ScenarioSummary{})
		}
		if len(result.Scenarios) == 0 {
			fmt.Fprintln(command.ErrOrStderr(), "No scenarios found.")
			return clicontract.WriteJSON(command.OutOrStdout(), []aoeval.ScenarioSummary{})
		}
		return clicontract.WriteJSON(command.OutOrStdout(), result.Scenarios)
	}
	return command
}

func (module Module) scenarioValidateCommand() *cobra.Command {
	command := &cobra.Command{Use: "validate", Short: "Validate holdout scenarios against schema"}
	command.RunE = func(command *cobra.Command, _ []string) error {
		result, err := module.useCases.Scenario.Validate(command.Context())
		if err != nil {
			return err
		}
		if result.MissingDirectory {
			fmt.Fprintln(command.OutOrStdout(), "No holdout directory found. Run 'ao eval scenario init' first.")
			return nil
		}
		if len(result.Errors) > 0 {
			fmt.Fprintf(command.ErrOrStderr(), "Validation errors:\n%s\n", strings.Join(result.Errors, "\n"))
			return fmt.Errorf("%d validation error(s) found", len(result.Errors))
		}
		if result.Validated == 0 {
			fmt.Fprintln(command.OutOrStdout(), "No scenario files found to validate.")
		} else {
			fmt.Fprintf(command.OutOrStdout(), "Validated %d scenario(s): all pass\n", result.Validated)
		}
		return nil
	}
	return command
}

func (module Module) scenarioEvaluateCommand() *cobra.Command {
	options := aoeval.ScenarioEvaluateRequest{Timeout: 2 * time.Minute, RunID: "ao-scenario-evaluate"}
	var jsonOutput bool
	command := &cobra.Command{Use: "evaluate", Short: "Evaluate directive-linked scenarios and record satisfaction results", Args: cobra.NoArgs}
	command.Flags().StringVar(&options.DirectiveID, "directive", "", "Evaluate only the directive with this stable Directive ID")
	command.Flags().BoolVar(&options.All, "all", false, "Evaluate every directive's linked scenarios")
	command.Flags().BoolVar(&jsonOutput, "json", false, "Emit the machine-readable evaluation report")
	command.Flags().DurationVar(&options.Timeout, "timeout", 2*time.Minute, "Per-check execution timeout")
	command.Flags().StringVar(&options.RunID, "run-id", "ao-scenario-evaluate", "run_id recorded in the results artifact")
	command.RunE = func(command *cobra.Command, _ []string) error {
		if module.host.ProjectRoot != nil {
			options.ProjectRoot = module.host.ProjectRoot()
		}
		if module.host.GoalsPath != nil {
			options.GoalsPath = module.host.GoalsPath()
		}
		report, err := module.useCases.Scenario.Evaluate(command.Context(), options)
		if err != nil {
			return err
		}
		if module.outputMode() == "yaml" {
			return clicontract.WriteYAML(command.OutOrStdout(), report)
		}
		if jsonOutput || module.outputMode() == "json" {
			return clicontract.WriteJSON(command.OutOrStdout(), report)
		}
		return renderScenarioEvaluate(command, report)
	}
	return command
}

func renderScenarioEvaluate(command *cobra.Command, report *aoeval.ScenarioEvaluateReport) error {
	for _, evaluation := range report.Evaluations {
		if !evaluation.Recorded {
			fmt.Fprintf(command.OutOrStdout(), "%-20s %-10s not recorded — %s\n", evaluation.ScenarioID, "-", evaluation.Note)
		} else {
			fmt.Fprintf(command.OutOrStdout(), "%-20s %-10s shape=%s score=%.2f threshold=%.2f\n", evaluation.ScenarioID, evaluation.Verdict, evaluation.Shape, evaluation.Score, evaluation.Threshold)
		}
	}
	if report.Written == 0 {
		fmt.Fprintln(command.OutOrStdout(), "No results recorded; artifact untouched.")
		return nil
	}
	fmt.Fprintf(command.OutOrStdout(), "Wrote %d result(s) to %s (run %s, iteration %d)\n", report.Written, report.Artifact, report.RunID, report.Iteration)
	return nil
}

func (module Module) scenarioABCommand() *cobra.Command {
	var options aoeval.ScenarioABRequest
	command := &cobra.Command{Use: "scenario-ab", Short: "Run a knowledge-reuse holdout scenario with vs. without the gold pull (the discriminating A/B)", Args: cobra.NoArgs, SilenceUsage: true}
	command.Flags().StringVar(&options.ScenarioPath, "scenario", "", "Path to the scenario.v1 JSON file (required)")
	command.Flags().StringVar(&options.OutputPath, "out", "", "Write the ScenarioDeltaScorecard JSON to this path")
	command.Flags().IntVar(&options.TokenBudget, "token-budget", 0, "Fail the gate if summed arm token cost exceeds this (0 = default 200000)")
	command.Flags().DurationVar(&options.Timeout, "timeout", 0, "Per-arm timeout (0 = default 5m)")
	command.Flags().BoolVar(&options.ControlOnly, "control-only", false, "Run only the without-gold control arm and fail on ceiling/no-headroom")
	command.RunE = func(command *cobra.Command, _ []string) error {
		result, err := module.useCases.ScenarioAB.Run(command.Context(), options)
		if result.Card.ScenarioID != "" {
			if module.outputMode() == "yaml" {
				if renderErr := clicontract.WriteYAML(command.OutOrStdout(), result.Card); renderErr != nil {
					return renderErr
				}
			} else if module.outputMode() == "json" {
				if renderErr := clicontract.WriteJSON(command.OutOrStdout(), result.Card); renderErr != nil {
					return renderErr
				}
			} else {
				renderScenarioAB(command, result.Card)
			}
		}
		if err != nil {
			var gate *aoeval.ScenarioABGateError
			if errors.As(err, &gate) {
				for _, reason := range gate.Reasons {
					fmt.Fprintf(command.ErrOrStderr(), "  FAIL: %s\n", reason)
				}
			}
		}
		return err
	}
	return command
}
func renderScenarioAB(command *cobra.Command, card aoeval.ScenarioDeltaScorecard) {
	if card.CeilingViolation {
		fmt.Fprintf(command.OutOrStdout(), "scenario-ab %s [%s]: CEILING VIOLATION (without=%.4f >= threshold=%.4f) — invalid scenario, no delta emitted\n", card.ScenarioID, card.VerdictClass, card.Without.Score, card.SatisfactionThreshold)
	} else if card.ControlOnly {
		fmt.Fprintf(command.OutOrStdout(), "scenario-ab %s [%s]: control-only headroom=PASS (without=%.4f < threshold=%.4f) tokens=%d gate=%s\n", card.ScenarioID, card.VerdictClass, card.Without.Score, card.SatisfactionThreshold, card.Without.TokenCost, passLabel(card.Gate.Pass))
	} else {
		fmt.Fprintf(command.OutOrStdout(), "scenario-ab %s [%s]: delta=%.4f (with=%.4f without=%.4f) tokens=%d gate=%s %s\n", card.ScenarioID, card.VerdictClass, card.AggregateDelta, card.With.Score, card.Without.Score, card.With.TokenCost+card.Without.TokenCost, passLabel(card.Gate.Pass), moatLabel(card.MoatEligible))
	}
}
func passLabel(pass bool) string {
	if pass {
		return "PASS"
	}
	return "FAIL"
}
func moatLabel(eligible bool) string {
	if eligible {
		return "moat-eligible"
	}
	return "NOT-moat-evidence(plumbing)"
}

func (module Module) scenarioMoatCommand() *cobra.Command {
	var options aoeval.ScenarioMoatRequest
	command := &cobra.Command{Use: "scenario-moat", Short: "Aggregate moat-eligible scenario A/B scorecards into a publication verdict", Args: cobra.NoArgs, SilenceUsage: true}
	command.Flags().StringArrayVar(&options.ScorecardPaths, "scorecard", nil, "Path to a ScenarioDeltaScorecard JSON (repeatable)")
	command.Flags().StringVar(&options.OutputPath, "out", "", "Write the MoatClaimResult JSON to this path")
	command.RunE = func(command *cobra.Command, _ []string) error {
		result, err := module.useCases.ScenarioAB.Moat(command.Context(), options)
		if err != nil {
			var ineligible aoeval.ErrMoatIneligibleScorecard
			if errors.As(err, &ineligible) {
				fmt.Fprintf(command.ErrOrStderr(), "REJECTED: %s\n", err)
			}
			return err
		}
		if module.outputMode() == "yaml" {
			return clicontract.WriteYAML(command.OutOrStdout(), result)
		}
		if module.outputMode() == "json" {
			return clicontract.WriteJSON(command.OutOrStdout(), result)
		}
		fmt.Fprintf(command.OutOrStdout(), "scenario-moat: verdict=%s scenarios=%d mean_delta=%.4f\n  %s\n", result.Verdict, result.ScenarioCount, result.MeanAggregateDelta, result.Reason)
		if options.OutputPath != "" {
			fmt.Fprintf(command.OutOrStdout(), "Moat claim result: %s\n", options.OutputPath)
		}
		return nil
	}
	return command
}

func (module Module) sessionOutcomeCommand() *cobra.Command {
	var sessionID string
	command := &cobra.Command{Use: "session-outcome [transcript-path]", Short: "Analyze session transcript to derive reward signal", Args: cobra.MaximumNArgs(1)}
	command.Flags().StringVar(&sessionID, "session", "", "Session ID (extracted from transcript if not provided)")
	command.RunE = func(command *cobra.Command, args []string) error {
		path := ""
		if len(args) > 0 {
			path = args[0]
		}
		dryRun := module.host.DryRun != nil && module.host.DryRun()
		result, err := module.useCases.Aliases.SessionOutcome(command.Context(), aoeval.SessionOutcomeRequest{TranscriptPath: path, SessionID: sessionID, DryRun: dryRun})
		if err != nil {
			return err
		}
		if result.DryRun {
			fmt.Fprintf(command.OutOrStdout(), "[dry-run] Would analyze transcript: %s\n", result.Transcript)
			return nil
		}
		if module.outputMode() == "yaml" {
			return clicontract.WriteYAML(command.OutOrStdout(), result)
		}
		if module.outputMode() == "json" {
			return clicontract.WriteJSON(command.OutOrStdout(), result)
		}
		fmt.Fprintln(command.OutOrStdout(), "Session Outcome Analysis\n========================")
		fmt.Fprintf(command.OutOrStdout(), "Session ID:  %s\nReward:      %.2f\nLines:       %d\n\nSignals detected:\n", result.SessionID, result.Reward, result.TotalLines)
		for _, signal := range result.Signals {
			status := "✗"
			if signal.Value {
				status = "✓"
			}
			fmt.Fprintf(command.OutOrStdout(), "  %s %-20s (weight: %+.2f)\n", status, signal.Name, signal.Weight)
		}
		return nil
	}
	return command
}

func (module Module) chaosCommand() *cobra.Command {
	command := &cobra.Command{Use: "chaos", Short: "Run a read-only smoke test of the tick membrane", Args: cobra.NoArgs}
	command.RunE = func(command *cobra.Command, _ []string) error {
		output, err := module.useCases.Aliases.Chaos(command.Context())
		if output.Stdout != "" {
			fmt.Fprint(command.OutOrStdout(), output.Stdout)
		}
		if output.Stderr != "" {
			fmt.Fprint(command.ErrOrStderr(), output.Stderr)
		}
		return err
	}
	return command
}

func (module Module) benchCommand() *cobra.Command {
	options := aoeval.BenchRequest{K: 3, SearchBackend: "local-lexical"}
	command := &cobra.Command{Use: "bench", Short: "Run retrieval quality benchmarks"}
	command.Flags().StringVar(&options.Corpus, "corpus", "", "Path to benchmark corpus directory")
	command.Flags().BoolVar(&options.JSON, "json", false, "JSON output")
	command.Flags().IntVar(&options.K, "k", 3, "K for Precision@K")
	command.Flags().BoolVar(&options.Live, "live", false, "Benchmark against real .agents/learnings/ instead of synthetic corpus")
	command.Flags().BoolVar(&options.Global, "global", false, "Include ~/.agents/learnings/ (cross-rig aggregated store, requires --live)")
	command.Flags().StringVar(&options.SearchEval, "search-eval", "", "Path to an ao-search eval manifest with queries and ground_truth paths")
	command.Flags().StringVar(&options.SearchRoot, "search-root", "", "Repo root to search for --search-eval (defaults to current directory)")
	command.Flags().StringVar(&options.SearchBackend, "search-backend", "local-lexical", "Search backend for --search-eval (local-lexical, ao-auto, agentic-rg, wiki-link-expand, rerank-llamacpp)")
	command.Flags().StringVar(&options.CompareBackends, "search-compare-backends", "", "Comma-separated search backends to compare for --search-eval")
	command.RunE = func(command *cobra.Command, _ []string) error {
		options.KChanged = command.Flags().Changed("k")
		options.JSON = options.JSON || module.outputMode() == "json"
		output, err := module.useCases.Bench.Bench(command.Context(), options)
		if output.Stdout != "" {
			fmt.Fprint(command.OutOrStdout(), output.Stdout)
		}
		if output.Stderr != "" {
			fmt.Fprint(command.ErrOrStderr(), output.Stderr)
		}
		return err
	}
	return command
}

func (module Module) renderRun(command *cobra.Command, result aoeval.CoreRunResult) error {
	var value any = result.Run
	if result.Mode == aoeval.CoreRunBaselineAB {
		value = result.Delta
	}
	if result.Mode == aoeval.CoreRunContextAB {
		value = result.ContextDelta
	}
	if module.outputMode() == "yaml" {
		return clicontract.WriteYAML(command.OutOrStdout(), value)
	}
	if module.outputMode() == "json" {
		return clicontract.WriteJSON(command.OutOrStdout(), value)
	}
	switch result.Mode {
	case aoeval.CoreRunContextAB:
		card := result.ContextDelta
		fmt.Fprintf(command.OutOrStdout(), "Eval context-AB %s: context-off=%.4f (%s) context-on=%.4f (%s) delta=%+.4f cases=%d\n", card.SuiteID, result.FirstRun.AggregateScore, result.FirstRun.Status, result.SecondRun.AggregateScore, result.SecondRun.Status, card.AggregateDelta, len(card.PerCase))
		if result.OutputPath != "" {
			fmt.Fprintf(command.OutOrStdout(), "Context delta scorecard: %s\n", result.OutputPath)
		}
	case aoeval.CoreRunBaselineAB:
		card := result.Delta
		fmt.Fprintf(command.OutOrStdout(), "Eval baseline-AB %s: skill-on=%.4f (%s) skill-off=%.4f (%s) delta=%+.4f cases=%d\n", card.SuiteID, result.FirstRun.AggregateScore, result.FirstRun.Status, result.SecondRun.AggregateScore, result.SecondRun.Status, card.AggregateDelta, len(card.PerCase))
		if result.OutputPath != "" {
			fmt.Fprintf(command.OutOrStdout(), "Delta scorecard: %s\n", result.OutputPath)
		}
	default:
		run := result.Run
		fmt.Fprintf(command.OutOrStdout(), "Eval %s: %s (aggregate %.4f, cases %d)\n", run.RunID, run.Status, run.AggregateScore, len(run.CaseResults))
		if result.OutputPath != "" {
			fmt.Fprintf(command.OutOrStdout(), "Run record: %s\n", result.OutputPath)
		}
	}
	return nil
}

func renderCoverage(command *cobra.Command, label string, missing, required []string) {
	if len(missing) > 0 {
		fmt.Fprintf(command.OutOrStdout(), "Missing required %s: %s\n", label, strings.Join(missing, ", "))
		return
	}
	if len(required) > 0 {
		fmt.Fprintf(command.OutOrStdout(), "Required %s covered\n", label)
	}
}

// suiteSchemaHint cites the authoritative suite schema and a working example so
// a failed `ao eval run` points the caller at the exact shape to fix.
const suiteSchemaHint = "see schemas/eval-suite.v1.schema.json for the suite schema and evals/agentops-core for a working example"

// annotateSuiteParseError appends the schema/example citation to the three
// suite-load failure modes surfaced by cli/internal/eval.LoadSuite — missing
// file ("read eval suite"), malformed JSON ("decode eval suite"), and
// schema-invalid ("eval suite validation failed"). Command-layer decoration
// only; the underlying error text in cli/internal/eval is unchanged.
func annotateSuiteParseError(err error) error {
	if err == nil {
		return nil
	}
	msg := err.Error()
	for _, sig := range []string{"read eval suite", "decode eval suite", "eval suite validation failed"} {
		if strings.Contains(msg, sig) {
			return fmt.Errorf("%w (%s)", err, suiteSchemaHint)
		}
	}
	return err
}

func staticCompletion(values ...string) func(*cobra.Command, []string, string) ([]string, cobra.ShellCompDirective) {
	return func(*cobra.Command, []string, string) ([]string, cobra.ShellCompDirective) {
		return append([]string(nil), values...), cobra.ShellCompDirectiveNoFileComp
	}
}