mirror of
https://github.com/infiniflow/ragflow.git
synced 2026-07-17 05:07:23 +08:00
dd20561fca
### Motivation This PR evolves the harness from a pure execution runtime into an **observable, replayable agent evaluation platform**. The current `harness/graph` checkpoint mechanism is insufficient for true event-sourced introspection—we need append-only event logs capturing every tool call, state transition, memory write, and approval decision, enabling deterministic replay, fork/diff, postmortem analysis, and time-travel debugging. ### Key Design Goals 1. **Event-Sourced Execution Model** Replace coarse checkpoints with granular, append-only event logs. Every operation becomes a durable event: tool invocation, state mutation, memory update, human approval. This unlocks deterministic replay, branching execution histories, and regression datasets derived directly from production failures. 2. **First-Class Replay & Evaluation Loop** Replay is not an afterthought—it is a core primitive. A single live run seeds an offline corpus that supports: repeated playback, model substitution, tool result mocking, and strategy comparison. The harness graduates from "executor" to "continuous evaluation platform" where failed production traces convert directly into offline regression suites. 3. **Operational Observability** Beyond raw traces, expose metrics that prove stability over time: - Tool success / failure rates - Approval latency distributions - Retry frequencies - Checkpoint restore reliability - Memory retrieval quality - Cost per completed task - Fork replay pass rates The underlying thesis: the bottleneck for most agent systems is not execution capability, but the inability to **demonstrate continuous, measurable improvement**. ### Type of change - [x] New Feature (non-breaking change which adds functionality)
79 lines
2.7 KiB
Go
79 lines
2.7 KiB
Go
package metrics
|
|
|
|
import (
|
|
"fmt"
|
|
"strings"
|
|
)
|
|
|
|
// Exporter formats agent metrics for output. This is a lightweight
|
|
// alternative to a full Prometheus client, avoiding the dependency.
|
|
type Exporter struct {
|
|
namespace string
|
|
}
|
|
|
|
// NewExporter creates a new metrics exporter.
|
|
func NewExporter(namespace string) *Exporter {
|
|
return &Exporter{namespace: namespace}
|
|
}
|
|
|
|
// ExportText formats metrics as Prometheus-style text output.
|
|
func (e *Exporter) ExportText(m *AgentMetrics) string {
|
|
snap := m.Snapshot()
|
|
var b strings.Builder
|
|
|
|
ns := e.namespace
|
|
if ns != "" {
|
|
ns += "_"
|
|
}
|
|
|
|
// Tool metrics.
|
|
fmt.Fprintf(&b, "# HELP %stool_calls_total Total tool invocations\n", ns)
|
|
fmt.Fprintf(&b, "# TYPE %stool_calls_total counter\n", ns)
|
|
fmt.Fprintf(&b, "%stool_calls_total %d\n", ns, snap.ToolCalls)
|
|
|
|
fmt.Fprintf(&b, "# HELP %stool_success_rate Tool success rate\n", ns)
|
|
fmt.Fprintf(&b, "# TYPE %stool_success_rate gauge\n", ns)
|
|
fmt.Fprintf(&b, "%stool_success_rate %.4f\n", ns, snap.ToolSuccessRate)
|
|
|
|
fmt.Fprintf(&b, "# HELP %stool_retry_rate Tool retry rate\n", ns)
|
|
fmt.Fprintf(&b, "# TYPE %stool_retry_rate gauge\n", ns)
|
|
fmt.Fprintf(&b, "%stool_retry_rate %.4f\n", ns, snap.ToolRetryRate)
|
|
|
|
// Checkpoint metrics.
|
|
fmt.Fprintf(&b, "# HELP %scheckpoint_saves_total Total checkpoint saves\n", ns)
|
|
fmt.Fprintf(&b, "# TYPE %scheckpoint_saves_total counter\n", ns)
|
|
fmt.Fprintf(&b, "%scheckpoint_saves_total %d\n", ns, snap.CheckpointSaves)
|
|
|
|
fmt.Fprintf(&b, "# HELP %scheckpoint_restore_success Checkpoint restore success rate\n", ns)
|
|
fmt.Fprintf(&b, "# TYPE %scheckpoint_restore_success gauge\n", ns)
|
|
fmt.Fprintf(&b, "%scheckpoint_restore_success %.4f\n", ns, snap.CheckpointRestoreSuccess)
|
|
|
|
// Execution metrics.
|
|
fmt.Fprintf(&b, "# HELP %ssteps_total Total supersteps executed\n", ns)
|
|
fmt.Fprintf(&b, "# TYPE %ssteps_total counter\n", ns)
|
|
fmt.Fprintf(&b, "%ssteps_total %d\n", ns, snap.Steps)
|
|
|
|
fmt.Fprintf(&b, "# HELP %snodes_executed_total Total nodes executed\n", ns)
|
|
fmt.Fprintf(&b, "# TYPE %snodes_executed_total counter\n", ns)
|
|
fmt.Fprintf(&b, "%snodes_executed_total %d\n", ns, snap.NodesExecuted)
|
|
|
|
fmt.Fprintf(&b, "# HELP %sinterrupts_total Total interrupts\n", ns)
|
|
fmt.Fprintf(&b, "# TYPE %sinterrupts_total counter\n", ns)
|
|
fmt.Fprintf(&b, "%sinterrupts_total %d\n", ns, snap.InterruptCount)
|
|
|
|
return b.String()
|
|
}
|
|
|
|
// ExportCSV formats metrics as a single CSV row.
|
|
func (e *Exporter) ExportCSV(m *AgentMetrics) string {
|
|
snap := m.Snapshot()
|
|
return fmt.Sprintf("%s,%d,%d,%d,%.4f,%.4f,%d,%d,%.4f,%d,%d,%d,%.6f",
|
|
snap.TraceID,
|
|
snap.ToolCalls, snap.ToolSuccesses, snap.ToolFailures,
|
|
snap.ToolSuccessRate, snap.ToolRetryRate,
|
|
snap.CheckpointSaves, snap.CheckpointRestores,
|
|
snap.CheckpointRestoreSuccess,
|
|
snap.Steps, snap.NodesExecuted, snap.InterruptCount,
|
|
snap.CostPerTask)
|
|
}
|