mirror of
https://github.com/infiniflow/ragflow.git
synced 2026-07-09 12:54:47 +08:00
dd20561fca
### Motivation This PR evolves the harness from a pure execution runtime into an **observable, replayable agent evaluation platform**. The current `harness/graph` checkpoint mechanism is insufficient for true event-sourced introspection—we need append-only event logs capturing every tool call, state transition, memory write, and approval decision, enabling deterministic replay, fork/diff, postmortem analysis, and time-travel debugging. ### Key Design Goals 1. **Event-Sourced Execution Model** Replace coarse checkpoints with granular, append-only event logs. Every operation becomes a durable event: tool invocation, state mutation, memory update, human approval. This unlocks deterministic replay, branching execution histories, and regression datasets derived directly from production failures. 2. **First-Class Replay & Evaluation Loop** Replay is not an afterthought—it is a core primitive. A single live run seeds an offline corpus that supports: repeated playback, model substitution, tool result mocking, and strategy comparison. The harness graduates from "executor" to "continuous evaluation platform" where failed production traces convert directly into offline regression suites. 3. **Operational Observability** Beyond raw traces, expose metrics that prove stability over time: - Tool success / failure rates - Approval latency distributions - Retry frequencies - Checkpoint restore reliability - Memory retrieval quality - Cost per completed task - Fork replay pass rates The underlying thesis: the bottleneck for most agent systems is not execution capability, but the inability to **demonstrate continuous, measurable improvement**. ### Type of change - [x] New Feature (non-breaking change which adds functionality)
60 lines
1.5 KiB
Go
60 lines
1.5 KiB
Go
package core
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"time"
|
|
|
|
"ragflow/internal/harness/core/schema"
|
|
"ragflow/internal/harness/events"
|
|
)
|
|
|
|
// NewEventRecorderToolMiddleware creates a ToolInvokeMiddleware that records
|
|
// every tool invocation to the EventRecorder found in the context.
|
|
//
|
|
// Usage:
|
|
//
|
|
// cfg := &ReActConfig[*schema.Message]{
|
|
// ToolsConfig: &ToolsNodeConfig{
|
|
// ToolInvokeMiddlewares: []ToolInvokeMiddleware{
|
|
// NewEventRecorderToolMiddleware(),
|
|
// },
|
|
// },
|
|
// }
|
|
// ctx = events.ContextWithRecorder(ctx, recorder)
|
|
func NewEventRecorderToolMiddleware() ToolInvokeMiddleware {
|
|
return func(next InvokeTool) InvokeTool {
|
|
return func(ctx context.Context, ictx *ToolInvocationContext) (*schema.ToolResult, error) {
|
|
rec := events.RecorderFromContext(ctx)
|
|
|
|
start := time.Now()
|
|
result, err := next(ctx, ictx)
|
|
durMs := time.Since(start).Milliseconds()
|
|
|
|
if rec == nil {
|
|
return result, err
|
|
}
|
|
|
|
// Extract tool arguments as map (parse from JSON string).
|
|
var args map[string]any
|
|
if ictx.Arguments != nil && ictx.Arguments.Arguments != "" {
|
|
json.Unmarshal([]byte(ictx.Arguments.Arguments), &args)
|
|
}
|
|
|
|
errStr := ""
|
|
retryCount := 0
|
|
if ictx.RetryConfig != nil {
|
|
retryCount = ictx.RetryConfig.MaxAttempts
|
|
}
|
|
if err != nil {
|
|
errStr = err.Error()
|
|
} else if result != nil && result.Error != "" {
|
|
errStr = result.Error
|
|
}
|
|
|
|
rec.RecordToolCall(ctx, ictx.Name, args, result, durMs, retryCount, errStr)
|
|
return result, err
|
|
}
|
|
}
|
|
}
|