mirror of
https://github.com/infiniflow/ragflow.git
synced 2026-07-18 13:47:21 +08:00
dd20561fca
### Motivation This PR evolves the harness from a pure execution runtime into an **observable, replayable agent evaluation platform**. The current `harness/graph` checkpoint mechanism is insufficient for true event-sourced introspection—we need append-only event logs capturing every tool call, state transition, memory write, and approval decision, enabling deterministic replay, fork/diff, postmortem analysis, and time-travel debugging. ### Key Design Goals 1. **Event-Sourced Execution Model** Replace coarse checkpoints with granular, append-only event logs. Every operation becomes a durable event: tool invocation, state mutation, memory update, human approval. This unlocks deterministic replay, branching execution histories, and regression datasets derived directly from production failures. 2. **First-Class Replay & Evaluation Loop** Replay is not an afterthought—it is a core primitive. A single live run seeds an offline corpus that supports: repeated playback, model substitution, tool result mocking, and strategy comparison. The harness graduates from "executor" to "continuous evaluation platform" where failed production traces convert directly into offline regression suites. 3. **Operational Observability** Beyond raw traces, expose metrics that prove stability over time: - Tool success / failure rates - Approval latency distributions - Retry frequencies - Checkpoint restore reliability - Memory retrieval quality - Cost per completed task - Fork replay pass rates The underlying thesis: the bottleneck for most agent systems is not execution capability, but the inability to **demonstrate continuous, measurable improvement**. ### Type of change - [x] New Feature (non-breaking change which adds functionality)
120 lines
3.1 KiB
Go
120 lines
3.1 KiB
Go
package replay
|
|
|
|
import (
|
|
"encoding/json"
|
|
"fmt"
|
|
|
|
"ragflow/internal/harness/events"
|
|
)
|
|
|
|
// ---- common overrides ----
|
|
|
|
// ReplayExactTools returns a ToolOverrideFunc that uses the recorded result
|
|
// unchanged. This is the default behaviour for deterministic replay.
|
|
func ReplayExactTools() ToolOverrideFunc {
|
|
return func(toolName string, args map[string]any, recordedResult any) (any, error) {
|
|
return recordedResult, nil
|
|
}
|
|
}
|
|
|
|
// ReplayLiveTools returns a ToolOverrideFunc that always returns nil,
|
|
// signalling the replay to execute the tool with the real implementation.
|
|
func ReplayLiveTools() ToolOverrideFunc {
|
|
return func(toolName string, args map[string]any, recordedResult any) (any, error) {
|
|
// Return nil to indicate "execute live".
|
|
return nil, nil
|
|
}
|
|
}
|
|
|
|
// ReplaySubstituteModel returns a ModelOverrideFunc that replaces the
|
|
// recorded LLM response with a fixed string. Use this to compare how
|
|
// a different model would change behaviour while keeping tool results frozen.
|
|
//
|
|
// The callback receives the original recorded response and should return
|
|
// the substitute response. Return ("", nil) to suppress the response.
|
|
type ReplayModelCallback func(recordedResponse string) string
|
|
|
|
// ReplaySubstituteModel creates a ModelOverrideFunc from a callback.
|
|
func ReplaySubstituteModel(fn ReplayModelCallback) ModelOverrideFunc {
|
|
return func(_ []any, recordedResponse string) (*string, error) {
|
|
substituted := fn(recordedResponse)
|
|
return &substituted, nil
|
|
}
|
|
}
|
|
|
|
// ---- error types ----
|
|
|
|
type replayError struct {
|
|
msg string
|
|
}
|
|
|
|
func (e *replayError) Error() string { return e.msg }
|
|
|
|
func errorf(format string, args ...any) error {
|
|
return &replayError{msg: fmt.Sprintf(format, args...)}
|
|
}
|
|
|
|
// ---- helpers ----
|
|
|
|
func jsonUnmarshal(data []byte, target any) error {
|
|
return json.Unmarshal(data, target)
|
|
}
|
|
|
|
func jsonMarshal(v any) ([]byte, error) {
|
|
return json.Marshal(v)
|
|
}
|
|
|
|
// copyEvent creates a shallow copy of an Event with a deep copy of Payload and Metadata.
|
|
func copyEvent(ev *events.Event) *events.Event {
|
|
cp := *ev
|
|
if ev.Payload != nil {
|
|
cp.Payload = make([]byte, len(ev.Payload))
|
|
copy(cp.Payload, ev.Payload)
|
|
}
|
|
if ev.Metadata != nil {
|
|
cp.Metadata = make(map[string]any, len(ev.Metadata))
|
|
for k, v := range ev.Metadata {
|
|
cp.Metadata[k] = v
|
|
}
|
|
}
|
|
if ev.CausedBy != nil {
|
|
cp.CausedBy = make([]events.EventID, len(ev.CausedBy))
|
|
copy(cp.CausedBy, ev.CausedBy)
|
|
}
|
|
return &cp
|
|
}
|
|
|
|
// ---- event helpers for test assertions ----
|
|
|
|
// FindEventsOfType filters events by type.
|
|
func FindEventsOfType(evts []*events.Event, typ events.EventType) []*events.Event {
|
|
var result []*events.Event
|
|
for _, ev := range evts {
|
|
if ev.Type == typ {
|
|
result = append(result, ev)
|
|
}
|
|
}
|
|
return result
|
|
}
|
|
|
|
// EventsContains checks if any event has the given type.
|
|
func EventsContains(evts []*events.Event, typ events.EventType) bool {
|
|
for _, ev := range evts {
|
|
if ev.Type == typ {
|
|
return true
|
|
}
|
|
}
|
|
return false
|
|
}
|
|
|
|
// EventCount counts events of a given type.
|
|
func EventCount(evts []*events.Event, typ events.EventType) int {
|
|
count := 0
|
|
for _, ev := range evts {
|
|
if ev.Type == typ {
|
|
count++
|
|
}
|
|
}
|
|
return count
|
|
}
|