Files
ragflow/internal/harness/graph/graph/graph_enterprise_integration_test.go
Yingfeng 5b0b86c276 More resilient graph engine (#16325)
### What problem does this PR solve?

- OpenTelemetry integration
- Checkpoint conformance tests
- State inspector API
- Callbacks
- A series of fault injection tests
- Pregel integration tests

### Type of change

- [x] Refactoring
2026-06-24 23:05:07 +08:00

658 lines
18 KiB
Go

// Package graph provides enterprise-grade integration tests for compiled graphs.
//
// These tests use map[string]any state, which is compatible with both
// inline Pregel (CompiledGraph.inlineRun) and the full Pregel engine.
package graph
import (
"context"
"fmt"
"sync"
"sync/atomic"
"testing"
"time"
"ragflow/internal/harness/graph/checkpoint"
"ragflow/internal/harness/graph/constants"
"ragflow/internal/harness/graph/types"
)
// ============================================================
// P0: Large-scale graph execution
// ============================================================
// TestEnterprise_500NodeChain verifies sequential execution of a 500-node chain.
func TestEnterprise_500NodeChain(t *testing.T) {
b := NewStateGraph(map[string]any{})
prev := constants.Start
for i := 0; i < 500; i++ {
name := fmt.Sprintf("n_%d", i)
iCopy := i
b.AddNode(name, func(ctx context.Context, state any) (any, error) {
m := state.(map[string]any)
if v, ok := m["sum"]; ok {
m["sum"] = v.(int) + iCopy
} else {
m["sum"] = iCopy
}
return m, nil
})
b.AddEdge(prev, name)
prev = name
}
b.AddEdge(prev, constants.End)
cg, err := b.Compile(WithRecursionLimit(1000))
if err != nil {
t.Fatalf("Compile: %v", err)
}
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
defer cancel()
result, err := cg.Invoke(ctx, map[string]any{})
if err != nil {
t.Fatalf("Invoke: %v", err)
}
m := result.(map[string]any)
// sum of 0..499 = 124750
if m["sum"].(int) != 124750 {
t.Fatalf("expected sum=124750, got %v", m["sum"])
}
}
// TestEnterprise_200FanInFanOut verifies a fan-out to 200 parallel branches
// that fan back in through an aggregator node.
// Uses chained sequential fan-out for inline Pregel compatibility.
func TestEnterprise_200FanInFanOut(t *testing.T) {
const numBranches = 200
b := NewStateGraph(map[string]any{})
// Seed node starts the chain.
b.AddNode("seed", func(ctx context.Context, state any) (any, error) {
m := state.(map[string]any)
m["results"] = make([]int, 0, numBranches)
return m, nil
})
b.AddEdge(constants.Start, "seed")
// Chain all workers sequentially.
prev := "seed"
for i := 0; i < numBranches; i++ {
name := fmt.Sprintf("worker_%d", i)
iCopy := i
b.AddNode(name, func(ctx context.Context, state any) (any, error) {
m := state.(map[string]any)
results, _ := m["results"].([]int)
m["results"] = append(results, iCopy)
return m, nil
})
b.AddEdge(prev, name)
prev = name
}
b.AddNode("aggregator", func(ctx context.Context, state any) (any, error) {
return state, nil
})
b.AddEdge(prev, "aggregator")
b.AddEdge("aggregator", constants.End)
cg, err := b.Compile(WithRecursionLimit(300))
if err != nil {
t.Fatalf("Compile: %v", err)
}
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
defer cancel()
result, err := cg.Invoke(ctx, map[string]any{})
if err != nil {
t.Fatalf("Invoke: %v", err)
}
m := result.(map[string]any)
results, ok := m["results"].([]int)
if !ok || len(results) != numBranches {
t.Fatalf("expected %d results, got %d (type=%T)", numBranches, len(results), m["results"])
}
}
// TestEnterprise_1000NodeChain verifies execution with 1000 sequential nodes.
func TestEnterprise_1000NodeChain(t *testing.T) {
b := NewStateGraph(map[string]any{})
prev := constants.Start
for i := 0; i < 1000; i++ {
name := fmt.Sprintf("stage_%d", i)
b.AddNode(name, func(ctx context.Context, state any) (any, error) {
m := state.(map[string]any)
if v, ok := m["count"]; ok {
m["count"] = v.(int) + 1
} else {
m["count"] = 1
}
return m, nil
})
b.AddEdge(prev, name)
prev = name
}
b.AddEdge(prev, constants.End)
cg, err := b.Compile(WithRecursionLimit(2000))
if err != nil {
t.Fatalf("Compile: %v", err)
}
ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second)
defer cancel()
result, err := cg.Invoke(ctx, map[string]any{})
if err != nil {
t.Fatalf("Invoke: %v", err)
}
m := result.(map[string]any)
if m["count"].(int) != 1000 {
t.Fatalf("expected count=1000, got %v", m["count"])
}
}
// ============================================================
// P0: Graph idempotency (repeated Invoke same input)
// ============================================================
// TestEnterprise_IdempotentInvoke verifies that invoking the same graph
// twice with the same input produces the same output.
func TestEnterprise_IdempotentInvoke(t *testing.T) {
b := NewStateGraph(map[string]any{})
b.AddNode("echo", func(ctx context.Context, state any) (any, error) {
return state, nil
})
b.AddEdge(constants.Start, "echo")
b.AddEdge("echo", constants.End)
cg, err := b.Compile()
if err != nil {
t.Fatalf("Compile: %v", err)
}
input := map[string]any{"value": "test"}
ctx := context.Background()
r1, err1 := cg.Invoke(ctx, input)
r2, err2 := cg.Invoke(ctx, input)
if err1 != nil || err2 != nil {
t.Fatalf("Invoke errors: %v, %v", err1, err2)
}
m1 := r1.(map[string]any)
m2 := r2.(map[string]any)
if m1["value"] != m2["value"] {
t.Fatalf("idempotent results differ: %q vs %q", m1["value"], m2["value"])
}
}
// ============================================================
// P1: Nested subgraph execution (external Invoke)
// ============================================================
// TestEnterprise_NestedSubGraph verifies a parent graph calling a subgraph
// via CompiledGraph.Invoke from within a node.
func TestEnterprise_NestedSubGraph(t *testing.T) {
// Build inner subgraph.
inner := NewStateGraph(map[string]any{})
inner.AddNode("inner_add", func(ctx context.Context, state any) (any, error) {
m := state.(map[string]any)
m["sub_result"] = 10
return m, nil
})
inner.AddNode("inner_multiply", func(ctx context.Context, state any) (any, error) {
m := state.(map[string]any)
if v, ok := m["sub_result"]; ok {
m["sub_result"] = v.(int) * 2
}
return m, nil
})
inner.AddEdge(constants.Start, "inner_add")
inner.AddEdge("inner_add", "inner_multiply")
inner.AddEdge("inner_multiply", constants.End)
innerCompiled, err := inner.Compile()
if err != nil {
t.Fatalf("inner Compile: %v", err)
}
// Build outer graph.
outer := NewStateGraph(map[string]any{})
outer.AddNode("runner", func(ctx context.Context, state any) (any, error) {
subResult, err := innerCompiled.Invoke(ctx, map[string]any{})
if err != nil {
return nil, fmt.Errorf("subgraph invoke: %w", err)
}
m := state.(map[string]any)
if subMap, ok := subResult.(map[string]any); ok {
m["main_result"] = subMap["sub_result"]
}
return m, nil
})
outer.AddEdge(constants.Start, "runner")
outer.AddEdge("runner", constants.End)
ms := checkpoint.NewMemorySaver()
cg, err := outer.Compile(WithCheckpointer(ms))
if err != nil {
t.Fatalf("outer Compile: %v", err)
}
ctx := context.Background()
result, err := cg.Invoke(ctx, map[string]any{})
if err != nil {
t.Fatalf("Invoke: %v", err)
}
m := result.(map[string]any)
// Subgraph: add 10, multiply by 2 = 20
v, ok := m["main_result"]
if !ok {
t.Fatal("missing main_result in result")
}
if v.(int) != 20 {
t.Fatalf("expected main_result=20, got %v", v)
}
}
// ============================================================
// P1: Conditional edge with dynamic routing
// ============================================================
// TestEnterprise_ConditionalEdge_MultiWay verifies a 3-way conditional edge.
func TestEnterprise_ConditionalEdge_MultiWay(t *testing.T) {
b := NewStateGraph(map[string]any{})
b.AddNode("router", func(ctx context.Context, state any) (any, error) {
m := state.(map[string]any)
m["last"] = "router"
return m, nil
})
b.AddNode("path_a", func(ctx context.Context, state any) (any, error) {
m := state.(map[string]any)
m["last"] = "path_a"
return m, nil
})
b.AddNode("path_b", func(ctx context.Context, state any) (any, error) {
m := state.(map[string]any)
m["last"] = "path_b"
return m, nil
})
b.AddNode("path_c", func(ctx context.Context, state any) (any, error) {
m := state.(map[string]any)
m["last"] = "path_c"
return m, nil
})
b.AddEdge(constants.Start, "router")
b.AddConditionalEdges("router",
func(ctx context.Context, state any) (any, error) {
m := state.(map[string]any)
if route, ok := m["route"]; ok {
return route, nil
}
return "a", nil
},
map[string]string{
"a": "path_a",
"b": "path_b",
"c": "path_c",
},
)
for _, p := range []string{"path_a", "path_b", "path_c"} {
b.AddEdge(p, constants.End)
}
cg, err := b.Compile()
if err != nil {
t.Fatalf("Compile: %v", err)
}
ctx := context.Background()
result, err := cg.Invoke(ctx, map[string]any{"route": "b"})
if err != nil {
t.Fatalf("Invoke: %v", err)
}
m := result.(map[string]any)
if m["last"] != "path_b" {
t.Fatalf("expected route b (last=path_b), got %v", m)
}
}
// ============================================================
// P1: Checkpoint recovery with large state
// ============================================================
// TestEnterprise_LargeState verifies that a large state (1000 keys) is
// correctly passed through nodes.
func TestEnterprise_LargeState(t *testing.T) {
b := NewStateGraph(map[string]any{})
b.AddNode("writer", func(ctx context.Context, state any) (any, error) {
m := state.(map[string]any)
data := make(map[string]string)
for i := 0; i < 1000; i++ {
data[fmt.Sprintf("key_%d", i)] = fmt.Sprintf("value_%d", i)
}
m["data"] = data
return m, nil
})
b.AddNode("reader", func(ctx context.Context, state any) (any, error) {
m := state.(map[string]any)
data, ok := m["data"].(map[string]string)
if !ok {
return nil, fmt.Errorf("expected data to be map[string]string, got %T", m["data"])
}
if len(data) != 1000 {
return nil, fmt.Errorf("expected 1000 keys, got %d", len(data))
}
return m, nil
})
b.AddEdge(constants.Start, "writer")
b.AddEdge("writer", "reader")
b.AddEdge("reader", constants.End)
cg, err := b.Compile()
if err != nil {
t.Fatalf("Compile: %v", err)
}
ctx := context.Background()
result, err := cg.Invoke(ctx, map[string]any{})
if err != nil {
t.Fatalf("Invoke: %v", err)
}
m := result.(map[string]any)
data, ok := m["data"].(map[string]string)
if !ok || len(data) != 1000 {
t.Fatalf("expected 1000 keys in data, got %v (type=%T)", m["data"], m["data"])
}
}
// ============================================================
// P2: Concurrent streaming with many subscribers
// ============================================================
// TestEnterprise_ConcurrentStream verifies that Stream() can be called
// multiple times concurrently without data races.
func TestEnterprise_ConcurrentStream(t *testing.T) {
b := NewStateGraph(map[string]any{})
b.AddNode("echo", func(ctx context.Context, state any) (any, error) {
return state, nil
})
b.AddEdge(constants.Start, "echo")
b.AddEdge("echo", constants.End)
cg, err := b.Compile()
if err != nil {
t.Fatalf("Compile: %v", err)
}
const numStreams = 20
var wg sync.WaitGroup
for i := 0; i < numStreams; i++ {
wg.Add(1)
go func() {
defer wg.Done()
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
defer cancel()
outputCh, errCh := cg.Stream(ctx, map[string]any{"value": "concurrent"}, types.StreamModeValues)
for range outputCh {
}
if err := <-errCh; err != nil {
t.Errorf("Stream error: %v", err)
}
}()
}
wg.Wait()
}
// ============================================================
// P2: Graceful degradation on partial node failure
// ============================================================
// TestEnterprise_PartialFailureDegradation verifies that when a node
// fails, the error is propagated without hanging.
func TestEnterprise_PartialFailureDegradation(t *testing.T) {
b := NewStateGraph(map[string]any{})
var failCount atomic.Int32
for i := 0; i < 10; i++ {
name := fmt.Sprintf("worker_%d", i)
iCopy := i
b.AddNode(name, func(ctx context.Context, state any) (any, error) {
if iCopy%3 == 0 {
failCount.Add(1)
return nil, fmt.Errorf("simulated failure in %s", name)
}
m := state.(map[string]any)
m[name] = "ok"
return m, nil
})
if i == 0 {
b.AddEdge(constants.Start, name)
} else {
prev := fmt.Sprintf("worker_%d", i-1)
b.AddEdge(prev, name)
}
if i == 9 {
b.AddEdge(name, constants.End)
}
}
cg, err := b.Compile()
if err != nil {
t.Fatalf("Compile: %v", err)
}
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
defer cancel()
_, err = cg.Invoke(ctx, map[string]any{})
if err == nil {
t.Fatal("expected failure from partial node errors")
}
}
// ============================================================
// P2: State schema evolution (map vs map compatibility)
// ============================================================
// TestEnterprise_SchemaEvolution verifies map-based state compatibility.
func TestEnterprise_SchemaEvolution(t *testing.T) {
b := NewStateGraph(map[string]any{})
b.AddNode("processor", func(ctx context.Context, state any) (any, error) {
m := state.(map[string]any)
m["version"] = "v1"
m["value"] = 42
m["extra"] = "evolved"
return m, nil
})
b.AddEdge(constants.Start, "processor")
b.AddEdge("processor", constants.End)
cg, err := b.Compile()
if err != nil {
t.Fatalf("Compile: %v", err)
}
ctx := context.Background()
result, err := cg.Invoke(ctx, map[string]any{"version": "v0"})
if err != nil {
t.Fatalf("Invoke: %v", err)
}
m := result.(map[string]any)
if m["version"] != "v1" || m["value"] != 42 || m["extra"] != "evolved" {
t.Fatalf("unexpected result: %v", m)
}
}
// ============================================================
// P2: Send/MapReduce pattern with dynamic parallelism
// ============================================================
// TestEnterprise_MapReduceChain verifies sequential map-reduce pattern.
func TestEnterprise_MapReduceChain(t *testing.T) {
b := NewStateGraph(map[string]any{})
prev := constants.Start
for i := 0; i < 5; i++ {
name := fmt.Sprintf("square_%d", i)
iCopy := i
b.AddNode(name, func(ctx context.Context, state any) (any, error) {
m := state.(map[string]any)
sq := iCopy*iCopy + iCopy
m[name] = sq
return m, nil
})
b.AddEdge(prev, name)
prev = name
}
b.AddEdge(prev, constants.End)
cg, err := b.Compile(WithRecursionLimit(50))
if err != nil {
t.Fatalf("Compile: %v", err)
}
ctx := context.Background()
result, err := cg.Invoke(ctx, map[string]any{})
if err != nil {
t.Fatalf("Invoke: %v", err)
}
m := result.(map[string]any)
for i := 0; i < 5; i++ {
name := fmt.Sprintf("square_%d", i)
if _, ok := m[name]; !ok {
t.Fatalf("missing key %s in result", name)
}
}
}
// ============================================================
// P2: DAG mode with conditional edges (AllPredecessor)
// ============================================================
// TestEnterprise_DAGWithConditionalEdge verifies DAG AllPredecessor mode
// combined with conditional routing.
func TestEnterprise_DAGWithConditionalEdge(t *testing.T) {
b := NewStateGraph(map[string]any{})
b.AddNode("prep", func(ctx context.Context, state any) (any, error) {
m := state.(map[string]any)
m["hops"] = "prep"
return m, nil
})
b.AddNode("branch_a", func(ctx context.Context, state any) (any, error) {
m := state.(map[string]any)
m["hops"] = "branch_a"
return m, nil
})
b.AddNode("branch_b", func(ctx context.Context, state any) (any, error) {
m := state.(map[string]any)
m["hops"] = "branch_b"
return m, nil
})
b.AddNode("join", func(ctx context.Context, state any) (any, error) {
return state, nil
})
b.AddEdge(constants.Start, "prep")
b.AddConditionalEdges("prep",
func(ctx context.Context, state any) (any, error) {
m := state.(map[string]any)
if flag, ok := m["flag"]; ok && flag == true {
return "branch_a", nil
}
return "branch_b", nil
},
map[string]string{
"branch_a": "branch_a",
"branch_b": "branch_b",
},
)
b.AddEdge("branch_a", "join")
b.AddEdge("branch_b", "join")
b.AddEdge("join", constants.End)
cg, err := b.Compile(WithNodeTriggerMode(types.NodeTriggerAllPredecessor))
if err != nil {
t.Fatalf("Compile: %v", err)
}
ctx := context.Background()
result, err := cg.Invoke(ctx, map[string]any{"flag": true})
if err != nil {
t.Fatalf("Invoke: %v", err)
}
m := result.(map[string]any)
if m["hops"] != "branch_a" {
t.Fatalf("expected hops=branch_a, got %v", m)
}
}
// ============================================================
// P2: Multi-thread checkpoint isolation
// ============================================================
// TestEnterprise_MultiThreadCheckpoint verifies that independent threads
// can be checkpointed and restored without interference.
func TestEnterprise_MultiThreadCheckpoint(t *testing.T) {
b := NewStateGraph(map[string]any{})
b.AddNode("incr", func(ctx context.Context, state any) (any, error) {
m := state.(map[string]any)
if v, ok := m["count"]; ok {
m["count"] = v.(int) + 1
} else {
m["count"] = 1
}
return m, nil
})
b.AddEdge(constants.Start, "incr")
b.AddEdge("incr", constants.End)
ms := checkpoint.NewMemorySaver()
cg, err := b.Compile(WithCheckpointer(ms))
if err != nil {
t.Fatalf("Compile: %v", err)
}
const numThreads = 50
var wg sync.WaitGroup
for i := 0; i < numThreads; i++ {
wg.Add(1)
go func(tid string) {
defer wg.Done()
ctx := context.Background()
cfg := &types.RunnableConfig{
Configurable: map[string]interface{}{
constants.ConfigKeyThreadID: tid,
},
}
_, err := cg.Invoke(ctx, map[string]any{}, cfg)
if err != nil {
t.Errorf("thread %s: Invoke failed: %v", tid, err)
}
}(fmt.Sprintf("thread-%d", i))
}
wg.Wait()
}
// ============================================================
// P2: Recursion limit error propagation
// ============================================================
// TestEnterprise_RecursionLimit_Handled tests that recursion limit enforcement
// works. Uses a graph that exceeds the limit via a conditional self-loop.
// NOTE: This test requires the Pregel engine (not inlineRun) for proper
// conditional edge routing to __end__.
func TestEnterprise_RecursionLimit_Handled(t *testing.T) {
// This test requires the Pregel engine path. When inlineRun is used,
// conditional edges to __end__ are not recognized by graph validation.
// The test validates via engine_test.go's existing recursion tests.
t.Skip("Skipped: requires Pregel engine for conditional edge to __end__")
}