// Package component — retry decorator tests. // // These tests exercise the retryInvoker wrapper directly. The wrapper // is the chat-level retry loop introduced to mirror Python's // max_retries/delay_after_error semantics (agent/component/llm.py, // driven by LLMBundle in rag/llm/chat_model.py). Unlike the // existing one-shot structured-output retry (in LLMComponent.Invoke), // the retry loop lives at the ChatInvoker boundary so it covers // every chat path: LLM, Agent, citation grounding. package component import ( "context" "errors" "strings" "sync/atomic" "testing" "time" ) // scriptedInvoker fails the first failTimes calls then succeeds. // err is returned on every failing call (asserted via errors.Is). type scriptedInvoker struct { failTimes int32 err error resp *ChatInvokeResponse calls int32 } func (s *scriptedInvoker) Invoke(_ context.Context, _ ChatInvokeRequest) (*ChatInvokeResponse, error) { n := atomic.AddInt32(&s.calls, 1) if n <= atomic.LoadInt32(&s.failTimes) { return nil, s.err } return s.resp, nil } func (s *scriptedInvoker) callCount() int { return int(atomic.LoadInt32(&s.calls)) } // alwaysFailInvoker returns err on every call. Used to exercise the // exhaustion path. type alwaysFailInvoker struct { err error calls int32 } func (a *alwaysFailInvoker) Invoke(_ context.Context, _ ChatInvokeRequest) (*ChatInvokeResponse, error) { atomic.AddInt32(&a.calls, 1) return nil, a.err } func (a *alwaysFailInvoker) callCount() int { return int(atomic.LoadInt32(&a.calls)) } // TestRetryInvoker_SucceedsOnSecondAttempt: 1 failure, 1 success — // the loop must retry exactly once and return the success response // without surfacing the error. func TestRetryInvoker_SucceedsOnSecondAttempt(t *testing.T) { want := &ChatInvokeResponse{Content: "ok", Model: "m", Stopped: true} inner := &scriptedInvoker{failTimes: 1, err: errors.New("transient"), resp: want} r := newRetryInvoker(inner, 3, time.Millisecond) resp, err := r.Invoke(context.Background(), ChatInvokeRequest{ModelName: "m"}) if err != nil { t.Fatalf("Invoke: unexpected err: %v", err) } if resp != want { t.Errorf("resp=%v, want %v", resp, want) } if got := inner.callCount(); got != 2 { t.Errorf("inner.calls=%d, want 2 (1 fail + 1 success)", got) } } // TestRetryInvoker_ExhaustsRetries: failures exceed the budget — // the loop must stop after maxRetries+1 attempts and wrap the last // error with the retry count. func TestRetryInvoker_ExhaustsRetries(t *testing.T) { sentinel := errors.New("permanent") inner := &alwaysFailInvoker{err: sentinel} r := newRetryInvoker(inner, 3, time.Millisecond) _, err := r.Invoke(context.Background(), ChatInvokeRequest{ModelName: "m"}) if err == nil { t.Fatal("expected error after exhaustion") } if !errors.Is(err, sentinel) { t.Errorf("err does not wrap sentinel: %v", err) } if got, want := inner.callCount(), 4; got != want { // 1 initial + 3 retries t.Errorf("inner.calls=%d, want %d (1 + maxRetries)", got, want) } if !strings.Contains(err.Error(), "3 retries") { t.Errorf("error message missing retry count: %q", err.Error()) } } // TestRetryInvoker_HonorsContextCancellation: a ctx cancelled // during backoff must abort the sleep and return ctx.Err() promptly, // not wait out the full delay. // // The semantic assertion is two-fold: // 1. The returned error wraps context.Canceled. // 2. The inner invoker was not called more than twice (the first // attempt + possibly a second one if cancel landed between the // attempt and the backoff sleep). A regression that ignored // ctx during backoff would burn through the full 5-attempt // budget and the inner counter would climb to 5. // // We do NOT assert on wall-clock elapsed time. The 30s backoff is // the test's own punishment for regressions — if the loop hangs, // the call simply takes 30s and the Go test runner's timeout (or // the surrounding CI budget) catches it. There is no load-bearing // information in "elapsed < 2s" that isn't already covered by the // counter check. func TestRetryInvoker_HonorsContextCancellation(t *testing.T) { inner := &alwaysFailInvoker{err: errors.New("transient")} // 30s delay so the test would obviously hang if ctx were not // honored. The cancellation lands within milliseconds. r := newRetryInvoker(inner, 5, 30*time.Second) ctx, cancel := context.WithCancel(context.Background()) // Cancel after a short delay, while the retry is sleeping // through the first backoff. go func() { time.Sleep(20 * time.Millisecond) cancel() }() _, err := r.Invoke(ctx, ChatInvokeRequest{ModelName: "m"}) if err == nil { t.Fatal("expected error from cancelled context") } if !errors.Is(err, context.Canceled) { t.Errorf("err=%v, want context.Canceled", err) } // First call happens, then we cancel during the first backoff. // The retry loop should not have made more than 2 calls. if got := inner.callCount(); got > 2 { t.Errorf("inner.calls=%d, want <= 2 (ctx cancel should abort backoff)", got) } } // TestRetryInvoker_ExponentialBackoff: the retry loop's backoff // function must grow exponentially (with full jitter) — a constant // backoff would burn the same 20ms per attempt and the loop would // finish far too quickly. // // Semantic check: the LOWER bound on elapsed wall-clock time is // the load-bearing assertion (proves the loop actually slept // between attempts with growing delays). A regression to a // constant backoff would slip below the lower bound. We assert // the upper bound semantically via the call counter — the loop // must make exactly 4 calls (1 initial + 3 retries), and each // retry must observe a non-zero backoff. We do NOT assert on a // fragile upper wall-clock bound: a slow CI runner could push // elapsed past 2s without any real regression. func TestRetryInvoker_ExponentialBackoff(t *testing.T) { inner := &alwaysFailInvoker{err: errors.New("transient")} const initial = 20 * time.Millisecond r := newRetryInvoker(inner, 3, initial) start := time.Now() _, _ = r.Invoke(context.Background(), ChatInvokeRequest{ModelName: "m"}) elapsed := time.Since(start) // 20 + 40 + 80 = 140ms of backoff (3 retries, 4 attempts total). // Use 130ms as the lower bound to avoid CI flakes from clock // granularity. A constant-delay regression would slip below this. if elapsed < 130*time.Millisecond { t.Errorf("elapsed=%v, want >= 130ms (proves exponential doubling, not constant delay)", elapsed) } if got, want := inner.callCount(), 4; got != want { t.Errorf("inner.calls=%d, want %d (1 initial + 3 retries)", got, want) } } // TestRetryInvoker_NoRetriesWhenZero: maxRetries=0 means a single // attempt with no retry on failure. Mirrors LLMParam.MaxRetries=0 // for latency-sensitive flows. func TestRetryInvoker_NoRetriesWhenZero(t *testing.T) { inner := &alwaysFailInvoker{err: errors.New("nope")} r := newRetryInvoker(inner, 0, 50*time.Millisecond) _, err := r.Invoke(context.Background(), ChatInvokeRequest{ModelName: "m"}) if err == nil { t.Fatal("expected error") } if got := inner.callCount(); got != 1 { t.Errorf("inner.calls=%d, want 1 (no retries)", got) } } // TestRetryInvoker_NilInner: a defensive nil check — the wrapper // should not panic when constructed with nil inner. func TestRetryInvoker_NilInner(t *testing.T) { r := newRetryInvoker(nil, 3, time.Millisecond) _, err := r.Invoke(context.Background(), ChatInvokeRequest{ModelName: "m"}) if err == nil { t.Fatal("expected error for nil inner") } } // TestLLMParam_RespectsMaxRetries: an LLMComponent configured with // MaxRetries=5 should exhaust after 6 attempts (1 initial + 5 // retries) when the invoker always fails. This is the integration // test for the param-override path through resolveChatInvoker. func TestLLMParam_RespectsMaxRetries(t *testing.T) { inner := &alwaysFailInvoker{err: errors.New("downstream dead")} withStubInvoker(t, inner) c := NewLLMComponent(LLMParam{ ModelID: "m", MaxRetries: 5, }) // Force the param to a tiny delay so the test is fast. The // zero-value default is 2s, which would make this test slow. c.param.DelayAfterError = time.Millisecond _, err := c.Invoke(context.Background(), map[string]any{"user_prompt": "x"}) if err == nil { t.Fatal("expected error from exhausted retries") } // 1 initial + 5 retries = 6 invoker calls. if got, want := inner.callCount(), 6; got != want { t.Errorf("inner.calls=%d, want %d", got, want) } } // TestLLMParam_ZeroRetriesMeansOneAttempt: MaxRetries=0 must bypass // retries entirely (the param-override path passes through // resolveChatInvoker and a fresh retryInvoker with maxRetries=0). func TestLLMParam_ZeroRetriesMeansOneAttempt(t *testing.T) { inner := &alwaysFailInvoker{err: errors.New("once")} withStubInvoker(t, inner) c := NewLLMComponent(LLMParam{ ModelID: "m", MaxRetries: 0, }) // MaxRetries=0 with default zero-value DelayAfterError triggers // the "no param override" path through resolveChatInvoker, which // returns the package default (3 retries). To genuinely request // zero retries we set DelayAfterError to a non-zero sentinel so // resolveChatInvoker wraps the default in a new retryInvoker // with maxRetries=0. c.param.DelayAfterError = time.Millisecond _, err := c.Invoke(context.Background(), map[string]any{"user_prompt": "x"}) if err == nil { t.Fatal("expected error") } if got := inner.callCount(); got != 1 { t.Errorf("inner.calls=%d, want 1 (zero retries)", got) } } // TestLLMParam_DefaultRetries: with MaxRetries and // DelayAfterError both zero (the v1 fixture default), the // component should still retry up to the package default // (retryInvokerDefaultRetries=3). This protects against // regressions where a future change accidentally bypasses the // retry loop on the hot path. func TestLLMParam_DefaultRetries(t *testing.T) { inner := &alwaysFailInvoker{err: errors.New("flaky")} withStubInvoker(t, inner) c := NewLLMComponent(LLMParam{ModelID: "m"}) // Both fields zero — the test relies on the package default // being applied. The default initial delay is 2s, which is too // slow for a unit test, so we mutate the package default // indirectly: the test cannot reach into the retry decorator // (it's wrapped by resolveChatInvoker), so we instead assert // behavior with a manually-fast retryInvoker injected via // SetDefaultChatInvoker. This is the more honest test. fastInner := &alwaysFailInvoker{err: errors.New("flaky")} SetDefaultChatInvoker(newRetryInvoker(fastInner, 2, time.Millisecond)) t.Cleanup(func() { SetDefaultChatInvoker(nil) }) _, err := c.Invoke(context.Background(), map[string]any{"user_prompt": "x"}) if err == nil { t.Fatal("expected error after default retries") } // 1 initial + 2 retries = 3. if got, want := fastInner.callCount(), 3; got != want { t.Errorf("inner.calls=%d, want %d", got, want) } // Original (un-wrapped) inner should have been called 0 times // because resolveChatInvoker returned the package default (the // one we just installed), not the one passed via withStubInvoker. if got := inner.callCount(); got != 0 { t.Errorf("unused inner.calls=%d, want 0", got) } } // TestUnwrapChatInvoker_StripsSingleRetryLayer is the unit-level // test for the unwrapChatInvoker helper. It must peel off a // single retryInvoker layer to return the bare invoker // underneath, so the param-override path can install a fresh // retryInvoker with the operator's literal MaxRetries without // multiplicatively stacking on the boot retry. func TestUnwrapChatInvoker_StripsSingleRetryLayer(t *testing.T) { bare := &alwaysFailInvoker{err: errors.New("bare")} wrapped := newRetryInvoker(bare, 3, time.Millisecond) if got := unwrapChatInvoker(wrapped); got != bare { t.Errorf("unwrapChatInvoker(retry(bare)) = %v, want %v (bare invoker)", got, bare) } } // TestUnwrapChatInvoker_NoRetryLayer verifies that a bare // (non-retry) invoker passes through unwrapChatInvoker // unchanged. The function must not wrap or modify the input // when no retry layers are present. func TestUnwrapChatInvoker_NoRetryLayer(t *testing.T) { bare := &alwaysFailInvoker{err: errors.New("bare")} if got := unwrapChatInvoker(bare); got != bare { t.Errorf("unwrapChatInvoker(bare) = %v, want %v (unchanged passthrough)", got, bare) } } // TestUnwrapChatInvoker_StripsMultipleLayers is the defensive // case: a chain of retryInvokers (production only installs one, // but pathological callers could nest) is peeled down to the // bare invoker. The loop bounds the walk at the first // non-retryInvoker layer. func TestUnwrapChatInvoker_StripsMultipleLayers(t *testing.T) { bare := &alwaysFailInvoker{err: errors.New("bare")} double := newRetryInvoker(newRetryInvoker(bare, 3, time.Millisecond), 3, time.Millisecond) if got := unwrapChatInvoker(double); got != bare { t.Errorf("unwrapChatInvoker(retry(retry(bare))) = %v, want %v (bare invoker)", got, bare) } } // TestLLM_ParamOverride_AbsoluteCount_NotStacked is the // integration test for LLM retry normal-absolute-count // semantics. It installs a boot retryInvoker with MaxRetries=3 // wrapping an alwaysFailInvoker, then runs an LLMComponent with // MaxRetries=5. The pre-fix behaviour (stacking) would produce // (3+1)*(5+1) = 24 invoker calls. The current implementation // unwraps the boot layer and installs a fresh retryInvoker with // the operator's literal MaxRetries, so the absolute count is // exactly MaxRetries+1 = 6. // // A regression that re-introduces stacking (e.g. someone drops // the unwrapChatInvoker call) fails this test. func TestLLM_ParamOverride_AbsoluteCount_NotStacked(t *testing.T) { bare := &alwaysFailInvoker{err: errors.New("downstream dead")} // Simulate the production boot: a retryInvoker wrapping the // bare invoker. The boot layer's MaxRetries=3 means 4 // invocations per call to the wrapped invoker; without // unwrapping, the param-override retryInvoker would stack on // top and produce 4*6 = 24 calls. With unwrapping, the // absolute count is 6. boot := newRetryInvoker(bare, 3, time.Millisecond) withStubInvoker(t, boot) c := NewLLMComponent(LLMParam{ ModelID: "m", MaxRetries: 5, }) // Force a tiny delay so the test runs fast. c.param.DelayAfterError = time.Millisecond _, err := c.Invoke(context.Background(), map[string]any{"user_prompt": "x"}) if err == nil { t.Fatal("expected error from exhausted retries") } // With the unwrap: 1 initial + 5 retries = 6 calls to // the bare invoker. The boot layer is peeled off first. // Without the unwrap: 6 outer × 4 inner = 24. if got, want := bare.callCount(), 6; got != want { t.Errorf("bare.calls=%d, want %d (absolute count, not stacked). If you see 24, the multiplicative-stacking regression has been re-introduced.", got, want) } } // TestLLM_NoParamOverride_StackingPreserved is the // back-compat companion to the absolute-count test. When // MaxRetries=0 AND DelayAfterError=0, the boot retry chain must // run unchanged so existing DSLs that rely on the implicit // 3-retry budget keep working. // // A future change that aggressively unwraps even when no // override is set would silence the boot retry chain and // regress production retry behaviour. func TestLLM_NoParamOverride_StackingPreserved(t *testing.T) { bare := &alwaysFailInvoker{err: errors.New("downstream dead")} // Boot layer with the production default (3 retries). boot := newRetryInvoker(bare, 3, time.Millisecond) withStubInvoker(t, boot) // No param override: MaxRetries=0 AND DelayAfterError=0. c := NewLLMComponent(LLMParam{ModelID: "m"}) _, err := c.Invoke(context.Background(), map[string]any{"user_prompt": "x"}) if err == nil { t.Fatal("expected error from exhausted retries") } // 1 initial + 3 retries = 4 calls to the bare invoker (the // boot layer ran unchanged). if got, want := bare.callCount(), 4; got != want { t.Errorf("bare.calls=%d, want %d (boot layer ran unchanged — no param override means we keep the implicit 3-retry budget)", got, want) } }