mirror of
https://github.com/infiniflow/ragflow.git
synced 2026-07-29 12:09:31 +08:00
## Summary
This PR hardens the Go ingestion **Extractor** and **LLM retry** paths
and closes several Python->Go parity gaps in the keyword/question/tag
extraction flow.
- **Generic retry utility** (`internal/common/retry.go`):
`RetryWithBackoff` with exponential backoff (default 3 retries, 2s
initial delay, capped at 1m), context-aware sleep, and a `maxRetries<=0`
fast path. Covered by `internal/common/retry_test.go`.
- **LLM retry reuse**: `agent/component/llm_retry.go` now delegates to
`common.RetryWithBackoff` instead of an inline loop (behavior preserved:
ctx cancellation short-circuits the backoff).
- **Extractor LLM calls** (`extractor.go`):
- `call()` now retries transient LLM failures via `RetryWithBackoff`
(retry exhaustion fails the chunk instead of silently skipping).
- Sets `temperature = 0.2`, matching Python `generator.py:230,245`.
- Runs keyword and question extraction **concurrently** per chunk when
both are enabled (`task_executor.py:444-448`), with mutex-guarded map
writes to avoid data races.
- Substitutes `{field_name}` placeholders (including `{chunks}` -> chunk
text) in `prompt`/`system_prompt` before the call, mirroring Python
`string_format` (`extractor.py:102-103`); unmatched placeholders are
left as-is.
- Falls back to the **tenant default chat model** when `llm_id` is empty
(`task_executor.py:573-574`).
- Strips `` **greedily** (`strings.LastIndex`) in
`cleanExtractionResult`.
- **Auto-tagging** (`extractor_tag.go`): drops the `in.llmID != ""`
guards so an empty `llm_id` no longer skips tagging (uses the tenant
default model), and strips `` greedily in `parseTaggerResponse`.
- **Docs**: fixes a misleading `PresentationChunker` docstring that
claimed per-slide `image`/`position` output (the PPTX path emits none —
unlike PDF), and removes a stale `docs/migration_python_go_diff.md`
reference in `media_dispatch.go`.
## Test plan
- `bash build.sh --test ./internal/common/...` — passes (new retry
utility + tests).
- `bash build.sh --test ./internal/ingestion/component/...` — passes
(extractor/chunker/schema).
- `gofmt` and lefthook pre-commit checks pass.
Note: the personal `docs/migration_python_go_diff.md` working notebook
in the tree is intentionally **not** part of this PR.
---------
Co-authored-by: CodeBuddy <noreply@codebuddy.ai>
158 lines
4.2 KiB
Go
158 lines
4.2 KiB
Go
package common
|
|
|
|
import (
|
|
"context"
|
|
"errors"
|
|
"strings"
|
|
"sync/atomic"
|
|
"testing"
|
|
"time"
|
|
)
|
|
|
|
func TestRetryWithBackoff_SuccessOnFirstAttempt(t *testing.T) {
|
|
var calls atomic.Int32
|
|
err := RetryWithBackoff(t.Context(), 3, time.Millisecond, func() error {
|
|
calls.Add(1)
|
|
return nil
|
|
})
|
|
if err != nil {
|
|
t.Fatalf("unexpected error: %v", err)
|
|
}
|
|
if calls.Load() != 1 {
|
|
t.Errorf("calls = %d, want 1", calls.Load())
|
|
}
|
|
}
|
|
|
|
func TestRetryWithBackoff_SuccessOnThirdAttempt(t *testing.T) {
|
|
var calls atomic.Int32
|
|
err := RetryWithBackoff(t.Context(), 3, time.Millisecond, func() error {
|
|
if calls.Add(1) < 3 {
|
|
return errors.New("transient")
|
|
}
|
|
return nil
|
|
})
|
|
if err != nil {
|
|
t.Fatalf("unexpected error: %v", err)
|
|
}
|
|
if calls.Load() != 3 {
|
|
t.Errorf("calls = %d, want 3", calls.Load())
|
|
}
|
|
}
|
|
|
|
func TestRetryWithBackoff_AllRetriesExhausted(t *testing.T) {
|
|
sentinel := errors.New("persistent failure")
|
|
err := RetryWithBackoff(t.Context(), 3, time.Millisecond, func() error {
|
|
return sentinel
|
|
})
|
|
if err == nil {
|
|
t.Fatal("expected error, got nil")
|
|
}
|
|
if !errors.Is(err, sentinel) {
|
|
t.Errorf("error should wrap sentinel, got %v", err)
|
|
}
|
|
if !strings.Contains(err.Error(), "after 3 retries") {
|
|
t.Errorf("error should mention retry count: %v", err)
|
|
}
|
|
}
|
|
|
|
func TestRetryWithBackoff_ContextCancelled(t *testing.T) {
|
|
ctx, cancel := context.WithCancel(t.Context())
|
|
var calls atomic.Int32
|
|
errChan := make(chan error, 1)
|
|
go func() {
|
|
errChan <- RetryWithBackoff(ctx, 3, 10*time.Second, func() error {
|
|
calls.Add(1)
|
|
cancel() // cancel before retry sleep
|
|
return errors.New("fail")
|
|
})
|
|
}()
|
|
err := <-errChan
|
|
if !errors.Is(err, context.Canceled) {
|
|
t.Errorf("expected context.Canceled, got %v", err)
|
|
}
|
|
if calls.Load() != 1 {
|
|
t.Errorf("calls = %d, want 1 (no retry after cancel)", calls.Load())
|
|
}
|
|
}
|
|
|
|
func TestRetryWithBackoff_ContextDeadlineExceeded(t *testing.T) {
|
|
ctx, cancel := context.WithTimeout(t.Context(), 10*time.Millisecond)
|
|
defer cancel()
|
|
var calls atomic.Int32
|
|
err := RetryWithBackoff(ctx, 3, time.Second, func() error {
|
|
calls.Add(1)
|
|
return errors.New("fail")
|
|
})
|
|
if !errors.Is(err, context.DeadlineExceeded) {
|
|
t.Errorf("expected DeadlineExceeded, got %v", err)
|
|
}
|
|
// Should have made at least 1 call; ctx deadline stops further retries
|
|
if calls.Load() < 1 {
|
|
t.Errorf("expected at least 1 call, got %d", calls.Load())
|
|
}
|
|
}
|
|
|
|
func TestRetryWithBackoff_ZeroMaxRetries(t *testing.T) {
|
|
var calls atomic.Int32
|
|
err := RetryWithBackoff(t.Context(), 0, 0, func() error {
|
|
calls.Add(1)
|
|
return errors.New("fail")
|
|
})
|
|
if err == nil {
|
|
t.Fatal("expected error, got nil")
|
|
}
|
|
if calls.Load() != 1 {
|
|
t.Errorf("calls = %d, want 1 (single attempt)", calls.Load())
|
|
}
|
|
}
|
|
|
|
func TestRetryWithBackoff_NegativeMaxRetries(t *testing.T) {
|
|
var calls atomic.Int32
|
|
err := RetryWithBackoff(t.Context(), -1, 0, func() error {
|
|
calls.Add(1)
|
|
return errors.New("fail")
|
|
})
|
|
if err == nil {
|
|
t.Fatal("expected error, got nil")
|
|
}
|
|
if calls.Load() != 1 {
|
|
t.Errorf("calls = %d, want 1 (negative → single attempt)", calls.Load())
|
|
}
|
|
}
|
|
|
|
// TestRetryWithBackoff_NonRetryableStops verifies that when a
|
|
// shouldRetry predicate returns false, RetryWithBackoff aborts
|
|
// immediately instead of burning the remaining retries/backoff.
|
|
func TestRetryWithBackoff_NonRetryableStops(t *testing.T) {
|
|
var calls atomic.Int32
|
|
sentinel := errors.New("auth failure")
|
|
err := RetryWithBackoff(t.Context(), 3, 10*time.Second, func() error {
|
|
calls.Add(1)
|
|
return sentinel
|
|
}, func(err error) bool {
|
|
return !errors.Is(err, sentinel)
|
|
})
|
|
if !errors.Is(err, sentinel) {
|
|
t.Errorf("expected sentinel, got %v", err)
|
|
}
|
|
if calls.Load() != 1 {
|
|
t.Errorf("calls = %d, want 1 (non-retryable → single attempt)", calls.Load())
|
|
}
|
|
}
|
|
|
|
// TestRetryWithBackoff_NilPredicateRetriesAll confirms that omitting
|
|
// the predicate preserves the original blind-retry behavior.
|
|
func TestRetryWithBackoff_NilPredicateRetriesAll(t *testing.T) {
|
|
var calls atomic.Int32
|
|
err := RetryWithBackoff(t.Context(), 2, time.Millisecond, func() error {
|
|
calls.Add(1)
|
|
return errors.New("transient")
|
|
})
|
|
if err == nil {
|
|
t.Fatal("expected error, got nil")
|
|
}
|
|
if calls.Load() != 3 {
|
|
t.Errorf("calls = %d, want 3 (all retries attempted)", calls.Load())
|
|
}
|
|
}
|