mirror of
https://github.com/infiniflow/ragflow.git
synced 2026-08-04 06:40:29 +08:00
fix(ingestion): improve Extractor/LLM robustness and Python->Go parity (#17470)
## Summary
This PR hardens the Go ingestion **Extractor** and **LLM retry** paths
and closes several Python->Go parity gaps in the keyword/question/tag
extraction flow.
- **Generic retry utility** (`internal/common/retry.go`):
`RetryWithBackoff` with exponential backoff (default 3 retries, 2s
initial delay, capped at 1m), context-aware sleep, and a `maxRetries<=0`
fast path. Covered by `internal/common/retry_test.go`.
- **LLM retry reuse**: `agent/component/llm_retry.go` now delegates to
`common.RetryWithBackoff` instead of an inline loop (behavior preserved:
ctx cancellation short-circuits the backoff).
- **Extractor LLM calls** (`extractor.go`):
- `call()` now retries transient LLM failures via `RetryWithBackoff`
(retry exhaustion fails the chunk instead of silently skipping).
- Sets `temperature = 0.2`, matching Python `generator.py:230,245`.
- Runs keyword and question extraction **concurrently** per chunk when
both are enabled (`task_executor.py:444-448`), with mutex-guarded map
writes to avoid data races.
- Substitutes `{field_name}` placeholders (including `{chunks}` -> chunk
text) in `prompt`/`system_prompt` before the call, mirroring Python
`string_format` (`extractor.py:102-103`); unmatched placeholders are
left as-is.
- Falls back to the **tenant default chat model** when `llm_id` is empty
(`task_executor.py:573-574`).
- Strips `` **greedily** (`strings.LastIndex`) in
`cleanExtractionResult`.
- **Auto-tagging** (`extractor_tag.go`): drops the `in.llmID != ""`
guards so an empty `llm_id` no longer skips tagging (uses the tenant
default model), and strips `` greedily in `parseTaggerResponse`.
- **Docs**: fixes a misleading `PresentationChunker` docstring that
claimed per-slide `image`/`position` output (the PPTX path emits none —
unlike PDF), and removes a stale `docs/migration_python_go_diff.md`
reference in `media_dispatch.go`.
## Test plan
- `bash build.sh --test ./internal/common/...` — passes (new retry
utility + tests).
- `bash build.sh --test ./internal/ingestion/component/...` — passes
(extractor/chunker/schema).
- `gofmt` and lefthook pre-commit checks pass.
Note: the personal `docs/migration_python_go_diff.md` working notebook
in the tree is intentionally **not** part of this PR.
---------
Co-authored-by: CodeBuddy <noreply@codebuddy.ai>
This commit is contained in:
@@ -26,6 +26,8 @@ import (
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"ragflow/internal/common"
|
||||
|
||||
"gorm.io/gorm"
|
||||
)
|
||||
|
||||
@@ -119,34 +121,16 @@ func (r *retryInvoker) Invoke(ctx context.Context, db *gorm.DB, req ChatInvokeRe
|
||||
if r.inner == nil {
|
||||
return nil, fmt.Errorf("component: retryInvoker: nil inner")
|
||||
}
|
||||
delay := r.initialDelay
|
||||
var lastErr error
|
||||
for attempt := 0; attempt <= r.maxRetries; attempt++ {
|
||||
resp, err := r.inner.Invoke(ctx, db, req)
|
||||
if err == nil {
|
||||
return resp, nil
|
||||
}
|
||||
lastErr = err
|
||||
if attempt == r.maxRetries {
|
||||
break
|
||||
}
|
||||
// Honour ctx cancellation during backoff. A short-circuited
|
||||
// sleep avoids hanging on shutdown when a long initialDelay
|
||||
// would otherwise block the goroutine.
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return nil, ctx.Err()
|
||||
case <-time.After(delay):
|
||||
}
|
||||
// Cap the doubling at a sane upper bound (1 minute). Without
|
||||
// this a misconfigured initialDelay (e.g. 10s) plus 5 retries
|
||||
// would sleep 10+20+40+80+160 = 310s before giving up.
|
||||
if delay > 0 {
|
||||
delay *= 2
|
||||
if delay > time.Minute {
|
||||
delay = time.Minute
|
||||
}
|
||||
}
|
||||
var resp *ChatInvokeResponse
|
||||
err := common.RetryWithBackoff(ctx, r.maxRetries, r.initialDelay, func() error {
|
||||
r, e := r.inner.Invoke(ctx, db, req)
|
||||
resp = r
|
||||
return e
|
||||
})
|
||||
// On failure, return nil (not the last partial response) so callers
|
||||
// that check err first never dereference a half-formed resp.
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return nil, fmt.Errorf("component: LLM: chat failed after %d retries: %w", r.maxRetries, lastErr)
|
||||
return resp, nil
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user