mirror of
https://github.com/infiniflow/ragflow.git
synced 2026-07-11 06:05:45 +08:00
feat(ingestion): mirror Go pipeline progress into the document table;
harden resume guards
- pipeline: bind the owning document via WithDocumentID; after each
TrackProgress event aggregate ingestion_task_log progress and mirror
progress/run/progress_msg back into the document table, so GET
/api/v1/datasets/{dataset_id}/documents reflects live Go pipeline
progress without a bespoke endpoint.
- canvas: extend the S3 resume guard to reject legacy no-op nodes (e.g.
ExitLoop) so component_total equals the count of progress-reporting
components and the aggregate percent can reach 100%.
- runtime/canvas: route progress through TrackProgress; add interrupt
test coverage (r3_interrupt_test.go).
- dao/entity: add IngestionTask.DocumentID column and AggregateProgress
support used by the mirror; IngestionTaskLog keeps a Checkpoint column
alongside the progress fields.
feat(deepdoc): cache DocAnalyzer inference results in Redis (1h TTL)
- Redis-backed DocAnalyzerCache decorator over inference.Client; cache
key = "ddoc:cache:<method>:" + sha256 of the JPEG-encoded image bytes
(deterministic).
- TTL = 1h; hits skip the inner HTTP call and return cached JSON; inner
errors are not cached.
refactor(deepdoc): align figure cropping with Python cropout + bounded
page caches
- CropSectionByDLA mirrors Python cropout: best-overlap DLA
figure/equation region, fallback to section bbox per page, vertical
concat on gray background.
- sliding-window page-image cache bounds peak memory to the recent
window instead of the whole PDF.
- rename DLADebug -> DLARegions across parser/chunker/tests.
refactor(parser): drop lib_type selector; align NewXxxParser with
NewPDFParser
- remove config["lib_type"] lookup and the libType param/field/switch
from all nine constructors; surface the CGO-required error at
ParseWithResult time instead of construction time; drop resolveLibType,
its test, and the four lib_type constants.
feat(utility): add a reusable workerpool for bounded concurrent
execution
- internal/utility/workerpool.go (+ tests).
refactor: translate Chinese prose comments to English in non-harness Go
files.
chore: upgrade github.com/cloudwego/eino from v0.9.9 to v0.9.12.
110 lines
2.6 KiB
Go
110 lines
2.6 KiB
Go
package utility
|
|
|
|
import (
|
|
"context"
|
|
"sync/atomic"
|
|
"testing"
|
|
"time"
|
|
)
|
|
|
|
func TestWorkerPoolSubmitAndStats(t *testing.T) {
|
|
pool := NewWorkerPool[int, int](2, 4, func(_ context.Context, in int) (int, error) {
|
|
return in * 2, nil
|
|
})
|
|
defer pool.StopWait()
|
|
|
|
f1, err := pool.Submit(context.Background(), 2)
|
|
if err != nil {
|
|
t.Fatalf("Submit(2): %v", err)
|
|
}
|
|
f2, err := pool.Submit(context.Background(), 3)
|
|
if err != nil {
|
|
t.Fatalf("Submit(3): %v", err)
|
|
}
|
|
|
|
r1, err := f1.Wait(context.Background())
|
|
if err != nil {
|
|
t.Fatalf("Wait(2): %v", err)
|
|
}
|
|
r2, err := f2.Wait(context.Background())
|
|
if err != nil {
|
|
t.Fatalf("Wait(3): %v", err)
|
|
}
|
|
if r1.Value != 4 || r2.Value != 6 {
|
|
t.Fatalf("unexpected results: %+v %+v", r1, r2)
|
|
}
|
|
|
|
stats := pool.Stats()
|
|
if stats.DesiredWorkers != 2 {
|
|
t.Fatalf("DesiredWorkers = %d, want 2", stats.DesiredWorkers)
|
|
}
|
|
if stats.SubmittedTotal != 2 || stats.CompletedTotal != 2 {
|
|
t.Fatalf("stats totals = %+v, want submitted=2 completed=2", stats)
|
|
}
|
|
if stats.FailedTotal != 0 || stats.PendingTotal != 0 {
|
|
t.Fatalf("stats failure/pending = %+v, want 0", stats)
|
|
}
|
|
}
|
|
|
|
func TestWorkerPoolSubmitToCanceledTaskReturnsContextError(t *testing.T) {
|
|
started := make(chan struct{})
|
|
release := make(chan struct{})
|
|
var ranSecond atomic.Uint64
|
|
|
|
pool := NewWorkerPool[int, int](1, 2, func(ctx context.Context, in int) (int, error) {
|
|
if in == 1 {
|
|
close(started)
|
|
<-release
|
|
return in, nil
|
|
}
|
|
ranSecond.Add(1)
|
|
return in, ctx.Err()
|
|
})
|
|
defer pool.StopWait()
|
|
|
|
firstCh := make(chan WorkerPoolResult[int, int], 1)
|
|
if err := pool.SubmitTo(context.Background(), 1, firstCh); err != nil {
|
|
t.Fatalf("SubmitTo(first): %v", err)
|
|
}
|
|
<-started
|
|
|
|
ctx, cancel := context.WithCancel(context.Background())
|
|
secondCh := make(chan WorkerPoolResult[int, int], 1)
|
|
if err := pool.SubmitTo(ctx, 2, secondCh); err != nil {
|
|
t.Fatalf("SubmitTo(second): %v", err)
|
|
}
|
|
cancel()
|
|
close(release)
|
|
|
|
select {
|
|
case <-firstCh:
|
|
case <-time.After(2 * time.Second):
|
|
t.Fatal("timed out waiting for first result")
|
|
}
|
|
|
|
select {
|
|
case res := <-secondCh:
|
|
if res.Err == nil {
|
|
t.Fatal("second result error = nil, want context cancellation")
|
|
}
|
|
if ranSecond.Load() != 0 {
|
|
t.Fatalf("second handler ran %d times, want 0", ranSecond.Load())
|
|
}
|
|
case <-time.After(2 * time.Second):
|
|
t.Fatal("timed out waiting for second result")
|
|
}
|
|
}
|
|
|
|
func TestWorkerPoolResize(t *testing.T) {
|
|
pool := NewWorkerPool[int, int](1, 2, func(_ context.Context, in int) (int, error) {
|
|
return in, nil
|
|
})
|
|
defer pool.StopWait()
|
|
|
|
pool.Resize(3)
|
|
stats := pool.Stats()
|
|
if stats.DesiredWorkers != 3 {
|
|
t.Fatalf("DesiredWorkers = %d, want 3", stats.DesiredWorkers)
|
|
}
|
|
}
|