mirror of
https://github.com/infiniflow/ragflow.git
synced 2026-07-11 22:25:41 +08:00
12787996d1
feat(ingestion): mirror Go pipeline progress into the document table;
harden resume guards
- pipeline: bind the owning document via WithDocumentID; after each
TrackProgress event aggregate ingestion_task_log progress and mirror
progress/run/progress_msg back into the document table, so GET
/api/v1/datasets/{dataset_id}/documents reflects live Go pipeline
progress without a bespoke endpoint.
- canvas: extend the S3 resume guard to reject legacy no-op nodes (e.g.
ExitLoop) so component_total equals the count of progress-reporting
components and the aggregate percent can reach 100%.
- runtime/canvas: route progress through TrackProgress; add interrupt
test coverage (r3_interrupt_test.go).
- dao/entity: add IngestionTask.DocumentID column and AggregateProgress
support used by the mirror; IngestionTaskLog keeps a Checkpoint column
alongside the progress fields.
feat(deepdoc): cache DocAnalyzer inference results in Redis (1h TTL)
- Redis-backed DocAnalyzerCache decorator over inference.Client; cache
key = "ddoc:cache:<method>:" + sha256 of the JPEG-encoded image bytes
(deterministic).
- TTL = 1h; hits skip the inner HTTP call and return cached JSON; inner
errors are not cached.
refactor(deepdoc): align figure cropping with Python cropout + bounded
page caches
- CropSectionByDLA mirrors Python cropout: best-overlap DLA
figure/equation region, fallback to section bbox per page, vertical
concat on gray background.
- sliding-window page-image cache bounds peak memory to the recent
window instead of the whole PDF.
- rename DLADebug -> DLARegions across parser/chunker/tests.
refactor(parser): drop lib_type selector; align NewXxxParser with
NewPDFParser
- remove config["lib_type"] lookup and the libType param/field/switch
from all nine constructors; surface the CGO-required error at
ParseWithResult time instead of construction time; drop resolveLibType,
its test, and the four lib_type constants.
feat(utility): add a reusable workerpool for bounded concurrent
execution
- internal/utility/workerpool.go (+ tests).
refactor: translate Chinese prose comments to English in non-harness Go
files.
chore: upgrade github.com/cloudwego/eino from v0.9.9 to v0.9.12.
59 lines
1.9 KiB
Go
59 lines
1.9 KiB
Go
package util
|
|
|
|
import (
|
|
"testing"
|
|
)
|
|
|
|
// TestPositionsFromMatrix_ConvertsToZeroBased verifies the JSON matrix
|
|
// (1-based page numbers, per normalizePDFPageNumber) is decoded to the
|
|
// engine's 0-based page indices. This matches Python's crop(), which
|
|
// indexes page_images[page_number-1].
|
|
func TestPositionsFromMatrix_ConvertsToZeroBased(t *testing.T) {
|
|
m := [][]any{
|
|
{1.0, 10.0, 100.0, 10.0, 100.0},
|
|
{2.0, 0.0, 50.0, 0.0, 50.0},
|
|
}
|
|
got := PositionsFromMatrix(m)
|
|
if len(got) != 2 {
|
|
t.Fatalf("len = %d, want 2", len(got))
|
|
}
|
|
if got[0].PageNumbers[0] != 0 {
|
|
t.Errorf("got[0].PageNumbers[0] = %d, want 0 (1-based -> 0-based)", got[0].PageNumbers[0])
|
|
}
|
|
if got[1].PageNumbers[0] != 1 {
|
|
t.Errorf("got[1].PageNumbers[0] = %d, want 1", got[1].PageNumbers[0])
|
|
}
|
|
}
|
|
|
|
// TestPositionsFromMatrix_ListPageNumbers exercises the multi-page list
|
|
// form (cross-page sections), confirming each entry is shifted -1.
|
|
func TestPositionsFromMatrix_ListPageNumbers(t *testing.T) {
|
|
m := [][]any{
|
|
{[]any{1.0, 2.0}, 10.0, 100.0, 10.0, 100.0},
|
|
}
|
|
got := PositionsFromMatrix(m)
|
|
if len(got) != 1 {
|
|
t.Fatalf("len = %d, want 1", len(got))
|
|
}
|
|
if len(got[0].PageNumbers) != 2 || got[0].PageNumbers[0] != 0 || got[0].PageNumbers[1] != 1 {
|
|
t.Errorf("PageNumbers = %v, want [0 1]", got[0].PageNumbers)
|
|
}
|
|
}
|
|
|
|
// TestPositionsFromMatrix_SkipsMalformedRows confirms rows shorter than 5
|
|
// fields and non-numeric pages are dropped rather than producing garbage.
|
|
func TestPositionsFromMatrix_SkipsMalformedRows(t *testing.T) {
|
|
m := [][]any{
|
|
{1.0, 10.0, 100.0, 10.0}, // too short
|
|
{"x", 10.0, 100.0, 10.0, 100.0}, // non-numeric page
|
|
{3.0, 10.0, 100.0, 10.0, 100.0}, // valid -> 0-based 2
|
|
}
|
|
got := PositionsFromMatrix(m)
|
|
if len(got) != 1 {
|
|
t.Fatalf("len = %d, want 1 (malformed rows skipped)", len(got))
|
|
}
|
|
if got[0].PageNumbers[0] != 2 {
|
|
t.Errorf("got[0].PageNumbers[0] = %d, want 2", got[0].PageNumbers[0])
|
|
}
|
|
}
|