mirror of
https://github.com/infiniflow/ragflow.git
synced 2026-07-11 14:15:40 +08:00
feat(ingestion): mirror Go pipeline progress into the document table;
harden resume guards
- pipeline: bind the owning document via WithDocumentID; after each
TrackProgress event aggregate ingestion_task_log progress and mirror
progress/run/progress_msg back into the document table, so GET
/api/v1/datasets/{dataset_id}/documents reflects live Go pipeline
progress without a bespoke endpoint.
- canvas: extend the S3 resume guard to reject legacy no-op nodes (e.g.
ExitLoop) so component_total equals the count of progress-reporting
components and the aggregate percent can reach 100%.
- runtime/canvas: route progress through TrackProgress; add interrupt
test coverage (r3_interrupt_test.go).
- dao/entity: add IngestionTask.DocumentID column and AggregateProgress
support used by the mirror; IngestionTaskLog keeps a Checkpoint column
alongside the progress fields.
feat(deepdoc): cache DocAnalyzer inference results in Redis (1h TTL)
- Redis-backed DocAnalyzerCache decorator over inference.Client; cache
key = "ddoc:cache:<method>:" + sha256 of the JPEG-encoded image bytes
(deterministic).
- TTL = 1h; hits skip the inner HTTP call and return cached JSON; inner
errors are not cached.
refactor(deepdoc): align figure cropping with Python cropout + bounded
page caches
- CropSectionByDLA mirrors Python cropout: best-overlap DLA
figure/equation region, fallback to section bbox per page, vertical
concat on gray background.
- sliding-window page-image cache bounds peak memory to the recent
window instead of the whole PDF.
- rename DLADebug -> DLARegions across parser/chunker/tests.
refactor(parser): drop lib_type selector; align NewXxxParser with
NewPDFParser
- remove config["lib_type"] lookup and the libType param/field/switch
from all nine constructors; surface the CGO-required error at
ParseWithResult time instead of construction time; drop resolveLibType,
its test, and the four lib_type constants.
feat(utility): add a reusable workerpool for bounded concurrent
execution
- internal/utility/workerpool.go (+ tests).
refactor: translate Chinese prose comments to English in non-harness Go
files.
chore: upgrade github.com/cloudwego/eino from v0.9.9 to v0.9.12.
234 lines
6.2 KiB
Go
234 lines
6.2 KiB
Go
package table
|
||
|
||
import (
|
||
"context"
|
||
"image"
|
||
pdf "ragflow/internal/deepdoc/parser/pdf/type"
|
||
"testing"
|
||
)
|
||
|
||
// mockRotationDoc implements DocAnalyzer with deterministic OCR results per angle.
|
||
// The mock tracks the call sequence: evaluateTableOrientation tests angles in
|
||
// order 0°, 90°, 180°, 270°. Each call to OCRDetect increments an internal
|
||
// counter and returns data for the corresponding angle.
|
||
type mockRotationDoc struct {
|
||
// angle → {regions count, average confidence, error}
|
||
angles map[int]struct {
|
||
regions int
|
||
avgConf float64
|
||
err error
|
||
}
|
||
callSeq int // incremented per OCRDetect call, selects the angle's data
|
||
}
|
||
|
||
var rotationOrder = []int{0, 90, 180, 270}
|
||
|
||
func (m *mockRotationDoc) DLA(_ context.Context, _ image.Image) ([]pdf.DLARegion, error) {
|
||
return nil, nil
|
||
}
|
||
func (m *mockRotationDoc) TSR(_ context.Context, _ image.Image) ([]pdf.TSRCell, error) {
|
||
return nil, nil
|
||
}
|
||
func (m *mockRotationDoc) OCR(_ image.Image) (string, error) { return "", nil }
|
||
func (m *mockRotationDoc) Health() bool { return true }
|
||
|
||
func (m *mockRotationDoc) currentAngle() int {
|
||
idx := m.callSeq % len(rotationOrder)
|
||
return rotationOrder[idx]
|
||
}
|
||
|
||
func (m *mockRotationDoc) OCRDetect(_ context.Context, img image.Image) ([]pdf.OCRBox, error) {
|
||
defer func() { m.callSeq++ }()
|
||
angle := m.currentAngle()
|
||
cfg, ok := m.angles[angle]
|
||
if !ok {
|
||
cfg = m.angles[0] // fallback to 0° config
|
||
}
|
||
if cfg.err != nil {
|
||
return nil, cfg.err
|
||
}
|
||
if cfg.regions == 0 {
|
||
return nil, nil
|
||
}
|
||
w, h := img.Bounds().Dx(), img.Bounds().Dy()
|
||
boxes := make([]pdf.OCRBox, cfg.regions)
|
||
step := w / (cfg.regions + 1)
|
||
for i := 0; i < cfg.regions; i++ {
|
||
x := step * (i + 1)
|
||
boxes[i] = pdf.OCRBox{
|
||
X0: float64(x), Y0: float64(h / 4),
|
||
X1: float64(x + 20), Y1: float64(h / 4),
|
||
X2: float64(x + 20), Y2: float64(h * 3 / 4),
|
||
X3: float64(x), Y3: float64(h * 3 / 4),
|
||
}
|
||
}
|
||
return boxes, nil
|
||
}
|
||
|
||
func (m *mockRotationDoc) OCRRecognize(_ context.Context, _ image.Image) ([]pdf.OCRText, error) {
|
||
angle := rotationOrder[(m.callSeq-1)%len(rotationOrder)] // use angle from last Detect call
|
||
cfg, ok := m.angles[angle]
|
||
if !ok {
|
||
cfg = m.angles[0]
|
||
}
|
||
if cfg.err != nil {
|
||
return nil, cfg.err
|
||
}
|
||
if cfg.regions == 0 {
|
||
return nil, nil
|
||
}
|
||
texts := make([]pdf.OCRText, cfg.regions)
|
||
for i := 0; i < cfg.regions; i++ {
|
||
texts[i] = pdf.OCRText{Text: "X", Confidence: cfg.avgConf}
|
||
}
|
||
return texts, nil
|
||
}
|
||
|
||
func makeTestTableImage() image.Image {
|
||
return image.NewRGBA(image.Rect(0, 0, 200, 100))
|
||
}
|
||
|
||
func TestEvaluateTableOrientation(t *testing.T) {
|
||
t.Run("normal table 0° wins", func(t *testing.T) {
|
||
doc := &mockRotationDoc{
|
||
angles: map[int]struct {
|
||
regions int
|
||
avgConf float64
|
||
err error
|
||
}{
|
||
0: {regions: 10, avgConf: 0.9},
|
||
},
|
||
}
|
||
angle, _, scores := EvaluateTableOrientation(context.Background(), makeTestTableImage(), doc)
|
||
if angle != 0 {
|
||
t.Errorf("expected 0°, got %d° (scores: %v)", angle, scores)
|
||
}
|
||
})
|
||
|
||
t.Run("90° rotated table wins", func(t *testing.T) {
|
||
doc := &mockRotationDoc{
|
||
angles: map[int]struct {
|
||
regions int
|
||
avgConf float64
|
||
err error
|
||
}{
|
||
0: {regions: 2, avgConf: 0.2},
|
||
90: {regions: 10, avgConf: 0.9},
|
||
180: {regions: 2, avgConf: 0.2},
|
||
270: {regions: 2, avgConf: 0.2},
|
||
},
|
||
}
|
||
angle, _, scores := EvaluateTableOrientation(context.Background(), makeTestTableImage(), doc)
|
||
if angle != 90 {
|
||
t.Errorf("expected 90°, got %d° (scores: %v)", angle, scores)
|
||
}
|
||
})
|
||
|
||
t.Run("180° rotated table wins", func(t *testing.T) {
|
||
doc := &mockRotationDoc{
|
||
angles: map[int]struct {
|
||
regions int
|
||
avgConf float64
|
||
err error
|
||
}{
|
||
0: {regions: 1, avgConf: 0.1},
|
||
90: {regions: 1, avgConf: 0.1},
|
||
180: {regions: 8, avgConf: 0.85},
|
||
270: {regions: 1, avgConf: 0.1},
|
||
},
|
||
}
|
||
angle, _, scores := EvaluateTableOrientation(context.Background(), makeTestTableImage(), doc)
|
||
if angle != 180 {
|
||
t.Errorf("expected 180°, got %d° (scores: %v)", angle, scores)
|
||
}
|
||
})
|
||
|
||
t.Run("270° rotated table wins", func(t *testing.T) {
|
||
doc := &mockRotationDoc{
|
||
angles: map[int]struct {
|
||
regions int
|
||
avgConf float64
|
||
err error
|
||
}{
|
||
0: {regions: 1, avgConf: 0.1},
|
||
90: {regions: 1, avgConf: 0.1},
|
||
180: {regions: 1, avgConf: 0.1},
|
||
270: {regions: 9, avgConf: 0.88},
|
||
},
|
||
}
|
||
angle, _, scores := EvaluateTableOrientation(context.Background(), makeTestTableImage(), doc)
|
||
if angle != 270 {
|
||
t.Errorf("expected 270°, got %d° (scores: %v)", angle, scores)
|
||
}
|
||
})
|
||
|
||
t.Run("threshold protection — 0° keeps when diff too small", func(t *testing.T) {
|
||
// Region-count scoring: 8 vs 9 is too close (< 1.4×) → 0° wins.
|
||
doc := &mockRotationDoc{
|
||
angles: map[int]struct {
|
||
regions int
|
||
avgConf float64
|
||
err error
|
||
}{
|
||
0: {regions: 8},
|
||
90: {regions: 9},
|
||
},
|
||
}
|
||
angle, _, _ := EvaluateTableOrientation(context.Background(), makeTestTableImage(), doc)
|
||
if angle != 0 {
|
||
t.Errorf("expected 0° (threshold protection), got %d°", angle)
|
||
}
|
||
})
|
||
|
||
t.Run("threshold pass — 90° wins when region count is clearly higher", func(t *testing.T) {
|
||
// 0° has few regions AND 90° has ≥1.4× more → 90° wins.
|
||
doc := &mockRotationDoc{
|
||
angles: map[int]struct {
|
||
regions int
|
||
avgConf float64
|
||
err error
|
||
}{
|
||
0: {regions: 4},
|
||
90: {regions: 10},
|
||
},
|
||
}
|
||
angle, _, _ := EvaluateTableOrientation(context.Background(), makeTestTableImage(), doc)
|
||
if angle != 90 {
|
||
t.Errorf("expected 90° (threshold passed), got %d°", angle)
|
||
}
|
||
})
|
||
|
||
t.Run("all angles fail OCR → fallback 0°", func(t *testing.T) {
|
||
doc := &mockRotationDoc{
|
||
angles: map[int]struct {
|
||
regions int
|
||
avgConf float64
|
||
err error
|
||
}{
|
||
0: {err: errMockOCR},
|
||
90: {err: errMockOCR},
|
||
180: {err: errMockOCR},
|
||
270: {err: errMockOCR},
|
||
},
|
||
}
|
||
angle, img, scores := EvaluateTableOrientation(context.Background(), makeTestTableImage(), doc)
|
||
if angle != 0 {
|
||
t.Errorf("expected 0° fallback, got %d°", angle)
|
||
}
|
||
if img == nil {
|
||
t.Error("expected non-nil fallback image")
|
||
}
|
||
for _, s := range scores {
|
||
if s != 0 {
|
||
t.Error("all scores should be 0 on OCR failure")
|
||
}
|
||
}
|
||
})
|
||
}
|
||
|
||
var errMockOCR = &mockError{"mock OCR failure"}
|
||
|
||
type mockError struct{ msg string }
|
||
|
||
func (e *mockError) Error() string { return e.msg }
|