mirror of
https://github.com/infiniflow/ragflow.git
synced 2026-07-11 06:05:45 +08:00
12787996d1
feat(ingestion): mirror Go pipeline progress into the document table;
harden resume guards
- pipeline: bind the owning document via WithDocumentID; after each
TrackProgress event aggregate ingestion_task_log progress and mirror
progress/run/progress_msg back into the document table, so GET
/api/v1/datasets/{dataset_id}/documents reflects live Go pipeline
progress without a bespoke endpoint.
- canvas: extend the S3 resume guard to reject legacy no-op nodes (e.g.
ExitLoop) so component_total equals the count of progress-reporting
components and the aggregate percent can reach 100%.
- runtime/canvas: route progress through TrackProgress; add interrupt
test coverage (r3_interrupt_test.go).
- dao/entity: add IngestionTask.DocumentID column and AggregateProgress
support used by the mirror; IngestionTaskLog keeps a Checkpoint column
alongside the progress fields.
feat(deepdoc): cache DocAnalyzer inference results in Redis (1h TTL)
- Redis-backed DocAnalyzerCache decorator over inference.Client; cache
key = "ddoc:cache:<method>:" + sha256 of the JPEG-encoded image bytes
(deterministic).
- TTL = 1h; hits skip the inner HTTP call and return cached JSON; inner
errors are not cached.
refactor(deepdoc): align figure cropping with Python cropout + bounded
page caches
- CropSectionByDLA mirrors Python cropout: best-overlap DLA
figure/equation region, fallback to section bbox per page, vertical
concat on gray background.
- sliding-window page-image cache bounds peak memory to the recent
window instead of the whole PDF.
- rename DLADebug -> DLARegions across parser/chunker/tests.
refactor(parser): drop lib_type selector; align NewXxxParser with
NewPDFParser
- remove config["lib_type"] lookup and the libType param/field/switch
from all nine constructors; surface the CGO-required error at
ParseWithResult time instead of construction time; drop resolveLibType,
its test, and the four lib_type constants.
feat(utility): add a reusable workerpool for bounded concurrent
execution
- internal/utility/workerpool.go (+ tests).
refactor: translate Chinese prose comments to English in non-harness Go
files.
chore: upgrade github.com/cloudwego/eino from v0.9.9 to v0.9.12.
91 lines
2.9 KiB
Go
91 lines
2.9 KiB
Go
//go:build cgo && manual
|
|
|
|
package pdf
|
|
|
|
import (
|
|
"context"
|
|
"image"
|
|
"os"
|
|
"path/filepath"
|
|
pdf "ragflow/internal/deepdoc/parser/pdf/type"
|
|
"testing"
|
|
)
|
|
|
|
func TestParse_PdfiumRender(t *testing.T) {
|
|
// Use a small controlled test PDF from the testdata/pdfs directory.
|
|
pdfPath := filepath.Join("testdata", "pdfs", "01_english_simple.pdf")
|
|
data, err := os.ReadFile(pdfPath)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
|
|
eng, err := NewEngine(data)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
defer eng.Close()
|
|
|
|
// Verify RawData is available and correct.
|
|
raw := eng.RawData()
|
|
if len(raw) == 0 {
|
|
t.Fatal("RawData() returned empty data")
|
|
}
|
|
if len(raw) != len(data) {
|
|
t.Fatalf("RawData() length %d != original %d", len(raw), len(data))
|
|
}
|
|
|
|
// Render a page through pdfium (via the parser's RenderPageToImage).
|
|
img, err := RenderPageToImage(eng, 0)
|
|
if err != nil {
|
|
t.Skipf("pdfium render not available: %v", err)
|
|
}
|
|
b := img.Bounds()
|
|
t.Logf("01_english_simple.pdf page 0: %dx%d", b.Dx(), b.Dy())
|
|
if b.Dx() <= 0 || b.Dy() <= 0 {
|
|
t.Errorf("expected non-zero dimensions from pdfium render, got %dx%d", b.Dx(), b.Dy())
|
|
}
|
|
|
|
// Run Parse with pdfium rendering — BATCH_SKIP_DEEPDOC=1 to avoid HTTP calls.
|
|
t.Setenv("BATCH_SKIP_DEEPDOC", "1")
|
|
cfg := pdf.DefaultParserConfig()
|
|
p := NewParser(cfg)
|
|
result, err := p.ParseRaw(context.Background(), eng, &MockDocAnalyzer{Healthy: true})
|
|
if err != nil {
|
|
t.Fatalf("Parse: %v", err)
|
|
}
|
|
t.Logf("Parse: %d sections, %d tables, %d pages", len(result.Sections), len(result.Tables), len(result.PageHeight))
|
|
|
|
if len(result.Sections) == 0 {
|
|
t.Error("expected at least one section")
|
|
}
|
|
if len(result.PageHeight) == 0 {
|
|
t.Error("expected at least one page dimension entry")
|
|
}
|
|
}
|
|
|
|
func TestParse_PdfiumRender_NoData(t *testing.T) {
|
|
// When engine has no raw PDF bytes, RenderPageToImage falls back to
|
|
// engine.RenderPageImage(). Stub returns (nil, nil) → guard converts
|
|
// to ErrNoPDFData so callers never receive a nil image with nil error.
|
|
img, err := RenderPageToImage(&pythonCharEngineStub{}, 0)
|
|
if err != ErrNoPDFData {
|
|
t.Errorf("expected ErrNoPDFData, got %v", err)
|
|
}
|
|
if img != nil {
|
|
t.Error("expected nil image")
|
|
}
|
|
}
|
|
|
|
// pythonCharEngineStub implements pdf.PDFEngine with RawData() returning nil.
|
|
type pythonCharEngineStub struct{}
|
|
|
|
func (e *pythonCharEngineStub) ExtractChars(_ int) ([]pdf.TextChar, error) { return nil, nil }
|
|
func (e *pythonCharEngineStub) RenderPage(_ int, _ float64) ([]byte, error) { return nil, nil }
|
|
func (e *pythonCharEngineStub) RenderPageImage(_ int, _ float64) (image.Image, error) {
|
|
return nil, nil
|
|
}
|
|
func (e *pythonCharEngineStub) RawData() []byte { return nil }
|
|
func (e *pythonCharEngineStub) PageCount() (int, error) { return 0, nil }
|
|
func (e *pythonCharEngineStub) Close() error { return nil }
|
|
func (e *pythonCharEngineStub) Outlines() ([]pdf.Outline, error) { return nil, nil }
|