zlei6/ragflow
1
0
Fork 0
mirror of https://github.com/infiniflow/ragflow.git synced 2026-07-26 10:23:28 +08:00
Code Issues Packages Projects Releases Wiki Activity

Refactor: refactor dataflow_service.go and guard nats message re-delivery (#16826)

Browse Source 
### Summary

1. refactor dataflow_service.go 
2. guard nats message re-delivery
3. support document parse cancelling & re-run
This commit is contained in:
Jack
2026-07-13 11:08:04 +08:00
committed by GitHub
parent 347a45967e
commit 3c0f8fc192
70 changed files with 4901 additions and 2401 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
internal/deepdoc/parser/pdf/parser.go
View File

@@ -187,7 +187,7 @@ func (p *Parser) processPage(ctx context.Context, engine pdf.PDFEngine, pg int,
// render to an unsafe DPI and spike memory on large pages.
const maxRetryZoom = 9.0
retryZoom := math.Min(p.Config.Zoom*pdf.DlaScale, maxRetryZoom)
slog.Info("per-page zoom retry", "page", pg, "zoom", retryZoom)
slog.Debug("per-page zoom retry", "page", pg, "zoom", retryZoom)
retryImg, retryRenderErr := p.renderAtDPI(ctx, engine, pg, retryZoom*72)
if retryRenderErr == nil && retryImg != nil {
ocrBoxes, updatedChars, ocrUsed = p.processPageBoxes(ctx, retryImg, chars, pg, retryRenderErr, isScanNoise, docAnalyzer)

26
internal/deepdoc/parser/pdf/parser_pipeline_integration_test.go
View File

@@ -1,4 +1,4 @@
//go:build cgo && integration
//go:build cgo && manual
package pdf
@@ -372,30 +372,6 @@ func TestIntegration_Idempotency(t *testing.T) {
})
}
// cropImageRect crops a rectangular region from an image.
func cropImageRect(img image.Image, x0, y0, x1, y1 int) image.Image {
b := img.Bounds()
if x0 < b.Min.X {
x0 = b.Min.X
}
if y0 < b.Min.Y {
y0 = b.Min.Y
}
if x1 > b.Max.X {
x1 = b.Max.X
}
if y1 > b.Max.Y {
y1 = b.Max.Y
}
out := image.NewRGBA(image.Rect(0, 0, x1-x0, y1-y0))
for y := y0; y < y1; y++ {
for x := x0; x < x1; x++ {
out.Set(x-x0, y-y0, img.At(x, y))
}
}
return out
}
const coordEpsilon = 1.0 // pixels
const confEpsilon = 0.01

24
internal/deepdoc/parser/pdf/test_helpers_test.go
View File

@@ -6,6 +6,30 @@ import (
pdf "ragflow/internal/deepdoc/parser/pdf/type"
)
// cropImageRect crops a rectangular region from an image.
func cropImageRect(img image.Image, x0, y0, x1, y1 int) image.Image {
b := img.Bounds()
if x0 < b.Min.X {
x0 = b.Min.X
}
if y0 < b.Min.Y {
y0 = b.Min.Y
}
if x1 > b.Max.X {
x1 = b.Max.X
}
if y1 > b.Max.Y {
y1 = b.Max.Y
}
out := image.NewRGBA(image.Rect(0, 0, x1-x0, y1-y0))
for y := y0; y < y1; y++ {
for x := x0; x < x1; x++ {
out.Set(x-x0, y-y0, img.At(x, y))
}
}
return out
}
// ── testPageImg: small test image for ocrMergeChars tests ─────────────
// 90×120 px at 216 DPI → 30×40 pt in PDF space after /3.0 scaling.