zlei6/ragflow
1
0
Fork 0
mirror of https://github.com/infiniflow/ragflow.git synced 2026-08-19 06:48:25 +08:00
Code Issues Packages Projects Releases Wiki Activity

deepdoc(pdf): WarpCrop de-skew + score-based layer-2 rotation (Go OCR parity with Python) (#18299)

Browse Source 
Supersedes / folds in #18305. The score-based layer-2 rotation selection from #18305 now lives here, on top of `WarpCrop` (layer 1), applied to **all three** Go OCR paths, together with the Python score plumbing the Go side depends on. #18305 is closed in favor of this PR.
This commit is contained in:
Jack
2026-08-17 15:13:56 +08:00
committed by GitHub
parent 272645a27a
commit 8d20cbd0b3
15 changed files with 1938 additions and 19 deletions
Download Patch File Download Diff File Expand all files Collapse all files

11
deepdoc/server/adapters/ocr_adapter.py
View File

@@ -15,9 +15,6 @@ from deepdoc.vision.ocr import OCR
logger = logging.getLogger(__name__)
# Confidence fill value — OSS recognize_batch does not return confidence scores.
_CONFIDENCE_FILL = 1.0
class OCRAdapter:
"""Calls OCR.detect() and OCR.recognize_batch(), converts to wire format."""
@@ -83,10 +80,12 @@ class OCRAdapter:
img = self._decode_bgr(image_data)
# OCR.recognize_batch() returns List[str]; single cropped image → list of 1 image
texts = self._ocr.recognize_batch([img])
# OCR.recognize_batch_with_score() returns List[(text, score)] so the
# client can do score-based layer-2 rotation selection. The score is
# the real recognition confidence (previously a constant 1.0 fill).
scored = self._ocr.recognize_batch_with_score([img])
items = [[text, _CONFIDENCE_FILL] for text in texts]
items = [[text, score] for text, score in scored]
# 4-level nesting matching Go [][][][]any:
# batch → page → items list → pair [text, confidence]

26
deepdoc/vision/ocr.py
View File

@@ -162,7 +162,6 @@ class TextRecognizer:
return padding_im
def resize_norm_img_vl(self, img, image_shape):
imgC, imgH, imgW = image_shape
img = img[:, :, ::-1] # bgr2rgb
resized_image = cv2.resize(img, (imgW, imgH), interpolation=cv2.INTER_LINEAR)
@@ -197,7 +196,6 @@ class TextRecognizer:
return np.reshape(img_black, (c, row, col)).astype(np.float32)
def srn_other_inputs(self, image_shape, num_heads, max_text_length):
imgC, imgH, imgW = image_shape
feature_dim = int((imgH / 8) * (imgW / 8))
@@ -281,7 +279,6 @@ class TextRecognizer:
return img
def resize_norm_img_svtr(self, img, image_shape):
imgC, imgH, imgW = image_shape
resized_image = cv2.resize(img, (imgW, imgH), interpolation=cv2.INTER_LINEAR)
resized_image = resized_image.astype("float32")
@@ -291,7 +288,6 @@ class TextRecognizer:
return resized_image
def resize_norm_img_abinet(self, img, image_shape):
imgC, imgH, imgW = image_shape
resized_image = cv2.resize(img, (imgW, imgH), interpolation=cv2.INTER_LINEAR)
@@ -307,7 +303,6 @@ class TextRecognizer:
return resized_image
def norm_img_can(self, img, image_shape):
img = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY) # CAN only predict gray scale image
if self.rec_image_shape[0] == 1:
@@ -641,6 +636,27 @@ class OCR:
texts.append(text)
return texts
def recognize_batch_with_score(self, img_list, device_id: int | None = None):
"""Like recognize_batch but keeps the per-item recognition score.
Returns a list of (text, score) tuples. Text below drop_score is
blanked, matching recognize_batch, but the score is preserved so the
caller (the OCR HTTP adapter) can surface it for score-based layer-2
rotation selection. Adding this method instead of changing
recognize_batch keeps the in-process __ocr path (pdf_parser.py) on the
original text-only contract.
"""
if device_id is None:
device_id = 0
rec_res, elapse = self.text_recognizer[device_id](img_list)
out = []
for i in range(len(rec_res)):
text, score = rec_res[i]
if score < self.drop_score:
text = ""
out.append((text, score))
return out
def __call__(self, img, device_id=0, cls=True):
time_dict = {"det": 0, "rec": 0, "cls": 0, "all": 0}
if device_id is None:

74
deepdoc/vision/test_ocr_score.py Normal file
View File

@@ -0,0 +1,74 @@
"""Offline unit tests for score-bearing OCR recognition.
These bypass model loading by constructing OCR / OCRAdapter via
object.__new__ and injecting fake recognizers, so they run without the
DeepDoc weights. They verify:
- recognize_batch_with_score returns (text, score) tuples,
- text below drop_score is blanked while the score is preserved,
- the original recognize_batch (text-only) contract is unchanged,
- OCRAdapter.recognize surfaces the real score instead of a 1.0 fill.
"""
import unittest
from deepdoc.server.adapters.ocr_adapter import OCRAdapter
from deepdoc.vision.ocr import OCR
class FakeRecognizer:
"""Callable stand-in for OCR.text_recognizer[device_id].
results is a list of recognizer outputs, one per call; each output is a
list of (text, score) tuples as produced by TextRecognizer.__call__.
"""
def __init__(self, results):
self._results = results
self.calls = 0
def __call__(self, img_list):
res = self._results[self.calls]
self.calls += 1
return res, 0.0
def make_ocr(results, drop_score=0.5):
ocr = object.__new__(OCR)
ocr.text_recognizer = [FakeRecognizer(results)]
ocr.drop_score = drop_score
return ocr
class TestRecognizeBatchWithScore(unittest.TestCase):
def test_returns_score_tuples(self):
ocr = make_ocr([[["hello", 0.92]]])
self.assertEqual(ocr.recognize_batch_with_score(["img"]), [("hello", 0.92)])
def test_blanks_below_drop_score_keeps_score(self):
ocr = make_ocr([[["garble", 0.10]]], drop_score=0.5)
# text is blanked by drop_score, but the score is preserved for
# layer-2 selection.
self.assertEqual(ocr.recognize_batch_with_score(["img"]), [("", 0.10)])
def test_original_recognize_batch_unchanged(self):
# The in-process __ocr path keeps its text-only contract.
ocr = make_ocr([[["hello", 0.92]]])
self.assertEqual(ocr.recognize_batch(["img"]), ["hello"])
class TestOCRAdapterScore(unittest.TestCase):
def test_recognize_surfaces_real_score(self):
adapter = object.__new__(OCRAdapter)
class FakeOCR:
def recognize_batch_with_score(self, imgs):
return [("world", 0.88)]
adapter._ocr = FakeOCR()
adapter._decode_bgr = lambda data: "img"
out = adapter.recognize(b"fake")
self.assertEqual(out, {"output": [[[["world", 0.88]]]]})
if __name__ == "__main__":
unittest.main()

99
internal/deepdoc/parser/pdf/parser_ocr.go
View File

@@ -37,8 +37,20 @@ func (p *Parser) ocrDetectAndRecognize(ctx context.Context, pageImg image.Image,
if x0 >= x1 || y0 >= y1 {
continue
}
cropped := util.FastCrop(pageImg, x0, y0, x1, y1)
texts, recErr := p.inferOCRRecognize(ctx, doc, cropped)
// De-skew the quad with a perspective transform (WarpCrop, layer 1),
// then recognize via ocrRecognizeWithRotation which applies layer-2
// score-based 0/CW90/CCW90 selection for tall crops (get_rotate_crop_image
// parity), so the recognizer receives a rectangular, horizontal crop
// instead of the slanted detection region. The emitted box bounds below
// still use the axis-aligned detection bbox (x0..y1); only the crop fed
// to recognition is transformed.
cropped := util.WarpCrop(pageImg, [4]util.Pt{
{X: b.X0, Y: b.Y0},
{X: b.X1, Y: b.Y1},
{X: b.X2, Y: b.Y2},
{X: b.X3, Y: b.Y3},
})
texts, recErr := p.ocrRecognizeWithRotation(ctx, doc, cropped)
if recErr != nil {
slog.Warn(logLabel+" OCR recognize failed", "page", pageNum, "err", recErr)
continue
@@ -79,6 +91,58 @@ func (p *Parser) ocrDetectAndRecognize(ctx context.Context, pageImg image.Image,
return result
}
// ocrRecognizeWithRotation recognizes a single cropped text region, applying
// layer-2 rotation selection for tall, narrow crops.
//
// When a crop's height is at least 1.5x its width, the text is most likely a
// vertical line and the recognizer — trained on horizontal text — only reads
// it cleanly after a 90 deg rotation. The crop is recognized at 0, CW90, and
// CCW90 and the orientation with the highest recognition confidence is kept.
// The emitted box bounds are unchanged; only the recognized text is
// effectively rotated. Layer 1 (the perspective de-skew) is applied at crop
// time by WarpCrop before this runs.
//
// The DeepDoc rec service surfaces the real recognition confidence, so the
// orientation is picked by score rather than by a fixed rotation.
func (p *Parser) ocrRecognizeWithRotation(ctx context.Context, doc pdf.DocAnalyzer, cropped image.Image) ([]pdf.OCRText, error) {
b := cropped.Bounds()
// Short / wide crops are already horizontal — recognize once at 0 deg.
if float64(b.Dy()) < 1.5*float64(b.Dx()) {
return p.inferOCRRecognize(ctx, doc, cropped)
}
candidates := []image.Image{
cropped,
util.RotateImageCW(cropped, 90), // CW90
util.RotateImageCW(cropped, 270), // CCW90
}
var best []pdf.OCRText
bestScore := -1.0
for _, c := range candidates {
texts, err := p.inferOCRRecognize(ctx, doc, c)
if err != nil {
return nil, err
}
if s := ocrBestScore(texts); s > bestScore {
bestScore = s
best = texts
}
}
return best, nil
}
// ocrBestScore is the layer-2 orientation score: the highest recognition
// confidence among the recognized items. A correctly oriented line reads with
// high confidence; a mis-rotated line reads with low confidence.
func ocrBestScore(texts []pdf.OCRText) float64 {
best := 0.0
for _, t := range texts {
if t.Confidence > best {
best = t.Confidence
}
}
return best
}
// ocrMergeChars runs full-page detect on a page that has embedded chars,
// merges the chars into detect regions, and OCRs any regions without chars.
// Matches Python's __ocr: detect → match chars to boxes → use char text
@@ -231,8 +295,17 @@ func (p *Parser) ocrTableCells(ctx context.Context, cells []pdf.TSRCell, tableIm
if x0 >= x1 || y0 >= y1 {
continue
}
cropped := util.FastCrop(tableImg, x0, y0, x1, y1)
texts, err := p.inferOCRRecognize(ctx, doc, cropped)
// De-skew via WarpCrop and recognize via ocrRecognizeWithRotation like
// the other OCR paths. Table cells are axis-aligned, so WarpCrop
// early-exits to FastCrop and ocrRecognizeWithRotation recognizes once
// at 0 deg; the bounds-clamp / non-finite guard is still inherited.
cropped := util.WarpCrop(tableImg, [4]util.Pt{
{X: float64(x0), Y: float64(y0)},
{X: float64(x1), Y: float64(y0)},
{X: float64(x1), Y: float64(y1)},
{X: float64(x0), Y: float64(y1)},
})
texts, err := p.ocrRecognizeWithRotation(ctx, doc, cropped)
if err != nil {
slog.Warn("table cell OCR failed", "err", err)
continue
@@ -290,10 +363,20 @@ func (p *Parser) buildTextBoxes(ctx context.Context, pageImg image.Image,
}
if len(needOCR) > 0 && doc != nil && doc.Health() {
for _, idx := range needOCR {
cropped := util.FastCrop(pageImg,
int(boxes[idx].x0*scale), int(boxes[idx].y0*scale),
int(boxes[idx].x1*scale), int(boxes[idx].y1*scale))
texts, err := p.inferOCRRecognize(ctx, doc, cropped)
// De-skew via WarpCrop and recognize via ocrRecognizeWithRotation
// the same way ocrDetectAndRecognize does, so all Go OCR paths feed
// the recognizer an identical geometry. Char/table-derived boxes are
// axis-aligned, so WarpCrop early-exits to FastCrop and
// ocrRecognizeWithRotation recognizes once at 0 deg here, while still
// inheriting WarpCrop's bounds-clamp / non-finite guard on the
// untrusted detector box.
cropped := util.WarpCrop(pageImg, [4]util.Pt{
{X: boxes[idx].x0 * scale, Y: boxes[idx].y0 * scale},
{X: boxes[idx].x1 * scale, Y: boxes[idx].y0 * scale},
{X: boxes[idx].x1 * scale, Y: boxes[idx].y1 * scale},
{X: boxes[idx].x0 * scale, Y: boxes[idx].y1 * scale},
})
texts, err := p.ocrRecognizeWithRotation(ctx, doc, cropped)
if err != nil {
slog.Warn("ocr merge: recognize failed", "page", pageNum, "err", err)
continue

104
internal/deepdoc/parser/pdf/parser_ocr_rotate_integration_test.go Normal file
View File

File diff suppressed because one or more lines are too long

134
internal/deepdoc/parser/pdf/parser_ocr_rotate_test.go Normal file
View File

@@ -0,0 +1,134 @@
package pdf
import (
"context"
"image"
"sync"
"testing"
pdf "ragflow/internal/deepdoc/parser/pdf/type"
)
// fakeRotateDoc is a DocAnalyzer whose OCRRecognize returns text and a
// recognition score keyed on the image orientation: a "wide" image (w >= h)
// reads as a clean coherent line with wideScore, a "tall" image (h > w) reads
// as a low-score / garbled result with tallScore. This mirrors reality: a
// correctly oriented horizontal line reads with high confidence, while a
// 90°-rotated line reads with low confidence. It also counts calls so the
// tests can assert the layer-2 fan-out (1 call for short crops, 3 for tall
// crops) and the score-based selection outcome.
type fakeRotateDoc struct {
mu sync.Mutex
calls int
wideText string
tallText string
wideScore float64
tallScore float64
}
func (f *fakeRotateDoc) OCRRecognize(_ context.Context, img image.Image) ([]pdf.OCRText, error) {
f.mu.Lock()
f.calls++
f.mu.Unlock()
b := img.Bounds()
txt := f.tallText
score := f.tallScore
if b.Dx() >= b.Dy() {
txt = f.wideText
score = f.wideScore
}
if txt == "" {
return nil, nil
}
return []pdf.OCRText{{Text: txt, Confidence: score}}, nil
}
func (f *fakeRotateDoc) Health() bool { return true }
func (f *fakeRotateDoc) DLA(context.Context, image.Image) ([]pdf.DLARegion, error) {
return nil, nil
}
func (f *fakeRotateDoc) TSR(context.Context, image.Image) ([]pdf.TSRCell, error) {
return nil, nil
}
func (f *fakeRotateDoc) OCRDetect(context.Context, image.Image) ([]pdf.OCRBox, error) {
return nil, nil
}
// TestOCRRecognizeWithRotation_ShortCrop_NoRotation verifies that crops whose
// height is below 1.5x their width are recognized once at 0° with no rotation
// fan-out — i.e. layer 2 is a no-op for the common horizontal case.
func TestOCRRecognizeWithRotation_ShortCrop_NoRotation(t *testing.T) {
crop := image.NewRGBA(image.Rect(0, 0, 60, 20)) // w=60, h=20 -> ratio 0.33
doc := &fakeRotateDoc{wideText: "Hello", tallText: "x"}
p := &Parser{}
texts, err := p.ocrRecognizeWithRotation(context.Background(), doc, crop)
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
if doc.calls != 1 {
t.Fatalf("expected 1 rec call for short crop, got %d", doc.calls)
}
if len(texts) != 1 || texts[0].Text != "Hello" {
t.Fatalf("expected [Hello], got %+v", texts)
}
}
// TestOCRRecognizeWithRotation_TallCrop_PicksBestScore verifies that a tall
// narrow crop (h/w >= 1.5) is tried at 0°, CW90°, CCW90° and the orientation
// with the highest recognition score wins (score-based, matching Python).
func TestOCRRecognizeWithRotation_TallCrop_PicksBestScore(t *testing.T) {
crop := image.NewRGBA(image.Rect(0, 0, 20, 60)) // w=20, h=60 -> ratio 3.0
doc := &fakeRotateDoc{wideText: "Hello world", tallText: "x", wideScore: 0.95, tallScore: 0.30}
p := &Parser{}
texts, err := p.ocrRecognizeWithRotation(context.Background(), doc, crop)
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
if doc.calls != 3 {
t.Fatalf("expected 3 rec calls for tall crop, got %d", doc.calls)
}
if len(texts) != 1 || texts[0].Text != "Hello world" {
t.Fatalf("expected best-score text [Hello world], got %+v", texts)
}
}
// TestOCRRecognizeWithRotation_TallCrop_TieKeepsZero verifies the stable
// tie-break: when all three orientations yield equal score, the 0° result is
// kept (matches Python's first-wins selection order).
func TestOCRRecognizeWithRotation_TallCrop_TieKeepsZero(t *testing.T) {
crop := image.NewRGBA(image.Rect(0, 0, 20, 60)) // w=20, h=60 -> ratio 3.0
doc := &fakeRotateDoc{wideText: "AB", tallText: "CD", wideScore: 0.5, tallScore: 0.5}
p := &Parser{}
texts, err := p.ocrRecognizeWithRotation(context.Background(), doc, crop)
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
if doc.calls != 3 {
t.Fatalf("expected 3 rec calls, got %d", doc.calls)
}
if len(texts) != 1 || texts[0].Text != "CD" {
t.Fatalf("expected tie to keep 0° text [CD], got %+v", texts)
}
}
// TestOCRBestScore verifies the orientation score is the max recognition
// confidence across items.
func TestOCRBestScore(t *testing.T) {
cases := []struct {
name string
texts []pdf.OCRText
want float64
}{
{"empty", nil, 0},
{"single", []pdf.OCRText{{Text: "ab", Confidence: 0.9}}, 0.9},
{"multi", []pdf.OCRText{{Text: "ab", Confidence: 0.7}, {Text: "c", Confidence: 0.95}}, 0.95},
{"keeps max", []pdf.OCRText{{Text: "a", Confidence: 0.2}, {Text: "b", Confidence: 0.8}}, 0.8},
}
for _, c := range cases {
if got := ocrBestScore(c.texts); got != c.want {
t.Fatalf("%s: got %v, want %v", c.name, got, c.want)
}
}
}

211
internal/deepdoc/parser/pdf/parser_ocr_warp_test.go Normal file
View File

@@ -0,0 +1,211 @@
package pdf
import (
"context"
"image"
"math"
"testing"
pdf "ragflow/internal/deepdoc/parser/pdf/type"
util "ragflow/internal/deepdoc/parser/pdf/util"
)
// captureAnalyzer records the image handed to OCRRecognize so a test can
// assert which crop geometry was fed to recognition.
type captureAnalyzer struct {
healthy bool
boxes []pdf.OCRBox
texts []pdf.OCRText
recImage image.Image
}
func (c *captureAnalyzer) Health() bool { return c.healthy }
func (c *captureAnalyzer) DLA(context.Context, image.Image) ([]pdf.DLARegion, error) {
return nil, nil
}
func (c *captureAnalyzer) TSR(context.Context, image.Image) ([]pdf.TSRCell, error) {
return nil, nil
}
func (c *captureAnalyzer) OCRDetect(context.Context, image.Image) ([]pdf.OCRBox, error) {
return c.boxes, nil
}
func (c *captureAnalyzer) OCRRecognize(_ context.Context, img image.Image) ([]pdf.OCRText, error) {
c.recImage = img
return c.texts, nil
}
func sameImage(a, b *image.RGBA) bool {
if a == nil || b == nil {
return a == b
}
if a.Bounds() != b.Bounds() {
return false
}
for y := 0; y < a.Bounds().Dy(); y++ {
for x := 0; x < a.Bounds().Dx(); x++ {
if a.RGBAAt(x, y) != b.RGBAAt(x, y) {
return false
}
}
}
return true
}
// TestOCRDetectAndRecognize_WarpsCrop locks that ocrDetectAndRecognize feeds
// the perspective-de-skewed (WarpCrop) crop to recognition, not the old
// axis-aligned FastCrop of the detection bbox. This is the live-path wiring
// of Step 4 / layer 1.
func TestOCRDetectAndRecognize_WarpsCrop(t *testing.T) {
p := newTestParser()
page := image.NewRGBA(image.Rect(0, 0, 200, 140))
// A clearly skewed (perspective) quad: TL, TR, BR, BL. It is intentionally
// wide (W=120, H=60 after the warp, h/w < 1.5) so layer 2 is a no-op and
// recognition receives the de-skewed crop unchanged; a tall quad would be
// rotated by ocrRecognizeWithRotation and break the equality below.
quad := [4]util.Pt{{X: 50, Y: 40}, {X: 150, Y: 30}, {X: 160, Y: 90}, {X: 40, Y: 100}}
box := pdf.OCRBox{
X0: quad[0].X, Y0: quad[0].Y,
X1: quad[1].X, Y1: quad[1].Y,
X2: quad[2].X, Y2: quad[2].Y,
X3: quad[3].X, Y3: quad[3].Y,
}
cap := &captureAnalyzer{
healthy: true,
boxes: []pdf.OCRBox{box},
texts: []pdf.OCRText{{Text: "x", Confidence: 0.9}},
}
got := p.ocrDetectAndRecognize(context.Background(), page, cap, 0, "warp")
if len(got) != 1 {
t.Fatalf("expected 1 text box, got %d", len(got))
}
if cap.recImage == nil {
t.Fatal("OCRRecognize was not called with a crop")
}
// The crop passed to recognition must be exactly the WarpCrop output.
want := util.WarpCrop(page, quad)
rec, ok := cap.recImage.(*image.RGBA)
if !ok {
t.Fatalf("rec crop is %T, want *image.RGBA", cap.recImage)
}
if !sameImage(rec, want) {
t.Errorf("rec crop is not the WarpCrop output (size got=%v want=%v)",
cap.recImage.Bounds(), want.Bounds())
}
// And it must NOT be the axis-aligned FastCrop of the detection bbox,
// proving de-skew actually happened on the live path.
minX := int(min4(quad[0].X, quad[1].X, quad[2].X, quad[3].X))
minY := int(min4(quad[0].Y, quad[1].Y, quad[2].Y, quad[3].Y))
maxX := int(max4(quad[0].X, quad[1].X, quad[2].X, quad[3].X))
maxY := int(max4(quad[0].Y, quad[1].Y, quad[2].Y, quad[3].Y))
bboxCrop := util.FastCrop(page, minX, minY, maxX, maxY)
if sameImage(rec, bboxCrop) {
t.Errorf("rec crop equals the axis-aligned bbox crop; warp was not applied")
}
// Safety property: the emitted TextBox must stay the axis-aligned
// detection bbox — only the crop fed to recognition is de-skewed, the box
// geometry itself is never transformed. If warp ever leaked into the
// emitted coordinates this would drift from the original detection bbox.
got0 := got[0]
if math.Abs(got0.X0-float64(minX)/pdf.DlaScale) > 1e-6 ||
math.Abs(got0.Top-float64(minY)/pdf.DlaScale) > 1e-6 ||
math.Abs(got0.X1-float64(maxX)/pdf.DlaScale) > 1e-6 ||
math.Abs(got0.Bottom-float64(maxY)/pdf.DlaScale) > 1e-6 {
t.Errorf("emitted TextBox is not the axis-aligned detection bbox: got=(%.4f,%.4f,%.4f,%.4f) want=(%.4f,%.4f,%.4f,%.4f)",
got0.X0, got0.Top, got0.X1, got0.Bottom,
float64(minX)/pdf.DlaScale, float64(minY)/pdf.DlaScale,
float64(maxX)/pdf.DlaScale, float64(maxY)/pdf.DlaScale)
}
}
func min4(a, b, c, d float64) float64 {
m := a
if b < m {
m = b
}
if c < m {
m = c
}
if d < m {
m = d
}
return m
}
// TestOCRMergeChars_WarpsEmptyBoxCrop locks that the char-merge path
// (buildTextBoxes) also de-skews the boxes it re-recognizes, matching the
// ocrDetectAndRecognize path, and that for the axis-aligned boxes this path
// actually produces (the WarpCrop is behavior-equivalent to the old FastCrop,
// so this doubles as a no-regression guard for the wiring change).
func TestOCRMergeChars_WarpsEmptyBoxCrop(t *testing.T) {
p := newTestParser()
page := image.NewRGBA(image.Rect(0, 0, 200, 140))
// Axis-aligned detection box (pixel space). Char boxes are axis-aligned,
// so the merge path only ever sees rectangles like this.
quad := [4]util.Pt{{X: 40, Y: 40}, {X: 140, Y: 40}, {X: 140, Y: 100}, {X: 40, Y: 100}}
box := pdf.OCRBox{
X0: quad[0].X, Y0: quad[0].Y,
X1: quad[1].X, Y1: quad[1].Y,
X2: quad[2].X, Y2: quad[2].Y,
X3: quad[3].X, Y3: quad[3].Y,
}
// A single space char inside the box -> matched, but its text trims to
// empty, so the box is pushed to the need-OCR path.
chars := []pdf.TextChar{{
Text: " ",
X0: 45,
X1: 135,
Top: 45,
Bottom: 95,
PageNumber: 0,
}}
cap := &captureAnalyzer{
healthy: true,
boxes: []pdf.OCRBox{box},
texts: []pdf.OCRText{{Text: "x", Confidence: 0.9}},
}
got := p.ocrMergeChars(context.Background(), page, chars, cap, 0)
if len(got) == 0 {
t.Fatal("expected at least one text box from the merge path")
}
if cap.recImage == nil {
t.Fatal("OCRRecognize was not called with a crop on the merge path")
}
rec, ok := cap.recImage.(*image.RGBA)
if !ok {
t.Fatalf("rec crop is %T, want *image.RGBA", cap.recImage)
}
// Axis-aligned box: WarpCrop must equal FastCrop (no behavioral change),
// and the crop fed to recognition must be exactly the warped rectangle.
wantWarp := util.WarpCrop(page, quad)
wantFast := util.FastCrop(page, 40, 40, 140, 100)
if !sameImage(rec, wantWarp) {
t.Errorf("merge-path rec crop is not the WarpCrop output (size got=%v want=%v)",
cap.recImage.Bounds(), wantWarp.Bounds())
}
if !sameImage(rec, wantFast) {
t.Errorf("merge-path crop diverged from the old FastCrop behavior; regression risk")
}
}
func max4(a, b, c, d float64) float64 {
m := a
if b > m {
m = b
}
if c > m {
m = c
}
if d > m {
m = d
}
return m
}

151
internal/deepdoc/parser/pdf/util/testdata/gen_warp_golden.py vendored Normal file
View File

@@ -0,0 +1,151 @@
"""Generate golden data for the Go util.WarpCrop unit test.
Produces, under this directory:
* warp_src.png - a synthetic source image with high-frequency content
* warp_expected.png - the perspective-de-skewed crop, computed with PIL's
PERSPECTIVE transform (BICUBIC)
* warp_meta.json - the 4 source corners (TL,TR,BR,BL) and the expected
output size (w,h) consumed by warp_test.go.
The reference perspective transform and the Go WarpCrop implementation compute
the same homogeneous mapping (destination -> source for the backward sampler);
any minor resampling-kernel difference (PIL-bicubic vs the Go Catmull-Rom
sampler) is absorbed by the MSE tolerance in the test.
"""
import base64
import io
import json
import math
import os
from PIL import Image, ImageDraw
HERE = os.path.dirname(os.path.abspath(__file__))
# A general quadrilateral (true perspective, not a parallelogram) inside the
# source image. Order: top-left, top-right, bottom-right, bottom-left.
SRC = [(50, 40), (260, 25), (250, 170), (40, 150)]
def dist(a, b):
return math.hypot(a[0] - b[0], a[1] - b[1])
def out_size(src):
w = int(max(dist(src[0], src[1]), dist(src[2], src[3])))
h = int(max(dist(src[0], src[3]), dist(src[1], src[2])))
return w, h
def solve_homography(src, dst):
"""Solve the 8-DOF homography mapping src->dst with bottom-right fixed to 1.
Returns coeffs [a,b,c,d,e,f,g,h] for PIL's PERSPECTIVE:
x' = (a*x + b*y + c) / (g*x + h*y + 1)
y' = (d*x + e*y + f) / (g*x + h*y + 1)
"""
A = [[0.0] * 9 for _ in range(8)]
b = [0.0] * 8
for i in range(4):
sx, sy = src[i]
dx, dy = dst[i]
# x' equation.
A[2 * i][0] = sx
A[2 * i][1] = sy
A[2 * i][2] = 1.0
A[2 * i][6] = -sx * dx
A[2 * i][7] = -sy * dx
b[2 * i] = dx
# y' equation.
A[2 * i + 1][3] = sx
A[2 * i + 1][4] = sy
A[2 * i + 1][5] = 1.0
A[2 * i + 1][6] = -sx * dy
A[2 * i + 1][7] = -sy * dy
b[2 * i + 1] = dy
# Gaussian elimination with partial pivoting.
for col in range(8):
pivot = max(range(col, 8), key=lambda r: abs(A[r][col]))
A[col], A[pivot] = A[pivot], A[col]
b[col], b[pivot] = b[pivot], b[col]
piv = A[col][col]
for r in range(col + 1, 8):
f = A[r][col] / piv
for c in range(col, 9):
A[r][c] -= f * A[col][c]
b[r] -= f * b[col]
x = [0.0] * 8
for r in range(7, -1, -1):
s = b[r]
for c in range(r + 1, 8):
s -= A[r][c] * x[c]
x[r] = s / A[r][r]
return x # [a,b,c,d,e,f,g,h]
def make_source(path):
img = Image.new("RGB", (320, 210), (255, 255, 255))
d = ImageDraw.Draw(img)
# Border.
d.rectangle([4, 4, 315, 205], outline=(0, 0, 0), width=2)
# Solid color blocks (smooth edges -> small resampling-kernel differences).
d.rectangle([20, 20, 90, 90], fill=(200, 30, 30))
d.rectangle([110, 30, 170, 100], fill=(30, 160, 40))
d.rectangle([200, 20, 300, 80], fill=(30, 60, 200))
# Circle outline (interpolation signal, smooth curvature).
d.ellipse([40, 120, 130, 200], outline=(0, 0, 0), width=3)
# A few thick diagonal bars (width 3) to exercise bicubic sampling without
# pushing content to the Nyquist limit.
for k in range(0, 160, 28):
d.line([(175 + k, 110), (175 + k + 60, 200)], fill=(0, 0, 0), width=3)
img.save(path)
def png_b64(img):
"""Encode a PIL image as a single-line base64 PNG string.
Golden fixtures are committed as base64 TEXT rather than binary PNG so the
repo's pre-commit text filters (mixed-line-ending / end-of-file-fixer) can
never corrupt the binary signature. A trailing newline added to the .b64
file is harmless: base64 decode ignores surrounding whitespace.
"""
buf = io.BytesIO()
img.save(buf, format="PNG")
return base64.b64encode(buf.getvalue()).decode("ascii")
def main():
src_path = os.path.join(HERE, "warp_src.png")
exp_path = os.path.join(HERE, "warp_expected.png")
src_b64 = os.path.join(HERE, "warp_src.b64")
exp_b64 = os.path.join(HERE, "warp_expected.b64")
meta_path = os.path.join(HERE, "warp_meta.json")
make_source(src_path)
w, h = out_size(SRC)
dst = [(0, 0), (w, 0), (w, h), (0, h)]
# PIL's PERSPECTIVE coeffs map DESTINATION -> SOURCE directly. So solve the
# homography dst->src, matching the Go WarpCrop implementation (which
# computes src->dst, then uses its inverse for backward mapping).
coeffs = solve_homography(dst, SRC)
img = Image.open(src_path).convert("RGB")
warped = img.transform((w, h), Image.PERSPECTIVE, coeffs, resample=Image.BICUBIC)
warped.save(exp_path)
# Committed (text) golden fixtures.
with open(src_b64, "w") as f:
f.write(png_b64(img))
with open(exp_b64, "w") as f:
f.write(png_b64(warped))
with open(meta_path, "w") as f:
json.dump({"src": SRC, "w": w, "h": h}, f, indent=2)
print(f"wrote {src_path} ({img.size}), {exp_path} ({warped.size}), {src_b64}, {exp_b64}, {meta_path}")
if __name__ == "__main__":
main()

1
internal/deepdoc/parser/pdf/util/testdata/warp_expected.b64 vendored Normal file
View File

File diff suppressed because one or more lines are too long

22
internal/deepdoc/parser/pdf/util/testdata/warp_meta.json vendored Normal file
View File

@@ -0,0 +1,22 @@
{
"src": [
[
50,
40
],
[
260,
25
],
[
250,
170
],
[
40,
150
]
],
"w": 210,
"h": 145
}

1
internal/deepdoc/parser/pdf/util/testdata/warp_src.b64 vendored Normal file
View File

@@ -0,0 +1 @@
iVBORw0KGgoAAAANSUhEUgAAAUAAAADSCAIAAAC0K44BAAAHK0lEQVR4nO3dwXXcRhBFUcCHeXDtCLhyLI7NsWilCLyeSOBFiyQ0DcIg0I3+v+rdneSjEjDmU9dAI2lelmUC4OmP0RcA4DwCBowRMGDsZf2NeZ5HXQeAg9bPrTiBAWMv9XfxXBrQVO/InMCAMQIGjBEwYIyAAWMEDBgjYMAYAQPGCBgwRsCAMQIGjBEwYIyAAWMEDBjb+NNIx/18fW11HTreHo/RlxDB618/R1+Ck8ePt3M/kBMYMEbAgDECBowRMGCMgAFjBAwYI2DAGAEDxggYMHbpk1hYe/3nz9GXcMbj739HXwLO4wQGjBEwYIyAAWMEDBgjYMAYAQPGCBgwRsCAMQIGjBEwYIyAAWMEDBgjYMAYAQPGCBgwRsCAMQIGjBEwYIyAAWMEDBgjYMAYAQPGCBgwRsCAMQIGjBEwYIyAAWMEDBgjYMAYAQPGCBgwRsCAMQIGjBEwYIyAAWMEDBgjYMAYAQPGCBgwRsCAMQIGjBEwYIyAAWMEDBgjYMAYAQPGCBgwRsCAMQIGjBEwYOxl9AUgpsePt9GXkAInMGCMgAFjBAwYI2DAGAEDxggYMEbAgDECBowRMGDs0iex3h6PVtcB4AROYMAYAQPGCBgwRsCAMQIGjBEwYIyAAWMEDBgjYMDYvCzL5zfmeZqm9fcA0FEXygkMGCNgwBgBA8YIGDBGwIAxAgaMETBgjIDxm/I7jcx0mUnA+FS+Mtp+zTGza8MEjF8cz59sM2sEjF/qD9UyU23mWplJwNjmcigln0nA+NTjz7EwsysCxm9clklmFgSMPUbLZM6ZBIxnLsskMycCxibHZTLnTALG/7NYJnPOvPSPm3n51qvG3yu0LEvzr11mtp8Z+O/Eavh6hXlNvmv9GrZ6EZh5emaKvxNrfic+0474MplzZpwV+sjLcfDXvJ1RH/8pyZnstEymnBlhhd55RZrcS+/5+gSXyZwz60K9T+DNtJpHtfN7AKa/5F0xz3Pz+2Xm6R/u+h64fke6vOv6827+LOHfHtt9vCHPTL+Av0r35svIlrHXxxvyzDQLWCHd/QsI3PCa7FPZbDNtAn4634anu/Z0MVGPYvFlMudMj4Drg3fUlezIcBQrL5M5ZxoELHvw1uqjeODF3EBqmcw5Uz3gHr8L11vshmWXyZwzpQN2rLfI07DOMplzpm7AvvUWsRteE1kmc84UDdi93iJww4LLZM6ZigHHqLdI0rDCMplzplzAkeotAje8NnyZzDlTK+B49RZRG5ZaJnPO1Ar4Q6R6i3h3VOgskzlnCgUc/s/Kf9xXpEP4SbAFVX+mSsCBv6Y3RbpfkWUy50yVgD9EPX6LqHensEzmnCkRcKTj6LjAdx1mQdWfKRHwh6gH1FrUexy+TOacOT7g8M+ualGfZsVbUPVnjg8YUQVYUPVnDg442BF0QrBXINiCqj9T5QTOsz8Xge830oKqNrOmEjCisl5QBWc+GRlwsO3xtHivQ5gFVX+mxAkceJ/cEfuuHRdUx5kSASM8lwXVbiYBo5cAC6r+TAJGR+4LqtrMtTJzWMAJP4BVi/qRrK/YLaj6MzmB0Zf1gio48wkBozuXBdVxJgHjbkYLqv5MAsYdXBZUu5kEjJs4Lqj6MwkYY1gsqPozCRj3sVtQ9WcSMG7ltaDqzyRgjCS+oOrPJGDczWhB1Z85LOBsnyLclPbzpC4Lqv5MTmCMJ7ug6s8kYIxhsaDqzyRgDKO/oOrPlAg459vgnHe9Q3BB1Z85MuBsT26+kvl1EF9Q9WdKnMDITHlB1Z+pEnC2fTLb/R4ntaDqzxwccObtseAVmIQXVP2ZKicwktNcUPVnjg844Uey0n4A6ziRBVV/5viA1zI0nOEezxFcUPVnSgSc8yDKedf71BZU/ZkSAa/FPqBi311zwxdU/ZkqAWc7jrLd73FSC6r+TJWApwRPs3h2dZDOgqo/UyjgtXgNx7uj2wRbetvO1Aq4978HNcr6Xjh+jxBZUPVnagU8RWyYes9RWFD1Z8oFPMVqmHpbCbP0tp2pGPAUpWHqvWj4gqo/UzTgyb9h6m0i3tLbdqZuwJNzw9TbSYClt+1M6YAnz4apt61gS2/bmeoBT1XDyhk/XR71thJp6b0ys2YQ8FTdg2bDT1dFvf1YL71teQQ8TdOyLLJHcX3wUm9zYZbetjNtAi7qo3hsxvUFkG4/jktv75lmAU9b59uQjDfTpd47uSy9XWf6BVx8lXHvkjd/FtK9TYClt+3MuT6a7b4Wd6Jtci+95+O7ejzqt5hZnxwv14cOV16azczOvUE9coyTroh5npv/vzCaGSHg4sjfB3B9waZbBcuyNH+v5DLziet74B3LO/GZuML96XGrmXFO4NqVj3/QqhHrp1AXRXiIhbSM3qw2fIj18dBnWZaAKzTycDkn+80kYMDYxntgnc8YA9jHCQwYa/9+HcBtOIEBYwQMGCNgwNh/xBMe6SmdX9EAAAAASUVORK5CYII=

353
internal/deepdoc/parser/pdf/util/warp.go Normal file
View File

@@ -0,0 +1,353 @@
package util
import (
"image"
"image/color"
"image/draw"
"math"
)
// Pt is a 2D float point used for warp corners.
type Pt struct {
X, Y float64
}
// WarpCrop de-skews a quadrilateral region from src using a perspective
// transform, producing the rectangular crop fed to text recognition.
//
// points must be the 4 corners in order: top-left, top-right, bottom-right,
// bottom-left (the DBNet quad order emitted by the OCR detector). The output
// size is (W, H) where
//
// W = int(max(|p0-p1|, |p2-p3|))
// H = int(max(|p0-p3|, |p1-p2|))
//
// Each destination pixel is mapped back to the source via the inverse
// homography and sampled with Catmull-Rom (bicubic) interpolation. Out-of-
// bounds source coordinates use BORDER_REPLICATE semantics (edge pixels
// repeated).
//
// WarpCrop performs NO rotation selection (the h/w >= 1.5 branch) — that
// belongs to the caller / layer 2.
//
// If the quad is degenerate (collinear / non-invertible homography), WarpCrop
// falls back to an axis-aligned crop of the quad's bounding box so callers
// stay safe.
// maxWarpDim bounds the allocated crop so a (clamped) quad can never drive an
// unbounded image.NewRGBA. Detector boxes arrive from a remote DocAnalyzer /
// DEEPDOC_URL and are treated as untrusted; this ceiling is a last line of
// defence against an unexpectedly large source image even after clamping.
const maxWarpDim = 1 << 16
func WarpCrop(src image.Image, points [4]Pt) *image.RGBA {
// Detection boxes come from a remote DocAnalyzer / DEEPDOC_URL and are
// effectively untrusted. FastCrop clamps its rectangle to the source
// bounds before allocating; this path must do the same on its four
// corners and must reject non-finite coordinates, so a malformed or
// out-of-range response cannot drive an unbounded image.NewRGBA (panic /
// OOM). On a normal in-bounds quad the clamp is a no-op, so detection
// accuracy is unchanged.
if !pointsFinite(points) {
return image.NewRGBA(image.Rect(0, 0, 1, 1))
}
rgba := toRGBA(src)
b := rgba.Bounds()
pts := clampQuad(points, b)
// Axis-aligned fast path: an axis-parallel quad is just a sub-rectangle,
// so the perspective warp degenerates to a copy. FastCrop does exactly
// that with a direct Pix slice copy (no per-pixel bicubic resampling),
// which is far cheaper. Table cells and char-derived boxes are always
// axis-aligned, so this short-circuits the common OCR paths to the cheap
// copy — the de-skew is only paid for genuinely slanted detection quads.
if axisAligned(pts) {
minX := int(math.Min(pts[0].X, math.Min(pts[1].X, math.Min(pts[2].X, pts[3].X))))
minY := int(math.Min(pts[0].Y, math.Min(pts[1].Y, math.Min(pts[2].Y, pts[3].Y))))
maxX := int(math.Max(pts[0].X, math.Max(pts[1].X, math.Max(pts[2].X, pts[3].X))))
maxY := int(math.Max(pts[0].Y, math.Max(pts[1].Y, math.Max(pts[2].Y, pts[3].Y))))
return FastCrop(rgba, minX, minY, maxX, maxY)
}
w := int(math.Max(dist(pts[0], pts[1]), dist(pts[2], pts[3])))
h := int(math.Max(dist(pts[0], pts[3]), dist(pts[1], pts[2])))
if w <= 0 || h <= 0 || w > maxWarpDim || h > maxWarpDim {
return axisFallback(src, pts)
}
dst := [4]Pt{{0, 0}, {float64(w), 0}, {float64(w), float64(h)}, {0, float64(h)}}
hMat, ok := perspectiveTransform(pts, dst)
if !ok {
return axisFallback(src, pts)
}
inv, ok := invert3x3(hMat)
if !ok {
return axisFallback(src, pts)
}
out := image.NewRGBA(image.Rect(0, 0, w, h))
for y := 0; y < h; y++ {
for x := 0; x < w; x++ {
// Backward map: src = inv * [x, y, 1].
den := inv[6]*float64(x) + inv[7]*float64(y) + inv[8]
if den == 0 {
continue
}
sx := (inv[0]*float64(x) + inv[1]*float64(y) + inv[2]) / den
sy := (inv[3]*float64(x) + inv[4]*float64(y) + inv[5]) / den
out.SetRGBA(x, y, sampleBicubic(rgba, sx, sy, b))
}
}
return out
}
// perspectiveTransform solves the 8-DOF homography H (row-major 3x3 with
// H[8]=1) such that dst_i = H * src_i in homogeneous coordinates. It fixes the
// bottom-right homography element to 1 (the 8-DOF normalization). Returns
// ok=false if the linear system is singular.
func perspectiveTransform(src, dst [4]Pt) ([9]float64, bool) {
var A [8][9]float64
for i := 0; i < 4; i++ {
sx, sy := src[i].X, src[i].Y
dx, dy := dst[i].X, dst[i].Y
// x' equation.
A[2*i][0] = sx
A[2*i][1] = sy
A[2*i][2] = 1
A[2*i][6] = -sx * dx
A[2*i][7] = -sy * dx
A[2*i][8] = dx
// y' equation.
A[2*i+1][3] = sx
A[2*i+1][4] = sy
A[2*i+1][5] = 1
A[2*i+1][6] = -sx * dy
A[2*i+1][7] = -sy * dy
A[2*i+1][8] = dy
}
x, ok := solveLinear8(A)
if !ok {
return [9]float64{}, false
}
return [9]float64{x[0], x[1], x[2], x[3], x[4], x[5], x[6], x[7], 1}, true
}
// solveLinear8 solves A * x = b for an 8x8 system via Gaussian elimination
// with partial pivoting. b is stored in the last column of A.
func solveLinear8(A [8][9]float64) ([8]float64, bool) {
for col := 0; col < 8; col++ {
// Partial pivot.
pivot := col
maxAbs := math.Abs(A[col][col])
for r := col + 1; r < 8; r++ {
if v := math.Abs(A[r][col]); v > maxAbs {
maxAbs = v
pivot = r
}
}
if maxAbs < 1e-12 {
return [8]float64{}, false
}
A[col], A[pivot] = A[pivot], A[col]
// Eliminate below.
for r := col + 1; r < 8; r++ {
f := A[r][col] / A[col][col]
for c := col; c < 9; c++ {
A[r][c] -= f * A[col][c]
}
}
}
// Back-substitution.
var x [8]float64
for r := 7; r >= 0; r-- {
sum := A[r][8]
for c := r + 1; c < 8; c++ {
sum -= A[r][c] * x[c]
}
x[r] = sum / A[r][r]
}
return x, true
}
// invert3x3 returns the inverse of the row-major 3x3 matrix m. Returns
// ok=false if singular.
func invert3x3(m [9]float64) ([9]float64, bool) {
det := m[0]*(m[4]*m[8]-m[5]*m[7]) -
m[1]*(m[3]*m[8]-m[5]*m[6]) +
m[2]*(m[3]*m[7]-m[4]*m[6])
if math.Abs(det) < 1e-12 {
return [9]float64{}, false
}
invDet := 1.0 / det
return [9]float64{
(m[4]*m[8] - m[5]*m[7]) * invDet,
(m[2]*m[7] - m[1]*m[8]) * invDet,
(m[1]*m[5] - m[2]*m[4]) * invDet,
(m[5]*m[6] - m[3]*m[8]) * invDet,
(m[0]*m[8] - m[2]*m[6]) * invDet,
(m[2]*m[3] - m[0]*m[5]) * invDet,
(m[3]*m[7] - m[4]*m[6]) * invDet,
(m[1]*m[6] - m[0]*m[7]) * invDet,
(m[0]*m[4] - m[1]*m[3]) * invDet,
}, true
}
// sampleBicubic returns the bicubic-interpolated (Catmull-Rom) color at the
// (possibly sub-pixel, out-of-bounds) location (x, y). Out-of-bounds
// coordinates use BORDER_REPLICATE semantics (edge pixels repeated). b is the
// source image bounds; sampling indices are offset by b.Min so a non-zero
// origin image samples correctly.
func sampleBicubic(img *image.RGBA, x, y float64, b image.Rectangle) color.RGBA {
ox, oy := float64(b.Min.X), float64(b.Min.Y)
x0 := int(math.Floor(x - ox))
y0 := int(math.Floor(y - oy))
tx := x - ox - float64(x0)
ty := y - oy - float64(y0)
maxX, maxY := b.Dx()-1, b.Dy()-1
// Interpolate each of the 4 source rows horizontally, then combine
// the 4 results vertically.
colX := func(cy int) (uint8, uint8, uint8, uint8) {
r0, g0, b0, a0 := pxAt(img, b.Min.X+clampIdx(x0-1, maxX), b.Min.Y+clampIdx(cy, maxY))
r1, g1, b1, a1 := pxAt(img, b.Min.X+clampIdx(x0, maxX), b.Min.Y+clampIdx(cy, maxY))
r2, g2, b2, a2 := pxAt(img, b.Min.X+clampIdx(x0+1, maxX), b.Min.Y+clampIdx(cy, maxY))
r3, g3, b3, a3 := pxAt(img, b.Min.X+clampIdx(x0+2, maxX), b.Min.Y+clampIdx(cy, maxY))
return uint8(clampByte(cubic(tx, [4]float64{float64(r0), float64(r1), float64(r2), float64(r3)}))),
uint8(clampByte(cubic(tx, [4]float64{float64(g0), float64(g1), float64(g2), float64(g3)}))),
uint8(clampByte(cubic(tx, [4]float64{float64(b0), float64(b1), float64(b2), float64(b3)}))),
uint8(clampByte(cubic(tx, [4]float64{float64(a0), float64(a1), float64(a2), float64(a3)})))
}
rA, gA, bA, aA := colX(y0 - 1)
rB, gB, bB, aB := colX(y0)
rC, gC, bC, aC := colX(y0 + 1)
rD, gD, bD, aD := colX(y0 + 2)
return color.RGBA{
R: uint8(clampByte(cubic(ty, [4]float64{float64(rA), float64(rB), float64(rC), float64(rD)}))),
G: uint8(clampByte(cubic(ty, [4]float64{float64(gA), float64(gB), float64(gC), float64(gD)}))),
B: uint8(clampByte(cubic(ty, [4]float64{float64(bA), float64(bB), float64(bC), float64(bD)}))),
A: uint8(clampByte(cubic(ty, [4]float64{float64(aA), float64(aB), float64(aC), float64(aD)}))),
}
}
// pxAt returns the RGBA bytes at (x, y), with coordinates already clamped by
// the caller (BORDER_REPLICATE).
func pxAt(img *image.RGBA, x, y int) (r, g, b, a uint8) {
c := img.RGBAAt(x, y)
return c.R, c.G, c.B, c.A
}
func clampIdx(i, max int) int {
if i < 0 {
return 0
}
if i > max {
return max
}
return i
}
func clampByte(v float64) float64 {
if v < 0 {
return 0
}
if v > 255 {
return 255
}
return v
}
// cubic is the Catmull-Rom cubic basis for parameter t in [0,1] over the four
// control samples p0..p3.
func cubic(t float64, p [4]float64) float64 {
t2 := t * t
t3 := t2 * t
return 0.5 * ((2 * p[1]) +
(-p[0]+p[2])*t +
(2*p[0]-5*p[1]+4*p[2]-p[3])*t2 +
(-p[0]+3*p[1]-3*p[2]+p[3])*t3)
}
// toRGBA returns src as *image.RGBA, converting when necessary.
func toRGBA(src image.Image) *image.RGBA {
if r, ok := src.(*image.RGBA); ok {
return r
}
b := src.Bounds()
out := image.NewRGBA(b)
draw.Draw(out, b, src, b.Min, draw.Src)
return out
}
// axisFallback crops the bounding box of the quad with FastCrop.
func axisFallback(src image.Image, points [4]Pt) *image.RGBA {
minX, minY := math.MaxFloat64, math.MaxFloat64
maxX, maxY := -math.MaxFloat64, -math.MaxFloat64
for _, p := range points {
minX = math.Min(minX, p.X)
minY = math.Min(minY, p.Y)
maxX = math.Max(maxX, p.X)
maxY = math.Max(maxY, p.Y)
}
return FastCrop(src, int(minX), int(minY), int(maxX), int(maxY))
}
func dist(a, b Pt) float64 {
return math.Hypot(a.X-b.X, a.Y-b.Y)
}
// pointsFinite reports whether all four corner coordinates are finite. A
// non-finite value from a malformed detector response must be rejected before
// any dimension derivation or allocation.
func pointsFinite(p [4]Pt) bool {
for _, q := range p {
if math.IsNaN(q.X) || math.IsNaN(q.Y) || math.IsInf(q.X, 0) || math.IsInf(q.Y, 0) {
return false
}
}
return true
}
// clampQuad clamps every corner to the source image bounds. FastCrop performs
// the equivalent clamp on its axis-aligned rectangle; WarpCrop must do the same
// on its four corners so an out-of-range detector box cannot produce an
// out-of-bounds or unbounded crop. Corners already inside the bounds are
// returned unchanged, so a well-formed detection box is unaffected.
func clampQuad(p [4]Pt, b image.Rectangle) [4]Pt {
out := p
minX, minY := float64(b.Min.X), float64(b.Min.Y)
maxX, maxY := float64(b.Max.X), float64(b.Max.Y)
for i := range out {
if out[i].X < minX {
out[i].X = minX
} else if out[i].X > maxX {
out[i].X = maxX
}
if out[i].Y < minY {
out[i].Y = minY
} else if out[i].Y > maxY {
out[i].Y = maxY
}
}
return out
}
// axisAligned reports whether the quad is axis-parallel: its left/right edges
// are vertical and its top/bottom edges are horizontal, within a small epsilon.
// The OCR detector can emit sub-pixel jitter on an otherwise upright box; that
// jitter is negligible for recognition, so the cheap FastCrop path is still
// correct for it. A genuinely slanted detection quad fails this test and pays
// the full perspective warp instead.
func axisAligned(p [4]Pt) bool {
const eps = 1e-3
// Quad order is TL, TR, BR, BL.
// Left edge TL-BL vertical: p0.X == p3.X
// Right edge TR-BR vertical: p1.X == p2.X
// Top edge TL-TR horizontal: p0.Y == p1.Y
// Bottom edge BL-BR horizonal: p3.Y == p2.Y
return approxEq(p[0].X, p[3].X, eps) &&
approxEq(p[1].X, p[2].X, eps) &&
approxEq(p[0].Y, p[1].Y, eps) &&
approxEq(p[3].Y, p[2].Y, eps)
}
func approxEq(a, b, eps float64) bool { return math.Abs(a-b) <= eps }

278
internal/deepdoc/parser/pdf/util/warp_test.go Normal file
View File

@@ -0,0 +1,278 @@
package util
import (
"bytes"
"encoding/base64"
"encoding/json"
"image"
"image/color"
"image/png"
"math"
"os"
"path/filepath"
"strings"
"testing"
)
// TestWarpCropMatchesGolden locks the perspective de-skew behaviour of
// WarpCrop against a reference warp (perspective transform with bicubic
// resampling) generated offline by testdata/gen_warp_golden.py. The reference
// uses the same homogeneous mapping as WarpCrop, so this test pins the
// geometry (output size + de-skew) of the implementation. Minor
// resampling-kernel differences between the reference sampler and the Go
// Catmull-Rom sampler are absorbed by the MSE tolerance.
//
// This is the unit-tier (model-free) lock for the warp step: the perspective
// de-skew applied to OCR detection quads before recognition.
func TestWarpCropMatchesGolden(t *testing.T) {
metaPath := filepath.Join("testdata", "warp_meta.json")
metaBytes, err := os.ReadFile(metaPath)
if err != nil {
t.Fatalf("read meta: %v", err)
}
var meta struct {
Src [4][2]float64 `json:"src"`
W int `json:"w"`
H int `json:"h"`
}
if err := json.Unmarshal(metaBytes, &meta); err != nil {
t.Fatalf("parse meta: %v", err)
}
var pts [4]Pt
for i := range meta.Src {
pts[i] = Pt{X: meta.Src[i][0], Y: meta.Src[i][1]}
}
src := loadGolden(t, filepath.Join("testdata", "warp_src.b64"))
expected := loadGolden(t, filepath.Join("testdata", "warp_expected.b64"))
got := WarpCrop(src, pts)
// Output size must match the reference contract exactly:
// W = int(max(|p0-p1|,|p2-p3|)), H = int(max(|p0-p3|,|p1-p2|)).
if got.Bounds().Dx() != meta.W || got.Bounds().Dy() != meta.H {
t.Fatalf("output size = %dx%d, want %dx%d",
got.Bounds().Dx(), got.Bounds().Dy(), meta.W, meta.H)
}
if expected.Bounds().Dx() != meta.W || expected.Bounds().Dy() != meta.H {
t.Fatalf("golden size = %dx%d, want %dx%d",
expected.Bounds().Dx(), expected.Bounds().Dy(), meta.W, meta.H)
}
mse := imageMSE(got, expected)
t.Logf("WarpCrop vs golden MSE = %.4f (RMSE/channel = %.4f)", mse, math.Sqrt(mse))
// Generous enough to absorb resampling-kernel differences, tight enough
// to catch a grossly wrong implementation (e.g. an axis-aligned crop of
// the same quad would diverge by orders of magnitude on this skewed input).
const maxMSE = 30.0
if mse > maxMSE {
t.Errorf("WarpCrop de-skew diverges from golden: MSE=%.4f > %.4f", mse, maxMSE)
}
// Sanity: WarpCrop must actually de-skew, not just return an axis-aligned
// bbox crop of the quad. On this perspective (non-parallelogram) input the
// output dimensions differ from the axis-aligned bbox, so the two are
// trivially unequal — confirm that rather than asserting a number.
bbox := axisFallback(src, pts)
if bbox.Bounds().Dx() == meta.W && bbox.Bounds().Dy() == meta.H {
t.Errorf("WarpCrop output size %dx%d equals the axis-aligned fallback size; warp may not be de-skewing",
meta.W, meta.H)
}
}
// TestWarpCropDegenerateQuadIsSafe checks that a collinear (degenerate) quad
// does not panic and returns a non-nil crop (falls back to axis-aligned).
func TestWarpCropDegenerateQuadIsSafe(t *testing.T) {
src := image.NewRGBA(image.Rect(0, 0, 50, 50))
// All four corners on a single line -> singular homography.
pts := [4]Pt{{10, 10}, {20, 10}, {30, 10}, {40, 10}}
got := WarpCrop(src, pts)
if got == nil {
t.Fatal("WarpCrop returned nil for degenerate quad")
}
if got.Bounds().Dx() <= 0 || got.Bounds().Dy() <= 0 {
t.Errorf("WarpCrop returned empty crop for degenerate quad: %dx%d",
got.Bounds().Dx(), got.Bounds().Dy())
}
}
// TestWarpCropAxisAlignedQuadIsStable checks that an already axis-aligned,
// axis-parallel quad is reproduced (up to bicubic resampling) without
// distortion — i.e. the output matches the source sub-rect.
func TestWarpCropAxisAlignedQuadIsStable(t *testing.T) {
src := image.NewRGBA(image.Rect(0, 0, 100, 100))
// Fill with a checkerboard so resampling has signal.
for y := 0; y < 100; y++ {
for x := 0; x < 100; x++ {
if ((x/10)+(y/10))%2 == 0 {
src.SetRGBA(x, y, color.RGBA{0, 0, 0, 255})
} else {
src.SetRGBA(x, y, color.RGBA{255, 255, 255, 255})
}
}
}
// Exact axis-aligned rectangle -> output should match the source sub-rect.
pts := [4]Pt{{20, 20}, {80, 20}, {80, 70}, {20, 70}}
got := WarpCrop(src, pts)
if got.Bounds().Dx() != 60 || got.Bounds().Dy() != 50 {
t.Fatalf("axis-aligned output size = %dx%d, want 60x50",
got.Bounds().Dx(), got.Bounds().Dy())
}
// For an axis-parallel quad the warp is identity (just a sub-rect copy),
// so it must match FastCrop of the same bbox up to resampling error.
want := FastCrop(src, 20, 20, 80, 70)
mse := imageMSE(got, want)
t.Logf("axis-aligned WarpCrop vs FastCrop MSE = %.4f", mse)
if mse > 5.0 {
t.Errorf("axis-aligned warp diverged from the source sub-rect: MSE=%.4f > 5.0", mse)
}
}
// loadGolden reads a single-line base64-encoded PNG fixture (committed as
// text so pre-commit text filters cannot corrupt the binary signature).
func loadGolden(t *testing.T, path string) *image.RGBA {
t.Helper()
raw, err := os.ReadFile(path)
if err != nil {
t.Fatalf("read %s: %v", path, err)
}
dec, err := base64.StdEncoding.DecodeString(strings.TrimSpace(string(raw)))
if err != nil {
t.Fatalf("base64 decode %s: %v", path, err)
}
img, err := png.Decode(bytes.NewReader(dec))
if err != nil {
t.Fatalf("decode %s: %v", path, err)
}
return toRGBA(img)
}
// imageMSE returns the mean squared error across all RGBA channels between a
// and b (both must have identical dimensions).
func imageMSE(a, b *image.RGBA) float64 {
ba, bb := a.Bounds(), b.Bounds()
if ba.Dx() != bb.Dx() || ba.Dy() != bb.Dy() {
return math.MaxFloat64
}
var acc float64
n := ba.Dx() * ba.Dy()
for y := 0; y < ba.Dy(); y++ {
for x := 0; x < ba.Dx(); x++ {
ca := a.RGBAAt(x, y)
cb := b.RGBAAt(x, y)
acc += sqDiff(ca.R, cb.R) + sqDiff(ca.G, cb.G) + sqDiff(ca.B, cb.B) + sqDiff(ca.A, cb.A)
}
}
return acc / float64(n*4)
}
func sqDiff(x, y uint8) float64 {
d := float64(x) - float64(y)
return d * d
}
// TestWarpCropRespectsNonZeroOrigin guards the source-image bounds handling in
// sampleBicubic: a source with a non-zero origin (Min != (0,0)) must be sampled
// at its absolute coordinates, not relative to (0,0). WarpCrop on such an image
// must produce the same crop as WarpCrop on an equivalent (0,0)-origin image
// holding identical pixels at the same absolute coordinates.
//
// The quad is interior to both images so the sampler never reaches either
// image's edge; this isolates the origin handling from edge-replication
// differences and exercises the far-edge clamp where a zero-origin assumption
// would clamp too early.
func TestWarpCropRespectsNonZeroOrigin(t *testing.T) {
gradient := func(x, y int) color.RGBA {
return color.RGBA{uint8(x % 256), uint8(y % 256), uint8((x + y) % 256), 255}
}
// Non-zero-origin source with its own pixel buffer.
origin := image.Pt(50, 50)
sub := image.NewRGBA(image.Rect(origin.X, origin.Y, origin.X+200, origin.Y+200))
for y := origin.Y; y < origin.Y+200; y++ {
for x := origin.X; x < origin.X+200; x++ {
sub.SetRGBA(x, y, gradient(x, y))
}
}
// Equivalent (0,0)-origin image holding the same pixels at the same
// absolute coordinates.
flat := image.NewRGBA(image.Rect(0, 0, 300, 300))
for y := 0; y < 300; y++ {
for x := 0; x < 300; x++ {
flat.SetRGBA(x, y, gradient(x, y))
}
}
// Interior quad in absolute coordinates (so sampling stays away from both
// images' edges).
pts := [4]Pt{
{X: 60, Y: 60},
{X: 240, Y: 60},
{X: 240, Y: 240},
{X: 60, Y: 240},
}
gotSub := WarpCrop(sub, pts)
gotFlat := WarpCrop(flat, pts)
if gotSub.Bounds() != gotFlat.Bounds() {
t.Fatalf("output size mismatch: sub=%v flat=%v", gotSub.Bounds(), gotFlat.Bounds())
}
// With correct origin handling the two are pixel-identical; a zero-origin
// assumption clamps ~20% of the crop too early and diverges by orders of
// magnitude.
if mse := imageMSE(gotSub, gotFlat); mse > 1e-3 {
t.Errorf("WarpCrop ignored the source image origin: MSE between sub- and flat-frame warps = %v", mse)
}
}
// TestWarpCropRejectsMalformedQuad guards against a process-crashing panic /
// OOM on an out-of-range or non-finite detector quad. The old FastCrop path
// clamped coordinates to the source bounds before allocating; WarpCrop must be
// equally safe. A finite but absurd coordinate (e.g. 3e18) would otherwise
// reach image.NewRGBA and panic with "huge or negative dimensions", and a
// non-finite coordinate would drive an undefined-size allocation.
//
// This is a regression guard for the untrusted-boundary contract: OCRDetect
// accepts coordinates from a configured DocAnalyzer / DEEPDOC_URL, and the
// first-party Python detector clips its points, but that invariant is not
// enforced at this Go boundary.
func TestWarpCropRejectsMalformedQuad(t *testing.T) {
src := image.NewRGBA(image.Rect(0, 0, 10, 10))
cases := []struct {
name string
pts [4]Pt
}{
{"huge x", [4]Pt{{0, 0}, {3e18, 0}, {3e18, 2}, {0, 2}}},
{"huge negative", [4]Pt{{-3e18, 0}, {0, 0}, {0, 2}, {-3e18, 2}}},
{"nan", [4]Pt{{0, 0}, {math.NaN(), 0}, {10, 10}, {0, 10}}},
{"inf", [4]Pt{{0, 0}, {math.Inf(1), 0}, {10, 10}, {0, 10}}},
{"outside bounds", [4]Pt{{-100, -100}, {200, -100}, {200, 200}, {-100, 200}}},
}
for _, tc := range cases {
t.Run(tc.name, func(t *testing.T) {
var got *image.RGBA
func() {
defer func() {
if r := recover(); r != nil {
t.Fatalf("WarpCrop panicked on %q: %v", tc.name, r)
}
}()
got = WarpCrop(src, tc.pts)
}()
if got == nil {
t.Fatalf("WarpCrop returned nil on %q", tc.name)
}
w, h := got.Bounds().Dx(), got.Bounds().Dy()
if w <= 0 || h <= 0 {
t.Errorf("WarpCrop returned an empty crop on %q: %dx%d", tc.name, w, h)
}
if w > maxWarpDim || h > maxWarpDim {
t.Errorf("WarpCrop returned an unbounded crop on %q: %dx%d", tc.name, w, h)
}
})
}
}

133
internal/deepdoc/parser/pdf/warp_align_test.go Normal file
View File

@@ -0,0 +1,133 @@
//go:build cgo && manual
package pdf
import (
"encoding/json"
"fmt"
"image"
"math"
"os"
"path/filepath"
"testing"
util "ragflow/internal/deepdoc/parser/pdf/util"
)
// TestWarpAlignGo renders a real PDF page with the production pdfium render
// path (the exact image the OCR detector receives) and, once a quads.json has
// been produced by tools/render_diff/warp_align.py's detect phase, also writes
// the Go WarpCrop and FastCrop crops for every detected quad.
//
// The crops land under WARP_OUT (default /tmp/render_diff/align) so nothing is
// committed and the worktree stays isolated from the rest of the repo.
//
// Run (two passes):
//
// # pass 1: render the page (WARP_PDF is required by the Go test)
// WARP_PDF=test/benchmark/test_docs/Doc1.pdf \
// bash build.sh --test-manual ./internal/deepdoc/parser/pdf/ \
// -run TestWarpAlignGo
// # derive quads + reference crops from the rendered page
// .venv/bin/python tools/render_diff/warp_align.py genquads \
// --pdf test/benchmark/test_docs/Doc1.pdf --go-page-png /tmp/render_diff/align/page0.png
// # pass 2: now quads.json exists -> write the Go crops
// WARP_PDF=test/benchmark/test_docs/Doc1.pdf \
// bash build.sh --test-manual ./internal/deepdoc/parser/pdf/ \
// -run TestWarpAlignGo
// .venv/bin/python tools/render_diff/warp_align.py compare
//
// Env:
//
// WARP_PDF (required on pass 1) input PDF path
// WARP_PAGE (default 0) 0-based page index
// WARP_OUT (default /tmp/render_diff/align) output directory
// WARP_QUADS (default <WARP_OUT>/quads.json) detect-phase output
type warpAlignBox struct {
Quad []float64 `json:"quad"` // [x0,y0,x1,y1,x2,y2,x3,y3]
}
type warpAlignQuads struct {
Boxes []warpAlignBox `json:"boxes"`
}
func TestWarpAlignGo(t *testing.T) {
pdfPath := os.Getenv("WARP_PDF")
if pdfPath == "" {
t.Skip("WARP_PDF not set; nothing to render")
}
page := 0
if v := os.Getenv("WARP_PAGE"); v != "" {
fmt.Sscanf(v, "%d", &page)
}
out := os.Getenv("WARP_OUT")
if out == "" {
out = filepath.Join("/tmp", "render_diff", "align")
}
if err := os.MkdirAll(out, 0o755); err != nil {
t.Fatalf("mkdir: %v", err)
}
data, err := os.ReadFile(pdfPath)
if err != nil {
t.Fatalf("read pdf: %v", err)
}
engine, err := NewEngine(data)
if err != nil {
t.Fatalf("new engine: %v", err)
}
defer engine.Close()
img, err := RenderPageToImage(engine, page)
if err != nil {
t.Fatalf("render page %d: %v", page, err)
}
if err := writeAlignPNG(filepath.Join(out, fmt.Sprintf("page%d.png", page)), img); err != nil {
t.Fatalf("write page: %v", err)
}
t.Logf("wrote page%d.png (%dx%d)", page, img.Bounds().Dx(), img.Bounds().Dy())
quadsPath := os.Getenv("WARP_QUADS")
if quadsPath == "" {
quadsPath = filepath.Join(out, "quads.json")
}
raw, err := os.ReadFile(quadsPath)
if err != nil {
t.Skipf("quads not found at %s; run warp_align.py detect first", quadsPath)
}
var q warpAlignQuads
if err := json.Unmarshal(raw, &q); err != nil {
t.Fatalf("parse quads: %v", err)
}
for i, b := range q.Boxes {
if len(b.Quad) != 8 {
t.Fatalf("box %d: bad quad len %d", i, len(b.Quad))
}
pts := [4]util.Pt{
{X: b.Quad[0], Y: b.Quad[1]},
{X: b.Quad[2], Y: b.Quad[3]},
{X: b.Quad[4], Y: b.Quad[5]},
{X: b.Quad[6], Y: b.Quad[7]},
}
warp := util.WarpCrop(img, pts)
if err := writeAlignPNG(filepath.Join(out, fmt.Sprintf("go_warp_%d.png", i)), warp); err != nil {
t.Fatalf("write warp %d: %v", i, err)
}
x0 := int(math.Min(b.Quad[0], math.Min(b.Quad[2], math.Min(b.Quad[4], b.Quad[6]))))
y0 := int(math.Min(b.Quad[1], math.Min(b.Quad[3], math.Min(b.Quad[5], b.Quad[7]))))
x1 := int(math.Max(b.Quad[0], math.Max(b.Quad[2], math.Max(b.Quad[4], b.Quad[6]))))
y1 := int(math.Max(b.Quad[1], math.Max(b.Quad[3], math.Max(b.Quad[5], b.Quad[7]))))
fc := util.FastCrop(img, x0, y0, x1, y1)
if err := writeAlignPNG(filepath.Join(out, fmt.Sprintf("go_fastcrop_%d.png", i)), fc); err != nil {
t.Fatalf("write fastcrop %d: %v", i, err)
}
}
t.Logf("wrote %d go crops (warp + fastcrop)", len(q.Boxes))
}
func writeAlignPNG(path string, img image.Image) error {
b, err := util.EncodePNG(img)
if err != nil {
return err
}
return os.WriteFile(path, b, 0o644)
}

359
tools/render_diff/warp_align.py Executable file
View File

@@ -0,0 +1,359 @@
#!/usr/bin/env python3
"""WarpCrop (Go) vs an independent perspective-crop reference (cv2) pixel
alignment on REAL document text, with no model weights required.
This exercises the perspective de-skew step (layer 1 of the OCR warp work). It
answers the geometric part of the regression question: for real text regions on
a real PDF page, does Go's util.WarpCrop reproduce the same crop geometry as an
independent perspective transform?
Detection and recognition need the deepdoc ONNX weights (downloaded from
HuggingFace, unavailable offline). They are intentionally NOT used here:
- quads come from pdfplumber word boxes (rotated to simulate skew) -- no model
- the reference crop is pure geometry (getPerspectiveTransform +
warpPerspective with BORDER_REPLICATE + INTER_CUBIC) -- no model
So this runs fully offline and isolates the warp geometry, which is the part
WarpCrop actually implements.
Pipeline
--------
1. genquads : from the Go-rendered page, derive real word boxes (pdfplumber),
rotate each by a sweep of angles to build skewed quads, write
quads.json and the reference crops (py_warp_*.png).
2. (Go) : TestWarpAlignGo pass 2 reads quads.json and writes go_warp_*.png
(WarpCrop, de-skewed) and go_fastcrop_*.png (FastCrop, axis-aligned bbox).
3. compare : pixel-MSE Go-WarpCrop vs reference and Go-FastCrop vs reference,
aggregated (layer-1 geometry; no model weights needed).
4. rec : run recognize_batch (the actual Go->service path, NO layer-2
rotation) on all three crops and compare the recognized text
(layer-1 + recognizer; needs the deepdoc ONNX weights).
All outputs go under --out-dir (default /tmp/render_diff/align); the worktree
is never touched and nothing is committed.
Run
---
WARP_PDF=test/benchmark/test_docs/Doc1.pdf \
bash build.sh --test-manual ./internal/deepdoc/parser/pdf/ -run TestWarpAlignGo
.venv/bin/python tools/render_diff/warp_align.py genquads \
--pdf test/benchmark/test_docs/Doc1.pdf --go-page-png /tmp/render_diff/align/page0.png
WARP_PDF=test/benchmark/test_docs/Doc1.pdf \
bash build.sh --test-manual ./internal/deepdoc/parser/pdf/ -run TestWarpAlignGo
.venv/bin/python tools/render_diff/warp_align.py compare
# (needs HF weights in rag/res/deepdoc)
.venv/bin/python tools/render_diff/warp_align.py rec
"""
import argparse
import difflib
import json
import logging
import math
import os
import sys
import cv2
import numpy as np
import pdfplumber
logging.basicConfig(level=logging.INFO, format="%(message)s", stream=sys.stdout)
# Go renders the page at exactly 3x (216 DPI); pdfplumber coords are in points.
SCALE = 3.0
def _sim(a, b):
"""Normalized similarity in [0,1] between two recognized strings."""
return difflib.SequenceMatcher(None, a or "", b or "").ratio()
def _quad_from_word(w, angle_deg, W, H, margin, perturb=4.0):
"""Return [x0,y0,x1,y1,x2,y2,x3,y3] (TL,TR,BR,BL) rotated by angle_deg, or
None if the rotated quad leaves the page.
A small deterministic perturbation is applied to the bottom-left corner so
the quad is a genuine trapezoid rather than a parallelogram. A plain
rotated rectangle maps to a parallelogram, whose getPerspectiveTransform has
no projective term -- the 8-DOF (true perspective) path in WarpCrop would
then go untested against cv2. The perturbation makes the cv2 reference
exercise the full homography the Go code implements.
"""
x0, top, x1, bottom = w["x0"], w["top"], w["x1"], w["bottom"]
pts = [
(x0 * SCALE, top * SCALE),
(x1 * SCALE, top * SCALE),
(x1 * SCALE, bottom * SCALE),
(x0 * SCALE, bottom * SCALE),
]
cx = (x0 + x1) / 2 * SCALE
cy = (top + bottom) / 2 * SCALE
a = math.radians(angle_deg)
ca, sa = math.cos(a), math.sin(a)
rot = []
for x, y in pts:
dx, dy = x - cx, y - cy
rot.append((cx + dx * ca - dy * sa, cy + dx * sa + dy * ca))
# Break the parallelogram so cv2 exercises the non-zero projective terms.
bx, by = rot[3]
rot[3] = (bx + perturb, by - perturb)
# in-bounds check
for x, y in rot:
if not (margin <= x <= W - margin and margin <= y <= H - margin):
return None
return [v for p in rot for v in p]
def _cv2_warp(page, quad):
p = np.array([[quad[0], quad[1]], [quad[2], quad[3]], [quad[4], quad[5]], [quad[6], quad[7]]], dtype=np.float32)
w = int(max(np.linalg.norm(p[0] - p[1]), np.linalg.norm(p[2] - p[3])))
h = int(max(np.linalg.norm(p[0] - p[3]), np.linalg.norm(p[1] - p[2])))
std = np.array([[0, 0], [w, 0], [w, h], [0, h]], dtype=np.float32)
M = cv2.getPerspectiveTransform(p, std)
return cv2.warpPerspective(page, M, (w, h), borderMode=cv2.BORDER_REPLICATE, flags=cv2.INTER_CUBIC)
def cmd_genquads(args):
page = cv2.imread(args.go_page_png)
if page is None:
raise SystemExit(f"cannot read {args.go_page_png}")
H, W = page.shape[:2]
with pdfplumber.open(args.pdf) as pdf:
pg = pdf.pages[args.page]
words = pg.extract_words()
angles = [float(x) for x in args.angles.split(",")]
boxes = []
for w in words:
if w["bottom"] - w["top"] < 6: # skip tiny fragments
continue
for ang in angles:
if len(boxes) >= args.max_quads:
break
q = _quad_from_word(w, ang, W, H, args.margin)
if q is None:
continue
idx = len(boxes)
boxes.append({"quad": q, "text": w.get("text", ""), "angle": ang})
cv2.imwrite(os.path.join(args.out_dir, f"py_warp_{idx}.png"), _cv2_warp(page, q))
if len(boxes) >= args.max_quads:
break
with open(os.path.join(args.out_dir, "quads.json"), "w") as f:
json.dump({"boxes": [{"quad": b["quad"]} for b in boxes]}, f, indent=2)
with open(os.path.join(args.out_dir, "quads_meta.json"), "w") as f:
json.dump(boxes, f, indent=2)
logging.info("[genquads] %d quads (from %d words, angles=%s) -> quads.json + py_warp_*.png", len(boxes), len(words), angles)
def pixel_mse(a_path, b_path):
"""Pixel MSE between two crops that MUST share identical dimensions.
Width/height are part of the WarpCrop <-> cv2 parity contract: a 1px
dimension difference (e.g. float64 vs float32 truncation in the norm) is a
real divergence and must surface, not be hidden by resizing. Callers that
compare the de-skewed WarpCrop crop against the cv2 reference use this and
treat a shape mismatch as a contract failure.
"""
A = np.asarray(cv2.imread(a_path), dtype=np.float64)
B = np.asarray(cv2.imread(b_path), dtype=np.float64)
if A.shape != B.shape:
raise ValueError(f"shape mismatch for {os.path.basename(a_path)} vs {os.path.basename(b_path)}: {A.shape} != {B.shape} (dimensions are part of the parity contract)")
diff = A - B
return float((diff**2).mean())
def pixel_mse_baseline(a_path, b_path):
"""Pixel MSE for the FastCrop baseline vs cv2.
FastCrop is axis-aligned while cv2 de-skews, so the two crops legitimately
differ in size. Resize to a common size to obtain a scalar divergence
estimate (the baseline is known-wrong; only its magnitude matters).
"""
A = np.asarray(cv2.imread(a_path), dtype=np.float64)
B = np.asarray(cv2.imread(b_path), dtype=np.float64)
if A.shape != B.shape:
B = cv2.resize(B, (A.shape[1], A.shape[0]))
diff = A - B
return float((diff**2).mean())
def cmd_compare(args):
with open(os.path.join(args.out_dir, "quads.json")) as f:
boxes = json.load(f)["boxes"]
try:
with open(os.path.join(args.out_dir, "quads_meta.json")) as f:
meta = json.load(f)
except FileNotFoundError:
meta = [{}] * len(boxes)
limit = args.limit if args.limit and args.limit > 0 else len(boxes)
mse_warp, mse_fc = [], []
warp_results = [] # (box_index, mw) so the per-angle summary stays aligned
better = 0 # cases where WarpCrop is closer to cv2 than FastCrop
warp_dim_mismatch = 0 # WarpCrop vs cv2 dimension contract failures
for i in range(min(limit, len(boxes))):
gw = os.path.join(args.out_dir, f"go_warp_{i}.png")
gf = os.path.join(args.out_dir, f"go_fastcrop_{i}.png")
pw = os.path.join(args.out_dir, f"py_warp_{i}.png")
if not (os.path.exists(gw) and os.path.exists(gf) and os.path.exists(pw)):
continue
# WarpCrop must match cv2 exactly in size; a mismatch is a contract
# failure (previously hidden by resizing) and is not counted as "closer".
try:
mw = pixel_mse(gw, pw)
except ValueError as e:
warp_dim_mismatch += 1
logging.warning("[compare] box %d: %s", i, e)
continue
mf = pixel_mse_baseline(gf, pw)
mse_warp.append(mw)
mse_fc.append(mf)
warp_results.append((i, mw))
if mw < mf:
better += 1
n = len(mse_warp)
if n == 0:
logging.info("[compare] no go crops found; run TestWarpAlignGo pass 2 first")
return
mean_w = sum(mse_warp) / n
mean_f = sum(mse_fc) / n
logging.info("\n[compare] %d boxes (limit=%s)", n, limit)
logging.info(" pixel MSE WarpCrop vs cv2 : mean=%.2f max=%.2f", mean_w, max(mse_warp))
logging.info(" pixel MSE FastCrop vs cv2 : mean=%.2f max=%.2f", mean_f, max(mse_fc))
logging.info(" WarpCrop closer to cv2 than FastCrop : %d/%d", better, n)
if warp_dim_mismatch:
logging.info(" WarpCrop vs cv2 DIMENSION MISMATCH (contract fail) : %d", warp_dim_mismatch)
# show per-angle summary if meta present; pair each MSE with its box index
# so a skipped/missing crop cannot misalign the angle buckets.
by_angle = {}
for idx, mw in warp_results:
ang = meta[idx].get("angle", 0) if idx < len(meta) else 0
by_angle.setdefault(ang, []).append(mw)
if by_angle:
logging.info("\n -- mean MSE(WarpCrop vs cv2) by rotation angle --")
for ang in sorted(by_angle):
vals = by_angle[ang]
logging.info(" angle %+6.1fdeg : n=%3d meanMSE=%.2f", ang, len(vals), sum(vals) / len(vals))
def cmd_rec(args):
"""Recognition-level alignment (the decisive "no regression" evidence).
The Go->Python OCR service feeds cropped image BYTES (no box) to the
recognizer via recognize_batch(), which does NOT perform layer-2 rotation
selection. So the apples-to-apples comparison is recognize_batch() on each
of the three crops already on disk:
go_warp_*.png : Go util.WarpCrop (de-skewed, layer-1 only)
go_fastcrop_*.png : Go util.FastCrop (axis-aligned bounding box)
py_warp_*.png : cv2.getPerspectiveTransform + warpPerspective
(the de-skewed crop Python produces before recognition)
If go_warp matches py_warp at the text level and both beat go_fastcrop,
WarpCrop reproduces the de-skewed crop geometry that FastCrop misses.
"""
from deepdoc.vision.ocr import OCR
ocr = OCR()
limit = args.limit if args.limit and args.limit > 0 else 10**9
n = 0
warp_eq_py = 0 # WarpCrop text == cv2 text (exact)
fc_eq_py = 0 # FastCrop text == cv2 text (exact)
warp_sim_total = 0.0
fc_sim_total = 0.0
warp_closer = 0 # WarpCrop text closer to cv2 than FastCrop text
by_angle = {}
worst = [] # (sim_warp, i, text_warp, text_py, text_fc)
# Per-box rotation angle, if the genquads metadata is present.
meta = []
meta_path = os.path.join(args.out_dir, "quads_meta.json")
if os.path.exists(meta_path):
with open(meta_path) as f:
meta = json.load(f)
for i in range(limit):
gw = os.path.join(args.out_dir, f"go_warp_{i}.png")
gf = os.path.join(args.out_dir, f"go_fastcrop_{i}.png")
pw = os.path.join(args.out_dir, f"py_warp_{i}.png")
if not (os.path.exists(gw) and os.path.exists(gf) and os.path.exists(pw)):
break
img_w = cv2.imread(gw)
img_f = cv2.imread(gf)
img_p = cv2.imread(pw)
tw = ocr.recognize_batch([img_w])[0]
tf = ocr.recognize_batch([img_f])[0]
tp = ocr.recognize_batch([img_p])[0]
sw = _sim(tw, tp)
sf = _sim(tf, tp)
warp_sim_total += sw
fc_sim_total += sf
if tw == tp:
warp_eq_py += 1
if tf == tp:
fc_eq_py += 1
if sw > sf:
warp_closer += 1
# per-angle bucket
ang = meta[i].get("angle", 0.0) if i < len(meta) else 0.0
by_angle.setdefault(ang, []).append(sw)
worst.append((sw, i, tw, tp, tf))
n += 1
if n == 0:
logging.info("[rec] no crops found; run genquads + TestWarpAlignGo pass 2 first")
return
logging.info("\n[rec] %d boxes (recognize_batch, no layer-2 rotation)", n)
logging.info(" exact text match WarpCrop vs cv2 : %d/%d (%.1f%%)", warp_eq_py, n, 100 * warp_eq_py / n)
logging.info(" exact text match FastCrop vs cv2 : %d/%d (%.1f%%)", fc_eq_py, n, 100 * fc_eq_py / n)
logging.info(" mean text similarity WarpCrop vs cv2 : %.3f", warp_sim_total / n)
logging.info(" mean text similarity FastCrop vs cv2 : %.3f", fc_sim_total / n)
logging.info(" WarpCrop text strictly closer to cv2 than FastCrop : %d/%d (%.1f%%)", warp_closer, n, 100 * warp_closer / n)
if by_angle:
logging.info("\n -- mean text similarity(WarpCrop vs cv2) by rotation angle --")
for ang in sorted(by_angle):
vals = by_angle[ang]
logging.info(" angle %+6.1fdeg : n=%3d sim=%.3f", ang, len(vals), sum(vals) / len(vals))
logging.info("\n -- 8 worst WarpCrop-vs-cv2 cases --")
worst.sort(key=lambda x: x[0])
for sw, i, tw, tp, tf in worst[:8]:
logging.info(" #%3d sim=%.2f | warp=%r py=%r fastcrop=%r", i, sw, tw, tp, tf)
def main():
ap = argparse.ArgumentParser()
sub = ap.add_subparsers(dest="cmd", required=True)
g = sub.add_parser("genquads")
g.add_argument("--pdf", required=True)
g.add_argument("--page", type=int, default=0)
g.add_argument("--go-page-png", required=True)
g.add_argument("--out-dir", default="/tmp/render_diff/align")
g.add_argument("--angles", default="0,15,-15,30,-30")
g.add_argument("--max-quads", type=int, default=120)
g.add_argument("--margin", type=int, default=24)
c = sub.add_parser("compare")
c.add_argument("--out-dir", default="/tmp/render_diff/align")
c.add_argument("--limit", type=int, default=0, help="max boxes to compare (0 = all)")
r = sub.add_parser("rec", help="recognition-level alignment (needs deepdoc weights)")
r.add_argument("--out-dir", default="/tmp/render_diff/align")
r.add_argument("--limit", type=int, default=0, help="max boxes to recognize (0 = all)")
args = ap.parse_args()
os.makedirs(args.out_dir, exist_ok=True)
if args.cmd == "genquads":
cmd_genquads(args)
elif args.cmd == "rec":
cmd_rec(args)
else:
cmd_compare(args)
if __name__ == "__main__":
main()