mirror of
https://github.com/infiniflow/ragflow.git
synced 2026-07-31 21:13:49 +08:00
Refactor: migrate pdf_parser.py to golang (#16323)
### What problem does this PR solve? Http API based on onnx model. pdf_parser.py to golang ### Type of change - [x] Refactoring
This commit is contained in:
109
internal/deepdoc/parser/pdf/pdfoxide/cropbox.go
Normal file
109
internal/deepdoc/parser/pdf/pdfoxide/cropbox.go
Normal file
@@ -0,0 +1,109 @@
|
||||
package pdfoxide
|
||||
|
||||
import "strconv"
|
||||
|
||||
// parseCropBoxFromRaw scans raw PDF bytes for /CropBox entries and
|
||||
// returns the array [x0, y0, x1, y1] for the given page index (0-based).
|
||||
// The second return value is false if no /CropBox was found.
|
||||
//
|
||||
// Algorithm: sequential scan of "/CropBox [...]" patterns — same approach
|
||||
// as parsePageRotationFromRaw. Works for all common PDF generators.
|
||||
func parseCropBoxFromRaw(data []byte, pageIdx int) ([4]float64, bool) {
|
||||
type cb [4]float64
|
||||
var boxes []cb
|
||||
rest := data
|
||||
for {
|
||||
idx := indexAfter(rest, "/CropBox")
|
||||
if idx < 0 {
|
||||
break
|
||||
}
|
||||
rest = rest[idx:]
|
||||
// Skip whitespace, expect '['
|
||||
for len(rest) > 0 && isSpace(rest[0]) {
|
||||
rest = rest[1:]
|
||||
}
|
||||
if len(rest) == 0 || rest[0] != '[' {
|
||||
continue
|
||||
}
|
||||
rest = rest[1:]
|
||||
// Parse 4 float values inside [...]
|
||||
var vals [4]float64
|
||||
ok := true
|
||||
for i := 0; i < 4; i++ {
|
||||
for len(rest) > 0 && isSpace(rest[0]) {
|
||||
rest = rest[1:]
|
||||
}
|
||||
v, n := parseFloat(rest)
|
||||
if n == 0 {
|
||||
ok = false
|
||||
break
|
||||
}
|
||||
vals[i] = v
|
||||
rest = rest[n:]
|
||||
}
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
boxes = append(boxes, cb(vals))
|
||||
}
|
||||
if pageIdx < len(boxes) {
|
||||
return boxes[pageIdx], true
|
||||
}
|
||||
return [4]float64{}, false
|
||||
}
|
||||
|
||||
// indexAfter finds the byte position right after the first occurrence of s in
|
||||
// data. Returns -1 if not found.
|
||||
func indexAfter(data []byte, s string) int {
|
||||
for i := 0; i < len(data)-len(s); i++ {
|
||||
match := true
|
||||
for j := 0; j < len(s); j++ {
|
||||
if data[i+j] != s[j] {
|
||||
match = false
|
||||
break
|
||||
}
|
||||
}
|
||||
if match {
|
||||
return i + len(s)
|
||||
}
|
||||
}
|
||||
return -1
|
||||
}
|
||||
|
||||
func isSpace(b byte) bool {
|
||||
return b == ' ' || b == '\t' || b == '\n' || b == '\r'
|
||||
}
|
||||
|
||||
// parseFloat parses a decimal number from the beginning of s.
|
||||
// Returns the value and the number of bytes consumed (0 on failure).
|
||||
func parseFloat(s []byte) (float64, int) {
|
||||
i := 0
|
||||
for i < len(s) && isSpace(s[i]) {
|
||||
i++
|
||||
}
|
||||
j := i
|
||||
// Scan: optional sign, digits, optional decimal point + digits
|
||||
if j < len(s) && (s[j] == '+' || s[j] == '-') {
|
||||
j++
|
||||
}
|
||||
hasDigit := false
|
||||
for j < len(s) && s[j] >= '0' && s[j] <= '9' {
|
||||
j++
|
||||
hasDigit = true
|
||||
}
|
||||
if j < len(s) && s[j] == '.' {
|
||||
j++
|
||||
for j < len(s) && s[j] >= '0' && s[j] <= '9' {
|
||||
j++
|
||||
hasDigit = true
|
||||
}
|
||||
}
|
||||
if !hasDigit || j == i {
|
||||
return 0, 0
|
||||
}
|
||||
v, err := strconv.ParseFloat(string(s[i:j]), 64)
|
||||
if err != nil {
|
||||
return 0, 0
|
||||
}
|
||||
return v, j
|
||||
}
|
||||
128
internal/deepdoc/parser/pdf/pdfoxide/cropbox_test.go
Normal file
128
internal/deepdoc/parser/pdf/pdfoxide/cropbox_test.go
Normal file
@@ -0,0 +1,128 @@
|
||||
package pdfoxide
|
||||
|
||||
import (
|
||||
"math"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestParseCropBoxFromRaw(t *testing.T) {
|
||||
eps := 1e-6
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
raw string
|
||||
pageIdx int
|
||||
want [4]float64
|
||||
ok bool
|
||||
}{
|
||||
{
|
||||
name: "standard A4 portrait",
|
||||
raw: "/CropBox [0 0 595.28 841.89]",
|
||||
want: [4]float64{0, 0, 595.28, 841.89},
|
||||
ok: true,
|
||||
},
|
||||
{
|
||||
name: "non-zero origin",
|
||||
raw: "/CropBox [30 20 575 832]",
|
||||
want: [4]float64{30, 20, 575, 832},
|
||||
ok: true,
|
||||
},
|
||||
{
|
||||
name: "with extra whitespace",
|
||||
raw: "/CropBox [ 0.5 10.25 595.3 842.0 ]",
|
||||
want: [4]float64{0.5, 10.25, 595.3, 842.0},
|
||||
ok: true,
|
||||
},
|
||||
{
|
||||
name: "no spaces inside brackets",
|
||||
raw: "/CropBox[0 0 595 842]",
|
||||
want: [4]float64{0, 0, 595, 842},
|
||||
ok: true,
|
||||
},
|
||||
{
|
||||
name: "page index 1 picks second CropBox",
|
||||
raw: "/CropBox [0 0 1 1] /Rotate 90 /CropBox [2 2 3 3]",
|
||||
pageIdx: 1,
|
||||
want: [4]float64{2, 2, 3, 3},
|
||||
ok: true,
|
||||
},
|
||||
{
|
||||
name: "page index out of range",
|
||||
raw: "/CropBox [0 0 1 1]",
|
||||
pageIdx: 5,
|
||||
want: [4]float64{},
|
||||
ok: false,
|
||||
},
|
||||
{
|
||||
name: "no cropbox",
|
||||
raw: "/MediaBox [0 0 595 842] /Rotate 90",
|
||||
want: [4]float64{},
|
||||
ok: false,
|
||||
},
|
||||
{
|
||||
name: "empty input",
|
||||
raw: "",
|
||||
want: [4]float64{},
|
||||
ok: false,
|
||||
},
|
||||
{
|
||||
name: "incomplete array — fewer than 4 values",
|
||||
raw: "/CropBox [0 0 595]",
|
||||
want: [4]float64{},
|
||||
ok: false,
|
||||
},
|
||||
{
|
||||
name: "negative values",
|
||||
raw: "/CropBox [-10 -20 595 842]",
|
||||
want: [4]float64{-10, -20, 595, 842},
|
||||
ok: true,
|
||||
},
|
||||
{
|
||||
name: "real pypdf output format (multiple spaces, decimals)",
|
||||
raw: "/Type /Page /MediaBox [0 0 595.2756 841.8898] /CropBox [30.0 20.0 575.0 832.0] /Rotate 90",
|
||||
want: [4]float64{30.0, 20.0, 575.0, 832.0},
|
||||
ok: true,
|
||||
},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got, ok := parseCropBoxFromRaw([]byte(tt.raw), tt.pageIdx)
|
||||
if ok != tt.ok {
|
||||
t.Fatalf("ok=%v want %v", ok, tt.ok)
|
||||
}
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
for i := 0; i < 4; i++ {
|
||||
if math.Abs(got[i]-tt.want[i]) > eps {
|
||||
t.Errorf("[%d]: got %.4f, want %.4f", i, got[i], tt.want[i])
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseFloat(t *testing.T) {
|
||||
tests := []struct {
|
||||
s string
|
||||
want float64
|
||||
n int
|
||||
}{
|
||||
{"0", 0, 1},
|
||||
{"595.28", 595.28, 6},
|
||||
{" 42", 42, 4},
|
||||
{"-10.5", -10.5, 5},
|
||||
{"+3.14", 3.14, 5},
|
||||
{"123abc", 123, 3},
|
||||
{"abc", 0, 0},
|
||||
{"", 0, 0},
|
||||
{".5", 0.5, 2},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
v, n := parseFloat([]byte(tt.s))
|
||||
if n != tt.n || math.Abs(v-tt.want) > 1e-6 {
|
||||
t.Errorf("parseFloat(%q) = (%.4f, %d), want (%.4f, %d)",
|
||||
tt.s, v, n, tt.want, tt.n)
|
||||
}
|
||||
}
|
||||
}
|
||||
375
internal/deepdoc/parser/pdf/pdfoxide/pdf_oxide_adapter.go
Normal file
375
internal/deepdoc/parser/pdf/pdfoxide/pdf_oxide_adapter.go
Normal file
@@ -0,0 +1,375 @@
|
||||
//go:build cgo
|
||||
|
||||
// Package pdfparser provides pdf_oxide-based PDF types and functions.
|
||||
//
|
||||
// This file wraps github.com/yfedoseev/pdf_oxide/go (pdf_oxide) to provide
|
||||
// pdfplumber-style character extraction, page rendering, and RAGFlow-compatible
|
||||
// utility functions. It is maintained as a standalone adapter layer so that
|
||||
// the pdfplumber compatibility code can be modified independently of the
|
||||
// pdf_oxide backend.
|
||||
//
|
||||
// Originally derived from github.com/yingfeng/pdfplumber-go.
|
||||
|
||||
package pdfoxide
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"image"
|
||||
"image/color"
|
||||
"math"
|
||||
"sort"
|
||||
"strings"
|
||||
|
||||
pdfoxide "github.com/yfedoseev/pdf_oxide/go"
|
||||
)
|
||||
|
||||
// ── pdf_oxide-based types ──────────────────────────────────────────
|
||||
|
||||
// Char represents a single character extracted from a PDF page,
|
||||
// matching pdfplumber's char dict format.
|
||||
type char struct {
|
||||
Text string `json:"text"`
|
||||
Fontname string `json:"fontname"`
|
||||
Size float64 `json:"size"`
|
||||
X0 float64 `json:"x0"`
|
||||
X1 float64 `json:"x1"`
|
||||
Top float64 `json:"top"`
|
||||
Bottom float64 `json:"bottom"`
|
||||
Width float64 `json:"width"`
|
||||
Height float64 `json:"height"`
|
||||
Doctop float64 `json:"doctop"`
|
||||
Matrix [6]float64 `json:"matrix"`
|
||||
Upright bool `json:"upright"`
|
||||
StrokingColor string `json:"stroking_color"`
|
||||
NonStrokingColor string `json:"non_stroking_color"`
|
||||
Ncs string `json:"ncs"`
|
||||
Adv float64 `json:"adv"`
|
||||
PageNumber int `json:"page_number"`
|
||||
}
|
||||
|
||||
// Document wraps pdf_oxide's PdfDocument with pdf_oxide-based methods.
|
||||
type Document struct {
|
||||
Inner *pdfoxide.PdfDocument
|
||||
}
|
||||
|
||||
// RenderResult holds the result of rendering a PDF page.
|
||||
type RenderResult struct {
|
||||
Data []byte
|
||||
Width int
|
||||
Height int
|
||||
Channels int
|
||||
}
|
||||
|
||||
// ── Document methods ─────────────────────────────────────────────────────
|
||||
|
||||
// Open opens a PDF file from a file path.
|
||||
func Open(path string) (*Document, error) {
|
||||
doc, err := pdfoxide.Open(path)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("pdfplumber: open %s: %w", path, err)
|
||||
}
|
||||
return &Document{Inner: doc}, nil
|
||||
}
|
||||
|
||||
// OpenBytes opens a PDF from raw bytes in memory.
|
||||
func OpenBytes(data []byte) (*Document, error) {
|
||||
doc, err := pdfoxide.OpenFromBytes(data)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("pdfplumber: open from bytes: %w", err)
|
||||
}
|
||||
return &Document{Inner: doc}, nil
|
||||
}
|
||||
|
||||
// Close releases the document handle.
|
||||
func (d *Document) Close() {
|
||||
if d.Inner != nil {
|
||||
d.Inner.Close()
|
||||
d.Inner = nil
|
||||
}
|
||||
}
|
||||
|
||||
// PageCount returns the number of pages in the document.
|
||||
func (d *Document) PageCount() (int, error) {
|
||||
if d.Inner == nil {
|
||||
return 0, fmt.Errorf("pdfplumber: document is closed")
|
||||
}
|
||||
return d.Inner.PageCount()
|
||||
}
|
||||
|
||||
// PageSize returns the pre-rotation page dimensions from pdf_oxide in PDF
|
||||
// points (1/72 inch). For a page with /Rotate 90, this returns the original
|
||||
// (unrotated) MediaBox dimensions — not the post-rotation visual size.
|
||||
// Compare with pdfium.PageSize to detect rotation.
|
||||
func (d *Document) PageSize(pageIdx int) (width, height float64, err error) {
|
||||
if d.Inner == nil {
|
||||
return 0, 0, fmt.Errorf("pdfplumber: document is closed")
|
||||
}
|
||||
info, err := d.Inner.PageInfo(pageIdx)
|
||||
if err != nil {
|
||||
return 0, 0, err
|
||||
}
|
||||
return float64(info.Width), float64(info.Height), nil
|
||||
}
|
||||
|
||||
// GetPageChars returns all characters on a page (0-indexed).
|
||||
func (d *Document) GetPageChars(pageIdx int) ([]char, error) {
|
||||
if d.Inner == nil {
|
||||
return nil, fmt.Errorf("pdfplumber: document is closed")
|
||||
}
|
||||
n, err := d.PageCount()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("pdfplumber: page count: %w", err)
|
||||
}
|
||||
if pageIdx < 0 || pageIdx >= n {
|
||||
return nil, fmt.Errorf("pdfplumber: page index %d out of range (pages: %d)", pageIdx, n)
|
||||
}
|
||||
raw, err := d.Inner.ExtractChars(pageIdx)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("pdfplumber: extract chars page %d: %w", pageIdx, err)
|
||||
}
|
||||
|
||||
// pdf_oxide returns Y in PDF coordinate system (origin bottom-left, Y↑).
|
||||
// Python pdfplumber internally flips to top-left origin (Y↓), matching
|
||||
// "top" = distance from page top. We replicate that here so that
|
||||
// sortByPageThenY produces top-to-bottom reading order.
|
||||
info, err := d.Inner.PageInfo(pageIdx)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("pdfplumber: page info %d: %w", pageIdx, err)
|
||||
}
|
||||
// Page height: use CropBox (matches pdfplumber's page.height).
|
||||
// pdf_oxide bbox: [baseline, baseline + font_size] — no descent
|
||||
// below baseline. pdfplumber bbox: [baseline - descent, baseline
|
||||
// + ascent]. Both have height = font_size, but the Y origin
|
||||
// differs. We keep the raw pdf_oxide bbox and sort by Bottom
|
||||
// (= pageHeight - c.Y) in groupCharsToLines so all chars on the
|
||||
// same baseline share the same sort key regardless of font size.
|
||||
pageHeight := float64(info.CropBox.Height)
|
||||
if pageHeight <= 0 {
|
||||
pageHeight = float64(info.Height) // fallback
|
||||
}
|
||||
|
||||
chars := make([]char, len(raw))
|
||||
for i, c := range raw {
|
||||
x0 := float64(c.X)
|
||||
fs := float64(c.FontSize)
|
||||
top := pageHeight - float64(c.Y) - float64(c.Height)
|
||||
w := float64(c.Width)
|
||||
h := float64(c.Height)
|
||||
chars[i] = char{
|
||||
Text: string(c.Char),
|
||||
Fontname: c.FontName,
|
||||
Size: fs,
|
||||
X0: x0,
|
||||
X1: x0 + w,
|
||||
Top: top,
|
||||
Bottom: top + h,
|
||||
Width: w,
|
||||
Height: h,
|
||||
Doctop: top,
|
||||
Matrix: [6]float64{fs, 0, 0, fs, x0, top},
|
||||
Upright: true,
|
||||
StrokingColor: "",
|
||||
NonStrokingColor: "",
|
||||
Ncs: "",
|
||||
Adv: fs * 0.5,
|
||||
PageNumber: pageIdx + 1,
|
||||
}
|
||||
}
|
||||
return chars, nil
|
||||
}
|
||||
|
||||
// GetDedupePageChars returns deduplicated characters on a page (0-indexed).
|
||||
// tolerance controls how close two chars must be to be considered duplicates.
|
||||
func (d *Document) GetDedupePageChars(pageIdx int, tolerance float64) ([]char, error) {
|
||||
chars, err := d.GetPageChars(pageIdx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return dedupeChars(chars, tolerance), nil
|
||||
}
|
||||
|
||||
// GetPageText extracts plain text from a page (0-indexed), in reading order (top → x0).
|
||||
func (d *Document) GetPageText(pageIdx int) (string, error) {
|
||||
chars, err := d.GetPageChars(pageIdx)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
if len(chars) == 0 {
|
||||
return "", nil
|
||||
}
|
||||
sorted := make([]char, len(chars))
|
||||
copy(sorted, chars)
|
||||
sort.Slice(sorted, func(i, j int) bool {
|
||||
if sorted[i].Top != sorted[j].Top {
|
||||
return sorted[i].Top < sorted[j].Top
|
||||
}
|
||||
return sorted[i].X0 < sorted[j].X0
|
||||
})
|
||||
var b strings.Builder
|
||||
for i, c := range sorted {
|
||||
b.WriteString(c.Text)
|
||||
if i+1 < len(sorted) {
|
||||
next := sorted[i+1]
|
||||
if math.Abs(next.Top-c.Top) < 0.5 {
|
||||
gap := next.X0 - c.X1
|
||||
if gap > c.Width*0.3 {
|
||||
b.WriteByte(' ')
|
||||
}
|
||||
} else {
|
||||
b.WriteByte('\n')
|
||||
}
|
||||
}
|
||||
}
|
||||
return b.String(), nil
|
||||
}
|
||||
|
||||
// ── Deduplication ────────────────────────────────────────────────────────
|
||||
func dedupeChars(chars []char, tolerance float64) []char {
|
||||
if len(chars) == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
// Sort by X0 so we only need a sliding window of nearby chars.
|
||||
sorted := make([]char, len(chars))
|
||||
copy(sorted, chars)
|
||||
sort.Slice(sorted, func(i, j int) bool { return sorted[i].X0 < sorted[j].X0 })
|
||||
|
||||
result := make([]char, 0, len(sorted))
|
||||
// maxCharWidth is the maximum X-span we've seen; chars further apart
|
||||
// than this cannot overlap. Update as we go.
|
||||
maxCharWidth := 0.0
|
||||
|
||||
for _, ch := range sorted {
|
||||
cw := ch.X1 - ch.X0
|
||||
if cw > maxCharWidth {
|
||||
maxCharWidth = cw
|
||||
}
|
||||
|
||||
dup := false
|
||||
// Only scan backwards within maxCharWidth; chars further away
|
||||
// cannot possibly overlap.
|
||||
for i := len(result) - 1; i >= 0; i-- {
|
||||
existing := &result[i]
|
||||
if ch.X0-existing.X1 > maxCharWidth {
|
||||
break // too far left to overlap
|
||||
}
|
||||
ox := math.Max(0, math.Min(ch.X1, existing.X1)-math.Max(ch.X0, existing.X0))
|
||||
oy := math.Max(0, math.Min(ch.Bottom, existing.Bottom)-math.Max(ch.Top, existing.Top))
|
||||
oa := ox * oy
|
||||
if oa <= 0 {
|
||||
continue
|
||||
}
|
||||
ca := cw * (ch.Bottom - ch.Top)
|
||||
ea := (existing.X1 - existing.X0) * (existing.Bottom - existing.Top)
|
||||
maxA := math.Max(ca, ea)
|
||||
ratio := oa / maxA
|
||||
sameFont := ch.Fontname == existing.Fontname
|
||||
sameSize := math.Abs(ch.Size-existing.Size) <= tolerance
|
||||
if ratio > 0.5 && sameFont && sameSize {
|
||||
dup = true
|
||||
break
|
||||
}
|
||||
}
|
||||
if !dup {
|
||||
result = append(result, ch)
|
||||
}
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
// ── Rendering ────────────────────────────────────────────────────────────
|
||||
|
||||
// RenderPage renders a PDF page to RGBA pixels using pdf_oxide.
|
||||
// pdfData must be the raw PDF bytes, pageIdx is 0-based, dpi is the resolution.
|
||||
// Prefer Document.RenderPage when you already have an open Document to avoid re-parsing.
|
||||
func RenderPage(pdfData []byte, pageIdx int, dpi float64) (*RenderResult, error) {
|
||||
if len(pdfData) == 0 {
|
||||
return nil, fmt.Errorf("pdfplumber: empty PDF data for rendering")
|
||||
}
|
||||
doc, err := pdfoxide.OpenFromBytes(pdfData)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("pdfplumber: open for render: %w", err)
|
||||
}
|
||||
defer doc.Close()
|
||||
|
||||
return renderPageFromDoc(doc, pageIdx, dpi)
|
||||
}
|
||||
|
||||
// RenderPage renders a single page using the already-open document.
|
||||
// Unlike the standalone RenderPage function, this reuses the open handle
|
||||
// and does not re-parse the PDF on every call.
|
||||
func (d *Document) RenderPage(pageIdx int, dpi float64) (*RenderResult, error) {
|
||||
if d.Inner == nil {
|
||||
return nil, fmt.Errorf("pdfplumber: document is closed")
|
||||
}
|
||||
return renderPageFromDoc(d.Inner, pageIdx, dpi)
|
||||
}
|
||||
|
||||
// renderPageFromDoc is the shared rendering core: calls RenderPageRaw and
|
||||
// converts premultiplied alpha to straight alpha.
|
||||
func renderPageFromDoc(doc *pdfoxide.PdfDocument, pageIdx int, dpi float64) (*RenderResult, error) {
|
||||
pixmap, err := doc.RenderPageRaw(pageIdx, int(math.Round(dpi)))
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("pdfplumber: render page %d: %w", pageIdx, err)
|
||||
}
|
||||
|
||||
data := make([]byte, len(pixmap.Data))
|
||||
for i := 0; i < len(pixmap.Data); i += 4 {
|
||||
a := pixmap.Data[i+3]
|
||||
if a == 0 {
|
||||
data[i], data[i+1], data[i+2], data[i+3] = 0, 0, 0, 0
|
||||
} else {
|
||||
data[i] = uint8(math.Min(255, float64(pixmap.Data[i])*255/float64(a)))
|
||||
data[i+1] = uint8(math.Min(255, float64(pixmap.Data[i+1])*255/float64(a)))
|
||||
data[i+2] = uint8(math.Min(255, float64(pixmap.Data[i+2])*255/float64(a)))
|
||||
data[i+3] = a
|
||||
}
|
||||
}
|
||||
return &RenderResult{Data: data, Width: pixmap.Width, Height: pixmap.Height, Channels: 4}, nil
|
||||
}
|
||||
|
||||
// InitRenderer is a no-op for pdf_oxide (renderer is initialized internally).
|
||||
func InitRenderer(path string) error { return nil }
|
||||
|
||||
// ToImage converts a RenderResult to an image.RGBA.
|
||||
func (r *RenderResult) ToImage() *image.RGBA {
|
||||
img := image.NewRGBA(image.Rect(0, 0, r.Width, r.Height))
|
||||
copy(img.Pix, r.Data)
|
||||
return img
|
||||
}
|
||||
|
||||
// ColorModel implements image.Image.
|
||||
func (r *RenderResult) ColorModel() color.Model { return color.RGBAModel }
|
||||
|
||||
// Bounds implements image.Image.
|
||||
func (r *RenderResult) Bounds() image.Rectangle { return image.Rect(0, 0, r.Width, r.Height) }
|
||||
|
||||
// At implements image.Image.
|
||||
func (r *RenderResult) At(x, y int) color.Color {
|
||||
if x < 0 || x >= r.Width || y < 0 || y >= r.Height {
|
||||
return color.RGBA{}
|
||||
}
|
||||
idx := (y*r.Width + x) * r.Channels
|
||||
if r.Channels >= 4 {
|
||||
return color.RGBA{R: r.Data[idx], G: r.Data[idx+1], B: r.Data[idx+2], A: r.Data[idx+3]}
|
||||
}
|
||||
return color.RGBA{R: r.Data[idx], G: r.Data[idx+1], B: r.Data[idx+2], A: 255}
|
||||
}
|
||||
|
||||
// ── Utility ──────────────────────────────────────────────────────────────
|
||||
|
||||
// TotalPageNumber opens a PDF and returns the page count.
|
||||
func TotalPageNumber(path string, data []byte) (int, error) {
|
||||
var doc *Document
|
||||
var err error
|
||||
if data != nil {
|
||||
doc, err = OpenBytes(data)
|
||||
} else {
|
||||
doc, err = Open(path)
|
||||
}
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
defer doc.Close()
|
||||
return doc.PageCount()
|
||||
}
|
||||
758
internal/deepdoc/parser/pdf/pdfoxide/pdf_oxide_adapter_test.go
Normal file
758
internal/deepdoc/parser/pdf/pdfoxide/pdf_oxide_adapter_test.go
Normal file
@@ -0,0 +1,758 @@
|
||||
//go:build cgo
|
||||
|
||||
package pdfoxide
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"math"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
var fixtureDir = filepath.Join("..", "parser", "testdata", "pdfs")
|
||||
|
||||
// ── Document opening ─────────────────────────────────────────────────────
|
||||
|
||||
func TestOpen(t *testing.T) {
|
||||
path := filepath.Join(fixtureDir, "01_english_simple.pdf")
|
||||
doc, err := Open(path)
|
||||
if err != nil {
|
||||
t.Fatalf("Open: %v", err)
|
||||
}
|
||||
defer doc.Close()
|
||||
if pc, _ := doc.PageCount(); pc != 1 {
|
||||
t.Fatalf("expected 1 page, got %d", pc)
|
||||
}
|
||||
}
|
||||
|
||||
func TestOpenBytes(t *testing.T) {
|
||||
data, err := os.ReadFile(filepath.Join(fixtureDir, "01_english_simple.pdf"))
|
||||
if err != nil {
|
||||
t.Fatalf("ReadFile: %v", err)
|
||||
}
|
||||
doc, err := OpenBytes(data)
|
||||
if err != nil {
|
||||
t.Fatalf("OpenBytes: %v", err)
|
||||
}
|
||||
defer doc.Close()
|
||||
if pc, _ := doc.PageCount(); pc != 1 {
|
||||
t.Fatalf("expected 1 page, got %d", pc)
|
||||
}
|
||||
}
|
||||
|
||||
func TestOpenBytes_Empty(t *testing.T) {
|
||||
_, err := OpenBytes(nil)
|
||||
if err == nil {
|
||||
t.Error("expected error for nil data")
|
||||
}
|
||||
_, err = OpenBytes([]byte{})
|
||||
if err == nil {
|
||||
t.Error("expected error for empty data")
|
||||
}
|
||||
}
|
||||
|
||||
func TestOpen_InvalidPath(t *testing.T) {
|
||||
_, err := Open(filepath.Join(fixtureDir, "nonexistent.pdf"))
|
||||
if err == nil {
|
||||
t.Error("expected error for nonexistent file")
|
||||
}
|
||||
}
|
||||
|
||||
// ── PageCount ────────────────────────────────────────────────────────────
|
||||
|
||||
func TestPageCount(t *testing.T) {
|
||||
doc := openFixture(t, "01_english_simple.pdf")
|
||||
defer doc.Close()
|
||||
pc, err := doc.PageCount()
|
||||
if err != nil {
|
||||
t.Fatalf("PageCount: %v", err)
|
||||
}
|
||||
if pc != 1 {
|
||||
t.Errorf("expected 1 page, got %d", pc)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPageCount_MultiPage(t *testing.T) {
|
||||
doc := openFixture(t, "03_multipage.pdf")
|
||||
defer doc.Close()
|
||||
pc, err := doc.PageCount()
|
||||
if err != nil {
|
||||
t.Fatalf("PageCount: %v", err)
|
||||
}
|
||||
if pc < 2 {
|
||||
t.Errorf("expected >= 2 pages, got %d", pc)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPageCount_AfterClose(t *testing.T) {
|
||||
doc := openFixture(t, "01_english_simple.pdf")
|
||||
doc.Close()
|
||||
pc, err := doc.PageCount()
|
||||
if err == nil {
|
||||
t.Error("expected error after close")
|
||||
}
|
||||
if pc != 0 {
|
||||
t.Errorf("expected 0 after close, got %d", pc)
|
||||
}
|
||||
}
|
||||
|
||||
// ── Close ────────────────────────────────────────────────────────────────
|
||||
|
||||
func TestClose_DoubleClose(t *testing.T) {
|
||||
doc := openFixture(t, "01_english_simple.pdf")
|
||||
doc.Close()
|
||||
// Second Close should not panic
|
||||
doc.Close()
|
||||
}
|
||||
|
||||
// ── GetPageChars ─────────────────────────────────────────────────────────
|
||||
|
||||
func TestGetPageChars(t *testing.T) {
|
||||
doc := openFixture(t, "01_english_simple.pdf")
|
||||
defer doc.Close()
|
||||
|
||||
chars, err := doc.GetPageChars(0)
|
||||
if err != nil {
|
||||
t.Fatalf("GetPageChars: %v", err)
|
||||
}
|
||||
if len(chars) == 0 {
|
||||
t.Fatal("expected non-empty chars")
|
||||
}
|
||||
|
||||
c := chars[0]
|
||||
if c.Text == "" {
|
||||
t.Error("expected non-empty text")
|
||||
}
|
||||
if c.Fontname == "" {
|
||||
t.Error("expected non-empty fontname")
|
||||
}
|
||||
if c.X0 >= c.X1 {
|
||||
t.Errorf("expected x0 < x1, got %f >= %f", c.X0, c.X1)
|
||||
}
|
||||
if c.Top >= c.Bottom {
|
||||
t.Errorf("expected top < bottom, got %f >= %f", c.Top, c.Bottom)
|
||||
}
|
||||
if c.PageNumber < 1 {
|
||||
t.Errorf("expected page_number >= 1, got %d", c.PageNumber)
|
||||
}
|
||||
if c.Size <= 0 {
|
||||
t.Errorf("expected positive font size, got %f", c.Size)
|
||||
}
|
||||
}
|
||||
|
||||
func TestGetPageChars_InvalidPage(t *testing.T) {
|
||||
doc := openFixture(t, "01_english_simple.pdf")
|
||||
defer doc.Close()
|
||||
|
||||
// Negative page
|
||||
_, err := doc.GetPageChars(-1)
|
||||
if err == nil {
|
||||
t.Error("expected error for negative page")
|
||||
}
|
||||
|
||||
// Out of range
|
||||
_, err = doc.GetPageChars(999)
|
||||
if err == nil {
|
||||
t.Error("expected error for out-of-range page")
|
||||
}
|
||||
}
|
||||
|
||||
func TestGetPageChars_AfterClose(t *testing.T) {
|
||||
doc := openFixture(t, "01_english_simple.pdf")
|
||||
doc.Close()
|
||||
|
||||
_, err := doc.GetPageChars(0)
|
||||
if err == nil {
|
||||
t.Error("expected error after close")
|
||||
}
|
||||
}
|
||||
|
||||
// ── GetDedupePageChars ───────────────────────────────────────────────────
|
||||
|
||||
func TestGetDedupePageChars(t *testing.T) {
|
||||
doc := openFixture(t, "01_english_simple.pdf")
|
||||
defer doc.Close()
|
||||
|
||||
raw, err := doc.GetPageChars(0)
|
||||
if err != nil {
|
||||
t.Fatalf("GetPageChars: %v", err)
|
||||
}
|
||||
|
||||
deduped, err := doc.GetDedupePageChars(0, 1.0)
|
||||
if err != nil {
|
||||
t.Fatalf("GetDedupePageChars: %v", err)
|
||||
}
|
||||
if len(deduped) > len(raw) {
|
||||
t.Errorf("expected deduped <= raw (%d > %d)", len(deduped), len(raw))
|
||||
}
|
||||
if len(deduped) == 0 && len(raw) > 0 {
|
||||
t.Error("expected non-empty deduped when raw is non-empty")
|
||||
}
|
||||
}
|
||||
|
||||
func TestGetDedupePageChars_Tolerance(t *testing.T) {
|
||||
doc := openFixture(t, "01_english_simple.pdf")
|
||||
defer doc.Close()
|
||||
|
||||
// tolerance=0 should preserve all (no dedup)
|
||||
t0, _ := doc.GetDedupePageChars(0, 0)
|
||||
// high tolerance may merge more
|
||||
tHi, _ := doc.GetDedupePageChars(0, 100.0)
|
||||
|
||||
raw, _ := doc.GetPageChars(0)
|
||||
if len(t0) != len(raw) {
|
||||
t.Logf("tolerance=0: %d chars (raw=%d) — some exact overlaps removed", len(t0), len(raw))
|
||||
}
|
||||
if len(tHi) > len(t0) {
|
||||
t.Errorf("high tolerance (%d) should not produce more chars than zero tolerance (%d)", len(tHi), len(t0))
|
||||
}
|
||||
}
|
||||
|
||||
// ── GetPageText ──────────────────────────────────────────────────────────
|
||||
|
||||
func TestGetPageText(t *testing.T) {
|
||||
doc := openFixture(t, "01_english_simple.pdf")
|
||||
defer doc.Close()
|
||||
|
||||
text, err := doc.GetPageText(0)
|
||||
if err != nil {
|
||||
t.Fatalf("GetPageText: %v", err)
|
||||
}
|
||||
if len(strings.TrimSpace(text)) == 0 {
|
||||
t.Error("expected non-empty text")
|
||||
}
|
||||
// This fixture is multi-line — verify newlines are present.
|
||||
if !strings.Contains(text, "\n") {
|
||||
t.Error("expected multi-line text to contain newlines")
|
||||
}
|
||||
// Verify no consecutive newlines (no blank lines from gaps).
|
||||
if strings.Contains(text, "\n\n") {
|
||||
t.Log("text contains blank lines (may be expected for this layout)")
|
||||
}
|
||||
}
|
||||
|
||||
func TestGetPageTextMultiLine(t *testing.T) {
|
||||
doc := openFixture(t, "03_multipage.pdf")
|
||||
defer doc.Close()
|
||||
|
||||
hasNewline := false
|
||||
pc, _ := doc.PageCount()
|
||||
for i := 0; i < pc; i++ {
|
||||
text, err := doc.GetPageText(i)
|
||||
if err != nil {
|
||||
t.Fatalf("GetPageText(%d): %v", i, err)
|
||||
}
|
||||
if len(text) == 0 {
|
||||
t.Errorf("page %d: expected non-empty text", i)
|
||||
}
|
||||
if strings.Contains(text, "\n") {
|
||||
hasNewline = true
|
||||
}
|
||||
}
|
||||
if !hasNewline {
|
||||
t.Error("expected at least one page to have multi-line text")
|
||||
}
|
||||
}
|
||||
|
||||
// ── RenderPage ───────────────────────────────────────────────────────────
|
||||
|
||||
func TestRenderPage(t *testing.T) {
|
||||
data, err := os.ReadFile(filepath.Join(fixtureDir, "01_english_simple.pdf"))
|
||||
if err != nil {
|
||||
t.Fatalf("ReadFile: %v", err)
|
||||
}
|
||||
res, err := RenderPage(data, 0, 72.0)
|
||||
if err != nil {
|
||||
t.Fatalf("RenderPage: %v", err)
|
||||
}
|
||||
if res.Width <= 0 || res.Height <= 0 {
|
||||
t.Errorf("invalid dimensions: %dx%d", res.Width, res.Height)
|
||||
}
|
||||
if res.Channels != 4 {
|
||||
t.Errorf("expected 4 channels, got %d", res.Channels)
|
||||
}
|
||||
expectedLen := res.Width * res.Height * res.Channels
|
||||
if len(res.Data) != expectedLen {
|
||||
t.Errorf("data length %d != %d", len(res.Data), expectedLen)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRenderPage_EmptyData(t *testing.T) {
|
||||
_, err := RenderPage(nil, 0, 72.0)
|
||||
if err == nil {
|
||||
t.Error("expected error for nil data")
|
||||
}
|
||||
_, err = RenderPage([]byte{}, 0, 72.0)
|
||||
if err == nil {
|
||||
t.Error("expected error for empty data")
|
||||
}
|
||||
}
|
||||
|
||||
func TestRenderPage_MultiPage(t *testing.T) {
|
||||
data, err := os.ReadFile(filepath.Join(fixtureDir, "03_multipage.pdf"))
|
||||
if err != nil {
|
||||
t.Fatalf("ReadFile: %v", err)
|
||||
}
|
||||
for i := 0; i < 2; i++ {
|
||||
res, err := RenderPage(data, i, 72.0)
|
||||
if err != nil {
|
||||
t.Fatalf("RenderPage page %d: %v", i, err)
|
||||
}
|
||||
if res.Width <= 0 || res.Height <= 0 {
|
||||
t.Errorf("page %d: invalid dimensions", i)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ── RenderResult methods ─────────────────────────────────────────────────
|
||||
|
||||
func TestRenderResult_ToImage(t *testing.T) {
|
||||
data, _ := os.ReadFile(filepath.Join(fixtureDir, "01_english_simple.pdf"))
|
||||
res, err := RenderPage(data, 0, 72.0)
|
||||
if err != nil {
|
||||
t.Fatalf("RenderPage: %v", err)
|
||||
}
|
||||
img := res.ToImage()
|
||||
if img.Bounds().Dx() != res.Width || img.Bounds().Dy() != res.Height {
|
||||
t.Errorf("image size %v != %dx%d", img.Bounds(), res.Width, res.Height)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRenderResult_At(t *testing.T) {
|
||||
data, _ := os.ReadFile(filepath.Join(fixtureDir, "01_english_simple.pdf"))
|
||||
res, err := RenderPage(data, 0, 72.0)
|
||||
if err != nil {
|
||||
t.Fatalf("RenderPage: %v", err)
|
||||
}
|
||||
// In-bounds: should return a non-nil color
|
||||
c := res.At(0, 0)
|
||||
if c == nil {
|
||||
t.Error("At(0,0) returned nil")
|
||||
}
|
||||
// Out-of-bounds: should not panic and return zero color
|
||||
out := res.At(-1, 0)
|
||||
if out == nil {
|
||||
t.Error("At(-1,0) returned nil")
|
||||
}
|
||||
out2 := res.At(res.Width, res.Height)
|
||||
if out2 == nil {
|
||||
t.Error("At(width,height) returned nil")
|
||||
}
|
||||
}
|
||||
|
||||
func TestRenderResult_Bounds(t *testing.T) {
|
||||
data, _ := os.ReadFile(filepath.Join(fixtureDir, "01_english_simple.pdf"))
|
||||
res, err := RenderPage(data, 0, 72.0)
|
||||
if err != nil {
|
||||
t.Fatalf("RenderPage: %v", err)
|
||||
}
|
||||
b := res.Bounds()
|
||||
if b.Min.X != 0 || b.Min.Y != 0 {
|
||||
t.Errorf("expected origin at (0,0), got (%d,%d)", b.Min.X, b.Min.Y)
|
||||
}
|
||||
if b.Dx() != res.Width || b.Dy() != res.Height {
|
||||
t.Errorf("bounds %v != %dx%d", b, res.Width, res.Height)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRenderResult_ColorModel(t *testing.T) {
|
||||
data, _ := os.ReadFile(filepath.Join(fixtureDir, "01_english_simple.pdf"))
|
||||
res, _ := RenderPage(data, 0, 72.0)
|
||||
// ColorModel should return a non-nil model
|
||||
if res.ColorModel() == nil {
|
||||
t.Error("ColorModel returned nil")
|
||||
}
|
||||
}
|
||||
|
||||
// ── TotalPageNumber ──────────────────────────────────────────────────────
|
||||
|
||||
func TestTotalPageNumber(t *testing.T) {
|
||||
data, err := os.ReadFile(filepath.Join(fixtureDir, "03_multipage.pdf"))
|
||||
if err != nil {
|
||||
t.Fatalf("ReadFile: %v", err)
|
||||
}
|
||||
n, err := TotalPageNumber("", data)
|
||||
if err != nil {
|
||||
t.Fatalf("TotalPageNumber: %v", err)
|
||||
}
|
||||
if n < 2 {
|
||||
t.Errorf("expected >= 2 pages, got %d", n)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTotalPageNumber_File(t *testing.T) {
|
||||
path := filepath.Join(fixtureDir, "01_english_simple.pdf")
|
||||
n, err := TotalPageNumber(path, nil)
|
||||
if err != nil {
|
||||
t.Fatalf("TotalPageNumber: %v", err)
|
||||
}
|
||||
if n != 1 {
|
||||
t.Errorf("expected 1 page, got %d", n)
|
||||
}
|
||||
}
|
||||
|
||||
// ── InitRenderer ─────────────────────────────────────────────────────────
|
||||
|
||||
func TestInitRenderer(t *testing.T) {
|
||||
if err := InitRenderer(""); err != nil {
|
||||
t.Errorf("InitRenderer should be no-op, got: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// ── Multiple PDFs smoke test ─────────────────────────────────────────────
|
||||
|
||||
func TestMultiplePDFs(t *testing.T) {
|
||||
entries, err := os.ReadDir(fixtureDir)
|
||||
if err != nil {
|
||||
t.Fatalf("ReadDir: %v", err)
|
||||
}
|
||||
count := 0
|
||||
for _, e := range entries {
|
||||
if e.IsDir() || filepath.Ext(e.Name()) != ".pdf" {
|
||||
continue
|
||||
}
|
||||
name := e.Name()
|
||||
t.Run(name, func(t *testing.T) {
|
||||
doc, err := Open(filepath.Join(fixtureDir, name))
|
||||
if err != nil {
|
||||
t.Fatalf("Open: %v", err)
|
||||
}
|
||||
defer doc.Close()
|
||||
|
||||
pc, _ := doc.PageCount()
|
||||
if pc == 0 {
|
||||
t.Error("PageCount returned 0")
|
||||
}
|
||||
for i := 0; i < pc; i++ {
|
||||
chars, err := doc.GetPageChars(i)
|
||||
if err != nil {
|
||||
t.Errorf("GetPageChars(%d): %v", i, err)
|
||||
continue
|
||||
}
|
||||
if len(chars) == 0 {
|
||||
t.Logf("page %d: 0 chars (may be image-only or sparse)", i)
|
||||
}
|
||||
}
|
||||
})
|
||||
count++
|
||||
}
|
||||
if count == 0 {
|
||||
t.Error("no PDFs found in fixture directory")
|
||||
}
|
||||
t.Logf("Tested %d PDFs", count)
|
||||
}
|
||||
|
||||
// ── Engine-level tests ───────────────────────────────────────────────────
|
||||
|
||||
func TestPDFPlumber_RenderPage(t *testing.T) {
|
||||
data, err := os.ReadFile(filepath.Join(fixtureDir, "01_english_simple.pdf"))
|
||||
if err != nil {
|
||||
t.Fatalf("ReadFile: %v", err)
|
||||
}
|
||||
eng, err := NewEngine(data)
|
||||
if err != nil {
|
||||
t.Fatalf("NewEngine: %v", err)
|
||||
}
|
||||
defer eng.Close()
|
||||
|
||||
img, err := eng.RenderPage(0, 72.0)
|
||||
if err != nil {
|
||||
t.Fatalf("RenderPage: %v", err)
|
||||
}
|
||||
if len(img) == 0 {
|
||||
t.Error("RenderPage returned empty image data")
|
||||
}
|
||||
}
|
||||
|
||||
func TestPDFPlumber_MultiPage(t *testing.T) {
|
||||
data, err := os.ReadFile(filepath.Join(fixtureDir, "03_multipage.pdf"))
|
||||
if err != nil {
|
||||
t.Fatalf("ReadFile: %v", err)
|
||||
}
|
||||
eng, err := NewEngine(data)
|
||||
if err != nil {
|
||||
t.Fatalf("NewEngine: %v", err)
|
||||
}
|
||||
defer eng.Close()
|
||||
|
||||
pc, _ := eng.PageCount()
|
||||
if pc < 2 {
|
||||
t.Fatalf("expected >= 2 pages, got %d", pc)
|
||||
}
|
||||
for i := 0; i < pc; i++ {
|
||||
chars, err := eng.ExtractChars(i)
|
||||
if err != nil {
|
||||
t.Errorf("ExtractChars(%d): %v", i, err)
|
||||
}
|
||||
if len(chars) == 0 {
|
||||
t.Logf("page %d: 0 chars extracted", i)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ── Char extraction comparison with Python pdfplumber ────────────────────
|
||||
|
||||
// pyChar mirrors the per-character dict that Python pdfplumber writes into
|
||||
// snapshots (stages.__images__.page_chars).
|
||||
type pyChar struct {
|
||||
Text string `json:"text"`
|
||||
FontName string `json:"fontname"`
|
||||
Size float64 `json:"size"`
|
||||
X0 float64 `json:"x0"`
|
||||
X1 float64 `json:"x1"`
|
||||
Top float64 `json:"top"`
|
||||
Bottom float64 `json:"bottom"`
|
||||
PageNumber int `json:"page_number"`
|
||||
}
|
||||
|
||||
// TestCharExtraction_CompareWithPython uses Go pdf_oxide to extract chars from
|
||||
// the 16 test PDFs and compares against Python pdfplumber golden data in
|
||||
// testdata/snapshots/*.json.
|
||||
//
|
||||
// pdf_oxide and pdfplumber are different engines with different internal
|
||||
// ordering and coordinate origins, so we compare:
|
||||
// - char count per page (should match closely)
|
||||
// - text content (as sorted sets, ignoring order differences)
|
||||
// - coordinate ranges (min/max, since absolute positions differ by engine)
|
||||
func TestCharExtraction_CompareWithPython(t *testing.T) {
|
||||
snapDir := filepath.Join("..", "parser", "testdata", "snapshots")
|
||||
|
||||
entries, err := os.ReadDir(snapDir)
|
||||
if err != nil {
|
||||
t.Fatalf("ReadDir: %v", err)
|
||||
}
|
||||
|
||||
totalPDFs := 0
|
||||
for _, e := range entries {
|
||||
if !strings.HasSuffix(e.Name(), ".json") {
|
||||
continue
|
||||
}
|
||||
name := strings.TrimSuffix(e.Name(), ".json")
|
||||
pdfPath := filepath.Join(fixtureDir, name+".pdf")
|
||||
if _, err := os.Stat(pdfPath); err != nil {
|
||||
t.Logf("SKIP %s: PDF not found", name)
|
||||
continue
|
||||
}
|
||||
|
||||
t.Run(name, func(t *testing.T) {
|
||||
pyChars := loadPyPageChars(t, filepath.Join(snapDir, e.Name()))
|
||||
|
||||
pdfData, err := os.ReadFile(pdfPath)
|
||||
if err != nil {
|
||||
t.Fatalf("ReadFile: %v", err)
|
||||
}
|
||||
eng, err := NewEngine(pdfData)
|
||||
if err != nil {
|
||||
t.Fatalf("NewEngine: %v", err)
|
||||
}
|
||||
defer eng.Close()
|
||||
|
||||
goPageCount, _ := eng.PageCount()
|
||||
pyPageCount := len(pyChars)
|
||||
|
||||
if goPageCount != pyPageCount {
|
||||
t.Logf("page count: Go=%d Python=%d", goPageCount, pyPageCount)
|
||||
}
|
||||
|
||||
totalPy, totalGo := 0, 0
|
||||
textInBoth, textOnlyPy, textOnlyGo := 0, 0, 0
|
||||
maxPages := goPageCount
|
||||
if pyPageCount > maxPages {
|
||||
maxPages = pyPageCount
|
||||
}
|
||||
|
||||
for pg := 0; pg < maxPages; pg++ {
|
||||
var pyPage []pyChar
|
||||
if pg < len(pyChars) {
|
||||
pyPage = pyChars[pg]
|
||||
}
|
||||
goPage, err := eng.ExtractChars(pg)
|
||||
if err != nil {
|
||||
t.Logf("page %d: Go ExtractChars error: %v", pg, err)
|
||||
continue
|
||||
}
|
||||
|
||||
totalPy += len(pyPage)
|
||||
totalGo += len(goPage)
|
||||
|
||||
// Build text sets (sorted by position order differs between engines)
|
||||
pyTexts := make(map[string]int)
|
||||
for _, c := range pyPage {
|
||||
pyTexts[c.Text]++
|
||||
}
|
||||
goTexts := make(map[string]int)
|
||||
for _, c := range goPage {
|
||||
goTexts[c.Text]++
|
||||
}
|
||||
|
||||
// Count texts that appear in both
|
||||
for t, pyCount := range pyTexts {
|
||||
goCount := goTexts[t]
|
||||
if goCount > 0 {
|
||||
m := pyCount
|
||||
if goCount < m {
|
||||
m = goCount
|
||||
}
|
||||
textInBoth += m
|
||||
} else {
|
||||
textOnlyPy += pyCount
|
||||
}
|
||||
}
|
||||
for t, goCount := range goTexts {
|
||||
if pyTexts[t] == 0 {
|
||||
textOnlyGo += goCount
|
||||
}
|
||||
}
|
||||
|
||||
if len(pyPage) != len(goPage) {
|
||||
t.Logf("page %d: char count Go=%d Python=%d", pg, len(goPage), len(pyPage))
|
||||
}
|
||||
}
|
||||
|
||||
// Summary
|
||||
totalCompared := textInBoth + textOnlyPy + textOnlyGo
|
||||
overlapRate := 0.0
|
||||
if totalCompared > 0 {
|
||||
overlapRate = float64(textInBoth) / float64(totalCompared) * 100
|
||||
}
|
||||
|
||||
t.Logf("chars: Go=%d Python=%d | text overlap: %.1f%% (shared=%d, only_py=%d, only_go=%d)",
|
||||
totalGo, totalPy, overlapRate, textInBoth, textOnlyPy, textOnlyGo)
|
||||
|
||||
if totalPy > 0 && totalGo > 0 {
|
||||
countDiff := float64(math.Abs(float64(totalGo-totalPy))) / float64(totalPy) * 100
|
||||
if countDiff > 5 {
|
||||
t.Errorf("char count differs by %.1f%% (>5%%)", countDiff)
|
||||
}
|
||||
}
|
||||
})
|
||||
totalPDFs++
|
||||
}
|
||||
|
||||
if totalPDFs == 0 {
|
||||
t.Error("no PDF/snapshot pairs found")
|
||||
}
|
||||
}
|
||||
|
||||
// loadPyPageChars reads Python pdfplumber page_chars from a snapshot JSON.
|
||||
func loadPyPageChars(t *testing.T, path string) [][]pyChar {
|
||||
t.Helper()
|
||||
data, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
t.Fatalf("read: %v", err)
|
||||
}
|
||||
var s struct {
|
||||
Stages map[string]struct {
|
||||
PageChars [][]pyChar `json:"page_chars"`
|
||||
} `json:"stages"`
|
||||
}
|
||||
if err := json.Unmarshal(data, &s); err != nil {
|
||||
t.Fatalf("parse: %v", err)
|
||||
}
|
||||
stage, ok := s.Stages["__images__"]
|
||||
if !ok {
|
||||
t.Fatal("no __images__ stage in snapshot")
|
||||
}
|
||||
return stage.PageChars
|
||||
}
|
||||
|
||||
// ── Helpers ──────────────────────────────────────────────────────────────
|
||||
|
||||
func openFixture(t *testing.T, name string) *Document {
|
||||
t.Helper()
|
||||
doc, err := Open(filepath.Join(fixtureDir, name))
|
||||
if err != nil {
|
||||
t.Fatalf("Open(%s): %v", name, err)
|
||||
}
|
||||
return doc
|
||||
}
|
||||
|
||||
func TestGetPageChars_RadicalNormalization(t *testing.T) {
|
||||
// Verify that GetPageChars applies normalizeRadicals to every char.
|
||||
// Uses any available fixture PDF — just checking no radical leaks through.
|
||||
doc := openFixture(t, "01_english_simple.pdf")
|
||||
defer doc.Close()
|
||||
|
||||
n, _ := doc.PageCount()
|
||||
foundRadical := false
|
||||
for pg := 0; pg < n && !foundRadical; pg++ {
|
||||
chars, err := doc.GetPageChars(pg)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
for _, c := range chars {
|
||||
for _, r := range c.Text {
|
||||
if r >= 0x2F00 && r <= 0x2FDF {
|
||||
t.Errorf("Kangxi Radical U+%04X found in page %d: %q — normalization NOT applied",
|
||||
r, pg, c.Text)
|
||||
foundRadical = true
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
if !foundRadical {
|
||||
t.Log("No Kangxi Radicals found — normalization applied (or none in source)")
|
||||
}
|
||||
}
|
||||
|
||||
// TestExtractChars_RotatedPages_CoordsInBounds verifies that character
|
||||
// coordinates from rotated pages stay within page bounds. pdf_oxide
|
||||
// already applies /Rotate internally; the Go engine must not rotate
|
||||
// a second time (double rotation pushes coords out of bounds).
|
||||
func TestExtractChars_RotatedPages_CoordsInBounds(t *testing.T) {
|
||||
angles := []struct {
|
||||
name string
|
||||
rot int
|
||||
}{
|
||||
{"rotate_0", 0},
|
||||
{"rotate_90", 90},
|
||||
{"rotate_180", 180},
|
||||
{"rotate_270", 270},
|
||||
}
|
||||
|
||||
for _, a := range angles {
|
||||
t.Run(a.name, func(t *testing.T) {
|
||||
data, err := os.ReadFile(filepath.Join(fixtureDir, a.name+".pdf"))
|
||||
if err != nil {
|
||||
t.Fatalf("ReadFile: %v", err)
|
||||
}
|
||||
eng, err := NewEngine(data)
|
||||
if err != nil {
|
||||
t.Fatalf("NewEngine: %v", err)
|
||||
}
|
||||
defer eng.Close()
|
||||
|
||||
chars, err := eng.ExtractChars(0)
|
||||
if err != nil {
|
||||
t.Fatalf("ExtractChars: %v", err)
|
||||
}
|
||||
if len(chars) == 0 {
|
||||
// Some rotated pages may legitimately have no extractable
|
||||
// characters. The critical requirement: if chars ARE
|
||||
// returned, every one must be within page bounds.
|
||||
t.Skipf("0 chars extracted — skipping bounds check")
|
||||
}
|
||||
|
||||
w, h, err := eng.PageSize(0)
|
||||
if err != nil {
|
||||
t.Fatalf("PageSize: %v", err)
|
||||
}
|
||||
|
||||
outOfBounds := 0
|
||||
for _, c := range chars {
|
||||
if c.X0 < -1 || c.X1 > w+1 || c.Top < -1 || c.Bottom > h+1 {
|
||||
t.Errorf("char %q out of bounds: (%.0f,%.0f)-(%.0f,%.0f) page=(%.0f,%.0f) rot=%d",
|
||||
c.Text, c.X0, c.Top, c.X1, c.Bottom, w, h, a.rot)
|
||||
outOfBounds++
|
||||
}
|
||||
}
|
||||
if outOfBounds > 0 {
|
||||
t.Errorf("%d/%d chars are out of bounds (rotation=%d°)",
|
||||
outOfBounds, len(chars), a.rot)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
56
internal/deepdoc/parser/pdf/pdfoxide/pdf_oxide_bench_test.go
Normal file
56
internal/deepdoc/parser/pdf/pdfoxide/pdf_oxide_bench_test.go
Normal file
@@ -0,0 +1,56 @@
|
||||
//go:build cgo
|
||||
|
||||
package pdfoxide
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestPDFPlumber_Basic(t *testing.T) {
|
||||
pdfDir := filepath.Join("..", "parser", "testdata", "pdfs")
|
||||
path := filepath.Join(pdfDir, "01_english_simple.pdf")
|
||||
data, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
t.Fatalf("read PDF: %v", err)
|
||||
}
|
||||
|
||||
eng, err := NewEngine(data)
|
||||
if err != nil {
|
||||
t.Fatalf("NewEngine: %v", err)
|
||||
}
|
||||
defer eng.Close()
|
||||
|
||||
pc, _ := eng.PageCount()
|
||||
t.Logf("Pages: %d", pc)
|
||||
|
||||
chars, err := eng.ExtractChars(0)
|
||||
if err != nil {
|
||||
t.Fatalf("ExtractChars: %v", err)
|
||||
}
|
||||
t.Logf("Page 0: %d chars extracted", len(chars))
|
||||
if len(chars) == 0 {
|
||||
t.Error("got 0 chars")
|
||||
}
|
||||
|
||||
// Show first few chars
|
||||
for i := 0; i < min(5, len(chars)); i++ {
|
||||
t.Logf(" char[%d]: text=%q x0=%.1f x1=%.1f top=%.1f bottom=%.1f font=%q",
|
||||
i, chars[i].Text, chars[i].X0, chars[i].X1, chars[i].Top, chars[i].Bottom, chars[i].FontName)
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkPDFPlumber_ExtractChars(b *testing.B) {
|
||||
pdfDir := filepath.Join("..", "parser", "testdata", "pdfs")
|
||||
path := filepath.Join(pdfDir, "01_english_simple.pdf")
|
||||
data, _ := os.ReadFile(path)
|
||||
|
||||
eng, _ := NewEngine(data)
|
||||
defer eng.Close()
|
||||
|
||||
b.ResetTimer()
|
||||
for i := 0; i < b.N; i++ {
|
||||
eng.ExtractChars(0)
|
||||
}
|
||||
}
|
||||
248
internal/deepdoc/parser/pdf/pdfoxide/pdf_oxide_engine.go
Normal file
248
internal/deepdoc/parser/pdf/pdfoxide/pdf_oxide_engine.go
Normal file
@@ -0,0 +1,248 @@
|
||||
//go:build cgo
|
||||
|
||||
package pdfoxide
|
||||
|
||||
import (
|
||||
"image"
|
||||
"math"
|
||||
|
||||
"ragflow/internal/deepdoc/parser/pdf/pdfium"
|
||||
)
|
||||
|
||||
// Char represents a single character extracted from a PDF page.
|
||||
type Char struct {
|
||||
X0, X1 float64
|
||||
Top, Bottom float64
|
||||
Text string
|
||||
FontName string
|
||||
FontSize float64
|
||||
PageNumber int
|
||||
}
|
||||
|
||||
// Engine wraps pdf_oxide to extract chars and render pages.
|
||||
type Engine struct {
|
||||
doc *Document
|
||||
rawData []byte
|
||||
}
|
||||
|
||||
// NewEngine opens a PDF from bytes and returns an Engine.
|
||||
func NewEngine(pdfBytes []byte) (*Engine, error) {
|
||||
doc, err := OpenBytes(pdfBytes)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &Engine{doc: doc, rawData: pdfBytes}, nil
|
||||
}
|
||||
|
||||
func (e *Engine) RawData() []byte { return e.rawData }
|
||||
|
||||
func (e *Engine) ExtractChars(pageNum int) ([]Char, error) {
|
||||
chars, err := e.doc.GetDedupePageChars(pageNum, 0.5)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// pdf_oxide returns characters in the original (unrotated) PDF
|
||||
// coordinate space. Rotate to match pdfium's effective (post-
|
||||
// /Rotate) coordinate space used for rendering and DLA/OCR.
|
||||
//
|
||||
// Rotation detection uses two sources:
|
||||
// 1. Byte-scan for explicit /Rotate (finds directly-defined values).
|
||||
// 2. Dimension comparison: pdf_oxide raw vs pdfium effective.
|
||||
// If dimensions are swapped, the page has implicit rotation
|
||||
// (inherited /Rotate or ContentBox rotation).
|
||||
rawW, rawH, _ := e.doc.PageSize(pageNum)
|
||||
effW, effH, pdfErr := pdfium.PageSize(e.rawData, pageNum)
|
||||
if pdfErr != nil {
|
||||
effW, effH = rawW, rawH
|
||||
}
|
||||
|
||||
dimSwapped := rawW > 0 && rawH > 0 && effW > 0 && effH > 0 &&
|
||||
math.Abs(rawW-effH) < 1 && math.Abs(rawH-effW) < 1
|
||||
|
||||
rawRot := parsePageRotationFromRaw(e.rawData, pageNum)
|
||||
|
||||
needsRotate := false
|
||||
rotation90 := false
|
||||
rotation180 := false
|
||||
|
||||
if dimSwapped {
|
||||
needsRotate = true
|
||||
if rawRot == 270 {
|
||||
rotation90 = false
|
||||
} else {
|
||||
rotation90 = true
|
||||
}
|
||||
} else if rawRot == 90 || rawRot == 270 {
|
||||
// Explicit /Rotate found but dimension-swap check failed
|
||||
// (e.g. CropBox alters effective dimensions). Trust the
|
||||
// explicit /Rotate value.
|
||||
needsRotate = true
|
||||
rotation90 = (rawRot != 270)
|
||||
} else if rawRot == 180 {
|
||||
needsRotate = true
|
||||
rotation180 = true
|
||||
}
|
||||
|
||||
// CropBox correction — shift origin if CropBox differs from MediaBox.
|
||||
var cropDX, cropDY float64
|
||||
realCrop, hasCrop := parseCropBoxFromRaw(e.rawData, pageNum)
|
||||
if hasCrop {
|
||||
cropH := realCrop[3] - realCrop[1]
|
||||
oxideCropH := rawH
|
||||
if cropH > 0 && (realCrop[0] != 0 || realCrop[1] != 0 ||
|
||||
math.Abs(realCrop[3]-oxideCropH) > 0.5) {
|
||||
cropDX = -realCrop[0]
|
||||
cropDY = -(oxideCropH - realCrop[3])
|
||||
}
|
||||
}
|
||||
|
||||
// When rotation is applied, the crop shift must be applied AFTER
|
||||
// rotation, using the correct axes for the rotated coordinate space.
|
||||
rotateCropDX, rotateCropDY := cropDX, cropDY
|
||||
if needsRotate && (cropDX != 0 || cropDY != 0) {
|
||||
switch {
|
||||
case rotation90:
|
||||
// rotate(x+cropDX,y+cropDY) = (rawH-(y+cropDY),x+cropDX)
|
||||
// = rotate(x,y) + (-cropDY, +cropDX)
|
||||
// cropDX=-30,cropDY=-10 => post-rotate shift = (+10,-30)
|
||||
rotateCropDX = -cropDY
|
||||
rotateCropDY = cropDX
|
||||
case rotation180:
|
||||
rotateCropDX = -cropDX
|
||||
rotateCropDY = -cropDY
|
||||
default: // 270 CW
|
||||
rotateCropDX = cropDY
|
||||
rotateCropDY = -cropDX
|
||||
}
|
||||
cropDX, cropDY = 0, 0
|
||||
}
|
||||
|
||||
result := make([]Char, len(chars))
|
||||
for i, c := range chars {
|
||||
x0, x1 := c.X0, c.X1
|
||||
top, bottom := c.Top, c.Bottom
|
||||
|
||||
x0 += cropDX
|
||||
x1 += cropDX
|
||||
top += cropDY
|
||||
bottom += cropDY
|
||||
|
||||
if needsRotate {
|
||||
origX0, origX1 := x0, x1
|
||||
origTop, origBottom := top, bottom
|
||||
|
||||
switch {
|
||||
case rotation90:
|
||||
x0 = rawH - origBottom
|
||||
x1 = rawH - origTop
|
||||
top = origX0
|
||||
bottom = origX1
|
||||
case rotation180:
|
||||
x0 = rawW - origX1
|
||||
x1 = rawW - origX0
|
||||
top = rawH - origBottom
|
||||
bottom = rawH - origTop
|
||||
default: // 270 CW
|
||||
x0 = origTop
|
||||
x1 = origBottom
|
||||
top = rawW - origX1
|
||||
bottom = rawW - origX0
|
||||
}
|
||||
|
||||
if x0 > x1 {
|
||||
x0, x1 = x1, x0
|
||||
}
|
||||
if top > bottom {
|
||||
top, bottom = bottom, top
|
||||
}
|
||||
}
|
||||
|
||||
// Apply crop correction in the final coordinate space.
|
||||
x0 += rotateCropDX
|
||||
x1 += rotateCropDX
|
||||
top += rotateCropDY
|
||||
bottom += rotateCropDY
|
||||
|
||||
result[i] = Char{
|
||||
X0: x0, X1: x1, Top: top, Bottom: bottom,
|
||||
Text: c.Text, FontName: c.Fontname, FontSize: c.Size,
|
||||
PageNumber: pageNum,
|
||||
}
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
|
||||
// parsePageRotationFromRaw scans raw PDF bytes for /Rotate entries.
|
||||
// Returns the rotation value for the given page index, or 0 if not found.
|
||||
// NOTE: This only finds /Rotate defined directly on page objects.
|
||||
// Inherited /Rotate (from parent Pages dict) is not detected here but
|
||||
// is caught by the dimension-comparison fallback in ExtractChars.
|
||||
func parsePageRotationFromRaw(data []byte, pageIdx int) int {
|
||||
var rotations []int
|
||||
rest := data
|
||||
for {
|
||||
idx := -1
|
||||
for i := 0; i < len(rest)-7; i++ {
|
||||
if rest[i] == '/' && rest[i+1] == 'R' && rest[i+2] == 'o' &&
|
||||
rest[i+3] == 't' && rest[i+4] == 'a' && rest[i+5] == 't' &&
|
||||
rest[i+6] == 'e' {
|
||||
idx = i
|
||||
break
|
||||
}
|
||||
}
|
||||
if idx < 0 {
|
||||
break
|
||||
}
|
||||
rest = rest[idx+7:]
|
||||
for len(rest) > 0 && (rest[0] == ' ' || rest[0] == '\t' || rest[0] == '\n' || rest[0] == '\r') {
|
||||
rest = rest[1:]
|
||||
}
|
||||
if len(rest) == 0 {
|
||||
break
|
||||
}
|
||||
val := 0
|
||||
i := 0
|
||||
for i < len(rest) && rest[i] >= '0' && rest[i] <= '9' {
|
||||
val = val*10 + int(rest[i]-'0')
|
||||
i++
|
||||
}
|
||||
if i > 0 {
|
||||
rotations = append(rotations, val)
|
||||
}
|
||||
rest = rest[i:]
|
||||
}
|
||||
if pageIdx < len(rotations) {
|
||||
return rotations[pageIdx]
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
// RenderPageImage uses pdfium for page rendering — pdfium correctly
|
||||
// applies /Rotate so the output matches character coordinates and DLA.
|
||||
// There is no pdf_oxide fallback because pdf_oxide does not apply
|
||||
// /Rotate, producing images in a different coordinate space.
|
||||
func (e *Engine) RenderPageImage(pageNum int, dpi float64) (image.Image, error) {
|
||||
return pdfium.RenderPage(e.rawData, pageNum, dpi)
|
||||
}
|
||||
|
||||
func (e *Engine) RenderPage(pageNum int, dpi float64) ([]byte, error) {
|
||||
result, err := e.doc.RenderPage(pageNum, dpi)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return result.Data, nil
|
||||
}
|
||||
|
||||
// PageSize returns the effective page dimensions via pdfium, which
|
||||
// correctly applies /Rotate. pdf_oxide's own PageSize returns raw
|
||||
// (unrotated) dimensions.
|
||||
func (e *Engine) PageSize(pageNum int) (float64, float64, error) {
|
||||
w, h, err := pdfium.PageSize(e.rawData, pageNum)
|
||||
if err != nil {
|
||||
return e.doc.PageSize(pageNum)
|
||||
}
|
||||
return w, h, nil
|
||||
}
|
||||
func (e *Engine) PageCount() (int, error) { return e.doc.PageCount() }
|
||||
func (e *Engine) Close() error { e.doc.Close(); return nil }
|
||||
Reference in New Issue
Block a user