mirror of
https://github.com/infiniflow/ragflow.git
synced 2026-07-16 04:37:21 +08:00
Refactor: migrate pdf_parser.py to golang (#16323)
### What problem does this PR solve? Http API based on onnx model. pdf_parser.py to golang ### Type of change - [x] Refactoring
This commit is contained in:
Jack
645
internal/deepdoc/parser/pdf/tools/compare.go
Normal file
645
internal/deepdoc/parser/pdf/tools/compare.go
Normal file
@@ -0,0 +1,645 @@
|
||||
package tools
|
||||
|
||||
import (
|
||||
"encoding/csv"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"math"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/xuri/excelize/v2"
|
||||
"golang.org/x/text/unicode/norm"
|
||||
)
|
||||
|
||||
// Diff stores per-PDF comparison metrics between Go and Python output.
|
||||
type Diff struct {
|
||||
File string
|
||||
PagesOk bool
|
||||
BoxesInitDiffPct float64
|
||||
BoxesTMDiffPct float64
|
||||
BoxesVMDiffPct float64
|
||||
SectionsDiffPct float64
|
||||
TextLenDiffPct float64
|
||||
CharsDiffPct float64
|
||||
TablesDiff int
|
||||
CharSim float64
|
||||
LcsSim float64
|
||||
RawCharSim float64 // CharSim without NFKC normalization
|
||||
RawLcsSim float64 // LcsSim without space stripping
|
||||
}
|
||||
|
||||
// CompareWithPython compares Go results against Python reference.
|
||||
func CompareWithPython(log TLogger, goResults []BatchResult, pyResults []PyResult, goTextDir, pyTextDir string) {
|
||||
pyMap := make(map[string]PyResult, len(pyResults))
|
||||
for _, pr := range pyResults {
|
||||
pyMap[pr.File] = pr
|
||||
}
|
||||
goMap := make(map[string]BatchResult, len(goResults))
|
||||
for _, r := range goResults {
|
||||
goMap[r.File] = r
|
||||
}
|
||||
|
||||
var diffs []Diff
|
||||
matched, mismatched := 0, 0
|
||||
|
||||
for _, r := range goResults {
|
||||
py, ok := pyMap[r.File]
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
d := Diff{File: r.File, TablesDiff: r.TSTables - py.Tables}
|
||||
if py.Pages > 0 {
|
||||
d.PagesOk = r.Pages == py.Pages
|
||||
if r.Pages == py.Pages {
|
||||
matched++
|
||||
} else {
|
||||
mismatched++
|
||||
}
|
||||
}
|
||||
if py.BoxesInitial > 0 {
|
||||
d.BoxesInitDiffPct = math.Abs(float64(r.BoxesInitial-py.BoxesInitial)) / float64(py.BoxesInitial) * 100
|
||||
}
|
||||
if py.BoxesTextMerge > 0 {
|
||||
d.BoxesTMDiffPct = math.Abs(float64(r.BoxesTextMerg-py.BoxesTextMerge)) / float64(py.BoxesTextMerge) * 100
|
||||
}
|
||||
if py.BoxesVertMerge > 0 {
|
||||
d.BoxesVMDiffPct = math.Abs(float64(r.BoxesVertMerg-py.BoxesVertMerge)) / float64(py.BoxesVertMerge) * 100
|
||||
}
|
||||
if py.Sections > 0 {
|
||||
d.SectionsDiffPct = math.Abs(float64(r.Sections-py.Sections)) / float64(py.Sections) * 100
|
||||
}
|
||||
if py.TextLen > 0 {
|
||||
d.TextLenDiffPct = math.Abs(float64(r.TextLen-py.TextLen)) / float64(py.TextLen) * 100
|
||||
}
|
||||
if py.Chars > 0 {
|
||||
d.CharsDiffPct = math.Abs(float64(r.Chars-py.Chars)) / float64(py.Chars) * 100
|
||||
}
|
||||
|
||||
goTextPath := filepath.Join(goTextDir, r.File+".txt")
|
||||
pyTextPath := filepath.Join(pyTextDir, r.File+".txt")
|
||||
if goTxt, err := os.ReadFile(goTextPath); err == nil {
|
||||
if pyTxt, err := os.ReadFile(pyTextPath); err == nil {
|
||||
goStr, pyStr := string(goTxt), string(pyTxt)
|
||||
// NFKC normalisation: fullwidth→halfwidth (e.g. "，（" → ",(")
|
||||
goStr = norm.NFKC.String(goStr)
|
||||
pyStr = norm.NFKC.String(pyStr)
|
||||
d.CharSim = CharSimilarity(goStr, pyStr)
|
||||
// Section-level LCS: align sections by position window,
|
||||
// compute per-section LCS, bidirectional F1.
|
||||
d.LcsSim = SectionAlignedScore(goStr, pyStr)
|
||||
// Raw metrics without NFKC / space stripping.
|
||||
d.RawCharSim = RawCharSimilarity(string(goTxt), string(pyTxt))
|
||||
d.RawLcsSim = SectionAlignedScore(string(goTxt), string(pyTxt))
|
||||
}
|
||||
}
|
||||
diffs = append(diffs, d)
|
||||
log.Logf(" [%d/%d] %s CharDiff=D%.1f%% LcsDiff=D%.1f%% RawCharDiff=D%.1f%% RawLcsDiff=D%.1f%%",
|
||||
len(diffs), len(goResults), r.File, 100-d.CharSim, 100-d.LcsSim, 100-d.RawCharSim, 100-d.RawLcsSim)
|
||||
}
|
||||
|
||||
sort.Slice(diffs, func(i, j int) bool { return diffs[i].SectionsDiffPct < diffs[j].SectionsDiffPct })
|
||||
|
||||
log.Logf("\n=== Go vs Python (%d PDFs) ===", len(diffs))
|
||||
log.Logf("Pages match: %d/%d", matched, matched+mismatched)
|
||||
log.Logf("%-40s %-18s %-18s %s %s %s %s %s %s %s %s %s %s",
|
||||
"file", "Go:init->tm->vm->sec", "Py:init->tm->vm->sec",
|
||||
"Init%", "TM%", "VM%", "Sec%", "Txt%", "TabD", "CharDiff%", "LcsDiff%", "RawCharDiff%", "RawLcsDiff%")
|
||||
log.Logf("%s", strings.Repeat("-", 168))
|
||||
|
||||
for _, d := range diffs {
|
||||
py := pyMap[d.File]
|
||||
gr := goMap[d.File]
|
||||
goStages := fmt.Sprintf("%3d->%3d->%3d->%3d", gr.BoxesInitial, gr.BoxesTextMerg, gr.BoxesVertMerg, gr.Sections)
|
||||
pyStages := fmt.Sprintf("%3d->%3d->%3d->%3d", py.BoxesInitial, py.BoxesTextMerge, py.BoxesVertMerge, py.Sections)
|
||||
log.Logf("%-40s %-18s %-18s %4.0f%% %4.0f%% %4.0f%% %4.0f%% %4.0f%% %+4d %.0f%% %.0f%% %.0f%% %.0f%%",
|
||||
d.File, goStages, pyStages,
|
||||
d.BoxesInitDiffPct, d.BoxesTMDiffPct, d.BoxesVMDiffPct,
|
||||
d.SectionsDiffPct, d.TextLenDiffPct, d.TablesDiff,
|
||||
100-d.CharSim, 100-d.LcsSim,
|
||||
100-d.RawCharSim, 100-d.RawLcsSim)
|
||||
}
|
||||
|
||||
n := len(diffs)
|
||||
if n == 0 {
|
||||
return
|
||||
}
|
||||
|
||||
type stats struct {
|
||||
median, mean, max, min float64
|
||||
over5, over10 int
|
||||
}
|
||||
computeStats := func(get func(Diff) float64) stats {
|
||||
sort.Slice(diffs, func(i, j int) bool { return get(diffs[i]) < get(diffs[j]) })
|
||||
s := stats{min: 1e9}
|
||||
if n%2 == 0 {
|
||||
s.median = (get(diffs[n/2-1]) + get(diffs[n/2])) / 2
|
||||
} else {
|
||||
s.median = get(diffs[n/2])
|
||||
}
|
||||
var sum float64
|
||||
for _, d := range diffs {
|
||||
v := get(d)
|
||||
sum += v
|
||||
if v > s.max {
|
||||
s.max = v
|
||||
}
|
||||
if v < s.min {
|
||||
s.min = v
|
||||
}
|
||||
if v > 5 {
|
||||
s.over5++
|
||||
}
|
||||
if v > 10 {
|
||||
s.over10++
|
||||
}
|
||||
}
|
||||
s.mean = sum / float64(n)
|
||||
return s
|
||||
}
|
||||
|
||||
label := func(name string, s stats) string {
|
||||
return fmt.Sprintf("%s Med=%.1f%% Mean=%.1f%% Min=%.0f%% Max=%.0f%% >5%%:%d >10%%:%d",
|
||||
name, s.median, s.mean, s.min, s.max, s.over5, s.over10)
|
||||
}
|
||||
|
||||
log.Logf("\nSummary (n=%d):", n)
|
||||
log.Logf(" %s", label("BoxesInit ", computeStats(func(d Diff) float64 { return d.BoxesInitDiffPct })))
|
||||
log.Logf(" %s", label("TextMerge", computeStats(func(d Diff) float64 { return d.BoxesTMDiffPct })))
|
||||
log.Logf(" %s", label("VertMerge", computeStats(func(d Diff) float64 { return d.BoxesVMDiffPct })))
|
||||
log.Logf(" %s", label("Sections ", computeStats(func(d Diff) float64 { return d.SectionsDiffPct })))
|
||||
log.Logf(" %s", label("TextLen ", computeStats(func(d Diff) float64 { return d.TextLenDiffPct })))
|
||||
log.Logf(" %s", label("CharDiff ", computeStats(func(d Diff) float64 { return 100 - d.CharSim })))
|
||||
log.Logf(" %s", label("LcsDiff ", computeStats(func(d Diff) float64 { return 100 - d.LcsSim })))
|
||||
log.Logf(" %s", label("RawCharDiff", computeStats(func(d Diff) float64 { return 100 - d.RawCharSim })))
|
||||
log.Logf(" %s", label("RawLcsDiff ", computeStats(func(d Diff) float64 { return 100 - d.RawLcsSim })))
|
||||
|
||||
// Auto-generate xlsx report with timestamp.
|
||||
mode := filepath.Base(filepath.Dir(goTextDir)) // "ocr"
|
||||
ts := time.Now().Format("20060102_1504")
|
||||
xlsxDir := filepath.Join("testdata", "output")
|
||||
os.MkdirAll(xlsxDir, 0755)
|
||||
xlsxPath := filepath.Join(xlsxDir, fmt.Sprintf("compare_%s_%s.xlsx", mode, ts))
|
||||
if err := WriteExcel(xlsxPath, diffs); err != nil {
|
||||
log.Logf("Excel write error: %v", err)
|
||||
} else {
|
||||
log.Logf("Excel report: %s", xlsxPath)
|
||||
}
|
||||
|
||||
// Also write CSV if BATCH_CSV env is set (backward compat).
|
||||
if csvPath := os.Getenv("BATCH_CSV"); csvPath != "" {
|
||||
if err := WriteCSV(csvPath, diffs); err != nil {
|
||||
log.Logf("CSV write error: %v", err)
|
||||
} else {
|
||||
log.Logf("CSV written to %s", csvPath)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// WriteCSV writes comparison results to a CSV file using encoding/csv
|
||||
// for proper field escaping (filenames may contain commas/quotes).
|
||||
func WriteCSV(path string, diffs []Diff) error {
|
||||
f, err := os.Create(path)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer f.Close()
|
||||
|
||||
w := csv.NewWriter(f)
|
||||
defer w.Flush()
|
||||
|
||||
if err := w.Write([]string{"file", "init%", "tm%", "vm%", "sec%", "txt%", "tabsD", "chrdiff%", "lcsdiff%", "rawChr%", "rawLcs%"}); err != nil {
|
||||
return err
|
||||
}
|
||||
for _, d := range diffs {
|
||||
row := []string{
|
||||
d.File,
|
||||
strconv.FormatFloat(d.BoxesInitDiffPct, 'f', 1, 64),
|
||||
strconv.FormatFloat(d.BoxesTMDiffPct, 'f', 1, 64),
|
||||
strconv.FormatFloat(d.BoxesVMDiffPct, 'f', 1, 64),
|
||||
strconv.FormatFloat(d.SectionsDiffPct, 'f', 1, 64),
|
||||
strconv.FormatFloat(d.TextLenDiffPct, 'f', 1, 64),
|
||||
strconv.Itoa(d.TablesDiff),
|
||||
strconv.FormatFloat(100-d.CharSim, 'f', 1, 64),
|
||||
strconv.FormatFloat(100-d.LcsSim, 'f', 1, 64),
|
||||
strconv.FormatFloat(100-d.RawCharSim, 'f', 1, 64),
|
||||
strconv.FormatFloat(100-d.RawLcsSim, 'f', 1, 64),
|
||||
}
|
||||
if err := w.Write(row); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
w.Flush()
|
||||
return w.Error()
|
||||
}
|
||||
|
||||
// WriteExcel writes comparison results to an xlsx file with formatting.
|
||||
func WriteExcel(path string, diffs []Diff) error {
|
||||
f := excelize.NewFile()
|
||||
defer f.Close()
|
||||
sheet := "Comparison"
|
||||
f.SetSheetName("Sheet1", sheet)
|
||||
|
||||
// Styles.
|
||||
headerStyle, _ := f.NewStyle(&excelize.Style{
|
||||
Font: &excelize.Font{Bold: true},
|
||||
Fill: excelize.Fill{Type: "pattern", Pattern: 1, Color: []string{"D9E1F2"}},
|
||||
Alignment: &excelize.Alignment{Horizontal: "center"},
|
||||
})
|
||||
greenStyle, _ := f.NewStyle(&excelize.Style{
|
||||
Fill: excelize.Fill{Type: "pattern", Pattern: 1, Color: []string{"C6EFCE"}},
|
||||
NumFmt: 2,
|
||||
})
|
||||
yellowStyle, _ := f.NewStyle(&excelize.Style{
|
||||
Fill: excelize.Fill{Type: "pattern", Pattern: 1, Color: []string{"FFEB9C"}},
|
||||
NumFmt: 2,
|
||||
})
|
||||
redStyle, _ := f.NewStyle(&excelize.Style{
|
||||
Fill: excelize.Fill{Type: "pattern", Pattern: 1, Color: []string{"FFC7CE"}},
|
||||
NumFmt: 2,
|
||||
})
|
||||
|
||||
// Header row.
|
||||
headers := []string{"File", "Init%", "TM%", "VM%", "Sec%", "Txt%", "TabsD", "ChrDiff%", "LcsDiff%"}
|
||||
for i, h := range headers {
|
||||
cell, _ := excelize.CoordinatesToCellName(i+1, 1)
|
||||
f.SetCellValue(sheet, cell, h)
|
||||
f.SetCellStyle(sheet, cell, cell, headerStyle)
|
||||
}
|
||||
|
||||
// Data rows.
|
||||
for row, d := range diffs {
|
||||
r := row + 2 // 1-indexed, skip header
|
||||
vals := []float64{
|
||||
0, // placeholder for file
|
||||
d.BoxesInitDiffPct, d.BoxesTMDiffPct, d.BoxesVMDiffPct,
|
||||
d.SectionsDiffPct, d.TextLenDiffPct, float64(d.TablesDiff),
|
||||
100 - d.CharSim, 100 - d.LcsSim,
|
||||
}
|
||||
|
||||
// File name (column A).
|
||||
f.SetCellValue(sheet, cellName(1, r), d.File)
|
||||
|
||||
// Numeric columns (B-I).
|
||||
for col := 2; col <= 9; col++ {
|
||||
cell := cellName(col, r)
|
||||
v := vals[col-1]
|
||||
f.SetCellValue(sheet, cell, v)
|
||||
// Color: green <5, yellow 5-20, red >=20.
|
||||
if col == 7 { // TabsD is a count, not percentage
|
||||
continue
|
||||
}
|
||||
abs := math.Abs(v)
|
||||
switch {
|
||||
case abs < 5:
|
||||
f.SetCellStyle(sheet, cell, cell, greenStyle)
|
||||
case abs < 20:
|
||||
f.SetCellStyle(sheet, cell, cell, yellowStyle)
|
||||
default:
|
||||
f.SetCellStyle(sheet, cell, cell, redStyle)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Column widths.
|
||||
f.SetColWidth(sheet, "A", "A", 45)
|
||||
f.SetColWidth(sheet, "B", "I", 12)
|
||||
|
||||
// Freeze header row.
|
||||
f.SetPanes(sheet, &excelize.Panes{
|
||||
Freeze: true,
|
||||
Split: false,
|
||||
XSplit: 0,
|
||||
YSplit: 1,
|
||||
TopLeftCell: "A2",
|
||||
ActivePane: "bottomLeft",
|
||||
})
|
||||
|
||||
return f.SaveAs(path)
|
||||
}
|
||||
|
||||
func cellName(col, row int) string {
|
||||
s, _ := excelize.CoordinatesToCellName(col, row)
|
||||
return s
|
||||
}
|
||||
|
||||
// including per-cell text comparison.
|
||||
func CompareTablesWithPython(log TLogger, goTablesDir, pyTablesDir string) {
|
||||
goEntries, err := os.ReadDir(goTablesDir)
|
||||
if err != nil {
|
||||
log.Logf("Tables compare: no Go tables dir %s", goTablesDir)
|
||||
return
|
||||
}
|
||||
|
||||
type goTable struct {
|
||||
Rows [][]string `json:"rows"`
|
||||
}
|
||||
type pyCell struct {
|
||||
X0 float64 `json:"x0"`
|
||||
X1 float64 `json:"x1"`
|
||||
Top float64 `json:"top"`
|
||||
Bottom float64 `json:"bottom"`
|
||||
Text string `json:"text"`
|
||||
Page int `json:"page"`
|
||||
}
|
||||
type pyResult struct {
|
||||
Cells []pyCell `json:"cells"`
|
||||
Page int `json:"page"`
|
||||
Rows [][]string `json:"rows"`
|
||||
}
|
||||
type pyFile struct {
|
||||
Tables int `json:"tables"`
|
||||
Results []pyResult `json:"results"`
|
||||
}
|
||||
|
||||
matched, tableDiffs, cellDiffs, textMismatches := 0, 0, 0, 0
|
||||
totalCellsCompared, totalCellsMatched := 0, 0
|
||||
|
||||
log.Logf("\n=== Table Comparison (Go vs Python) ===")
|
||||
log.Logf("%-40s %6s %6s %6s %6s %8s %s",
|
||||
"file", "GoTbl", "PyTbl", "GoCel", "PyCel", "TxtMatch", "Result")
|
||||
log.Logf("%s", strings.Repeat("-", 100))
|
||||
|
||||
for _, e := range goEntries {
|
||||
if e.IsDir() || !strings.HasSuffix(e.Name(), ".json") {
|
||||
continue
|
||||
}
|
||||
|
||||
goPath := filepath.Join(goTablesDir, e.Name())
|
||||
pyPath := filepath.Join(pyTablesDir, e.Name())
|
||||
if !FileExists(pyPath) {
|
||||
continue
|
||||
}
|
||||
|
||||
// Read Go tables.
|
||||
goData, _ := os.ReadFile(goPath)
|
||||
var goTables []goTable
|
||||
if err := json.Unmarshal(goData, &goTables); err != nil {
|
||||
log.Logf(" %s: Go JSON parse error: %v", e.Name(), err)
|
||||
continue
|
||||
}
|
||||
|
||||
// Read Python tables.
|
||||
pyData, _ := os.ReadFile(pyPath)
|
||||
var pyF pyFile
|
||||
if err := json.Unmarshal(pyData, &pyF); err != nil {
|
||||
log.Logf(" %s: Py JSON parse error: %v", e.Name(), err)
|
||||
continue
|
||||
}
|
||||
|
||||
matched++
|
||||
|
||||
// Count cells.
|
||||
goTotalCells := 0
|
||||
for _, t := range goTables {
|
||||
for _, row := range t.Rows {
|
||||
goTotalCells += len(row)
|
||||
}
|
||||
}
|
||||
pyTotalCells := 0
|
||||
for _, r := range pyF.Results {
|
||||
if len(r.Cells) > 0 {
|
||||
pyTotalCells += len(r.Cells)
|
||||
} else {
|
||||
for _, row := range r.Rows {
|
||||
pyTotalCells += len(row)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Cell-level text comparison (table by table, row by row, cell by cell).
|
||||
cellsCompared, cellsMatched := 0, 0
|
||||
nTables := min(len(goTables), len(pyF.Results))
|
||||
for ti := 0; ti < nTables; ti++ {
|
||||
goRows := goTables[ti].Rows
|
||||
pyRows := pyF.Results[ti].Rows
|
||||
nRows := min(len(goRows), len(pyRows))
|
||||
for ri := 0; ri < nRows; ri++ {
|
||||
nCols := min(len(goRows[ri]), len(pyRows[ri]))
|
||||
for ci := 0; ci < nCols; ci++ {
|
||||
cellsCompared++
|
||||
if strings.TrimSpace(goRows[ri][ci]) == strings.TrimSpace(pyRows[ri][ci]) {
|
||||
cellsMatched++
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
totalCellsCompared += cellsCompared
|
||||
totalCellsMatched += cellsMatched
|
||||
|
||||
// Status.
|
||||
status := "✅"
|
||||
txtMatch := ""
|
||||
if len(goTables) != len(pyF.Results) {
|
||||
tableDiffs++
|
||||
status = "❌ tables"
|
||||
}
|
||||
if goTotalCells != pyTotalCells {
|
||||
cellDiffs++
|
||||
if status == "✅" {
|
||||
status = "⚠️ cells"
|
||||
}
|
||||
}
|
||||
if cellsCompared > 0 {
|
||||
pct := float64(cellsMatched) / float64(cellsCompared) * 100
|
||||
txtMatch = fmt.Sprintf("%.0f%%", pct)
|
||||
if pct < 100 && status == "✅" {
|
||||
status = "⚠️ text"
|
||||
textMismatches++
|
||||
}
|
||||
if pct < 100 && status != "✅" {
|
||||
textMismatches++
|
||||
}
|
||||
} else {
|
||||
txtMatch = "-"
|
||||
}
|
||||
|
||||
name := strings.TrimSuffix(e.Name(), ".json")
|
||||
log.Logf("%-40s %6d %6d %6d %6d %8s %s",
|
||||
name, len(goTables), len(pyF.Results), goTotalCells, pyTotalCells, txtMatch, status)
|
||||
}
|
||||
|
||||
if matched == 0 {
|
||||
log.Logf("No matching table files found")
|
||||
return
|
||||
}
|
||||
|
||||
txtPct := 0.0
|
||||
if totalCellsCompared > 0 {
|
||||
txtPct = float64(totalCellsMatched) / float64(totalCellsCompared) * 100
|
||||
}
|
||||
log.Logf("\nTable Summary: %d PDFs, %d table diffs, %d cell diffs, %d text mismatches",
|
||||
matched, tableDiffs, cellDiffs, textMismatches)
|
||||
log.Logf("Cell text match: %d/%d (%.1f%%)", totalCellsMatched, totalCellsCompared, txtPct)
|
||||
}
|
||||
|
||||
// ── DLA intermediate comparison ──────────────────────────────────────────
|
||||
|
||||
type jsonDlaPage struct {
|
||||
Page int `json:"page"`
|
||||
Regions []jsonDlaRegion `json:"regions"`
|
||||
}
|
||||
type jsonDlaRegion struct {
|
||||
Label string `json:"label"` // Go uses "label"
|
||||
Type string `json:"type"` // Python uses "type"
|
||||
X0 float64 `json:"x0"`
|
||||
Y0 float64 `json:"y0"`
|
||||
X1 float64 `json:"x1"`
|
||||
Y1 float64 `json:"y1"`
|
||||
}
|
||||
|
||||
// CompareDLAWithPython compares per-page DLA layout regions.
|
||||
// Both dirs contain {pdf}.json files with []dlaPageRegion.
|
||||
func CompareDLAWithPython(log TLogger, goDLADir, pyDLADir string) {
|
||||
goEntries, _ := os.ReadDir(goDLADir)
|
||||
pyEntries, _ := os.ReadDir(pyDLADir)
|
||||
pySet := map[string]bool{}
|
||||
for _, e := range pyEntries {
|
||||
pySet[e.Name()] = true
|
||||
}
|
||||
|
||||
matched := 0
|
||||
log.Logf("\n=== DLA Comparison (Go vs Python) ===")
|
||||
log.Logf("%-40s %6s %6s %6s %6s %6s",
|
||||
"file", "GoPg", "PyPg", "GoReg", "PyReg", "TblReg")
|
||||
log.Logf("%s", strings.Repeat("-", 80))
|
||||
|
||||
for _, e := range goEntries {
|
||||
if !strings.HasSuffix(e.Name(), ".json") || !pySet[e.Name()] {
|
||||
continue
|
||||
}
|
||||
goData, _ := os.ReadFile(filepath.Join(goDLADir, e.Name()))
|
||||
pyData, _ := os.ReadFile(filepath.Join(pyDLADir, e.Name()))
|
||||
|
||||
var goPages []jsonDlaPage
|
||||
json.Unmarshal(goData, &goPages)
|
||||
var pyPages []jsonDlaPage
|
||||
json.Unmarshal(pyData, &pyPages)
|
||||
|
||||
matched++
|
||||
goRegions, pyRegions := 0, 0
|
||||
goTables, pyTables := 0, 0
|
||||
for _, p := range goPages {
|
||||
goRegions += len(p.Regions)
|
||||
for _, r := range p.Regions {
|
||||
if dlaRegionIsTable(r) {
|
||||
goTables++
|
||||
}
|
||||
}
|
||||
}
|
||||
for _, p := range pyPages {
|
||||
pyRegions += len(p.Regions)
|
||||
for _, r := range p.Regions {
|
||||
if dlaRegionIsTable(r) {
|
||||
pyTables++
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
name := strings.TrimSuffix(e.Name(), ".json")
|
||||
log.Logf("%-40s %6d %6d %6d %6d %6d",
|
||||
name, len(goPages), len(pyPages), goRegions, pyRegions, goTables-pyTables)
|
||||
}
|
||||
if matched == 0 {
|
||||
log.Logf("No matching DLA files found (go=%s py=%s)", goDLADir, pyDLADir)
|
||||
}
|
||||
}
|
||||
|
||||
// ── TSR raw intermediate comparison ──────────────────────────────────────
|
||||
|
||||
type tsrRawCell struct {
|
||||
TableIndex int `json:"table_index"`
|
||||
Page int `json:"page"`
|
||||
Label string `json:"label"`
|
||||
X0, Y0 float64 `json:"x0" y0:"y0"`
|
||||
X1, Y1 float64 `json:"x1" y1:"y1"`
|
||||
Text string `json:"text"`
|
||||
}
|
||||
|
||||
// CompareTSRRawWithPython compares raw TSR cells per table.
|
||||
// Both dirs contain {pdf}.json files with []tsrRawCell (Go) or []tsrRawCell (Py).
|
||||
func CompareTSRRawWithPython(log TLogger, goTSRDir, pyTSRDir string) {
|
||||
goEntries, _ := os.ReadDir(goTSRDir)
|
||||
pyEntries, _ := os.ReadDir(pyTSRDir)
|
||||
pySet := map[string]bool{}
|
||||
for _, e := range pyEntries {
|
||||
pySet[e.Name()] = true
|
||||
}
|
||||
|
||||
matched := 0
|
||||
totalDiffs := 0
|
||||
log.Logf("\n=== TSR Raw Comparison (Go vs Python) ===")
|
||||
log.Logf("%-40s %6s %6s %8s %8s %6s",
|
||||
"file", "GoTbl", "PyTbl", "GoCell", "PyCell", "LabelD")
|
||||
log.Logf("%s", strings.Repeat("-", 85))
|
||||
|
||||
for _, e := range goEntries {
|
||||
if !strings.HasSuffix(e.Name(), ".json") || !pySet[e.Name()] {
|
||||
continue
|
||||
}
|
||||
goData, _ := os.ReadFile(filepath.Join(goTSRDir, e.Name()))
|
||||
pyData, _ := os.ReadFile(filepath.Join(pyTSRDir, e.Name()))
|
||||
|
||||
var goCells []tsrRawCell
|
||||
json.Unmarshal(goData, &goCells)
|
||||
var pyCells []tsrRawCell
|
||||
json.Unmarshal(pyData, &pyCells)
|
||||
|
||||
// Group by table.
|
||||
goByTable := map[int][]tsrRawCell{}
|
||||
pyByTable := map[int][]tsrRawCell{}
|
||||
for _, c := range goCells {
|
||||
goByTable[c.TableIndex] = append(goByTable[c.TableIndex], c)
|
||||
}
|
||||
for _, c := range pyCells {
|
||||
pyByTable[c.TableIndex] = append(pyByTable[c.TableIndex], c)
|
||||
}
|
||||
|
||||
matched++
|
||||
labelDiffs := 0
|
||||
goTotal, pyTotal := len(goCells), len(pyCells)
|
||||
for ti := range goByTable {
|
||||
goTab := goByTable[ti]
|
||||
pyTab := pyByTable[ti]
|
||||
n := min(len(goTab), len(pyTab))
|
||||
for i := 0; i < n; i++ {
|
||||
if goTab[i].Label != pyTab[i].Label {
|
||||
labelDiffs++
|
||||
}
|
||||
}
|
||||
labelDiffs += abs(len(goTab) - len(pyTab))
|
||||
}
|
||||
if labelDiffs > 0 {
|
||||
totalDiffs++
|
||||
}
|
||||
|
||||
name := strings.TrimSuffix(e.Name(), ".json")
|
||||
log.Logf("%-40s %6d %6d %8d %8d %6d",
|
||||
name, len(goByTable), len(pyByTable), goTotal, pyTotal, labelDiffs)
|
||||
}
|
||||
if matched == 0 {
|
||||
log.Logf("No matching TSR raw files found (go=%s py=%s)", goTSRDir, pyTSRDir)
|
||||
} else {
|
||||
log.Logf("TSR Raw Summary: %d PDFs, %d with label diffs", matched, totalDiffs)
|
||||
}
|
||||
}
|
||||
|
||||
func dlaRegionIsTable(r jsonDlaRegion) bool {
|
||||
label := r.Label
|
||||
if label == "" {
|
||||
label = r.Type
|
||||
}
|
||||
return label == "table"
|
||||
}
|
||||
|
||||
func abs(x int) int {
|
||||
if x < 0 {
|
||||
return -x
|
||||
}
|
||||
return x
|
||||
}
|
||||
66
internal/deepdoc/parser/pdf/tools/config.go
Normal file
66
internal/deepdoc/parser/pdf/tools/config.go
Normal file
@@ -0,0 +1,66 @@
|
||||
package tools
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
"time"
|
||||
)
|
||||
|
||||
type Config struct {
|
||||
Count int
|
||||
Single string
|
||||
SkipOCR bool // DLA+TSR but no image OCR
|
||||
CompareOnly bool
|
||||
CompareFilter string
|
||||
CSVOutput string
|
||||
GoTextDir string
|
||||
PyTextDir string
|
||||
TablesDir string
|
||||
GoSuffix string
|
||||
}
|
||||
|
||||
func LoadConfig() Config {
|
||||
goVariant := "ocr"
|
||||
pyVariant := "ocr"
|
||||
td := filepath.Join("testdata")
|
||||
return Config{
|
||||
Count: envInt("BATCH_COUNT", 0),
|
||||
Single: os.Getenv("BATCH_SINGLE"),
|
||||
SkipOCR: os.Getenv("BATCH_SKIP_OCR") == "1",
|
||||
CompareOnly: os.Getenv("BATCH_COMPARE_ONLY") == "1",
|
||||
CompareFilter: os.Getenv("BATCH_COMPARE_FILTER"),
|
||||
CSVOutput: envStr("BATCH_COMPARE_CSV", filepath.Join(td, "output", fmt.Sprintf("compare_%s.csv", time.Now().Format("20060102_150405")))),
|
||||
GoTextDir: filepath.Join(td, "output", "go", goVariant, "text"),
|
||||
PyTextDir: filepath.Join(td, "output", "py", pyVariant, "text"),
|
||||
TablesDir: filepath.Join(td, "output", "go", goVariant, "tables"),
|
||||
GoSuffix: goVariant,
|
||||
}
|
||||
}
|
||||
|
||||
func envInt(key string, def int) int {
|
||||
v := os.Getenv(key)
|
||||
if v == "" {
|
||||
return def
|
||||
}
|
||||
n, err := strconv.Atoi(v)
|
||||
if err != nil {
|
||||
return def
|
||||
}
|
||||
return n
|
||||
}
|
||||
|
||||
func envStr(key, def string) string {
|
||||
v := os.Getenv(key)
|
||||
if v == "" {
|
||||
return def
|
||||
}
|
||||
return v
|
||||
}
|
||||
|
||||
// FileExists returns true if the path exists.
|
||||
func FileExists(path string) bool {
|
||||
_, err := os.Stat(path)
|
||||
return err == nil
|
||||
}
|
||||
90
internal/deepdoc/parser/pdf/tools/metadata.go
Normal file
90
internal/deepdoc/parser/pdf/tools/metadata.go
Normal file
@@ -0,0 +1,90 @@
|
||||
package tools
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"unicode/utf8"
|
||||
)
|
||||
|
||||
// ReadPythonTextMeta reads Python pipeline stage data from #@meta lines.
|
||||
func ReadPythonTextMeta(pyTextDir string) ([]PyResult, error) {
|
||||
entries, err := os.ReadDir(pyTextDir)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var results []PyResult
|
||||
for _, e := range entries {
|
||||
if !strings.HasSuffix(e.Name(), ".txt") {
|
||||
continue
|
||||
}
|
||||
data, err := os.ReadFile(filepath.Join(pyTextDir, e.Name()))
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
py := PyResult{File: strings.TrimSuffix(e.Name(), ".txt"), TextLen: utf8.RuneCount(data)}
|
||||
if idx := strings.LastIndex(string(data), "\n#@meta"); idx >= 0 {
|
||||
var meta struct {
|
||||
Chars int `json:"chars"`
|
||||
BoxesInitial int `json:"boxes_initial"`
|
||||
BoxesTextMerge int `json:"boxes_text_merge"`
|
||||
BoxesVertMerge int `json:"boxes_vertical_merge"`
|
||||
Sections int `json:"sections"`
|
||||
}
|
||||
if json.Unmarshal(data[idx+7:], &meta) == nil {
|
||||
py.Chars = meta.Chars
|
||||
py.BoxesInitial = meta.BoxesInitial
|
||||
py.BoxesTextMerge = meta.BoxesTextMerge
|
||||
py.BoxesVertMerge = meta.BoxesVertMerge
|
||||
py.Sections = meta.Sections
|
||||
py.Pages = 0
|
||||
py.TextLen = utf8.RuneCount(data[:idx])
|
||||
}
|
||||
}
|
||||
results = append(results, py)
|
||||
}
|
||||
return results, nil
|
||||
}
|
||||
|
||||
// ReadGoTextMeta reads Go pipeline stage data from #@meta lines.
|
||||
func ReadGoTextMeta(goTextDir string) ([]BatchResult, error) {
|
||||
entries, err := os.ReadDir(goTextDir)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var results []BatchResult
|
||||
for _, e := range entries {
|
||||
if !strings.HasSuffix(e.Name(), ".txt") {
|
||||
continue
|
||||
}
|
||||
data, err := os.ReadFile(filepath.Join(goTextDir, e.Name()))
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
r := BatchResult{
|
||||
File: strings.TrimSuffix(e.Name(), ".txt"),
|
||||
Pages: 1,
|
||||
TextLen: utf8.RuneCount(data),
|
||||
}
|
||||
if idx := strings.LastIndex(string(data), "\n#@meta"); idx >= 0 {
|
||||
r.TextLen = utf8.RuneCount(data[:idx]) // text only, exclude #@meta
|
||||
var meta struct {
|
||||
Chars int `json:"chars"`
|
||||
BoxesIn int `json:"boxes_initial"`
|
||||
BoxesTM int `json:"boxes_text_merge"`
|
||||
BoxesVM int `json:"boxes_vertical_merge"`
|
||||
Sections int `json:"sections"`
|
||||
}
|
||||
if json.Unmarshal(data[idx+7:], &meta) == nil {
|
||||
r.Chars = meta.Chars
|
||||
r.BoxesInitial = meta.BoxesIn
|
||||
r.BoxesTextMerg = meta.BoxesTM
|
||||
r.BoxesVertMerg = meta.BoxesVM
|
||||
r.Sections = meta.Sections
|
||||
}
|
||||
}
|
||||
results = append(results, r)
|
||||
}
|
||||
return results, nil
|
||||
}
|
||||
277
internal/deepdoc/parser/pdf/tools/similarity.go
Normal file
277
internal/deepdoc/parser/pdf/tools/similarity.go
Normal file
@@ -0,0 +1,277 @@
|
||||
package tools
|
||||
|
||||
import (
|
||||
"sort"
|
||||
"strings"
|
||||
"unicode"
|
||||
)
|
||||
|
||||
func StripMeta(s string) string {
|
||||
if idx := strings.LastIndex(s, "\n#@meta"); idx >= 0 {
|
||||
return s[:idx]
|
||||
}
|
||||
return s
|
||||
}
|
||||
|
||||
func CharSimilarity(a, b string) float64 {
|
||||
a = StripMeta(a)
|
||||
b = StripMeta(b)
|
||||
extract := func(s string) map[rune]int {
|
||||
m := make(map[rune]int)
|
||||
for _, r := range s {
|
||||
if !unicode.IsSpace(r) {
|
||||
m[r]++
|
||||
}
|
||||
}
|
||||
return m
|
||||
}
|
||||
ca, cb := extract(a), extract(b)
|
||||
if len(ca) == 0 && len(cb) == 0 {
|
||||
return 100
|
||||
}
|
||||
common, totalA, totalB := 0, 0, 0
|
||||
for r, n := range ca {
|
||||
totalA += n
|
||||
if n2, ok := cb[r]; ok {
|
||||
common += min(n, n2)
|
||||
}
|
||||
}
|
||||
for _, n := range cb {
|
||||
totalB += n
|
||||
}
|
||||
if totalA+totalB == 0 {
|
||||
return 100
|
||||
}
|
||||
return float64(common*2) / float64(totalA+totalB) * 100
|
||||
}
|
||||
|
||||
func lcsRunes(a, b []rune) int {
|
||||
if len(a) < len(b) {
|
||||
a, b = b, a
|
||||
}
|
||||
m, n := len(b), len(a)
|
||||
prev := make([]int, m+1)
|
||||
cur := make([]int, m+1)
|
||||
for i := 1; i <= n; i++ {
|
||||
for j := 1; j <= m; j++ {
|
||||
if a[i-1] == b[j-1] {
|
||||
cur[j] = prev[j-1] + 1
|
||||
} else {
|
||||
cur[j] = max(cur[j-1], prev[j])
|
||||
}
|
||||
}
|
||||
prev, cur = cur, prev
|
||||
}
|
||||
return prev[m]
|
||||
}
|
||||
|
||||
func LcsSimilarity(a, b string) float64 {
|
||||
a = StripMeta(a)
|
||||
b = StripMeta(b)
|
||||
ra := make([]rune, 0)
|
||||
for _, r := range a {
|
||||
if !unicode.IsSpace(r) {
|
||||
ra = append(ra, r)
|
||||
}
|
||||
}
|
||||
rb := make([]rune, 0)
|
||||
for _, r := range b {
|
||||
if !unicode.IsSpace(r) {
|
||||
rb = append(rb, r)
|
||||
}
|
||||
}
|
||||
if len(ra) == 0 && len(rb) == 0 {
|
||||
return 100
|
||||
}
|
||||
if len(ra) == 0 || len(rb) == 0 {
|
||||
return 0
|
||||
}
|
||||
return float64(lcsRunes(ra, rb)) / float64(max(len(ra), len(rb))) * 100
|
||||
}
|
||||
|
||||
// RawCharSimilarity is CharSimilarity without space stripping — spaces
|
||||
// count as characters. Still strips #@meta lines.
|
||||
func RawCharSimilarity(a, b string) float64 {
|
||||
a = StripMeta(a)
|
||||
b = StripMeta(b)
|
||||
ca := make(map[rune]int)
|
||||
for _, r := range a {
|
||||
ca[r]++
|
||||
}
|
||||
cb := make(map[rune]int)
|
||||
for _, r := range b {
|
||||
cb[r]++
|
||||
}
|
||||
if len(ca) == 0 && len(cb) == 0 {
|
||||
return 100
|
||||
}
|
||||
common, totalA, totalB := 0, 0, 0
|
||||
for r, n := range ca {
|
||||
totalA += n
|
||||
if n2, ok := cb[r]; ok {
|
||||
common += min(n, n2)
|
||||
}
|
||||
}
|
||||
for _, n := range cb {
|
||||
totalB += n
|
||||
}
|
||||
if totalA+totalB == 0 {
|
||||
return 100
|
||||
}
|
||||
return float64(common*2) / float64(totalA+totalB) * 100
|
||||
}
|
||||
|
||||
// RawLcsSimilarity is LcsSimilarity without space stripping — whitespace
|
||||
// is kept in the LCS comparison. Still strips #@meta lines.
|
||||
func RawLcsSimilarity(a, b string) float64 {
|
||||
a = StripMeta(a)
|
||||
b = StripMeta(b)
|
||||
ra := []rune(a)
|
||||
rb := []rune(b)
|
||||
if len(ra) == 0 && len(rb) == 0 {
|
||||
return 100
|
||||
}
|
||||
if len(ra) == 0 || len(rb) == 0 {
|
||||
return 0
|
||||
}
|
||||
return float64(lcsRunes(ra, rb)) / float64(max(len(ra), len(rb))) * 100
|
||||
}
|
||||
|
||||
// SectionAlignedScore computes a two-phase LCS similarity:
|
||||
//
|
||||
// Phase 1: One-to-one section matching — pair Go and Python sections by
|
||||
// CharSimilarity (greedy, highest first). For matched pairs, compute
|
||||
// per-section LCS ratio.
|
||||
//
|
||||
// Phase 2: Residual — concatenate all unmatched sections from both sides
|
||||
// into one string each, compute LCS ratio once. This handles cases where
|
||||
// one side merges sections that the other side keeps separate.
|
||||
//
|
||||
// Final score is a char-weighted average of matched and residual scores.
|
||||
func SectionAlignedScore(goText, pyText string) float64 {
|
||||
split := func(s string) []string {
|
||||
s = StripMeta(s)
|
||||
return strings.Split(strings.TrimSpace(s), "\n")
|
||||
}
|
||||
gs := split(goText)
|
||||
ps := split(pyText)
|
||||
if len(gs) == 0 && len(ps) == 0 {
|
||||
return 100
|
||||
}
|
||||
if len(gs) == 0 || len(ps) == 0 {
|
||||
return 0
|
||||
}
|
||||
|
||||
// Phase 1: Position-window greedy matching.
|
||||
// Sections are ordered top-to-bottom by page position, so a global
|
||||
// match beyond a small positional offset is extremely unlikely.
|
||||
// Constrain candidates to ±window to avoid O(n×m) blow-up on large docs.
|
||||
const alignWindow = 5
|
||||
type candidate struct {
|
||||
gi, pi int
|
||||
sim float64
|
||||
}
|
||||
// Precompute rune lengths for length-ratio gating.
|
||||
glens := make([]int, len(gs))
|
||||
plens := make([]int, len(ps))
|
||||
for i, s := range gs {
|
||||
glens[i] = len([]rune(s))
|
||||
}
|
||||
for i, s := range ps {
|
||||
plens[i] = len([]rune(s))
|
||||
}
|
||||
|
||||
candidates := make([]candidate, 0, len(gs)*(alignWindow*2+1))
|
||||
for i, g := range gs {
|
||||
lo := max(0, i-alignWindow)
|
||||
hi := min(len(ps)-1, i+alignWindow)
|
||||
for j := lo; j <= hi; j++ {
|
||||
// Skip pairs with >2x length difference — a 500-char section
|
||||
// matching a 30-char section produces near-zero LCS.
|
||||
if glens[i] > plens[j]*2 || plens[j] > glens[i]*2 {
|
||||
continue
|
||||
}
|
||||
if sim := CharSimilarity(g, ps[j]); sim > 30 {
|
||||
candidates = append(candidates, candidate{i, j, sim})
|
||||
}
|
||||
}
|
||||
}
|
||||
// Sort descending by similarity — best matches first.
|
||||
sort.Slice(candidates, func(a, b int) bool {
|
||||
return candidates[a].sim > candidates[b].sim
|
||||
})
|
||||
|
||||
goUsed := make([]bool, len(gs))
|
||||
pyUsed := make([]bool, len(ps))
|
||||
matchedScore := 0.0
|
||||
matchedChars := 0
|
||||
|
||||
for _, c := range candidates {
|
||||
if goUsed[c.gi] || pyUsed[c.pi] {
|
||||
continue
|
||||
}
|
||||
goUsed[c.gi] = true
|
||||
pyUsed[c.pi] = true
|
||||
|
||||
// Compute LCS ratio for matched pair.
|
||||
ra := nonSpaceRunes(gs[c.gi])
|
||||
rb := nonSpaceRunes(ps[c.pi])
|
||||
lcsScore := 0.0
|
||||
if len(ra) > 0 && len(rb) > 0 {
|
||||
lcsScore = float64(lcsRunes(ra, rb)) / float64(max(len(ra), len(rb))) * 100
|
||||
} else if len(ra) == 0 && len(rb) == 0 {
|
||||
lcsScore = 100
|
||||
}
|
||||
chars := max(len(ra), len(rb))
|
||||
matchedScore += lcsScore * float64(chars)
|
||||
matchedChars += chars
|
||||
}
|
||||
|
||||
// Phase 2: Residual — concat unmatched sections, compute LCS once.
|
||||
var goRes, pyRes strings.Builder
|
||||
for i, g := range gs {
|
||||
if !goUsed[i] {
|
||||
goRes.WriteString(g)
|
||||
goRes.WriteByte(' ')
|
||||
}
|
||||
}
|
||||
for j, p := range ps {
|
||||
if !pyUsed[j] {
|
||||
pyRes.WriteString(p)
|
||||
pyRes.WriteByte(' ')
|
||||
}
|
||||
}
|
||||
|
||||
residualScore := 0.0
|
||||
residualChars := 0
|
||||
goResRunes := nonSpaceRunes(goRes.String())
|
||||
pyResRunes := nonSpaceRunes(pyRes.String())
|
||||
residualChars = max(len(goResRunes), len(pyResRunes))
|
||||
if residualChars > 0 {
|
||||
if len(goResRunes) > 5000 || len(pyResRunes) > 5000 {
|
||||
// Residual too large for O(n²) LCS — fall back to CharSimilarity.
|
||||
residualScore = CharSimilarity(goRes.String(), pyRes.String())
|
||||
} else {
|
||||
residualScore = float64(lcsRunes(goResRunes, pyResRunes)) / float64(residualChars) * 100
|
||||
}
|
||||
} else if len(goResRunes) == 0 && len(pyResRunes) == 0 {
|
||||
residualScore = 100
|
||||
}
|
||||
|
||||
// Weighted average.
|
||||
totalChars := matchedChars + residualChars
|
||||
if totalChars == 0 {
|
||||
return 100
|
||||
}
|
||||
return (matchedScore + residualScore*float64(residualChars)) / float64(totalChars)
|
||||
}
|
||||
|
||||
func nonSpaceRunes(s string) []rune {
|
||||
out := make([]rune, 0, len(s))
|
||||
for _, r := range s {
|
||||
if !unicode.IsSpace(r) {
|
||||
out = append(out, r)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
70
internal/deepdoc/parser/pdf/tools/types.go
Normal file
70
internal/deepdoc/parser/pdf/tools/types.go
Normal file
@@ -0,0 +1,70 @@
|
||||
package tools
|
||||
|
||||
// BatchResult stores per-PDF pipeline stage output.
|
||||
type BatchResult struct {
|
||||
File string `json:"file"`
|
||||
Pages int `json:"pages"`
|
||||
Chars int `json:"chars"`
|
||||
BoxesInitial int `json:"boxes_initial"`
|
||||
BoxesTextMerg int `json:"boxes_text_merge"`
|
||||
BoxesVertMerg int `json:"boxes_vertical_merge"`
|
||||
Sections int `json:"sections"`
|
||||
TSTables int `json:"tsr_tables,omitempty"`
|
||||
TextLen int `json:"text_len"`
|
||||
TimeS float64 `json:"time_s"`
|
||||
Error string `json:"error,omitempty"`
|
||||
}
|
||||
|
||||
// PyResult mirrors Python dump_py_results.py output.
|
||||
type PyResult struct {
|
||||
File string `json:"file"`
|
||||
Pages int `json:"pages"`
|
||||
Chars int `json:"chars"`
|
||||
BoxesInitial int `json:"boxes_initial"`
|
||||
BoxesTextMerge int `json:"boxes_text_merge"`
|
||||
BoxesVertMerge int `json:"boxes_vertical_merge"`
|
||||
Sections int `json:"sections"`
|
||||
Tables int `json:"tables"`
|
||||
TextLen int `json:"text_len"`
|
||||
IsEnglish *bool `json:"is_english"`
|
||||
TimeS float64 `json:"time_s"`
|
||||
Error string `json:"error,omitempty"`
|
||||
}
|
||||
|
||||
// TableItem stores per-table output.
|
||||
type TableItem struct {
|
||||
ImageB64 string `json:"image_b64"`
|
||||
Rows [][]string `json:"rows"`
|
||||
Cells []TSRCell `json:"cells,omitempty"`
|
||||
Positions []Position `json:"positions"`
|
||||
}
|
||||
|
||||
// TSRCell mirrors parser.TSRCell for serialization.
|
||||
type TSRCell struct {
|
||||
X0, Y0, X1, Y1 float64 `json:"x0,y0,x1,y1"`
|
||||
Text string `json:"text"`
|
||||
Label string `json:"label"`
|
||||
}
|
||||
|
||||
// Position stores a bounding box.
|
||||
type Position struct {
|
||||
Left, Right, Top, Bottom float64
|
||||
}
|
||||
|
||||
// RealPDFResult holds per-PDF stats for Go vs Python comparison.
|
||||
type RealPDFResult struct {
|
||||
File string `json:"file"`
|
||||
Pages int `json:"pages"`
|
||||
Chars int `json:"chars"`
|
||||
Sections int `json:"sections"`
|
||||
TextLen int `json:"text_len"`
|
||||
Error string `json:"error,omitempty"`
|
||||
}
|
||||
|
||||
// TLogger is a minimal interface for logging in comparison functions.
|
||||
type TLogger interface {
|
||||
Logf(format string, args ...any)
|
||||
Errorf(format string, args ...any)
|
||||
Fatalf(format string, args ...any)
|
||||
Skipf(format string, args ...any)
|
||||
}
|
||||
Reference in New Issue
Block a user