")
totalTdTh := strings.Count(html, " 1.5 × median_height ≈ 15pt).
// Each figure text box → separate section in result.Sections.
// CollectFigures collects them into result.Figures but doesn't re-insert.
var figureSections []Section
for _, s := range result.Sections {
if s.LayoutType == "figure" {
figureSections = append(figureSections, s)
}
}
// Assert 1: Python expects exactly 1 consolidated figure section.
// Go currently produces 2 (one per unmerged text box) — this FAILS.
if len(figureSections) != 1 {
t.Errorf("FIGURE INSERTION BUG: expected 1 consolidated figure section (Python insert_table_figures), got %d. Go does not consolidate figure text boxes into a single block.", len(figureSections))
}
// Assert 2: The single figure section must contain BOTH text fragments.
if len(figureSections) == 1 {
combined := figureSections[0].Text
if !strings.Contains(combined, "架构图") || !strings.Contains(combined, "系统模块") {
t.Errorf("FIGURE INSERTION BUG: figure section text=%q should contain both fragments. Python merges all figure-region text.", combined)
}
}
t.Logf("figure sections in Sections: %d", len(figureSections))
t.Logf("result.Figures count: %d", len(result.Figures))
t.Logf("result.Sections total: %d", len(result.Sections))
for i, s := range result.Sections {
t.Logf(" section[%d] layout=%q text=%q", i, s.LayoutType, s.Text)
}
}
// =============================================================================
// Issue 3: Multi-page table merging
// Python's _extract_table_figure merges tables with same layoutno across
// consecutive pages (gap ≤ 1 page, Y-dis ≤ 23× median height).
// Go's extractTableAndReplace does NOT merge tables across pages.
// =============================================================================
// TestExtractTableAndReplace_NoCrossPageMerge exposes that extractTableAndReplace
// does not merge tables from consecutive pages even with the same layoutno.
func TestExtractTableAndReplace_NoCrossPageMerge(t *testing.T) {
// Simulate a table spanning pages 0 and 1.
// Python would merge these because: same layoutno, consecutive pages,
// Y-distance ≤ 23× median_height.
boxes := []TextBox{
{X0: 10, X1: 200, Top: 500, Bottom: 530, Text: "续表内容", LayoutType: "table", PageNumber: 0, LayoutNo: "0"},
{X0: 10, X1: 200, Top: 50, Bottom: 80, Text: "表尾内容", LayoutType: "table", PageNumber: 1, LayoutNo: "0"},
}
// Two separate TableItems — one per page. Python would merge these
// before insert_table_figures.
tables := []TableItem{
{
Cells: []TSRCell{{X0: 0, Y0: 0, X1: 100, Y1: 30, Text: "Page0", Label: "table row"}},
Positions: []Position{{PageNumbers: []int{0}, Left: 0, Right: 300, Top: 500, Bottom: 530}},
Scale: 1.0,
},
{
Cells: []TSRCell{{X0: 0, Y0: 0, X1: 100, Y1: 30, Text: "Page1", Label: "table row"}},
Positions: []Position{{PageNumbers: []int{1}, Left: 0, Right: 300, Top: 50, Bottom: 80}},
Scale: 1.0,
},
}
result := extractTableAndReplace(boxes, tables)
// Go produces 2 separate HTML table boxes (one per page).
// Python would produce 1 merged table with cells from both pages.
tableCount := 0
for _, b := range result {
if strings.Contains(b.Text, "") {
tableCount++
}
}
if tableCount == 2 {
t.Errorf("CROSS-PAGE TABLE MERGE BUG: got %d separate HTML tables across pages. Python would merge same-layoutno tables on consecutive pages into 1 consolidated table.", tableCount)
}
t.Logf("table HTML boxes: %d (Python would merge into 1)", tableCount)
}
// =============================================================================
// Issue 3a: nomerge_lout_no — don't merge tables separated by captions
// Python's _extract_table_figure tracks nomerge_lout_no: when a table box
// is followed by a caption/title/reference, the table's key is added to
// nomerge_lout_no. Later, cross-page merge skips tables in nomerge_lout_no.
//
// Example:
// Page 0: table "0-table-3" → caption "表1：..." → table "0-table-4"
// Page 1: table "1-table-3" (same layoutNo)
// → Page 0's table-3 should NOT merge with Page 1's table-3,
// because the caption on page 0 indicates the table ended.
// → Go's mergeTablesAcrossPages has no nomerge_lout_no check.
// =============================================================================
// TestMergeTablesAcrossPages_NomergeAfterCaption_Missing exposes that
// mergeTablesAcrossPages unconditionally merges consecutive-page tables,
// even when Python's nomerge_lout_no would prevent it.
func TestMergeTablesAcrossPages_NomergeAfterCaption_Missing(t *testing.T) {
// Simulate: page 0 has table at top, followed by a caption,
// then another table. Page 1 has the same-layoutNo table continuing.
// In Python, page 0's first table goes into nomerge_lout_no because
// the next box is a caption → no cross-page merge for that table group.
tables := []TableItem{
{
Cells: []TSRCell{{Text: "Page0-first", Label: "table row"}},
Positions: []Position{{
PageNumbers: []int{0},
Left: 0, Right: 300,
Top: 0, Bottom: 50,
}},
NoMerge: true, // Set when caption follows this table on the page
},
{
Cells: []TSRCell{{Text: "Page1-cont", Label: "table row"}},
Positions: []Position{{
PageNumbers: []int{1},
Left: 0, Right: 300,
Top: 0, Bottom: 50,
}},
},
}
result := mergeTablesAcrossPages(tables, nil)
// Verify NoMerge prevents cross-page merging.
if len(result) != 2 {
t.Errorf("NOMERGE BUG: expected 2 separate table groups, got %d.", len(result))
}
t.Log("NoMerge flag correctly prevents cross-page merge.")
}
// =============================================================================
// Issue 3b: insert position — min_rectangle_distance vs anchor
// Python's insert_table_figures uses min_rectangle_distance to find the
// spatially nearest text box and inserts the table/figure next to it.
// Go's extractTableAndReplace uses the first replaced table box index as
// the anchor (insert position).
//
// When the DLA table region extends beyond the anchor box's bottom and
// overlaps a text box below the table, Python puts the table next to that
// overlapping text box (distance=0); Go puts it at the anchor position.
// =============================================================================
// TestExtractTableAndReplace_InsertionPosition_DistanceBug exposes that
// extractTableAndReplace uses the first table box as anchor, rather than
// finding the spatially nearest text box like Python.
func TestExtractTableAndReplace_InsertionPosition_DistanceBug(t *testing.T) {
// Two text boxes above the table: L0 (left, near table) and R0 (right, far).
// Python: nearest to table is L0 (dx=0, dy=70). L0 bottom=30 < table top=100
// → insert AFTER L0. Result: [L0, table, R0, R1, L2].
// Go: anchor = first table box (L1 at index 2). Result: [L0, R0, table, R1, L2].
// The table is one position off.
boxes := []TextBox{
{X0: 10, X1: 100, Top: 10, Bottom: 30, Text: "L0", LayoutType: "text", PageNumber: 0},
{X0: 300, X1: 400, Top: 10, Bottom: 30, Text: "R0", LayoutType: "text", PageNumber: 0},
{X0: 10, X1: 100, Top: 100, Bottom: 130, Text: "table", LayoutType: "table", PageNumber: 0},
{X0: 300, X1: 400, Top: 100, Bottom: 130, Text: "R1", LayoutType: "text", PageNumber: 0},
{X0: 10, X1: 100, Top: 250, Bottom: 270, Text: "L2", LayoutType: "text", PageNumber: 0},
}
tables := []TableItem{{
Cells: []TSRCell{{Text: "cell", Label: "table row"}},
Positions: []Position{{Left: 10, Right: 100, Top: 100, Bottom: 130, PageNumbers: []int{0}}},
Scale: 1.0,
RegionLeft: 10, RegionRight: 100, RegionTop: 100, RegionBottom: 130,
}}
result := extractTableAndReplace(boxes, tables)
// Find L0 and table positions.
l0Idx, tableIdx := -1, -1
for i, b := range result {
if strings.TrimSpace(b.Text) == "L0" {
l0Idx = i
}
if b.LayoutType == "table" {
tableIdx = i
}
}
// BUG: table should immediately follow L0 (nearest neighbor, insert_after).
// Python: min_rectangle_distance → L0 nearest (dx=0, dy=70), L0 below table
// → insert_at+1 → table right after L0.
// Go: anchor = first table box index → table at original table box position.
if tableIdx != l0Idx+1 {
t.Errorf("INSERTION POSITION BUG: table (idx=%d) should immediately follow L0 (idx=%d). "+
"Python's min_rectangle_distance finds L0 as nearest text box and inserts table after it. "+
"Go anchors at first table box position (between R0 and R1).", tableIdx, l0Idx)
}
t.Logf("L0 at idx=%d, table at idx=%d", l0Idx, tableIdx)
t.Log("Fix: replace first-replaced-box anchor with min_rectangle_distance nearest-neighbor (Python pdf_parser.py:1608-1655).")
}
// =============================================================================
// Issue 4: page_cum_height coordinate system
// Python tracks cumulative page image heights for cross-page position tags
// and image cropping. Go uses per-page coordinates only.
// =============================================================================
// TestBoxesToSections_PerPageCoordinates confirms position tags use
// page-relative coordinates. Python's _line_tag also produces local
// coordinates (subtracts page_cum_height). The page number differentiates
// pages; page_cum_height is an internal implementation detail.
func TestBoxesToSections_PerPageCoordinates(t *testing.T) {
boxes := []TextBox{
{X0: 10, X1: 100, Top: 40, Bottom: 60, Text: "Page 0 text", LayoutType: "text", PageNumber: 0},
{X0: 10, X1: 100, Top: 40, Bottom: 60, Text: "Page 1 text", LayoutType: "text", PageNumber: 1},
}
sections := boxesToSections(boxes, nil)
if len(sections) != 2 {
t.Fatalf("expected 2 sections, got %d", len(sections))
}
s0, s1 := sections[0], sections[1]
if len(s0.Positions) > 0 && len(s1.Positions) > 0 {
p0, p1 := s0.Positions[0], s1.Positions[0]
// Both Python and Go use local (page-relative) coordinates.
// Python's _line_tag: top = bx["top"] - page_cum_height[pn-1]
// gives local coordinate. Same as Go.
if p0.Top != p1.Top || p0.Bottom != p1.Bottom {
t.Errorf("expected same local coords, got Top=(%.0f,%.0f) Bottom=(%.0f,%.0f)", p0.Top, p1.Top, p0.Bottom, p1.Bottom)
}
t.Logf("page 0: Page=%v Top=%.0f Bottom=%.0f", p0.PageNumbers, p0.Top, p0.Bottom)
t.Logf("page 1: Page=%v Top=%.0f Bottom=%.0f", p1.PageNumbers, p1.Top, p1.Bottom)
t.Log("OK: position tags use page-relative coordinates in both Go and Python.")
}
}
// =============================================================================
// Issue 6: cropSectionImage padding logic
// Python's self.crop adds 120px context above first segment, 120px context
// below last segment, 6px gap between pages, and overlay transparency.
// Go has simpler crop logic.
// =============================================================================
// TestCropSectionImage_PaddingVsPython documents that Go's cropSectionImage
// adds context padding differently from Python's self.crop.
func TestCropSectionImage_PaddingVsPython(t *testing.T) {
// Create a page image and position tag for a small text region.
img := image.NewRGBA(image.Rect(0, 0, 300, 800)) // 300×800 page at zoom=3 → PDF 100×267
pageImages := map[int]image.Image{0: img}
// Position tag for a small text box near the top of the page.
posTag := FormatPositionTag(0, 50.0, 100.0, 10.0, 30.0)
result := cropSectionImage(posTag, pageImages, 3.0)
if result == "" {
t.Error("cropSectionImage returned empty string for valid position")
}
// Decode result to check image dimensions.
data, err := base64.StdEncoding.DecodeString(result)
if err != nil {
t.Fatalf("failed to decode base64: %v", err)
}
cropped, _, err := image.Decode(bytes.NewReader(data))
if err != nil {
t.Fatalf("failed to decode PNG: %v", err)
}
croppedH := cropped.Bounds().Dy()
// Original text region: Top=10, Bottom=30 → height=20 at PDF points.
// zoom=3 → 60px text height.
// Python adds 120px context above + 120px below + 6px gap → ~306px.
// Go adds contextPad=120 points above/below at PDF scale → with zoom=3: 360+60+360=780px.
// Python uses pixel-space padding (120px literally), Go uses PDF-point padding (120pt).
expectedMin := 60 // bare minimum: text region itself
if croppedH <= expectedMin {
t.Errorf("CROP PADDING BUG: cropped image height=%dpx, expected >%dpx with context padding. Python adds 120px above and below for context.", croppedH, expectedMin)
}
t.Logf("cropped image: %dx%d (text region 60px, expecting padding)", cropped.Bounds().Dx(), croppedH)
t.Log("NOTE: Python's self.crop adds 120px context padding in pixel space, multi-page stitching, and overlay transparency. Go's cropSectionImage uses PDF-point padding and simpler stitching.")
}
// =============================================================================
// Issue 7: Data-source filter missing
// Python's _extract_table_figure pops table/figure boxes matching
// r"(数据|资料|图表)*来源[:： ]" (pdf_parser.py:1040-1042, 1050-1052).
// These boxes are discarded — not extracted, not inserted back.
// Go has no equivalent filter in extractTableAndReplace or consolidateFigures.
// =============================================================================
// dataSourcePattern is a Go translation of Python's
// r"(数据|资料|图表)*来源[:： ]" used with re.match (anchored at start).
var dataSourcePattern = `^(数据|资料|图表)*来源[:： ]`
// TestDataSourcePattern_RegexCoverage validates the Python regex behavior
// that should be adopted. Documents which strings match and which don't.
func TestDataSourcePattern_RegexCoverage(t *testing.T) {
tests := []struct {
text string
want bool // Python re.match truthiness
}{
// ── Matching patterns (should be filtered) ──
{"数据来源：国家统计局", true}, // 数据 + 来源 + fullwidth colon
{"资料来源: 某报告", true}, // 资料 + 来源 + halfwidth colon
{"图表来源：某数据库", true}, // 图表 + 来源 + fullwidth colon
{"来源：权威机构", true}, // zero prefix + 来源 + fullwidth colon
{"来源: 参考数据", true}, // zero prefix + 来源 + halfwidth colon
{"数据来源 说明", true}, // 数据 + 来源 + space
// ── Non-matching patterns (should NOT be filtered) ──
{"数据来源明细", false}, // 来源 followed by 明, not :：space
{"普通来源说明", false}, // doesn't start with keyword
{"数据", false}, // too short
{"来源", false}, // 来源 but no :：space after
{"资料来源说明", false}, // 来源 followed by 说, not :：space
{"", false}, // empty
{"TABLE 1: 数据来源统计", false}, // doesn't start with keyword
}
for _, tt := range tests {
matched := regexp.MustCompile(dataSourcePattern).MatchString(tt.text)
if matched != tt.want {
t.Errorf("dataSourcePattern.MatchString(%q) = %v, want %v", tt.text, matched, tt.want)
}
}
t.Log("NOTE: Python re.match(r\"(数据|资料|图表)*来源[:： ]\", text) — anchored at start.")
t.Log("Go regexp.MatchString equivalent with ^ prefix.")
}
// TestExtractTableAndReplace_DataSourceFilter_Missing exposes that Go does NOT
// filter out table boxes whose text matches r"(数据|资料|图表)*来源[:： ]".
// Python's _extract_table_figure pops these boxes from self.boxes without
// adding them to the tables dict (pdf_parser.py:1040-1042).
func TestExtractTableAndReplace_DataSourceFilter_Missing(t *testing.T) {
// A table box with data-source text and a normal table box.
// Both overlap a TableItem position, so both would be replaced with HTML.
boxes := []TextBox{
{X0: 10, X1: 200, Top: 0, Bottom: 50, Text: "数据来源：国家统计局", LayoutType: "table", PageNumber: 0},
{X0: 10, X1: 200, Top: 60, Bottom: 80, Text: "表1：正常数据", LayoutType: "table", PageNumber: 0},
}
// Two TableItems — one per table box — so each would independently produce HTML.
tables := []TableItem{
{
Cells: []TSRCell{{X0: 0, Y0: 0, X1: 100, Y1: 30, Text: "来源", Label: "table row"}},
Positions: []Position{{Left: 0, Right: 300, Top: 0, Bottom: 50}},
Scale: 1.0,
},
{
Cells: []TSRCell{{X0: 0, Y0: 0, X1: 100, Y1: 30, Text: "正常", Label: "table row"}},
Positions: []Position{{Left: 0, Right: 300, Top: 60, Bottom: 80}},
Scale: 1.0,
},
}
result := extractTableAndReplace(boxes, tables)
// Python behavior: "数据来源：国家统计局" is popped from self.boxes,
// NOT added to tables dict, NOT replaced with HTML. Gone entirely.
// "表1：正常数据" is replaced with HTML as usual.
// Expected result: exactly 1 HTML table box for the normal table.
//
// BUG: Go replaces both boxes with HTML tables. The data-source box
// produces an HTML table with cell text "来源" — this should NOT exist.
htmlTableCount := 0
hasDataSourceTable := false
for _, b := range result {
if strings.Contains(b.Text, "") {
htmlTableCount++
// The data-source table's cell text "来源" ends up in the HTML.
// c.f. constructTable which uses TSRCell text, not box text.
if strings.Contains(b.Text, ">来源<") {
hasDataSourceTable = true
}
}
}
if htmlTableCount != 1 {
t.Errorf("DATA SOURCE FILTER BUG: expected 1 HTML table (normal only), got %d. Python pops data-source table box entirely in _extract_table_figure (pdf_parser.py:1040-1042). Go replaces it with an HTML table.", htmlTableCount)
}
if hasDataSourceTable {
t.Errorf("DATA SOURCE FILTER BUG: data-source table should NOT produce HTML output. Cell '来源' appears in HTML: Python discards these boxes, Go incorrectly constructs a table for them.")
}
t.Log("NOTE: Python filters table boxes matching r\"(数据|资料|图表)*来源[:： ]\" in _extract_table_figure.")
t.Log("Go's extractTableAndReplace has no equivalent filter — data-source boxes get replaced with HTML instead of being discarded.")
}
// TestExtractTableAndReplace_DataSourceVariants tests multiple variants of
// the data-source pattern that should all be filtered.
func TestExtractTableAndReplace_DataSourceVariants(t *testing.T) {
variants := []string{
"数据来源：国家统计局",
"资料来源: 某报告",
"图表来源：某数据库",
"来源：权威机构",
"来源: 参考数据",
}
for _, variant := range variants {
t.Run(variant, func(t *testing.T) {
boxes := []TextBox{
{X0: 10, X1: 200, Top: 0, Bottom: 50, Text: variant, LayoutType: "table", PageNumber: 0},
}
tables := []TableItem{{
Cells: []TSRCell{{X0: 0, Y0: 0, X1: 100, Y1: 30, Text: "A", Label: "table row"}},
Positions: []Position{{Left: 0, Right: 300, Top: 0, Bottom: 50}},
Scale: 1.0,
}}
result := extractTableAndReplace(boxes, tables)
// BUG: box with data-source text should be REMOVED entirely —
// zero HTML output. Python pops these boxes without replacement.
for _, b := range result {
if strings.Contains(b.Text, "") {
t.Errorf("DATA SOURCE FILTER BUG: variant %q should be removed without HTML replacement. Python pops data-source table boxes entirely.", variant)
}
}
})
}
t.Log("NOTE: All variants of r\"(数据|资料|图表)*来源[:： ]\" should be filtered by extractTableAndReplace.")
}
// TestConsolidateFigures_DataSourceFilter_Missing exposes that Go does NOT
// filter out figure boxes whose text matches r"(数据|资料|图表)*来源[:： ]".
// Python's _extract_table_figure pops these boxes from self.boxes without
// adding them to the figures dict (pdf_parser.py:1050-1052).
func TestConsolidateFigures_DataSourceFilter_Missing(t *testing.T) {
boxes := []TextBox{
{X0: 10, X1: 200, Top: 0, Bottom: 50, Text: "数据来源：某机构", LayoutType: "figure", PageNumber: 0, LayoutNo: "figure-0"},
{X0: 10, X1: 200, Top: 60, Bottom: 80, Text: "架构图", LayoutType: "figure", PageNumber: 0, LayoutNo: "figure-0"},
}
result := consolidateFigures(boxes)
// Python behavior: "数据来源：某机构" is popped from self.boxes,
// NOT added to figures dict → gone entirely.
// "架构图" is extracted normally.
// Expected result: exactly 1 figure box with "架构图" text only.
for _, b := range result {
if strings.Contains(b.Text, "数据来源") || strings.Contains(b.Text, "某机构") {
t.Errorf("DATA SOURCE FIGURE FILTER BUG: '数据来源：某机构' figure box should be removed entirely. Python pops data-source figure boxes in _extract_table_figure (pdf_parser.py:1050-1052). Go still includes it.")
}
}
// Verify the normal figure box IS still present.
foundFigure := false
for _, b := range result {
if strings.Contains(b.Text, "架构图") {
foundFigure = true
}
}
if !foundFigure {
t.Error("normal figure box '架构图' should still be present")
}
t.Log("NOTE: Python filters figure boxes matching r\"(数据|资料|图表)*来源[:： ]\" in _extract_table_figure.")
t.Log("Go's consolidateFigures has no equivalent filter.")
}
|