From 7646afab1ba476395ae8d5d247ede1f6470717fc Mon Sep 17 00:00:00 2001 From: Jack Date: Wed, 12 Aug 2026 17:59:50 +0800 Subject: [PATCH] fix(pdf/layout): recover title-bridged & gutter-less double columns (2D rescue + L3) (#18150) Incremental follow-up to #18023 (gap + balance-gate hybrid). Adds two complementary column detectors to `AssignColumn` that run **only after** the gap detector and the balance gate both fail, so already-correct pages are never touched. --- .../parser/pdf/layout/combined_column.go | 486 +++++++++++++++++- .../layout/combined_column_defensive_test.go | 160 ++++++ .../layout/combined_column_synthetic_test.go | 88 +++- 3 files changed, 718 insertions(+), 16 deletions(-) create mode 100644 internal/deepdoc/parser/pdf/layout/combined_column_defensive_test.go diff --git a/internal/deepdoc/parser/pdf/layout/combined_column.go b/internal/deepdoc/parser/pdf/layout/combined_column.go index fe1b0fcc3c..9ca6ce51c0 100644 --- a/internal/deepdoc/parser/pdf/layout/combined_column.go +++ b/internal/deepdoc/parser/pdf/layout/combined_column.go @@ -4,6 +4,7 @@ import ( "math" "math/rand" "sort" + "unicode/utf8" pdf "ragflow/internal/deepdoc/parser/pdf/type" util "ragflow/internal/deepdoc/parser/pdf/util" @@ -74,6 +75,33 @@ const maxColumnCount = 4 // touching those. const minColLineFrac = 0.12 +// maxPageExtent caps the X span a single line may plausibly occupy. A line +// wider than this is treated as a malformed coordinate (mirrors pdf-inspector's +// MAX_PAGE_EXTENT=14400 guard) and excluded from the column projection so it +// cannot balloon the page extent and collapse multi-column detection. +const maxPageExtent = 14400.0 + +// maxTrimFraction is the largest fraction of lines robustPageExtent may discard +// as outliers. If more than this fraction is anomalous, the page is trusted +// as-is: the "anomalies" are the norm, not noise. +const maxTrimFraction = 0.10 + +// maxBins caps the histogram allocation in gapColumnCount/detectColumnCount2D so +// a malformed (ballooned) page extent cannot trigger an OOM-scale allocation +// (mirrors pdf-inspector's bin cap). When the extent is huge, the bin is +// widened so the projection still resolves real gutters. +const maxBins = 65536 + +// gapMinFrac: a horizontal run of low coverage counts as a column gap only if +// it is at least this fraction of the page width. Reused by both gapColumnCount +// and the 2D both-sides gutter rescue so the two detectors agree on what a +// "real" gutter width is. +const gapMinFrac = 0.04 + +// binPt: x-binning resolution (points) for the 1D gap histogram and the 2D +// gutter scan. Sharing it keeps the gap and gutter detectors aligned. +const binPt = 2.0 + // detectColumnCount returns (columnCount, centroids) for one page. // columnCount is 1, 2, or up to maxColumnCount; centroids are the k cluster // means in x0 space (snapshot of the gate decision) and are reused for ColID @@ -83,7 +111,7 @@ func detectColumnCount(boxes []pdf.TextBox, indices []int) (int, []float64) { for i, idx := range indices { lines[i] = boxes[idx] } - g := gapColumnCount(lines, 0.04, 0.15, 2.0) + g := gapColumnCount(lines, gapMinFrac, 0.15, binPt) if g >= 2 { _, width := pageExtent(lines) if width > 0 { @@ -133,6 +161,20 @@ func detectColumnCount(boxes []pdf.TextBox, indices []int) (int, []float64) { } return 1, nil } + // 2D rescue: a clean vertical gutter the 1D projection masks via bridging + // rows (full-width front matter + in-body headings/captions). Recovers + // title-bridged doubles the balance gate correctly rejects (sparse + // minority). Runs only after both gap>=2 and the balance gate fail, so it + // never touches already-correct pages. + if k, cents := detectColumnCount2D(lines); k >= 2 { + return k, cents + } + // L3: median-width-ratio complement (PR #10475). Fires only after gap, + // balance, and the 2D valley rescue all returned 1, so it never touches + // the pages they already handle. Targets "gutter-less" doubles/triples. + if k, cents := detectColumnCountMedian(lines); k >= 2 { + return k, cents + } return 1, nil } @@ -231,19 +273,38 @@ func gapColumnCount(lines []pdf.TextBox, gapMinFrac, crossTol, binPt float64) in if n == 0 { return 1 } - minX0, width := pageExtent(lines) + // A1: image/equation placeholders must not feed the projection — a figure + // spanning the gutter would otherwise fill the gap and mask a real column + // boundary. + lines = textProjectionLines(lines) + if len(lines) == 0 { + return 1 + } + // A2: robust page extent discards malformed/outlier lines so a single bad + // box cannot balloon the width and collapse detection to one column. + minX0, width := robustPageExtent(lines) if width <= 0 { return 1 } minGap := gapMinFrac * width - nb := int(width/binPt) + 1 + // Cap the bin count so a ballooned extent cannot allocate an OOM-scale + // histogram. When the extent is huge, widen the bin so the projection still + // resolves real gutters. + effBin := binPt + if width/float64(maxBins) > effBin { + effBin = width / float64(maxBins) + } + nb := int(width/effBin) + 1 + if nb < 1 { + nb = 1 + } cov := make([]int, nb) for _, b := range lines { - i0 := int((b.X0 - minX0) / binPt) + i0 := int((b.X0 - minX0) / effBin) if i0 < 0 { i0 = 0 } - i1 := int((b.X1 - minX0) / binPt) + i1 := int((b.X1 - minX0) / effBin) if i1 > nb-1 { i1 = nb - 1 } @@ -256,7 +317,7 @@ func gapColumnCount(lines []pdf.TextBox, gapMinFrac, crossTol, binPt float64) in run := 0.0 for _, c := range cov { if float64(c) < thr { - run += binPt + run += effBin } else { if run >= minGap { cols++ @@ -270,6 +331,323 @@ func gapColumnCount(lines []pdf.TextBox, gapMinFrac, crossTol, binPt float64) in return cols } +// bridgingFrac: lines wider than this fraction of the page text width are +// treated as bridging elements — full-width front matter (already dropped by +// dropFullWidth at 0.9) plus partially-wide in-body headings/captions that +// span the gutter. Dropping them before the valley scan is what exposes the +// clean gutter of a title-bridged double column. 0.60 is the sweet spot +// measured on the 70-page corpus: lower (0.50) leaves too few real column +// lines on single pages and keeps enough bridging width to still hide some +// gutters; higher (0.65) lets the sparse bridging lines that hide the target +// gutters survive. +const bridgingFrac = 0.60 + +// medianFullWidthFrac is the full-width threshold for the L3 median-width +// detector. It is lower than dropFullWidth's 0.9 because the median path +// buckets by normalized center x and only needs to exclude lines that would +// otherwise dominate every bucket; lines between 0.8 and 0.9 width are rare +// and keeping them out of the buckets avoids a single wide line skewing cents. +const medianFullWidthFrac = 0.80 + +// medianColCap: max column count the median-width-ratio signal (L3, from PR +// #10475's page_w/median_w) may assign. Capped at 3 so a raw_cols estimate of +// 4 (common on 3-column pages) does not over-shoot, and well under +// maxColumnCount. +const medianColCap = 3 + +// shortLineFrac: L3 requires at least one line spanning >= this fraction of +// the page width. A real multi-column page has lines that span a column +// (~page_w/N); a single page of uniformly short lines also has a small median +// width, but no near-full-width line — this guard filters those false doubles. +const shortLineFrac = 0.45 + +// detectColumnCount2D is a rescue detector for title-bridged double columns: +// pages whose two body columns are separated by a clean gutter that the 1D x0 +// projection loses once full-width front matter (and in-body bridging +// headings/captions) spans it. It runs only after gap>=2 and the balance gate +// both fail, so it never touches already-correct pages. +// +// Method (faithful to tool-py/column_detectors.gap_glyph_body_column_counts, +// extended with bridging removal): project the BODY — full-width lines dropped +// by dropFullWidth, then any still-wide bridging line dropped at +// bridgingFrac*width — onto the x-axis with glyph-count-per-bin weighting +// (each covered bin receives the line's full rune count, so a wide line +// contributes proportionally more), and look +// for INTERIOR valleys (low-ink runs bounded by high ink on both sides, wider +// than gapMinFrac*width, and not at the page edge). A single clean gutter +// splits the page into two real columns. +// +// The rescue ACCEPTS only exactly one interior valley (k=2). Zero valleys +// means no clean gutter (keep single). Two or more valleys means either a +// multi-column layout (already handled by the gap path) or a single page with +// a vertical blank band (figure/equation) — both are rejected so the rescue +// never over-splits a single column into 3+. The both-sides prune gate +// (pruneColumns, minColLineFrac) is the final guard: a spurious second block +// with too few lines is dropped. +func detectColumnCount2D(lines []pdf.TextBox) (int, []float64) { + // A1: strip figure/equation boxes (they span the gutter and would fill the + // projection, hiding a real column boundary). A2: robust extent so a + // malformed box cannot balloon the page width / histogram allocation. + projLines := textProjectionLines(lines) + minX0, width := robustPageExtent(projLines) + if width <= 0 { + return 0, nil + } + body := dropFullWidth(projLines, width) + body = dropWide(body, width, bridgingFrac) + if len(body) < 4 { + return 0, nil + } + // Cap the bin count so a ballooned extent cannot allocate an OOM-scale + // histogram. When the extent is huge, widen the bin so the projection still + // resolves real gutters. + effBin := binPt + if width/float64(maxBins) > effBin { + effBin = width / float64(maxBins) + } + nb := int(width/effBin) + 1 + proj := make([]int, nb) + for _, b := range body { + w := utf8.RuneCountInString(b.Text) + if w <= 0 { + w = 1 + } + i0 := clampInt(int((b.X0-minX0)/effBin), 0, nb-1) + i1 := clampInt(int((b.X1-minX0)/effBin), 0, nb-1) + for i := i0; i <= i1; i++ { + proj[i] += w + } + } + pk := 0 + for _, p := range proj { + if p > pk { + pk = p + } + } + if pk == 0 { + return 0, nil + } + // A gutter is a run of bins whose glyph-weight is below valleyFrac of the + // page peak. Measured on the 70-page corpus this relative threshold (the + // tool-py reference value) is what actually recovers title-bridged doubles: + // their gutter is clean (≈0 glyphs) and the minority column still carries + // enough ink to sit above valleyFrac*peak and bound the gutter. A minority + // column below ~30% of peak ink (e.g. a very sparse 6-line column) merges + // with the gutter and is NOT recovered — that is a real limitation, not a + // bug; such pages fall back to the confidence-labeling track (issue #18079). + const valleyFrac = 0.30 + minGap := gapMinFrac * width + edge := int(0.05 * width / effBin) + if edge < 0 { + edge = 0 + } + // Find interior valleys; accept ONLY a single clean gutter (k=2). Zero + // valleys means no clean gutter (keep single). Two or more valleys means + // either a multi-column layout (handled by the gap path) or a single page + // with a vertical blank band (figure/equation) — both are rejected so the + // rescue never over-splits a single column into 3+. + var valleyC float64 + count := 0 + i := 0 + for i < nb { + if float64(proj[i]) < valleyFrac*float64(pk) { + j := i + for j < nb && float64(proj[j]) < valleyFrac*float64(pk) { + j++ + } + runW := float64(j-i) * effBin + isInterior := i > edge && j-1 < nb-1-edge + if runW >= minGap && isInterior { + count++ + valleyC = minX0 + float64(i+j)*effBin/2 + } + i = j + } else { + i++ + } + } + if count != 1 { + return 0, nil + } + // Split the body at the single valley into left/right blocks; each + // centroid is the mean X0 of its lines. Classify by b.X0 (not the center) + // so the split agrees with pruneColumns' X0-based assignment — lines whose + // center and X0 fall on opposite sides of the valley would otherwise be + // counted differently by the two steps. + var leftSum, rightSum float64 + lc, rc := 0, 0 + for _, b := range body { + if b.X0 < valleyC { + leftSum += b.X0 + lc++ + } else { + rightSum += b.X0 + rc++ + } + } + if lc == 0 || rc == 0 { + return 0, nil + } + cents := []float64{leftSum / float64(lc), rightSum / float64(rc)} + if pk2, pc, ok := pruneColumns(body, cents); ok { + return pk2, pc + } + return 0, nil +} + +// dropWide removes lines whose width spans >= frac of the page text width. +// Used by detectColumnCount2D to strip in-body bridging headings/captions +// (partially-wide lines that span the gutter but are not full-width front +// matter) before the valley scan. Returns nil if every line is wide so the +// caller treats the page as single rather than pushing an empty body through. +func dropWide(lines []pdf.TextBox, width, frac float64) []pdf.TextBox { + if frac >= 1 { + return lines + } + thr := frac * width + out := make([]pdf.TextBox, 0, len(lines)) + for _, b := range lines { + if b.X1-b.X0 < thr { + out = append(out, b) + } + } + if len(out) == 0 { + return nil + } + return out +} + +// medianWidth returns the median box width on the page. Used by the L3 +// median-width-ratio column signal. +func medianWidth(lines []pdf.TextBox) float64 { + if len(lines) == 0 { + return 1.0 + } + ws := make([]float64, len(lines)) + for i, b := range lines { + ws[i] = b.X1 - b.X0 + if ws[i] < 1 { + ws[i] = 1 + } + } + sort.Float64s(ws) + n := len(ws) + if n%2 == 1 { + return ws[n/2] + } + return (ws[n/2-1] + ws[n/2]) / 2.0 +} + +// maxWidth returns the widest box on the page. +func maxWidth(lines []pdf.TextBox) float64 { + m := 0.0 + for _, b := range lines { + if w := b.X1 - b.X0; w > m { + m = w + } + } + return m +} + +// detectColumnCountMedian is the L3 complementary signal, inspired by PR +// #10475's _assign_column (page_w / median_line_width). It fires only after +// gap, the balance gate, and the 2D valley rescue have ALL returned a single +// column, so it never touches the pages they already handle correctly. +// +// It targets "gutter-less" doubles/triples: pages whose two (or three) body +// columns are separated by a gutter so narrow/bridged that the x-projection +// has no clean ink dip — so the geometric detectors miss them, yet each line +// is only ~page_w/N wide, giving raw_cols = page_w/median_w >= 2. +// +// Two gates keep it safe (measured on the 70-page corpus): +// - raw_cols > maxColumnCount (4): a huge ratio means table cells, not text +// columns (narrow cells yield a tiny median width) -> skip. +// - no line spanning >= shortLineFrac*page_w: the page is one column of +// uniformly short lines whose small median width is not a real multi-column +// signal -> skip. +// +// When it fires, columns are assigned by normalized center-x bucketing +// (matching PR #10475's col_id assignment); the bucket means become centroids. +func detectColumnCountMedian(lines []pdf.TextBox) (int, []float64) { + minX0, width := pageExtent(lines) + if width <= 0 { + return 0, nil + } + mw := medianWidth(lines) + if mw < 1 { + mw = 1 + } + raw := int(width / mw) + if raw < 2 { + return 0, nil + } + if raw > maxColumnCount { + // Table-like (narrow cells): not a text layout. + return 0, nil + } + if maxWidth(lines) < shortLineFrac*width { + // Uniformly short lines, not real columns. + return 0, nil + } + k := raw + if k > medianColCap { + k = medianColCap + } + if k < 2 { + k = 2 + } + // Bucket non-full-width lines by normalized center x, mirroring PR #10475. + // Collect the SAME non-full-width lines into body so the prune step counts + // the line set that produced the centroids — otherwise full-width + // titles/abstracts (which the bucket loop skips) would be re-counted by + // pruneColumns, inflate one column, and let a real multi-column page + // collapse to one. This is the same discipline balancedBodyK2 and + // detectColumnCount2D already follow. + fwThr := medianFullWidthFrac * width + body := make([]pdf.TextBox, 0, len(lines)) + buckets := make([][]float64, k) + for _, b := range lines { + if b.X1-b.X0 >= fwThr { + continue + } + body = append(body, b) + cx := 0.5 * (b.X0 + b.X1) + norm := (cx - minX0) / width + if norm < 0 { + norm = 0 + } + if norm > 0.999999 { + norm = 0.999999 + } + bkt := int(norm * float64(k)) + if bkt > k-1 { + bkt = k - 1 + } + buckets[bkt] = append(buckets[bkt], b.X0) + } + cents := make([]float64, 0, k) + for _, bx := range buckets { + if len(bx) == 0 { + continue + } + var s float64 + for _, x := range bx { + s += x + } + cents = append(cents, s/float64(len(bx))) + } + if len(cents) < 2 { + return 0, nil + } + // Require each column to hold enough lines (the same guard used by the + // rest of the detector) so a sparse side column does not become a false + // split. + if pk, pc, ok := pruneColumns(body, cents); ok { + return pk, pc + } + return 0, nil +} + // balancedBodyK2 runs a forced k=2 KMeans on the BODY x0 (full-width front // matter excluded) and reports whether the split is a real two-column: two // clusters each holding >= minModeFrac of body lines, separated by >= @@ -351,6 +729,102 @@ func pageExtent(lines []pdf.TextBox) (minX0, width float64) { return minX0, maxX1 - minX0 } +// textProjectionLines returns the lines that should feed the column +// projection. Image and equation placeholders are excluded because a figure +// that spans the gutter would otherwise fill the gutter's gap and mask a real +// column boundary (mirrors pdf-inspector stripping image placeholders). Table +// boxes are intentionally kept: the existing tableMaxColFrac gate already +// handles narrow table columns, and dropping them here would widen the blast +// radius unnecessarily. +func textProjectionLines(lines []pdf.TextBox) []pdf.TextBox { + out := make([]pdf.TextBox, 0, len(lines)) + for _, b := range lines { + switch b.LayoutType { + case pdf.LayoutTypeFigure, pdf.LayoutTypeEquation: + continue + default: + out = append(out, b) + } + } + return out +} + +// robustPageExtent returns the X extent (minX0, width) of the text body, +// discarding a small number of outlier lines (malformed coordinates / stray +// boxes) that would otherwise balloon the extent and collapse multi-column +// detection. Normal pages return exactly pageExtent's result. +func robustPageExtent(lines []pdf.TextBox) (minX0, width float64) { + if len(lines) == 0 { + return 0, 0 + } + // Drop implausibly wide (malformed) lines outright: a single box with an + // absurd X1 (e.g. 1e6) would otherwise set the page width to 1e6. + work := make([]pdf.TextBox, 0, len(lines)) + dropped := 0 + for _, b := range lines { + if b.X1-b.X0 > maxPageExtent { + dropped++ + continue + } + work = append(work, b) + } + if len(work) == 0 || float64(dropped)/float64(len(lines)) >= maxTrimFraction { + // Everything malformed, or too many dropped: the outliers are the + // norm. Fall back to the raw extent so the page is never altered. + return pageExtent(lines) + } + // Cluster lines by left edge; discard clusters separated from the main + // body by more than a full page width, provided they are a minority. + return clusteredExtent(work) +} + +// clusteredExtent keeps the largest cluster of lines (by count) and returns its +// extent, but only when the discarded minority is below maxTrimFraction; +// otherwise it returns the raw extent of all lines. This catches outliers whose +// width alone is plausible (e.g. a tiny box placed at an absurd X coordinate). +func clusteredExtent(lines []pdf.TextBox) (minX0, width float64) { + if len(lines) <= 1 { + return pageExtent(lines) + } + xs := make([]float64, len(lines)) + for i, b := range lines { + xs[i] = b.X0 + } + sort.Float64s(xs) + // Split into clusters at gaps larger than a full page. + clusters := [][]float64{{xs[0]}} + for i := 1; i < len(xs); i++ { + if xs[i]-xs[i-1] > maxPageExtent { + clusters = append(clusters, []float64{xs[i]}) + } else { + clusters[len(clusters)-1] = append(clusters[len(clusters)-1], xs[i]) + } + } + if len(clusters) == 1 { + return pageExtent(lines) + } + best := 0 + for i := 1; i < len(clusters); i++ { + if len(clusters[i]) > len(clusters[best]) { + best = i + } + } + dropped := len(lines) - len(clusters[best]) + if float64(dropped)/float64(len(lines)) >= maxTrimFraction { + return pageExtent(lines) + } + lo, hi := clusters[best][0], clusters[best][0] + for _, x := range clusters[best] { + if x < lo { + lo = x + } + if x > hi { + hi = x + } + } + return lo, hi - lo +} + // snapX0s pulls x0 values within indentTol of minX0 back to minX0, so slightly // indented lines still cluster with the left edge (mirrors _assign_column). func snapX0s(x0s []float64, minX0, indentTol float64) []float64 { diff --git a/internal/deepdoc/parser/pdf/layout/combined_column_defensive_test.go b/internal/deepdoc/parser/pdf/layout/combined_column_defensive_test.go new file mode 100644 index 0000000000..3f0c26b030 --- /dev/null +++ b/internal/deepdoc/parser/pdf/layout/combined_column_defensive_test.go @@ -0,0 +1,160 @@ +package layout + +import ( + "testing" + + pdf "ragflow/internal/deepdoc/parser/pdf/type" +) + +// columnCountOf returns the number of distinct columns reported by AssignColumn +// for the given page boxes (mirrors the harness's ColID tally). +func columnCountOf(boxes []pdf.TextBox) int { + res := AssignColumn(boxes) + k := 1 + for _, b := range res { + if b.ColID+1 > k { + k = b.ColID + 1 + } + } + return k +} + +// addBox appends a text box spanning [x0,x1] at the next vertical slot. +func addBox(boxes *[]pdf.TextBox, x0, x1 float64) { + top := float64(len(*boxes)) * 12 + *boxes = append(*boxes, pdf.TextBox{ + PageNumber: 0, X0: x0, X1: x1, Top: top, Bottom: top + 10, + LayoutType: pdf.LayoutTypeText, Text: "b", + }) +} + +// TestTextProjectionLinesStripsFigures proves A1: image and equation +// placeholders must not feed the column projection (a figure spanning the +// gutter would otherwise mask a real column boundary). Text and table boxes +// are kept. +func TestTextProjectionLinesStripsFigures(t *testing.T) { + lines := []pdf.TextBox{ + {LayoutType: pdf.LayoutTypeText, X0: 100, X1: 300}, + {LayoutType: pdf.LayoutTypeFigure, X0: 50, X1: 700}, + {LayoutType: pdf.LayoutTypeEquation, X0: 50, X1: 700}, + {LayoutType: pdf.LayoutTypeTable, X0: 100, X1: 300}, + {LayoutType: pdf.LayoutTypeText, X0: 400, X1: 600}, + {LayoutType: "", X0: 100, X1: 300}, // empty type is treated as text + } + out := textProjectionLines(lines) + if len(out) != 4 { + t.Fatalf("textProjectionLines kept %d lines, want 4 (text+table+text+empty), dropped %d", + len(out), len(lines)-len(out)) + } + for _, b := range out { + if b.LayoutType == pdf.LayoutTypeFigure || b.LayoutType == pdf.LayoutTypeEquation { + t.Errorf("figure/equation box leaked into projection: %+v", b) + } + } +} + +// TestRobustPageExtentDropsOutlier proves A2: a single malformed box with an +// absurd width must not inflate the page extent; the extent of the real text +// body must be returned instead. Uses a realistic page (many real lines) so the +// lone outlier is a true minority (< maxTrimFraction). +func TestRobustPageExtentDropsOutlier(t *testing.T) { + var lines []pdf.TextBox + for i := 0; i < 20; i++ { + lines = append(lines, pdf.TextBox{X0: 100, X1: 300}) // left column + } + for i := 0; i < 20; i++ { + lines = append(lines, pdf.TextBox{X0: 400, X1: 600}) // right column + } + lines = append(lines, pdf.TextBox{X0: 100, X1: 1e6}) // malformed outlier + minX0, width := robustPageExtent(lines) + wantMin, wantWidth := 100.0, 500.0 + if minX0 != wantMin || width != wantWidth { + t.Fatalf("robustPageExtent = (%.0f, %.0f), want (%.0f, %.0f) (outlier must be dropped)", + minX0, width, wantMin, wantWidth) + } +} + +// TestRobustPageExtentNotTrimsRealFarColumns proves the A2 guard: two real +// columns placed far apart (but within a sane page span) must NOT be trimmed +// to one. The gap between them is large but below maxPageExtent, so no split. +func TestRobustPageExtentNotTrimsRealFarColumns(t *testing.T) { + lines := []pdf.TextBox{ + {X0: 100, X1: 300}, // left column + {X0: 2000, X1: 2200}, // right column, far but < maxPageExtent away + } + minX0, width := robustPageExtent(lines) + // Extent must span BOTH columns: 100 .. 2200. + if minX0 != 100.0 || width != 2100.0 { + t.Fatalf("robustPageExtent = (%.0f, %.0f), want (100, 2100) (real far columns kept)", + minX0, width) + } +} + +// TestSyntheticOutlierBoxKeepsColumns proves A2 end-to-end: a clean double +// column plus a malformed box must still be detected as 2 columns (the outlier +// must not collapse detection to 1 via a ballooned extent). +func TestSyntheticOutlierBoxKeepsColumns(t *testing.T) { + var boxes []pdf.TextBox + for i := 0; i < 14; i++ { + addBox(&boxes, 100, 350) // left column + } + for i := 0; i < 6; i++ { + addBox(&boxes, 450, 700) // right column + } + // Malformed box: normal left edge but absurd right edge. + boxes = append(boxes, pdf.TextBox{ + PageNumber: 0, X0: 100, X1: 1e6, Top: 9999, Bottom: 10009, + LayoutType: pdf.LayoutTypeText, Text: "bad", + }) + k := gapColumnCount(boxes, 0.04, 0.15, 2.0) + if k != 2 { + t.Fatalf("outlier box collapsed detection to %d column(s); want 2", k) + } +} + +// TestSyntheticFigureSpanningGutter proves A1 end-to-end on the 1D gap path: a +// full-width figure that bridges the gutter must not fill the projection and +// mask the two real text columns. +func TestSyntheticFigureSpanningGutter(t *testing.T) { + var boxes []pdf.TextBox + for i := 0; i < 14; i++ { + addBox(&boxes, 100, 350) // left column + } + for i := 0; i < 6; i++ { + addBox(&boxes, 450, 700) // right column + } + // Full-width figure spanning the gutter. + boxes = append(boxes, pdf.TextBox{ + PageNumber: 0, X0: 100, X1: 700, Top: 5000, Bottom: 5100, + LayoutType: pdf.LayoutTypeFigure, Text: "fig", + }) + k := gapColumnCount(boxes, 0.04, 0.15, 2.0) + if k != 2 { + t.Fatalf("figure spanning gutter collapsed detection to %d column(s); want 2", k) + } +} + +// TestDetectColumnCount2D_IgnoresFigureSpanningGutter proves A1 extends to the +// 2D rescue projection: a figure that spans the gutter must not fill the +// projection and hide a real two-column valley. Without A1 the figure masks +// the gutter and detectColumnCount2D returns 0 (miss); with A1 it returns 2. +func TestDetectColumnCount2D_IgnoresFigureSpanningGutter(t *testing.T) { + var boxes []pdf.TextBox + for i := 0; i < 14; i++ { + addBox(&boxes, 100, 340) // left column + } + for i := 0; i < 6; i++ { + addBox(&boxes, 460, 700) // right column + } + // Figure spanning the gutter: wide enough to survive dropWide (>=0.6*width + // would drop it, so keep it just under) but not full-width. It fills the + // projection across the real column boundary and hides the valley pre-A1. + boxes = append(boxes, pdf.TextBox{ + PageNumber: 0, X0: 100, X1: 455, Top: 5000, Bottom: 5100, + LayoutType: pdf.LayoutTypeFigure, Text: "fig", + }) + k, _ := detectColumnCount2D(boxes) + if k != 2 { + t.Fatalf("figure spanning gutter hid the 2D valley: detectColumnCount2D=%d, want 2", k) + } +} diff --git a/internal/deepdoc/parser/pdf/layout/combined_column_synthetic_test.go b/internal/deepdoc/parser/pdf/layout/combined_column_synthetic_test.go index a81a0c1254..2db81a100c 100644 --- a/internal/deepdoc/parser/pdf/layout/combined_column_synthetic_test.go +++ b/internal/deepdoc/parser/pdf/layout/combined_column_synthetic_test.go @@ -87,23 +87,91 @@ func TestSyntheticSparseSecondColumn(t *testing.T) { } } -// TODO #18079: a 2D spatial column detector should split this into 2. +// TestSyntheticTitleBridgedDouble models a title-bridged double column: a +// CLEAN vertical gutter plus a full-width title block at the very top that +// bridges the gutter. The two body columns do NOT overlap in x0. // -// The page has a full-width title block at the top that bridges the gutter, -// and below it two side-by-side columns where the right column is sparse and -// its x0 overlaps the left column's x0 range. The x0-based balance gate -// therefore rejects it (minority < 30%, x0 overlap) and the page is reported -// as single. Skipped until #18079 lands; then unskip and assert got == 2. +// The heading bridge is NOT a discriminant signal. What recovers the page is: +// (1) dropFullWidth drops the full-width title, and (2) detectColumnCount2D +// drops the still-wide bridging line at bridgingFrac*width, exposing the clean +// gutter; an interior-valley scan then finds exactly one gutter -> 2 columns. +// +// Geometry here: title [50,450] at top (bridges gutter only at the top); +// left body column [50,240] (30 lines); right body column [260,450] (12 lines) +// — a clean 20-unit gutter at 240–260 with no x0 overlap. +// +// Right-column line count is set to 12 on purpose: +// - body = 30 left + 12 right = 42; 12/42 = 0.286 < 0.30 (minModeFrac) so +// the 1D balance gate correctly rejects it (minority too small), and +// crossTol=0.15 collapses the gutter in 1D projection -> reports 1 until +// the 2D rescue lands. +// - The right column carries 12/30 = 0.40 of the page peak glyph ink, i.e. +// above valleyFrac*peak (0.30), so it BOUNDS the gutter and the valley +// scan recovers it -> 2 columns. NOTE: this is the real capability of the +// rescue — it needs the minority column above ~30% of peak ink. A truly +// sparse column (e.g. 6 lines = 0.20 of peak) merges with the gutter and +// is NOT recovered; those fall back to the confidence-labeling track. +// - 12 >= 0.12*42 = 5.04, so the both-sides prune gate (minColLineFrac) +// accepts it. TestSyntheticSparseSecondColumn (right=3, 3 < 0.12*33) stays +// 1, so the recover/stay-1 split is carried by right-column count. func TestSyntheticTitleBridgedDouble(t *testing.T) { - t.Skip("TODO #18079: title-bridged double should be detected as 2 columns") boxes := concat( concat( - stackedColumn(50, 440, 3, 10, 10), // full-width title (bridges the gutter) - stackedColumn(50, 240, 30, 40, 10), // left column + stackedColumn(50, 450, 3, 10, 10), // full-width title (bridges gutter only at top) + stackedColumn(50, 240, 30, 40, 10), // left body column [50,240] ), - stackedColumn(200, 440, 4, 40, 20), // right column: sparse, x0 overlaps left + stackedColumn(260, 450, 12, 40, 20), // right body column [260,450]: 12 lines, clean gutter 240–260 ) if got := columnCount(boxes); got != 2 { t.Errorf("title-bridged double: got %d, want 2", got) } } + +// TestSyntheticMedianWidthDouble locks the L3 signal (PR #10475's +// page_w/median_w): a gutter-less double whose two columns are ADJACENT in x0 +// (no whitespace gutter), so there is no clean ink dip for gap/balance/2D- +// rescue to find, yet each line is only ~half the page wide (raw_cols = +// page_w/median_w = 2). +// +// Geometry: left body column [50,250] (30 lines), right body column [250,450] +// (12 lines) — adjacent at x=250, so the x-projection is one continuous ink +// band with no >= gapMinFrac run -> gap=1. Balance rejects it (right is +// 12/42 = 0.286 < minModeFrac 0.30). The 2D rescue also fails (no clean +// interior valley). Only the median-width ratio (raw_cols=2, a line spans 200 +// >= 0.45*400=180) recovers it -> 2 columns. +// +// This pins the #18079 acceptance target that the x0-based detectors alone +// cannot reach: a real double with no geometric gutter. +func TestSyntheticMedianWidthDouble(t *testing.T) { + boxes := concat( + stackedColumn(50, 250, 30, 10, 10), // left body column [50,250] + stackedColumn(250, 450, 12, 10, 20), // right body column [250,450]: adjacent, no gutter + ) + if got := columnCount(boxes); got != 2 { + t.Errorf("median-width gutter-less double: got %d, want 2", got) + } +} + +// TestSyntheticMedianWidthDoubleWithTitle locks the L3 median detector's prune +// discipline: it must prune on the SAME non-full-width lines that produced the +// centroids, not the full line set. A gutter-less double with a full-width +// title exercises the path where, before the fix, the title (counted by +// pruneColumns into the left column) inflated n and collapsed the sparse right +// column to a single column. After the fix the detector recovers 2. +// +// Geometry: left [50,250] (29 lines), right [250,450] (4 lines) — adjacent, +// no gutter; plus one full-width title [50,450]. Without the title the right +// column is 4/33 = 0.121 >= minColLineFrac (0.12) -> 2; with the title counted +// in, 4/34 = 0.118 < 0.12 -> 1 (the pre-fix bug). +func TestSyntheticMedianWidthDoubleWithTitle(t *testing.T) { + boxes := concat( + concat( + stackedColumn(50, 250, 29, 10, 10), // left body column [50,250] + stackedColumn(250, 450, 4, 10, 20), // right body column [250,450]: adjacent, no gutter + ), + stackedColumn(50, 450, 1, 10, 10), // full-width title [50,450] + ) + if got := columnCount(boxes); got != 2 { + t.Errorf("median-width gutter-less double with title: got %d, want 2", got) + } +}