diff --git a/internal/deepdoc/parser/pdf/inference/client.go b/internal/deepdoc/parser/pdf/inference/client.go index 451c2fdde1..5dc646f4c6 100644 --- a/internal/deepdoc/parser/pdf/inference/client.go +++ b/internal/deepdoc/parser/pdf/inference/client.go @@ -76,24 +76,6 @@ type bboxesResponse struct { BBoxes [][]float64 `json:"bboxes"` } -// dlaGarbageLayouts mirrors Python LayoutRecognizer's garbage gate -// (deepdoc/vision/layout_recognizer.py:97 and :379), which drops any region -// whose type is in garbage_layouts=["footer","header","reference"] AND whose -// confidence is below 0.4. We apply the same gate at the DLA source so every -// consumer of Client.DLA (not just the table-annotation path, which re-applies -// it downstream) sees the Python-aligned region set. -// -// Of the three types, only "reference" is reachable with the OSS default -// 10-class DLA taxonomy (DefaultDLALabels has no footer/header classes), so in -// practice this gate only fires on low-confidence references. footer/header are -// included defensively to match Python's full garbage set for any deployment -// whose DLA label taxonomy emits them. -var dlaGarbageLayouts = map[string]bool{ - string(pdf.LayoutTypeFooter): true, - string(pdf.LayoutTypeHeader): true, - string(pdf.LayoutTypeReference): true, -} - // DLA analyzes a full page image and returns labeled regions. func (c *Client) DLA(ctx context.Context, pageImage image.Image) ([]pdf.DLARegion, error) { data, err := util.EncodePNG(pageImage) @@ -114,8 +96,9 @@ func (c *Client) DLA(ctx context.Context, pageImage image.Image) ([]pdf.DLARegio if clsID := int(b[5]); clsID >= 0 && clsID < len(labels) { label = labels[clsID] } - // Drop low-confidence garbage-layout regions (Python parity: 0.4 gate). - if dlaGarbageLayouts[label] && b[4] < 0.4 { + // Drop low-confidence garbage-layout regions (Python parity: 0.4 gate, + // deepdoc/vision/layout_recognizer.py:97). + if pdf.GarbageLayoutTypes[label] && b[4] < pdf.GarbageLayoutScoreThreshold { continue } regions = append(regions, pdf.DLARegion{ diff --git a/internal/deepdoc/parser/pdf/inference/client_dla_garbage_test.go b/internal/deepdoc/parser/pdf/inference/client_dla_garbage_test.go index 199de8e5c4..090ce44fba 100644 --- a/internal/deepdoc/parser/pdf/inference/client_dla_garbage_test.go +++ b/internal/deepdoc/parser/pdf/inference/client_dla_garbage_test.go @@ -9,14 +9,14 @@ import ( ) // TestDeepDocHTTP_DLA_GarbageGate pins the Python-parity 0.4 garbage gate -// (LayoutRecognizer.__call__, deepdoc/vision/layout_recognizer.py:97 and :379): -// a region whose layout type is a garbage layout (footer/header/reference) and -// whose confidence is strictly below 0.4 is dropped; everything else is kept. +// (deepdoc/vision/layout_recognizer.py:97): a region whose layout type is a +// garbage layout (footer/header/reference) and whose confidence is strictly +// below 0.4 is dropped; everything else is kept. // // The OSS default 10-class DLA taxonomy only emits "reference" as a garbage -// type, but footer/header are covered defensively (see dlaGarbageLayouts in -// client.go). Each subtest drives a mock /predict/dla backend returning one -// bbox and asserts the resulting region set. +// type, but footer/header are covered defensively via pdf.GarbageLayoutTypes. +// Each subtest drives a mock /predict/dla backend returning one bbox and +// asserts the resulting region set. func TestDeepDocHTTP_DLA_GarbageGate(t *testing.T) { newClient := func(t *testing.T, bboxes [][]float64) *Client { t.Helper() @@ -78,7 +78,8 @@ func TestDeepDocHTTP_DLA_GarbageGate(t *testing.T) { t.Run("low_conf_garbage_and_text", func(t *testing.T) { // A low-confidence reference is dropped while an unrelated text region - // is kept — Python keeps only the high-confidence non-garbage region. + // (non-garbage) is kept — the gate only removes garbage-layout regions + // below 0.4; non-garbage regions pass through at any confidence. client := newClient(t, [][]float64{ {50, 10, 500, 50, 0.30, referenceClass}, // reference, low confidence -> dropped {50, 100, 500, 300, 0.90, 1}, // text, high confidence -> kept diff --git a/internal/deepdoc/parser/pdf/table/table_annotate.go b/internal/deepdoc/parser/pdf/table/table_annotate.go index 7335e61ca0..911a97460a 100644 --- a/internal/deepdoc/parser/pdf/table/table_annotate.go +++ b/internal/deepdoc/parser/pdf/table/table_annotate.go @@ -307,7 +307,7 @@ func FilteredDLARegions(regions []pdf.DLARegion, boxes []pdf.TextBox) []pdf.DLAR } kept := regions[:0] for _, r := range regions { - if r.Confidence >= 0.4 || !isGarbageLayoutType(r.Label) { + if r.Confidence >= pdf.GarbageLayoutScoreThreshold || !pdf.GarbageLayoutTypes[r.Label] { kept = append(kept, r) } } @@ -409,7 +409,7 @@ func AnnotateBoxLayouts(boxes []pdf.TextBox, regions []pdf.DLARegion, scale floa } if bestJ >= 0 && bestOverlap >= 0.4 { // Garbage layout not at page edge -> pop (Python: bxs.pop(i)). - if isGarbageLayoutType(ty) && pageImgHeight > 0 && !garbageKeepFeat(ty, boxes[i], pageImgHeight/scale) { + if pdf.GarbageLayoutTypes[ty] && pageImgHeight > 0 && !garbageKeepFeat(ty, boxes[i], pageImgHeight/scale) { dropped[i] = true continue } @@ -475,15 +475,6 @@ func AnnotateBoxLayouts(boxes []pdf.TextBox, regions []pdf.DLARegion, scale floa } // ── garbage layout helpers ──────────────────────────────────────────── -// garbageLayoutTypes matches Python's self.garbage_layouts. -var garbageLayoutTypes = map[string]bool{ - pdf.LayoutTypeFooter: true, pdf.LayoutTypeHeader: true, pdf.LayoutTypeReference: true, -} - -func isGarbageLayoutType(ty string) bool { - return garbageLayoutTypes[ty] -} - // garbageKeepFeat matches Python's keep_feats in LayoutRecognizer.__call__: // footer near page bottom (>90% of page height) or header near page top (<10%) // are real page decorations - keep them. Others are DLA noise. diff --git a/internal/deepdoc/parser/pdf/type/types.go b/internal/deepdoc/parser/pdf/type/types.go index ae0cd7447e..f0eed00b10 100644 --- a/internal/deepdoc/parser/pdf/type/types.go +++ b/internal/deepdoc/parser/pdf/type/types.go @@ -44,6 +44,8 @@ const ( LayoutTypeHeader = doctype.LayoutTypeHeader DLALabelFigureCaption = doctype.DLALabelFigureCaption DLALabelTableCaption = doctype.DLALabelTableCaption + + GarbageLayoutScoreThreshold = doctype.GarbageLayoutScoreThreshold ) // ── Re-export functions and variables ────────────────────────────────────── @@ -52,4 +54,5 @@ var ( CollectFigures = doctype.CollectFigures DefaultParserConfig = doctype.DefaultParserConfig IsCJK = doctype.IsCJK + GarbageLayoutTypes = doctype.GarbageLayoutTypes ) diff --git a/internal/deepdoc/parser/type/types.go b/internal/deepdoc/parser/type/types.go index 5c9f79d56f..428f23b838 100644 --- a/internal/deepdoc/parser/type/types.go +++ b/internal/deepdoc/parser/type/types.go @@ -251,6 +251,20 @@ const ( DLALabelTableCaption = "table caption" ) +// GarbageLayoutScoreThreshold is the minimum confidence a garbage-layout +// region must reach to survive; below it the region is dropped. Mirrors +// Python's garbage gate in LayoutRecognizer (deepdoc/vision/layout_recognizer.py:97). +const GarbageLayoutScoreThreshold = 0.4 + +// GarbageLayoutTypes are layout types dropped when their confidence is below +// GarbageLayoutScoreThreshold. Mirrors Python's self.garbage_layouts +// = ["footer", "header", "reference"]. +var GarbageLayoutTypes = map[string]bool{ + LayoutTypeFooter: true, + LayoutTypeHeader: true, + LayoutTypeReference: true, +} + // ── Interfaces ──────────────────────────────────────────────────────────── // DocAnalyzer abstracts DeepDoc vision operations.