diff --git a/deepdoc/parser/pdf_parser.py b/deepdoc/parser/pdf_parser.py index 1dbb11b753..5953829291 100644 --- a/deepdoc/parser/pdf_parser.py +++ b/deepdoc/parser/pdf_parser.py @@ -196,6 +196,60 @@ class RAGFlowPdfParser: # CID pattern regex for unmapped font characters from pdfminer _CID_PATTERN = re.compile(r"\(cid\s*:\s*\d+\s*\)") + _OCR_ALPHABET = None + + @classmethod + def _ocr_can_represent(cls, text, min_coverage=0.8): + """True if the OCR recogniser's alphabet covers this text well enough to be worth OCRing.""" + if not text: + return True + if cls._OCR_ALPHABET is None: + res = os.path.join(get_project_base_directory(), "rag/res/deepdoc/ocr.res") + try: + with open(res, encoding="utf-8") as f: + cls._OCR_ALPHABET = set(f.read()) + except (OSError, UnicodeDecodeError) as e: + logging.warning("Could not load OCR alphabet from %s: %s; treating all text as representable.", res, e) + cls._OCR_ALPHABET = set() + if not cls._OCR_ALPHABET: + return True # unknown alphabet: preserve existing behaviour + letters = [c for c in text if c.strip()] + if not letters: + return True + covered = sum(1 for c in letters if c in cls._OCR_ALPHABET) + return covered / len(letters) >= min_coverage + + # CJK scripts (Han, Hiragana, Katakana, Hangul) do not separate words with + # spaces, so a geometric gap between their glyphs must not become one. + _CJK_PATTERN = re.compile(r"[ᄀ-ᇿ぀-ヿ㄰-㆏㐀-䶿一-鿿가-힯豈-﫿]|[\U00020000-\U0002fa1f]") + + @classmethod + def _insert_word_spaces(cls, chars, gap_ratio=0.25): + """Recover missing spaces from character geometry. + + Many PDFs encode no space glyphs and separate words by positioning alone. + Append a space to a char when the gap to the next exceeds ``gap_ratio`` of + the mean char width; intra-word kerns fall well below that. CJK is skipped: + it does not write inter-word spaces, so a gap between CJK glyphs is ordinary + tracking, not a boundary. ``chars`` is a list of pdfplumber-style dicts and + is mutated in place. + """ + widths = [c["width"] for c in chars if c["text"] and c["text"].strip()] + mean_w = sum(widths) / len(widths) if widths else 0 + if mean_w <= 0: + return + for cur, nxt in zip(chars, chars[1:]): + if ( + cur["text"] + and nxt["text"] + and cur["text"].strip() + and nxt["text"].strip() + and not cls._CJK_PATTERN.search(cur["text"]) + and not cls._CJK_PATTERN.search(nxt["text"]) + and nxt["x0"] - cur["x1"] > mean_w * gap_ratio + ): + cur["text"] += " " + @staticmethod def _is_garbled_char(ch): """Check if a single character is garbled (unmappable from PDF font encoding). @@ -747,9 +801,9 @@ class RAGFlowPdfParser: if self._is_garbled_char(ch): garbled_count += 1 del b["chars"] - # If the majority of characters from pdfplumber are garbled, - # clear the text so OCR recognition will be used as fallback. - # Strategy 1: PUA / unmapped CID characters + + # Strategy 1: PUA / unmapped CID characters. These are genuine garbage, + # so re-OCR regardless of script. if total_count > 0 and garbled_count / total_count >= 0.5: logging.info( "Page %d: detected garbled pdfplumber text (garbled=%d/%d), falling back to OCR for box at (%.1f, %.1f)", @@ -761,6 +815,12 @@ class RAGFlowPdfParser: ) b["text"] = "" continue + + # Keep a clean text layer the recogniser cannot spell: ocr.res is + # CJK+Latin, so re-OCRing e.g. a Cyrillic page only produces garbage. + if total_count > 0 and not self._ocr_can_represent(b["text"]): + continue + # Strategy 2: font-encoding garbling — all chars are ASCII # punctuation from subset fonts (no CJK output) if total_count > 0 and self._is_garbled_by_font_encoding(box_chars, min_chars=5): @@ -1592,16 +1652,7 @@ class RAGFlowPdfParser: self.is_english = False async def __img_ocr(i, id, img, chars, limiter): - j = 0 - while j + 1 < len(chars): - if ( - chars[j]["text"] - and chars[j + 1]["text"] - and re.match(r"[0-9a-zA-Z,.:;!%]+", chars[j]["text"] + chars[j + 1]["text"]) - and chars[j + 1]["x0"] - chars[j]["x1"] >= min(chars[j + 1]["width"], chars[j]["width"]) / 2 - ): - chars[j]["text"] += " " - j += 1 + self._insert_word_spaces(chars) if limiter: async with limiter: diff --git a/internal/deepdoc/parser/pdf/layout/chars_boxes.go b/internal/deepdoc/parser/pdf/layout/chars_boxes.go index e905094169..c25f684a7f 100644 --- a/internal/deepdoc/parser/pdf/layout/chars_boxes.go +++ b/internal/deepdoc/parser/pdf/layout/chars_boxes.go @@ -2,7 +2,6 @@ package layout import ( "math" - "regexp" "sort" "strings" @@ -10,6 +9,17 @@ import ( util "ragflow/internal/deepdoc/parser/pdf/util" ) +// hasCJK reports whether s contains a CJK rune. CJK scripts do not separate +// words with spaces, so geometric gaps between their glyphs must not become one. +func hasCJK(s string) bool { + for _, r := range s { + if pdf.IsCJK(r) { + return true + } + } + return false +} + // CharsToBoxes converts raw characters to initial text boxes by grouping // characters into lines based on vertical overlap. // @@ -169,12 +179,6 @@ func verticalOverlap(a, b pdf.TextChar) bool { } // lineToTextBox converts a line of characters to a single pdf.TextBox. -// asciiWordPattern matches strings composed entirely of ASCII word -// characters. Python uses re.match (prefix match) — the stricter -// full-string match here is equivalent in practice because each -// pdf.TextChar.Text is a single rune, so prevText+currText ≤ 2 chars. -// Python: pdf_parser.py:1528 re.match(r"[0-9a-zA-Z,.:;!%]+", ...) -var asciiWordPattern = regexp.MustCompile(`^[0-9a-zA-Z,.:;!%]+$`) func LineToTextBox(chars []pdf.TextChar) pdf.TextBox { if len(chars) == 0 { @@ -186,21 +190,32 @@ func LineToTextBox(chars []pdf.TextChar) pdf.TextBox { Top: chars[0].Top, Bottom: chars[0].Bottom, } + // Recover missing spaces from geometry: many PDFs encode no space glyphs and + // separate words by positioning alone. For a script that separates words with + // spaces, a gap wider than a fraction of the mean char width is a word + // boundary; intra-word kerns fall well below it. CJK is excluded: it does not + // write inter-word spaces, so a gap between CJK glyphs is ordinary tracking, + // not a boundary. + var sumWidth float64 + var nWidth int + for _, c := range chars { + if strings.TrimSpace(c.Text) != "" { + sumWidth += c.X1 - c.X0 + nWidth++ + } + } + var spaceGap float64 + if nWidth > 0 { + spaceGap = (sumWidth / float64(nWidth)) * 0.25 + } var textParts []string for i, c := range chars { - // Insert space between adjacent ASCII words with a visible gap. - // Python: pdf_parser.py:1524-1532 __img_ocr space insertion. - if i > 0 { + if i > 0 && spaceGap > 0 { prev := chars[i-1] - prevText := strings.TrimSpace(prev.Text) - currText := strings.TrimSpace(c.Text) - if prevText != "" && currText != "" { - gap := c.X0 - prev.X1 - minWidth := math.Min(c.X1-c.X0, prev.X1-prev.X0) - if gap >= minWidth/2 && - asciiWordPattern.MatchString(prevText+currText) { - textParts = append(textParts, " ") - } + if strings.TrimSpace(prev.Text) != "" && strings.TrimSpace(c.Text) != "" && + !hasCJK(prev.Text) && !hasCJK(c.Text) && + c.X0-prev.X1 > spaceGap { + textParts = append(textParts, " ") } } box.X0 = math.Min(box.X0, c.X0) diff --git a/internal/deepdoc/parser/pdf/layout/chars_boxes_test.go b/internal/deepdoc/parser/pdf/layout/chars_boxes_test.go index 354d11d32c..dd0059c3fb 100644 --- a/internal/deepdoc/parser/pdf/layout/chars_boxes_test.go +++ b/internal/deepdoc/parser/pdf/layout/chars_boxes_test.go @@ -104,4 +104,44 @@ func TestLineToTextBox(t *testing.T) { t.Errorf("Text = %q, want 'X'", box.Text) } }) + // Word boundaries must be recovered for non-Latin scripts too. Before the fix + // the gap rule was gated on an ASCII-only regex, so Cyrillic chars never got a + // space regardless of the gap between them (a whole page welded into one token). + t.Run("cyrillic word gap", func(t *testing.T) { + chars := []pdf.TextChar{ + // "Ок" — kern-sized gap (2, ~0.25 width): same word + {X0: 50, X1: 58, Top: 100, Bottom: 112, Text: "О"}, + {X0: 60, X1: 68, Top: 100, Bottom: 112, Text: "к"}, + // large gap before "н": a word boundary + {X0: 120, X1: 128, Top: 100, Bottom: 112, Text: "н"}, + } + box := LineToTextBox(chars) + if box.Text != "Ок н" { + t.Errorf("Text = %q, want 'Ок н' (space inserted only at the word gap)", box.Text) + } + }) + // CJK does not separate words with spaces, so a gap between CJK glyphs is + // ordinary tracking and must NOT become one, however wide. + t.Run("cjk gap is not a word boundary", func(t *testing.T) { + chars := []pdf.TextChar{ + {X0: 12, X1: 18, Top: 100, Bottom: 112, Text: "乙"}, + // gap of 4 on 6-wide glyphs — far over the 0.25*mean threshold + {X0: 22, X1: 28, Top: 100, Bottom: 112, Text: "丙"}, + } + box := LineToTextBox(chars) + if box.Text != "乙丙" { + t.Errorf("Text = %q, want '乙丙' (no space between CJK glyphs)", box.Text) + } + }) + // A CJK/Latin boundary is likewise not a space the geometry may invent. + t.Run("cjk adjacent to latin gets no invented space", func(t *testing.T) { + chars := []pdf.TextChar{ + {X0: 12, X1: 18, Top: 100, Bottom: 112, Text: "乙"}, + {X0: 22, X1: 28, Top: 100, Bottom: 112, Text: "A"}, + } + box := LineToTextBox(chars) + if box.Text != "乙A" { + t.Errorf("Text = %q, want '乙A'", box.Text) + } + }) } diff --git a/internal/deepdoc/parser/pdf/parser_ocr.go b/internal/deepdoc/parser/pdf/parser_ocr.go index 4c2114d516..6353fda4cd 100644 --- a/internal/deepdoc/parser/pdf/parser_ocr.go +++ b/internal/deepdoc/parser/pdf/parser_ocr.go @@ -273,10 +273,12 @@ func (p *Parser) buildTextBoxes(ctx context.Context, pageImg image.Image, } } } + // PUA / unmapped-glyph garbage: genuine noise, re-OCR regardless of script. if totalCnt > 0 && float64(garbledCnt)/float64(totalCnt) >= 0.5 { tb.Text = "" - } - if tb.Text != "" && util.IsGarbledByFontEncoding(boxChars[i], 5) { + } else if tb.Text != "" && util.OcrCanRepresent(tb.Text) && util.IsGarbledByFontEncoding(boxChars[i], 5) { + // Font-encoding garbling, but skipped for a script the recogniser + // cannot spell -- OCR would only produce garbage. tb.Text = "" } } diff --git a/internal/deepdoc/parser/pdf/util/garbled.go b/internal/deepdoc/parser/pdf/util/garbled.go index d044e30c2e..ad28caa0b3 100644 --- a/internal/deepdoc/parser/pdf/util/garbled.go +++ b/internal/deepdoc/parser/pdf/util/garbled.go @@ -1,11 +1,15 @@ package util import ( + "os" + "path/filepath" "regexp" "strings" + "sync" "unicode" pdf "ragflow/internal/deepdoc/parser/pdf/type" + "ragflow/internal/utility" ) // CIDPattern matches pdfminer's CID placeholder like "(cid:123)". @@ -201,6 +205,66 @@ func IsGarbledByFontEncoding(chars []pdf.TextChar, minChars int) bool { return cjkRatio < 0.05 && punctRatio > 0.4 } +// ocrCoverageThreshold is the minimum fraction of a text's characters that the +// OCR recogniser's alphabet must cover for an OCR fallback to be worthwhile. +const ocrCoverageThreshold = 0.8 + +var ( + ocrAlphabetOnce sync.Once + ocrAlphabet map[rune]struct{} +) + +// loadOCRAlphabet reads the recogniser's character dictionary once +// (rag/res/deepdoc/ocr.res, provisioned next to the OCR model). On any error it +// returns an empty set, which OcrCanRepresent treats as "unknown alphabet" and +// so allows the existing fallback behaviour. +func loadOCRAlphabet() map[rune]struct{} { + ocrAlphabetOnce.Do(func() { + ocrAlphabet = map[rune]struct{}{} + data, err := os.ReadFile(filepath.Join(utility.GetProjectRoot(), "rag", "res", "deepdoc", "ocr.res")) + if err != nil { + return + } + for _, r := range string(data) { + ocrAlphabet[r] = struct{}{} + } + }) + return ocrAlphabet +} + +// OcrCanRepresent reports whether the OCR recogniser's alphabet covers text +// well enough that an OCR pass could improve on it. ocr.res is CJK+Latin +// (~6300 CJK / 52 Latin / 6 Cyrillic), so it covers ~100% of an English page but +// only ~6% of a Cyrillic one. Discarding a usable text layer in favour of a +// recogniser that cannot spell the script only produces garbage, so the +// garbled-text fallbacks are skipped when this returns false. +func OcrCanRepresent(text string) bool { + return ocrCanRepresent(text, loadOCRAlphabet(), ocrCoverageThreshold) +} + +func ocrCanRepresent(text string, alphabet map[rune]struct{}, minCoverage float64) bool { + if strings.TrimSpace(text) == "" { + return true + } + if len(alphabet) == 0 { // unknown alphabet: preserve existing behaviour + return true + } + letters, covered := 0, 0 + for _, r := range text { + if unicode.IsSpace(r) { + continue + } + letters++ + if _, ok := alphabet[r]; ok { + covered++ + } + } + if letters == 0 { + return true + } + return float64(covered)/float64(letters) >= minCoverage +} + // catOf returns "Cs" for surrogates, "Cn" for unassigned code points // (not in any Unicode category), and "" for everything else. // Python unicodedata.category() returns "Cc" for control chars, "Cn" only @@ -239,12 +303,18 @@ func IsGarbledPage(chars []pdf.TextChar) bool { fullText.WriteString(c.Text) } text := fullText.String() + // PUA / unmapped-glyph garbage: genuine noise, re-OCR regardless of script. if IsGarbledText(text, 0.3) { return true } if PdfOxideUnmappedGarbled(text) && IsScanNoise(text) { return true } + // Beyond genuine garbage, keep a clean text layer the recogniser cannot spell: + // OCR of a script absent from ocr.res only produces garbage. + if !OcrCanRepresent(text) { + return false + } if IsGarbledByFontEncoding(chars, 20) { return true } diff --git a/internal/deepdoc/parser/pdf/util/garbled_ocr_test.go b/internal/deepdoc/parser/pdf/util/garbled_ocr_test.go new file mode 100644 index 0000000000..000716e914 --- /dev/null +++ b/internal/deepdoc/parser/pdf/util/garbled_ocr_test.go @@ -0,0 +1,45 @@ +package util + +import "testing" + +// runeSet builds an alphabet from a string, mirroring how loadOCRAlphabet reads +// ocr.res into a rune set. +func runeSet(s string) map[rune]struct{} { + m := make(map[rune]struct{}) + for _, r := range s { + m[r] = struct{}{} + } + return m +} + +// TestOcrCanRepresent covers the guard that keeps a usable text layer when the +// OCR recogniser's alphabet cannot spell the script. The real ocr.res is +// CJK+Latin (52 Latin / 6 Cyrillic), giving English ~100% coverage and Russian +// ~6% — so a Latin-only alphabet reproduces the split here. +func TestOcrCanRepresent(t *testing.T) { + latin := runeSet("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789 .,-") + + tests := []struct { + name string + text string + alphabet map[rune]struct{} + minCoverage float64 + want bool + }{ + {"english is representable", "Minimum edit distance", latin, 0.8, true}, + {"russian is not representable", "О книге как-то иначе", latin, 0.8, false}, + {"empty text is representable", "", latin, 0.8, true}, + {"whitespace only is representable", " ", latin, 0.8, true}, + {"unknown alphabet preserves behaviour", "О книге", runeSet(""), 0.8, true}, + {"exactly at threshold", "abcdx", runeSet("abcd"), 0.8, true}, // 4/5 == 0.80 + {"below threshold", "abcxx", runeSet("abcd"), 0.8, false}, // 3/5 == 0.60 + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := ocrCanRepresent(tt.text, tt.alphabet, tt.minCoverage) + if got != tt.want { + t.Errorf("ocrCanRepresent(%q, ...) = %v, want %v", tt.text, got, tt.want) + } + }) + } +} diff --git a/test/unit_test/deepdoc/parser/test_pdf_garbled_detection.py b/test/unit_test/deepdoc/parser/test_pdf_garbled_detection.py index 43c572e2eb..d6ea1597df 100644 --- a/test/unit_test/deepdoc/parser/test_pdf_garbled_detection.py +++ b/test/unit_test/deepdoc/parser/test_pdf_garbled_detection.py @@ -454,3 +454,91 @@ class TestLayoutRecognizerIsGarbage: def test_cid_with_large_number(self): assert _is_garbage({"text": "(cid:99999)"}) is True + + +# --------------------------------------------------------------------------- +# Tests for _ocr_can_represent +# --------------------------------------------------------------------------- + + +class TestOcrCanRepresent: + """The OCR fallback is skipped for a script the recogniser cannot spell. + + ocr.res is CJK+Latin, so it covers ~100% of an English page but only ~6% of a + Cyrillic one; re-OCRing a script it cannot spell only produces garbage. + """ + + _LATIN = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789 .,-" + + def test_english_is_representable(self, monkeypatch): + monkeypatch.setattr(_Parser, "_OCR_ALPHABET", set(self._LATIN)) + assert _Parser._ocr_can_represent("Minimum edit distance") is True + + def test_russian_is_not_representable(self, monkeypatch): + monkeypatch.setattr(_Parser, "_OCR_ALPHABET", set(self._LATIN)) + assert _Parser._ocr_can_represent("О книге как-то иначе") is False + + def test_empty_text_is_representable(self, monkeypatch): + monkeypatch.setattr(_Parser, "_OCR_ALPHABET", set(self._LATIN)) + assert _Parser._ocr_can_represent("") is True + assert _Parser._ocr_can_represent(" ") is True + + def test_unknown_alphabet_is_representable(self, monkeypatch): + monkeypatch.setattr(_Parser, "_OCR_ALPHABET", set()) + assert _Parser._ocr_can_represent("О книге") is True + + def test_coverage_threshold_boundary(self, monkeypatch): + monkeypatch.setattr(_Parser, "_OCR_ALPHABET", set("abcd")) + assert _Parser._ocr_can_represent("abcdx", min_coverage=0.8) is True # 4/5 + assert _Parser._ocr_can_represent("abcxx", min_coverage=0.8) is False # 3/5 + + +# --------------------------------------------------------------------------- +# Tests for _insert_word_spaces (geometric word-boundary recovery) +# --------------------------------------------------------------------------- + + +def _line(words, char_w=5.0, word_gap=2.0, kern=0.0, x0=0.0): + """Build a spaceless pdfplumber-style char stream: fixed-width glyphs, + ``word_gap`` pt between words, ``kern`` pt between glyphs inside a word.""" + chars = [] + x = x0 + for wi, w in enumerate(words): + if wi: + x += word_gap + for ci, ch in enumerate(w): + if ci: + x += kern + chars.append({"text": ch, "x0": x, "x1": x + char_w, "width": char_w}) + x += char_w + return chars + + +def _spaced(chars): + _Parser._insert_word_spaces(chars) + return "".join(c["text"] for c in chars) + + +class TestInsertWordSpaces: + def test_cyrillic_words_are_split(self): + # mean width 5 -> threshold 0.25*5 = 1.25pt; the 2.0pt word gap exceeds it. + assert _spaced(_line(["Окниге", "както"])).split() == ["Окниге", "както"] + + def test_latin_word_gap_becomes_a_space(self): + assert _spaced(_line(["Hello", "World"])) == "Hello World" + + def test_intra_word_kern_is_not_a_space(self): + # A 0.3pt kern inside a word is well below the threshold. + assert _spaced(_line(["Large"], kern=0.3)) == "Large" + + def test_cjk_gap_is_not_a_word_boundary(self): + # CJK does not separate words with spaces: a wide gap between CJK glyphs + # is ordinary tracking, not a boundary. + assert _spaced(_line(["乙", "丙"])) == "乙丙" + + def test_cjk_adjacent_to_latin_gets_no_invented_space(self): + assert _spaced(_line(["乙", "A"])) == "乙A" + + def test_japanese_and_korean_gaps_are_not_boundaries(self): + assert _spaced(_line(["ア", "イ"])) == "アイ" + assert _spaced(_line(["한", "글"])) == "한글"