fix(deepdoc): recover word boundaries for non-Latin scripts; skip OCR fallback the recogniser can't serve (#16958)

deepdoc/parser/pdf_parser.py

@@ -196,6 +196,60 @@ class RAGFlowPdfParser:
     # CID pattern regex for unmapped font characters from pdfminer
     _CID_PATTERN = re.compile(r"\(cid\s*:\s*\d+\s*\)")
 
+    _OCR_ALPHABET = None
+
+    @classmethod
+    def _ocr_can_represent(cls, text, min_coverage=0.8):
+        """True if the OCR recogniser's alphabet covers this text well enough to be worth OCRing."""
+        if not text:
+            return True
+        if cls._OCR_ALPHABET is None:
+            res = os.path.join(get_project_base_directory(), "rag/res/deepdoc/ocr.res")
+            try:
+                with open(res, encoding="utf-8") as f:
+                    cls._OCR_ALPHABET = set(f.read())
+            except (OSError, UnicodeDecodeError) as e:
+                logging.warning("Could not load OCR alphabet from %s: %s; treating all text as representable.", res, e)
+                cls._OCR_ALPHABET = set()
+        if not cls._OCR_ALPHABET:
+            return True                      # unknown alphabet: preserve existing behaviour
+        letters = [c for c in text if c.strip()]
+        if not letters:
+            return True
+        covered = sum(1 for c in letters if c in cls._OCR_ALPHABET)
+        return covered / len(letters) >= min_coverage
+
+    # CJK scripts (Han, Hiragana, Katakana, Hangul) do not separate words with
+    # spaces, so a geometric gap between their glyphs must not become one.
+    _CJK_PATTERN = re.compile(r"[ᄀ-ᇿ぀-ヿ㄰-㆏㐀-䶿一-鿿가-힯豈-﫿]|[\U00020000-\U0002fa1f]")
+
+    @classmethod
+    def _insert_word_spaces(cls, chars, gap_ratio=0.25):
+        """Recover missing spaces from character geometry.
+
+        Many PDFs encode no space glyphs and separate words by positioning alone.
+        Append a space to a char when the gap to the next exceeds ``gap_ratio`` of
+        the mean char width; intra-word kerns fall well below that. CJK is skipped:
+        it does not write inter-word spaces, so a gap between CJK glyphs is ordinary
+        tracking, not a boundary. ``chars`` is a list of pdfplumber-style dicts and
+        is mutated in place.
+        """
+        widths = [c["width"] for c in chars if c["text"] and c["text"].strip()]
+        mean_w = sum(widths) / len(widths) if widths else 0
+        if mean_w <= 0:
+            return
+        for cur, nxt in zip(chars, chars[1:]):
+            if (
+                cur["text"]
+                and nxt["text"]
+                and cur["text"].strip()
+                and nxt["text"].strip()
+                and not cls._CJK_PATTERN.search(cur["text"])
+                and not cls._CJK_PATTERN.search(nxt["text"])
+                and nxt["x0"] - cur["x1"] > mean_w * gap_ratio
+            ):
+                cur["text"] += " "
+
     @staticmethod
     def _is_garbled_char(ch):
         """Check if a single character is garbled (unmappable from PDF font encoding).
@@ -747,9 +801,9 @@ class RAGFlowPdfParser:
                             if self._is_garbled_char(ch):
                                 garbled_count += 1
             del b["chars"]
-            # If the majority of characters from pdfplumber are garbled,
-            # clear the text so OCR recognition will be used as fallback.
-            # Strategy 1: PUA / unmapped CID characters
+
+            # Strategy 1: PUA / unmapped CID characters. These are genuine garbage,
+            # so re-OCR regardless of script.
             if total_count > 0 and garbled_count / total_count >= 0.5:
                 logging.info(
                     "Page %d: detected garbled pdfplumber text (garbled=%d/%d), falling back to OCR for box at (%.1f, %.1f)",
@@ -761,6 +815,12 @@ class RAGFlowPdfParser:
                 )
                 b["text"] = ""
                 continue
+
+            # Keep a clean text layer the recogniser cannot spell: ocr.res is
+            # CJK+Latin, so re-OCRing e.g. a Cyrillic page only produces garbage.
+            if total_count > 0 and not self._ocr_can_represent(b["text"]):
+                continue
+
             # Strategy 2: font-encoding garbling — all chars are ASCII
             # punctuation from subset fonts (no CJK output)
             if total_count > 0 and self._is_garbled_by_font_encoding(box_chars, min_chars=5):
@@ -1592,16 +1652,7 @@ class RAGFlowPdfParser:
             self.is_english = False
 
         async def __img_ocr(i, id, img, chars, limiter):
-            j = 0
-            while j + 1 < len(chars):
-                if (
-                    chars[j]["text"]
-                    and chars[j + 1]["text"]
-                    and re.match(r"[0-9a-zA-Z,.:;!%]+", chars[j]["text"] + chars[j + 1]["text"])
-                    and chars[j + 1]["x0"] - chars[j]["x1"] >= min(chars[j + 1]["width"], chars[j]["width"]) / 2
-                ):
-                    chars[j]["text"] += " "
-                j += 1
+            self._insert_word_spaces(chars)
 
             if limiter:
                 async with limiter:

internal/deepdoc/parser/pdf/layout/chars_boxes.go

@@ -2,7 +2,6 @@ package layout
 
 import (
 	"math"
-	"regexp"
 	"sort"
 	"strings"
 
@@ -10,6 +9,17 @@ import (
 	util "ragflow/internal/deepdoc/parser/pdf/util"
 )
 
+// hasCJK reports whether s contains a CJK rune. CJK scripts do not separate
+// words with spaces, so geometric gaps between their glyphs must not become one.
+func hasCJK(s string) bool {
+	for _, r := range s {
+		if pdf.IsCJK(r) {
+			return true
+		}
+	}
+	return false
+}
+
 // CharsToBoxes converts raw characters to initial text boxes by grouping
 // characters into lines based on vertical overlap.
 //
@@ -169,12 +179,6 @@ func verticalOverlap(a, b pdf.TextChar) bool {
 }
 
 // lineToTextBox converts a line of characters to a single pdf.TextBox.
-// asciiWordPattern matches strings composed entirely of ASCII word
-// characters. Python uses re.match (prefix match) — the stricter
-// full-string match here is equivalent in practice because each
-// pdf.TextChar.Text is a single rune, so prevText+currText ≤ 2 chars.
-// Python: pdf_parser.py:1528 re.match(r"[0-9a-zA-Z,.:;!%]+", ...)
-var asciiWordPattern = regexp.MustCompile(`^[0-9a-zA-Z,.:;!%]+$`)
 
 func LineToTextBox(chars []pdf.TextChar) pdf.TextBox {
 	if len(chars) == 0 {
@@ -186,21 +190,32 @@ func LineToTextBox(chars []pdf.TextChar) pdf.TextBox {
 		Top:    chars[0].Top,
 		Bottom: chars[0].Bottom,
 	}
+	// Recover missing spaces from geometry: many PDFs encode no space glyphs and
+	// separate words by positioning alone. For a script that separates words with
+	// spaces, a gap wider than a fraction of the mean char width is a word
+	// boundary; intra-word kerns fall well below it. CJK is excluded: it does not
+	// write inter-word spaces, so a gap between CJK glyphs is ordinary tracking,
+	// not a boundary.
+	var sumWidth float64
+	var nWidth int
+	for _, c := range chars {
+		if strings.TrimSpace(c.Text) != "" {
+			sumWidth += c.X1 - c.X0
+			nWidth++
+		}
+	}
+	var spaceGap float64
+	if nWidth > 0 {
+		spaceGap = (sumWidth / float64(nWidth)) * 0.25
+	}
 	var textParts []string
 	for i, c := range chars {
-		// Insert space between adjacent ASCII words with a visible gap.
-		// Python: pdf_parser.py:1524-1532 __img_ocr space insertion.
-		if i > 0 {
+		if i > 0 && spaceGap > 0 {
 			prev := chars[i-1]
-			prevText := strings.TrimSpace(prev.Text)
-			currText := strings.TrimSpace(c.Text)
-			if prevText != "" && currText != "" {
-				gap := c.X0 - prev.X1
-				minWidth := math.Min(c.X1-c.X0, prev.X1-prev.X0)
-				if gap >= minWidth/2 &&
-					asciiWordPattern.MatchString(prevText+currText) {
-					textParts = append(textParts, " ")
-				}
+			if strings.TrimSpace(prev.Text) != "" && strings.TrimSpace(c.Text) != "" &&
+				!hasCJK(prev.Text) && !hasCJK(c.Text) &&
+				c.X0-prev.X1 > spaceGap {
+				textParts = append(textParts, " ")
 			}
 		}
 		box.X0 = math.Min(box.X0, c.X0)

internal/deepdoc/parser/pdf/layout/chars_boxes_test.go

@@ -104,4 +104,44 @@ func TestLineToTextBox(t *testing.T) {
 			t.Errorf("Text = %q, want 'X'", box.Text)
 		}
 	})
+	// Word boundaries must be recovered for non-Latin scripts too. Before the fix
+	// the gap rule was gated on an ASCII-only regex, so Cyrillic chars never got a
+	// space regardless of the gap between them (a whole page welded into one token).
+	t.Run("cyrillic word gap", func(t *testing.T) {
+		chars := []pdf.TextChar{
+			// "Ок" — kern-sized gap (2, ~0.25 width): same word
+			{X0: 50, X1: 58, Top: 100, Bottom: 112, Text: "О"},
+			{X0: 60, X1: 68, Top: 100, Bottom: 112, Text: "к"},
+			// large gap before "н": a word boundary
+			{X0: 120, X1: 128, Top: 100, Bottom: 112, Text: "н"},
+		}
+		box := LineToTextBox(chars)
+		if box.Text != "Ок н" {
+			t.Errorf("Text = %q, want 'Ок н' (space inserted only at the word gap)", box.Text)
+		}
+	})
+	// CJK does not separate words with spaces, so a gap between CJK glyphs is
+	// ordinary tracking and must NOT become one, however wide.
+	t.Run("cjk gap is not a word boundary", func(t *testing.T) {
+		chars := []pdf.TextChar{
+			{X0: 12, X1: 18, Top: 100, Bottom: 112, Text: "乙"},
+			// gap of 4 on 6-wide glyphs — far over the 0.25*mean threshold
+			{X0: 22, X1: 28, Top: 100, Bottom: 112, Text: "丙"},
+		}
+		box := LineToTextBox(chars)
+		if box.Text != "乙丙" {
+			t.Errorf("Text = %q, want '乙丙' (no space between CJK glyphs)", box.Text)
+		}
+	})
+	// A CJK/Latin boundary is likewise not a space the geometry may invent.
+	t.Run("cjk adjacent to latin gets no invented space", func(t *testing.T) {
+		chars := []pdf.TextChar{
+			{X0: 12, X1: 18, Top: 100, Bottom: 112, Text: "乙"},
+			{X0: 22, X1: 28, Top: 100, Bottom: 112, Text: "A"},
+		}
+		box := LineToTextBox(chars)
+		if box.Text != "乙A" {
+			t.Errorf("Text = %q, want '乙A'", box.Text)
+		}
+	})
 }

internal/deepdoc/parser/pdf/parser_ocr.go

@@ -273,10 +273,12 @@ func (p *Parser) buildTextBoxes(ctx context.Context, pageImg image.Image,
 					}
 				}
 			}
+			// PUA / unmapped-glyph garbage: genuine noise, re-OCR regardless of script.
 			if totalCnt > 0 && float64(garbledCnt)/float64(totalCnt) >= 0.5 {
 				tb.Text = ""
-			}
-			if tb.Text != "" && util.IsGarbledByFontEncoding(boxChars[i], 5) {
+			} else if tb.Text != "" && util.OcrCanRepresent(tb.Text) && util.IsGarbledByFontEncoding(boxChars[i], 5) {
+				// Font-encoding garbling, but skipped for a script the recogniser
+				// cannot spell -- OCR would only produce garbage.
 				tb.Text = ""
 			}
 		}

internal/deepdoc/parser/pdf/util/garbled.go

@@ -1,11 +1,15 @@
 package util
 
 import (
+	"os"
+	"path/filepath"
 	"regexp"
 	"strings"
+	"sync"
 	"unicode"
 
 	pdf "ragflow/internal/deepdoc/parser/pdf/type"
+	"ragflow/internal/utility"
 )
 
 // CIDPattern matches pdfminer's CID placeholder like "(cid:123)".
@@ -201,6 +205,66 @@ func IsGarbledByFontEncoding(chars []pdf.TextChar, minChars int) bool {
 	return cjkRatio < 0.05 && punctRatio > 0.4
 }
 
+// ocrCoverageThreshold is the minimum fraction of a text's characters that the
+// OCR recogniser's alphabet must cover for an OCR fallback to be worthwhile.
+const ocrCoverageThreshold = 0.8
+
+var (
+	ocrAlphabetOnce sync.Once
+	ocrAlphabet     map[rune]struct{}
+)
+
+// loadOCRAlphabet reads the recogniser's character dictionary once
+// (rag/res/deepdoc/ocr.res, provisioned next to the OCR model). On any error it
+// returns an empty set, which OcrCanRepresent treats as "unknown alphabet" and
+// so allows the existing fallback behaviour.
+func loadOCRAlphabet() map[rune]struct{} {
+	ocrAlphabetOnce.Do(func() {
+		ocrAlphabet = map[rune]struct{}{}
+		data, err := os.ReadFile(filepath.Join(utility.GetProjectRoot(), "rag", "res", "deepdoc", "ocr.res"))
+		if err != nil {
+			return
+		}
+		for _, r := range string(data) {
+			ocrAlphabet[r] = struct{}{}
+		}
+	})
+	return ocrAlphabet
+}
+
+// OcrCanRepresent reports whether the OCR recogniser's alphabet covers text
+// well enough that an OCR pass could improve on it. ocr.res is CJK+Latin
+// (~6300 CJK / 52 Latin / 6 Cyrillic), so it covers ~100% of an English page but
+// only ~6% of a Cyrillic one. Discarding a usable text layer in favour of a
+// recogniser that cannot spell the script only produces garbage, so the
+// garbled-text fallbacks are skipped when this returns false.
+func OcrCanRepresent(text string) bool {
+	return ocrCanRepresent(text, loadOCRAlphabet(), ocrCoverageThreshold)
+}
+
+func ocrCanRepresent(text string, alphabet map[rune]struct{}, minCoverage float64) bool {
+	if strings.TrimSpace(text) == "" {
+		return true
+	}
+	if len(alphabet) == 0 { // unknown alphabet: preserve existing behaviour
+		return true
+	}
+	letters, covered := 0, 0
+	for _, r := range text {
+		if unicode.IsSpace(r) {
+			continue
+		}
+		letters++
+		if _, ok := alphabet[r]; ok {
+			covered++
+		}
+	}
+	if letters == 0 {
+		return true
+	}
+	return float64(covered)/float64(letters) >= minCoverage
+}
+
 // catOf returns "Cs" for surrogates, "Cn" for unassigned code points
 // (not in any Unicode category), and "" for everything else.
 // Python unicodedata.category() returns "Cc" for control chars, "Cn" only
@@ -239,12 +303,18 @@ func IsGarbledPage(chars []pdf.TextChar) bool {
 		fullText.WriteString(c.Text)
 	}
 	text := fullText.String()
+	// PUA / unmapped-glyph garbage: genuine noise, re-OCR regardless of script.
 	if IsGarbledText(text, 0.3) {
 		return true
 	}
 	if PdfOxideUnmappedGarbled(text) && IsScanNoise(text) {
 		return true
 	}
+	// Beyond genuine garbage, keep a clean text layer the recogniser cannot spell:
+	// OCR of a script absent from ocr.res only produces garbage.
+	if !OcrCanRepresent(text) {
+		return false
+	}
 	if IsGarbledByFontEncoding(chars, 20) {
 		return true
 	}

internal/deepdoc/parser/pdf/util/garbled_ocr_test.go

@@ -0,0 +1,45 @@
+package util
+
+import "testing"
+
+// runeSet builds an alphabet from a string, mirroring how loadOCRAlphabet reads
+// ocr.res into a rune set.
+func runeSet(s string) map[rune]struct{} {
+	m := make(map[rune]struct{})
+	for _, r := range s {
+		m[r] = struct{}{}
+	}
+	return m
+}
+
+// TestOcrCanRepresent covers the guard that keeps a usable text layer when the
+// OCR recogniser's alphabet cannot spell the script. The real ocr.res is
+// CJK+Latin (52 Latin / 6 Cyrillic), giving English ~100% coverage and Russian
+// ~6% — so a Latin-only alphabet reproduces the split here.
+func TestOcrCanRepresent(t *testing.T) {
+	latin := runeSet("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789 .,-")
+
+	tests := []struct {
+		name        string
+		text        string
+		alphabet    map[rune]struct{}
+		minCoverage float64
+		want        bool
+	}{
+		{"english is representable", "Minimum edit distance", latin, 0.8, true},
+		{"russian is not representable", "О книге как-то иначе", latin, 0.8, false},
+		{"empty text is representable", "", latin, 0.8, true},
+		{"whitespace only is representable", "   ", latin, 0.8, true},
+		{"unknown alphabet preserves behaviour", "О книге", runeSet(""), 0.8, true},
+		{"exactly at threshold", "abcdx", runeSet("abcd"), 0.8, true}, // 4/5 == 0.80
+		{"below threshold", "abcxx", runeSet("abcd"), 0.8, false},     // 3/5 == 0.60
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got := ocrCanRepresent(tt.text, tt.alphabet, tt.minCoverage)
+			if got != tt.want {
+				t.Errorf("ocrCanRepresent(%q, ...) = %v, want %v", tt.text, got, tt.want)
+			}
+		})
+	}
+}

test/unit_test/deepdoc/parser/test_pdf_garbled_detection.py

@@ -454,3 +454,91 @@ class TestLayoutRecognizerIsGarbage:
 
     def test_cid_with_large_number(self):
         assert _is_garbage({"text": "(cid:99999)"}) is True
+
+
+# ---------------------------------------------------------------------------
+# Tests for _ocr_can_represent
+# ---------------------------------------------------------------------------
+
+
+class TestOcrCanRepresent:
+    """The OCR fallback is skipped for a script the recogniser cannot spell.
+
+    ocr.res is CJK+Latin, so it covers ~100% of an English page but only ~6% of a
+    Cyrillic one; re-OCRing a script it cannot spell only produces garbage.
+    """
+
+    _LATIN = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789 .,-"
+
+    def test_english_is_representable(self, monkeypatch):
+        monkeypatch.setattr(_Parser, "_OCR_ALPHABET", set(self._LATIN))
+        assert _Parser._ocr_can_represent("Minimum edit distance") is True
+
+    def test_russian_is_not_representable(self, monkeypatch):
+        monkeypatch.setattr(_Parser, "_OCR_ALPHABET", set(self._LATIN))
+        assert _Parser._ocr_can_represent("О книге как-то иначе") is False
+
+    def test_empty_text_is_representable(self, monkeypatch):
+        monkeypatch.setattr(_Parser, "_OCR_ALPHABET", set(self._LATIN))
+        assert _Parser._ocr_can_represent("") is True
+        assert _Parser._ocr_can_represent("   ") is True
+
+    def test_unknown_alphabet_is_representable(self, monkeypatch):
+        monkeypatch.setattr(_Parser, "_OCR_ALPHABET", set())
+        assert _Parser._ocr_can_represent("О книге") is True
+
+    def test_coverage_threshold_boundary(self, monkeypatch):
+        monkeypatch.setattr(_Parser, "_OCR_ALPHABET", set("abcd"))
+        assert _Parser._ocr_can_represent("abcdx", min_coverage=0.8) is True  # 4/5
+        assert _Parser._ocr_can_represent("abcxx", min_coverage=0.8) is False  # 3/5
+
+
+# ---------------------------------------------------------------------------
+# Tests for _insert_word_spaces (geometric word-boundary recovery)
+# ---------------------------------------------------------------------------
+
+
+def _line(words, char_w=5.0, word_gap=2.0, kern=0.0, x0=0.0):
+    """Build a spaceless pdfplumber-style char stream: fixed-width glyphs,
+    ``word_gap`` pt between words, ``kern`` pt between glyphs inside a word."""
+    chars = []
+    x = x0
+    for wi, w in enumerate(words):
+        if wi:
+            x += word_gap
+        for ci, ch in enumerate(w):
+            if ci:
+                x += kern
+            chars.append({"text": ch, "x0": x, "x1": x + char_w, "width": char_w})
+            x += char_w
+    return chars
+
+
+def _spaced(chars):
+    _Parser._insert_word_spaces(chars)
+    return "".join(c["text"] for c in chars)
+
+
+class TestInsertWordSpaces:
+    def test_cyrillic_words_are_split(self):
+        # mean width 5 -> threshold 0.25*5 = 1.25pt; the 2.0pt word gap exceeds it.
+        assert _spaced(_line(["Окниге", "както"])).split() == ["Окниге", "както"]
+
+    def test_latin_word_gap_becomes_a_space(self):
+        assert _spaced(_line(["Hello", "World"])) == "Hello World"
+
+    def test_intra_word_kern_is_not_a_space(self):
+        # A 0.3pt kern inside a word is well below the threshold.
+        assert _spaced(_line(["Large"], kern=0.3)) == "Large"
+
+    def test_cjk_gap_is_not_a_word_boundary(self):
+        # CJK does not separate words with spaces: a wide gap between CJK glyphs
+        # is ordinary tracking, not a boundary.
+        assert _spaced(_line(["乙", "丙"])) == "乙丙"
+
+    def test_cjk_adjacent_to_latin_gets_no_invented_space(self):
+        assert _spaced(_line(["乙", "A"])) == "乙A"
+
+    def test_japanese_and_korean_gaps_are_not_boundaries(self):
+        assert _spaced(_line(["ア", "イ"])) == "アイ"
+        assert _spaced(_line(["한", "글"])) == "한글"