fix(deepdoc): recover word boundaries for non-Latin scripts; skip OCR fallback the recogniser can't serve (#16958)

internal/deepdoc/parser/pdf/layout/chars_boxes.go

@@ -2,7 +2,6 @@ package layout
 
 import (
 	"math"
-	"regexp"
 	"sort"
 	"strings"
 
@@ -10,6 +9,17 @@ import (
 	util "ragflow/internal/deepdoc/parser/pdf/util"
 )
 
+// hasCJK reports whether s contains a CJK rune. CJK scripts do not separate
+// words with spaces, so geometric gaps between their glyphs must not become one.
+func hasCJK(s string) bool {
+	for _, r := range s {
+		if pdf.IsCJK(r) {
+			return true
+		}
+	}
+	return false
+}
+
 // CharsToBoxes converts raw characters to initial text boxes by grouping
 // characters into lines based on vertical overlap.
 //
@@ -169,12 +179,6 @@ func verticalOverlap(a, b pdf.TextChar) bool {
 }
 
 // lineToTextBox converts a line of characters to a single pdf.TextBox.
-// asciiWordPattern matches strings composed entirely of ASCII word
-// characters. Python uses re.match (prefix match) — the stricter
-// full-string match here is equivalent in practice because each
-// pdf.TextChar.Text is a single rune, so prevText+currText ≤ 2 chars.
-// Python: pdf_parser.py:1528 re.match(r"[0-9a-zA-Z,.:;!%]+", ...)
-var asciiWordPattern = regexp.MustCompile(`^[0-9a-zA-Z,.:;!%]+$`)
 
 func LineToTextBox(chars []pdf.TextChar) pdf.TextBox {
 	if len(chars) == 0 {
@@ -186,21 +190,32 @@ func LineToTextBox(chars []pdf.TextChar) pdf.TextBox {
 		Top:    chars[0].Top,
 		Bottom: chars[0].Bottom,
 	}
+	// Recover missing spaces from geometry: many PDFs encode no space glyphs and
+	// separate words by positioning alone. For a script that separates words with
+	// spaces, a gap wider than a fraction of the mean char width is a word
+	// boundary; intra-word kerns fall well below it. CJK is excluded: it does not
+	// write inter-word spaces, so a gap between CJK glyphs is ordinary tracking,
+	// not a boundary.
+	var sumWidth float64
+	var nWidth int
+	for _, c := range chars {
+		if strings.TrimSpace(c.Text) != "" {
+			sumWidth += c.X1 - c.X0
+			nWidth++
+		}
+	}
+	var spaceGap float64
+	if nWidth > 0 {
+		spaceGap = (sumWidth / float64(nWidth)) * 0.25
+	}
 	var textParts []string
 	for i, c := range chars {
-		// Insert space between adjacent ASCII words with a visible gap.
-		// Python: pdf_parser.py:1524-1532 __img_ocr space insertion.
-		if i > 0 {
+		if i > 0 && spaceGap > 0 {
 			prev := chars[i-1]
-			prevText := strings.TrimSpace(prev.Text)
-			currText := strings.TrimSpace(c.Text)
-			if prevText != "" && currText != "" {
-				gap := c.X0 - prev.X1
-				minWidth := math.Min(c.X1-c.X0, prev.X1-prev.X0)
-				if gap >= minWidth/2 &&
-					asciiWordPattern.MatchString(prevText+currText) {
-					textParts = append(textParts, " ")
-				}
+			if strings.TrimSpace(prev.Text) != "" && strings.TrimSpace(c.Text) != "" &&
+				!hasCJK(prev.Text) && !hasCJK(c.Text) &&
+				c.X0-prev.X1 > spaceGap {
+				textParts = append(textParts, " ")
 			}
 		}
 		box.X0 = math.Min(box.X0, c.X0)

internal/deepdoc/parser/pdf/layout/chars_boxes_test.go

@@ -104,4 +104,44 @@ func TestLineToTextBox(t *testing.T) {
 			t.Errorf("Text = %q, want 'X'", box.Text)
 		}
 	})
+	// Word boundaries must be recovered for non-Latin scripts too. Before the fix
+	// the gap rule was gated on an ASCII-only regex, so Cyrillic chars never got a
+	// space regardless of the gap between them (a whole page welded into one token).
+	t.Run("cyrillic word gap", func(t *testing.T) {
+		chars := []pdf.TextChar{
+			// "Ок" — kern-sized gap (2, ~0.25 width): same word
+			{X0: 50, X1: 58, Top: 100, Bottom: 112, Text: "О"},
+			{X0: 60, X1: 68, Top: 100, Bottom: 112, Text: "к"},
+			// large gap before "н": a word boundary
+			{X0: 120, X1: 128, Top: 100, Bottom: 112, Text: "н"},
+		}
+		box := LineToTextBox(chars)
+		if box.Text != "Ок н" {
+			t.Errorf("Text = %q, want 'Ок н' (space inserted only at the word gap)", box.Text)
+		}
+	})
+	// CJK does not separate words with spaces, so a gap between CJK glyphs is
+	// ordinary tracking and must NOT become one, however wide.
+	t.Run("cjk gap is not a word boundary", func(t *testing.T) {
+		chars := []pdf.TextChar{
+			{X0: 12, X1: 18, Top: 100, Bottom: 112, Text: "乙"},
+			// gap of 4 on 6-wide glyphs — far over the 0.25*mean threshold
+			{X0: 22, X1: 28, Top: 100, Bottom: 112, Text: "丙"},
+		}
+		box := LineToTextBox(chars)
+		if box.Text != "乙丙" {
+			t.Errorf("Text = %q, want '乙丙' (no space between CJK glyphs)", box.Text)
+		}
+	})
+	// A CJK/Latin boundary is likewise not a space the geometry may invent.
+	t.Run("cjk adjacent to latin gets no invented space", func(t *testing.T) {
+		chars := []pdf.TextChar{
+			{X0: 12, X1: 18, Top: 100, Bottom: 112, Text: "乙"},
+			{X0: 22, X1: 28, Top: 100, Bottom: 112, Text: "A"},
+		}
+		box := LineToTextBox(chars)
+		if box.Text != "乙A" {
+			t.Errorf("Text = %q, want '乙A'", box.Text)
+		}
+	})
 }

internal/deepdoc/parser/pdf/parser_ocr.go

@@ -273,10 +273,12 @@ func (p *Parser) buildTextBoxes(ctx context.Context, pageImg image.Image,
 					}
 				}
 			}
+			// PUA / unmapped-glyph garbage: genuine noise, re-OCR regardless of script.
 			if totalCnt > 0 && float64(garbledCnt)/float64(totalCnt) >= 0.5 {
 				tb.Text = ""
-			}
-			if tb.Text != "" && util.IsGarbledByFontEncoding(boxChars[i], 5) {
+			} else if tb.Text != "" && util.OcrCanRepresent(tb.Text) && util.IsGarbledByFontEncoding(boxChars[i], 5) {
+				// Font-encoding garbling, but skipped for a script the recogniser
+				// cannot spell -- OCR would only produce garbage.
 				tb.Text = ""
 			}
 		}

internal/deepdoc/parser/pdf/util/garbled.go

@@ -1,11 +1,15 @@
 package util
 
 import (
+	"os"
+	"path/filepath"
 	"regexp"
 	"strings"
+	"sync"
 	"unicode"
 
 	pdf "ragflow/internal/deepdoc/parser/pdf/type"
+	"ragflow/internal/utility"
 )
 
 // CIDPattern matches pdfminer's CID placeholder like "(cid:123)".
@@ -201,6 +205,66 @@ func IsGarbledByFontEncoding(chars []pdf.TextChar, minChars int) bool {
 	return cjkRatio < 0.05 && punctRatio > 0.4
 }
 
+// ocrCoverageThreshold is the minimum fraction of a text's characters that the
+// OCR recogniser's alphabet must cover for an OCR fallback to be worthwhile.
+const ocrCoverageThreshold = 0.8
+
+var (
+	ocrAlphabetOnce sync.Once
+	ocrAlphabet     map[rune]struct{}
+)
+
+// loadOCRAlphabet reads the recogniser's character dictionary once
+// (rag/res/deepdoc/ocr.res, provisioned next to the OCR model). On any error it
+// returns an empty set, which OcrCanRepresent treats as "unknown alphabet" and
+// so allows the existing fallback behaviour.
+func loadOCRAlphabet() map[rune]struct{} {
+	ocrAlphabetOnce.Do(func() {
+		ocrAlphabet = map[rune]struct{}{}
+		data, err := os.ReadFile(filepath.Join(utility.GetProjectRoot(), "rag", "res", "deepdoc", "ocr.res"))
+		if err != nil {
+			return
+		}
+		for _, r := range string(data) {
+			ocrAlphabet[r] = struct{}{}
+		}
+	})
+	return ocrAlphabet
+}
+
+// OcrCanRepresent reports whether the OCR recogniser's alphabet covers text
+// well enough that an OCR pass could improve on it. ocr.res is CJK+Latin
+// (~6300 CJK / 52 Latin / 6 Cyrillic), so it covers ~100% of an English page but
+// only ~6% of a Cyrillic one. Discarding a usable text layer in favour of a
+// recogniser that cannot spell the script only produces garbage, so the
+// garbled-text fallbacks are skipped when this returns false.
+func OcrCanRepresent(text string) bool {
+	return ocrCanRepresent(text, loadOCRAlphabet(), ocrCoverageThreshold)
+}
+
+func ocrCanRepresent(text string, alphabet map[rune]struct{}, minCoverage float64) bool {
+	if strings.TrimSpace(text) == "" {
+		return true
+	}
+	if len(alphabet) == 0 { // unknown alphabet: preserve existing behaviour
+		return true
+	}
+	letters, covered := 0, 0
+	for _, r := range text {
+		if unicode.IsSpace(r) {
+			continue
+		}
+		letters++
+		if _, ok := alphabet[r]; ok {
+			covered++
+		}
+	}
+	if letters == 0 {
+		return true
+	}
+	return float64(covered)/float64(letters) >= minCoverage
+}
+
 // catOf returns "Cs" for surrogates, "Cn" for unassigned code points
 // (not in any Unicode category), and "" for everything else.
 // Python unicodedata.category() returns "Cc" for control chars, "Cn" only
@@ -239,12 +303,18 @@ func IsGarbledPage(chars []pdf.TextChar) bool {
 		fullText.WriteString(c.Text)
 	}
 	text := fullText.String()
+	// PUA / unmapped-glyph garbage: genuine noise, re-OCR regardless of script.
 	if IsGarbledText(text, 0.3) {
 		return true
 	}
 	if PdfOxideUnmappedGarbled(text) && IsScanNoise(text) {
 		return true
 	}
+	// Beyond genuine garbage, keep a clean text layer the recogniser cannot spell:
+	// OCR of a script absent from ocr.res only produces garbage.
+	if !OcrCanRepresent(text) {
+		return false
+	}
 	if IsGarbledByFontEncoding(chars, 20) {
 		return true
 	}

internal/deepdoc/parser/pdf/util/garbled_ocr_test.go

@@ -0,0 +1,45 @@
+package util
+
+import "testing"
+
+// runeSet builds an alphabet from a string, mirroring how loadOCRAlphabet reads
+// ocr.res into a rune set.
+func runeSet(s string) map[rune]struct{} {
+	m := make(map[rune]struct{})
+	for _, r := range s {
+		m[r] = struct{}{}
+	}
+	return m
+}
+
+// TestOcrCanRepresent covers the guard that keeps a usable text layer when the
+// OCR recogniser's alphabet cannot spell the script. The real ocr.res is
+// CJK+Latin (52 Latin / 6 Cyrillic), giving English ~100% coverage and Russian
+// ~6% — so a Latin-only alphabet reproduces the split here.
+func TestOcrCanRepresent(t *testing.T) {
+	latin := runeSet("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789 .,-")
+
+	tests := []struct {
+		name        string
+		text        string
+		alphabet    map[rune]struct{}
+		minCoverage float64
+		want        bool
+	}{
+		{"english is representable", "Minimum edit distance", latin, 0.8, true},
+		{"russian is not representable", "О книге как-то иначе", latin, 0.8, false},
+		{"empty text is representable", "", latin, 0.8, true},
+		{"whitespace only is representable", "   ", latin, 0.8, true},
+		{"unknown alphabet preserves behaviour", "О книге", runeSet(""), 0.8, true},
+		{"exactly at threshold", "abcdx", runeSet("abcd"), 0.8, true}, // 4/5 == 0.80
+		{"below threshold", "abcxx", runeSet("abcd"), 0.8, false},     // 3/5 == 0.60
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got := ocrCanRepresent(tt.text, tt.alphabet, tt.minCoverage)
+			if got != tt.want {
+				t.Errorf("ocrCanRepresent(%q, ...) = %v, want %v", tt.text, got, tt.want)
+			}
+		})
+	}
+}