fix(nlp): differentiate alphabetic OOV term weights (#18470)

### Summary

Closes #18414.

`rag/res/term.freq` is not shipped, and both term-weight implementations
therefore assigned the same `300` fallback frequency to every lowercase
Latin token. With no tokenizer frequency, NER, or POS signal, function
words and content words received identical lexical boosts.

This PR adds the same bounded out-of-vocabulary prior to Python and Go:

- Use it only when the explicit DF dictionary or tokenizer has no
frequency.
- Count Latin, Greek, and Cyrillic letters, including uppercase and
accented forms.
- Keep the existing frequency of `300` for words up to three letters,
halve it every two additional letters, and clamp it at `10`.
- Reject digits, underscores, and logographic terms so Chinese and other
existing fine-grained-tokenizer paths are unchanged.
- Treat an absent optional `term.freq` as the supported fallback path
without a startup warning, while still logging inaccessible or malformed
dictionaries.

A corpus-derived table was intentionally not added: that would require
provenance/licensing decisions, language detection, and handling
cross-language homographs. The bounded prior is deterministic,
dependency-free, and fixes the equal-weight degradation for
whitespace-delimited alphabetic languages without claiming
corpus-specific precision.

Python and Go consume one shared fixture covering ASCII, uppercase,
accented Latin, Greek, Cyrillic, separators, invalid mixed tokens, and a
CJK non-match. Both sides also verify the issue's ordering (`was <
largest < supplier < equipment`) and that an explicit dictionary entry
still takes precedence.


Co-authored-by: Loong <184861530+yzl0ng@users.noreply.github.com>

internal/service/nlp/term_weight.go

@@ -23,6 +23,7 @@ import (
 	"regexp"
 	"strconv"
 	"strings"
+	"unicode"
 
 	"ragflow/internal/tokenizer"
 
@@ -87,13 +88,39 @@ func initStopWords() map[string]struct{} {
 	return stopWords
 }
 
+// alphabeticOOVFrequency estimates frequency for an alphabetic out-of-vocabulary
+// term. It preserves the previous frequency of 300 for short words, then halves
+// it every two letters, with a floor of 10. Latin, Greek, and Cyrillic scripts
+// are supported while logographic terms keep their existing tokenizer path.
+func alphabeticOOVFrequency(term string) (float64, bool) {
+	letterCount := 0
+	for _, r := range term {
+		if unicode.IsLetter(r) && unicode.In(r, unicode.Latin, unicode.Greek, unicode.Cyrillic) {
+			letterCount++
+			continue
+		}
+		if r != ' ' && r != '.' && r != '-' {
+			return 0, false
+		}
+	}
+	if letterCount == 0 {
+		return 0, false
+	}
+
+	exponent := float64(max(0, letterCount-3)) / 2
+	frequency := math.Round(300 / math.Pow(2, exponent))
+	return math.Max(10, frequency), true
+}
+
 // loadDict loads a dictionary file
 // Format: term\tfreq or just term
 func loadDict(fnm string) map[string]int {
 	res := make(map[string]int)
 	data, err := os.ReadFile(fnm)
 	if err != nil {
-		common.Warn("Failed to load dictionary", zap.String("file", fnm), zap.Error(err))
+		if !os.IsNotExist(err) {
+			common.Warn("Failed to load dictionary", zap.String("file", fnm), zap.Error(err))
+		}
 		return res
 	}
 
@@ -285,7 +312,6 @@ func (d *TermWeightDealer) Weights(tks []string, preprocess bool) []TermWeight {
 	numPattern := regexp.MustCompile("^[0-9,.]{2,}$")
 	shortLetterPattern := regexp.MustCompile("^[a-z]{1,2}$")
 	numSpacePattern := regexp.MustCompile("^[0-9. -]{2,}$")
-	letterPattern := regexp.MustCompile("^[a-z. -]+$")
 
 	// ner weight function
 	nerWeight := func(t string) float64 {
@@ -338,8 +364,10 @@ func (d *TermWeightDealer) Weights(tks []string, preprocess bool) []TermWeight {
 		}
 		// Use tokenizer's freq function
 		s := tokenizer.GetTermFreq(t)
-		if s == 0 && letterPattern.MatchString(t) {
-			return 300
+		if s == 0 {
+			if oovFrequency, ok := alphabeticOOVFrequency(t); ok {
+				return oovFrequency
+			}
 		}
 		if s == 0 && len([]rune(t)) >= 4 {
 			// Try fine-grained tokenization
@@ -385,8 +413,8 @@ func (d *TermWeightDealer) Weights(tks []string, preprocess bool) []TermWeight {
 		if v, ok := d.df[t]; ok {
 			return float64(v) + 3
 		}
-		if letterPattern.MatchString(t) {
-			return 300
+		if oovFrequency, ok := alphabeticOOVFrequency(t); ok {
+			return oovFrequency
 		}
 		if len([]rune(t)) >= 4 {
 			// Use fine-grained tokenization

internal/service/nlp/term_weight_test.go

@@ -15,6 +15,7 @@
 package nlp
 
 import (
+	"encoding/json"
 	"os"
 	"path/filepath"
 	"reflect"
@@ -23,6 +24,74 @@ import (
 	"testing"
 )
 
+type oovFrequencyFixture struct {
+	Term      string   `json:"term"`
+	Frequency *float64 `json:"frequency"`
+}
+
+func repositoryRoot(t *testing.T) string {
+	t.Helper()
+	dir, err := os.Getwd()
+	if err != nil {
+		t.Fatalf("get working directory: %v", err)
+	}
+	for {
+		if _, err := os.Stat(filepath.Join(dir, "go.mod")); err == nil {
+			return dir
+		}
+		parent := filepath.Dir(dir)
+		if parent == dir {
+			t.Fatal("repository root not found")
+		}
+		dir = parent
+	}
+}
+
+func TestAlphabeticOOVFrequencyMatchesSharedFixture(t *testing.T) {
+	fixturePath := filepath.Join(repositoryRoot(t), "test", "fixtures", "term_weight_oov.json")
+	data, err := os.ReadFile(fixturePath)
+	if err != nil {
+		t.Fatalf("read OOV fixture: %v", err)
+	}
+
+	var cases []oovFrequencyFixture
+	if err := json.Unmarshal(data, &cases); err != nil {
+		t.Fatalf("decode OOV fixture: %v", err)
+	}
+	for _, testCase := range cases {
+		got, ok := alphabeticOOVFrequency(testCase.Term)
+		if testCase.Frequency == nil {
+			if ok {
+				t.Errorf("alphabeticOOVFrequency(%q) = %v, true; want no fallback", testCase.Term, got)
+			}
+			continue
+		}
+		if !ok || got != *testCase.Frequency {
+			t.Errorf("alphabeticOOVFrequency(%q) = %v, %v; want %v, true", testCase.Term, got, ok, *testCase.Frequency)
+		}
+	}
+}
+
+func TestOOVWeightsFavorLongerContentTerms(t *testing.T) {
+	d := NewTermWeightDealer(t.TempDir())
+	terms := []string{"was", "largest", "supplier", "equipment"}
+	weights := d.Weights(terms, false)
+	if len(weights) != len(terms) {
+		t.Fatalf("Weights returned %d terms; want %d", len(weights), len(terms))
+	}
+	for i := 1; i < len(weights); i++ {
+		if weights[i-1].Weight >= weights[i].Weight {
+			t.Fatalf("weights are not increasing with OOV term length: %v", weights)
+		}
+	}
+
+	d.df["equipment"] = 1_000_000
+	weights = d.Weights([]string{"supplier", "equipment"}, false)
+	if weights[1].Weight >= weights[0].Weight {
+		t.Fatalf("dictionary frequency should take precedence over the OOV prior: %v", weights)
+	}
+}
+
 // TestNewTermWeightDealer tests the constructor
 func TestNewTermWeightDealer(t *testing.T) {
 	// Test with empty resPath

rag/nlp/term_weight.py

@@ -14,14 +14,43 @@
 #  limitations under the License.
 #
 
+import json
 import logging
 import math
-import json
-import re
 import os
+import re
+import unicodedata
+
 import numpy as np
-from rag.nlp import rag_tokenizer
+
 from common.file_utils import get_project_base_directory
+from rag.nlp import rag_tokenizer
+
+
+def _alphabetic_oov_frequency(term):
+    """Estimate frequency for a Latin, Greek, or Cyrillic OOV term."""
+    letter_count = 0
+    for char in term:
+        if char.isascii():
+            if char.isalpha():
+                letter_count += 1
+            elif char not in " .-":
+                return None
+            continue
+        script_name = unicodedata.name(char, "").split(" ", 1)[0]
+        if char.isalpha() and script_name in {"LATIN", "GREEK", "CYRILLIC"}:
+            letter_count += 1
+        elif char not in " .-":
+            return None
+    if not letter_count:
+        return None
+
+    # Preserve the old frequency (300) for short words, then halve it every
+    # two letters. This is a bounded, language-neutral prior: longer unknown
+    # words are usually more informative, without outranking known terms by an
+    # unbounded amount.
+    exponent = max(0, letter_count - 3) / 2
+    return max(10, round(300 / (2**exponent)))
 
 
 class Dealer:
@@ -89,10 +118,13 @@ class Dealer:
                 self.ne = json.load(f)
         except Exception:
             logging.warning("Load ner.json FAIL!")
+        freq_path = os.path.join(fnm, "term.freq")
         try:
-            self.df = load_dict(os.path.join(fnm, "term.freq"))
-        except Exception:
-            logging.warning("Load term.freq FAIL!")
+            self.df = load_dict(freq_path)
+        except FileNotFoundError:
+            pass
+        except (OSError, ValueError):
+            logging.warning("Load term.freq FAIL!", exc_info=True)
 
     def pretoken(self, txt, num=False, stpwd=True):
         patt = [r"[~—\t @#%!<>,\.\?\":;'\{\}\[\]_=\(\)\|，。？》•●○↓《；‘’：“”【¥ 】…￥！、·（）×`&\\/「」\\]"]
@@ -161,7 +193,6 @@ class Dealer:
         num_pattern = re.compile(r"[0-9,.]{2,}$")
         short_letter_pattern = re.compile(r"[a-z]{1,2}$")
         num_space_pattern = re.compile(r"[0-9. -]{2,}$")
-        letter_pattern = re.compile(r"[a-z. -]+$")
 
         def ner(t):
             if num_pattern.match(t):
@@ -189,8 +220,10 @@ class Dealer:
             if num_space_pattern.match(t):
                 return 3
             s = rag_tokenizer.freq(t)
-            if not s and letter_pattern.match(t):
-                return 300
+            if not s:
+                oov_frequency = _alphabetic_oov_frequency(t)
+                if oov_frequency is not None:
+                    return oov_frequency
             if not s:
                 s = 0
 
@@ -208,9 +241,10 @@ class Dealer:
                 return 5
             if t in self.df:
                 return self.df[t] + 3
-            elif letter_pattern.match(t):
-                return 300
-            elif len(t) >= 4:
+            oov_frequency = _alphabetic_oov_frequency(t)
+            if oov_frequency is not None:
+                return oov_frequency
+            if len(t) >= 4:
                 s = [tt for tt in rag_tokenizer.fine_grained_tokenize(t).split() if len(tt) > 1]
                 if len(s) > 1:
                     return max(3, np.min([df(tt) for tt in s]) / 6.0)

test/fixtures/term_weight_oov.json

@@ -0,0 +1,17 @@
+[
+  {"term": "a", "frequency": 300},
+  {"term": "the", "frequency": 300},
+  {"term": "Café", "frequency": 212},
+  {"term": "maior", "frequency": 150},
+  {"term": "κόσμος", "frequency": 106},
+  {"term": "привет", "frequency": 106},
+  {"term": "largest", "frequency": 75},
+  {"term": "supplier", "frequency": 53},
+  {"term": "equipment", "frequency": 38},
+  {"term": "équipement", "frequency": 27},
+  {"term": "hospital-equipment", "frequency": 10},
+  {"term": "北京", "frequency": null},
+  {"term": "abc123", "frequency": null},
+  {"term": "hello_world", "frequency": null},
+  {"term": "...", "frequency": null}
+]

test/unit_test/rag/nlp/test_term_weight_oov.py

@@ -0,0 +1,74 @@
+#
+#  Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+#
+
+import json
+import logging
+from pathlib import Path
+
+from rag.nlp import rag_tokenizer, term_weight
+from rag.nlp.term_weight import Dealer, _alphabetic_oov_frequency
+
+
+def test_alphabetic_oov_frequency_matches_shared_fixture():
+    """Keep the Python fallback aligned with the fixture consumed by Go tests."""
+    fixture_path = Path(__file__).parents[3] / "fixtures" / "term_weight_oov.json"
+    cases = json.loads(fixture_path.read_text(encoding="utf-8"))
+
+    for case in cases:
+        assert _alphabetic_oov_frequency(case["term"]) == case["frequency"]
+
+
+def test_oov_weights_favor_longer_content_terms(monkeypatch):
+    """Avoid equal weights when no dictionary or tokenizer frequency is available."""
+    monkeypatch.setattr(rag_tokenizer, "freq", lambda _term: 0)
+    monkeypatch.setattr(rag_tokenizer, "tag", lambda _term: "")
+    dealer = Dealer()
+    dealer.df = {}
+
+    weights = dict(dealer.weights(["was", "largest", "supplier", "equipment"], preprocess=False))
+
+    assert weights["was"] < weights["largest"] < weights["supplier"] < weights["equipment"]
+
+    dealer.df["equipment"] = 1_000_000
+    weights = dict(dealer.weights(["supplier", "equipment"], preprocess=False))
+    assert weights["equipment"] < weights["supplier"]
+
+
+def test_missing_frequency_dictionary_is_silent(tmp_path, monkeypatch, caplog):
+    """Treat an absent optional frequency dictionary as an expected fallback."""
+    resource_dir = tmp_path / "rag" / "res"
+    resource_dir.mkdir(parents=True)
+    (resource_dir / "ner.json").write_text("{}", encoding="utf-8")
+    monkeypatch.setattr(term_weight, "get_project_base_directory", lambda: str(tmp_path))
+
+    with caplog.at_level(logging.WARNING):
+        Dealer()
+
+    assert "Load term.freq FAIL!" not in caplog.text
+
+
+def test_malformed_frequency_dictionary_logs_warning(tmp_path, monkeypatch, caplog):
+    """Expose malformed dictionaries instead of silently enabling the fallback."""
+    resource_dir = tmp_path / "rag" / "res"
+    resource_dir.mkdir(parents=True)
+    (resource_dir / "ner.json").write_text("{}", encoding="utf-8")
+    (resource_dir / "term.freq").write_text("term\tnot-a-number\n", encoding="utf-8")
+    monkeypatch.setattr(term_weight, "get_project_base_directory", lambda: str(tmp_path))
+
+    with caplog.at_level(logging.WARNING):
+        Dealer()
+
+    assert "Load term.freq FAIL!" in caplog.text