diff --git a/internal/service/nlp/term_weight.go b/internal/service/nlp/term_weight.go index 362f656b34..984b2df613 100644 --- a/internal/service/nlp/term_weight.go +++ b/internal/service/nlp/term_weight.go @@ -23,6 +23,7 @@ import ( "regexp" "strconv" "strings" + "unicode" "ragflow/internal/tokenizer" @@ -87,13 +88,39 @@ func initStopWords() map[string]struct{} { return stopWords } +// alphabeticOOVFrequency estimates frequency for an alphabetic out-of-vocabulary +// term. It preserves the previous frequency of 300 for short words, then halves +// it every two letters, with a floor of 10. Latin, Greek, and Cyrillic scripts +// are supported while logographic terms keep their existing tokenizer path. +func alphabeticOOVFrequency(term string) (float64, bool) { + letterCount := 0 + for _, r := range term { + if unicode.IsLetter(r) && unicode.In(r, unicode.Latin, unicode.Greek, unicode.Cyrillic) { + letterCount++ + continue + } + if r != ' ' && r != '.' && r != '-' { + return 0, false + } + } + if letterCount == 0 { + return 0, false + } + + exponent := float64(max(0, letterCount-3)) / 2 + frequency := math.Round(300 / math.Pow(2, exponent)) + return math.Max(10, frequency), true +} + // loadDict loads a dictionary file // Format: term\tfreq or just term func loadDict(fnm string) map[string]int { res := make(map[string]int) data, err := os.ReadFile(fnm) if err != nil { - common.Warn("Failed to load dictionary", zap.String("file", fnm), zap.Error(err)) + if !os.IsNotExist(err) { + common.Warn("Failed to load dictionary", zap.String("file", fnm), zap.Error(err)) + } return res } @@ -285,7 +312,6 @@ func (d *TermWeightDealer) Weights(tks []string, preprocess bool) []TermWeight { numPattern := regexp.MustCompile("^[0-9,.]{2,}$") shortLetterPattern := regexp.MustCompile("^[a-z]{1,2}$") numSpacePattern := regexp.MustCompile("^[0-9. -]{2,}$") - letterPattern := regexp.MustCompile("^[a-z. -]+$") // ner weight function nerWeight := func(t string) float64 { @@ -338,8 +364,10 @@ func (d *TermWeightDealer) Weights(tks []string, preprocess bool) []TermWeight { } // Use tokenizer's freq function s := tokenizer.GetTermFreq(t) - if s == 0 && letterPattern.MatchString(t) { - return 300 + if s == 0 { + if oovFrequency, ok := alphabeticOOVFrequency(t); ok { + return oovFrequency + } } if s == 0 && len([]rune(t)) >= 4 { // Try fine-grained tokenization @@ -385,8 +413,8 @@ func (d *TermWeightDealer) Weights(tks []string, preprocess bool) []TermWeight { if v, ok := d.df[t]; ok { return float64(v) + 3 } - if letterPattern.MatchString(t) { - return 300 + if oovFrequency, ok := alphabeticOOVFrequency(t); ok { + return oovFrequency } if len([]rune(t)) >= 4 { // Use fine-grained tokenization diff --git a/internal/service/nlp/term_weight_test.go b/internal/service/nlp/term_weight_test.go index 078e5b0009..c3007af6b2 100644 --- a/internal/service/nlp/term_weight_test.go +++ b/internal/service/nlp/term_weight_test.go @@ -15,6 +15,7 @@ package nlp import ( + "encoding/json" "os" "path/filepath" "reflect" @@ -23,6 +24,74 @@ import ( "testing" ) +type oovFrequencyFixture struct { + Term string `json:"term"` + Frequency *float64 `json:"frequency"` +} + +func repositoryRoot(t *testing.T) string { + t.Helper() + dir, err := os.Getwd() + if err != nil { + t.Fatalf("get working directory: %v", err) + } + for { + if _, err := os.Stat(filepath.Join(dir, "go.mod")); err == nil { + return dir + } + parent := filepath.Dir(dir) + if parent == dir { + t.Fatal("repository root not found") + } + dir = parent + } +} + +func TestAlphabeticOOVFrequencyMatchesSharedFixture(t *testing.T) { + fixturePath := filepath.Join(repositoryRoot(t), "test", "fixtures", "term_weight_oov.json") + data, err := os.ReadFile(fixturePath) + if err != nil { + t.Fatalf("read OOV fixture: %v", err) + } + + var cases []oovFrequencyFixture + if err := json.Unmarshal(data, &cases); err != nil { + t.Fatalf("decode OOV fixture: %v", err) + } + for _, testCase := range cases { + got, ok := alphabeticOOVFrequency(testCase.Term) + if testCase.Frequency == nil { + if ok { + t.Errorf("alphabeticOOVFrequency(%q) = %v, true; want no fallback", testCase.Term, got) + } + continue + } + if !ok || got != *testCase.Frequency { + t.Errorf("alphabeticOOVFrequency(%q) = %v, %v; want %v, true", testCase.Term, got, ok, *testCase.Frequency) + } + } +} + +func TestOOVWeightsFavorLongerContentTerms(t *testing.T) { + d := NewTermWeightDealer(t.TempDir()) + terms := []string{"was", "largest", "supplier", "equipment"} + weights := d.Weights(terms, false) + if len(weights) != len(terms) { + t.Fatalf("Weights returned %d terms; want %d", len(weights), len(terms)) + } + for i := 1; i < len(weights); i++ { + if weights[i-1].Weight >= weights[i].Weight { + t.Fatalf("weights are not increasing with OOV term length: %v", weights) + } + } + + d.df["equipment"] = 1_000_000 + weights = d.Weights([]string{"supplier", "equipment"}, false) + if weights[1].Weight >= weights[0].Weight { + t.Fatalf("dictionary frequency should take precedence over the OOV prior: %v", weights) + } +} + // TestNewTermWeightDealer tests the constructor func TestNewTermWeightDealer(t *testing.T) { // Test with empty resPath diff --git a/rag/nlp/term_weight.py b/rag/nlp/term_weight.py index 43b018a429..9477945680 100644 --- a/rag/nlp/term_weight.py +++ b/rag/nlp/term_weight.py @@ -14,14 +14,43 @@ # limitations under the License. # +import json import logging import math -import json -import re import os +import re +import unicodedata + import numpy as np -from rag.nlp import rag_tokenizer + from common.file_utils import get_project_base_directory +from rag.nlp import rag_tokenizer + + +def _alphabetic_oov_frequency(term): + """Estimate frequency for a Latin, Greek, or Cyrillic OOV term.""" + letter_count = 0 + for char in term: + if char.isascii(): + if char.isalpha(): + letter_count += 1 + elif char not in " .-": + return None + continue + script_name = unicodedata.name(char, "").split(" ", 1)[0] + if char.isalpha() and script_name in {"LATIN", "GREEK", "CYRILLIC"}: + letter_count += 1 + elif char not in " .-": + return None + if not letter_count: + return None + + # Preserve the old frequency (300) for short words, then halve it every + # two letters. This is a bounded, language-neutral prior: longer unknown + # words are usually more informative, without outranking known terms by an + # unbounded amount. + exponent = max(0, letter_count - 3) / 2 + return max(10, round(300 / (2**exponent))) class Dealer: @@ -89,10 +118,13 @@ class Dealer: self.ne = json.load(f) except Exception: logging.warning("Load ner.json FAIL!") + freq_path = os.path.join(fnm, "term.freq") try: - self.df = load_dict(os.path.join(fnm, "term.freq")) - except Exception: - logging.warning("Load term.freq FAIL!") + self.df = load_dict(freq_path) + except FileNotFoundError: + pass + except (OSError, ValueError): + logging.warning("Load term.freq FAIL!", exc_info=True) def pretoken(self, txt, num=False, stpwd=True): patt = [r"[~—\t @#%!<>,\.\?\":;'\{\}\[\]_=\(\)\|,。?》•●○↓《;‘’:“”【¥ 】…¥!、·()×`&\\/「」\\]"] @@ -161,7 +193,6 @@ class Dealer: num_pattern = re.compile(r"[0-9,.]{2,}$") short_letter_pattern = re.compile(r"[a-z]{1,2}$") num_space_pattern = re.compile(r"[0-9. -]{2,}$") - letter_pattern = re.compile(r"[a-z. -]+$") def ner(t): if num_pattern.match(t): @@ -189,8 +220,10 @@ class Dealer: if num_space_pattern.match(t): return 3 s = rag_tokenizer.freq(t) - if not s and letter_pattern.match(t): - return 300 + if not s: + oov_frequency = _alphabetic_oov_frequency(t) + if oov_frequency is not None: + return oov_frequency if not s: s = 0 @@ -208,9 +241,10 @@ class Dealer: return 5 if t in self.df: return self.df[t] + 3 - elif letter_pattern.match(t): - return 300 - elif len(t) >= 4: + oov_frequency = _alphabetic_oov_frequency(t) + if oov_frequency is not None: + return oov_frequency + if len(t) >= 4: s = [tt for tt in rag_tokenizer.fine_grained_tokenize(t).split() if len(tt) > 1] if len(s) > 1: return max(3, np.min([df(tt) for tt in s]) / 6.0) diff --git a/test/fixtures/term_weight_oov.json b/test/fixtures/term_weight_oov.json new file mode 100644 index 0000000000..c7adbbafdc --- /dev/null +++ b/test/fixtures/term_weight_oov.json @@ -0,0 +1,17 @@ +[ + {"term": "a", "frequency": 300}, + {"term": "the", "frequency": 300}, + {"term": "Café", "frequency": 212}, + {"term": "maior", "frequency": 150}, + {"term": "κόσμος", "frequency": 106}, + {"term": "привет", "frequency": 106}, + {"term": "largest", "frequency": 75}, + {"term": "supplier", "frequency": 53}, + {"term": "equipment", "frequency": 38}, + {"term": "équipement", "frequency": 27}, + {"term": "hospital-equipment", "frequency": 10}, + {"term": "北京", "frequency": null}, + {"term": "abc123", "frequency": null}, + {"term": "hello_world", "frequency": null}, + {"term": "...", "frequency": null} +] diff --git a/test/unit_test/rag/nlp/test_term_weight_oov.py b/test/unit_test/rag/nlp/test_term_weight_oov.py new file mode 100644 index 0000000000..d220bf726b --- /dev/null +++ b/test/unit_test/rag/nlp/test_term_weight_oov.py @@ -0,0 +1,74 @@ +# +# Copyright 2026 The InfiniFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import json +import logging +from pathlib import Path + +from rag.nlp import rag_tokenizer, term_weight +from rag.nlp.term_weight import Dealer, _alphabetic_oov_frequency + + +def test_alphabetic_oov_frequency_matches_shared_fixture(): + """Keep the Python fallback aligned with the fixture consumed by Go tests.""" + fixture_path = Path(__file__).parents[3] / "fixtures" / "term_weight_oov.json" + cases = json.loads(fixture_path.read_text(encoding="utf-8")) + + for case in cases: + assert _alphabetic_oov_frequency(case["term"]) == case["frequency"] + + +def test_oov_weights_favor_longer_content_terms(monkeypatch): + """Avoid equal weights when no dictionary or tokenizer frequency is available.""" + monkeypatch.setattr(rag_tokenizer, "freq", lambda _term: 0) + monkeypatch.setattr(rag_tokenizer, "tag", lambda _term: "") + dealer = Dealer() + dealer.df = {} + + weights = dict(dealer.weights(["was", "largest", "supplier", "equipment"], preprocess=False)) + + assert weights["was"] < weights["largest"] < weights["supplier"] < weights["equipment"] + + dealer.df["equipment"] = 1_000_000 + weights = dict(dealer.weights(["supplier", "equipment"], preprocess=False)) + assert weights["equipment"] < weights["supplier"] + + +def test_missing_frequency_dictionary_is_silent(tmp_path, monkeypatch, caplog): + """Treat an absent optional frequency dictionary as an expected fallback.""" + resource_dir = tmp_path / "rag" / "res" + resource_dir.mkdir(parents=True) + (resource_dir / "ner.json").write_text("{}", encoding="utf-8") + monkeypatch.setattr(term_weight, "get_project_base_directory", lambda: str(tmp_path)) + + with caplog.at_level(logging.WARNING): + Dealer() + + assert "Load term.freq FAIL!" not in caplog.text + + +def test_malformed_frequency_dictionary_logs_warning(tmp_path, monkeypatch, caplog): + """Expose malformed dictionaries instead of silently enabling the fallback.""" + resource_dir = tmp_path / "rag" / "res" + resource_dir.mkdir(parents=True) + (resource_dir / "ner.json").write_text("{}", encoding="utf-8") + (resource_dir / "term.freq").write_text("term\tnot-a-number\n", encoding="utf-8") + monkeypatch.setattr(term_weight, "get_project_base_directory", lambda: str(tmp_path)) + + with caplog.at_level(logging.WARNING): + Dealer() + + assert "Load term.freq FAIL!" in caplog.text