From 86c520a33617cc5c1fee5244d31bc48cdedd7eac Mon Sep 17 00:00:00 2001 From: Loong <213221738@seu.edu.cn> Date: Wed, 19 Aug 2026 18:37:04 +0800 Subject: [PATCH] fix(nlp): differentiate alphabetic OOV term weights (#18470) ### Summary Closes #18414. `rag/res/term.freq` is not shipped, and both term-weight implementations therefore assigned the same `300` fallback frequency to every lowercase Latin token. With no tokenizer frequency, NER, or POS signal, function words and content words received identical lexical boosts. This PR adds the same bounded out-of-vocabulary prior to Python and Go: - Use it only when the explicit DF dictionary or tokenizer has no frequency. - Count Latin, Greek, and Cyrillic letters, including uppercase and accented forms. - Keep the existing frequency of `300` for words up to three letters, halve it every two additional letters, and clamp it at `10`. - Reject digits, underscores, and logographic terms so Chinese and other existing fine-grained-tokenizer paths are unchanged. - Treat an absent optional `term.freq` as the supported fallback path without a startup warning, while still logging inaccessible or malformed dictionaries. A corpus-derived table was intentionally not added: that would require provenance/licensing decisions, language detection, and handling cross-language homographs. The bounded prior is deterministic, dependency-free, and fixes the equal-weight degradation for whitespace-delimited alphabetic languages without claiming corpus-specific precision. Python and Go consume one shared fixture covering ASCII, uppercase, accented Latin, Greek, Cyrillic, separators, invalid mixed tokens, and a CJK non-match. Both sides also verify the issue's ordering (`was < largest < supplier < equipment`) and that an explicit dictionary entry still takes precedence. Co-authored-by: Loong <184861530+yzl0ng@users.noreply.github.com> --- internal/service/nlp/term_weight.go | 40 ++++++++-- internal/service/nlp/term_weight_test.go | 69 +++++++++++++++++ rag/nlp/term_weight.py | 58 ++++++++++++--- test/fixtures/term_weight_oov.json | 17 +++++ .../unit_test/rag/nlp/test_term_weight_oov.py | 74 +++++++++++++++++++ 5 files changed, 240 insertions(+), 18 deletions(-) create mode 100644 test/fixtures/term_weight_oov.json create mode 100644 test/unit_test/rag/nlp/test_term_weight_oov.py diff --git a/internal/service/nlp/term_weight.go b/internal/service/nlp/term_weight.go index 362f656b34..984b2df613 100644 --- a/internal/service/nlp/term_weight.go +++ b/internal/service/nlp/term_weight.go @@ -23,6 +23,7 @@ import ( "regexp" "strconv" "strings" + "unicode" "ragflow/internal/tokenizer" @@ -87,13 +88,39 @@ func initStopWords() map[string]struct{} { return stopWords } +// alphabeticOOVFrequency estimates frequency for an alphabetic out-of-vocabulary +// term. It preserves the previous frequency of 300 for short words, then halves +// it every two letters, with a floor of 10. Latin, Greek, and Cyrillic scripts +// are supported while logographic terms keep their existing tokenizer path. +func alphabeticOOVFrequency(term string) (float64, bool) { + letterCount := 0 + for _, r := range term { + if unicode.IsLetter(r) && unicode.In(r, unicode.Latin, unicode.Greek, unicode.Cyrillic) { + letterCount++ + continue + } + if r != ' ' && r != '.' && r != '-' { + return 0, false + } + } + if letterCount == 0 { + return 0, false + } + + exponent := float64(max(0, letterCount-3)) / 2 + frequency := math.Round(300 / math.Pow(2, exponent)) + return math.Max(10, frequency), true +} + // loadDict loads a dictionary file // Format: term\tfreq or just term func loadDict(fnm string) map[string]int { res := make(map[string]int) data, err := os.ReadFile(fnm) if err != nil { - common.Warn("Failed to load dictionary", zap.String("file", fnm), zap.Error(err)) + if !os.IsNotExist(err) { + common.Warn("Failed to load dictionary", zap.String("file", fnm), zap.Error(err)) + } return res } @@ -285,7 +312,6 @@ func (d *TermWeightDealer) Weights(tks []string, preprocess bool) []TermWeight { numPattern := regexp.MustCompile("^[0-9,.]{2,}$") shortLetterPattern := regexp.MustCompile("^[a-z]{1,2}$") numSpacePattern := regexp.MustCompile("^[0-9. -]{2,}$") - letterPattern := regexp.MustCompile("^[a-z. -]+$") // ner weight function nerWeight := func(t string) float64 { @@ -338,8 +364,10 @@ func (d *TermWeightDealer) Weights(tks []string, preprocess bool) []TermWeight { } // Use tokenizer's freq function s := tokenizer.GetTermFreq(t) - if s == 0 && letterPattern.MatchString(t) { - return 300 + if s == 0 { + if oovFrequency, ok := alphabeticOOVFrequency(t); ok { + return oovFrequency + } } if s == 0 && len([]rune(t)) >= 4 { // Try fine-grained tokenization @@ -385,8 +413,8 @@ func (d *TermWeightDealer) Weights(tks []string, preprocess bool) []TermWeight { if v, ok := d.df[t]; ok { return float64(v) + 3 } - if letterPattern.MatchString(t) { - return 300 + if oovFrequency, ok := alphabeticOOVFrequency(t); ok { + return oovFrequency } if len([]rune(t)) >= 4 { // Use fine-grained tokenization diff --git a/internal/service/nlp/term_weight_test.go b/internal/service/nlp/term_weight_test.go index 078e5b0009..c3007af6b2 100644 --- a/internal/service/nlp/term_weight_test.go +++ b/internal/service/nlp/term_weight_test.go @@ -15,6 +15,7 @@ package nlp import ( + "encoding/json" "os" "path/filepath" "reflect" @@ -23,6 +24,74 @@ import ( "testing" ) +type oovFrequencyFixture struct { + Term string `json:"term"` + Frequency *float64 `json:"frequency"` +} + +func repositoryRoot(t *testing.T) string { + t.Helper() + dir, err := os.Getwd() + if err != nil { + t.Fatalf("get working directory: %v", err) + } + for { + if _, err := os.Stat(filepath.Join(dir, "go.mod")); err == nil { + return dir + } + parent := filepath.Dir(dir) + if parent == dir { + t.Fatal("repository root not found") + } + dir = parent + } +} + +func TestAlphabeticOOVFrequencyMatchesSharedFixture(t *testing.T) { + fixturePath := filepath.Join(repositoryRoot(t), "test", "fixtures", "term_weight_oov.json") + data, err := os.ReadFile(fixturePath) + if err != nil { + t.Fatalf("read OOV fixture: %v", err) + } + + var cases []oovFrequencyFixture + if err := json.Unmarshal(data, &cases); err != nil { + t.Fatalf("decode OOV fixture: %v", err) + } + for _, testCase := range cases { + got, ok := alphabeticOOVFrequency(testCase.Term) + if testCase.Frequency == nil { + if ok { + t.Errorf("alphabeticOOVFrequency(%q) = %v, true; want no fallback", testCase.Term, got) + } + continue + } + if !ok || got != *testCase.Frequency { + t.Errorf("alphabeticOOVFrequency(%q) = %v, %v; want %v, true", testCase.Term, got, ok, *testCase.Frequency) + } + } +} + +func TestOOVWeightsFavorLongerContentTerms(t *testing.T) { + d := NewTermWeightDealer(t.TempDir()) + terms := []string{"was", "largest", "supplier", "equipment"} + weights := d.Weights(terms, false) + if len(weights) != len(terms) { + t.Fatalf("Weights returned %d terms; want %d", len(weights), len(terms)) + } + for i := 1; i < len(weights); i++ { + if weights[i-1].Weight >= weights[i].Weight { + t.Fatalf("weights are not increasing with OOV term length: %v", weights) + } + } + + d.df["equipment"] = 1_000_000 + weights = d.Weights([]string{"supplier", "equipment"}, false) + if weights[1].Weight >= weights[0].Weight { + t.Fatalf("dictionary frequency should take precedence over the OOV prior: %v", weights) + } +} + // TestNewTermWeightDealer tests the constructor func TestNewTermWeightDealer(t *testing.T) { // Test with empty resPath diff --git a/rag/nlp/term_weight.py b/rag/nlp/term_weight.py index 43b018a429..9477945680 100644 --- a/rag/nlp/term_weight.py +++ b/rag/nlp/term_weight.py @@ -14,14 +14,43 @@ # limitations under the License. # +import json import logging import math -import json -import re import os +import re +import unicodedata + import numpy as np -from rag.nlp import rag_tokenizer + from common.file_utils import get_project_base_directory +from rag.nlp import rag_tokenizer + + +def _alphabetic_oov_frequency(term): + """Estimate frequency for a Latin, Greek, or Cyrillic OOV term.""" + letter_count = 0 + for char in term: + if char.isascii(): + if char.isalpha(): + letter_count += 1 + elif char not in " .-": + return None + continue + script_name = unicodedata.name(char, "").split(" ", 1)[0] + if char.isalpha() and script_name in {"LATIN", "GREEK", "CYRILLIC"}: + letter_count += 1 + elif char not in " .-": + return None + if not letter_count: + return None + + # Preserve the old frequency (300) for short words, then halve it every + # two letters. This is a bounded, language-neutral prior: longer unknown + # words are usually more informative, without outranking known terms by an + # unbounded amount. + exponent = max(0, letter_count - 3) / 2 + return max(10, round(300 / (2**exponent))) class Dealer: @@ -89,10 +118,13 @@ class Dealer: self.ne = json.load(f) except Exception: logging.warning("Load ner.json FAIL!") + freq_path = os.path.join(fnm, "term.freq") try: - self.df = load_dict(os.path.join(fnm, "term.freq")) - except Exception: - logging.warning("Load term.freq FAIL!") + self.df = load_dict(freq_path) + except FileNotFoundError: + pass + except (OSError, ValueError): + logging.warning("Load term.freq FAIL!", exc_info=True) def pretoken(self, txt, num=False, stpwd=True): patt = [r"[~—\t @#%!<>,\.\?\":;'\{\}\[\]_=\(\)\|，。？》•●○↓《；‘’：“”【¥ 】…￥！、·（）×`&\\/「」\\]"] @@ -161,7 +193,6 @@ class Dealer: num_pattern = re.compile(r"[0-9,.]{2,}$") short_letter_pattern = re.compile(r"[a-z]{1,2}$") num_space_pattern = re.compile(r"[0-9. -]{2,}$") - letter_pattern = re.compile(r"[a-z. -]+$") def ner(t): if num_pattern.match(t): @@ -189,8 +220,10 @@ class Dealer: if num_space_pattern.match(t): return 3 s = rag_tokenizer.freq(t) - if not s and letter_pattern.match(t): - return 300 + if not s: + oov_frequency = _alphabetic_oov_frequency(t) + if oov_frequency is not None: + return oov_frequency if not s: s = 0 @@ -208,9 +241,10 @@ class Dealer: return 5 if t in self.df: return self.df[t] + 3 - elif letter_pattern.match(t): - return 300 - elif len(t) >= 4: + oov_frequency = _alphabetic_oov_frequency(t) + if oov_frequency is not None: + return oov_frequency + if len(t) >= 4: s = [tt for tt in rag_tokenizer.fine_grained_tokenize(t).split() if len(tt) > 1] if len(s) > 1: return max(3, np.min([df(tt) for tt in s]) / 6.0) diff --git a/test/fixtures/term_weight_oov.json b/test/fixtures/term_weight_oov.json new file mode 100644 index 0000000000..c7adbbafdc --- /dev/null +++ b/test/fixtures/term_weight_oov.json @@ -0,0 +1,17 @@ +[ + {"term": "a", "frequency": 300}, + {"term": "the", "frequency": 300}, + {"term": "Café", "frequency": 212}, + {"term": "maior", "frequency": 150}, + {"term": "κόσμος", "frequency": 106}, + {"term": "привет", "frequency": 106}, + {"term": "largest", "frequency": 75}, + {"term": "supplier", "frequency": 53}, + {"term": "equipment", "frequency": 38}, + {"term": "équipement", "frequency": 27}, + {"term": "hospital-equipment", "frequency": 10}, + {"term": "北京", "frequency": null}, + {"term": "abc123", "frequency": null}, + {"term": "hello_world", "frequency": null}, + {"term": "...", "frequency": null} +] diff --git a/test/unit_test/rag/nlp/test_term_weight_oov.py b/test/unit_test/rag/nlp/test_term_weight_oov.py new file mode 100644 index 0000000000..d220bf726b --- /dev/null +++ b/test/unit_test/rag/nlp/test_term_weight_oov.py @@ -0,0 +1,74 @@ +# +# Copyright 2026 The InfiniFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import json +import logging +from pathlib import Path + +from rag.nlp import rag_tokenizer, term_weight +from rag.nlp.term_weight import Dealer, _alphabetic_oov_frequency + + +def test_alphabetic_oov_frequency_matches_shared_fixture(): + """Keep the Python fallback aligned with the fixture consumed by Go tests.""" + fixture_path = Path(__file__).parents[3] / "fixtures" / "term_weight_oov.json" + cases = json.loads(fixture_path.read_text(encoding="utf-8")) + + for case in cases: + assert _alphabetic_oov_frequency(case["term"]) == case["frequency"] + + +def test_oov_weights_favor_longer_content_terms(monkeypatch): + """Avoid equal weights when no dictionary or tokenizer frequency is available.""" + monkeypatch.setattr(rag_tokenizer, "freq", lambda _term: 0) + monkeypatch.setattr(rag_tokenizer, "tag", lambda _term: "") + dealer = Dealer() + dealer.df = {} + + weights = dict(dealer.weights(["was", "largest", "supplier", "equipment"], preprocess=False)) + + assert weights["was"] < weights["largest"] < weights["supplier"] < weights["equipment"] + + dealer.df["equipment"] = 1_000_000 + weights = dict(dealer.weights(["supplier", "equipment"], preprocess=False)) + assert weights["equipment"] < weights["supplier"] + + +def test_missing_frequency_dictionary_is_silent(tmp_path, monkeypatch, caplog): + """Treat an absent optional frequency dictionary as an expected fallback.""" + resource_dir = tmp_path / "rag" / "res" + resource_dir.mkdir(parents=True) + (resource_dir / "ner.json").write_text("{}", encoding="utf-8") + monkeypatch.setattr(term_weight, "get_project_base_directory", lambda: str(tmp_path)) + + with caplog.at_level(logging.WARNING): + Dealer() + + assert "Load term.freq FAIL!" not in caplog.text + + +def test_malformed_frequency_dictionary_logs_warning(tmp_path, monkeypatch, caplog): + """Expose malformed dictionaries instead of silently enabling the fallback.""" + resource_dir = tmp_path / "rag" / "res" + resource_dir.mkdir(parents=True) + (resource_dir / "ner.json").write_text("{}", encoding="utf-8") + (resource_dir / "term.freq").write_text("term\tnot-a-number\n", encoding="utf-8") + monkeypatch.setattr(term_weight, "get_project_base_directory", lambda: str(tmp_path)) + + with caplog.at_level(logging.WARNING): + Dealer() + + assert "Load term.freq FAIL!" in caplog.text