zlei6/ragflow
1
0
Fork 0
mirror of https://github.com/infiniflow/ragflow.git synced 2026-07-29 04:08:12 +08:00
Code Issues Packages Projects Releases Wiki Activity

Fix tokenizer in cpp (#13735)

Browse Source 
### What problem does this PR solve?

Tokenzier in Infinity is modified in
https://github.com/infiniflow/infinity/pull/3330, sync the code change
to cpp files in ragflow

### Type of change

- [x] Bug Fix (non-breaking change which fixes an issue)
This commit is contained in:
qinling0210
2026-03-23 15:40:35 +08:00
committed by GitHub
parent 7b86f577be
commit ac542da505
3 changed files with 52 additions and 46 deletions
Download Patch File Download Diff File Expand all files Collapse all files

44
internal/cpp/wordnet_lemmatizer.cpp
View File

@@ -182,28 +182,14 @@ std::vector<std::string> WordNetLemmatizer::Morphy(const std::string &form, cons
return FilterForms(forms, pos);
}
// Apply morphological rules with recursion (like Java version)
// Apply morphological rules (only ONE level, not recursive like Java)
// This matches Python NLTK WordNet behavior
std::vector<std::string> forms = CollectSubstitutions(form, pos);
std::vector<std::string> combined_forms = forms;
combined_forms.push_back(form);
// First attempt with original form and first-level substitutions
auto results = FilterForms(combined_forms, pos);
if (!results.empty()) {
return results;
}
// Recursively apply rules (Java version's while loop)
while (!forms.empty()) {
forms = CollectSubstitutions(forms, pos);
results = FilterForms(forms, pos);
if (!results.empty()) {
return results;
}
}
// Return empty result if no valid lemma found
return {};
return results;
}
std::string WordNetLemmatizer::Lemmatize(const std::string &form, const std::string &pos) {
@@ -211,13 +197,33 @@ std::string WordNetLemmatizer::Lemmatize(const std::string &form, const std::str
if (!pos.empty()) {
parts_of_speech.push_back(pos);
} else {
parts_of_speech = POS_LIST;
// Use only NOUN to match Python NLTK default behavior
parts_of_speech = {NOUN};
}
for (const auto &part : parts_of_speech) {
auto analyses = Morphy(form, part);
if (!analyses.empty()) {
return analyses[0];
// Python NLTK returns the SHORTEST lemma: min(lemmas, key=len)
// For "as" -> ["as", "a"] -> returns "a"
// For "data" -> ["data", "datum"] -> returns "data"
// For "men" -> ["men", "man"] -> returns "men" (original form preferred when same length)
std::string shortest = analyses[0];
for (const auto &analysis : analyses) {
if (analysis.length() < shortest.length()) {
shortest = analysis;
}
}
// If original form is in the results and has same length as shortest, prefer original form
if (shortest != form) {
for (const auto &analysis : analyses) {
if (analysis == form && analysis.length() == shortest.length()) {
shortest = analysis;
break;
}
}
}
return shortest;
}
}