mirror of
https://github.com/infiniflow/ragflow.git
synced 2026-07-09 12:54:47 +08:00
d8cefcf052
## Summary - Add language-aware Snowball stemmer to `RagTokenizer` supporting 16 languages (Dutch, German, French, Spanish, etc.) - Thread the KB `language` parameter through the full tokenization pipeline (14 parser modules + task executor) - Add Dutch to the frontend language lists and cross-language form ## Problem RAGFlow uses the English Porter stemmer + WordNet lemmatizer for **all** BM25 tokenization, regardless of the knowledge base language setting. This produces incorrect stems for non-English text. For example: | Dutch word | Dutch stemmer | English Porter | |---|---|---| | documenten | document | documenten (unchanged!) | | gebruikers | gebruiker | gebruik (over-stemmed) | | instellingen | instell | instellingen (unchanged!) | This degrades BM25 recall for any non-English knowledge base. ## Solution NLTK already ships Snowball stemmers for 16 languages. This PR: 1. **`rag/nlp/rag_tokenizer.py`**: Overrides `tokenize()` with `set_language()` and `_normalize_token()` that selects the correct NLTK Snowball stemmer. Falls back to Porter for unmapped languages (Chinese, Japanese, Korean, etc. — these use character-based tokenization anyway). 2. **`rag/nlp/__init__.py`** + **14 `rag/app/*.py` parsers** + **`rag/svr/task_executor.py`**: Threads the `language` parameter through `tokenize()`, `tokenize_chunks()`, `tokenize_table()`, and all callers. 3. **Frontend**: Adds Dutch (`Nederlands`) to `LanguageList`, `LanguageMap`, `LanguageAbbreviationMap`, `LanguageTranslationMap`, cross-language form field, and `en.ts` locale. ## Backward Compatibility - Default language is `"English"`, preserving existing behavior for all current users - Languages without a Snowball stemmer mapping fall back to Porter (no change) - No new dependencies — NLTK Snowball is already bundled
220 lines
4.5 KiB
TypeScript
220 lines
4.5 KiB
TypeScript
export const fileIconMap = {
|
|
aep: 'aep.svg',
|
|
ai: 'ai.svg',
|
|
avi: 'avi.svg',
|
|
css: 'css.svg',
|
|
csv: 'csv.svg',
|
|
dmg: 'dmg.svg',
|
|
doc: 'doc.svg',
|
|
docx: 'docx.svg',
|
|
eps: 'eps.svg',
|
|
exe: 'exe.svg',
|
|
fig: 'fig.svg',
|
|
gif: 'gif.svg',
|
|
html: 'html.svg',
|
|
indd: 'indd.svg',
|
|
java: 'java.svg',
|
|
jpeg: 'jpeg.svg',
|
|
jpg: 'jpg.svg',
|
|
js: 'js.svg',
|
|
json: 'json.svg',
|
|
md: 'md.svg',
|
|
mdx: 'mdx.svg',
|
|
mkv: 'mkv.svg',
|
|
mp3: 'mp3.svg',
|
|
mp4: 'mp4.svg',
|
|
mpeg: 'mpeg.svg',
|
|
pdf: 'pdf.svg',
|
|
png: 'png.svg',
|
|
ppt: 'ppt.svg',
|
|
pptx: 'pptx.svg',
|
|
psd: 'psd.svg',
|
|
rss: 'rss.svg',
|
|
sql: 'sql.svg',
|
|
svg: 'svg.svg',
|
|
tiff: 'tiff.svg',
|
|
txt: 'txt.svg',
|
|
wav: 'wav.svg',
|
|
webp: 'webp.svg',
|
|
xls: 'xls.svg',
|
|
xlsx: 'xlsx.svg',
|
|
xml: 'xml.svg',
|
|
};
|
|
|
|
// TODO: Use standard BCP 47 language tag and display names
|
|
export const LanguageList = [
|
|
'English',
|
|
'Chinese',
|
|
'Traditional Chinese',
|
|
'Russian',
|
|
'Indonesian',
|
|
'Spanish',
|
|
'Vietnamese',
|
|
'Japanese',
|
|
'Portuguese BR',
|
|
'German',
|
|
'French',
|
|
'Italian',
|
|
'Bulgarian',
|
|
'Arabic',
|
|
'Turkish',
|
|
'Dutch',
|
|
];
|
|
export const LanguageMap = {
|
|
English: 'English',
|
|
Chinese: '简体中文',
|
|
'Traditional Chinese': '繁體中文',
|
|
Russian: 'Русский',
|
|
Indonesian: 'Bahasa Indonesia',
|
|
Indonesia: 'Indonesia',
|
|
Spanish: 'Español',
|
|
Vietnamese: 'Tiếng việt',
|
|
Japanese: '日本語',
|
|
'Portuguese BR': 'Português BR',
|
|
German: 'Deutsch',
|
|
French: 'Français',
|
|
Italian: 'Italiano',
|
|
Bulgarian: 'Български',
|
|
Arabic: 'العربية',
|
|
Turkish: 'Türkçe',
|
|
Dutch: 'Nederlands',
|
|
};
|
|
|
|
export enum LanguageAbbreviation {
|
|
En = 'en',
|
|
Zh = 'zh-Hans',
|
|
ZhTraditional = 'zh-Hant',
|
|
Ru = 'ru',
|
|
Id = 'id',
|
|
Ja = 'ja',
|
|
Es = 'es',
|
|
Vi = 'vi',
|
|
PtBr = 'pt-BR',
|
|
De = 'de',
|
|
Fr = 'fr',
|
|
It = 'it',
|
|
Bg = 'bg',
|
|
Ar = 'ar',
|
|
Tr = 'tr',
|
|
Ko = 'ko',
|
|
Nl = 'nl',
|
|
}
|
|
|
|
export const LanguageAbbreviationMap = {
|
|
[LanguageAbbreviation.En]: 'English',
|
|
[LanguageAbbreviation.Zh]: '简体中文',
|
|
[LanguageAbbreviation.ZhTraditional]: '繁體中文',
|
|
[LanguageAbbreviation.Ru]: 'Русский',
|
|
[LanguageAbbreviation.Id]: 'Bahasa Indonesia',
|
|
[LanguageAbbreviation.Es]: 'Español',
|
|
[LanguageAbbreviation.Vi]: 'Tiếng việt',
|
|
[LanguageAbbreviation.Ja]: '日本語',
|
|
[LanguageAbbreviation.PtBr]: 'Português BR',
|
|
[LanguageAbbreviation.De]: 'Deutsch',
|
|
[LanguageAbbreviation.Fr]: 'Français',
|
|
[LanguageAbbreviation.It]: 'Italiano',
|
|
[LanguageAbbreviation.Bg]: 'Български',
|
|
[LanguageAbbreviation.Ar]: 'العربية',
|
|
[LanguageAbbreviation.Tr]: 'Türkçe',
|
|
[LanguageAbbreviation.Ko]: '한국어',
|
|
[LanguageAbbreviation.Nl]: 'Nederlands',
|
|
};
|
|
|
|
export const LanguageTranslationMap = {
|
|
English: 'en',
|
|
Chinese: 'zh-Hans',
|
|
'Traditional Chinese': 'zh-Hant',
|
|
Russian: 'ru',
|
|
Indonesian: 'id',
|
|
Indonesia: 'id',
|
|
Spanish: 'es',
|
|
Vietnamese: 'vi',
|
|
Japanese: 'ja',
|
|
Korean: 'ko',
|
|
'Portuguese BR': 'pt-BR',
|
|
'pt-br': 'pt-BR',
|
|
'pt-BR': 'pt-BR',
|
|
German: 'de',
|
|
French: 'fr',
|
|
Italian: 'it',
|
|
Tamil: 'ta',
|
|
Telugu: 'te',
|
|
Kannada: 'ka',
|
|
Thai: 'th',
|
|
Greek: 'el',
|
|
Hindi: 'hi',
|
|
Ukrainian: 'uk',
|
|
Bulgarian: 'bg',
|
|
Arabic: 'ar',
|
|
Turkish: 'tr',
|
|
Dutch: 'nl',
|
|
};
|
|
|
|
export enum FileMimeType {
|
|
Bmp = 'image/bmp',
|
|
Csv = 'text/csv',
|
|
Odt = 'application/vnd.oasis.opendocument.text',
|
|
Doc = 'application/msword',
|
|
Docx = 'application/vnd.openxmlformats-officedocument.wordprocessingml.document',
|
|
Gif = 'image/gif',
|
|
Htm = 'text/htm',
|
|
Html = 'text/html',
|
|
Jpg = 'image/jpg',
|
|
Jpeg = 'image/jpeg',
|
|
Pdf = 'application/pdf',
|
|
Png = 'image/png',
|
|
Ppt = 'application/vnd.ms-powerpoint',
|
|
Pptx = 'application/vnd.openxmlformats-officedocument.presentationml.presentation',
|
|
Tiff = 'image/tiff',
|
|
Txt = 'text/plain',
|
|
Xls = 'application/vnd.ms-excel',
|
|
Xlsx = 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet',
|
|
Mp4 = 'video/mp4',
|
|
Json = 'application/json',
|
|
Md = 'text/markdown',
|
|
Mdx = 'text/mdx',
|
|
}
|
|
|
|
export const Domain = 'cloud.ragflow.io';
|
|
|
|
//#region file preview
|
|
export const Images = [
|
|
'jpg',
|
|
'jpeg',
|
|
'png',
|
|
'gif',
|
|
'bmp',
|
|
'tif',
|
|
'tiff',
|
|
'webp',
|
|
// 'svg',
|
|
'ico',
|
|
];
|
|
|
|
// Without FileViewer
|
|
export const ExceptiveType = [
|
|
'xlsx',
|
|
'xls',
|
|
'pdf',
|
|
'docx',
|
|
'md',
|
|
'mdx',
|
|
...Images,
|
|
];
|
|
|
|
export const SupportedPreviewDocumentTypes = [...ExceptiveType];
|
|
//#endregion
|
|
|
|
export enum Platform {
|
|
RAGFlow = 'RAGFlow',
|
|
Dify = 'Dify',
|
|
FastGPT = 'FastGPT',
|
|
Coze = 'Coze',
|
|
}
|
|
|
|
export enum ThemeEnum {
|
|
Dark = 'dark',
|
|
Light = 'light',
|
|
System = 'system',
|
|
}
|