2024-02-27 19:05:50 +08:00
|
|
|
export const fileIconMap = {
|
|
|
|
|
aep: 'aep.svg',
|
|
|
|
|
ai: 'ai.svg',
|
|
|
|
|
avi: 'avi.svg',
|
|
|
|
|
css: 'css.svg',
|
|
|
|
|
csv: 'csv.svg',
|
|
|
|
|
dmg: 'dmg.svg',
|
|
|
|
|
doc: 'doc.svg',
|
|
|
|
|
docx: 'docx.svg',
|
|
|
|
|
eps: 'eps.svg',
|
|
|
|
|
exe: 'exe.svg',
|
|
|
|
|
fig: 'fig.svg',
|
|
|
|
|
gif: 'gif.svg',
|
|
|
|
|
html: 'html.svg',
|
|
|
|
|
indd: 'indd.svg',
|
|
|
|
|
java: 'java.svg',
|
|
|
|
|
jpeg: 'jpeg.svg',
|
|
|
|
|
jpg: 'jpg.svg',
|
|
|
|
|
js: 'js.svg',
|
|
|
|
|
json: 'json.svg',
|
2025-12-29 12:54:31 +08:00
|
|
|
md: 'md.svg',
|
|
|
|
|
mdx: 'mdx.svg',
|
2024-02-27 19:05:50 +08:00
|
|
|
mkv: 'mkv.svg',
|
|
|
|
|
mp3: 'mp3.svg',
|
|
|
|
|
mp4: 'mp4.svg',
|
|
|
|
|
mpeg: 'mpeg.svg',
|
|
|
|
|
pdf: 'pdf.svg',
|
|
|
|
|
png: 'png.svg',
|
|
|
|
|
ppt: 'ppt.svg',
|
|
|
|
|
pptx: 'pptx.svg',
|
|
|
|
|
psd: 'psd.svg',
|
|
|
|
|
rss: 'rss.svg',
|
|
|
|
|
sql: 'sql.svg',
|
|
|
|
|
svg: 'svg.svg',
|
|
|
|
|
tiff: 'tiff.svg',
|
|
|
|
|
txt: 'txt.svg',
|
|
|
|
|
wav: 'wav.svg',
|
|
|
|
|
webp: 'webp.svg',
|
|
|
|
|
xls: 'xls.svg',
|
|
|
|
|
xlsx: 'xlsx.svg',
|
|
|
|
|
xml: 'xml.svg',
|
|
|
|
|
};
|
2024-04-12 11:41:00 +08:00
|
|
|
|
2026-03-12 21:01:09 +08:00
|
|
|
// TODO: Use standard BCP 47 language tag and display names
|
2024-11-07 16:53:19 +07:00
|
|
|
export const LanguageList = [
|
|
|
|
|
'English',
|
|
|
|
|
'Chinese',
|
|
|
|
|
'Traditional Chinese',
|
2025-09-24 08:18:47 +06:00
|
|
|
'Russian',
|
2026-03-02 19:10:11 +08:00
|
|
|
'Indonesian',
|
2024-11-08 00:44:03 -05:00
|
|
|
'Spanish',
|
2024-11-21 17:43:08 +07:00
|
|
|
'Vietnamese',
|
2024-12-09 12:38:06 +09:00
|
|
|
'Japanese',
|
2025-01-21 00:22:29 -03:00
|
|
|
'Portuguese BR',
|
2025-03-11 09:13:58 +01:00
|
|
|
'German',
|
2025-08-06 04:22:32 +02:00
|
|
|
'French',
|
2025-12-09 10:59:21 +01:00
|
|
|
'Italian',
|
2026-02-14 10:51:29 +02:00
|
|
|
'Bulgarian',
|
2026-03-02 19:10:11 +08:00
|
|
|
'Arabic',
|
2026-03-18 16:09:32 +03:00
|
|
|
'Turkish',
|
feat: add native Dutch language support for BM25 tokenization (#14140)
## Summary
- Add language-aware Snowball stemmer to `RagTokenizer` supporting 16
languages (Dutch, German, French, Spanish, etc.)
- Thread the KB `language` parameter through the full tokenization
pipeline (14 parser modules + task executor)
- Add Dutch to the frontend language lists and cross-language form
## Problem
RAGFlow uses the English Porter stemmer + WordNet lemmatizer for **all**
BM25 tokenization, regardless of the knowledge base language setting.
This produces incorrect stems for non-English text. For example:
| Dutch word | Dutch stemmer | English Porter |
|---|---|---|
| documenten | document | documenten (unchanged!) |
| gebruikers | gebruiker | gebruik (over-stemmed) |
| instellingen | instell | instellingen (unchanged!) |
This degrades BM25 recall for any non-English knowledge base.
## Solution
NLTK already ships Snowball stemmers for 16 languages. This PR:
1. **`rag/nlp/rag_tokenizer.py`**: Overrides `tokenize()` with
`set_language()` and `_normalize_token()` that selects the correct NLTK
Snowball stemmer. Falls back to Porter for unmapped languages (Chinese,
Japanese, Korean, etc. — these use character-based tokenization anyway).
2. **`rag/nlp/__init__.py`** + **14 `rag/app/*.py` parsers** +
**`rag/svr/task_executor.py`**: Threads the `language` parameter through
`tokenize()`, `tokenize_chunks()`, `tokenize_table()`, and all callers.
3. **Frontend**: Adds Dutch (`Nederlands`) to `LanguageList`,
`LanguageMap`, `LanguageAbbreviationMap`, `LanguageTranslationMap`,
cross-language form field, and `en.ts` locale.
## Backward Compatibility
- Default language is `"English"`, preserving existing behavior for all
current users
- Languages without a Snowball stemmer mapping fall back to Porter (no
change)
- No new dependencies — NLTK Snowball is already bundled
2026-07-06 17:39:56 +02:00
|
|
|
'Dutch',
|
2024-11-07 16:53:19 +07:00
|
|
|
];
|
2024-10-16 15:57:39 +08:00
|
|
|
export const LanguageMap = {
|
|
|
|
|
English: 'English',
|
|
|
|
|
Chinese: '简体中文',
|
|
|
|
|
'Traditional Chinese': '繁體中文',
|
2025-09-24 08:18:47 +06:00
|
|
|
Russian: 'Русский',
|
2026-03-02 19:10:11 +08:00
|
|
|
Indonesian: 'Bahasa Indonesia',
|
2024-11-07 16:53:19 +07:00
|
|
|
Indonesia: 'Indonesia',
|
2024-11-08 00:44:03 -05:00
|
|
|
Spanish: 'Español',
|
2024-11-21 17:43:08 +07:00
|
|
|
Vietnamese: 'Tiếng việt',
|
2024-12-09 12:38:06 +09:00
|
|
|
Japanese: '日本語',
|
2025-01-21 00:22:29 -03:00
|
|
|
'Portuguese BR': 'Português BR',
|
2026-03-12 13:33:36 +08:00
|
|
|
German: 'Deutsch',
|
2025-08-06 04:22:32 +02:00
|
|
|
French: 'Français',
|
2025-12-09 10:59:21 +01:00
|
|
|
Italian: 'Italiano',
|
2026-02-14 10:51:29 +02:00
|
|
|
Bulgarian: 'Български',
|
2026-03-02 19:10:11 +08:00
|
|
|
Arabic: 'العربية',
|
2026-03-18 16:09:32 +03:00
|
|
|
Turkish: 'Türkçe',
|
feat: add native Dutch language support for BM25 tokenization (#14140)
## Summary
- Add language-aware Snowball stemmer to `RagTokenizer` supporting 16
languages (Dutch, German, French, Spanish, etc.)
- Thread the KB `language` parameter through the full tokenization
pipeline (14 parser modules + task executor)
- Add Dutch to the frontend language lists and cross-language form
## Problem
RAGFlow uses the English Porter stemmer + WordNet lemmatizer for **all**
BM25 tokenization, regardless of the knowledge base language setting.
This produces incorrect stems for non-English text. For example:
| Dutch word | Dutch stemmer | English Porter |
|---|---|---|
| documenten | document | documenten (unchanged!) |
| gebruikers | gebruiker | gebruik (over-stemmed) |
| instellingen | instell | instellingen (unchanged!) |
This degrades BM25 recall for any non-English knowledge base.
## Solution
NLTK already ships Snowball stemmers for 16 languages. This PR:
1. **`rag/nlp/rag_tokenizer.py`**: Overrides `tokenize()` with
`set_language()` and `_normalize_token()` that selects the correct NLTK
Snowball stemmer. Falls back to Porter for unmapped languages (Chinese,
Japanese, Korean, etc. — these use character-based tokenization anyway).
2. **`rag/nlp/__init__.py`** + **14 `rag/app/*.py` parsers** +
**`rag/svr/task_executor.py`**: Threads the `language` parameter through
`tokenize()`, `tokenize_chunks()`, `tokenize_table()`, and all callers.
3. **Frontend**: Adds Dutch (`Nederlands`) to `LanguageList`,
`LanguageMap`, `LanguageAbbreviationMap`, `LanguageTranslationMap`,
cross-language form field, and `en.ts` locale.
## Backward Compatibility
- Default language is `"English"`, preserving existing behavior for all
current users
- Languages without a Snowball stemmer mapping fall back to Porter (no
change)
- No new dependencies — NLTK Snowball is already bundled
2026-07-06 17:39:56 +02:00
|
|
|
Dutch: 'Nederlands',
|
2024-10-16 15:57:39 +08:00
|
|
|
};
|
|
|
|
|
|
2025-02-18 15:06:53 +08:00
|
|
|
export enum LanguageAbbreviation {
|
|
|
|
|
En = 'en',
|
2026-03-12 21:01:09 +08:00
|
|
|
Zh = 'zh-Hans',
|
|
|
|
|
ZhTraditional = 'zh-Hant',
|
2025-09-24 08:18:47 +06:00
|
|
|
Ru = 'ru',
|
2025-02-18 15:06:53 +08:00
|
|
|
Id = 'id',
|
|
|
|
|
Ja = 'ja',
|
|
|
|
|
Es = 'es',
|
|
|
|
|
Vi = 'vi',
|
|
|
|
|
PtBr = 'pt-BR',
|
2025-03-11 09:13:58 +01:00
|
|
|
De = 'de',
|
2025-08-06 04:22:32 +02:00
|
|
|
Fr = 'fr',
|
2025-12-09 10:59:21 +01:00
|
|
|
It = 'it',
|
2026-02-14 10:51:29 +02:00
|
|
|
Bg = 'bg',
|
2026-03-02 19:10:11 +08:00
|
|
|
Ar = 'ar',
|
2026-03-18 16:09:32 +03:00
|
|
|
Tr = 'tr',
|
2026-06-11 17:55:40 +09:00
|
|
|
Ko = 'ko',
|
feat: add native Dutch language support for BM25 tokenization (#14140)
## Summary
- Add language-aware Snowball stemmer to `RagTokenizer` supporting 16
languages (Dutch, German, French, Spanish, etc.)
- Thread the KB `language` parameter through the full tokenization
pipeline (14 parser modules + task executor)
- Add Dutch to the frontend language lists and cross-language form
## Problem
RAGFlow uses the English Porter stemmer + WordNet lemmatizer for **all**
BM25 tokenization, regardless of the knowledge base language setting.
This produces incorrect stems for non-English text. For example:
| Dutch word | Dutch stemmer | English Porter |
|---|---|---|
| documenten | document | documenten (unchanged!) |
| gebruikers | gebruiker | gebruik (over-stemmed) |
| instellingen | instell | instellingen (unchanged!) |
This degrades BM25 recall for any non-English knowledge base.
## Solution
NLTK already ships Snowball stemmers for 16 languages. This PR:
1. **`rag/nlp/rag_tokenizer.py`**: Overrides `tokenize()` with
`set_language()` and `_normalize_token()` that selects the correct NLTK
Snowball stemmer. Falls back to Porter for unmapped languages (Chinese,
Japanese, Korean, etc. — these use character-based tokenization anyway).
2. **`rag/nlp/__init__.py`** + **14 `rag/app/*.py` parsers** +
**`rag/svr/task_executor.py`**: Threads the `language` parameter through
`tokenize()`, `tokenize_chunks()`, `tokenize_table()`, and all callers.
3. **Frontend**: Adds Dutch (`Nederlands`) to `LanguageList`,
`LanguageMap`, `LanguageAbbreviationMap`, `LanguageTranslationMap`,
cross-language form field, and `en.ts` locale.
## Backward Compatibility
- Default language is `"English"`, preserving existing behavior for all
current users
- Languages without a Snowball stemmer mapping fall back to Porter (no
change)
- No new dependencies — NLTK Snowball is already bundled
2026-07-06 17:39:56 +02:00
|
|
|
Nl = 'nl',
|
2025-02-18 15:06:53 +08:00
|
|
|
}
|
|
|
|
|
|
|
|
|
|
export const LanguageAbbreviationMap = {
|
|
|
|
|
[LanguageAbbreviation.En]: 'English',
|
|
|
|
|
[LanguageAbbreviation.Zh]: '简体中文',
|
|
|
|
|
[LanguageAbbreviation.ZhTraditional]: '繁體中文',
|
2025-09-24 08:18:47 +06:00
|
|
|
[LanguageAbbreviation.Ru]: 'Русский',
|
2026-03-02 19:10:11 +08:00
|
|
|
[LanguageAbbreviation.Id]: 'Bahasa Indonesia',
|
2025-02-18 15:06:53 +08:00
|
|
|
[LanguageAbbreviation.Es]: 'Español',
|
|
|
|
|
[LanguageAbbreviation.Vi]: 'Tiếng việt',
|
|
|
|
|
[LanguageAbbreviation.Ja]: '日本語',
|
|
|
|
|
[LanguageAbbreviation.PtBr]: 'Português BR',
|
2025-03-11 09:13:58 +01:00
|
|
|
[LanguageAbbreviation.De]: 'Deutsch',
|
2025-08-06 04:22:32 +02:00
|
|
|
[LanguageAbbreviation.Fr]: 'Français',
|
2025-12-09 10:59:21 +01:00
|
|
|
[LanguageAbbreviation.It]: 'Italiano',
|
2026-02-14 10:51:29 +02:00
|
|
|
[LanguageAbbreviation.Bg]: 'Български',
|
2026-03-02 19:10:11 +08:00
|
|
|
[LanguageAbbreviation.Ar]: 'العربية',
|
2026-03-18 16:09:32 +03:00
|
|
|
[LanguageAbbreviation.Tr]: 'Türkçe',
|
2026-06-11 17:55:40 +09:00
|
|
|
[LanguageAbbreviation.Ko]: '한국어',
|
feat: add native Dutch language support for BM25 tokenization (#14140)
## Summary
- Add language-aware Snowball stemmer to `RagTokenizer` supporting 16
languages (Dutch, German, French, Spanish, etc.)
- Thread the KB `language` parameter through the full tokenization
pipeline (14 parser modules + task executor)
- Add Dutch to the frontend language lists and cross-language form
## Problem
RAGFlow uses the English Porter stemmer + WordNet lemmatizer for **all**
BM25 tokenization, regardless of the knowledge base language setting.
This produces incorrect stems for non-English text. For example:
| Dutch word | Dutch stemmer | English Porter |
|---|---|---|
| documenten | document | documenten (unchanged!) |
| gebruikers | gebruiker | gebruik (over-stemmed) |
| instellingen | instell | instellingen (unchanged!) |
This degrades BM25 recall for any non-English knowledge base.
## Solution
NLTK already ships Snowball stemmers for 16 languages. This PR:
1. **`rag/nlp/rag_tokenizer.py`**: Overrides `tokenize()` with
`set_language()` and `_normalize_token()` that selects the correct NLTK
Snowball stemmer. Falls back to Porter for unmapped languages (Chinese,
Japanese, Korean, etc. — these use character-based tokenization anyway).
2. **`rag/nlp/__init__.py`** + **14 `rag/app/*.py` parsers** +
**`rag/svr/task_executor.py`**: Threads the `language` parameter through
`tokenize()`, `tokenize_chunks()`, `tokenize_table()`, and all callers.
3. **Frontend**: Adds Dutch (`Nederlands`) to `LanguageList`,
`LanguageMap`, `LanguageAbbreviationMap`, `LanguageTranslationMap`,
cross-language form field, and `en.ts` locale.
## Backward Compatibility
- Default language is `"English"`, preserving existing behavior for all
current users
- Languages without a Snowball stemmer mapping fall back to Porter (no
change)
- No new dependencies — NLTK Snowball is already bundled
2026-07-06 17:39:56 +02:00
|
|
|
[LanguageAbbreviation.Nl]: 'Nederlands',
|
2025-02-18 15:06:53 +08:00
|
|
|
};
|
|
|
|
|
|
2024-04-12 11:41:00 +08:00
|
|
|
export const LanguageTranslationMap = {
|
|
|
|
|
English: 'en',
|
2026-03-12 21:01:09 +08:00
|
|
|
Chinese: 'zh-Hans',
|
|
|
|
|
'Traditional Chinese': 'zh-Hant',
|
2025-09-24 08:18:47 +06:00
|
|
|
Russian: 'ru',
|
2025-12-16 07:15:25 +02:00
|
|
|
Indonesian: 'id',
|
2026-03-02 19:10:11 +08:00
|
|
|
Indonesia: 'id',
|
2024-11-08 00:44:03 -05:00
|
|
|
Spanish: 'es',
|
2024-11-21 17:43:08 +07:00
|
|
|
Vietnamese: 'vi',
|
2024-12-09 12:38:06 +09:00
|
|
|
Japanese: 'ja',
|
2025-12-16 07:15:25 +02:00
|
|
|
Korean: 'ko',
|
2026-03-02 19:10:11 +08:00
|
|
|
'Portuguese BR': 'pt-BR',
|
|
|
|
|
'pt-br': 'pt-BR',
|
|
|
|
|
'pt-BR': 'pt-BR',
|
2025-03-11 09:13:58 +01:00
|
|
|
German: 'de',
|
2025-08-06 04:22:32 +02:00
|
|
|
French: 'fr',
|
2025-12-09 10:59:21 +01:00
|
|
|
Italian: 'it',
|
2025-12-16 07:15:25 +02:00
|
|
|
Tamil: 'ta',
|
|
|
|
|
Telugu: 'te',
|
|
|
|
|
Kannada: 'ka',
|
|
|
|
|
Thai: 'th',
|
|
|
|
|
Greek: 'el',
|
|
|
|
|
Hindi: 'hi',
|
|
|
|
|
Ukrainian: 'uk',
|
2026-02-14 10:51:29 +02:00
|
|
|
Bulgarian: 'bg',
|
2026-03-02 19:10:11 +08:00
|
|
|
Arabic: 'ar',
|
2026-03-18 16:09:32 +03:00
|
|
|
Turkish: 'tr',
|
feat: add native Dutch language support for BM25 tokenization (#14140)
## Summary
- Add language-aware Snowball stemmer to `RagTokenizer` supporting 16
languages (Dutch, German, French, Spanish, etc.)
- Thread the KB `language` parameter through the full tokenization
pipeline (14 parser modules + task executor)
- Add Dutch to the frontend language lists and cross-language form
## Problem
RAGFlow uses the English Porter stemmer + WordNet lemmatizer for **all**
BM25 tokenization, regardless of the knowledge base language setting.
This produces incorrect stems for non-English text. For example:
| Dutch word | Dutch stemmer | English Porter |
|---|---|---|
| documenten | document | documenten (unchanged!) |
| gebruikers | gebruiker | gebruik (over-stemmed) |
| instellingen | instell | instellingen (unchanged!) |
This degrades BM25 recall for any non-English knowledge base.
## Solution
NLTK already ships Snowball stemmers for 16 languages. This PR:
1. **`rag/nlp/rag_tokenizer.py`**: Overrides `tokenize()` with
`set_language()` and `_normalize_token()` that selects the correct NLTK
Snowball stemmer. Falls back to Porter for unmapped languages (Chinese,
Japanese, Korean, etc. — these use character-based tokenization anyway).
2. **`rag/nlp/__init__.py`** + **14 `rag/app/*.py` parsers** +
**`rag/svr/task_executor.py`**: Threads the `language` parameter through
`tokenize()`, `tokenize_chunks()`, `tokenize_table()`, and all callers.
3. **Frontend**: Adds Dutch (`Nederlands`) to `LanguageList`,
`LanguageMap`, `LanguageAbbreviationMap`, `LanguageTranslationMap`,
cross-language form field, and `en.ts` locale.
## Backward Compatibility
- Default language is `"English"`, preserving existing behavior for all
current users
- Languages without a Snowball stemmer mapping fall back to Porter (no
change)
- No new dependencies — NLTK Snowball is already bundled
2026-07-06 17:39:56 +02:00
|
|
|
Dutch: 'nl',
|
2024-04-12 11:41:00 +08:00
|
|
|
};
|
2024-05-08 10:30:18 +08:00
|
|
|
|
2024-12-06 13:43:17 +08:00
|
|
|
export enum FileMimeType {
|
|
|
|
|
Bmp = 'image/bmp',
|
|
|
|
|
Csv = 'text/csv',
|
|
|
|
|
Odt = 'application/vnd.oasis.opendocument.text',
|
|
|
|
|
Doc = 'application/msword',
|
|
|
|
|
Docx = 'application/vnd.openxmlformats-officedocument.wordprocessingml.document',
|
|
|
|
|
Gif = 'image/gif',
|
|
|
|
|
Htm = 'text/htm',
|
|
|
|
|
Html = 'text/html',
|
|
|
|
|
Jpg = 'image/jpg',
|
|
|
|
|
Jpeg = 'image/jpeg',
|
|
|
|
|
Pdf = 'application/pdf',
|
|
|
|
|
Png = 'image/png',
|
|
|
|
|
Ppt = 'application/vnd.ms-powerpoint',
|
|
|
|
|
Pptx = 'application/vnd.openxmlformats-officedocument.presentationml.presentation',
|
|
|
|
|
Tiff = 'image/tiff',
|
|
|
|
|
Txt = 'text/plain',
|
|
|
|
|
Xls = 'application/vnd.ms-excel',
|
|
|
|
|
Xlsx = 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet',
|
|
|
|
|
Mp4 = 'video/mp4',
|
|
|
|
|
Json = 'application/json',
|
2025-12-29 12:54:31 +08:00
|
|
|
Md = 'text/markdown',
|
2026-06-11 17:55:40 +09:00
|
|
|
Mdx = 'text/mdx',
|
2024-12-06 13:43:17 +08:00
|
|
|
}
|
2024-05-09 11:30:15 +08:00
|
|
|
|
2026-03-16 14:44:39 +08:00
|
|
|
export const Domain = 'cloud.ragflow.io';
|
2024-05-10 10:38:39 +08:00
|
|
|
|
2024-05-09 11:30:15 +08:00
|
|
|
//#region file preview
|
|
|
|
|
export const Images = [
|
|
|
|
|
'jpg',
|
|
|
|
|
'jpeg',
|
|
|
|
|
'png',
|
|
|
|
|
'gif',
|
|
|
|
|
'bmp',
|
|
|
|
|
'tif',
|
|
|
|
|
'tiff',
|
|
|
|
|
'webp',
|
|
|
|
|
// 'svg',
|
|
|
|
|
'ico',
|
|
|
|
|
];
|
|
|
|
|
|
|
|
|
|
// Without FileViewer
|
2025-12-29 12:54:31 +08:00
|
|
|
export const ExceptiveType = [
|
|
|
|
|
'xlsx',
|
|
|
|
|
'xls',
|
|
|
|
|
'pdf',
|
|
|
|
|
'docx',
|
|
|
|
|
'md',
|
|
|
|
|
'mdx',
|
|
|
|
|
...Images,
|
|
|
|
|
];
|
2024-05-09 11:30:15 +08:00
|
|
|
|
2024-05-10 10:38:39 +08:00
|
|
|
export const SupportedPreviewDocumentTypes = [...ExceptiveType];
|
2024-05-09 11:30:15 +08:00
|
|
|
//#endregion
|
2025-02-24 19:30:33 +08:00
|
|
|
|
|
|
|
|
export enum Platform {
|
|
|
|
|
RAGFlow = 'RAGFlow',
|
|
|
|
|
Dify = 'Dify',
|
|
|
|
|
FastGPT = 'FastGPT',
|
|
|
|
|
Coze = 'Coze',
|
|
|
|
|
}
|
2025-07-31 12:14:00 +08:00
|
|
|
|
|
|
|
|
export enum ThemeEnum {
|
|
|
|
|
Dark = 'dark',
|
|
|
|
|
Light = 'light',
|
|
|
|
|
System = 'system',
|
|
|
|
|
}
|