Files
ragflow/web/src/constants/common.ts

220 lines
4.5 KiB
TypeScript
Raw Normal View History

export const fileIconMap = {
aep: 'aep.svg',
ai: 'ai.svg',
avi: 'avi.svg',
css: 'css.svg',
csv: 'csv.svg',
dmg: 'dmg.svg',
doc: 'doc.svg',
docx: 'docx.svg',
eps: 'eps.svg',
exe: 'exe.svg',
fig: 'fig.svg',
gif: 'gif.svg',
html: 'html.svg',
indd: 'indd.svg',
java: 'java.svg',
jpeg: 'jpeg.svg',
jpg: 'jpg.svg',
js: 'js.svg',
json: 'json.svg',
md: 'md.svg',
mdx: 'mdx.svg',
mkv: 'mkv.svg',
mp3: 'mp3.svg',
mp4: 'mp4.svg',
mpeg: 'mpeg.svg',
pdf: 'pdf.svg',
png: 'png.svg',
ppt: 'ppt.svg',
pptx: 'pptx.svg',
psd: 'psd.svg',
rss: 'rss.svg',
sql: 'sql.svg',
svg: 'svg.svg',
tiff: 'tiff.svg',
txt: 'txt.svg',
wav: 'wav.svg',
webp: 'webp.svg',
xls: 'xls.svg',
xlsx: 'xlsx.svg',
xml: 'xml.svg',
};
// TODO: Use standard BCP 47 language tag and display names
export const LanguageList = [
'English',
'Chinese',
'Traditional Chinese',
'Russian',
'Indonesian',
'Spanish',
'Vietnamese',
'Japanese',
'Portuguese BR',
'German',
'French',
'Italian',
'Bulgarian',
'Arabic',
'Turkish',
feat: add native Dutch language support for BM25 tokenization (#14140) ## Summary - Add language-aware Snowball stemmer to `RagTokenizer` supporting 16 languages (Dutch, German, French, Spanish, etc.) - Thread the KB `language` parameter through the full tokenization pipeline (14 parser modules + task executor) - Add Dutch to the frontend language lists and cross-language form ## Problem RAGFlow uses the English Porter stemmer + WordNet lemmatizer for **all** BM25 tokenization, regardless of the knowledge base language setting. This produces incorrect stems for non-English text. For example: | Dutch word | Dutch stemmer | English Porter | |---|---|---| | documenten | document | documenten (unchanged!) | | gebruikers | gebruiker | gebruik (over-stemmed) | | instellingen | instell | instellingen (unchanged!) | This degrades BM25 recall for any non-English knowledge base. ## Solution NLTK already ships Snowball stemmers for 16 languages. This PR: 1. **`rag/nlp/rag_tokenizer.py`**: Overrides `tokenize()` with `set_language()` and `_normalize_token()` that selects the correct NLTK Snowball stemmer. Falls back to Porter for unmapped languages (Chinese, Japanese, Korean, etc. — these use character-based tokenization anyway). 2. **`rag/nlp/__init__.py`** + **14 `rag/app/*.py` parsers** + **`rag/svr/task_executor.py`**: Threads the `language` parameter through `tokenize()`, `tokenize_chunks()`, `tokenize_table()`, and all callers. 3. **Frontend**: Adds Dutch (`Nederlands`) to `LanguageList`, `LanguageMap`, `LanguageAbbreviationMap`, `LanguageTranslationMap`, cross-language form field, and `en.ts` locale. ## Backward Compatibility - Default language is `"English"`, preserving existing behavior for all current users - Languages without a Snowball stemmer mapping fall back to Porter (no change) - No new dependencies — NLTK Snowball is already bundled
2026-07-06 17:39:56 +02:00
'Dutch',
];
export const LanguageMap = {
English: 'English',
Chinese: '简体中文',
'Traditional Chinese': '繁體中文',
Russian: 'Русский',
Indonesian: 'Bahasa Indonesia',
Indonesia: 'Indonesia',
Spanish: 'Español',
Vietnamese: 'Tiếng việt',
Japanese: '日本語',
'Portuguese BR': 'Português BR',
German: 'Deutsch',
French: 'Français',
Italian: 'Italiano',
Bulgarian: 'Български',
Arabic: 'العربية',
Turkish: 'Türkçe',
feat: add native Dutch language support for BM25 tokenization (#14140) ## Summary - Add language-aware Snowball stemmer to `RagTokenizer` supporting 16 languages (Dutch, German, French, Spanish, etc.) - Thread the KB `language` parameter through the full tokenization pipeline (14 parser modules + task executor) - Add Dutch to the frontend language lists and cross-language form ## Problem RAGFlow uses the English Porter stemmer + WordNet lemmatizer for **all** BM25 tokenization, regardless of the knowledge base language setting. This produces incorrect stems for non-English text. For example: | Dutch word | Dutch stemmer | English Porter | |---|---|---| | documenten | document | documenten (unchanged!) | | gebruikers | gebruiker | gebruik (over-stemmed) | | instellingen | instell | instellingen (unchanged!) | This degrades BM25 recall for any non-English knowledge base. ## Solution NLTK already ships Snowball stemmers for 16 languages. This PR: 1. **`rag/nlp/rag_tokenizer.py`**: Overrides `tokenize()` with `set_language()` and `_normalize_token()` that selects the correct NLTK Snowball stemmer. Falls back to Porter for unmapped languages (Chinese, Japanese, Korean, etc. — these use character-based tokenization anyway). 2. **`rag/nlp/__init__.py`** + **14 `rag/app/*.py` parsers** + **`rag/svr/task_executor.py`**: Threads the `language` parameter through `tokenize()`, `tokenize_chunks()`, `tokenize_table()`, and all callers. 3. **Frontend**: Adds Dutch (`Nederlands`) to `LanguageList`, `LanguageMap`, `LanguageAbbreviationMap`, `LanguageTranslationMap`, cross-language form field, and `en.ts` locale. ## Backward Compatibility - Default language is `"English"`, preserving existing behavior for all current users - Languages without a Snowball stemmer mapping fall back to Porter (no change) - No new dependencies — NLTK Snowball is already bundled
2026-07-06 17:39:56 +02:00
Dutch: 'Nederlands',
};
export enum LanguageAbbreviation {
En = 'en',
Zh = 'zh-Hans',
ZhTraditional = 'zh-Hant',
Ru = 'ru',
Id = 'id',
Ja = 'ja',
Es = 'es',
Vi = 'vi',
PtBr = 'pt-BR',
De = 'de',
Fr = 'fr',
It = 'it',
Bg = 'bg',
Ar = 'ar',
Tr = 'tr',
Ko = 'ko',
feat: add native Dutch language support for BM25 tokenization (#14140) ## Summary - Add language-aware Snowball stemmer to `RagTokenizer` supporting 16 languages (Dutch, German, French, Spanish, etc.) - Thread the KB `language` parameter through the full tokenization pipeline (14 parser modules + task executor) - Add Dutch to the frontend language lists and cross-language form ## Problem RAGFlow uses the English Porter stemmer + WordNet lemmatizer for **all** BM25 tokenization, regardless of the knowledge base language setting. This produces incorrect stems for non-English text. For example: | Dutch word | Dutch stemmer | English Porter | |---|---|---| | documenten | document | documenten (unchanged!) | | gebruikers | gebruiker | gebruik (over-stemmed) | | instellingen | instell | instellingen (unchanged!) | This degrades BM25 recall for any non-English knowledge base. ## Solution NLTK already ships Snowball stemmers for 16 languages. This PR: 1. **`rag/nlp/rag_tokenizer.py`**: Overrides `tokenize()` with `set_language()` and `_normalize_token()` that selects the correct NLTK Snowball stemmer. Falls back to Porter for unmapped languages (Chinese, Japanese, Korean, etc. — these use character-based tokenization anyway). 2. **`rag/nlp/__init__.py`** + **14 `rag/app/*.py` parsers** + **`rag/svr/task_executor.py`**: Threads the `language` parameter through `tokenize()`, `tokenize_chunks()`, `tokenize_table()`, and all callers. 3. **Frontend**: Adds Dutch (`Nederlands`) to `LanguageList`, `LanguageMap`, `LanguageAbbreviationMap`, `LanguageTranslationMap`, cross-language form field, and `en.ts` locale. ## Backward Compatibility - Default language is `"English"`, preserving existing behavior for all current users - Languages without a Snowball stemmer mapping fall back to Porter (no change) - No new dependencies — NLTK Snowball is already bundled
2026-07-06 17:39:56 +02:00
Nl = 'nl',
}
export const LanguageAbbreviationMap = {
[LanguageAbbreviation.En]: 'English',
[LanguageAbbreviation.Zh]: '简体中文',
[LanguageAbbreviation.ZhTraditional]: '繁體中文',
[LanguageAbbreviation.Ru]: 'Русский',
[LanguageAbbreviation.Id]: 'Bahasa Indonesia',
[LanguageAbbreviation.Es]: 'Español',
[LanguageAbbreviation.Vi]: 'Tiếng việt',
[LanguageAbbreviation.Ja]: '日本語',
[LanguageAbbreviation.PtBr]: 'Português BR',
[LanguageAbbreviation.De]: 'Deutsch',
[LanguageAbbreviation.Fr]: 'Français',
[LanguageAbbreviation.It]: 'Italiano',
[LanguageAbbreviation.Bg]: 'Български',
[LanguageAbbreviation.Ar]: 'العربية',
[LanguageAbbreviation.Tr]: 'Türkçe',
[LanguageAbbreviation.Ko]: '한국어',
feat: add native Dutch language support for BM25 tokenization (#14140) ## Summary - Add language-aware Snowball stemmer to `RagTokenizer` supporting 16 languages (Dutch, German, French, Spanish, etc.) - Thread the KB `language` parameter through the full tokenization pipeline (14 parser modules + task executor) - Add Dutch to the frontend language lists and cross-language form ## Problem RAGFlow uses the English Porter stemmer + WordNet lemmatizer for **all** BM25 tokenization, regardless of the knowledge base language setting. This produces incorrect stems for non-English text. For example: | Dutch word | Dutch stemmer | English Porter | |---|---|---| | documenten | document | documenten (unchanged!) | | gebruikers | gebruiker | gebruik (over-stemmed) | | instellingen | instell | instellingen (unchanged!) | This degrades BM25 recall for any non-English knowledge base. ## Solution NLTK already ships Snowball stemmers for 16 languages. This PR: 1. **`rag/nlp/rag_tokenizer.py`**: Overrides `tokenize()` with `set_language()` and `_normalize_token()` that selects the correct NLTK Snowball stemmer. Falls back to Porter for unmapped languages (Chinese, Japanese, Korean, etc. — these use character-based tokenization anyway). 2. **`rag/nlp/__init__.py`** + **14 `rag/app/*.py` parsers** + **`rag/svr/task_executor.py`**: Threads the `language` parameter through `tokenize()`, `tokenize_chunks()`, `tokenize_table()`, and all callers. 3. **Frontend**: Adds Dutch (`Nederlands`) to `LanguageList`, `LanguageMap`, `LanguageAbbreviationMap`, `LanguageTranslationMap`, cross-language form field, and `en.ts` locale. ## Backward Compatibility - Default language is `"English"`, preserving existing behavior for all current users - Languages without a Snowball stemmer mapping fall back to Porter (no change) - No new dependencies — NLTK Snowball is already bundled
2026-07-06 17:39:56 +02:00
[LanguageAbbreviation.Nl]: 'Nederlands',
};
export const LanguageTranslationMap = {
English: 'en',
Chinese: 'zh-Hans',
'Traditional Chinese': 'zh-Hant',
Russian: 'ru',
Feature/mineru improvements (#11938) 我已在下面的评论中用中文重复说明。 ### What problem does this PR solve? ## Summary This PR enhances the MinerU document parser with additional configuration options, giving users more control over PDF parsing behavior and improving support for multilingual documents. ## Changes ### Backend (`deepdoc/parser/mineru_parser.py`) - Added configurable parsing options: - **Parse Method**: `auto`, `txt`, or `ocr` — allows users to choose the extraction strategy - **Formula Recognition**: Toggle for enabling/disabling formula extraction (useful to disable for Cyrillic documents where it may cause issues) - **Table Recognition**: Toggle for enabling/disabling table extraction - Added language code mapping (`LANGUAGE_TO_MINERU_MAP`) to translate RAGFlow language settings to MinerU-compatible language codes for better OCR accuracy - Improved parser configuration handling to pass these options through the processing pipeline ### Frontend (`web/`) - Created new `MinerUOptionsFormField` component that conditionally renders when MinerU is selected as the layout recognition engine - Added UI controls for: - Parse method selection (dropdown) - Formula recognition toggle (switch) - Table recognition toggle (switch) - Added i18n translations for English and Chinese - Integrated the options into both the dataset creation dialog and dataset settings page ### Integration - Updated `rag/app/naive.py` to forward MinerU options to the parser - Updated task service to handle the new configuration parameters ## Why MinerU is a powerful document parser, but the default settings don't work well for all document types. This PR allows users to: 1. Choose the best parsing method for their documents 2. Disable formula recognition for Cyrillic/non-Latin scripts where it causes issues 3. Control table extraction based on document needs 4. Benefit from automatic language detection for better OCR results ## Testing - [x] Tested MinerU parsing with different parse methods - [x] Verified UI renders correctly when MinerU is selected/deselected - [x] Confirmed settings persist correctly in dataset configuration ### Type of change - [x] Bug Fix (non-breaking change which fixes an issue) - [x] New Feature (non-breaking change which adds functionality) - [ ] Documentation Update - [x] Refactoring - [ ] Performance Improvement - [ ] Other (please describe): --------- Co-authored-by: user210 <user210@rt> Co-authored-by: Kevin Hu <kevinhu.sh@gmail.com>
2025-12-16 07:15:25 +02:00
Indonesian: 'id',
Indonesia: 'id',
Spanish: 'es',
Vietnamese: 'vi',
Japanese: 'ja',
Feature/mineru improvements (#11938) 我已在下面的评论中用中文重复说明。 ### What problem does this PR solve? ## Summary This PR enhances the MinerU document parser with additional configuration options, giving users more control over PDF parsing behavior and improving support for multilingual documents. ## Changes ### Backend (`deepdoc/parser/mineru_parser.py`) - Added configurable parsing options: - **Parse Method**: `auto`, `txt`, or `ocr` — allows users to choose the extraction strategy - **Formula Recognition**: Toggle for enabling/disabling formula extraction (useful to disable for Cyrillic documents where it may cause issues) - **Table Recognition**: Toggle for enabling/disabling table extraction - Added language code mapping (`LANGUAGE_TO_MINERU_MAP`) to translate RAGFlow language settings to MinerU-compatible language codes for better OCR accuracy - Improved parser configuration handling to pass these options through the processing pipeline ### Frontend (`web/`) - Created new `MinerUOptionsFormField` component that conditionally renders when MinerU is selected as the layout recognition engine - Added UI controls for: - Parse method selection (dropdown) - Formula recognition toggle (switch) - Table recognition toggle (switch) - Added i18n translations for English and Chinese - Integrated the options into both the dataset creation dialog and dataset settings page ### Integration - Updated `rag/app/naive.py` to forward MinerU options to the parser - Updated task service to handle the new configuration parameters ## Why MinerU is a powerful document parser, but the default settings don't work well for all document types. This PR allows users to: 1. Choose the best parsing method for their documents 2. Disable formula recognition for Cyrillic/non-Latin scripts where it causes issues 3. Control table extraction based on document needs 4. Benefit from automatic language detection for better OCR results ## Testing - [x] Tested MinerU parsing with different parse methods - [x] Verified UI renders correctly when MinerU is selected/deselected - [x] Confirmed settings persist correctly in dataset configuration ### Type of change - [x] Bug Fix (non-breaking change which fixes an issue) - [x] New Feature (non-breaking change which adds functionality) - [ ] Documentation Update - [x] Refactoring - [ ] Performance Improvement - [ ] Other (please describe): --------- Co-authored-by: user210 <user210@rt> Co-authored-by: Kevin Hu <kevinhu.sh@gmail.com>
2025-12-16 07:15:25 +02:00
Korean: 'ko',
'Portuguese BR': 'pt-BR',
'pt-br': 'pt-BR',
'pt-BR': 'pt-BR',
German: 'de',
French: 'fr',
Italian: 'it',
Feature/mineru improvements (#11938) 我已在下面的评论中用中文重复说明。 ### What problem does this PR solve? ## Summary This PR enhances the MinerU document parser with additional configuration options, giving users more control over PDF parsing behavior and improving support for multilingual documents. ## Changes ### Backend (`deepdoc/parser/mineru_parser.py`) - Added configurable parsing options: - **Parse Method**: `auto`, `txt`, or `ocr` — allows users to choose the extraction strategy - **Formula Recognition**: Toggle for enabling/disabling formula extraction (useful to disable for Cyrillic documents where it may cause issues) - **Table Recognition**: Toggle for enabling/disabling table extraction - Added language code mapping (`LANGUAGE_TO_MINERU_MAP`) to translate RAGFlow language settings to MinerU-compatible language codes for better OCR accuracy - Improved parser configuration handling to pass these options through the processing pipeline ### Frontend (`web/`) - Created new `MinerUOptionsFormField` component that conditionally renders when MinerU is selected as the layout recognition engine - Added UI controls for: - Parse method selection (dropdown) - Formula recognition toggle (switch) - Table recognition toggle (switch) - Added i18n translations for English and Chinese - Integrated the options into both the dataset creation dialog and dataset settings page ### Integration - Updated `rag/app/naive.py` to forward MinerU options to the parser - Updated task service to handle the new configuration parameters ## Why MinerU is a powerful document parser, but the default settings don't work well for all document types. This PR allows users to: 1. Choose the best parsing method for their documents 2. Disable formula recognition for Cyrillic/non-Latin scripts where it causes issues 3. Control table extraction based on document needs 4. Benefit from automatic language detection for better OCR results ## Testing - [x] Tested MinerU parsing with different parse methods - [x] Verified UI renders correctly when MinerU is selected/deselected - [x] Confirmed settings persist correctly in dataset configuration ### Type of change - [x] Bug Fix (non-breaking change which fixes an issue) - [x] New Feature (non-breaking change which adds functionality) - [ ] Documentation Update - [x] Refactoring - [ ] Performance Improvement - [ ] Other (please describe): --------- Co-authored-by: user210 <user210@rt> Co-authored-by: Kevin Hu <kevinhu.sh@gmail.com>
2025-12-16 07:15:25 +02:00
Tamil: 'ta',
Telugu: 'te',
Kannada: 'ka',
Thai: 'th',
Greek: 'el',
Hindi: 'hi',
Ukrainian: 'uk',
Bulgarian: 'bg',
Arabic: 'ar',
Turkish: 'tr',
feat: add native Dutch language support for BM25 tokenization (#14140) ## Summary - Add language-aware Snowball stemmer to `RagTokenizer` supporting 16 languages (Dutch, German, French, Spanish, etc.) - Thread the KB `language` parameter through the full tokenization pipeline (14 parser modules + task executor) - Add Dutch to the frontend language lists and cross-language form ## Problem RAGFlow uses the English Porter stemmer + WordNet lemmatizer for **all** BM25 tokenization, regardless of the knowledge base language setting. This produces incorrect stems for non-English text. For example: | Dutch word | Dutch stemmer | English Porter | |---|---|---| | documenten | document | documenten (unchanged!) | | gebruikers | gebruiker | gebruik (over-stemmed) | | instellingen | instell | instellingen (unchanged!) | This degrades BM25 recall for any non-English knowledge base. ## Solution NLTK already ships Snowball stemmers for 16 languages. This PR: 1. **`rag/nlp/rag_tokenizer.py`**: Overrides `tokenize()` with `set_language()` and `_normalize_token()` that selects the correct NLTK Snowball stemmer. Falls back to Porter for unmapped languages (Chinese, Japanese, Korean, etc. — these use character-based tokenization anyway). 2. **`rag/nlp/__init__.py`** + **14 `rag/app/*.py` parsers** + **`rag/svr/task_executor.py`**: Threads the `language` parameter through `tokenize()`, `tokenize_chunks()`, `tokenize_table()`, and all callers. 3. **Frontend**: Adds Dutch (`Nederlands`) to `LanguageList`, `LanguageMap`, `LanguageAbbreviationMap`, `LanguageTranslationMap`, cross-language form field, and `en.ts` locale. ## Backward Compatibility - Default language is `"English"`, preserving existing behavior for all current users - Languages without a Snowball stemmer mapping fall back to Porter (no change) - No new dependencies — NLTK Snowball is already bundled
2026-07-06 17:39:56 +02:00
Dutch: 'nl',
};
export enum FileMimeType {
Bmp = 'image/bmp',
Csv = 'text/csv',
Odt = 'application/vnd.oasis.opendocument.text',
Doc = 'application/msword',
Docx = 'application/vnd.openxmlformats-officedocument.wordprocessingml.document',
Gif = 'image/gif',
Htm = 'text/htm',
Html = 'text/html',
Jpg = 'image/jpg',
Jpeg = 'image/jpeg',
Pdf = 'application/pdf',
Png = 'image/png',
Ppt = 'application/vnd.ms-powerpoint',
Pptx = 'application/vnd.openxmlformats-officedocument.presentationml.presentation',
Tiff = 'image/tiff',
Txt = 'text/plain',
Xls = 'application/vnd.ms-excel',
Xlsx = 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet',
Mp4 = 'video/mp4',
Json = 'application/json',
Md = 'text/markdown',
Mdx = 'text/mdx',
}
export const Domain = 'cloud.ragflow.io';
//#region file preview
export const Images = [
'jpg',
'jpeg',
'png',
'gif',
'bmp',
'tif',
'tiff',
'webp',
// 'svg',
'ico',
];
// Without FileViewer
export const ExceptiveType = [
'xlsx',
'xls',
'pdf',
'docx',
'md',
'mdx',
...Images,
];
export const SupportedPreviewDocumentTypes = [...ExceptiveType];
//#endregion
export enum Platform {
RAGFlow = 'RAGFlow',
Dify = 'Dify',
FastGPT = 'FastGPT',
Coze = 'Coze',
}
export enum ThemeEnum {
Dark = 'dark',
Light = 'light',
System = 'system',
}