From d8cefcf052e5f16340da9a60b098f20b8b41ef91 Mon Sep 17 00:00:00 2001 From: Rodger Blom <151914184+rwb-truelime@users.noreply.github.com> Date: Mon, 6 Jul 2026 17:39:56 +0200 Subject: [PATCH] feat: add native Dutch language support for BM25 tokenization (#14140) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Summary - Add language-aware Snowball stemmer to `RagTokenizer` supporting 16 languages (Dutch, German, French, Spanish, etc.) - Thread the KB `language` parameter through the full tokenization pipeline (14 parser modules + task executor) - Add Dutch to the frontend language lists and cross-language form ## Problem RAGFlow uses the English Porter stemmer + WordNet lemmatizer for **all** BM25 tokenization, regardless of the knowledge base language setting. This produces incorrect stems for non-English text. For example: | Dutch word | Dutch stemmer | English Porter | |---|---|---| | documenten | document | documenten (unchanged!) | | gebruikers | gebruiker | gebruik (over-stemmed) | | instellingen | instell | instellingen (unchanged!) | This degrades BM25 recall for any non-English knowledge base. ## Solution NLTK already ships Snowball stemmers for 16 languages. This PR: 1. **`rag/nlp/rag_tokenizer.py`**: Overrides `tokenize()` with `set_language()` and `_normalize_token()` that selects the correct NLTK Snowball stemmer. Falls back to Porter for unmapped languages (Chinese, Japanese, Korean, etc. — these use character-based tokenization anyway). 2. **`rag/nlp/__init__.py`** + **14 `rag/app/*.py` parsers** + **`rag/svr/task_executor.py`**: Threads the `language` parameter through `tokenize()`, `tokenize_chunks()`, `tokenize_table()`, and all callers. 3. **Frontend**: Adds Dutch (`Nederlands`) to `LanguageList`, `LanguageMap`, `LanguageAbbreviationMap`, `LanguageTranslationMap`, cross-language form field, and `en.ts` locale. ## Backward Compatibility - Default language is `"English"`, preserving existing behavior for all current users - Languages without a Snowball stemmer mapping fall back to Porter (no change) - No new dependencies — NLTK Snowball is already bundled --- .gitignore | 1 + rag/app/audio.py | 2 +- rag/app/book.py | 4 +- rag/app/email.py | 2 +- rag/app/laws.py | 4 +- rag/app/manual.py | 8 ++-- rag/app/naive.py | 16 ++++---- rag/app/one.py | 2 +- rag/app/paper.py | 10 ++--- rag/app/picture.py | 6 +-- rag/app/presentation.py | 6 +-- rag/app/qa.py | 6 ++- rag/app/resume.py | 1 + rag/app/table.py | 4 +- rag/app/tag.py | 1 + rag/nlp/__init__.py | 39 ++++++++++--------- rag/svr/task_executor.py | 6 +++ .../components/cross-language-form-field.tsx | 1 + web/src/constants/common.ts | 5 +++ web/src/locales/en.ts | 2 + 20 files changed, 73 insertions(+), 53 deletions(-) diff --git a/.gitignore b/.gitignore index 4b339a811..e2a8cef07 100644 --- a/.gitignore +++ b/.gitignore @@ -22,6 +22,7 @@ Cargo.lock .idea/ .vscode/ .cursor/settings.json +.opencode/ # Exclude Mac generated files .DS_Store diff --git a/rag/app/audio.py b/rag/app/audio.py index 38294eb99..e5c600978 100644 --- a/rag/app/audio.py +++ b/rag/app/audio.py @@ -50,7 +50,7 @@ def chunk(filename, binary, tenant_id, lang, callback=None, **kwargs): ans = seq2txt_mdl.transcription(tmp_path) callback(0.8, "Sequence2Txt LLM respond: %s ..." % ans[:32]) - tokenize(doc, ans, is_english) + tokenize(doc, ans, is_english, language=lang) return [doc] except Exception as e: callback(prog=-1, msg=str(e)) diff --git a/rag/app/book.py b/rag/app/book.py index c19c0d257..a9718eb3b 100644 --- a/rag/app/book.py +++ b/rag/app/book.py @@ -169,8 +169,8 @@ def chunk(filename, binary=None, from_page=0, to_page=MAXIMUM_PAGE_NUMBER, lang= # is_english(random_choices([t for t, _ in sections], k=218)) eng = lang.lower() == "english" - res = tokenize_table(tbls, doc, eng) - res.extend(tokenize_chunks(chunks, doc, eng, pdf_parser)) + res = tokenize_table(tbls, doc, eng, language=lang) + res.extend(tokenize_chunks(chunks, doc, eng, pdf_parser, language=lang)) table_ctx = max(0, int(parser_config.get("table_context_size", 0) or 0)) image_ctx = max(0, int(parser_config.get("image_context_size", 0) or 0)) if table_ctx or image_ctx: diff --git a/rag/app/email.py b/rag/app/email.py index 0e9f4b041..90c926e62 100644 --- a/rag/app/email.py +++ b/rag/app/email.py @@ -102,7 +102,7 @@ def chunk( parser_config.get("delimiter", "\n!?。；！？"), ) - main_res.extend(tokenize_chunks(chunks, doc, eng, None)) + main_res.extend(tokenize_chunks(chunks, doc, eng, None, language=lang)) logging.debug("naive_merge({}): {}".format(filename, timer() - st)) # get the attachment info for part in msg.iter_attachments(): diff --git a/rag/app/laws.py b/rag/app/laws.py index d36534083..b2c518ff1 100644 --- a/rag/app/laws.py +++ b/rag/app/laws.py @@ -180,7 +180,7 @@ def chunk(filename, binary=None, from_page=0, to_page=MAXIMUM_PAGE_NUMBER, lang= callback(0.1, "Start to parse.") chunks = Docx()(filename, binary) callback(0.7, "Finish parsing.") - return tokenize_chunks(chunks, doc, eng, None) + return tokenize_chunks(chunks, doc, eng, None, language=lang) elif re.search(r"\.pdf$", filename, re.IGNORECASE): layout_recognizer, parser_model_name = normalize_layout_recognizer(parser_config.get("layout_recognize", "DeepDOC")) @@ -261,7 +261,7 @@ def chunk(filename, binary=None, from_page=0, to_page=MAXIMUM_PAGE_NUMBER, lang= if not res: callback(0.99, "No chunk parsed out.") - return tokenize_chunks(res, doc, eng, pdf_parser) + return tokenize_chunks(res, doc, eng, pdf_parser, language=lang) # chunks = hierarchical_merge(bull, sections, 5) # return tokenize_chunks(["\n".join(ck)for ck in chunks], doc, eng, pdf_parser) diff --git a/rag/app/manual.py b/rag/app/manual.py index 371755e3c..766055080 100644 --- a/rag/app/manual.py +++ b/rag/app/manual.py @@ -261,8 +261,8 @@ def chunk(filename, binary=None, from_page=0, to_page=MAXIMUM_PAGE_NUMBER, lang= callback=callback, **kwargs, ) - res = tokenize_table(tbls, doc, eng) - res.extend(tokenize_chunks(chunks, doc, eng, pdf_parser)) + res = tokenize_table(tbls, doc, eng, language=lang) + res.extend(tokenize_chunks(chunks, doc, eng, pdf_parser, language=lang)) table_ctx = max(0, int(parser_config.get("table_context_size", 0) or 0)) image_ctx = max(0, int(parser_config.get("image_context_size", 0) or 0)) if table_ctx or image_ctx: @@ -275,13 +275,13 @@ def chunk(filename, binary=None, from_page=0, to_page=MAXIMUM_PAGE_NUMBER, lang= docx_parser = Docx() ti_list, tbls = docx_parser(filename, binary, from_page=0, to_page=MAXIMUM_PAGE_NUMBER, callback=callback) tbls = vision_figure_parser_docx_wrapper(sections=ti_list, tbls=tbls, callback=callback, **kwargs) - res = tokenize_table(tbls, doc, eng) + res = tokenize_table(tbls, doc, eng, language=lang) for text, image in ti_list: d = copy.deepcopy(doc) if image: d["image"] = image d["doc_type_kwd"] = "image" - tokenize(d, text, eng) + tokenize(d, text, eng, language=lang) res.append(d) table_ctx = max(0, int(parser_config.get("table_context_size", 0) or 0)) image_ctx = max(0, int(parser_config.get("image_context_size", 0) or 0)) diff --git a/rag/app/naive.py b/rag/app/naive.py index fe6fc2d80..024c46b15 100644 --- a/rag/app/naive.py +++ b/rag/app/naive.py @@ -976,7 +976,7 @@ def chunk(filename, binary=None, from_page=0, to_page=MAXIMUM_PAGE_NUMBER, lang= callback(0.8, "Finish parsing.") st = timer() - res.extend(doc_tokenize_chunks_with_images(chunks, doc, is_english, child_delimiters_pattern=child_deli)) + res.extend(doc_tokenize_chunks_with_images(chunks, doc, is_english, child_delimiters_pattern=child_deli, language=lang)) logging.info("naive_merge({}): {}".format(filename, timer() - st)) res.extend(embed_res) res.extend(url_res) @@ -1024,7 +1024,7 @@ def chunk(filename, binary=None, from_page=0, to_page=MAXIMUM_PAGE_NUMBER, lang= if int(parser_config.get("chunk_token_num", 0)) <= 0: parser_config["chunk_token_num"] = 0 - res = tokenize_table(tables, doc, is_english) + res = tokenize_table(tables, doc, is_english, language=lang) callback(0.8, "Finish parsing.") elif re.search(r"\.(csv|xlsx?)$", filename, re.IGNORECASE): @@ -1046,7 +1046,7 @@ def chunk(filename, binary=None, from_page=0, to_page=MAXIMUM_PAGE_NUMBER, lang= sections, tables = tcadp_parser.parse_pdf(filepath=filename, binary=binary, callback=callback, output_dir=os.environ.get("TCADP_OUTPUT_DIR", ""), file_type=file_type) sections = _normalize_section_text_for_rtl_presentation_forms(sections) parser_config["chunk_token_num"] = 0 - res = tokenize_table(tables, doc, is_english) + res = tokenize_table(tables, doc, is_english, language=lang) callback(0.8, "Finish parsing.") else: # Default DeepDOC parser @@ -1116,7 +1116,7 @@ def chunk(filename, binary=None, from_page=0, to_page=MAXIMUM_PAGE_NUMBER, lang= soup = markdown_parser.md_to_html(section_text) hyperlink_urls = markdown_parser.get_hyperlink_urls(soup) urls.update(hyperlink_urls) - res = tokenize_table(tables, doc, is_english) + res = tokenize_table(tables, doc, is_english, language=lang) callback(0.8, "Finish parsing.") elif re.search(r"\.(htm|html)$", filename, re.IGNORECASE): @@ -1215,9 +1215,9 @@ def chunk(filename, binary=None, from_page=0, to_page=MAXIMUM_PAGE_NUMBER, lang= has_images = merged_images and any(img is not None for img in merged_images) if has_images: - res.extend(tokenize_chunks_with_images(chunks, doc, is_english, merged_images, child_delimiters_pattern=child_deli)) + res.extend(tokenize_chunks_with_images(chunks, doc, is_english, merged_images, child_delimiters_pattern=child_deli, language=lang)) else: - res.extend(tokenize_chunks(chunks, doc, is_english, pdf_parser, child_delimiters_pattern=child_deli)) + res.extend(tokenize_chunks(chunks, doc, is_english, pdf_parser, child_delimiters_pattern=child_deli, language=lang)) else: if section_images: if all(image is None for image in section_images): @@ -1225,11 +1225,11 @@ def chunk(filename, binary=None, from_page=0, to_page=MAXIMUM_PAGE_NUMBER, lang= if section_images: chunks, images = naive_merge_with_images(sections, section_images, int(parser_config.get("chunk_token_num", 128)), parser_config.get("delimiter", "\n!?。；！？"), overlapped_percent) - res.extend(tokenize_chunks_with_images(chunks, doc, is_english, images, child_delimiters_pattern=child_deli)) + res.extend(tokenize_chunks_with_images(chunks, doc, is_english, images, child_delimiters_pattern=child_deli, language=lang)) else: chunks = naive_merge(sections, int(parser_config.get("chunk_token_num", 128)), parser_config.get("delimiter", "\n!?。；！？"), overlapped_percent) - res.extend(tokenize_chunks(chunks, doc, is_english, pdf_parser, child_delimiters_pattern=child_deli)) + res.extend(tokenize_chunks(chunks, doc, is_english, pdf_parser, child_delimiters_pattern=child_deli, language=lang)) if urls and parser_config.get("analyze_hyperlink", False) and is_root: for index, url in enumerate(urls): diff --git a/rag/app/one.py b/rag/app/one.py index d5fbbfcc8..ba81a1d2a 100644 --- a/rag/app/one.py +++ b/rag/app/one.py @@ -163,7 +163,7 @@ def chunk(filename, binary=None, from_page=0, to_page=MAXIMUM_PAGE_NUMBER, lang= doc = {"docnm_kwd": filename, "title_tks": rag_tokenizer.tokenize(re.sub(r"\.[a-zA-Z]+$", "", filename))} doc["title_sm_tks"] = rag_tokenizer.fine_grained_tokenize(doc["title_tks"]) - tokenize(doc, "\n".join(sections), eng) + tokenize(doc, "\n".join(sections), eng, language=lang) return [doc] diff --git a/rag/app/paper.py b/rag/app/paper.py index cc3a898a6..5f30bc0da 100644 --- a/rag/app/paper.py +++ b/rag/app/paper.py @@ -188,7 +188,7 @@ def chunk(filename, binary=None, from_page=0, to_page=MAXIMUM_PAGE_NUMBER, lang= eng = lang.lower() == "english" # pdf_parser.is_english logging.debug("It's English.....{}".format(eng)) - res = tokenize_table(paper["tables"], doc, eng) + res = tokenize_table(paper["tables"], doc, eng, language=lang) if paper["abstract"]: d = copy.deepcopy(doc) @@ -197,7 +197,7 @@ def chunk(filename, binary=None, from_page=0, to_page=MAXIMUM_PAGE_NUMBER, lang= d["important_tks"] = " ".join(d["important_kwd"]) d["image"], poss = pdf_parser.crop(paper["abstract"], need_position=True) add_positions(d, poss) - tokenize(d, txt, eng) + tokenize(d, txt, eng, language=lang) res.append(d) sorted_sections = paper["sections"] @@ -223,7 +223,7 @@ def chunk(filename, binary=None, from_page=0, to_page=MAXIMUM_PAGE_NUMBER, lang= continue chunks.append(txt) last_sid = sec_id - res.extend(tokenize_chunks(chunks, doc, eng, pdf_parser)) + res.extend(tokenize_chunks(chunks, doc, eng, pdf_parser, language=lang)) table_ctx = max(0, int(parser_config.get("table_context_size", 0) or 0)) image_ctx = max(0, int(parser_config.get("image_context_size", 0) or 0)) if table_ctx or image_ctx: @@ -264,7 +264,7 @@ def chunk(filename, binary=None, from_page=0, to_page=MAXIMUM_PAGE_NUMBER, lang= txt += "\n" + pdf_parser.remove_tag(p) d["image"], poss = pdf_parser.crop(p, need_position=True) add_positions(d, poss) - tokenize(d, txt, eng) + tokenize(d, txt, eng, language=lang) res.append(d) i = 0 @@ -274,7 +274,7 @@ def chunk(filename, binary=None, from_page=0, to_page=MAXIMUM_PAGE_NUMBER, lang= nonlocal chunk, res, doc, pdf_parser, tk_cnt d = copy.deepcopy(doc) ck = "\n".join(chunk) - tokenize(d, pdf_parser.remove_tag(ck), pdf_parser.is_english) + tokenize(d, pdf_parser.remove_tag(ck), pdf_parser.is_english, language=lang) d["image"], poss = pdf_parser.crop(ck, need_position=True) add_positions(d, poss) res.append(d) diff --git a/rag/app/picture.py b/rag/app/picture.py index 9b1c27a4e..348df9abc 100644 --- a/rag/app/picture.py +++ b/rag/app/picture.py @@ -61,7 +61,7 @@ def chunk(filename, binary, tenant_id, lang, callback=None, **kwargs): ans = asyncio.run(cv_mdl.async_chat(system="", history=[], gen_conf={}, video_bytes=binary, filename=filename, video_prompt=video_prompt)) callback(0.8, "CV LLM respond: %s ..." % ans[:32]) ans += "\n" + ans - tokenize(doc, ans, eng) + tokenize(doc, ans, eng, language=lang) return [doc] except Exception as e: callback(prog=-1, msg=str(e)) @@ -84,7 +84,7 @@ def chunk(filename, binary, tenant_id, lang, callback=None, **kwargs): callback(0.4, "Finish OCR: (%s ...)" % txt[:12]) if (eng and len(txt.split()) > 32) or len(txt) > 32: - tokenize(doc, txt, eng) + tokenize(doc, txt, eng, language=lang) callback(0.8, "OCR results is too long to use CV LLM.") return attach_media_context([doc], 0, image_ctx) @@ -98,7 +98,7 @@ def chunk(filename, binary, tenant_id, lang, callback=None, **kwargs): ans = cv_mdl.describe(img_binary.read()) callback(0.8, "CV LLM respond: %s ..." % ans[:32]) txt += "\n" + ans - tokenize(doc, txt, eng) + tokenize(doc, txt, eng, language=lang) return attach_media_context([doc], 0, image_ctx) except Exception as e: callback(prog=-1, msg=str(e)) diff --git a/rag/app/presentation.py b/rag/app/presentation.py index c5b5025fb..b2875ee16 100644 --- a/rag/app/presentation.py +++ b/rag/app/presentation.py @@ -147,7 +147,7 @@ def chunk(filename, binary=None, from_page=0, to_page=MAXIMUM_PAGE_NUMBER, lang= d["page_num_int"] = [pn + 1] d["top_int"] = [0] d["position_int"] = [(pn + 1, 0, 0, 0, 0)] - tokenize(d, txt, eng) + tokenize(d, txt, eng, language=lang) res.append(d) return res except Exception as e: @@ -182,7 +182,7 @@ def chunk(filename, binary=None, from_page=0, to_page=MAXIMUM_PAGE_NUMBER, lang= d["page_num_int"] = [pn + 1] d["top_int"] = [0] d["position_int"] = [(pn + 1, 0, 0, 0, 0)] - tokenize(d, txt, eng) + tokenize(d, txt, eng, language=lang) res.append(d) if callback: @@ -237,7 +237,7 @@ def chunk(filename, binary=None, from_page=0, to_page=MAXIMUM_PAGE_NUMBER, lang= d["page_num_int"] = [pn + 1] d["top_int"] = [0] d["position_int"] = [(pn + 1, 0, img.size[0] if img else 0, 0, img.size[1] if img else 0)] - tokenize(d, txt, eng) + tokenize(d, txt, eng, language=lang) res.append(d) return res diff --git a/rag/app/qa.py b/rag/app/qa.py index 293bd5328..f663d7142 100644 --- a/rag/app/qa.py +++ b/rag/app/qa.py @@ -297,6 +297,7 @@ def chunk(filename, binary=None, from_page=0, to_page=MAXIMUM_PAGE_NUMBER, lang= Every pair of Q&A will be treated as a chunk. """ eng = lang.lower() == "english" + rag_tokenizer.tokenizer.set_language(lang) res = [] doc = {"docnm_kwd": filename, "title_tks": rag_tokenizer.tokenize(re.sub(r"\.[a-zA-Z]+$", "", filename))} if re.search(r"\.xlsx?$", filename, re.IGNORECASE): @@ -418,8 +419,9 @@ def chunk(filename, binary=None, from_page=0, to_page=MAXIMUM_PAGE_NUMBER, lang= elif re.search(r"\.docx$", filename, re.IGNORECASE): docx_parser = Docx() - qai_list, tbls = docx_parser(filename, binary, from_page=0, to_page=MAXIMUM_PAGE_NUMBER, callback=callback) - res = tokenize_table(tbls, doc, eng) + qai_list, tbls = docx_parser(filename, binary, + from_page=0, to_page=MAXIMUM_PAGE_NUMBER, callback=callback) + res = tokenize_table(tbls, doc, eng, language=lang) for i, (q, a, image) in enumerate(qai_list): res.append(beAdocDocx(deepcopy(doc), q, a, eng, image, i)) return res diff --git a/rag/app/resume.py b/rag/app/resume.py index e7aed73db..8475e172c 100644 --- a/rag/app/resume.py +++ b/rag/app/resume.py @@ -2504,6 +2504,7 @@ def chunk(filename, binary, tenant_id, from_page=0, to_page=MAXIMUM_PAGE_NUMBER, try: callback(0.1, "Starting resume parsing...") + rag_tokenizer.tokenizer.set_language(lang) # Parse resume resume, lines, line_positions = parse_resume(filename, binary, tenant_id, lang) diff --git a/rag/app/table.py b/rag/app/table.py index fd41e5e27..5e21d1e5e 100644 --- a/rag/app/table.py +++ b/rag/app/table.py @@ -548,7 +548,7 @@ def chunk(filename, binary=None, from_page=0, to_page=MAXIMUM_TASK_PAGE_NUMBER, else: d.update(stored) formatted_text = "\n".join([f"- {field}: {value}" for field, value in text_fields]) if text_fields else "" - tokenize(d, formatted_text, eng) + tokenize(d, formatted_text, eng, language=lang) if _debug_row_idx == 1: logger.debug(f"[TABLE_PARSER_DEBUG] Chunk content_with_weight length: {len(d.get('content_with_weight', '') or '')}") _cd = d.get("chunk_data") @@ -559,7 +559,7 @@ def chunk(filename, binary=None, from_page=0, to_page=MAXIMUM_TASK_PAGE_NUMBER, res.append(d) if tbls: doc = {"docnm_kwd": filename, "title_tks": rag_tokenizer.tokenize(re.sub(r"\.[a-zA-Z]+$", "", filename))} - res.extend(tokenize_table(tbls, doc, is_english)) + res.extend(tokenize_table(tbls, doc, is_english, language=lang)) callback(0.35, "") return res diff --git a/rag/app/tag.py b/rag/app/tag.py index 43d50b8ab..2c3a27785 100644 --- a/rag/app/tag.py +++ b/rag/app/tag.py @@ -47,6 +47,7 @@ def chunk(filename, binary=None, lang="Chinese", callback=None, **kwargs): Every pair will be treated as a chunk. """ eng = lang.lower() == "english" + rag_tokenizer.tokenizer.set_language(lang) res = [] doc = {"docnm_kwd": filename, "title_tks": rag_tokenizer.tokenize(re.sub(r"\.[a-zA-Z]+$", "", filename))} if re.search(r"\.xlsx?$", filename, re.IGNORECASE): diff --git a/rag/nlp/__init__.py b/rag/nlp/__init__.py index eb18d3e5c..d6b40e2e6 100644 --- a/rag/nlp/__init__.py +++ b/rag/nlp/__init__.py @@ -352,16 +352,16 @@ def is_chinese(text): return False -def tokenize(d, txt, eng): +def tokenize(d, txt, eng, language="English"): from . import rag_tokenizer - + rag_tokenizer.tokenizer.set_language(language) d["content_with_weight"] = txt t = re.sub(r"]{0,12})?>", " ", txt) d["content_ltks"] = rag_tokenizer.tokenize(t) d["content_sm_ltks"] = rag_tokenizer.fine_grained_tokenize(d["content_ltks"]) -def split_with_pattern(d, pattern: str, content: str, eng) -> list: +def split_with_pattern(d, pattern: str, content: str, eng, language="English") -> list: docs = [] # Validate and compile regex pattern before use @@ -371,7 +371,7 @@ def split_with_pattern(d, pattern: str, content: str, eng) -> list: logging.warning(f"Invalid delimiter regex pattern '{pattern}': {e}. Falling back to no split.") # Fallback: return content as single chunk dd = copy.deepcopy(d) - tokenize(dd, content, eng) + tokenize(dd, content, eng, language=language) return [dd] txts = [txt for txt in compiled_pattern.split(content)] @@ -382,12 +382,12 @@ def split_with_pattern(d, pattern: str, content: str, eng) -> list: if j + 1 < len(txts): txt += txts[j + 1] dd = copy.deepcopy(d) - tokenize(dd, txt, eng) + tokenize(dd, txt, eng, language=language) docs.append(dd) return docs -def tokenize_chunks(chunks, doc, eng, pdf_parser=None, child_delimiters_pattern=None): +def tokenize_chunks(chunks, doc, eng, pdf_parser=None, child_delimiters_pattern=None, language="English"): res = [] # wrap up as es documents for ii, ck in enumerate(chunks): @@ -407,15 +407,15 @@ def tokenize_chunks(chunks, doc, eng, pdf_parser=None, child_delimiters_pattern= if child_delimiters_pattern: d["mom_with_weight"] = ck - res.extend(split_with_pattern(d, child_delimiters_pattern, ck, eng)) + res.extend(split_with_pattern(d, child_delimiters_pattern, ck, eng, language=language)) continue - tokenize(d, ck, eng) + tokenize(d, ck, eng, language=language) res.append(d) return res -def doc_tokenize_chunks_with_images(chunks, doc, eng, child_delimiters_pattern=None, batch_size=10): +def doc_tokenize_chunks_with_images(chunks, doc, eng, child_delimiters_pattern=None, batch_size=10, language="English"): res = [] for ii, ck in enumerate(chunks): text = ck.get("context_above", "") + ck.get("text") + ck.get("context_below", "") @@ -430,18 +430,18 @@ def doc_tokenize_chunks_with_images(chunks, doc, eng, child_delimiters_pattern=N if ck.get("ck_type") == "text": if child_delimiters_pattern: d["mom_with_weight"] = text - res.extend(split_with_pattern(d, child_delimiters_pattern, text, eng)) + res.extend(split_with_pattern(d, child_delimiters_pattern, text, eng, language=language)) continue elif ck.get("ck_type") == "image": d["doc_type_kwd"] = "image" elif ck.get("ck_type") == "table": d["doc_type_kwd"] = "table" - tokenize(d, text, eng) + tokenize(d, text, eng, language=language) res.append(d) return res -def tokenize_chunks_with_images(chunks, doc, eng, images, child_delimiters_pattern=None): +def tokenize_chunks_with_images(chunks, doc, eng, images, child_delimiters_pattern=None, language="English"): res = [] # wrap up as es documents for ii, (ck, image) in enumerate(zip(chunks, images)): @@ -453,14 +453,14 @@ def tokenize_chunks_with_images(chunks, doc, eng, images, child_delimiters_patte add_positions(d, [[ii] * 5]) if child_delimiters_pattern: d["mom_with_weight"] = ck - res.extend(split_with_pattern(d, child_delimiters_pattern, ck, eng)) + res.extend(split_with_pattern(d, child_delimiters_pattern, ck, eng, language=language)) continue - tokenize(d, ck, eng) + tokenize(d, ck, eng, language=language) res.append(d) return res -def tokenize_table(tbls, doc, eng, batch_size=10): +def tokenize_table(tbls, doc, eng, batch_size=10, language="English"): res = [] # add tables for (img, rows), poss in tbls: @@ -468,7 +468,7 @@ def tokenize_table(tbls, doc, eng, batch_size=10): continue if isinstance(rows, str): d = copy.deepcopy(doc) - tokenize(d, rows, eng) + tokenize(d, rows, eng, language=language) d["content_with_weight"] = rows d["doc_type_kwd"] = "table" if img: @@ -479,11 +479,12 @@ def tokenize_table(tbls, doc, eng, batch_size=10): add_positions(d, poss) res.append(d) continue - de = "; " if eng else "； " + lang_key = (language or "English").strip().lower() + de = "； " if lang_key in {"chinese", "japanese"} else "; " for i in range(0, len(rows), batch_size): d = copy.deepcopy(doc) - r = de.join(rows[i : i + batch_size]) - tokenize(d, r, eng) + r = de.join(rows[i:i + batch_size]) + tokenize(d, r, eng, language=language) d["doc_type_kwd"] = "table" if img: d["image"] = img diff --git a/rag/svr/task_executor.py b/rag/svr/task_executor.py index a7e113b3f..2e85ef7e8 100644 --- a/rag/svr/task_executor.py +++ b/rag/svr/task_executor.py @@ -419,6 +419,8 @@ async def build_chunks(task, progress_callback): # Record docs after MinIO upload get_recording_context().record("docs_after_prep", docs) + rag_tokenizer.tokenizer.set_language(task["language"]) + if task["parser_config"].get("auto_keywords", 0): st = timer() progress_callback(msg="Start to generate keywords for every chunk ...") @@ -759,6 +761,7 @@ async def run_dataflow(task: dict): dsl = pipeline_log.dsl dataflow_id = pipeline_log.pipeline_id pipeline = Pipeline(dsl, tenant_id=task["tenant_id"], doc_id=doc_id, task_id=task_id, flow_id=dataflow_id) + rag_tokenizer.tokenizer.set_language(task.get("language", "English")) chunks = await pipeline.run(file=task["file"]) if task.get("file") else await pipeline.run() if doc_id == CANVAS_DEBUG_DOC_ID: get_recording_context().record("dataflow_debug_result", "canvas_debug_mode") @@ -1023,6 +1026,8 @@ async def run_raptor_for_kb(row, kb_parser_config, chat_mdl, embd_mdl, vector_si """Generate RAPTOR summaries for selected documents in a knowledge base.""" fake_doc_id = GRAPH_RAPTOR_FAKE_DOC_ID + rag_tokenizer.tokenizer.set_language(row.get("language", "English")) + raptor_config = kb_parser_config.get("raptor", {}) raptor_ext_config = raptor_config.get("ext") or {} tree_builder = get_raptor_tree_builder(raptor_config) @@ -1385,6 +1390,7 @@ async def do_handle_task(task): task_language = task.get("language") or "Chinese" if not task.get("language"): logging.warning("Task %s has no language set, falling back to Chinese", task_id) + rag_tokenizer.tokenizer.set_language(task_language) doc_task_llm_id = task["parser_config"].get("llm_id") or task["llm_id"] kb_task_llm_id = task["kb_parser_config"].get("llm_id") or task["llm_id"] task["llm_id"] = kb_task_llm_id diff --git a/web/src/components/cross-language-form-field.tsx b/web/src/components/cross-language-form-field.tsx index ab14255b4..bfc8ce4e1 100644 --- a/web/src/components/cross-language-form-field.tsx +++ b/web/src/components/cross-language-form-field.tsx @@ -22,6 +22,7 @@ const Languages = [ 'Vietnamese', 'Arabic', 'Turkish', + 'Dutch', ]; export const crossLanguageOptions = Languages.map((x) => ({ diff --git a/web/src/constants/common.ts b/web/src/constants/common.ts index 9498f41d2..45a6b6477 100644 --- a/web/src/constants/common.ts +++ b/web/src/constants/common.ts @@ -58,6 +58,7 @@ export const LanguageList = [ 'Bulgarian', 'Arabic', 'Turkish', + 'Dutch', ]; export const LanguageMap = { English: 'English', @@ -76,6 +77,7 @@ export const LanguageMap = { Bulgarian: 'Български', Arabic: 'العربية', Turkish: 'Türkçe', + Dutch: 'Nederlands', }; export enum LanguageAbbreviation { @@ -95,6 +97,7 @@ export enum LanguageAbbreviation { Ar = 'ar', Tr = 'tr', Ko = 'ko', + Nl = 'nl', } export const LanguageAbbreviationMap = { @@ -114,6 +117,7 @@ export const LanguageAbbreviationMap = { [LanguageAbbreviation.Ar]: 'العربية', [LanguageAbbreviation.Tr]: 'Türkçe', [LanguageAbbreviation.Ko]: '한국어', + [LanguageAbbreviation.Nl]: 'Nederlands', }; export const LanguageTranslationMap = { @@ -143,6 +147,7 @@ export const LanguageTranslationMap = { Bulgarian: 'bg', Arabic: 'ar', Turkish: 'tr', + Dutch: 'nl', }; export enum FileMimeType { diff --git a/web/src/locales/en.ts b/web/src/locales/en.ts index 4a4b47acd..ce90818cc 100644 --- a/web/src/locales/en.ts +++ b/web/src/locales/en.ts @@ -40,6 +40,7 @@ export default { bulgarian: 'Bulgarian', arabic: 'Arabic', turkish: 'Turkish', + dutch: 'Dutch', language: 'Language', languageMessage: 'Please input your language!', languagePlaceholder: 'select your language', @@ -3064,6 +3065,7 @@ Important structured information may include: names, dates, locations, events, k bulgarian: 'Bulgarian', arabic: 'Arabic', turkish: 'Turkish', + dutch: 'Dutch', }, pagination: { total: 'Total {{total}}',