feat: add native Dutch language support for BM25 tokenization (#14140)

## Summary - Add language-aware Snowball stemmer to `RagTokenizer` supporting 16 languages (Dutch, German, French, Spanish, etc.) - Thread the KB `language` parameter through the full tokenization pipeline (14 parser modules + task executor) - Add Dutch to the frontend language lists and cross-language form ## Problem RAGFlow uses the English Porter stemmer + WordNet lemmatizer for **all** BM25 tokenization, regardless of the knowledge base language setting. This produces incorrect stems for non-English text. For example: | Dutch word | Dutch stemmer | English Porter | |---|---|---| | documenten | document | documenten (unchanged!) | | gebruikers | gebruiker | gebruik (over-stemmed) | | instellingen | instell | instellingen (unchanged!) | This degrades BM25 recall for any non-English knowledge base. ## Solution NLTK already ships Snowball stemmers for 16 languages. This PR: 1. **`rag/nlp/rag_tokenizer.py`**: Overrides `tokenize()` with `set_language()` and `_normalize_token()` that selects the correct NLTK Snowball stemmer. Falls back to Porter for unmapped languages (Chinese, Japanese, Korean, etc. — these use character-based tokenization anyway). 2. **`rag/nlp/__init__.py`** + **14 `rag/app/*.py` parsers** + **`rag/svr/task_executor.py`**: Threads the `language` parameter through `tokenize()`, `tokenize_chunks()`, `tokenize_table()`, and all callers. 3. **Frontend**: Adds Dutch (`Nederlands`) to `LanguageList`, `LanguageMap`, `LanguageAbbreviationMap`, `LanguageTranslationMap`, cross-language form field, and `en.ts` locale. ## Backward Compatibility - Default language is `"English"`, preserving existing behavior for all current users - Languages without a Snowball stemmer mapping fall back to Porter (no change) - No new dependencies — NLTK Snowball is already bundled
2026-07-07 12:00:44 +08:00 · 2026-07-06 17:39:56 +02:00
parent dd20561fca
commit d8cefcf052
20 changed files with 73 additions and 53 deletions
--- a/.gitignore
+++ b/.gitignore
@@ -22,6 +22,7 @@ Cargo.lock
 .idea/
 .vscode/
 .cursor/settings.json
+.opencode/

 # Exclude Mac generated files
 .DS_Store
--- a/rag/app/audio.py
+++ b/rag/app/audio.py
@@ -50,7 +50,7 @@ def chunk(filename, binary, tenant_id, lang, callback=None, **kwargs):
        ans = seq2txt_mdl.transcription(tmp_path)
        callback(0.8, "Sequence2Txt LLM respond: %s ..." % ans[:32])

-        tokenize(doc, ans, is_english)
+        tokenize(doc, ans, is_english, language=lang)
        return [doc]
    except Exception as e:
        callback(prog=-1, msg=str(e))
--- a/rag/app/book.py
+++ b/rag/app/book.py
@@ -169,8 +169,8 @@ def chunk(filename, binary=None, from_page=0, to_page=MAXIMUM_PAGE_NUMBER, lang=
    # is_english(random_choices([t for t, _ in sections], k=218))
    eng = lang.lower() == "english"

-    res = tokenize_table(tbls, doc, eng)
-    res.extend(tokenize_chunks(chunks, doc, eng, pdf_parser))
+    res = tokenize_table(tbls, doc, eng, language=lang)
+    res.extend(tokenize_chunks(chunks, doc, eng, pdf_parser, language=lang))
    table_ctx = max(0, int(parser_config.get("table_context_size", 0) or 0))
    image_ctx = max(0, int(parser_config.get("image_context_size", 0) or 0))
    if table_ctx or image_ctx:
--- a/rag/app/email.py
+++ b/rag/app/email.py
@@ -102,7 +102,7 @@ def chunk(
        parser_config.get("delimiter", "\n!?。；！？"),
    )

-    main_res.extend(tokenize_chunks(chunks, doc, eng, None))
+    main_res.extend(tokenize_chunks(chunks, doc, eng, None, language=lang))
    logging.debug("naive_merge({}): {}".format(filename, timer() - st))
    # get the attachment info
    for part in msg.iter_attachments():
--- a/rag/app/laws.py
+++ b/rag/app/laws.py
@@ -180,7 +180,7 @@ def chunk(filename, binary=None, from_page=0, to_page=MAXIMUM_PAGE_NUMBER, lang=
        callback(0.1, "Start to parse.")
        chunks = Docx()(filename, binary)
        callback(0.7, "Finish parsing.")
-        return tokenize_chunks(chunks, doc, eng, None)
+        return tokenize_chunks(chunks, doc, eng, None, language=lang)

    elif re.search(r"\.pdf$", filename, re.IGNORECASE):
        layout_recognizer, parser_model_name = normalize_layout_recognizer(parser_config.get("layout_recognize", "DeepDOC"))
@@ -261,7 +261,7 @@ def chunk(filename, binary=None, from_page=0, to_page=MAXIMUM_PAGE_NUMBER, lang=
    if not res:
        callback(0.99, "No chunk parsed out.")

-    return tokenize_chunks(res, doc, eng, pdf_parser)
+    return tokenize_chunks(res, doc, eng, pdf_parser, language=lang)

    # chunks = hierarchical_merge(bull, sections, 5)
    #     return tokenize_chunks(["\n".join(ck)for ck in chunks], doc, eng, pdf_parser)
--- a/rag/app/manual.py
+++ b/rag/app/manual.py
@@ -261,8 +261,8 @@ def chunk(filename, binary=None, from_page=0, to_page=MAXIMUM_PAGE_NUMBER, lang=
            callback=callback,
            **kwargs,
        )
-        res = tokenize_table(tbls, doc, eng)
-        res.extend(tokenize_chunks(chunks, doc, eng, pdf_parser))
+        res = tokenize_table(tbls, doc, eng, language=lang)
+        res.extend(tokenize_chunks(chunks, doc, eng, pdf_parser, language=lang))
        table_ctx = max(0, int(parser_config.get("table_context_size", 0) or 0))
        image_ctx = max(0, int(parser_config.get("image_context_size", 0) or 0))
        if table_ctx or image_ctx:
@@ -275,13 +275,13 @@ def chunk(filename, binary=None, from_page=0, to_page=MAXIMUM_PAGE_NUMBER, lang=
        docx_parser = Docx()
        ti_list, tbls = docx_parser(filename, binary, from_page=0, to_page=MAXIMUM_PAGE_NUMBER, callback=callback)
        tbls = vision_figure_parser_docx_wrapper(sections=ti_list, tbls=tbls, callback=callback, **kwargs)
-        res = tokenize_table(tbls, doc, eng)
+        res = tokenize_table(tbls, doc, eng, language=lang)
        for text, image in ti_list:
            d = copy.deepcopy(doc)
            if image:
                d["image"] = image
                d["doc_type_kwd"] = "image"
-            tokenize(d, text, eng)
+            tokenize(d, text, eng, language=lang)
            res.append(d)
        table_ctx = max(0, int(parser_config.get("table_context_size", 0) or 0))
        image_ctx = max(0, int(parser_config.get("image_context_size", 0) or 0))
--- a/rag/app/naive.py
+++ b/rag/app/naive.py
@@ -976,7 +976,7 @@ def chunk(filename, binary=None, from_page=0, to_page=MAXIMUM_PAGE_NUMBER, lang=
        callback(0.8, "Finish parsing.")
        st = timer()

-        res.extend(doc_tokenize_chunks_with_images(chunks, doc, is_english, child_delimiters_pattern=child_deli))
+        res.extend(doc_tokenize_chunks_with_images(chunks, doc, is_english, child_delimiters_pattern=child_deli, language=lang))
        logging.info("naive_merge({}): {}".format(filename, timer() - st))
        res.extend(embed_res)
        res.extend(url_res)
@@ -1024,7 +1024,7 @@ def chunk(filename, binary=None, from_page=0, to_page=MAXIMUM_PAGE_NUMBER, lang=
            if int(parser_config.get("chunk_token_num", 0)) <= 0:
                parser_config["chunk_token_num"] = 0

-        res = tokenize_table(tables, doc, is_english)
+        res = tokenize_table(tables, doc, is_english, language=lang)
        callback(0.8, "Finish parsing.")

    elif re.search(r"\.(csv|xlsx?)$", filename, re.IGNORECASE):
@@ -1046,7 +1046,7 @@ def chunk(filename, binary=None, from_page=0, to_page=MAXIMUM_PAGE_NUMBER, lang=
            sections, tables = tcadp_parser.parse_pdf(filepath=filename, binary=binary, callback=callback, output_dir=os.environ.get("TCADP_OUTPUT_DIR", ""), file_type=file_type)
            sections = _normalize_section_text_for_rtl_presentation_forms(sections)
            parser_config["chunk_token_num"] = 0
-            res = tokenize_table(tables, doc, is_english)
+            res = tokenize_table(tables, doc, is_english, language=lang)
            callback(0.8, "Finish parsing.")
        else:
            # Default DeepDOC parser
@@ -1116,7 +1116,7 @@ def chunk(filename, binary=None, from_page=0, to_page=MAXIMUM_PAGE_NUMBER, lang=
                soup = markdown_parser.md_to_html(section_text)
                hyperlink_urls = markdown_parser.get_hyperlink_urls(soup)
                urls.update(hyperlink_urls)
-        res = tokenize_table(tables, doc, is_english)
+        res = tokenize_table(tables, doc, is_english, language=lang)
        callback(0.8, "Finish parsing.")

    elif re.search(r"\.(htm|html)$", filename, re.IGNORECASE):
@@ -1215,9 +1215,9 @@ def chunk(filename, binary=None, from_page=0, to_page=MAXIMUM_PAGE_NUMBER, lang=
        has_images = merged_images and any(img is not None for img in merged_images)

        if has_images:
-            res.extend(tokenize_chunks_with_images(chunks, doc, is_english, merged_images, child_delimiters_pattern=child_deli))
+            res.extend(tokenize_chunks_with_images(chunks, doc, is_english, merged_images, child_delimiters_pattern=child_deli, language=lang))
        else:
-            res.extend(tokenize_chunks(chunks, doc, is_english, pdf_parser, child_delimiters_pattern=child_deli))
+            res.extend(tokenize_chunks(chunks, doc, is_english, pdf_parser, child_delimiters_pattern=child_deli, language=lang))
    else:
        if section_images:
            if all(image is None for image in section_images):
@@ -1225,11 +1225,11 @@ def chunk(filename, binary=None, from_page=0, to_page=MAXIMUM_PAGE_NUMBER, lang=

        if section_images:
            chunks, images = naive_merge_with_images(sections, section_images, int(parser_config.get("chunk_token_num", 128)), parser_config.get("delimiter", "\n!?。；！？"), overlapped_percent)
-            res.extend(tokenize_chunks_with_images(chunks, doc, is_english, images, child_delimiters_pattern=child_deli))
+            res.extend(tokenize_chunks_with_images(chunks, doc, is_english, images, child_delimiters_pattern=child_deli, language=lang))
        else:
            chunks = naive_merge(sections, int(parser_config.get("chunk_token_num", 128)), parser_config.get("delimiter", "\n!?。；！？"), overlapped_percent)

-            res.extend(tokenize_chunks(chunks, doc, is_english, pdf_parser, child_delimiters_pattern=child_deli))
+            res.extend(tokenize_chunks(chunks, doc, is_english, pdf_parser, child_delimiters_pattern=child_deli, language=lang))

    if urls and parser_config.get("analyze_hyperlink", False) and is_root:
        for index, url in enumerate(urls):
--- a/rag/app/one.py
+++ b/rag/app/one.py
@@ -163,7 +163,7 @@ def chunk(filename, binary=None, from_page=0, to_page=MAXIMUM_PAGE_NUMBER, lang=

    doc = {"docnm_kwd": filename, "title_tks": rag_tokenizer.tokenize(re.sub(r"\.[a-zA-Z]+$", "", filename))}
    doc["title_sm_tks"] = rag_tokenizer.fine_grained_tokenize(doc["title_tks"])
-    tokenize(doc, "\n".join(sections), eng)
+    tokenize(doc, "\n".join(sections), eng, language=lang)
    return [doc]


--- a/rag/app/paper.py
+++ b/rag/app/paper.py
@@ -188,7 +188,7 @@ def chunk(filename, binary=None, from_page=0, to_page=MAXIMUM_PAGE_NUMBER, lang=
    eng = lang.lower() == "english"  # pdf_parser.is_english
    logging.debug("It's English.....{}".format(eng))

-    res = tokenize_table(paper["tables"], doc, eng)
+    res = tokenize_table(paper["tables"], doc, eng, language=lang)

    if paper["abstract"]:
        d = copy.deepcopy(doc)
@@ -197,7 +197,7 @@ def chunk(filename, binary=None, from_page=0, to_page=MAXIMUM_PAGE_NUMBER, lang=
        d["important_tks"] = " ".join(d["important_kwd"])
        d["image"], poss = pdf_parser.crop(paper["abstract"], need_position=True)
        add_positions(d, poss)
-        tokenize(d, txt, eng)
+        tokenize(d, txt, eng, language=lang)
        res.append(d)

    sorted_sections = paper["sections"]
@@ -223,7 +223,7 @@ def chunk(filename, binary=None, from_page=0, to_page=MAXIMUM_PAGE_NUMBER, lang=
                continue
        chunks.append(txt)
        last_sid = sec_id
-    res.extend(tokenize_chunks(chunks, doc, eng, pdf_parser))
+    res.extend(tokenize_chunks(chunks, doc, eng, pdf_parser, language=lang))
    table_ctx = max(0, int(parser_config.get("table_context_size", 0) or 0))
    image_ctx = max(0, int(parser_config.get("image_context_size", 0) or 0))
    if table_ctx or image_ctx:
@@ -264,7 +264,7 @@ def chunk(filename, binary=None, from_page=0, to_page=MAXIMUM_PAGE_NUMBER, lang=
            txt += "\n" + pdf_parser.remove_tag(p)
            d["image"], poss = pdf_parser.crop(p, need_position=True)
            add_positions(d, poss)
-            tokenize(d, txt, eng)
+            tokenize(d, txt, eng, language=lang)
            res.append(d)

    i = 0
@@ -274,7 +274,7 @@ def chunk(filename, binary=None, from_page=0, to_page=MAXIMUM_PAGE_NUMBER, lang=
        nonlocal chunk, res, doc, pdf_parser, tk_cnt
        d = copy.deepcopy(doc)
        ck = "\n".join(chunk)
-        tokenize(d, pdf_parser.remove_tag(ck), pdf_parser.is_english)
+        tokenize(d, pdf_parser.remove_tag(ck), pdf_parser.is_english, language=lang)
        d["image"], poss = pdf_parser.crop(ck, need_position=True)
        add_positions(d, poss)
        res.append(d)
--- a/rag/app/picture.py
+++ b/rag/app/picture.py
@@ -61,7 +61,7 @@ def chunk(filename, binary, tenant_id, lang, callback=None, **kwargs):
            ans = asyncio.run(cv_mdl.async_chat(system="", history=[], gen_conf={}, video_bytes=binary, filename=filename, video_prompt=video_prompt))
            callback(0.8, "CV LLM respond: %s ..." % ans[:32])
            ans += "\n" + ans
-            tokenize(doc, ans, eng)
+            tokenize(doc, ans, eng, language=lang)
            return [doc]
        except Exception as e:
            callback(prog=-1, msg=str(e))
@@ -84,7 +84,7 @@ def chunk(filename, binary, tenant_id, lang, callback=None, **kwargs):

        callback(0.4, "Finish OCR: (%s ...)" % txt[:12])
        if (eng and len(txt.split()) > 32) or len(txt) > 32:
-            tokenize(doc, txt, eng)
+            tokenize(doc, txt, eng, language=lang)
            callback(0.8, "OCR results is too long to use CV LLM.")
            return attach_media_context([doc], 0, image_ctx)

@@ -98,7 +98,7 @@ def chunk(filename, binary, tenant_id, lang, callback=None, **kwargs):
                ans = cv_mdl.describe(img_binary.read())
            callback(0.8, "CV LLM respond: %s ..." % ans[:32])
            txt += "\n" + ans
-            tokenize(doc, txt, eng)
+            tokenize(doc, txt, eng, language=lang)
            return attach_media_context([doc], 0, image_ctx)
        except Exception as e:
            callback(prog=-1, msg=str(e))
--- a/rag/app/presentation.py
+++ b/rag/app/presentation.py
@@ -147,7 +147,7 @@ def chunk(filename, binary=None, from_page=0, to_page=MAXIMUM_PAGE_NUMBER, lang=
                d["page_num_int"] = [pn + 1]
                d["top_int"] = [0]
                d["position_int"] = [(pn + 1, 0, 0, 0, 0)]
-                tokenize(d, txt, eng)
+                tokenize(d, txt, eng, language=lang)
                res.append(d)
            return res
        except Exception as e:
@@ -182,7 +182,7 @@ def chunk(filename, binary=None, from_page=0, to_page=MAXIMUM_PAGE_NUMBER, lang=
                    d["page_num_int"] = [pn + 1]
                    d["top_int"] = [0]
                    d["position_int"] = [(pn + 1, 0, 0, 0, 0)]
-                    tokenize(d, txt, eng)
+                    tokenize(d, txt, eng, language=lang)
                    res.append(d)

                if callback:
@@ -237,7 +237,7 @@ def chunk(filename, binary=None, from_page=0, to_page=MAXIMUM_PAGE_NUMBER, lang=
            d["page_num_int"] = [pn + 1]
            d["top_int"] = [0]
            d["position_int"] = [(pn + 1, 0, img.size[0] if img else 0, 0, img.size[1] if img else 0)]
-            tokenize(d, txt, eng)
+            tokenize(d, txt, eng, language=lang)
            res.append(d)
        return res

--- a/rag/app/qa.py
+++ b/rag/app/qa.py
@@ -297,6 +297,7 @@ def chunk(filename, binary=None, from_page=0, to_page=MAXIMUM_PAGE_NUMBER, lang=
    Every pair of Q&A will be treated as a chunk.
    """
    eng = lang.lower() == "english"
+    rag_tokenizer.tokenizer.set_language(lang)
    res = []
    doc = {"docnm_kwd": filename, "title_tks": rag_tokenizer.tokenize(re.sub(r"\.[a-zA-Z]+$", "", filename))}
    if re.search(r"\.xlsx?$", filename, re.IGNORECASE):
@@ -418,8 +419,9 @@ def chunk(filename, binary=None, from_page=0, to_page=MAXIMUM_PAGE_NUMBER, lang=

    elif re.search(r"\.docx$", filename, re.IGNORECASE):
        docx_parser = Docx()
-        qai_list, tbls = docx_parser(filename, binary, from_page=0, to_page=MAXIMUM_PAGE_NUMBER, callback=callback)
-        res = tokenize_table(tbls, doc, eng)
+        qai_list, tbls = docx_parser(filename, binary,
+                                     from_page=0, to_page=MAXIMUM_PAGE_NUMBER, callback=callback)
+        res = tokenize_table(tbls, doc, eng, language=lang)
        for i, (q, a, image) in enumerate(qai_list):
            res.append(beAdocDocx(deepcopy(doc), q, a, eng, image, i))
        return res
--- a/rag/app/resume.py
+++ b/rag/app/resume.py
@@ -2504,6 +2504,7 @@ def chunk(filename, binary, tenant_id, from_page=0, to_page=MAXIMUM_PAGE_NUMBER,

    try:
        callback(0.1, "Starting resume parsing...")
+        rag_tokenizer.tokenizer.set_language(lang)

        # Parse resume
        resume, lines, line_positions = parse_resume(filename, binary, tenant_id, lang)
--- a/rag/app/table.py
+++ b/rag/app/table.py
@@ -548,7 +548,7 @@ def chunk(filename, binary=None, from_page=0, to_page=MAXIMUM_TASK_PAGE_NUMBER,
            else:
                d.update(stored)
            formatted_text = "\n".join([f"- {field}: {value}" for field, value in text_fields]) if text_fields else ""
-            tokenize(d, formatted_text, eng)
+            tokenize(d, formatted_text, eng, language=lang)
            if _debug_row_idx == 1:
                logger.debug(f"[TABLE_PARSER_DEBUG] Chunk content_with_weight length: {len(d.get('content_with_weight', '') or '')}")
                _cd = d.get("chunk_data")
@@ -559,7 +559,7 @@ def chunk(filename, binary=None, from_page=0, to_page=MAXIMUM_TASK_PAGE_NUMBER,
            res.append(d)
    if tbls:
        doc = {"docnm_kwd": filename, "title_tks": rag_tokenizer.tokenize(re.sub(r"\.[a-zA-Z]+$", "", filename))}
-        res.extend(tokenize_table(tbls, doc, is_english))
+        res.extend(tokenize_table(tbls, doc, is_english, language=lang))
    callback(0.35, "")

    return res
--- a/rag/app/tag.py
+++ b/rag/app/tag.py
@@ -47,6 +47,7 @@ def chunk(filename, binary=None, lang="Chinese", callback=None, **kwargs):
    Every pair will be treated as a chunk.
    """
    eng = lang.lower() == "english"
+    rag_tokenizer.tokenizer.set_language(lang)
    res = []
    doc = {"docnm_kwd": filename, "title_tks": rag_tokenizer.tokenize(re.sub(r"\.[a-zA-Z]+$", "", filename))}
    if re.search(r"\.xlsx?$", filename, re.IGNORECASE):
--- a/rag/nlp/init.py
+++ b/rag/nlp/init.py
@@ -352,16 +352,16 @@ def is_chinese(text):
    return False


-def tokenize(d, txt, eng):
+def tokenize(d, txt, eng, language="English"):
    from . import rag_tokenizer
-
+    rag_tokenizer.tokenizer.set_language(language)
    d["content_with_weight"] = txt
    t = re.sub(r"</?(table|td|caption|tr|th)( [^<>]{0,12})?>", " ", txt)
    d["content_ltks"] = rag_tokenizer.tokenize(t)
    d["content_sm_ltks"] = rag_tokenizer.fine_grained_tokenize(d["content_ltks"])


-def split_with_pattern(d, pattern: str, content: str, eng) -> list:
+def split_with_pattern(d, pattern: str, content: str, eng, language="English") -> list:
    docs = []

    # Validate and compile regex pattern before use
@@ -371,7 +371,7 @@ def split_with_pattern(d, pattern: str, content: str, eng) -> list:
        logging.warning(f"Invalid delimiter regex pattern '{pattern}': {e}. Falling back to no split.")
        # Fallback: return content as single chunk
        dd = copy.deepcopy(d)
-        tokenize(dd, content, eng)
+        tokenize(dd, content, eng, language=language)
        return [dd]

    txts = [txt for txt in compiled_pattern.split(content)]
@@ -382,12 +382,12 @@ def split_with_pattern(d, pattern: str, content: str, eng) -> list:
        if j + 1 < len(txts):
            txt += txts[j + 1]
        dd = copy.deepcopy(d)
-        tokenize(dd, txt, eng)
+        tokenize(dd, txt, eng, language=language)
        docs.append(dd)
    return docs


-def tokenize_chunks(chunks, doc, eng, pdf_parser=None, child_delimiters_pattern=None):
+def tokenize_chunks(chunks, doc, eng, pdf_parser=None, child_delimiters_pattern=None, language="English"):
    res = []
    # wrap up as es documents
    for ii, ck in enumerate(chunks):
@@ -407,15 +407,15 @@ def tokenize_chunks(chunks, doc, eng, pdf_parser=None, child_delimiters_pattern=

        if child_delimiters_pattern:
            d["mom_with_weight"] = ck
-            res.extend(split_with_pattern(d, child_delimiters_pattern, ck, eng))
+            res.extend(split_with_pattern(d, child_delimiters_pattern, ck, eng, language=language))
            continue

-        tokenize(d, ck, eng)
+        tokenize(d, ck, eng, language=language)
        res.append(d)
    return res


-def doc_tokenize_chunks_with_images(chunks, doc, eng, child_delimiters_pattern=None, batch_size=10):
+def doc_tokenize_chunks_with_images(chunks, doc, eng, child_delimiters_pattern=None, batch_size=10, language="English"):
    res = []
    for ii, ck in enumerate(chunks):
        text = ck.get("context_above", "") + ck.get("text") + ck.get("context_below", "")
@@ -430,18 +430,18 @@ def doc_tokenize_chunks_with_images(chunks, doc, eng, child_delimiters_pattern=N
        if ck.get("ck_type") == "text":
            if child_delimiters_pattern:
                d["mom_with_weight"] = text
-                res.extend(split_with_pattern(d, child_delimiters_pattern, text, eng))
+                res.extend(split_with_pattern(d, child_delimiters_pattern, text, eng, language=language))
                continue
        elif ck.get("ck_type") == "image":
            d["doc_type_kwd"] = "image"
        elif ck.get("ck_type") == "table":
            d["doc_type_kwd"] = "table"
-        tokenize(d, text, eng)
+        tokenize(d, text, eng, language=language)
        res.append(d)
    return res


-def tokenize_chunks_with_images(chunks, doc, eng, images, child_delimiters_pattern=None):
+def tokenize_chunks_with_images(chunks, doc, eng, images, child_delimiters_pattern=None, language="English"):
    res = []
    # wrap up as es documents
    for ii, (ck, image) in enumerate(zip(chunks, images)):
@@ -453,14 +453,14 @@ def tokenize_chunks_with_images(chunks, doc, eng, images, child_delimiters_patte
        add_positions(d, [[ii] * 5])
        if child_delimiters_pattern:
            d["mom_with_weight"] = ck
-            res.extend(split_with_pattern(d, child_delimiters_pattern, ck, eng))
+            res.extend(split_with_pattern(d, child_delimiters_pattern, ck, eng, language=language))
            continue
-        tokenize(d, ck, eng)
+        tokenize(d, ck, eng, language=language)
        res.append(d)
    return res


-def tokenize_table(tbls, doc, eng, batch_size=10):
+def tokenize_table(tbls, doc, eng, batch_size=10, language="English"):
    res = []
    # add tables
    for (img, rows), poss in tbls:
@@ -468,7 +468,7 @@ def tokenize_table(tbls, doc, eng, batch_size=10):
            continue
        if isinstance(rows, str):
            d = copy.deepcopy(doc)
-            tokenize(d, rows, eng)
+            tokenize(d, rows, eng, language=language)
            d["content_with_weight"] = rows
            d["doc_type_kwd"] = "table"
            if img:
@@ -479,11 +479,12 @@ def tokenize_table(tbls, doc, eng, batch_size=10):
                add_positions(d, poss)
            res.append(d)
            continue
-        de = "; " if eng else "； "
+        lang_key = (language or "English").strip().lower()
+        de = "； " if lang_key in {"chinese", "japanese"} else "; "
        for i in range(0, len(rows), batch_size):
            d = copy.deepcopy(doc)
-            r = de.join(rows[i : i + batch_size])
-            tokenize(d, r, eng)
+            r = de.join(rows[i:i + batch_size])
+            tokenize(d, r, eng, language=language)
            d["doc_type_kwd"] = "table"
            if img:
                d["image"] = img
--- a/rag/svr/task_executor.py
+++ b/rag/svr/task_executor.py
@@ -419,6 +419,8 @@ async def build_chunks(task, progress_callback):
    # Record docs after MinIO upload
    get_recording_context().record("docs_after_prep", docs)

+    rag_tokenizer.tokenizer.set_language(task["language"])
+
    if task["parser_config"].get("auto_keywords", 0):
        st = timer()
        progress_callback(msg="Start to generate keywords for every chunk ...")
@@ -759,6 +761,7 @@ async def run_dataflow(task: dict):
        dsl = pipeline_log.dsl
        dataflow_id = pipeline_log.pipeline_id
    pipeline = Pipeline(dsl, tenant_id=task["tenant_id"], doc_id=doc_id, task_id=task_id, flow_id=dataflow_id)
+    rag_tokenizer.tokenizer.set_language(task.get("language", "English"))
    chunks = await pipeline.run(file=task["file"]) if task.get("file") else await pipeline.run()
    if doc_id == CANVAS_DEBUG_DOC_ID:
        get_recording_context().record("dataflow_debug_result", "canvas_debug_mode")
@@ -1023,6 +1026,8 @@ async def run_raptor_for_kb(row, kb_parser_config, chat_mdl, embd_mdl, vector_si
    """Generate RAPTOR summaries for selected documents in a knowledge base."""
    fake_doc_id = GRAPH_RAPTOR_FAKE_DOC_ID

+    rag_tokenizer.tokenizer.set_language(row.get("language", "English"))
+
    raptor_config = kb_parser_config.get("raptor", {})
    raptor_ext_config = raptor_config.get("ext") or {}
    tree_builder = get_raptor_tree_builder(raptor_config)
@@ -1385,6 +1390,7 @@ async def do_handle_task(task):
    task_language = task.get("language") or "Chinese"
    if not task.get("language"):
        logging.warning("Task %s has no language set, falling back to Chinese", task_id)
+    rag_tokenizer.tokenizer.set_language(task_language)
    doc_task_llm_id = task["parser_config"].get("llm_id") or task["llm_id"]
    kb_task_llm_id = task["kb_parser_config"].get("llm_id") or task["llm_id"]
    task["llm_id"] = kb_task_llm_id
--- a/web/src/components/cross-language-form-field.tsx
+++ b/web/src/components/cross-language-form-field.tsx
@@ -22,6 +22,7 @@ const Languages = [
  'Vietnamese',
  'Arabic',
  'Turkish',
+  'Dutch',
 ];

 export const crossLanguageOptions = Languages.map((x) => ({
--- a/web/src/constants/common.ts
+++ b/web/src/constants/common.ts
@@ -58,6 +58,7 @@ export const LanguageList = [
  'Bulgarian',
  'Arabic',
  'Turkish',
+  'Dutch',
 ];
 export const LanguageMap = {
  English: 'English',
@@ -76,6 +77,7 @@ export const LanguageMap = {
  Bulgarian: 'Български',
  Arabic: 'العربية',
  Turkish: 'Türkçe',
+  Dutch: 'Nederlands',
 };

 export enum LanguageAbbreviation {
@@ -95,6 +97,7 @@ export enum LanguageAbbreviation {
  Ar = 'ar',
  Tr = 'tr',
  Ko = 'ko',
+  Nl = 'nl',
 }

 export const LanguageAbbreviationMap = {
@@ -114,6 +117,7 @@ export const LanguageAbbreviationMap = {
  [LanguageAbbreviation.Ar]: 'العربية',
  [LanguageAbbreviation.Tr]: 'Türkçe',
  [LanguageAbbreviation.Ko]: '한국어',
+  [LanguageAbbreviation.Nl]: 'Nederlands',
 };

 export const LanguageTranslationMap = {
@@ -143,6 +147,7 @@ export const LanguageTranslationMap = {
  Bulgarian: 'bg',
  Arabic: 'ar',
  Turkish: 'tr',
+  Dutch: 'nl',
 };

 export enum FileMimeType {
--- a/web/src/locales/en.ts
+++ b/web/src/locales/en.ts
@@ -40,6 +40,7 @@ export default {
      bulgarian: 'Bulgarian',
      arabic: 'Arabic',
      turkish: 'Turkish',
+      dutch: 'Dutch',
      language: 'Language',
      languageMessage: 'Please input your language!',
      languagePlaceholder: 'select your language',
@@ -3064,6 +3065,7 @@ Important structured information may include: names, dates, locations, events, k
      bulgarian: 'Bulgarian',
      arabic: 'Arabic',
      turkish: 'Turkish',
+      dutch: 'Dutch',
    },
    pagination: {
      total: 'Total {{total}}',