From b9ed0773ecf5c77475f3ddc85a2a468d435c4eee Mon Sep 17 00:00:00 2001
From: Yingfeng <yingfeng.zhang@gmail.com>
Date: Thu, 9 Jul 2026 12:02:19 +0800
Subject: [PATCH] Fix naive chunking for windows (#16767)

`\r` is ignored for splitting boundaries
---
 rag/nlp/__init__.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/rag/nlp/__init__.py b/rag/nlp/__init__.py
index d6b40e2e6..fa4f5a5fa 100644
--- a/rag/nlp/__init__.py
+++ b/rag/nlp/__init__.py
@@ -1162,6 +1162,8 @@ def naive_merge(sections: str | list, chunk_token_num=128, delimiter="\n。；
         sections = [sections]
     if isinstance(sections[0], str):
         sections = [(s, "") for s in sections]
+    # Normalize line endings so delimiter ``\n`` matches ``\r\n`` and standalone ``\r``.
+    sections = [(s.replace("\r\n", "\n").replace("\r", "\n"), pos) for s, pos in sections]
     cks = [""]
     tk_nums = [0]