mirror of
https://github.com/infiniflow/ragflow.git
synced 2026-07-09 21:04:49 +08:00
Fix naive chunking for windows (#16767)
`\r` is ignored for splitting boundaries
This commit is contained in:
@@ -1162,6 +1162,8 @@ def naive_merge(sections: str | list, chunk_token_num=128, delimiter="\n。;
|
||||
sections = [sections]
|
||||
if isinstance(sections[0], str):
|
||||
sections = [(s, "") for s in sections]
|
||||
# Normalize line endings so delimiter ``\n`` matches ``\r\n`` and standalone ``\r``.
|
||||
sections = [(s.replace("\r\n", "\n").replace("\r", "\n"), pos) for s, pos in sections]
|
||||
cks = [""]
|
||||
tk_nums = [0]
|
||||
|
||||
|
||||
Reference in New Issue
Block a user