fix: align pipeline delimiter chunking (#17723)

2026-08-05 07:10:29 +08:00 · 2026-08-03 17:47:08 +08:00
parent 3e7cfbe052
commit 9ccb23e661
5 changed files with 62 additions and 219 deletions
--- a/rag/flow/parser/parser.py
+++ b/rag/flow/parser/parser.py
@@ -1131,6 +1131,7 @@ class Parser(ProcessBase):
            blob,
            conf.get("chunk_token_num", 128),
            conf.get("delimiter", "\n!?;。；！？"),
+            keep_delimiters=True,
        )
        if conf.get("output_format") == "json":
            self.set_output("json", [{"text": section[0], "doc_type_kwd": "text"} for section in sections if section[0]])