diff --git a/deepdoc/parser/txt_parser.py b/deepdoc/parser/txt_parser.py index 9e51ffc961..f6414bfc97 100644 --- a/deepdoc/parser/txt_parser.py +++ b/deepdoc/parser/txt_parser.py @@ -19,7 +19,6 @@ import re from common.token_utils import num_tokens_from_string from deepdoc.parser.utils import get_text -from rag.nlp import _split_oversized_unit from rag.nlp.delim import ( compile_delimiter_pattern, normalize_text_newlines, @@ -28,12 +27,12 @@ from rag.nlp.delim import ( class RAGFlowTxtParser: - def __call__(self, fnm, binary=None, chunk_token_num=128, delimiter="\n!?;。；！？"): + def __call__(self, fnm, binary=None, chunk_token_num=128, delimiter="\n!?;。；！？", keep_delimiters=False): txt = get_text(fnm, binary) - return self.parser_txt(txt, chunk_token_num, delimiter) + return self.parser_txt(txt, chunk_token_num, delimiter, keep_delimiters) @classmethod - def parser_txt(cls, txt, chunk_token_num=128, delimiter="\n!?;。；！？"): + def parser_txt(cls, txt, chunk_token_num=128, delimiter="\n!?;。；！？", keep_delimiters=False): if not isinstance(txt, str): raise TypeError("txt type should be str!") cks = [""] @@ -42,21 +41,15 @@ class RAGFlowTxtParser: def add_chunk(t): nonlocal cks, tk_nums tnum = num_tokens_from_string(t) - - if cks[-1] == "": - cks[-1] = t - tk_nums[-1] = tnum - return - - merged = cks[-1] + "\n" + t - merged_tnum = num_tokens_from_string(merged) - if merged_tnum <= chunk_token_num: - cks[-1] = merged - tk_nums[-1] = merged_tnum - return - - cks.append(t) - tk_nums.append(tnum) + if tk_nums[-1] > chunk_token_num: + cks.append(t) + tk_nums.append(tnum) + else: + if cks[-1]: + cks[-1] += "\n" + t + else: + cks[-1] += t + tk_nums[-1] += tnum txt = normalize_text_newlines(txt) parsed_dels = parse_delimiter_field(delimiter) @@ -67,18 +60,14 @@ class RAGFlowTxtParser: bool(dels), ) secs = re.split(r"(%s)" % dels, txt) if dels else [txt] - for sec in secs: + for index, sec in enumerate(secs): if dels and re.match(f"^{dels}$", sec): continue if not sec: continue - if num_tokens_from_string(sec) <= chunk_token_num: - add_chunk(sec) - continue - pieces = _split_oversized_unit(sec, chunk_token_num, token_count_fn=num_tokens_from_string) - logging.debug("parser_txt: split oversized section (%d tokens) into %d pieces", num_tokens_from_string(sec), len(pieces)) - for piece in pieces: - add_chunk(piece) + if keep_delimiters and index + 1 < len(secs) and re.match(f"^{dels}$", secs[index + 1]): + sec += secs[index + 1] + add_chunk(sec) logging.debug("parser_txt: %d sections -> %d chunks (chunk_token_num=%d)", len(secs), len(cks), chunk_token_num) return [[c, ""] for c in cks] diff --git a/rag/flow/chunker/token_chunker.py b/rag/flow/chunker/token_chunker.py index b7a75c5c3a..ab9063f43c 100644 --- a/rag/flow/chunker/token_chunker.py +++ b/rag/flow/chunker/token_chunker.py @@ -77,7 +77,7 @@ def _compile_delimiter_pattern(delimiters): def _split_text_by_pattern(text, pattern): - # Split text by the compiled delimiter pattern and keep delimiter text in each chunk. + # Split text by the compiled delimiter pattern and discard delimiters. if not pattern: return [text or ""] @@ -85,10 +85,6 @@ def _split_text_by_pattern(text, pattern): chunks = [] for i in range(0, len(split_texts), 2): chunk = split_texts[i] - if not chunk: - continue - if i + 1 < len(split_texts): - chunk += split_texts[i + 1] if chunk.strip(): chunks.append(chunk) return chunks @@ -317,16 +313,17 @@ class TokenChunker(ProcessBase): self.set_output("chunks", [{"text": payload}] if payload.strip() else []) self.callback(1, "Done.") return - cks = ( - _split_text_by_pattern(payload, delimiter_pattern) - if delimiter_pattern - else naive_merge( + if self._param.delimiter_mode == "delimiter": + cks = _split_text_by_pattern(payload, delimiter_pattern) + elif delimiter_pattern: + cks = _split_text_by_pattern(payload, delimiter_pattern) + else: + cks = naive_merge( payload, self._param.chunk_token_size, "".join(self._param.delimiters), overlapped_percent, ) - ) if custom_pattern: docs = [] for c in cks: @@ -357,11 +354,47 @@ class TokenChunker(ProcessBase): self.set_output("chunks", [{"text": merged_text}] if merged_text.strip() else []) self.callback(1, "Done.") return + + if self._param.delimiter_mode == "delimiter": + text_chunks = _build_json_chunks(json_result, "") + chunks = [] + text_buffer = [] + + def flush_text_buffer(): + if not text_buffer: + return + combined_text = "".join(text_buffer) + split_texts = _split_text_by_pattern(combined_text, delimiter_pattern) + chunks.extend( + { + "text": text, + "doc_type_kwd": "text", + "ck_type": "text", + "tk_nums": num_tokens_from_string(text), + } + for text in split_texts + if text.strip() + ) + text_buffer.clear() + + for chunk in text_chunks: + if chunk["ck_type"] == "text": + text_buffer.append(chunk["text"]) + else: + flush_text_buffer() + chunks.append(chunk) + flush_text_buffer() + _attach_context_to_media_chunks(chunks, self._param.table_context_size, self._param.image_context_size) + await restore_pdf_text_previews(chunks, from_upstream, self._canvas) + self.set_output("chunks", _finalize_json_chunks(chunks)) + self.callback(1, "Done.") + return + # Structured JSON input is normalized first, then optionally enriched with # media context, and finally merged only when delimiter splitting is inactive. chunks = _build_json_chunks(json_result, delimiter_pattern) _attach_context_to_media_chunks(chunks, self._param.table_context_size, self._param.image_context_size) - if not delimiter_pattern: + if self._param.delimiter_mode == "token_size" and not delimiter_pattern: chunks = _merge_text_chunks_by_token_size(chunks, self._param.chunk_token_size, overlapped_percent) if custom_pattern: diff --git a/rag/flow/parser/parser.py b/rag/flow/parser/parser.py index cf9048d02a..1d472ab4ff 100644 --- a/rag/flow/parser/parser.py +++ b/rag/flow/parser/parser.py @@ -1131,6 +1131,7 @@ class Parser(ProcessBase): blob, conf.get("chunk_token_num", 128), conf.get("delimiter", "\n!?;。；！？"), + keep_delimiters=True, ) if conf.get("output_format") == "json": self.set_output("json", [{"text": section[0], "doc_type_kwd": "text"} for section in sections if section[0]]) diff --git a/test/unit_test/deepdoc/parser/test_txt_parser.py b/test/unit_test/deepdoc/parser/test_txt_parser.py deleted file mode 100644 index 289f77ac8e..0000000000 --- a/test/unit_test/deepdoc/parser/test_txt_parser.py +++ /dev/null @@ -1,178 +0,0 @@ -# -# Copyright 2025 The InfiniFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -"""Unit tests for ``RAGFlowTxtParser.parser_txt`` strict-cap behaviour. - -The pre-fix ``add_chunk`` fired its size check *after* the append, so each -chunk could overshoot ``chunk_token_num`` by up to the size of one line. These -tests assert the proactive projected-total invariant: no produced chunk may -contain more than ``chunk_token_num`` tokens. -""" - -import importlib.util -import os -import sys -from unittest import mock - -_MOCK_MODULES = [ - "xgboost", - "pdfplumber", - "huggingface_hub", - "PIL", - "PIL.Image", - "pypdf", - "sklearn", - "deepdoc.vision", - "deepdoc", - "deepdoc.parser", - "deepdoc.parser.utils", -] -_orig_modules = {m: sys.modules.get(m) for m in _MOCK_MODULES} -_orig_get_text = getattr(sys.modules.get("deepdoc.parser.utils"), "get_text", None) - -try: - for _m in _MOCK_MODULES: - if _m not in sys.modules: - sys.modules[_m] = mock.MagicMock() - - # ``get_text`` is invoked by ``RAGFlowTxtParser.__call__`` only, not by - # ``parser_txt``. Provide a permissive stub so the module loads. - sys.modules["deepdoc.parser.utils"].get_text = lambda *a, **kw: "" - - def _find_project_root(marker="pyproject.toml"): - d = os.path.dirname(os.path.abspath(__file__)) - while d != os.path.dirname(d): - if os.path.exists(os.path.join(d, marker)): - return d - d = os.path.dirname(d) - return None - - _PROJECT_ROOT = _find_project_root() - - _spec = importlib.util.spec_from_file_location( - "deepdoc.parser._txt_parser_under_test", - os.path.join(_PROJECT_ROOT, "deepdoc", "parser", "txt_parser.py"), - ) - _mod = importlib.util.module_from_spec(_spec) - sys.modules["deepdoc.parser._txt_parser_under_test"] = _mod - _spec.loader.exec_module(_mod) - - RAGFlowTxtParser = _mod.RAGFlowTxtParser -finally: - for _m, _orig in _orig_modules.items(): - if _orig is None: - sys.modules.pop(_m, None) - else: - sys.modules[_m] = _orig - if _orig_modules.get("deepdoc.parser.utils") is not None and _orig_get_text is not None: - _orig_modules["deepdoc.parser.utils"].get_text = _orig_get_text - if _orig_modules.get("deepdoc.parser") is not None and _orig_modules.get("deepdoc.parser.utils") is not None: - _orig_modules["deepdoc.parser"].utils = _orig_modules["deepdoc.parser.utils"] - - -# A deterministic, tokenizer-free stand-in for ``num_tokens_from_string`` so -# the assertions below reason in plain words and are independent of tiktoken. - - -def _patch_word_count(monkeypatch_module): - def fake_num_tokens(s): - return len((s or "").split()) - - monkeypatch_module.setattr(_mod, "num_tokens_from_string", fake_num_tokens) - - -def test_no_overshoot_when_packing_short_lines(monkeypatch): - """Lines of 25 tokens, budget 100 — every chunk must be <= 100 tokens.""" - _patch_word_count(monkeypatch) - txt = " ".join(["alpha"] * 25) + "\n" + " ".join(["beta"] * 25) + "\n" + " ".join(["gamma"] * 25) - chunks = RAGFlowTxtParser.parser_txt(txt, chunk_token_num=100, delimiter="\n") - sizes = [len(c[0].split()) for c in chunks if c[0].strip()] - assert all(s <= 100 for s in sizes), sizes - # 75 tokens of content, expected a single 75-token chunk. - assert sum(sizes) == 75 - - -def test_no_overshoot_at_chunk_boundary(monkeypatch): - """Lines of 30 tokens, budget 100. Pre-fix the boundary chunk was 130 tokens.""" - _patch_word_count(monkeypatch) - lines = [" ".join([f"w{i}"] * 30) for i in range(10)] # 10 lines, 300 tokens - chunks = RAGFlowTxtParser.parser_txt("\n".join(lines), chunk_token_num=100, delimiter="\n") - sizes = [len(c[0].split()) for c in chunks if c[0].strip()] - assert all(s <= 100 for s in sizes), sizes - - -def test_atomic_oversized_line_is_sub_split_on_whitespace(monkeypatch): - """A single line that exceeds the budget is split on whitespace atoms.""" - _patch_word_count(monkeypatch) - huge_line = " ".join(["alpha"] * 80) # 80 tokens, no internal delimiter - chunks = RAGFlowTxtParser.parser_txt(huge_line, chunk_token_num=50, delimiter="\n") - sizes = [len(c[0].split()) for c in chunks if c[0].strip()] - assert all(s <= 50 for s in sizes), sizes - assert sum(sizes) == 80 - assert len(chunks) >= 2 - - -def test_empty_text_returns_empty(monkeypatch): - _patch_word_count(monkeypatch) - # Empty input produces a single empty chunk placeholder (existing - # behaviour the callers rely on). The hard-cap guarantee is that any - # chunk carrying content stays within the budget. - result = RAGFlowTxtParser.parser_txt("", chunk_token_num=128, delimiter="\n") - non_empty = [c for c in result if c[0].strip()] - assert non_empty == [] - result2 = RAGFlowTxtParser.parser_txt(" \n\n ", chunk_token_num=128, delimiter="\n") - non_empty2 = [c for c in result2 if c[0].strip()] - assert non_empty2 == [] - - -def test_unbroken_token_exceeding_budget_fallback(monkeypatch): - """A single unbroken non-whitespace string exceeding the budget is split - via the character-window/token-slicing fallback. - """ - - def char_count_tokens(s): - return len(s or "") - - monkeypatch.setattr(_mod, "num_tokens_from_string", char_count_tokens) - - huge_word = "a" * 80 # 80 characters/tokens, no whitespace - chunks = RAGFlowTxtParser.parser_txt(huge_word, chunk_token_num=30, delimiter="\n") - - non_empty = [c[0] for c in chunks if c[0].strip()] - assert len(non_empty) >= 3 - assert all(char_count_tokens(c) <= 30 for c in non_empty) - assert "".join(non_empty) == huge_word - - -def test_newline_join_token_count_strict_cap(monkeypatch): - """Verify that joining chunks with newline does not overshoot chunk_token_num - even when individual token counts sum to <= budget but the newline pushes it over. - """ - - def char_count_tokens(s): - return len(s or "") - - monkeypatch.setattr(_mod, "num_tokens_from_string", char_count_tokens) - - # Two lines of 10 chars each. Budget = 20. - # line1 + "\n" + line2 = 10 + 1 + 10 = 21 chars/tokens, exceeding budget of 20. - line1 = "a" * 10 - line2 = "b" * 10 - txt = f"{line1}\n{line2}" - chunks = RAGFlowTxtParser.parser_txt(txt, chunk_token_num=20, delimiter="\n") - non_empty = [c[0] for c in chunks if c[0].strip()] - assert all(char_count_tokens(c) <= 20 for c in non_empty) - assert len(non_empty) == 2 diff --git a/web/src/utils/__tests__/delimiter-preview.test.ts b/web/src/utils/__tests__/delimiter-preview.test.ts index f6e7768149..76facd72cc 100644 --- a/web/src/utils/__tests__/delimiter-preview.test.ts +++ b/web/src/utils/__tests__/delimiter-preview.test.ts @@ -34,9 +34,7 @@ describe('parseDelimitersForDisplay', () => { }); it('normalizes CRLF before parsing', () => { - expect(parseDelimitersForDisplay('\r\n').map((d) => d.raw)).toEqual([ - '\n', - ]); + expect(parseDelimitersForDisplay('\r\n').map((d) => d.raw)).toEqual(['\n']); expect(parseDelimitersForDisplay('`\r\n`').map((d) => d.raw)).toEqual([ '\n', ]);