mirror of
https://github.com/infiniflow/ragflow.git
synced 2026-08-03 14:27:32 +08:00
d4ceeee4ed
## Summary Six sites used to read the same `parser_config.delimiter` field with divergent grammars: - `rag.nlp.get_delimiters` (PDF/DOCX/HTML/EPUB/JSON/CSV/XLSX/email/book) - `rag.nlp.naive_merge` (custom-delimiter branch) - `rag.nlp.naive_merge_with_images` - `rag.nlp._build_cks` - `deepdoc.parser.txt_parser.parser_txt` (.txt, code) - `deepdoc.parser.markdown_parser.MarkdownElementExtractor.get_delimiters` The six implementations disagreed on bare-vs-wrapped chars, dedupe, sort order, CRLF normalization, and `re.I` (#17384). The shipped default `` `\n!?;。；！？` `` was a no-op for `.md` because the markdown path only matched backtick-wrapped tokens. ## Changes - **new:** `rag/nlp/delim.py` with `parse_delimiter_field` and `compile_delimiter_pattern`. Single source of truth. CRLF normalization at the top; longest-first stable sort; insertion-ordered dedupe; no `re.I`. - **refactor:** all six call sites delegate to the helper. - `rag/nlp/__init__.py::get_delimiters` becomes a thin shim. - `deepdoc/parser/txt_parser.py::parser_txt` drops the `[encode/decode/unicode_escape]` round-trip. - `deepdoc/parser/markdown_parser.py::get_delimiters` honors bare chars (fixes [1]). - **tests:** `test/unit_test/rag/test_delim.py` (85 tests) — helper, acceptance table, frontend parity, static guard against re-inlining. - **tests:** `test/unit_test/rag/test_delimiter_case_sensitive.py` (from #17386) updated to retarget the static check at the new helper + AST-based broader scan. ## Acceptance criteria - All six sites produce the same regex pattern for the same input. - Shipped default keeps working for `.txt` / `.pdf` / `.docx`. - Shipped default for `.md` now splits (was a silent no-op). - Tooltip example `` `\n##;` `` produces three effective delimiters regardless of file type. - Bare whitespace inputs split on every occurrence. - Backtick-wrapped whitespace splits only on the exact N-char sequence. - CRLF-line-ending documents split identically to LF-line-ending documents. - 123 tests pass (85 new + 38 existing). ## Rebase protocol As #17385 and #17386 evolve, this branch will be rebased on top. The only overlap between this PR's diff and the other two is `test_delimiter_case_sensitive.py`, where #17383 modifies the static check to point at the new helper location. --------- Co-authored-by: kiloconnect[bot] <240665456+kiloconnect[bot]@users.noreply.github.com>
85 lines
2.9 KiB
Python
85 lines
2.9 KiB
Python
#
|
|
# Copyright 2025 The InfiniFlow Authors. All Rights Reserved.
|
|
#
|
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
# you may not use this file except in compliance with the License.
|
|
# You may obtain a copy of the License at
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing, software
|
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
# See the License for the specific language governing permissions and
|
|
# limitations under the License.
|
|
#
|
|
|
|
import logging
|
|
import re
|
|
|
|
from common.token_utils import num_tokens_from_string
|
|
from deepdoc.parser.utils import get_text
|
|
from rag.nlp import _split_oversized_unit
|
|
from rag.nlp.delim import (
|
|
compile_delimiter_pattern,
|
|
normalize_text_newlines,
|
|
parse_delimiter_field,
|
|
)
|
|
|
|
|
|
class RAGFlowTxtParser:
|
|
def __call__(self, fnm, binary=None, chunk_token_num=128, delimiter="\n!?;。；！？"):
|
|
txt = get_text(fnm, binary)
|
|
return self.parser_txt(txt, chunk_token_num, delimiter)
|
|
|
|
@classmethod
|
|
def parser_txt(cls, txt, chunk_token_num=128, delimiter="\n!?;。；！？"):
|
|
if not isinstance(txt, str):
|
|
raise TypeError("txt type should be str!")
|
|
cks = [""]
|
|
tk_nums = [0]
|
|
|
|
def add_chunk(t):
|
|
nonlocal cks, tk_nums
|
|
tnum = num_tokens_from_string(t)
|
|
|
|
if cks[-1] == "":
|
|
cks[-1] = t
|
|
tk_nums[-1] = tnum
|
|
return
|
|
|
|
merged = cks[-1] + "\n" + t
|
|
merged_tnum = num_tokens_from_string(merged)
|
|
if merged_tnum <= chunk_token_num:
|
|
cks[-1] = merged
|
|
tk_nums[-1] = merged_tnum
|
|
return
|
|
|
|
cks.append(t)
|
|
tk_nums.append(tnum)
|
|
|
|
txt = normalize_text_newlines(txt)
|
|
parsed_dels = parse_delimiter_field(delimiter)
|
|
dels = compile_delimiter_pattern(parsed_dels)
|
|
logging.debug(
|
|
"RAGFlowTxtParser.parser_txt: delimiter_count=%d, splitting=%s",
|
|
len(parsed_dels),
|
|
bool(dels),
|
|
)
|
|
secs = re.split(r"(%s)" % dels, txt) if dels else [txt]
|
|
for sec in secs:
|
|
if dels and re.match(f"^{dels}$", sec):
|
|
continue
|
|
if not sec:
|
|
continue
|
|
if num_tokens_from_string(sec) <= chunk_token_num:
|
|
add_chunk(sec)
|
|
continue
|
|
pieces = _split_oversized_unit(sec, chunk_token_num, token_count_fn=num_tokens_from_string)
|
|
logging.debug("parser_txt: split oversized section (%d tokens) into %d pieces", num_tokens_from_string(sec), len(pieces))
|
|
for piece in pieces:
|
|
add_chunk(piece)
|
|
|
|
logging.debug("parser_txt: %d sections -> %d chunks (chunk_token_num=%d)", len(secs), len(cks), chunk_token_num)
|
|
return [[c, ""] for c in cks]
|