zlei6/ragflow
1
0
Fork 0
mirror of https://github.com/infiniflow/ragflow.git synced 2026-08-08 16:38:01 +08:00
Code Issues Packages Projects Releases Wiki Activity

test(token_chunker): drop unsupported chunk_token_size=0 case from delimiter-mode test (#17997)

Browse Source
This commit is contained in:
Jack
2026-08-07 17:43:04 +08:00
committed by GitHub
parent 2fef38f955
commit 869da9c7ad
1 changed files with 9 additions and 9 deletions
Download Patch File Download Diff File Expand all files Collapse all files

18
rag/flow/tests/test_token_chunker.py
View File

@@ -426,10 +426,10 @@ def test_json_delimiter_mode_consecutive_delimiter_keeps_boundary():
assert all("##" not in t for t in texts)
def test_text_delimiter_mode_zero_or_one_no_atom_split():
# chunk_token_size=0/1 must not atom-split delimiter segments into 1-token
def test_text_delimiter_mode_one_no_atom_split():
# chunk_token_size=1 must not atom-split delimiter segments into 1-token
# chunks; the delimiter path produces delimiter-boundary chunks regardless of cap.
for module, chunker in _build_json_chunker({"delimiter_mode": "delimiter", "delimiters": ["`|`"]}):
for _module, chunker in _build_json_chunker({"delimiter_mode": "delimiter", "delimiters": ["`|`"]}):
# text path: delimiter_mode is delimiter but a custom delimiter is
# present, so the delimiter branch (_split_text_by_pattern) is used.
kwargs = {
@@ -437,9 +437,9 @@ def test_text_delimiter_mode_zero_or_one_no_atom_split():
"output_format": "text",
"text": "aaa|bbb|ccc",
}
for chunk_token_size in (0, 1):
setattr(chunker._param, "chunk_token_size", chunk_token_size)
asyncio.run(chunker._invoke(**kwargs))
chunks = chunker._outputs["chunks"]
texts = [c["text"] for c in chunks]
assert texts == ["aaa", "bbb", "ccc"], f"chunk_token_size={chunk_token_size} atom-split a delimiter segment: {texts}"
chunk_token_size = 1
setattr(chunker._param, "chunk_token_size", chunk_token_size)
asyncio.run(chunker._invoke(**kwargs))
chunks = chunker._outputs["chunks"]
texts = [c["text"] for c in chunks]
assert texts == ["aaa", "bbb", "ccc"], f"chunk_token_size={chunk_token_size} atom-split a delimiter segment: {texts}"