fix(chunker): carry overlap-head PDF positions into new chunk (#18148) (#18227)

When `TokenChunker` starts a fresh chunk with an overlap prefix (Go `computeOverlapPrefix` / Python visible-text cut), the previous chunk's **tail PDF coordinates were dropped**. As a result, the overlap head of a PDF chunk is displayed but **not highlighted** — the highlight box is shifted/truncated relative to the displayed span (infiniflow/ragflow#18148).

rag/flow/chunker/token_chunker.py

@@ -106,7 +106,7 @@ def _split_text_by_pattern(text, pattern):
     if not pattern:
         return [text or ""]
 
-    split_texts = re.split(r"(%s)" % pattern, text or "", flags=re.DOTALL)
+    split_texts = re.split("(" + pattern + ")", text or "", flags=re.DOTALL)
     chunks = []
     for i in range(0, len(split_texts), 2):
         chunk = split_texts[i]
@@ -230,19 +230,59 @@ def _attach_context_to_media_chunks(chunks, table_context_size, image_context_si
         chunk["context_below"] = "".join(parts_below)
 
 
+def _overlap_tail_positions(prev_items, overlap_start):
+    # Given the source items a previous chunk was built from (each as
+    # ``(item_text, pos_group)`` where ``pos_group`` is that item's
+    # ``_pdf_positions`` list), return the flattened boxes whose item visible
+    # span intersects the overlap tail ``[overlap_start, total)``.
+    #
+    # PDF positions are per-item (coarse), so an item is included wholesale once
+    # any part of it falls in the overlap tail. This keeps the overlap prefix
+    # highlighted without over-inflating the box set with the previous chunk's
+    # non-overlap (head) coordinates (#18148).
+    if not prev_items:
+        return []
+    # Items are concatenated with a single "\n" separator, matching the merge
+    # join at _merge_text_chunks_by_token_size.
+    spans = []
+    offset = 0
+    for item_text, _pos_group in prev_items:
+        start = offset
+        end = offset + len(item_text)
+        spans.append((start, end))
+        offset = end + 1
+    if not spans:
+        return []
+    total = spans[-1][1]
+    out = []
+    for (start, end), (_text, pos_group) in zip(spans, prev_items, strict=True):
+        if start < total and end > overlap_start:
+            out.extend(pos_group or [])
+    return out
+
+
 def _merge_text_chunks_by_token_size(chunks, chunk_token_size, overlapped_percent):
     # Merge adjacent text chunks when delimiter-based splitting is not active.
     merged = []
+    # Parallel to ``merged``: for each merged chunk, the list of source items it
+    # was built from, each as ``(item_text, pos_group)``. Tracking items (not
+    # just the flattened position list) lets us map a visible-text offset range
+    # back to the exact coordinate boxes that belong to it, so the overlap
+    # prefix carries only the previous chunk's tail coordinates instead of
+    # dropping them (#18148).
+    merged_items = []
     prev_text_idx = -1
     threshold = chunk_token_size * (100 - overlapped_percent) / 100.0
 
     for chunk in chunks:
         if chunk["ck_type"] != "text":
             merged.append(deepcopy(chunk))
+            merged_items.append(None)
             prev_text_idx = -1
             continue
 
         current = deepcopy(chunk)
+        current_item = (current["text"], list(current.get(PDF_POSITIONS_KEY) or []))
         should_start_new = prev_text_idx < 0 or merged[prev_text_idx]["tk_nums"] > threshold
         # #17799: an over-budget unit stands alone — never merged into the
         # previous chunk. This matches Python naive_merge and the Go
@@ -262,11 +302,21 @@ def _merge_text_chunks_by_token_size(chunks, chunk_token_size, overlapped_percen
                 overlap_start = int(len(visible) * (100 - overlapped_percent) / 100.0)
                 if 0 <= overlap_start < len(visible):
                     overlap_text = visible[overlap_start:]
+                    # Carry the previous chunk's tail coordinates so the overlap
+                    # prefix is highlighted, not just the cur span (#18148).
+                    # Only the items intersecting the overlap tail keep their
+                    # boxes; the head (non-overlap) boxes are excluded so the
+                    # highlight is not over-inflated.
+                    overlap_positions = _overlap_tail_positions(merged_items[prev_text_idx], overlap_start)
                 else:
                     overlap_text = ""
+                    overlap_positions = []
                 current["text"] = overlap_text + current["text"]
+                current[PDF_POSITIONS_KEY] = overlap_positions + (current.get(PDF_POSITIONS_KEY) or [])
+                current_item = (current["text"], list(current.get(PDF_POSITIONS_KEY) or []))
                 current["tk_nums"] = num_tokens_from_string(current["text"])
             merged.append(current)
+            merged_items.append([current_item])
             prev_text_idx = len(merged) - 1
             continue
 
@@ -276,6 +326,7 @@ def _merge_text_chunks_by_token_size(chunks, chunk_token_size, overlapped_percen
             merged[prev_text_idx]["text"] += current["text"]
         merged[prev_text_idx][PDF_POSITIONS_KEY].extend(current.get(PDF_POSITIONS_KEY) or [])
         merged[prev_text_idx]["tk_nums"] += current["tk_nums"]
+        merged_items[prev_text_idx].append(current_item)
 
     return merged
 
@@ -340,8 +391,8 @@ class TokenChunker(ProcessBase):
     async def _invoke(self, **kwargs):
         try:
             from_upstream = TokenChunkerFromUpstream.model_validate(kwargs)
-        except Exception as e:
-            self.set_output("_ERROR", f"Input error: {str(e)}")
+        except Exception as e:  # noqa: BLE001
+            self.set_output("_ERROR", f"Input error: {e!s}")
             return
 
         # Build the primary delimiter regex. If no active custom delimiter exists,
@@ -437,7 +488,7 @@ class TokenChunker(ProcessBase):
                     offset += 1
                 combined_text = "".join(parts[:-1])  # drop the trailing glue
 
-                raw = re.split(r"(%s)" % delimiter_pattern, combined_text, flags=re.DOTALL)
+                raw = re.split("(" + delimiter_pattern + ")", combined_text, flags=re.DOTALL)
                 segments = []  # (text, start, end) within combined_text
                 pos = 0
                 for i in range(0, len(raw), 2):

rag/flow/tests/test_parser_pdf_positions.py

@@ -0,0 +1,72 @@
+"""Parser-layer regression guard for the Book builtin DSL highlight path.
+
+The Book builtin DSL parses PDF with ``parse_method=DeepDOC`` and
+``output_format=json``. The parser (rag/flow/parser/parser.py:781) runs
+``normalize_pdf_items_metadata(bboxes)`` on the deepdoc output, which must
+keep each box's PDF coordinates so the downstream TitleChunker can build
+``position_int`` and the parsing-result view can highlight the text.
+
+This test drives the REAL gate function with deepdoc-style boxes (carrying
+both ``positions`` and ``position_tag``, exactly as
+deepdoc/parser/pdf_parser.py:1900-1902 emits) and asserts the coordinates
+survive into the internal ``_pdf_positions`` field without stripping the
+original ``positions`` field. It is the parser-layer counterpart of
+rag/flow/tests/test_title_chunker_position_int.py (which proves the
+chunker layer keeps the coordinates). Together they pin down the full
+parser -> chunker -> position_int bridge for infiniflow/ragflow#18148.
+"""
+
+from rag.flow.parser.pdf_chunk_metadata import (
+    PDF_POSITIONS_KEY,
+    extract_pdf_positions,
+    normalize_pdf_items_metadata,
+)
+
+
+def _deepdoc_style_bboxes():
+    # Shape mirrors deepdoc RAGFlowPdfParser.parse_into_bboxes output:
+    # one entry with both position_tag + positions, two with positions only,
+    # spanning page 1 and page 2.
+    return [
+        {
+            "text": "Introduction paragraph on page one.",
+            "layout_type": "text",
+            "position_tag": "@@1\tIntroduction paragraph on page one.",
+            "positions": [[1, 10, 200, 50, 80]],
+        },
+        {
+            "text": "Body text continues on page one.",
+            "layout_type": "text",
+            "positions": [[1, 12, 205, 90, 120]],
+        },
+        {
+            "text": "Second chapter starts on page two.",
+            "layout_type": "text",
+            "positions": [[2, 15, 210, 40, 75]],
+        },
+    ]
+
+
+def test_parser_gate_preserves_bbox_coordinates():
+    bboxes = _deepdoc_style_bboxes()
+    # This is exactly what parser.py:781 calls for output_format == "json".
+    normalize_pdf_items_metadata(bboxes)
+
+    for box in bboxes:
+        # Coordinate bridge: the chunker reads PDF_POSITIONS_KEY.
+        assert box.get(PDF_POSITIONS_KEY), f"missing {PDF_POSITIONS_KEY}: {box}"
+        # The original field must NOT be stripped by normalization.
+        assert "positions" in box, f"positions stripped from box: {box}"
+
+    # Pages referenced by the downstream chunker must cover every source page.
+    pages = {int(p[0]) for box in bboxes for p in extract_pdf_positions(box)}
+    assert pages == {1, 2}, f"expected pages {{1,2}}, got {pages}"
+
+
+def test_parser_gate_produces_exact_coordinates():
+    bboxes = _deepdoc_style_bboxes()
+    normalize_pdf_items_metadata(bboxes)
+
+    # Each normalized box keeps the source (page, left, right, top, bottom).
+    assert extract_pdf_positions(bboxes[0]) == [[1, 10, 200, 50, 80]]
+    assert extract_pdf_positions(bboxes[2]) == [[2, 15, 210, 40, 75]]

rag/flow/tests/test_title_chunker_position_int.py

@@ -0,0 +1,263 @@
+import asyncio
+import importlib
+import sys
+import types
+from contextlib import contextmanager
+from pathlib import Path
+
+"""Reproduction test for the Book builtin DSL highlight path.
+
+The Book builtin DSL parses PDF with ``parse_method=DeepDOC`` and
+``output_format=json``, then chunks with ``TitleChunker`` (method=hierarchy).
+This test drives the real TitleChunker chain (extract -> merge -> finalize)
+with deepdoc-style JSON items that carry ``positions``, and asserts the
+emitted chunks keep a non-empty ``position_int`` covering every source page.
+
+It is a GREEN regression guard: it proves the TitleChunker layer preserves
+PDF coordinates for the Book DSL, which localizes the parsing-result "missing
+highlight" bug to the parser emission / dataset config rather than the
+chunker. Mirrors rag/flow/tests/test_token_chunker.py's
+``test_json_delimiter_mode_position_int_survives_full_chain``.
+"""
+
+
+@contextmanager
+def _load_title_chunker_with_stubs():
+    root = Path(__file__).resolve().parents[3]
+    original_modules = {}
+
+    def _install(name: str, module: types.ModuleType):
+        original_modules.setdefault(name, sys.modules.get(name))
+        sys.modules[name] = module
+
+    try:
+        rag_pkg = types.ModuleType("rag")
+        rag_pkg.__path__ = [str(root / "rag")]
+        _install("rag", rag_pkg)
+
+        rag_flow_pkg = types.ModuleType("rag.flow")
+        rag_flow_pkg.__package__ = "rag"
+        rag_flow_pkg.__path__ = [str(root / "rag" / "flow")]
+        _install("rag.flow", rag_flow_pkg)
+
+        rag_flow_chunker_pkg = types.ModuleType("rag.flow.chunker")
+        rag_flow_chunker_pkg.__package__ = "rag.flow"
+        rag_flow_chunker_pkg.__path__ = [str(root / "rag" / "flow" / "chunker")]
+        _install("rag.flow.chunker", rag_flow_chunker_pkg)
+
+        rag_flow_parser_pkg = types.ModuleType("rag.flow.parser")
+        rag_flow_parser_pkg.__package__ = "rag.flow"
+        rag_flow_parser_pkg.__path__ = [str(root / "rag" / "flow" / "parser")]
+        _install("rag.flow.parser", rag_flow_parser_pkg)
+
+        common_pkg = types.ModuleType("common")
+        common_pkg.__path__ = [str(root / "common")]
+        _install("common", common_pkg)
+
+        common_float_utils = types.ModuleType("common.float_utils")
+        common_float_utils.normalize_overlapped_percent = lambda value: value
+        _install("common.float_utils", common_float_utils)
+
+        common_token_utils = types.ModuleType("common.token_utils")
+        common_token_utils.num_tokens_from_string = lambda text: 1
+        _install("common.token_utils", common_token_utils)
+
+        rag_nlp = types.ModuleType("rag.nlp")
+        rag_nlp.naive_merge = lambda *args, **kwargs: []
+        rag_nlp.not_bullet = lambda text: False
+        rag_nlp.not_title = lambda text: True
+        _install("rag.nlp", rag_nlp)
+
+        deepdoc_pkg = types.ModuleType("deepdoc")
+        deepdoc_pkg.__path__ = [str(root / "deepdoc")]
+        _install("deepdoc", deepdoc_pkg)
+
+        deepdoc_parser_pkg = types.ModuleType("deepdoc.parser")
+        deepdoc_parser_pkg.__path__ = [str(root / "deepdoc" / "parser")]
+        _install("deepdoc.parser", deepdoc_parser_pkg)
+
+        class _RAGFlowPdfParser:
+            @staticmethod
+            def remove_tag(text):
+                return text
+
+            @staticmethod
+            def extract_positions(tag):
+                return []
+
+        deepdoc_pdf_parser = types.ModuleType("deepdoc.parser.pdf_parser")
+        deepdoc_pdf_parser.RAGFlowPdfParser = _RAGFlowPdfParser
+        _install("deepdoc.parser.pdf_parser", deepdoc_pdf_parser)
+
+        deepdoc_parser_utils = types.ModuleType("deepdoc.parser.utils")
+        deepdoc_parser_utils.extract_pdf_outlines = lambda *args, **kwargs: []
+        _install("deepdoc.parser.utils", deepdoc_parser_utils)
+
+        class ProcessParamBase:
+            def __init__(self):
+                pass
+
+            def check_valid_value(self, value, msg, allowed):
+                if value not in allowed:
+                    raise ValueError(msg)
+
+            def check_positive_integer(self, value, msg):
+                pass
+
+            def check_decimal_float(self, value, msg):
+                pass
+
+            def check_nonnegative_number(self, value, msg):
+                pass
+
+        class ProcessBase:
+            def __init__(self, _pipeline, _id, param):
+                self._pipeline = _pipeline
+                self._id = _id
+                self._param = param
+                self._outputs = {}
+                self.callback = lambda *_args, **_kwargs: None
+
+            def set_output(self, key, value):
+                self._outputs[key] = value
+
+        rag_flow_base = types.ModuleType("rag.flow.base")
+        rag_flow_base.ProcessBase = ProcessBase
+        rag_flow_base.ProcessParamBase = ProcessParamBase
+        _install("rag.flow.base", rag_flow_base)
+
+        pdf_chunk_metadata = types.ModuleType("rag.flow.parser.pdf_chunk_metadata")
+        pdf_chunk_metadata.PDF_POSITIONS_KEY = "pdf_positions"
+        pdf_chunk_metadata.extract_pdf_positions = lambda _item: []
+        pdf_chunk_metadata.merge_pdf_positions = lambda _records: []
+        pdf_chunk_metadata.finalize_pdf_chunk = lambda chunk: chunk
+        pdf_chunk_metadata.restore_pdf_text_previews = lambda *_a, **_k: None
+        _install("rag.flow.parser.pdf_chunk_metadata", pdf_chunk_metadata)
+
+        common_spec = importlib.util.spec_from_file_location(
+            "rag.flow.chunker.title_chunker.common",
+            root / "rag" / "flow" / "chunker" / "title_chunker" / "common.py",
+        )
+        common_module = importlib.util.module_from_spec(common_spec)
+        _install("rag.flow.chunker.title_chunker.common", common_module)
+        common_spec.loader.exec_module(common_module)
+
+        hierarchy_spec = importlib.util.spec_from_file_location(
+            "rag.flow.chunker.title_chunker.hierarchy_chunker",
+            root / "rag" / "flow" / "chunker" / "title_chunker" / "hierarchy_chunker.py",
+        )
+        hierarchy_module = importlib.util.module_from_spec(hierarchy_spec)
+        _install("rag.flow.chunker.title_chunker.hierarchy_chunker", hierarchy_module)
+        hierarchy_spec.loader.exec_module(hierarchy_module)
+
+        yield common_module, hierarchy_module
+    finally:
+        for module_name, original in original_modules.items():
+            if original is None:
+                sys.modules.pop(module_name, None)
+            else:
+                sys.modules[module_name] = original
+
+
+def _real_extract_pdf_positions(item):
+    # Faithful mirror of rag/flow/parser/pdf_chunk_metadata.extract_pdf_positions.
+    if not isinstance(item, dict):
+        return []
+    positions = item.get("pdf_positions")
+    if isinstance(positions, list):
+        return [list(p) for p in positions]
+    positions = item.get("positions")
+    if isinstance(positions, list):
+        return [list(p) for p in positions]
+    position_tag = item.get("position_tag")
+    if isinstance(position_tag, str) and position_tag:
+        return []  # RAGFlowPdfParser.extract_positions is stubbed out here.
+    position_int = item.get("position_int")
+    if isinstance(position_int, list):
+        return [list(p) for p in position_int if isinstance(p, (list, tuple)) and len(p) >= 5]
+    return []
+
+
+def _real_merge_pdf_positions(records):
+    # Faithful mirror of rag/flow/parser/pdf_chunk_metadata.merge_pdf_positions.
+    merged = []
+    for rec in records or []:
+        if not isinstance(rec, dict):
+            continue
+        for pos in rec.get("pdf_positions") or []:
+            if isinstance(pos, (list, tuple)) and len(pos) >= 5:
+                merged.append([pos[0], pos[1], pos[2], pos[3], pos[4]])
+    seen = set()
+    out = []
+    for pos in merged:
+        key = tuple(pos[:5])
+        if key not in seen:
+            seen.add(key)
+            out.append(pos)
+    out.sort(key=lambda item: (item[0], item[3], item[1]))
+    return out
+
+
+def _real_finalize_pdf_chunk(chunk):
+    # Faithful mirror of rag/flow/parser/pdf_chunk_metadata.finalize_pdf_chunk.
+    positions = _real_extract_pdf_positions(chunk)
+    if positions:
+        chunk["position_int"] = [list(p) for p in positions]
+        chunk.pop("pdf_positions", None)
+    return chunk
+
+
+def test_title_chunker_preserves_position_int_from_deepdoc_json():
+    # Reproduces the parsing-result highlight verification for the Book builtin
+    # DSL (infiniflow/ragflow#18148 follow-up): a deepdoc + json parser output
+    # carrying per-item ``positions`` must flow through TitleChunker (method=
+    # hierarchy) and reach ``position_int`` on the emitted chunks. This proves
+    # the TitleChunker layer is NOT the cause of the missing highlight.
+    with _load_title_chunker_with_stubs() as (common_module, hierarchy_module):
+        # Install faithful coordinate helpers so the REAL TitleChunker path runs.
+        common_module.extract_pdf_positions = _real_extract_pdf_positions
+        common_module.merge_pdf_positions = _real_merge_pdf_positions
+        common_module.finalize_pdf_chunk = _real_finalize_pdf_chunk
+
+        async def _restore_previews(*_a, **_k):
+            return None
+
+        common_module.restore_pdf_text_previews = _restore_previews
+
+        json_result = [
+            {"text": "Introduction paragraph on page one.", "doc_type_kwd": "text", "positions": [[1, 10, 200, 50, 80]]},
+            {"text": "Body text continues on page one.", "doc_type_kwd": "text", "positions": [[1, 12, 205, 90, 120]]},
+            {"text": "Second chapter starts on page two.", "doc_type_kwd": "text", "positions": [[2, 15, 210, 40, 75]]},
+        ]
+
+        from_upstream = types.SimpleNamespace(
+            output_format="json",
+            json_result=json_result,
+            markdown_result=None,
+            text_result=None,
+            html_result=None,
+            chunks=None,
+            file=None,
+            name="book-test",  # not *.pdf -> restore_pdf_text_previews early-returns
+        )
+
+        param = common_module.TitleChunkerParam()
+        param.method = "hierarchy"
+        param.hierarchy = 1
+        param.levels = []  # no headings -> all body -> single merged chunk
+        param.include_heading_content = False
+        param.root_chunk_as_heading = False
+
+        process = common_module.ProcessBase(None, "title_chunker", param)
+        process._canvas = types.SimpleNamespace(_doc_id="doc", _tenant_id="tenant")
+        process._outputs = {}
+
+        chunker = hierarchy_module.HierarchyTitleChunker(process, from_upstream)
+        asyncio.run(chunker.invoke())
+
+        chunks = process._outputs.get("chunks", [])
+        assert chunks, "TitleChunker produced no chunks"
+        pos_int = chunks[0].get("position_int")
+        assert pos_int, "position_int missing from TitleChunker output"
+        pages = {p[0] for p in pos_int}
+        assert pages == {1, 2}, f"expected pages {{1,2}}, got {pages}"

rag/flow/tests/test_token_chunker.py

@@ -1,10 +1,12 @@
-import importlib.util
 import asyncio
+import importlib.util
 import sys
 import types
 from contextlib import contextmanager
 from pathlib import Path
 
+import pytest
+
 
 @contextmanager
 def _load_token_chunker_with_stubs():
@@ -107,7 +109,7 @@ def _load_token_chunker_with_stubs():
             schema_module = importlib.util.module_from_spec(schema_spec)
             _install("rag.flow.chunker.schema", schema_module)
             schema_spec.loader.exec_module(schema_module)
-        except Exception:
+        except Exception:  # noqa: BLE001
             schema_module = types.ModuleType("rag.flow.chunker.schema")
 
             class TokenChunkerFromUpstream:
@@ -303,11 +305,8 @@ def test_token_size_mode_normalized_to_delimiter():
 
         bad = token_chunker_module.TokenChunkerParam()
         bad.delimiter_mode = "nope"
-        try:
+        with pytest.raises(ValueError):
             bad.check()
-            raise AssertionError("expected check() to reject unknown delimiter_mode")
-        except Exception:
-            pass
 
 
 def test_json_no_delimiter_mode_merges_to_token_cap():
@@ -375,7 +374,7 @@ def test_json_delimiter_mode_pdf_positions_per_segment_not_broadcast():
                 if key in preview_cache:
                     chunk["img_id"] = preview_cache[key]
                 else:
-                    new_id = "img-%d" % len(preview_cache)
+                    new_id = f"img-{len(preview_cache)}"
                     chunk["img_id"] = new_id
                     preview_cache[key] = new_id
 
@@ -410,6 +409,95 @@ def test_json_delimiter_mode_pdf_positions_per_segment_not_broadcast():
         assert len(set(img_ids)) == len(img_ids), img_ids
 
 
+def test_json_no_delimiter_mode_overlap_prefix_carries_prev_positions():
+    # TDD test for #18148. When the JSON path merges to the token cap with
+    # overlapped_percent>0, a fresh chunk starts with an overlap prefix copied
+    # from the previous chunk's tail (token_chunker.py:_merge_text_chunks_by_
+    # token_size, the should_start_new branch). That overlap text is part of the
+    # chunk's displayed content, so its PDF coordinates MUST also be carried
+    # forward into the new chunk's _pdf_positions -- exactly like the merge-into-
+    # prev path extends positions (token_chunker.py:277). Today the overlap
+    # branch only keeps cur's coordinates, so the overlap head is shown but not
+    # highlighted. Mirrors the Go test
+    # TestMergeByTokenSizeFromJSON_OverlapPrefixCarriesPrevPositions.
+    #
+    # overlapped_percent=100 makes the overlap prefix the ENTIRE previous chunk,
+    # so the expectation is crisp: every new chunk must carry the previous
+    # chunk's full coordinates. RED until the overlap branch carries coordinates.
+    for _module, chunker in _build_json_chunker({"delimiter_mode": "delimiter", "delimiters": [], "chunk_token_size": 5, "overlapped_percent": 100}):
+        kwargs = {
+            "name": "token_chunker",
+            "output_format": "json",
+            "json_result": [
+                {"text": "alpha", "doc_type_kwd": "text", "positions": [[1, 0, 10, 0, 5]]},
+                {"text": "beta", "doc_type_kwd": "text", "positions": [[2, 0, 20, 0, 8]]},
+                {"text": "gamma", "doc_type_kwd": "text", "positions": [[3, 0, 30, 0, 12]]},
+            ],
+        }
+        asyncio.run(chunker._invoke(**kwargs))
+        chunks = chunker._outputs["chunks"]
+        # Each unit starts a fresh chunk at overlapped_percent=100.
+        assert len(chunks) == 3, f"want 3 chunks, got {len(chunks)}"
+
+        # chunk[1] starts with the overlap prefix copied from chunk[0] ("alpha").
+        assert "alpha" in chunks[1]["text"], f"chunk[1] missing overlap prefix: {chunks[1]['text']!r}"
+        # The overlap prefix is shown, so chunk[1] must also carry chunk[0]'s
+        # coordinates. BUG: only chunk[1]'s own (posB) coordinates survive.
+        pos1 = chunks[1].get("pdf_positions") or []
+        assert [1, 0, 10, 0, 5] in pos1, f"chunk[1] dropped overlap-head coords (prev posA): {pos1}"
+        assert [2, 0, 20, 0, 8] in pos1, f"chunk[1] lost its own coords: {pos1}"
+
+        # chunk[2]'s overlap prefix is the full chunk[1] text; its coordinates
+        # must include chunk[0], chunk[1], and its own (overlap chain carried).
+        assert "alphabeta" in chunks[2]["text"], f"chunk[2] missing overlap prefix: {chunks[2]['text']!r}"
+        pos2 = chunks[2].get("pdf_positions") or []
+        for want in ([1, 0, 10, 0, 5], [2, 0, 20, 0, 8], [3, 0, 30, 0, 12]):
+            assert want in pos2, f"chunk[2] missing coords {want} (overlap chain not carried): {pos2}"
+
+
+def test_json_no_delimiter_mode_partial_overlap_prefix_carries_only_tail_positions():
+    # Partial-overlap companion to
+    # test_json_no_delimiter_mode_overlap_prefix_carries_prev_positions (#18148).
+    # overlapped_percent=100 (the full-overlap test) forces the ENTIRE previous
+    # chunk into the overlap prefix; here overlapped_percent=20 means the
+    # overlap prefix is only the TAIL ~20% of the previous chunk. The
+    # coordinates carried must be exactly the previous chunk's tail items whose
+    # span intersects that tail -- NOT the whole previous chunk. This locks the
+    # per-item tail-selection in _overlap_tail_positions (token_chunker.py:233):
+    # a regression that carried the entire previous chunk's coordinates
+    # (over-inflating the highlight box) or dropped overlap coordinates entirely
+    # would both fail this test.
+    for _module, chunker in _build_json_chunker({"delimiter_mode": "delimiter", "delimiters": [], "chunk_token_size": 5, "overlapped_percent": 20}):
+        kwargs = {
+            "name": "token_chunker",
+            "output_format": "json",
+            "json_result": [
+                {"text": "aaaaa", "doc_type_kwd": "text", "positions": [[1, 0, 10, 0, 5]]},
+                {"text": "bbbbb", "doc_type_kwd": "text", "positions": [[2, 0, 20, 0, 8]]},
+                {"text": "ccccc", "doc_type_kwd": "text", "positions": [[3, 0, 30, 0, 12]]},
+                {"text": "ddddd", "doc_type_kwd": "text", "positions": [[4, 0, 40, 0, 16]]},
+                {"text": "eeeee", "doc_type_kwd": "text", "positions": [[5, 0, 50, 0, 20]]},
+                {"text": "fffff", "doc_type_kwd": "text", "positions": [[6, 0, 60, 0, 24]]},
+            ],
+        }
+        asyncio.run(chunker._invoke(**kwargs))
+        chunks = chunker._outputs["chunks"]
+        # 5 items merge into one chunk (tk reaches 5); the 6th starts fresh with
+        # a partial overlap prefix.
+        assert len(chunks) == 2, f"want 2 chunks, got {len(chunks)}"
+
+        # The new chunk's overlap text is the tail of the previous chunk.
+        assert "eeeee" in chunks[1]["text"], f"chunk[1] missing overlap tail text: {chunks[1]['text']!r}"
+        pos1 = chunks[1].get("pdf_positions") or []
+        # The tail item's coordinates MUST be carried.
+        assert [5, 0, 50, 0, 20] in pos1, f"chunk[1] dropped tail-item coords (prev posE): {pos1}"
+        assert [6, 0, 60, 0, 24] in pos1, f"chunk[1] lost its own coords (posF): {pos1}"
+        # Partial overlap: the head items of the previous chunk must NOT be
+        # carried (that would over-inflate the highlight box).
+        for absent in ([1, 0, 10, 0, 5], [2, 0, 20, 0, 8], [3, 0, 30, 0, 12], [4, 0, 40, 0, 16]):
+            assert absent not in pos1, f"chunk[1] over-carried non-overlap head coords {absent}: {pos1}"
+
+
 def test_json_delimiter_mode_consecutive_delimiter_keeps_boundary():
     # Regression for #17723: "A####B" with pattern "##" must yield ["A", "B"],
     # both boundary-adjacent segments preserved (the bug collapsed it to "A##B").
@@ -438,7 +526,7 @@ def test_text_delimiter_mode_one_no_atom_split():
             "text": "aaa|bbb|ccc",
         }
         chunk_token_size = 1
-        setattr(chunker._param, "chunk_token_size", chunk_token_size)
+        chunker._param.chunk_token_size = chunk_token_size
         asyncio.run(chunker._invoke(**kwargs))
         chunks = chunker._outputs["chunks"]
         texts = [c["text"] for c in chunks]