feat: add LLM-guided semantic rechunking for knowledge compilation (#17546)

rag/advanced_rag/knowlege_compile/_common.py

@@ -57,6 +57,7 @@ def knowledge_compile_gen_conf(chat_mdl, gen_conf: Optional[dict] = None) -> dic
     model_name = str(model_config.get("llm_name") or getattr(chat_mdl, "llm_name", "")).lower()
 
     if "deepseek-v4" in model_name:
+        conf["max_completion_tokens"] = 32768
         extra_body = conf.get("extra_body")
         extra_body = dict(extra_body) if isinstance(extra_body, dict) else {}
         extra_body["thinking"] = {"type": "disabled"}

rag/advanced_rag/knowlege_compile/runner.py

@@ -65,6 +65,9 @@ DOC_STRUCTURE_COMPILE_BATCH_CHUNKS = 4
 # before the LLM call.
 STRUCTURE_CONTEXT_FRACTION = 0.5
 STRUCTURE_DEFAULT_CONTEXT = 100_000
+KNOWLEDGE_GRAPH_CONTEXT_FRACTION = 0.1
+KNOWLEDGE_GRAPH_MIN_BATCH_TOKENS = 2048
+KNOWLEDGE_GRAPH_MAX_BATCH_TOKENS = 4096
 
 # Bound the number of batch/template extraction calls in flight. Results are
 # committed in submission order so accumulator updates and merge flushes stay
@@ -310,7 +313,7 @@ async def run_structure_compile_over_batches(
     # compile_kwd(s) each template actually wrote, harvested from flush results
     # so a dataset-scope template can rebuild its dataset graph once at the end.
     compile_kwds_by_tid: dict[str, set[str]] = {tid: set() for tid, _ in active_templates}
-    agg_infos: dict[str, dict] = {tid: {"inserted": 0, "updated": 0, "duplicates_dropped": 0} for tid, _ in active_templates}
+    agg_infos: dict[str, dict] = {tid: {"inserted": 0, "updated": 0, "duplicates_dropped": 0, "rechunked_chunks": []} for tid, _ in active_templates}
     chunks_by_id: dict[str, str] = {}
     flush_sequence = 0
     flush_tasks: set[asyncio.Task[None]] = set()
@@ -414,6 +417,10 @@ async def run_structure_compile_over_batches(
     async def _commit_result(batch_no: int, batch_len: int, template_id: str, docs: list[dict]) -> None:
         if docs:
             accumulators[template_id].extend(docs)
+        rechunked_chunks = getattr(docs, "rechunked_chunks", None)
+        if rechunked_chunks:
+            known_ids = {chunk.get("id") for chunk in agg_infos[template_id]["rechunked_chunks"]}
+            agg_infos[template_id]["rechunked_chunks"].extend(chunk for chunk in rechunked_chunks if chunk.get("id") not in known_ids)
         if len(accumulators[template_id]) >= DOC_STRUCTURE_MERGE_MAX_DOCS:
             progress_cb(msg=f"  merge flush ({len(accumulators[template_id])} docs) for batch {batch_no} ({batch_len} chunks) for template ({template_ids_by_id[template_id]}/{total})")
             await _flush(template_id)
@@ -428,6 +435,11 @@ async def run_structure_compile_over_batches(
 
     def _dynamic_batch_budget(template_id: str) -> int:
         max_length = getattr(chat_mdl_by_tid[template_id], "max_length", None) or STRUCTURE_DEFAULT_CONTEXT
+        if template_kinds.get(template_id) == "knowledge_graph":
+            return min(
+                max(int(max_length * KNOWLEDGE_GRAPH_CONTEXT_FRACTION), KNOWLEDGE_GRAPH_MIN_BATCH_TOKENS),
+                KNOWLEDGE_GRAPH_MAX_BATCH_TOKENS,
+            )
         return max(int(max_length * STRUCTURE_CONTEXT_FRACTION), 1024)
 
     async def _commit_ready() -> None:
@@ -489,9 +501,6 @@ async def run_structure_compile_over_batches(
                 for template_id, parser_cfg in active_templates:
                     if cancel_check():
                         raise TaskCanceledException("Task was cancelled during document knowledge compilation")
-                    if template_kinds[template_id] == "knowledge_graph":
-                        await _submit_one(incoming_batch, template_id, parser_cfg)
-                        continue
                     buffer = dynamic_buffers[template_id]
                     budget = _dynamic_batch_budget(template_id)
                     buffer_tokens = dynamic_buffer_tokens[template_id]
@@ -622,8 +631,26 @@ async def run_structure_compile_over_batches(
             raise TaskCanceledException("Task was cancelled during document knowledge compilation")
         agg = agg_infos[template_id]
         if record:
-            record(f"document_structure_compile:{template_id}", agg)
-        progress_cb(msg=f"Document knowledge compilation done ({idx + 1}/{total}): {agg}")
+            recorded_agg = {key: value for key, value in agg.items() if key != "rechunked_chunks"}
+            recorded_agg["rechunked_chunk_count"] = len(agg.get("rechunked_chunks") or [])
+            record(f"document_structure_compile:{template_id}", recorded_agg)
+        rechunked_chunks = agg.get("rechunked_chunks") or []
+        if rechunked_chunks:
+            progress_cb(
+                msg=(
+                    f"Rechunk: {len(chunks_by_id)} -> {len(rechunked_chunks)} chunks; "
+                    f"inserted={agg.get('inserted', 0)}, updated={agg.get('updated', 0)}, "
+                    f"duplicates_dropped={agg.get('duplicates_dropped', 0)}"
+                )
+            )
+        else:
+            progress_cb(
+                msg=(
+                    f"Document knowledge compilation done ({idx + 1}/{total}): "
+                    f"inserted={agg.get('inserted', 0)}, updated={agg.get('updated', 0)}, "
+                    f"duplicates_dropped={agg.get('duplicates_dropped', 0)}"
+                )
+            )
 
         # ── Synthesis phase ──────────────────────────────────────────────
         # If the template has synthesis.enabled, run wiki PLAN+REFINE

rag/advanced_rag/knowlege_compile/structure.py

@@ -18,6 +18,8 @@ import asyncio
 import heapq
 import json
 import logging
+import re
+import uuid
 from typing import Awaitable, Callable, Tuple
 
 import xxhash
@@ -68,6 +70,14 @@ _STRUCT_MERGE_LOCK_TIMEOUT_S = 60
 _STRUCT_MERGE_LOCK_BLOCKING_TIMEOUT_S = 5
 
 
+class _RechunkedDocs(list):
+    """Compiled structure rows plus the formal chunks created by rechunking."""
+
+    def __init__(self, docs=None, rechunked_chunks=None):
+        super().__init__(docs or [])
+        self.rechunked_chunks = rechunked_chunks or []
+
+
 def _struct_merge_lock_key(kb_id: str, compilation_template_id: str | None) -> str:
     """Per-(kb, template) lock so different templates can merge in parallel."""
     return f"struct_merge:{kb_id}:{compilation_template_id or ''}"
@@ -280,7 +290,7 @@ def _struct_render_type_fields(fields: list, language: str, *, kind: str) -> Tup
     return "\n".join(lines), skeleton
 
 
-def _struct_hypergraph_prompts(parser_config: dict, language: str = "en") -> Tuple[str, str]:
+def _struct_hypergraph_prompts(parser_config: dict, language: str = "en", rechunk: bool = False) -> Tuple[str, str]:
     autotype = _struct_infer_type(parser_config)
     guideline = _struct_get(parser_config, "guideline", default={}) or {}
     output = _struct_get(parser_config, "output", default={}) or {}
@@ -292,6 +302,7 @@ def _struct_hypergraph_prompts(parser_config: dict, language: str = "en") -> Tup
     rules_r = _struct_localize(_struct_get(guideline, "rules_for_relations"), language)
     rules_t = _struct_localize(_struct_get(guideline, "rules_for_time"), language)
     global_rules = _struct_localize(_struct_get(parser_config, "global_rules"), language)
+    rechunk_rules = _struct_localize(_struct_get(parser_config, "rechunk_rules"), language)
 
     observation_time = _struct_get(options, "observation_time") or datetime.date.today().isoformat()
     if rules_t and "{observation_time}" in rules_t:
@@ -318,12 +329,26 @@ def _struct_hypergraph_prompts(parser_config: dict, language: str = "en") -> Tup
     if ent_desc:
         node_parts.append(f"## Entity Description:\n{ent_desc}")
     node_parts.append(f"## Entity Fields:\n{ent_fields_text}")
-    node_parts.append(
-        "## Response Format:\n"
-        "Reply with a single JSON object of the form: "
-        f'{{"items": [{ent_skel}, ...]}}.\n'
-        f'Auto-type: "{_struct_infer_type(parser_config)}". ' + ("Items must be unique. " if autotype == "set" else "") + "Return JSON only, no commentary."
-    )
+    if rechunk:
+        node_parts.append(
+            "## Semantic Chunking Rules:\n"
+            f"{rechunk_rules}\n\n"
+            "## Response Format:\n"
+            "First group the source chunks according to the rules above. "
+            "Do not return chunk text. Return temporary chunk ids (c1, c2, ...) and the source chunk ids included in each group. "
+            'Use compact inclusive ranges for consecutive source ids, for example ["t1-t3", "t8"]. '
+            "Then extract entities and use only those temporary chunk ids in each entity's source_chunk_ids. "
+            "Reply with a single JSON object of the form: "
+            f'{{"chunks": [{{"id": "c1", "source_chunk_ids": ["source-id", ...]}}], "items": [{ent_skel}, ...]}}.\n'
+            f'Auto-type: "{_struct_infer_type(parser_config)}". ' + ("Items must be unique. " if autotype == "set" else "") + "Return JSON only, no commentary."
+        )
+    else:
+        node_parts.append(
+            "## Response Format:\n"
+            "Reply with a single JSON object of the form: "
+            f'{{"items": [{ent_skel}, ...]}}.\n'
+            f'Auto-type: "{_struct_infer_type(parser_config)}". ' + ("Items must be unique. " if autotype == "set" else "") + "Return JSON only, no commentary."
+        )
     node_prompt = "\n\n".join(node_parts)
 
     if not relations_cfg:
@@ -383,8 +408,38 @@ def _struct_unwrap_items(res) -> list:
     return []
 
 
-async def _struct_extract_hypergraph(text: str, parser_config: dict, chat_mdl, language: str) -> Tuple[list[dict], list[dict]]:
-    node_prompt, edge_prompt_template = _struct_hypergraph_prompts(parser_config, language)
+def _struct_expand_source_chunk_ids(raw_ids, source_texts: dict[str, str]) -> list[str]:
+    """Expand compact positional source IDs such as t1-t3."""
+    if isinstance(raw_ids, str):
+        raw_ids = [raw_ids]
+    if not isinstance(raw_ids, list):
+        return []
+
+    expanded: list[str] = []
+    available = set(source_texts)
+    for raw_id in raw_ids:
+        value = str(raw_id).strip()
+        match = re.fullmatch(r"t(\d+)\s*-\s*t(\d+)", value, flags=re.IGNORECASE)
+        if match:
+            start, end = (int(match.group(1)), int(match.group(2)))
+            step = 1 if start <= end else -1
+            values = [f"t{index}" for index in range(start, end + step, step)]
+        else:
+            values = [value]
+        for chunk_id in values:
+            if chunk_id in available and chunk_id not in expanded:
+                expanded.append(chunk_id)
+    return expanded
+
+
+async def _struct_extract_hypergraph(
+    text: str,
+    parser_config: dict,
+    chat_mdl,
+    language: str,
+    rechunk: bool = False,
+) -> Tuple[list[dict], list[dict], dict[str, str], list[dict]]:
+    node_prompt, edge_prompt_template = _struct_hypergraph_prompts(parser_config, language, rechunk=rechunk)
 
     user_prompt = (
         "## Source Text:\n"
@@ -395,6 +450,56 @@ async def _struct_extract_hypergraph(text: str, parser_config: dict, chat_mdl, l
     )
     node_res = await gen_json(node_prompt, user_prompt, chat_mdl, gen_conf=_knowledge_compile_gen_conf(chat_mdl, {"temperature": 0.1}))
     nodes = _struct_unwrap_items(node_res)
+    chunk_id_map: dict[str, str] = {}
+    rechunked_chunks: list[dict] = []
+    relation_text = text
+
+    if rechunk:
+        source_texts = {match.group(1).strip(): match.group(2).strip() for match in re.finditer(r"\[CHUNK_ID:\s*([^\]]+)\]\n(.*?)\n\[END_CHUNK\]", text, flags=re.DOTALL)}
+        groups = node_res.get("chunks") if isinstance(node_res, dict) else None
+        valid_groups = []
+        claimed_sources: set[str] = set()
+        if isinstance(groups, list):
+            for group in groups:
+                if not isinstance(group, dict):
+                    continue
+                sources = [source for source in _struct_expand_source_chunk_ids(group.get("source_chunk_ids"), source_texts) if source not in claimed_sources]
+                if not sources:
+                    continue
+                claimed_sources.update(sources)
+                temp_id = str(group.get("id") or f"c{len(valid_groups) + 1}").strip()
+                if temp_id in {item[0] for item in valid_groups}:
+                    temp_id = f"c{len(valid_groups) + 1}"
+                valid_groups.append((temp_id, sources))
+
+        for source_id in source_texts:
+            if source_id not in claimed_sources:
+                valid_groups.append((f"c{len(valid_groups) + 1}", [source_id]))
+
+        relation_segments = []
+        temp_to_uuid: dict[str, str] = {}
+        for temp_id, source_ids in valid_groups:
+            new_id = uuid.uuid4().hex
+            temp_to_uuid[temp_id] = new_id
+            for source_id in source_ids:
+                chunk_id_map[source_id] = new_id
+            grouped_text = "\n\n".join(source_texts[source_id] for source_id in source_ids)
+            relation_segments.append(f"[CHUNK_ID: {new_id}]\n{grouped_text}\n[END_CHUNK]")
+            rechunked_chunks.append(
+                {
+                    "id": new_id,
+                    "text": grouped_text,
+                    "source_chunk_ids": source_ids,
+                }
+            )
+        relation_text = "\n\n".join(relation_segments) or text
+
+        for node in nodes:
+            raw_ids = node.get("source_chunk_ids")
+            if isinstance(raw_ids, str):
+                raw_ids = [raw_ids]
+            if isinstance(raw_ids, list):
+                node["source_chunk_ids"] = [temp_to_uuid.get(str(item).strip(), str(item).strip()) for item in raw_ids if temp_to_uuid.get(str(item).strip())]
 
     id_field = _struct_entity_id_field(parser_config)
     known_keys = []
@@ -408,13 +513,31 @@ async def _struct_extract_hypergraph(text: str, parser_config: dict, chat_mdl, l
     known_str = "- " + "\n- ".join(known_keys) if known_keys else "(none)"
 
     if not edge_prompt_template:
-        return nodes, []
+        return nodes, [], chunk_id_map, rechunked_chunks
 
     edge_prompt = edge_prompt_template.replace("{known_nodes}", known_str)
-    edge_res = await gen_json(edge_prompt, user_prompt, chat_mdl, gen_conf=_knowledge_compile_gen_conf(chat_mdl, {"temperature": 0.1}))
+    edge_user_prompt = (
+        user_prompt
+        if not rechunk
+        else (
+            "## Source Text:\n"
+            "Each source chunk is enclosed by [CHUNK_ID: ...] and [END_CHUNK]. "
+            "For every relation, return source_chunk_ids containing only the IDs of chunks that support that relation.\n"
+            f"{relation_text}\n\n## Output (JSON only):"
+        )
+    )
+    edge_res = await gen_json(edge_prompt, edge_user_prompt, chat_mdl, gen_conf=_knowledge_compile_gen_conf(chat_mdl, {"temperature": 0.1}))
     edges = _struct_unwrap_items(edge_res)
 
-    return nodes, edges
+    if rechunk:
+        for edge in edges:
+            raw_ids = edge.get("source_chunk_ids")
+            if isinstance(raw_ids, str):
+                raw_ids = [raw_ids]
+            if isinstance(raw_ids, list):
+                edge["source_chunk_ids"] = [item for item in raw_ids if isinstance(item, str) and item in set(chunk_id_map.values())]
+
+    return nodes, edges, chunk_id_map, rechunked_chunks
 
 
 def _struct_payload_chunk_ids(payload: dict, batch_ids: list) -> list:
@@ -677,7 +800,7 @@ async def _struct_process_batch(
     semaphore,
     compilation_template_id: str | None = None,
     compilation_template_kind: str | None = None,
-) -> list[dict]:
+) -> _RechunkedDocs:
     """Process one packed batch end-to-end (extract → embed → ES docs).
 
     ``packed`` is the per-batch shape produced by
@@ -690,50 +813,57 @@ async def _struct_process_batch(
     embedding work to bound peak concurrency.
     """
     if not packed:
-        return []
+        return _RechunkedDocs()
 
     batch_ids: list = [e["chunk_id"] for e in packed if e.get("chunk_id")]
     batch_segments: list[str] = [f"[CHUNK_ID: {e['chunk_id']}]\n{e['text']}\n[END_CHUNK]" for e in packed if e.get("chunk_id") and isinstance(e.get("text"), str)]
     combined_text = "\n\n".join(batch_segments)
-
     src_field, target_field = _struct_relation_member_fields(parser_config)
+    rechunk = bool(parser_config.get("rechunk"))
 
-    async def _run() -> list[dict]:
+    async def _run() -> _RechunkedDocs:
         # For hypergraph, entity extraction MUST complete before edge extraction
         # within the same batch, because the edge prompt's {known_nodes}
         # placeholder is filled from this batch's extracted nodes — see
         # _struct_extract_hypergraph. Parallelism across batches is fine; the
         # two stages within one batch are strictly sequential.
         try:
-            items, relations = await _struct_extract_hypergraph(combined_text, parser_config, chat_mdl, language)
+            items, relations, chunk_id_map, formal_chunks = await _struct_extract_hypergraph(
+                combined_text,
+                parser_config,
+                chat_mdl,
+                language,
+                rechunk=rechunk,
+            )
         except Exception as e:
             logging.exception(f"compile_structure_from_text: extraction failed for batch {batch_idx}: {e}")
-            return []
+            return _RechunkedDocs()
 
         payloads = items + relations
         kinds = ["entity"] * len(items) + ["relation"] * len(relations)
+        payload_chunk_ids = list(dict.fromkeys(chunk_id_map.values())) if chunk_id_map else batch_ids
         if not payloads:
             if callback:
                 callback((batch_idx + 1) / total, f"{batch_idx + 1}/{total} batches: 0 items")
-            return []
+            return _RechunkedDocs(rechunked_chunks=formal_chunks)
 
         embed_inputs = [_struct_payload_description(p) for p in payloads]
         try:
             embeddings = await _struct_embed(embd_mdl, embed_inputs)
         except Exception as e:
             logging.exception(f"compile_structure_from_text: embedding failed for batch {batch_idx}: {e}")
-            return []
+            return _RechunkedDocs(rechunked_chunks=formal_chunks)
 
         if len(embeddings) != len(payloads):
             logging.error(f"compile_structure_from_text: embedding count mismatch ({len(embeddings)} vs {len(payloads)}) for batch {batch_idx}")
-            return []
+            return _RechunkedDocs(rechunked_chunks=formal_chunks)
 
         docs = [
             _struct_to_doc_storage_doc(
                 payload,
                 autotype,
                 doc_id,
-                _struct_payload_chunk_ids(payload, batch_ids),
+                _struct_payload_chunk_ids(payload, payload_chunk_ids),
                 vec,
                 kind,
                 src_field=src_field,
@@ -747,7 +877,7 @@ async def _struct_process_batch(
         if callback:
             callback((batch_idx + 1) / total, f"{batch_idx + 1}/{total} batches: {len(payloads)} items")
 
-        return docs
+        return _RechunkedDocs(docs, formal_chunks)
 
     if semaphore is not None:
         async with semaphore:
@@ -810,7 +940,8 @@ async def compile_structure_from_text(
         logging.error(f"compile_structure_from_text: unsupported type '{autotype}'")
         return []
 
-    node_prompt, edge_prompt = _struct_hypergraph_prompts(parser_config, language)
+    rechunk = bool(parser_config.get("rechunk"))
+    node_prompt, edge_prompt = _struct_hypergraph_prompts(parser_config, language, rechunk=rechunk)
     prompt_overhead = max(num_tokens_from_string(node_prompt), num_tokens_from_string(edge_prompt))
 
     # ``kind`` for the row stamp follows the template's ``kind`` field if
@@ -847,12 +978,15 @@ async def compile_structure_from_text(
             compilation_template_kind=template_kind,
         )
 
-    def _flatten(per_batch: list) -> list[dict]:
+    def _flatten(per_batch: list) -> _RechunkedDocs:
         out: list[dict] = []
+        formal_chunks: list[dict] = []
         for br in per_batch or []:
-            if br:
-                out.extend(br)
-        return out
+            if br is None:
+                continue
+            out.extend(br)
+            formal_chunks.extend(getattr(br, "rechunked_chunks", []))
+        return _RechunkedDocs(out, formal_chunks)
 
     return await _run_chunked_pipeline(
         packed_batches,

rag/flow/compiler/compiler.py

@@ -14,6 +14,7 @@
 #  limitations under the License.
 import logging
 import random
+import re
 from copy import deepcopy
 from types import SimpleNamespace
 
@@ -26,6 +27,8 @@ from api.db.services.knowledgebase_service import KnowledgebaseService
 from api.db.services.llm_service import LLMBundle
 from api.db.services.task_service import has_canceled
 from common.constants import LLMType
+from common.token_utils import num_tokens_from_string
+from rag.nlp import naive_merge
 from rag.advanced_rag.knowlege_compile.runner import (
     DOC_STRUCTURE_COMPILE_BATCH_CHUNKS,
     load_active_templates,
@@ -58,6 +61,288 @@ class CompilerParam(ProcessParamBase, LLMParam):
 
 class Compiler(ProcessBase, LLM):
     component_name = "Compiler"
+    _PARSER_TEXT_CHUNK_TOKEN_SIZE = 1024
+    _PARSER_CANDIDATE_TOKEN_SIZE = 128
+    _PARSER_MIN_COARSE_CHUNK_TOKEN_SIZE = 256
+    _PARSER_MIN_SPLIT_CHARACTERS = 24
+
+    @classmethod
+    def _split_slumber_level(cls, text: str, delimiters: list[str]) -> list[str]:
+        """Split at boundaries while keeping delimiters with the previous part."""
+        if not text:
+            return []
+        pattern = "|".join(re.escape(delimiter) for delimiter in sorted(delimiters, key=len, reverse=True))
+        if not pattern:
+            return [text]
+
+        parts = []
+        start = 0
+        for match in re.finditer(pattern, text, flags=re.DOTALL):
+            end = match.end()
+            if end - start >= cls._PARSER_MIN_SPLIT_CHARACTERS:
+                parts.append(text[start:end])
+                start = end
+        if start < len(text):
+            parts.append(text[start:])
+
+        # Keep very short delimiter fragments with their neighbour, matching
+        # Chonkie's min_chars behavior instead of producing tiny candidates.
+        merged = []
+        for part in parts:
+            if merged and len(part.strip()) < cls._PARSER_MIN_SPLIT_CHARACTERS:
+                merged[-1] += part
+            else:
+                merged.append(part)
+        return [part for part in merged if part.strip()]
+
+    @classmethod
+    def _slumber_candidates(cls, text: str, level: int = 0, target_token_size: int | None = None) -> list[str]:
+        """Apply paragraph→sentence splitting with token fallback."""
+        target_token_size = target_token_size or cls._PARSER_TEXT_CHUNK_TOKEN_SIZE
+        levels = [
+            ["\n\n", "\r\n", "\n", "\r"],
+            [". ", "! ", "? ", "。", "！", "？", "；", ";"],
+        ]
+        if not text.strip():
+            return []
+        if level >= len(levels):
+            return [part for part in naive_merge(text, target_token_size, "", 0) if part.strip()]
+
+        splits = cls._split_slumber_level(text, levels[level])
+        if len(splits) <= 1 and level < len(levels) - 1:
+            return cls._slumber_candidates(text, level + 1, target_token_size)
+
+        candidates = []
+        for split in splits:
+            if num_tokens_from_string(split) > target_token_size:
+                candidates.extend(cls._slumber_candidates(split, level + 1, target_token_size))
+            else:
+                candidates.append(split)
+        return candidates
+
+    @staticmethod
+    def _is_atomic_json_record(record: dict) -> bool:
+        """Keep structured or position-bound Parser records intact."""
+        doc_type = str(record.get("doc_type_kwd") or "").strip().lower()
+        layout_type = str(record.get("layout_type") or "").strip().lower()
+        text = record.get("text") or record.get("content_with_weight")
+        has_markdown_table = False
+        has_html_table = False
+        has_fenced_code = False
+        if isinstance(text, str):
+            lines = text.splitlines()
+            has_markdown_table = any(index + 1 < len(lines) and re.match(r"^\s*\|", lines[index]) and Compiler._is_markdown_table_separator(lines[index + 1]) for index in range(len(lines)))
+            has_html_table = bool(re.search(r"<table(?:\s|>)", text, flags=re.IGNORECASE))
+            has_fenced_code = bool(re.search(r"^\s*(`{3,}|~{3,})", text, flags=re.MULTILINE))
+        return bool(
+            doc_type in {"table", "image"}
+            or record.get("img_id")
+            or record.get("image")
+            or layout_type in {"figure", "table"}
+            or record.get("bbox")
+            or has_markdown_table
+            or has_html_table
+            or has_fenced_code
+        )
+
+    @staticmethod
+    def _formalize_rechunked_chunks(chunks: list[dict], rechunked_chunks: list[dict], doc_id: str) -> list[dict]:
+        """Replace parser chunks with the semantic groups produced by rechunking."""
+        originals = {str(chunk.get("id")): chunk for chunk in chunks if chunk.get("id")}
+        result = []
+        for grouped in rechunked_chunks:
+            source_ids = [str(value) for value in grouped.get("source_chunk_ids") or []]
+            source = next((originals[source_id] for source_id in source_ids if source_id in originals), None)
+            if source is None:
+                logging.warning(
+                    "Compiler: skip rechunked group %s because none of its source chunks exist",
+                    grouped.get("id"),
+                )
+                continue
+            item = deepcopy(source)
+            item["id"] = str(grouped.get("id"))
+            item["doc_id"] = doc_id
+            item["text"] = grouped.get("text") or ""
+            for key in list(item):
+                if key.startswith("q_") and key.endswith("_vec"):
+                    del item[key]
+            for key in ("content_with_weight", "content_ltks", "content_sm_ltks", "position_int", "page_num_int", "top_int"):
+                item.pop(key, None)
+            result.append(item)
+        return result
+
+    @staticmethod
+    def _is_markdown_table_separator(line: str) -> bool:
+        cells = line.strip().strip("|").split("|")
+        return len(cells) >= 1 and all(re.fullmatch(r"\s*:?-{3,}:?\s*", cell) for cell in cells)
+
+    @classmethod
+    def _split_markdown_blocks(cls, text: str) -> list[tuple[str, str | None]]:
+        """Separate Markdown tables and fenced code blocks as atomic blocks."""
+        lines = text.splitlines(keepends=True)
+        blocks: list[tuple[str, str | None]] = []
+        ordinary: list[str] = []
+        index = 0
+
+        def flush_ordinary() -> None:
+            if ordinary:
+                blocks.append(("".join(ordinary), False))
+                ordinary.clear()
+
+        while index < len(lines):
+            fence_match = re.match(r"^\s*(`{3,}|~{3,})", lines[index])
+            if fence_match:
+                flush_ordinary()
+                fence = fence_match.group(1)
+                fence_char = fence[0]
+                fence_length = len(fence)
+                code = [lines[index]]
+                index += 1
+                while index < len(lines):
+                    code.append(lines[index])
+                    closing = re.match(
+                        rf"^\s*{re.escape(fence_char)}{{{fence_length},}}\s*$",
+                        lines[index].rstrip("\r\n"),
+                    )
+                    index += 1
+                    if closing:
+                        break
+                blocks.append(("".join(code), "fenced_code"))
+                continue
+
+            if index + 1 < len(lines) and re.match(r"^\s*\|", lines[index]) and cls._is_markdown_table_separator(lines[index + 1]):
+                flush_ordinary()
+                table = [lines[index], lines[index + 1]]
+                index += 2
+                while index < len(lines) and re.match(r"^\s*\|", lines[index]):
+                    table.append(lines[index])
+                    index += 1
+                blocks.append(("".join(table), "table"))
+                continue
+            ordinary.append(lines[index])
+            index += 1
+
+        flush_ordinary()
+        return blocks
+
+    @classmethod
+    def _merge_small_text_chunks(cls, chunks: list[dict], target_token_size: int) -> list[dict]:
+        """Merge adjacent ordinary text chunks up to the coarse target."""
+        if target_token_size < cls._PARSER_MIN_COARSE_CHUNK_TOKEN_SIZE:
+            return chunks
+
+        merged = []
+        pending = None
+        for chunk in chunks:
+            text = chunk.get("text")
+            if not isinstance(text, str) or not text.strip() or cls._is_atomic_json_record(chunk):
+                if pending is not None:
+                    merged.append(pending)
+                    pending = None
+                merged.append(chunk)
+                continue
+
+            if pending is None:
+                pending = chunk
+                continue
+
+            combined = f"{pending['text']}\n\n{text}"
+            if num_tokens_from_string(pending["text"]) < cls._PARSER_MIN_COARSE_CHUNK_TOKEN_SIZE and num_tokens_from_string(combined) <= target_token_size:
+                pending["text"] = combined
+            else:
+                merged.append(pending)
+                pending = chunk
+
+        if pending is not None:
+            merged.append(pending)
+        return merged
+
+    @classmethod
+    def _normalize_upstream_chunks(
+        cls,
+        upstream: dict,
+        split_json_text: bool = False,
+        target_token_size: int | None = None,
+    ) -> list[dict]:
+        """Normalize direct Parser output into Compiler chunks.
+
+        Parser JSON is a list of chunk-like records. JSON records are split by
+        the requested token target while retaining their metadata; upstream
+        ``chunks`` preserve atomic table/image records unless rechunking is
+        explicitly requested. Textual Parser outputs use the same split.
+        """
+        output_format = upstream.get("output_format")
+        if output_format == "chunks":
+            chunks = upstream.get("chunks") or []
+            if not isinstance(chunks, list):
+                return []
+            if not split_json_text:
+                normalized = deepcopy(chunks)
+                for chunk in normalized:
+                    if not isinstance(chunk, dict):
+                        continue
+                    text = chunk.get("text")
+                    if not isinstance(text, str):
+                        text = chunk.get("content_with_weight")
+                    chunk["text"] = text if isinstance(text, str) else ""
+                return normalized
+            records = chunks
+        elif output_format == "json":
+            records = upstream.get("json") or upstream.get("json_result") or []
+            if not isinstance(records, list):
+                return []
+        else:
+            records = None
+
+        if records is not None:
+            chunks = []
+            for record in records:
+                if not isinstance(record, dict):
+                    continue
+                chunk = deepcopy(record)
+                text = chunk.get("text")
+                if not isinstance(text, str):
+                    text = chunk.get("content_with_weight")
+                if isinstance(text, str) and text.strip():
+                    if Compiler._is_atomic_json_record(chunk) or not split_json_text:
+                        chunk["text"] = text
+                        chunks.append(chunk)
+                    else:
+                        for part in Compiler._slumber_candidates(text, target_token_size=target_token_size):
+                            split_chunk = deepcopy(chunk)
+                            split_chunk["text"] = part
+                            chunks.append(split_chunk)
+                else:
+                    # Keep metadata-only records (for example, an image
+                    # placeholder) instead of silently dropping them.
+                    chunk["text"] = text if isinstance(text, str) else ""
+                    chunks.append(chunk)
+            if split_json_text:
+                return cls._merge_small_text_chunks(chunks, target_token_size or cls._PARSER_TEXT_CHUNK_TOKEN_SIZE)
+            return chunks
+
+        if output_format in {"markdown", "text", "html"}:
+            text = upstream.get(output_format)
+            if not isinstance(text, str):
+                text = upstream.get(f"{output_format}_result")
+            if not isinstance(text, str) or not text.strip():
+                return []
+            if not split_json_text:
+                return [{"text": text}]
+            if output_format == "markdown":
+                chunks = []
+                for block, block_type in cls._split_markdown_blocks(text):
+                    if block_type == "table":
+                        chunks.append({"text": block, "doc_type_kwd": "table"})
+                    elif block_type == "fenced_code":
+                        chunks.append({"text": block})
+                    else:
+                        chunks.extend({"text": part} for part in cls._slumber_candidates(block, target_token_size=target_token_size) if part.strip())
+            else:
+                chunks = [{"text": part} for part in cls._slumber_candidates(text, target_token_size=target_token_size) if part.strip()]
+            return cls._merge_small_text_chunks(chunks, target_token_size or cls._PARSER_TEXT_CHUNK_TOKEN_SIZE)
+
+        return []
 
     def _compile_progress(self, prog=None, msg=""):
         """Adapt the knowledge-compile ``callback`` protocol to the flow
@@ -210,31 +495,35 @@ class Compiler(ProcessBase, LLM):
         # kwargs. Do not call LLM.get_input_elements() here: it resolves the
         # inherited prompt variables through Canvas.globals, while Pipeline
         # is a Graph and has no globals.
-        chunks = deepcopy(kwargs.get("chunks") or [])
-        if not chunks:
-            for val in kwargs.values():
-                if isinstance(val, list):
-                    chunks = deepcopy(val)
+        output_format = kwargs.get("output_format")
+        if output_format == "chunks":
+            raw_chunks = kwargs.get("chunks") or []
+            if not isinstance(raw_chunks, list) or not raw_chunks:
+                self.set_output("chunks", [])
+                return
+            # Normalize chunks only after the active templates are known, so
+            # rechunk mode does not normalize the same input twice.
+            chunks = None
+        else:
+            chunks = self._normalize_upstream_chunks(kwargs)
 
         tenant_id = self._canvas.get_tenant_id()
         doc_id = self._canvas._doc_id
         kb_id = getattr(self._canvas, "_kb_id", None) or DocumentService.get_knowledgebase_id(doc_id)
         language = self._compile_language(kwargs)
 
-        if not chunks:
+        if chunks is not None and not chunks:
             self.set_output("chunks", chunks)
             return
 
-        for ck in chunks:
-            ck["doc_id"] = doc_id
-            ck["id"] = xxhash.xxh64((ck["text"] + str(ck["doc_id"])).encode("utf-8")).hexdigest()
-
         # Resolve the configured template groups to concrete, active
         # (non-artifact) structure-compilation templates.
         template_ids = resolve_template_ids_from_groups(self._param.compilation_template_group_ids, tenant_id)
         active_templates = load_active_templates(template_ids, tenant_id)
         if not active_templates:
             self.callback(0, "No active compilation templates resolved from the configured groups.")
+            if chunks is None:
+                chunks = self._normalize_upstream_chunks(kwargs)
             self.set_output("chunks", chunks)
             return
 
@@ -279,10 +568,52 @@ class Compiler(ProcessBase, LLM):
             filtered_templates.append((template_id, parser_cfg))
 
         if not filtered_templates:
+            if chunks is None:
+                chunks = self._normalize_upstream_chunks(kwargs)
             self.set_output("chunks", chunks)
             return
         active_templates = filtered_templates
 
+        def _template_requests_rechunk(cfg: dict) -> bool:
+            if not isinstance(cfg, dict):
+                return False
+            # ``raptor.rechunk`` belongs to the Tree path, which is explicitly
+            # excluded from the semantic rechunk staging below. It must not
+            # shrink non-Tree parser chunks unless a structure template has
+            # requested the actual regrouping pass.
+            return bool(cfg.get("rechunk"))
+
+        should_rechunk = any(_template_requests_rechunk(cfg) for _, cfg in active_templates)
+        target_token_size = self._PARSER_CANDIDATE_TOKEN_SIZE if should_rechunk else self._PARSER_TEXT_CHUNK_TOKEN_SIZE
+        if output_format == "chunks":
+            chunks = self._normalize_upstream_chunks(
+                kwargs,
+                split_json_text=should_rechunk,
+                target_token_size=target_token_size,
+            )
+        elif output_format in {"markdown", "text", "html"}:
+            chunks = self._normalize_upstream_chunks(
+                kwargs,
+                split_json_text=should_rechunk,
+                target_token_size=target_token_size,
+            )
+        elif output_format == "json":
+            chunks = self._normalize_upstream_chunks(
+                kwargs,
+                split_json_text=should_rechunk,
+                target_token_size=target_token_size,
+            )
+            if not chunks:
+                self.set_output("chunks", chunks)
+                return
+        if not chunks:
+            self.set_output("chunks", chunks)
+            return
+
+        for ck in chunks:
+            ck["doc_id"] = doc_id
+            ck["id"] = xxhash.xxh64((ck["text"] + str(ck["doc_id"])).encode("utf-8")).hexdigest()
+
         if self._canvas._kb_id:
             e, kb = KnowledgebaseService.get_by_id(self._canvas._kb_id)
             if kb.tenant_embd_id:
@@ -315,6 +646,17 @@ class Compiler(ProcessBase, LLM):
             )
 
         if non_tree_templates:
+            first_rechunk_index = next(
+                (index for index, (_, cfg) in enumerate(non_tree_templates) if _template_requests_rechunk(cfg)),
+                None,
+            )
+            if first_rechunk_index is not None:
+                # Keep the stable hash IDs for Tree. Non-tree rechunking uses
+                # compact positional IDs to keep the LLM grouping response
+                # small (e.g. t1-t3 instead of three long hash IDs).
+                for index, chunk in enumerate(chunks, start=1):
+                    chunk["id"] = f"t{index}"
+
             task_id = getattr(self._canvas, "task_id", None)
 
             def _cancelled() -> bool:
@@ -324,17 +666,48 @@ class Compiler(ProcessBase, LLM):
                 for i in range(0, len(chunks), DOC_STRUCTURE_COMPILE_BATCH_CHUNKS):
                     yield chunks[i : i + DOC_STRUCTURE_COMPILE_BATCH_CHUNKS]
 
-            await run_structure_compile_over_batches(
-                active_templates=non_tree_templates,
-                chat_mdl_by_tid=chat_mdl_by_tid,
-                embedding_model=embedding_model,
-                tenant_id=tenant_id,
-                kb_id=kb_id,
-                doc_id=doc_id,
-                language=language,
-                chunk_batches=_chunk_batches(),
-                progress_cb=self._compile_progress,
-                cancel_check=_cancelled,
-            )
+            async def _run_templates(template_specs):
+                return await run_structure_compile_over_batches(
+                    active_templates=template_specs,
+                    chat_mdl_by_tid=chat_mdl_by_tid,
+                    embedding_model=embedding_model,
+                    tenant_id=tenant_id,
+                    kb_id=kb_id,
+                    doc_id=doc_id,
+                    language=language,
+                    chunk_batches=_chunk_batches(),
+                    progress_cb=self._compile_progress,
+                    cancel_check=_cancelled,
+                )
+
+            if first_rechunk_index is None:
+                await _run_templates(non_tree_templates)
+            else:
+                if first_rechunk_index:
+                    await _run_templates(non_tree_templates[:first_rechunk_index])
+                first_template = [non_tree_templates[first_rechunk_index]]
+                compile_info = await _run_templates(first_template)
+                first_template_id = first_template[0][0]
+                formal_chunks = (compile_info.get(first_template_id) or {}).get("rechunked_chunks")
+                if formal_chunks:
+                    formalized_chunks = self._formalize_rechunked_chunks(chunks, formal_chunks, doc_id)
+                    if formalized_chunks:
+                        chunks = formalized_chunks
+                    else:
+                        logging.warning(
+                            "Compiler: rechunking returned no valid formal chunks for doc %s; keeping original chunks",
+                            doc_id,
+                        )
+
+                # The first rechunk template defines the semantic boundaries
+                # for all later templates. They consume those formal chunks
+                # and must not perform another independent rechunking pass.
+                later_templates = []
+                for template_id, parser_cfg in non_tree_templates[first_rechunk_index + 1 :]:
+                    later_config = dict(parser_cfg or {})
+                    later_config["rechunk"] = False
+                    later_templates.append((template_id, later_config))
+                if later_templates:
+                    await _run_templates(later_templates)
 
         self.set_output("chunks", chunks)

rag/llm/chat_model.py

@@ -1691,8 +1691,30 @@ class LiteLLMBase(ABC):
             gen_conf=gen_conf,
         )
 
-        gen_conf.pop("max_tokens", None)
+        deepseek_max_tokens = None
+        if self.provider == SupportedLiteLLMProvider.DeepSeek:
+            # DeepSeek's API uses the legacy OpenAI-compatible max_tokens field.
+            # LiteLLM accepts max_completion_tokens generically, but does not
+            # translate it for DeepSeek and the provider then falls back to 8192.
+            # Knowledge compilation supplies max_completion_tokens explicitly
+            # through its model-specific generation configuration. Otherwise,
+            # preserve the legacy max_tokens value used by existing callers.
+            raw_max_completion_tokens = gen_conf.pop("max_completion_tokens", None)
+            raw_max_tokens = gen_conf.pop("max_tokens", None)
+            raw_limit = raw_max_completion_tokens if raw_max_completion_tokens is not None else raw_max_tokens
+            if raw_limit is not None and not isinstance(raw_limit, bool):
+                try:
+                    candidate = int(raw_limit)
+                except (TypeError, ValueError):
+                    candidate = 0
+                if candidate > 0:
+                    deepseek_max_tokens = candidate
+        else:
+            gen_conf.pop("max_tokens", None)
+
         gen_conf = {k: v for k, v in gen_conf.items() if k in LITELLM_ALLOWED_GEN_CONF_KEYS}
+        if deepseek_max_tokens is not None:
+            gen_conf["max_tokens"] = deepseek_max_tokens
         return gen_conf
 
     def _need_reasoning_content_back(self) -> bool:

rag/svr/task_executor_refactor/chunk_post_processor.py

@@ -378,8 +378,6 @@ from rag.advanced_rag.knowlege_compile.runner import (  # noqa: E402
     load_active_templates,
     run_structure_compile_over_batches,
 )
-
-
 # ----- parser_config helpers -----------------------------------------