From 44e13d1cb6ec8e53b3c76d69e27253d0ac52a3eb Mon Sep 17 00:00:00 2001 From: buua436 Date: Fri, 31 Jul 2026 13:33:30 +0800 Subject: [PATCH] feat: add LLM-guided semantic rechunking for knowledge compilation (#17546) --- .../knowledge_graph.yaml | 6 + .../compilation_templates/mind_map.yaml | 6 + .../compilation_templates/page_index.yaml | 8 + .../session_essence.yaml | 6 + .../compilation_templates/session_graph.yaml | 6 + .../compilation_templates/timeline.yaml | 6 + rag/advanced_rag/knowlege_compile/_common.py | 1 + rag/advanced_rag/knowlege_compile/runner.py | 39 +- .../knowlege_compile/structure.py | 190 ++++++-- rag/flow/compiler/compiler.py | 417 +++++++++++++++++- rag/llm/chat_model.py | 24 +- .../chunk_post_processor.py | 2 - .../database/compilation-template.ts | 2 + .../request/compilation-template.ts | 2 + web/src/locales/en.ts | 6 + web/src/locales/zh.ts | 5 + .../components/template-configuration.tsx | 84 ++-- .../create-next/utils.ts | 25 +- .../components/template-configuration.tsx | 84 ++-- .../compilation-templates/edit-next/utils.ts | 25 +- 20 files changed, 827 insertions(+), 117 deletions(-) diff --git a/api/db/init_data/compilation_templates/knowledge_graph.yaml b/api/db/init_data/compilation_templates/knowledge_graph.yaml index 27a8535dd1..c7d4386af8 100644 --- a/api/db/init_data/compilation_templates/knowledge_graph.yaml +++ b/api/db/init_data/compilation_templates/knowledge_graph.yaml @@ -2,6 +2,12 @@ kind: knowledge_graph display_name: Knowledge graph config: kind: knowledge_graph + rechunk_rules: | + - Group only adjacent source chunks into coherent semantic units for this knowledge-compilation task. + - Preserve source order: use every source chunk exactly once; do not omit, overlap, duplicate, or reorder chunks. + - Split when the topic, section, or semantic focus changes; do not merge unrelated content. + - Keep self-contained tables, lists, figures, and code blocks intact; do not split their internal structure. + - Do not invent, rewrite, or return source text; return only chunk grouping metadata. entity: description: >- You are a robust graph entity extractor for knowledge graphs. diff --git a/api/db/init_data/compilation_templates/mind_map.yaml b/api/db/init_data/compilation_templates/mind_map.yaml index 383be48e6e..136d79b44e 100644 --- a/api/db/init_data/compilation_templates/mind_map.yaml +++ b/api/db/init_data/compilation_templates/mind_map.yaml @@ -2,6 +2,12 @@ kind: mind_map display_name: Mind map - Radial concept hierarchy config: kind: mind_map + rechunk_rules: | + - Group only adjacent source chunks into coherent semantic units for this knowledge-compilation task. + - Preserve source order: use every source chunk exactly once; do not omit, overlap, duplicate, or reorder chunks. + - Keep each main idea with its supporting content, and split when the topic or branch changes. + - Keep self-contained tables, lists, figures, and code blocks intact; do not split their internal structure. + - Do not invent, rewrite, or return source text; return only chunk grouping metadata. entity: description: >- You are a robust mind-map extractor. Extract the central concept, diff --git a/api/db/init_data/compilation_templates/page_index.yaml b/api/db/init_data/compilation_templates/page_index.yaml index f17519daf1..18213b5f00 100644 --- a/api/db/init_data/compilation_templates/page_index.yaml +++ b/api/db/init_data/compilation_templates/page_index.yaml @@ -2,6 +2,14 @@ kind: page_index display_name: PageIndex — Hierarchical table of contents config: kind: page_index + rechunk_rules: | + - Group only adjacent source chunks into coherent semantic units for this page-index task. + - Preserve source order: use every source chunk exactly once; do not omit, overlap, duplicate, or reorder chunks. + - Keep each heading with the content it introduces, and split when the topic, section, or semantic focus changes. + - Preserve document heading hierarchy: keep a parent heading with its introductory content and nearby child-section context. + - Do not combine separate sections merely because their subjects are related; keep each section boundary recoverable. + - Keep self-contained tables, lists, figures, and code blocks intact; do not split their internal rows or items. + - Do not merge unrelated content. Do not invent, rewrite, or return source text; return only chunk grouping metadata. entity: description: >- You are a source-grounded document index extractor. First extract every diff --git a/api/db/init_data/compilation_templates/session_essence.yaml b/api/db/init_data/compilation_templates/session_essence.yaml index dfe7bb6888..62ddef4e4a 100644 --- a/api/db/init_data/compilation_templates/session_essence.yaml +++ b/api/db/init_data/compilation_templates/session_essence.yaml @@ -2,6 +2,12 @@ kind: session_essence display_name: Session Essence — Cross-source entity synopses from conversations config: kind: knowledge_graph + rechunk_rules: | + - Group only adjacent source chunks that discuss the same conversation topic or entity. + - Preserve source order: use every source chunk exactly once; do not omit, overlap, duplicate, or reorder chunks. + - Split when the speaker, topic, intent, or semantic focus changes. + - Keep self-contained tables, lists, figures, and code blocks intact; do not split their internal structure. + - Do not invent, rewrite, or return source text; return only chunk grouping metadata. entity: description: >- You are a robust entity and fact extractor for conversational data. diff --git a/api/db/init_data/compilation_templates/session_graph.yaml b/api/db/init_data/compilation_templates/session_graph.yaml index 3bbdb1efb5..d9dd23daf4 100644 --- a/api/db/init_data/compilation_templates/session_graph.yaml +++ b/api/db/init_data/compilation_templates/session_graph.yaml @@ -2,6 +2,12 @@ kind: session_graph display_name: Session Graph — Knowledge graph from conversations config: kind: knowledge_graph + rechunk_rules: | + - Group only adjacent source chunks that discuss the same conversation topic or entity. + - Preserve source order: use every source chunk exactly once; do not omit, overlap, duplicate, or reorder chunks. + - Split when the speaker, topic, intent, or semantic focus changes. + - Keep self-contained tables, lists, figures, and code blocks intact; do not split their internal structure. + - Do not invent, rewrite, or return source text; return only chunk grouping metadata. entity: description: >- You are a robust entity and fact extractor for conversational data. diff --git a/api/db/init_data/compilation_templates/timeline.yaml b/api/db/init_data/compilation_templates/timeline.yaml index 00bee1485d..728603a315 100644 --- a/api/db/init_data/compilation_templates/timeline.yaml +++ b/api/db/init_data/compilation_templates/timeline.yaml @@ -2,6 +2,12 @@ kind: timeline display_name: List (Timeline) — Chronological events / Graph config: kind: timeline + rechunk_rules: | + - Group only adjacent source chunks that belong to the same event or continuous time period. + - Preserve source order: use every source chunk exactly once; do not omit, overlap, duplicate, or reorder chunks. + - Split when the event, time period, or semantic focus changes. + - Keep self-contained tables, lists, figures, and code blocks intact; do not split their internal structure. + - Do not invent, rewrite, or return source text; return only chunk grouping metadata. entity: description: >- You are a robust events-timeline extractor. diff --git a/rag/advanced_rag/knowlege_compile/_common.py b/rag/advanced_rag/knowlege_compile/_common.py index f426b61071..77393d6427 100644 --- a/rag/advanced_rag/knowlege_compile/_common.py +++ b/rag/advanced_rag/knowlege_compile/_common.py @@ -57,6 +57,7 @@ def knowledge_compile_gen_conf(chat_mdl, gen_conf: Optional[dict] = None) -> dic model_name = str(model_config.get("llm_name") or getattr(chat_mdl, "llm_name", "")).lower() if "deepseek-v4" in model_name: + conf["max_completion_tokens"] = 32768 extra_body = conf.get("extra_body") extra_body = dict(extra_body) if isinstance(extra_body, dict) else {} extra_body["thinking"] = {"type": "disabled"} diff --git a/rag/advanced_rag/knowlege_compile/runner.py b/rag/advanced_rag/knowlege_compile/runner.py index 61d37cf6de..3f855b5d5c 100644 --- a/rag/advanced_rag/knowlege_compile/runner.py +++ b/rag/advanced_rag/knowlege_compile/runner.py @@ -65,6 +65,9 @@ DOC_STRUCTURE_COMPILE_BATCH_CHUNKS = 4 # before the LLM call. STRUCTURE_CONTEXT_FRACTION = 0.5 STRUCTURE_DEFAULT_CONTEXT = 100_000 +KNOWLEDGE_GRAPH_CONTEXT_FRACTION = 0.1 +KNOWLEDGE_GRAPH_MIN_BATCH_TOKENS = 2048 +KNOWLEDGE_GRAPH_MAX_BATCH_TOKENS = 4096 # Bound the number of batch/template extraction calls in flight. Results are # committed in submission order so accumulator updates and merge flushes stay @@ -310,7 +313,7 @@ async def run_structure_compile_over_batches( # compile_kwd(s) each template actually wrote, harvested from flush results # so a dataset-scope template can rebuild its dataset graph once at the end. compile_kwds_by_tid: dict[str, set[str]] = {tid: set() for tid, _ in active_templates} - agg_infos: dict[str, dict] = {tid: {"inserted": 0, "updated": 0, "duplicates_dropped": 0} for tid, _ in active_templates} + agg_infos: dict[str, dict] = {tid: {"inserted": 0, "updated": 0, "duplicates_dropped": 0, "rechunked_chunks": []} for tid, _ in active_templates} chunks_by_id: dict[str, str] = {} flush_sequence = 0 flush_tasks: set[asyncio.Task[None]] = set() @@ -414,6 +417,10 @@ async def run_structure_compile_over_batches( async def _commit_result(batch_no: int, batch_len: int, template_id: str, docs: list[dict]) -> None: if docs: accumulators[template_id].extend(docs) + rechunked_chunks = getattr(docs, "rechunked_chunks", None) + if rechunked_chunks: + known_ids = {chunk.get("id") for chunk in agg_infos[template_id]["rechunked_chunks"]} + agg_infos[template_id]["rechunked_chunks"].extend(chunk for chunk in rechunked_chunks if chunk.get("id") not in known_ids) if len(accumulators[template_id]) >= DOC_STRUCTURE_MERGE_MAX_DOCS: progress_cb(msg=f" merge flush ({len(accumulators[template_id])} docs) for batch {batch_no} ({batch_len} chunks) for template ({template_ids_by_id[template_id]}/{total})") await _flush(template_id) @@ -428,6 +435,11 @@ async def run_structure_compile_over_batches( def _dynamic_batch_budget(template_id: str) -> int: max_length = getattr(chat_mdl_by_tid[template_id], "max_length", None) or STRUCTURE_DEFAULT_CONTEXT + if template_kinds.get(template_id) == "knowledge_graph": + return min( + max(int(max_length * KNOWLEDGE_GRAPH_CONTEXT_FRACTION), KNOWLEDGE_GRAPH_MIN_BATCH_TOKENS), + KNOWLEDGE_GRAPH_MAX_BATCH_TOKENS, + ) return max(int(max_length * STRUCTURE_CONTEXT_FRACTION), 1024) async def _commit_ready() -> None: @@ -489,9 +501,6 @@ async def run_structure_compile_over_batches( for template_id, parser_cfg in active_templates: if cancel_check(): raise TaskCanceledException("Task was cancelled during document knowledge compilation") - if template_kinds[template_id] == "knowledge_graph": - await _submit_one(incoming_batch, template_id, parser_cfg) - continue buffer = dynamic_buffers[template_id] budget = _dynamic_batch_budget(template_id) buffer_tokens = dynamic_buffer_tokens[template_id] @@ -622,8 +631,26 @@ async def run_structure_compile_over_batches( raise TaskCanceledException("Task was cancelled during document knowledge compilation") agg = agg_infos[template_id] if record: - record(f"document_structure_compile:{template_id}", agg) - progress_cb(msg=f"Document knowledge compilation done ({idx + 1}/{total}): {agg}") + recorded_agg = {key: value for key, value in agg.items() if key != "rechunked_chunks"} + recorded_agg["rechunked_chunk_count"] = len(agg.get("rechunked_chunks") or []) + record(f"document_structure_compile:{template_id}", recorded_agg) + rechunked_chunks = agg.get("rechunked_chunks") or [] + if rechunked_chunks: + progress_cb( + msg=( + f"Rechunk: {len(chunks_by_id)} -> {len(rechunked_chunks)} chunks; " + f"inserted={agg.get('inserted', 0)}, updated={agg.get('updated', 0)}, " + f"duplicates_dropped={agg.get('duplicates_dropped', 0)}" + ) + ) + else: + progress_cb( + msg=( + f"Document knowledge compilation done ({idx + 1}/{total}): " + f"inserted={agg.get('inserted', 0)}, updated={agg.get('updated', 0)}, " + f"duplicates_dropped={agg.get('duplicates_dropped', 0)}" + ) + ) # ── Synthesis phase ────────────────────────────────────────────── # If the template has synthesis.enabled, run wiki PLAN+REFINE diff --git a/rag/advanced_rag/knowlege_compile/structure.py b/rag/advanced_rag/knowlege_compile/structure.py index bc45452bf7..66385c1aee 100644 --- a/rag/advanced_rag/knowlege_compile/structure.py +++ b/rag/advanced_rag/knowlege_compile/structure.py @@ -18,6 +18,8 @@ import asyncio import heapq import json import logging +import re +import uuid from typing import Awaitable, Callable, Tuple import xxhash @@ -68,6 +70,14 @@ _STRUCT_MERGE_LOCK_TIMEOUT_S = 60 _STRUCT_MERGE_LOCK_BLOCKING_TIMEOUT_S = 5 +class _RechunkedDocs(list): + """Compiled structure rows plus the formal chunks created by rechunking.""" + + def __init__(self, docs=None, rechunked_chunks=None): + super().__init__(docs or []) + self.rechunked_chunks = rechunked_chunks or [] + + def _struct_merge_lock_key(kb_id: str, compilation_template_id: str | None) -> str: """Per-(kb, template) lock so different templates can merge in parallel.""" return f"struct_merge:{kb_id}:{compilation_template_id or ''}" @@ -280,7 +290,7 @@ def _struct_render_type_fields(fields: list, language: str, *, kind: str) -> Tup return "\n".join(lines), skeleton -def _struct_hypergraph_prompts(parser_config: dict, language: str = "en") -> Tuple[str, str]: +def _struct_hypergraph_prompts(parser_config: dict, language: str = "en", rechunk: bool = False) -> Tuple[str, str]: autotype = _struct_infer_type(parser_config) guideline = _struct_get(parser_config, "guideline", default={}) or {} output = _struct_get(parser_config, "output", default={}) or {} @@ -292,6 +302,7 @@ def _struct_hypergraph_prompts(parser_config: dict, language: str = "en") -> Tup rules_r = _struct_localize(_struct_get(guideline, "rules_for_relations"), language) rules_t = _struct_localize(_struct_get(guideline, "rules_for_time"), language) global_rules = _struct_localize(_struct_get(parser_config, "global_rules"), language) + rechunk_rules = _struct_localize(_struct_get(parser_config, "rechunk_rules"), language) observation_time = _struct_get(options, "observation_time") or datetime.date.today().isoformat() if rules_t and "{observation_time}" in rules_t: @@ -318,12 +329,26 @@ def _struct_hypergraph_prompts(parser_config: dict, language: str = "en") -> Tup if ent_desc: node_parts.append(f"## Entity Description:\n{ent_desc}") node_parts.append(f"## Entity Fields:\n{ent_fields_text}") - node_parts.append( - "## Response Format:\n" - "Reply with a single JSON object of the form: " - f'{{"items": [{ent_skel}, ...]}}.\n' - f'Auto-type: "{_struct_infer_type(parser_config)}". ' + ("Items must be unique. " if autotype == "set" else "") + "Return JSON only, no commentary." - ) + if rechunk: + node_parts.append( + "## Semantic Chunking Rules:\n" + f"{rechunk_rules}\n\n" + "## Response Format:\n" + "First group the source chunks according to the rules above. " + "Do not return chunk text. Return temporary chunk ids (c1, c2, ...) and the source chunk ids included in each group. " + 'Use compact inclusive ranges for consecutive source ids, for example ["t1-t3", "t8"]. ' + "Then extract entities and use only those temporary chunk ids in each entity's source_chunk_ids. " + "Reply with a single JSON object of the form: " + f'{{"chunks": [{{"id": "c1", "source_chunk_ids": ["source-id", ...]}}], "items": [{ent_skel}, ...]}}.\n' + f'Auto-type: "{_struct_infer_type(parser_config)}". ' + ("Items must be unique. " if autotype == "set" else "") + "Return JSON only, no commentary." + ) + else: + node_parts.append( + "## Response Format:\n" + "Reply with a single JSON object of the form: " + f'{{"items": [{ent_skel}, ...]}}.\n' + f'Auto-type: "{_struct_infer_type(parser_config)}". ' + ("Items must be unique. " if autotype == "set" else "") + "Return JSON only, no commentary." + ) node_prompt = "\n\n".join(node_parts) if not relations_cfg: @@ -383,8 +408,38 @@ def _struct_unwrap_items(res) -> list: return [] -async def _struct_extract_hypergraph(text: str, parser_config: dict, chat_mdl, language: str) -> Tuple[list[dict], list[dict]]: - node_prompt, edge_prompt_template = _struct_hypergraph_prompts(parser_config, language) +def _struct_expand_source_chunk_ids(raw_ids, source_texts: dict[str, str]) -> list[str]: + """Expand compact positional source IDs such as t1-t3.""" + if isinstance(raw_ids, str): + raw_ids = [raw_ids] + if not isinstance(raw_ids, list): + return [] + + expanded: list[str] = [] + available = set(source_texts) + for raw_id in raw_ids: + value = str(raw_id).strip() + match = re.fullmatch(r"t(\d+)\s*-\s*t(\d+)", value, flags=re.IGNORECASE) + if match: + start, end = (int(match.group(1)), int(match.group(2))) + step = 1 if start <= end else -1 + values = [f"t{index}" for index in range(start, end + step, step)] + else: + values = [value] + for chunk_id in values: + if chunk_id in available and chunk_id not in expanded: + expanded.append(chunk_id) + return expanded + + +async def _struct_extract_hypergraph( + text: str, + parser_config: dict, + chat_mdl, + language: str, + rechunk: bool = False, +) -> Tuple[list[dict], list[dict], dict[str, str], list[dict]]: + node_prompt, edge_prompt_template = _struct_hypergraph_prompts(parser_config, language, rechunk=rechunk) user_prompt = ( "## Source Text:\n" @@ -395,6 +450,56 @@ async def _struct_extract_hypergraph(text: str, parser_config: dict, chat_mdl, l ) node_res = await gen_json(node_prompt, user_prompt, chat_mdl, gen_conf=_knowledge_compile_gen_conf(chat_mdl, {"temperature": 0.1})) nodes = _struct_unwrap_items(node_res) + chunk_id_map: dict[str, str] = {} + rechunked_chunks: list[dict] = [] + relation_text = text + + if rechunk: + source_texts = {match.group(1).strip(): match.group(2).strip() for match in re.finditer(r"\[CHUNK_ID:\s*([^\]]+)\]\n(.*?)\n\[END_CHUNK\]", text, flags=re.DOTALL)} + groups = node_res.get("chunks") if isinstance(node_res, dict) else None + valid_groups = [] + claimed_sources: set[str] = set() + if isinstance(groups, list): + for group in groups: + if not isinstance(group, dict): + continue + sources = [source for source in _struct_expand_source_chunk_ids(group.get("source_chunk_ids"), source_texts) if source not in claimed_sources] + if not sources: + continue + claimed_sources.update(sources) + temp_id = str(group.get("id") or f"c{len(valid_groups) + 1}").strip() + if temp_id in {item[0] for item in valid_groups}: + temp_id = f"c{len(valid_groups) + 1}" + valid_groups.append((temp_id, sources)) + + for source_id in source_texts: + if source_id not in claimed_sources: + valid_groups.append((f"c{len(valid_groups) + 1}", [source_id])) + + relation_segments = [] + temp_to_uuid: dict[str, str] = {} + for temp_id, source_ids in valid_groups: + new_id = uuid.uuid4().hex + temp_to_uuid[temp_id] = new_id + for source_id in source_ids: + chunk_id_map[source_id] = new_id + grouped_text = "\n\n".join(source_texts[source_id] for source_id in source_ids) + relation_segments.append(f"[CHUNK_ID: {new_id}]\n{grouped_text}\n[END_CHUNK]") + rechunked_chunks.append( + { + "id": new_id, + "text": grouped_text, + "source_chunk_ids": source_ids, + } + ) + relation_text = "\n\n".join(relation_segments) or text + + for node in nodes: + raw_ids = node.get("source_chunk_ids") + if isinstance(raw_ids, str): + raw_ids = [raw_ids] + if isinstance(raw_ids, list): + node["source_chunk_ids"] = [temp_to_uuid.get(str(item).strip(), str(item).strip()) for item in raw_ids if temp_to_uuid.get(str(item).strip())] id_field = _struct_entity_id_field(parser_config) known_keys = [] @@ -408,13 +513,31 @@ async def _struct_extract_hypergraph(text: str, parser_config: dict, chat_mdl, l known_str = "- " + "\n- ".join(known_keys) if known_keys else "(none)" if not edge_prompt_template: - return nodes, [] + return nodes, [], chunk_id_map, rechunked_chunks edge_prompt = edge_prompt_template.replace("{known_nodes}", known_str) - edge_res = await gen_json(edge_prompt, user_prompt, chat_mdl, gen_conf=_knowledge_compile_gen_conf(chat_mdl, {"temperature": 0.1})) + edge_user_prompt = ( + user_prompt + if not rechunk + else ( + "## Source Text:\n" + "Each source chunk is enclosed by [CHUNK_ID: ...] and [END_CHUNK]. " + "For every relation, return source_chunk_ids containing only the IDs of chunks that support that relation.\n" + f"{relation_text}\n\n## Output (JSON only):" + ) + ) + edge_res = await gen_json(edge_prompt, edge_user_prompt, chat_mdl, gen_conf=_knowledge_compile_gen_conf(chat_mdl, {"temperature": 0.1})) edges = _struct_unwrap_items(edge_res) - return nodes, edges + if rechunk: + for edge in edges: + raw_ids = edge.get("source_chunk_ids") + if isinstance(raw_ids, str): + raw_ids = [raw_ids] + if isinstance(raw_ids, list): + edge["source_chunk_ids"] = [item for item in raw_ids if isinstance(item, str) and item in set(chunk_id_map.values())] + + return nodes, edges, chunk_id_map, rechunked_chunks def _struct_payload_chunk_ids(payload: dict, batch_ids: list) -> list: @@ -677,7 +800,7 @@ async def _struct_process_batch( semaphore, compilation_template_id: str | None = None, compilation_template_kind: str | None = None, -) -> list[dict]: +) -> _RechunkedDocs: """Process one packed batch end-to-end (extract → embed → ES docs). ``packed`` is the per-batch shape produced by @@ -690,50 +813,57 @@ async def _struct_process_batch( embedding work to bound peak concurrency. """ if not packed: - return [] + return _RechunkedDocs() batch_ids: list = [e["chunk_id"] for e in packed if e.get("chunk_id")] batch_segments: list[str] = [f"[CHUNK_ID: {e['chunk_id']}]\n{e['text']}\n[END_CHUNK]" for e in packed if e.get("chunk_id") and isinstance(e.get("text"), str)] combined_text = "\n\n".join(batch_segments) - src_field, target_field = _struct_relation_member_fields(parser_config) + rechunk = bool(parser_config.get("rechunk")) - async def _run() -> list[dict]: + async def _run() -> _RechunkedDocs: # For hypergraph, entity extraction MUST complete before edge extraction # within the same batch, because the edge prompt's {known_nodes} # placeholder is filled from this batch's extracted nodes — see # _struct_extract_hypergraph. Parallelism across batches is fine; the # two stages within one batch are strictly sequential. try: - items, relations = await _struct_extract_hypergraph(combined_text, parser_config, chat_mdl, language) + items, relations, chunk_id_map, formal_chunks = await _struct_extract_hypergraph( + combined_text, + parser_config, + chat_mdl, + language, + rechunk=rechunk, + ) except Exception as e: logging.exception(f"compile_structure_from_text: extraction failed for batch {batch_idx}: {e}") - return [] + return _RechunkedDocs() payloads = items + relations kinds = ["entity"] * len(items) + ["relation"] * len(relations) + payload_chunk_ids = list(dict.fromkeys(chunk_id_map.values())) if chunk_id_map else batch_ids if not payloads: if callback: callback((batch_idx + 1) / total, f"{batch_idx + 1}/{total} batches: 0 items") - return [] + return _RechunkedDocs(rechunked_chunks=formal_chunks) embed_inputs = [_struct_payload_description(p) for p in payloads] try: embeddings = await _struct_embed(embd_mdl, embed_inputs) except Exception as e: logging.exception(f"compile_structure_from_text: embedding failed for batch {batch_idx}: {e}") - return [] + return _RechunkedDocs(rechunked_chunks=formal_chunks) if len(embeddings) != len(payloads): logging.error(f"compile_structure_from_text: embedding count mismatch ({len(embeddings)} vs {len(payloads)}) for batch {batch_idx}") - return [] + return _RechunkedDocs(rechunked_chunks=formal_chunks) docs = [ _struct_to_doc_storage_doc( payload, autotype, doc_id, - _struct_payload_chunk_ids(payload, batch_ids), + _struct_payload_chunk_ids(payload, payload_chunk_ids), vec, kind, src_field=src_field, @@ -747,7 +877,7 @@ async def _struct_process_batch( if callback: callback((batch_idx + 1) / total, f"{batch_idx + 1}/{total} batches: {len(payloads)} items") - return docs + return _RechunkedDocs(docs, formal_chunks) if semaphore is not None: async with semaphore: @@ -810,7 +940,8 @@ async def compile_structure_from_text( logging.error(f"compile_structure_from_text: unsupported type '{autotype}'") return [] - node_prompt, edge_prompt = _struct_hypergraph_prompts(parser_config, language) + rechunk = bool(parser_config.get("rechunk")) + node_prompt, edge_prompt = _struct_hypergraph_prompts(parser_config, language, rechunk=rechunk) prompt_overhead = max(num_tokens_from_string(node_prompt), num_tokens_from_string(edge_prompt)) # ``kind`` for the row stamp follows the template's ``kind`` field if @@ -847,12 +978,15 @@ async def compile_structure_from_text( compilation_template_kind=template_kind, ) - def _flatten(per_batch: list) -> list[dict]: + def _flatten(per_batch: list) -> _RechunkedDocs: out: list[dict] = [] + formal_chunks: list[dict] = [] for br in per_batch or []: - if br: - out.extend(br) - return out + if br is None: + continue + out.extend(br) + formal_chunks.extend(getattr(br, "rechunked_chunks", [])) + return _RechunkedDocs(out, formal_chunks) return await _run_chunked_pipeline( packed_batches, diff --git a/rag/flow/compiler/compiler.py b/rag/flow/compiler/compiler.py index b0dec7a1d4..eb1bbfb078 100644 --- a/rag/flow/compiler/compiler.py +++ b/rag/flow/compiler/compiler.py @@ -14,6 +14,7 @@ # limitations under the License. import logging import random +import re from copy import deepcopy from types import SimpleNamespace @@ -26,6 +27,8 @@ from api.db.services.knowledgebase_service import KnowledgebaseService from api.db.services.llm_service import LLMBundle from api.db.services.task_service import has_canceled from common.constants import LLMType +from common.token_utils import num_tokens_from_string +from rag.nlp import naive_merge from rag.advanced_rag.knowlege_compile.runner import ( DOC_STRUCTURE_COMPILE_BATCH_CHUNKS, load_active_templates, @@ -58,6 +61,288 @@ class CompilerParam(ProcessParamBase, LLMParam): class Compiler(ProcessBase, LLM): component_name = "Compiler" + _PARSER_TEXT_CHUNK_TOKEN_SIZE = 1024 + _PARSER_CANDIDATE_TOKEN_SIZE = 128 + _PARSER_MIN_COARSE_CHUNK_TOKEN_SIZE = 256 + _PARSER_MIN_SPLIT_CHARACTERS = 24 + + @classmethod + def _split_slumber_level(cls, text: str, delimiters: list[str]) -> list[str]: + """Split at boundaries while keeping delimiters with the previous part.""" + if not text: + return [] + pattern = "|".join(re.escape(delimiter) for delimiter in sorted(delimiters, key=len, reverse=True)) + if not pattern: + return [text] + + parts = [] + start = 0 + for match in re.finditer(pattern, text, flags=re.DOTALL): + end = match.end() + if end - start >= cls._PARSER_MIN_SPLIT_CHARACTERS: + parts.append(text[start:end]) + start = end + if start < len(text): + parts.append(text[start:]) + + # Keep very short delimiter fragments with their neighbour, matching + # Chonkie's min_chars behavior instead of producing tiny candidates. + merged = [] + for part in parts: + if merged and len(part.strip()) < cls._PARSER_MIN_SPLIT_CHARACTERS: + merged[-1] += part + else: + merged.append(part) + return [part for part in merged if part.strip()] + + @classmethod + def _slumber_candidates(cls, text: str, level: int = 0, target_token_size: int | None = None) -> list[str]: + """Apply paragraph→sentence splitting with token fallback.""" + target_token_size = target_token_size or cls._PARSER_TEXT_CHUNK_TOKEN_SIZE + levels = [ + ["\n\n", "\r\n", "\n", "\r"], + [". ", "! ", "? ", "。", "！", "？", "；", ";"], + ] + if not text.strip(): + return [] + if level >= len(levels): + return [part for part in naive_merge(text, target_token_size, "", 0) if part.strip()] + + splits = cls._split_slumber_level(text, levels[level]) + if len(splits) <= 1 and level < len(levels) - 1: + return cls._slumber_candidates(text, level + 1, target_token_size) + + candidates = [] + for split in splits: + if num_tokens_from_string(split) > target_token_size: + candidates.extend(cls._slumber_candidates(split, level + 1, target_token_size)) + else: + candidates.append(split) + return candidates + + @staticmethod + def _is_atomic_json_record(record: dict) -> bool: + """Keep structured or position-bound Parser records intact.""" + doc_type = str(record.get("doc_type_kwd") or "").strip().lower() + layout_type = str(record.get("layout_type") or "").strip().lower() + text = record.get("text") or record.get("content_with_weight") + has_markdown_table = False + has_html_table = False + has_fenced_code = False + if isinstance(text, str): + lines = text.splitlines() + has_markdown_table = any(index + 1 < len(lines) and re.match(r"^\s*\|", lines[index]) and Compiler._is_markdown_table_separator(lines[index + 1]) for index in range(len(lines))) + has_html_table = bool(re.search(r")", text, flags=re.IGNORECASE)) + has_fenced_code = bool(re.search(r"^\s*(`{3,}|~{3,})", text, flags=re.MULTILINE)) + return bool( + doc_type in {"table", "image"} + or record.get("img_id") + or record.get("image") + or layout_type in {"figure", "table"} + or record.get("bbox") + or has_markdown_table + or has_html_table + or has_fenced_code + ) + + @staticmethod + def _formalize_rechunked_chunks(chunks: list[dict], rechunked_chunks: list[dict], doc_id: str) -> list[dict]: + """Replace parser chunks with the semantic groups produced by rechunking.""" + originals = {str(chunk.get("id")): chunk for chunk in chunks if chunk.get("id")} + result = [] + for grouped in rechunked_chunks: + source_ids = [str(value) for value in grouped.get("source_chunk_ids") or []] + source = next((originals[source_id] for source_id in source_ids if source_id in originals), None) + if source is None: + logging.warning( + "Compiler: skip rechunked group %s because none of its source chunks exist", + grouped.get("id"), + ) + continue + item = deepcopy(source) + item["id"] = str(grouped.get("id")) + item["doc_id"] = doc_id + item["text"] = grouped.get("text") or "" + for key in list(item): + if key.startswith("q_") and key.endswith("_vec"): + del item[key] + for key in ("content_with_weight", "content_ltks", "content_sm_ltks", "position_int", "page_num_int", "top_int"): + item.pop(key, None) + result.append(item) + return result + + @staticmethod + def _is_markdown_table_separator(line: str) -> bool: + cells = line.strip().strip("|").split("|") + return len(cells) >= 1 and all(re.fullmatch(r"\s*:?-{3,}:?\s*", cell) for cell in cells) + + @classmethod + def _split_markdown_blocks(cls, text: str) -> list[tuple[str, str | None]]: + """Separate Markdown tables and fenced code blocks as atomic blocks.""" + lines = text.splitlines(keepends=True) + blocks: list[tuple[str, str | None]] = [] + ordinary: list[str] = [] + index = 0 + + def flush_ordinary() -> None: + if ordinary: + blocks.append(("".join(ordinary), False)) + ordinary.clear() + + while index < len(lines): + fence_match = re.match(r"^\s*(`{3,}|~{3,})", lines[index]) + if fence_match: + flush_ordinary() + fence = fence_match.group(1) + fence_char = fence[0] + fence_length = len(fence) + code = [lines[index]] + index += 1 + while index < len(lines): + code.append(lines[index]) + closing = re.match( + rf"^\s*{re.escape(fence_char)}{{{fence_length},}}\s*$", + lines[index].rstrip("\r\n"), + ) + index += 1 + if closing: + break + blocks.append(("".join(code), "fenced_code")) + continue + + if index + 1 < len(lines) and re.match(r"^\s*\|", lines[index]) and cls._is_markdown_table_separator(lines[index + 1]): + flush_ordinary() + table = [lines[index], lines[index + 1]] + index += 2 + while index < len(lines) and re.match(r"^\s*\|", lines[index]): + table.append(lines[index]) + index += 1 + blocks.append(("".join(table), "table")) + continue + ordinary.append(lines[index]) + index += 1 + + flush_ordinary() + return blocks + + @classmethod + def _merge_small_text_chunks(cls, chunks: list[dict], target_token_size: int) -> list[dict]: + """Merge adjacent ordinary text chunks up to the coarse target.""" + if target_token_size < cls._PARSER_MIN_COARSE_CHUNK_TOKEN_SIZE: + return chunks + + merged = [] + pending = None + for chunk in chunks: + text = chunk.get("text") + if not isinstance(text, str) or not text.strip() or cls._is_atomic_json_record(chunk): + if pending is not None: + merged.append(pending) + pending = None + merged.append(chunk) + continue + + if pending is None: + pending = chunk + continue + + combined = f"{pending['text']}\n\n{text}" + if num_tokens_from_string(pending["text"]) < cls._PARSER_MIN_COARSE_CHUNK_TOKEN_SIZE and num_tokens_from_string(combined) <= target_token_size: + pending["text"] = combined + else: + merged.append(pending) + pending = chunk + + if pending is not None: + merged.append(pending) + return merged + + @classmethod + def _normalize_upstream_chunks( + cls, + upstream: dict, + split_json_text: bool = False, + target_token_size: int | None = None, + ) -> list[dict]: + """Normalize direct Parser output into Compiler chunks. + + Parser JSON is a list of chunk-like records. JSON records are split by + the requested token target while retaining their metadata; upstream + ``chunks`` preserve atomic table/image records unless rechunking is + explicitly requested. Textual Parser outputs use the same split. + """ + output_format = upstream.get("output_format") + if output_format == "chunks": + chunks = upstream.get("chunks") or [] + if not isinstance(chunks, list): + return [] + if not split_json_text: + normalized = deepcopy(chunks) + for chunk in normalized: + if not isinstance(chunk, dict): + continue + text = chunk.get("text") + if not isinstance(text, str): + text = chunk.get("content_with_weight") + chunk["text"] = text if isinstance(text, str) else "" + return normalized + records = chunks + elif output_format == "json": + records = upstream.get("json") or upstream.get("json_result") or [] + if not isinstance(records, list): + return [] + else: + records = None + + if records is not None: + chunks = [] + for record in records: + if not isinstance(record, dict): + continue + chunk = deepcopy(record) + text = chunk.get("text") + if not isinstance(text, str): + text = chunk.get("content_with_weight") + if isinstance(text, str) and text.strip(): + if Compiler._is_atomic_json_record(chunk) or not split_json_text: + chunk["text"] = text + chunks.append(chunk) + else: + for part in Compiler._slumber_candidates(text, target_token_size=target_token_size): + split_chunk = deepcopy(chunk) + split_chunk["text"] = part + chunks.append(split_chunk) + else: + # Keep metadata-only records (for example, an image + # placeholder) instead of silently dropping them. + chunk["text"] = text if isinstance(text, str) else "" + chunks.append(chunk) + if split_json_text: + return cls._merge_small_text_chunks(chunks, target_token_size or cls._PARSER_TEXT_CHUNK_TOKEN_SIZE) + return chunks + + if output_format in {"markdown", "text", "html"}: + text = upstream.get(output_format) + if not isinstance(text, str): + text = upstream.get(f"{output_format}_result") + if not isinstance(text, str) or not text.strip(): + return [] + if not split_json_text: + return [{"text": text}] + if output_format == "markdown": + chunks = [] + for block, block_type in cls._split_markdown_blocks(text): + if block_type == "table": + chunks.append({"text": block, "doc_type_kwd": "table"}) + elif block_type == "fenced_code": + chunks.append({"text": block}) + else: + chunks.extend({"text": part} for part in cls._slumber_candidates(block, target_token_size=target_token_size) if part.strip()) + else: + chunks = [{"text": part} for part in cls._slumber_candidates(text, target_token_size=target_token_size) if part.strip()] + return cls._merge_small_text_chunks(chunks, target_token_size or cls._PARSER_TEXT_CHUNK_TOKEN_SIZE) + + return [] def _compile_progress(self, prog=None, msg=""): """Adapt the knowledge-compile ``callback`` protocol to the flow @@ -210,31 +495,35 @@ class Compiler(ProcessBase, LLM): # kwargs. Do not call LLM.get_input_elements() here: it resolves the # inherited prompt variables through Canvas.globals, while Pipeline # is a Graph and has no globals. - chunks = deepcopy(kwargs.get("chunks") or []) - if not chunks: - for val in kwargs.values(): - if isinstance(val, list): - chunks = deepcopy(val) + output_format = kwargs.get("output_format") + if output_format == "chunks": + raw_chunks = kwargs.get("chunks") or [] + if not isinstance(raw_chunks, list) or not raw_chunks: + self.set_output("chunks", []) + return + # Normalize chunks only after the active templates are known, so + # rechunk mode does not normalize the same input twice. + chunks = None + else: + chunks = self._normalize_upstream_chunks(kwargs) tenant_id = self._canvas.get_tenant_id() doc_id = self._canvas._doc_id kb_id = getattr(self._canvas, "_kb_id", None) or DocumentService.get_knowledgebase_id(doc_id) language = self._compile_language(kwargs) - if not chunks: + if chunks is not None and not chunks: self.set_output("chunks", chunks) return - for ck in chunks: - ck["doc_id"] = doc_id - ck["id"] = xxhash.xxh64((ck["text"] + str(ck["doc_id"])).encode("utf-8")).hexdigest() - # Resolve the configured template groups to concrete, active # (non-artifact) structure-compilation templates. template_ids = resolve_template_ids_from_groups(self._param.compilation_template_group_ids, tenant_id) active_templates = load_active_templates(template_ids, tenant_id) if not active_templates: self.callback(0, "No active compilation templates resolved from the configured groups.") + if chunks is None: + chunks = self._normalize_upstream_chunks(kwargs) self.set_output("chunks", chunks) return @@ -279,10 +568,52 @@ class Compiler(ProcessBase, LLM): filtered_templates.append((template_id, parser_cfg)) if not filtered_templates: + if chunks is None: + chunks = self._normalize_upstream_chunks(kwargs) self.set_output("chunks", chunks) return active_templates = filtered_templates + def _template_requests_rechunk(cfg: dict) -> bool: + if not isinstance(cfg, dict): + return False + # ``raptor.rechunk`` belongs to the Tree path, which is explicitly + # excluded from the semantic rechunk staging below. It must not + # shrink non-Tree parser chunks unless a structure template has + # requested the actual regrouping pass. + return bool(cfg.get("rechunk")) + + should_rechunk = any(_template_requests_rechunk(cfg) for _, cfg in active_templates) + target_token_size = self._PARSER_CANDIDATE_TOKEN_SIZE if should_rechunk else self._PARSER_TEXT_CHUNK_TOKEN_SIZE + if output_format == "chunks": + chunks = self._normalize_upstream_chunks( + kwargs, + split_json_text=should_rechunk, + target_token_size=target_token_size, + ) + elif output_format in {"markdown", "text", "html"}: + chunks = self._normalize_upstream_chunks( + kwargs, + split_json_text=should_rechunk, + target_token_size=target_token_size, + ) + elif output_format == "json": + chunks = self._normalize_upstream_chunks( + kwargs, + split_json_text=should_rechunk, + target_token_size=target_token_size, + ) + if not chunks: + self.set_output("chunks", chunks) + return + if not chunks: + self.set_output("chunks", chunks) + return + + for ck in chunks: + ck["doc_id"] = doc_id + ck["id"] = xxhash.xxh64((ck["text"] + str(ck["doc_id"])).encode("utf-8")).hexdigest() + if self._canvas._kb_id: e, kb = KnowledgebaseService.get_by_id(self._canvas._kb_id) if kb.tenant_embd_id: @@ -315,6 +646,17 @@ class Compiler(ProcessBase, LLM): ) if non_tree_templates: + first_rechunk_index = next( + (index for index, (_, cfg) in enumerate(non_tree_templates) if _template_requests_rechunk(cfg)), + None, + ) + if first_rechunk_index is not None: + # Keep the stable hash IDs for Tree. Non-tree rechunking uses + # compact positional IDs to keep the LLM grouping response + # small (e.g. t1-t3 instead of three long hash IDs). + for index, chunk in enumerate(chunks, start=1): + chunk["id"] = f"t{index}" + task_id = getattr(self._canvas, "task_id", None) def _cancelled() -> bool: @@ -324,17 +666,48 @@ class Compiler(ProcessBase, LLM): for i in range(0, len(chunks), DOC_STRUCTURE_COMPILE_BATCH_CHUNKS): yield chunks[i : i + DOC_STRUCTURE_COMPILE_BATCH_CHUNKS] - await run_structure_compile_over_batches( - active_templates=non_tree_templates, - chat_mdl_by_tid=chat_mdl_by_tid, - embedding_model=embedding_model, - tenant_id=tenant_id, - kb_id=kb_id, - doc_id=doc_id, - language=language, - chunk_batches=_chunk_batches(), - progress_cb=self._compile_progress, - cancel_check=_cancelled, - ) + async def _run_templates(template_specs): + return await run_structure_compile_over_batches( + active_templates=template_specs, + chat_mdl_by_tid=chat_mdl_by_tid, + embedding_model=embedding_model, + tenant_id=tenant_id, + kb_id=kb_id, + doc_id=doc_id, + language=language, + chunk_batches=_chunk_batches(), + progress_cb=self._compile_progress, + cancel_check=_cancelled, + ) + + if first_rechunk_index is None: + await _run_templates(non_tree_templates) + else: + if first_rechunk_index: + await _run_templates(non_tree_templates[:first_rechunk_index]) + first_template = [non_tree_templates[first_rechunk_index]] + compile_info = await _run_templates(first_template) + first_template_id = first_template[0][0] + formal_chunks = (compile_info.get(first_template_id) or {}).get("rechunked_chunks") + if formal_chunks: + formalized_chunks = self._formalize_rechunked_chunks(chunks, formal_chunks, doc_id) + if formalized_chunks: + chunks = formalized_chunks + else: + logging.warning( + "Compiler: rechunking returned no valid formal chunks for doc %s; keeping original chunks", + doc_id, + ) + + # The first rechunk template defines the semantic boundaries + # for all later templates. They consume those formal chunks + # and must not perform another independent rechunking pass. + later_templates = [] + for template_id, parser_cfg in non_tree_templates[first_rechunk_index + 1 :]: + later_config = dict(parser_cfg or {}) + later_config["rechunk"] = False + later_templates.append((template_id, later_config)) + if later_templates: + await _run_templates(later_templates) self.set_output("chunks", chunks) diff --git a/rag/llm/chat_model.py b/rag/llm/chat_model.py index d4015e1673..0ab017bd2f 100644 --- a/rag/llm/chat_model.py +++ b/rag/llm/chat_model.py @@ -1691,8 +1691,30 @@ class LiteLLMBase(ABC): gen_conf=gen_conf, ) - gen_conf.pop("max_tokens", None) + deepseek_max_tokens = None + if self.provider == SupportedLiteLLMProvider.DeepSeek: + # DeepSeek's API uses the legacy OpenAI-compatible max_tokens field. + # LiteLLM accepts max_completion_tokens generically, but does not + # translate it for DeepSeek and the provider then falls back to 8192. + # Knowledge compilation supplies max_completion_tokens explicitly + # through its model-specific generation configuration. Otherwise, + # preserve the legacy max_tokens value used by existing callers. + raw_max_completion_tokens = gen_conf.pop("max_completion_tokens", None) + raw_max_tokens = gen_conf.pop("max_tokens", None) + raw_limit = raw_max_completion_tokens if raw_max_completion_tokens is not None else raw_max_tokens + if raw_limit is not None and not isinstance(raw_limit, bool): + try: + candidate = int(raw_limit) + except (TypeError, ValueError): + candidate = 0 + if candidate > 0: + deepseek_max_tokens = candidate + else: + gen_conf.pop("max_tokens", None) + gen_conf = {k: v for k, v in gen_conf.items() if k in LITELLM_ALLOWED_GEN_CONF_KEYS} + if deepseek_max_tokens is not None: + gen_conf["max_tokens"] = deepseek_max_tokens return gen_conf def _need_reasoning_content_back(self) -> bool: diff --git a/rag/svr/task_executor_refactor/chunk_post_processor.py b/rag/svr/task_executor_refactor/chunk_post_processor.py index 3e6cb4b300..484ff20920 100644 --- a/rag/svr/task_executor_refactor/chunk_post_processor.py +++ b/rag/svr/task_executor_refactor/chunk_post_processor.py @@ -378,8 +378,6 @@ from rag.advanced_rag.knowlege_compile.runner import ( # noqa: E402 load_active_templates, run_structure_compile_over_batches, ) - - # ----- parser_config helpers ----------------------------------------- diff --git a/web/src/interfaces/database/compilation-template.ts b/web/src/interfaces/database/compilation-template.ts index 564611c3e6..b29876e69e 100644 --- a/web/src/interfaces/database/compilation-template.ts +++ b/web/src/interfaces/database/compilation-template.ts @@ -24,6 +24,8 @@ export interface ICompilationTemplateConfig { relation?: ICompilationTemplateSection; raptor?: ICompilationTemplateRaptorConfig; global_rules?: string; + rechunk?: boolean; + rechunk_rules?: string; [section: string]: | ICompilationTemplateSection | ICompilationTemplateRaptorConfig diff --git a/web/src/interfaces/request/compilation-template.ts b/web/src/interfaces/request/compilation-template.ts index 81058f246a..6ab69cb280 100644 --- a/web/src/interfaces/request/compilation-template.ts +++ b/web/src/interfaces/request/compilation-template.ts @@ -24,6 +24,8 @@ export interface ICompilationTemplateConfigRequest { relation?: ICompilationTemplateSectionRequest; raptor?: ICompilationTemplateRaptorConfigRequest; global_rules?: string; + rechunk?: boolean; + rechunk_rules?: string; [section: string]: | ICompilationTemplateSectionRequest | ICompilationTemplateRaptorConfigRequest diff --git a/web/src/locales/en.ts b/web/src/locales/en.ts index 7ad38f99cc..25be19c67e 100644 --- a/web/src/locales/en.ts +++ b/web/src/locales/en.ts @@ -1900,6 +1900,12 @@ Example: Virtual Hosted Style`, rechunkByTreeLeaves: 'Re-chunk by tree leaves', rechunkByTreeLeavesTip: "Merge each leaf cluster's source chunks into a single replacement chunk. Originals are kept but marked unavailable for retrieval. Only one tree template per group may enable this.", + rechunkInput: 'Re-chunk parser output', + rechunkInputTip: + 'Let the LLM determine chunk boundaries based on the knowledge compilation task.', + rechunkRules: 'Rechunking rules', + rechunkRulesPlaceholder: + 'Describe how the LLM should group source chunks for this compilation task.', jsonPreview: 'JSON preview', processFlow: 'Process flow', processFlowComingSoon: 'Process flow preview coming soon', diff --git a/web/src/locales/zh.ts b/web/src/locales/zh.ts index 9b0f22c1a5..68f3632ee9 100644 --- a/web/src/locales/zh.ts +++ b/web/src/locales/zh.ts @@ -1590,6 +1590,11 @@ NER：使用 spaCy NER 和基于规则的关键词提取来抽取实体和关系 rechunkByTreeLeaves: '按树叶重新分块', rechunkByTreeLeavesTip: '将每个叶簇的源数据块合并为单个替换数据块。原始数据块保留但标记为不可检索。每个分组最多只能有一个树模板启用此功能。', + rechunkInput: '重新切分 Parser 输出', + rechunkInputTip: '根据知识编译任务，由 LLM 决定 chunk 边界。', + rechunkRules: 'Rechunk 规则', + rechunkRulesPlaceholder: + '描述此知识编译任务下，LLM 应如何合并和划分源 chunk。', jsonPreview: 'JSON 预览', processFlow: '流程视图', processFlowComingSoon: '流程视图预览即将到来', diff --git a/web/src/pages/user-setting/compilation-templates/create-next/components/template-configuration.tsx b/web/src/pages/user-setting/compilation-templates/create-next/components/template-configuration.tsx index def1f126ac..65a826160f 100644 --- a/web/src/pages/user-setting/compilation-templates/create-next/components/template-configuration.tsx +++ b/web/src/pages/user-setting/compilation-templates/create-next/components/template-configuration.tsx @@ -1,6 +1,7 @@ import { ModelTreeSelectFormField } from '@/components/model-tree-select'; import { SelectWithSearch } from '@/components/originui/select-with-search'; import { RAGFlowFormItem } from '@/components/ragflow-form'; +import { SwitchFormField } from '@/components/switch-fom-field'; import { Button } from '@/components/ui/button'; import { Input } from '@/components/ui/input'; import { Tabs, TabsContent, TabsList, TabsTrigger } from '@/components/ui/tabs'; @@ -65,6 +66,10 @@ export function TemplateConfiguration({ control: form.control, name: `templates.${selectedTemplateIndex}.kind`, }); + const rechunk = useWatch({ + control: form.control, + name: `templates.${selectedTemplateIndex}.config.rechunk`, + }); const availableKindOptions = useAvailableKindOptions( form, @@ -198,39 +203,62 @@ export function TemplateConfiguration({ {kind === CompilationTemplateKind.Tree ? ( ) : ( - sectionNames.length > 0 && - activeSectionTab && ( - - + <> + {kind !== CompilationTemplateKind.Artifacts && ( + <> + + {rechunk && ( + +