From 48b2d3b914f91d121e6c8db343231448de305d38 Mon Sep 17 00:00:00 2001 From: buua436 Date: Fri, 7 Aug 2026 13:47:04 +0800 Subject: [PATCH] fix: preserve page index chapter chunk provenance (#17963) --- .../compilation_templates/page_index.yaml | 56 ++++------- .../knowlege_compile/structure.py | 98 ++++++++++++++++++- 2 files changed, 117 insertions(+), 37 deletions(-) diff --git a/api/db/init_data/compilation_templates/page_index.yaml b/api/db/init_data/compilation_templates/page_index.yaml index 18213b5f00..a0b055fec5 100644 --- a/api/db/init_data/compilation_templates/page_index.yaml +++ b/api/db/init_data/compilation_templates/page_index.yaml @@ -12,45 +12,29 @@ config: - Do not merge unrelated content. Do not invent, rewrite, or return source text; return only chunk grouping metadata. entity: description: >- - You are a source-grounded document index extractor. First extract every - supported heading in source order, then extract the important information - belonging to each heading. Headings form the hierarchy; facts, dates, - numbers, names, and conclusions are detail information that should be - captured in concise `fact` or `conclusion` entities attached below the - nearest relevant heading through `include` relations. Do not omit a - heading because its section is short, and do not return only headings. - - A title entity's `name` must be the exact clean heading text. A detail - entity's `name` must be a concise, source-grounded label or statement, - and its `description` must preserve the useful compressed information. - Keep detail entities atomic: do not combine unrelated facts, dates, - numbers, names, or conclusions into one item. Preserve important values, - units, dates, qualifiers, and conclusions. Do not invent content. - - For each title, write a strongly compressed summary of the content - belonging to that heading: normally 1–2 sentences, aiming for about - 50–100 English words or 80–180 Chinese characters when the source is - long enough. For detailed lists, give the overall result and only the - most important examples. Do not copy long passages verbatim and do not - reduce descriptions to generic labels. Do not write meta descriptions - such as `heading for ...`, `section heading`, or `subheading for ...`. - Do not include image captions, `Main article` links, or unrelated tables - and lists unless they are actual content of the heading. If a parent - heading has no direct body before the next child heading, generate a - short higher-level overview from the available child-section content; - do not copy the child description verbatim. Only use an empty - description when there is no usable source content at all. + First extract all actual headings in source order, then extract the + important facts or conclusions under them. Treat explicit chapter markers such as 第N回、 + 第N章、Chapter N, numbered headings, and Markdown headings as titles + even when they have no `#` marker. A `title` must appear in the source; + copy it exactly and never invent one from a fact or summary. Titles have + priority over facts and conclusions: complete title extraction first and + never replace a title with a fact or conclusion. Do not downgrade an actual + chapter heading to a fact. For other entities, use concise atomic, + source-grounded names and descriptions. Do not invent. fields: - type: title description: the heading text (clean, no page numbers or leader dots) rule: | - - Chinese titles ≤25 characters; English titles ≤80. Use clean heading text. - - The title must be non-empty; use `-1` only when no heading is supported. - - Extract every supported heading in source order; do not skip major or short sections. - - For multiple headings, preserve source order and summarize each section separately. - - Write 1–2 compressed sentences, retaining key facts, dates, numbers, names, locations, and conclusions. - - Summarize parent content from its children when it has no direct body; do not copy a child verbatim. - - Exclude captions, navigation links, and unrelated lists. Preserve numbering, language, and source meaning. + - Identify and emit every explicit heading or chapter marker first, in source order, including 第N回 even without Markdown syntax. + - Copy the clean heading text; do not invent, paraphrase, or promote facts into titles. + - Assign source_chunk_ids yourself: include every source chunk covered by the title's + section, not only the chunk containing the heading. A parent title includes the + chunks of its nested child sections; a child title includes only its own subtree. + Stop at the next title of the same or an ancestor level. + - A line beginning with 第N回/第N章/第N节 or Chapter N is a title when it appears in the source. + - Do not emit the same chapter heading as a fact or conclusion. + - Only after title extraction, emit facts or conclusions under the corresponding titles. + - Summarize the content under each title without copying long passages. - type: fact description: a concise source-grounded factual statement belonging to a title rule: | @@ -81,5 +65,5 @@ config: - Never use a source heading as an endpoint unless that heading was also emitted as a `title` entity. - Before returning, remove any relation whose endpoint is missing and verify that every detail entity has one parent relation. - Keep language of "title" the same as the input. - - 第N章 must include 第N条 or 第N节. + - 第N回、 第N章 must include their actual child sections or detail entities when present. global_rules: '' diff --git a/rag/advanced_rag/knowlege_compile/structure.py b/rag/advanced_rag/knowlege_compile/structure.py index 66385c1aee..2284518039 100644 --- a/rag/advanced_rag/knowlege_compile/structure.py +++ b/rag/advanced_rag/knowlege_compile/structure.py @@ -1165,6 +1165,71 @@ async def _struct_merge_pair(existing: dict, incoming: dict, chat_mdl) -> dict | return merged +def _struct_merge_exact_entity_payload(existing: dict, incoming: dict) -> dict | None: + """Merge same-name entity payloads without relying on vector similarity.""" + try: + left = json.loads(existing.get("content_with_weight") or "{}") + right = json.loads(incoming.get("content_with_weight") or "{}") + except Exception: + return None + if not isinstance(left, dict) or not isinstance(right, dict): + return None + + merged = dict(left) + for key, value in right.items(): + if key not in merged or merged[key] in (None, "", []): + merged[key] = value + + types = {str(left.get("type") or "").strip().casefold(), str(right.get("type") or "").strip().casefold()} + for preferred in ("title", "fact", "conclusion"): + if preferred in types: + merged["type"] = preferred + break + + descriptions = [left.get("description") or "", right.get("description") or ""] + merged["description"] = max(descriptions, key=lambda value: len(str(value))) + merged["source_chunk_ids"] = _struct_union_chunk_ids(left.get("source_chunk_ids"), right.get("source_chunk_ids")) + return merged + + +async def _struct_merge_exact_named_entities(docs: list[dict], embd_mdl) -> tuple[list[dict], int]: + """Collapse same-name entities before similarity-based dedup.""" + kept: dict[str, dict] = {} + order: list[str] = [] + unchanged: list[dict] = [] + dropped = 0 + + for doc in docs: + name = _struct_entity_name(doc).strip().casefold() + if not name: + unchanged.append(doc) + continue + if name not in kept: + kept[name] = doc + order.append(name) + continue + + existing = kept[name] + payload = _struct_merge_exact_entity_payload(existing, doc) + if payload is None: + unchanged.append(doc) + continue + vector = await _struct_reembed_payload(payload, embd_mdl) + if vector is None: + unchanged.append(doc) + continue + kept[name] = _struct_rebuild_doc_storage_doc( + payload, + existing, + vector, + _struct_union_chunk_ids(existing.get("source_chunk_ids"), doc.get("source_chunk_ids")), + preserve_id=True, + ) + dropped += 1 + + return [kept[name] for name in order] + unchanged, dropped + + def _struct_apply_merge_invariants(existing: dict, merged_payload: dict) -> dict: """For relations, force the source/target fields back to the existing payload's values — from_entity_kwd / to_entity_kwd must not change across a merge. @@ -1273,6 +1338,36 @@ async def _struct_doc_storage_knn_candidate( from common import settings from common.doc_store.doc_store_base import MatchDenseExpr, OrderByExpr + # Names are the entity identity used by the structure graph. Check the + # exact name before KNN so a new compile joins an existing canonical row + # even when title/fact/conclusion descriptions have low vector similarity. + if doc.get("knowledge_graph_kwd") == "entity": + name = str(doc.get("name_kwd") or _struct_entity_name(doc) or "").strip().casefold() + if name: + exact_condition = _struct_doc_storage_dedup_condition(doc, merge_scope) + exact_condition["name_kwd"] = [name] + try: + res = await thread_pool_exec( + settings.docStoreConn.search, + select_fields, + [], + exact_condition, + [], + OrderByExpr(), + 0, + 1, + index, + [kb_id], + ) + field_map = settings.docStoreConn.get_fields(res, select_fields) + if field_map: + old_id, old_doc = next(iter(field_map.items())) + old_doc = dict(old_doc) + old_doc.setdefault("id", old_id) + return old_doc + except Exception: + logging.exception("merge_compiled_structures: exact entity-name search failed") + vec_field, vec = _struct_doc_vec(doc) if not vec_field or vec is None: return None @@ -2027,6 +2122,7 @@ async def _struct_local_dedup_parallel( entity_docs = [doc for doc in docs if doc.get("knowledge_graph_kwd") != "relation"] relation_docs = [doc for doc in docs if doc.get("knowledge_graph_kwd") == "relation"] + entity_docs, exact_dropped = await _struct_merge_exact_named_entities(entity_docs, embd_mdl) entity_groups = _struct_entity_candidate_groups(entity_docs, similarity_threshold) group_semaphore = asyncio.Semaphore(_LOCAL_DEDUP_GROUP_CONCURRENCY) @@ -2045,7 +2141,7 @@ async def _struct_local_dedup_parallel( entity_results = await asyncio.gather(*(dedup_group(group) for group in entity_groups)) deduped_entities: list[dict] = [] entity_aliases: dict[str, str] = {} - dropped = 0 + dropped = exact_dropped for entity_result, group in zip(entity_results, entity_groups): group_docs, group_dropped, group_aliases = entity_result deduped_entities.extend(group_docs)