fix: preserve page index chapter chunk provenance (#17963)

api/db/init_data/compilation_templates/page_index.yaml

@@ -12,45 +12,29 @@ config:
     - Do not merge unrelated content. Do not invent, rewrite, or return source text; return only chunk grouping metadata.
   entity:
     description: >-
-      You are a source-grounded document index extractor. First extract every
-      supported heading in source order, then extract the important information
-      belonging to each heading. Headings form the hierarchy; facts, dates,
-      numbers, names, and conclusions are detail information that should be
-      captured in concise `fact` or `conclusion` entities attached below the
-      nearest relevant heading through `include` relations. Do not omit a
-      heading because its section is short, and do not return only headings.
-
-      A title entity's `name` must be the exact clean heading text. A detail
-      entity's `name` must be a concise, source-grounded label or statement,
-      and its `description` must preserve the useful compressed information.
-      Keep detail entities atomic: do not combine unrelated facts, dates,
-      numbers, names, or conclusions into one item. Preserve important values,
-      units, dates, qualifiers, and conclusions. Do not invent content.
-
-      For each title, write a strongly compressed summary of the content
-      belonging to that heading: normally 1–2 sentences, aiming for about
-      50–100 English words or 80–180 Chinese characters when the source is
-      long enough. For detailed lists, give the overall result and only the
-      most important examples. Do not copy long passages verbatim and do not
-      reduce descriptions to generic labels. Do not write meta descriptions
-      such as `heading for ...`, `section heading`, or `subheading for ...`.
-      Do not include image captions, `Main article` links, or unrelated tables
-      and lists unless they are actual content of the heading. If a parent
-      heading has no direct body before the next child heading, generate a
-      short higher-level overview from the available child-section content;
-      do not copy the child description verbatim. Only use an empty
-      description when there is no usable source content at all.
+      First extract all actual headings in source order, then extract the
+      important facts or conclusions under them. Treat explicit chapter markers such as 第N回、
+      第N章、Chapter N, numbered headings, and Markdown headings as titles
+      even when they have no `#` marker. A `title` must appear in the source;
+      copy it exactly and never invent one from a fact or summary. Titles have
+      priority over facts and conclusions: complete title extraction first and
+      never replace a title with a fact or conclusion. Do not downgrade an actual
+      chapter heading to a fact. For other entities, use concise atomic,
+      source-grounded names and descriptions. Do not invent.
     fields:
       - type: title
         description: the heading text (clean, no page numbers or leader dots)
         rule: |
-          - Chinese titles ≤25 characters; English titles ≤80. Use clean heading text.
-          - The title must be non-empty; use `-1` only when no heading is supported.
-          - Extract every supported heading in source order; do not skip major or short sections.
-          - For multiple headings, preserve source order and summarize each section separately.
-          - Write 1–2 compressed sentences, retaining key facts, dates, numbers, names, locations, and conclusions.
-          - Summarize parent content from its children when it has no direct body; do not copy a child verbatim.
-          - Exclude captions, navigation links, and unrelated lists. Preserve numbering, language, and source meaning.
+          - Identify and emit every explicit heading or chapter marker first, in source order, including 第N回 even without Markdown syntax.
+          - Copy the clean heading text; do not invent, paraphrase, or promote facts into titles.
+          - Assign source_chunk_ids yourself: include every source chunk covered by the title's
+            section, not only the chunk containing the heading. A parent title includes the
+            chunks of its nested child sections; a child title includes only its own subtree.
+            Stop at the next title of the same or an ancestor level.
+          - A line beginning with 第N回/第N章/第N节 or Chapter N is a title when it appears in the source.
+          - Do not emit the same chapter heading as a fact or conclusion.
+          - Only after title extraction, emit facts or conclusions under the corresponding titles.
+          - Summarize the content under each title without copying long passages.
       - type: fact
         description: a concise source-grounded factual statement belonging to a title
         rule: |
@@ -81,5 +65,5 @@ config:
           - Never use a source heading as an endpoint unless that heading was also emitted as a `title` entity.
           - Before returning, remove any relation whose endpoint is missing and verify that every detail entity has one parent relation.
           - Keep language of "title" the same as the input.
-          - 第N章 must include 第N条 or 第N节.
+          - 第N回、 第N章 must include their actual child sections or detail entities when present.
   global_rules: ''

rag/advanced_rag/knowlege_compile/structure.py

@@ -1165,6 +1165,71 @@ async def _struct_merge_pair(existing: dict, incoming: dict, chat_mdl) -> dict |
     return merged
 
 
+def _struct_merge_exact_entity_payload(existing: dict, incoming: dict) -> dict | None:
+    """Merge same-name entity payloads without relying on vector similarity."""
+    try:
+        left = json.loads(existing.get("content_with_weight") or "{}")
+        right = json.loads(incoming.get("content_with_weight") or "{}")
+    except Exception:
+        return None
+    if not isinstance(left, dict) or not isinstance(right, dict):
+        return None
+
+    merged = dict(left)
+    for key, value in right.items():
+        if key not in merged or merged[key] in (None, "", []):
+            merged[key] = value
+
+    types = {str(left.get("type") or "").strip().casefold(), str(right.get("type") or "").strip().casefold()}
+    for preferred in ("title", "fact", "conclusion"):
+        if preferred in types:
+            merged["type"] = preferred
+            break
+
+    descriptions = [left.get("description") or "", right.get("description") or ""]
+    merged["description"] = max(descriptions, key=lambda value: len(str(value)))
+    merged["source_chunk_ids"] = _struct_union_chunk_ids(left.get("source_chunk_ids"), right.get("source_chunk_ids"))
+    return merged
+
+
+async def _struct_merge_exact_named_entities(docs: list[dict], embd_mdl) -> tuple[list[dict], int]:
+    """Collapse same-name entities before similarity-based dedup."""
+    kept: dict[str, dict] = {}
+    order: list[str] = []
+    unchanged: list[dict] = []
+    dropped = 0
+
+    for doc in docs:
+        name = _struct_entity_name(doc).strip().casefold()
+        if not name:
+            unchanged.append(doc)
+            continue
+        if name not in kept:
+            kept[name] = doc
+            order.append(name)
+            continue
+
+        existing = kept[name]
+        payload = _struct_merge_exact_entity_payload(existing, doc)
+        if payload is None:
+            unchanged.append(doc)
+            continue
+        vector = await _struct_reembed_payload(payload, embd_mdl)
+        if vector is None:
+            unchanged.append(doc)
+            continue
+        kept[name] = _struct_rebuild_doc_storage_doc(
+            payload,
+            existing,
+            vector,
+            _struct_union_chunk_ids(existing.get("source_chunk_ids"), doc.get("source_chunk_ids")),
+            preserve_id=True,
+        )
+        dropped += 1
+
+    return [kept[name] for name in order] + unchanged, dropped
+
+
 def _struct_apply_merge_invariants(existing: dict, merged_payload: dict) -> dict:
     """For relations, force the source/target fields back to the existing payload's
     values — from_entity_kwd / to_entity_kwd must not change across a merge.
@@ -1273,6 +1338,36 @@ async def _struct_doc_storage_knn_candidate(
     from common import settings
     from common.doc_store.doc_store_base import MatchDenseExpr, OrderByExpr
 
+    # Names are the entity identity used by the structure graph. Check the
+    # exact name before KNN so a new compile joins an existing canonical row
+    # even when title/fact/conclusion descriptions have low vector similarity.
+    if doc.get("knowledge_graph_kwd") == "entity":
+        name = str(doc.get("name_kwd") or _struct_entity_name(doc) or "").strip().casefold()
+        if name:
+            exact_condition = _struct_doc_storage_dedup_condition(doc, merge_scope)
+            exact_condition["name_kwd"] = [name]
+            try:
+                res = await thread_pool_exec(
+                    settings.docStoreConn.search,
+                    select_fields,
+                    [],
+                    exact_condition,
+                    [],
+                    OrderByExpr(),
+                    0,
+                    1,
+                    index,
+                    [kb_id],
+                )
+                field_map = settings.docStoreConn.get_fields(res, select_fields)
+                if field_map:
+                    old_id, old_doc = next(iter(field_map.items()))
+                    old_doc = dict(old_doc)
+                    old_doc.setdefault("id", old_id)
+                    return old_doc
+            except Exception:
+                logging.exception("merge_compiled_structures: exact entity-name search failed")
+
     vec_field, vec = _struct_doc_vec(doc)
     if not vec_field or vec is None:
         return None
@@ -2027,6 +2122,7 @@ async def _struct_local_dedup_parallel(
 
     entity_docs = [doc for doc in docs if doc.get("knowledge_graph_kwd") != "relation"]
     relation_docs = [doc for doc in docs if doc.get("knowledge_graph_kwd") == "relation"]
+    entity_docs, exact_dropped = await _struct_merge_exact_named_entities(entity_docs, embd_mdl)
     entity_groups = _struct_entity_candidate_groups(entity_docs, similarity_threshold)
     group_semaphore = asyncio.Semaphore(_LOCAL_DEDUP_GROUP_CONCURRENCY)
 
@@ -2045,7 +2141,7 @@ async def _struct_local_dedup_parallel(
     entity_results = await asyncio.gather(*(dedup_group(group) for group in entity_groups))
     deduped_entities: list[dict] = []
     entity_aliases: dict[str, str] = {}
-    dropped = 0
+    dropped = exact_dropped
     for entity_result, group in zip(entity_results, entity_groups):
         group_docs, group_dropped, group_aliases = entity_result
         deduped_entities.extend(group_docs)