Feat: Add knowledge compilation workflows (#16515)

## Summary
- Add knowledge compilation template APIs, services, and builtin
template seed data
- Add advanced knowledge compile structure/artifact/RAPTOR workflow
support
- Update parsing, dataset/document APIs, and supporting services for
compilation workflows

api/db/services/document_service.py

@@ -373,10 +373,14 @@ class DocumentService(CommonService):
     @DB.connection_context()
     def list_doc_headers_by_kb_and_source_type(cls, kb_id, source_type, page_size=500):
         fields = [cls.model.id, cls.model.kb_id, cls.model.source_type, cls.model.name]
-        docs = cls.model.select(*fields).where(
-            cls.model.kb_id == kb_id,
-            cls.model.source_type == source_type,
-        ).order_by(cls.model.create_time.asc())
+        docs = (
+            cls.model.select(*fields)
+            .where(
+                cls.model.kb_id == kb_id,
+                cls.model.source_type == source_type,
+            )
+            .order_by(cls.model.create_time.asc())
+        )
         offset = 0
         res = []
         while True:
@@ -402,10 +406,14 @@ class DocumentService(CommonService):
         rows and the resulting map would silently miss entries.
         """
         fields = [cls.model.id, cls.model.content_hash]
-        docs = cls.model.select(*fields).where(
-            cls.model.kb_id == kb_id,
-            cls.model.source_type == source_type,
-        ).order_by(cls.model.create_time.asc())
+        docs = (
+            cls.model.select(*fields)
+            .where(
+                cls.model.kb_id == kb_id,
+                cls.model.source_type == source_type,
+            )
+            .order_by(cls.model.create_time.asc())
+        )
         offset = 0
         result: dict[str, str] = {}
         while True:
@@ -489,6 +497,20 @@ class DocumentService(CommonService):
         except Exception as e:
             logging.error(f"Failed to delete chunks from doc store for document {doc.id}: {e}")
 
+        # Prune this doc's line from the KB's tree-kind navigation
+        # markdown (best-effort — the markdown is a downstream artifact,
+        # and failure here must not block the document delete).
+        try:
+            from rag.advanced_rag.knowlege_compile.dataset_nav import (
+                remove_dataset_nav_doc_sync,
+            )
+
+            remove_dataset_nav_doc_sync(tenant_id, doc.kb_id, doc.id)
+        except Exception as e:
+            logging.warning(
+                f"Failed to prune dataset_nav for document {doc.id}: {e}",
+            )
+
         # Delete document metadata (non-critical, log and continue)
         try:
             DocMetadataService.delete_document_metadata(doc.id, doc.kb_id, tenant_id)
@@ -571,21 +593,20 @@ class DocumentService(CommonService):
     @classmethod
     @DB.connection_context()
     def get_unfinished_docs(cls):
-        fields = [cls.model.id, cls.model.process_begin_at, cls.model.parser_config, cls.model.progress_msg,
-                  cls.model.run, cls.model.parser_id]
-        unfinished_task_query = Task.select(Task.doc_id).where(
-            (Task.progress >= 0) & (Task.progress < 1)
-        )
+        fields = [cls.model.id, cls.model.process_begin_at, cls.model.parser_config, cls.model.progress_msg, cls.model.run, cls.model.parser_id]
+        unfinished_task_query = Task.select(Task.doc_id).where((Task.progress >= 0) & (Task.progress < 1))
         docs_with_non_failed_tasks = Task.select(Task.doc_id).where(Task.progress >= 0).distinct()
 
         docs = cls.model.select(*fields).where(
             cls.model.status == StatusEnum.VALID.value,
             ~(cls.model.type == FileType.VIRTUAL.value),
             ((cls.model.run.is_null(True)) | (cls.model.run != TaskStatus.CANCEL.value)),
-            (((cls.model.progress < 1) & (cls.model.progress > 0)) |
-             (cls.model.id.in_(unfinished_task_query)) |
-             ((cls.model.progress == -1) & (cls.model.run == TaskStatus.FAIL.value) &
-              (cls.model.id.in_(docs_with_non_failed_tasks)))))  # including GraphRAG/RAPTOR/Mindmap; re-sync failed docs
+            (
+                ((cls.model.progress < 1) & (cls.model.progress > 0))
+                | (cls.model.id.in_(unfinished_task_query))
+                | ((cls.model.progress == -1) & (cls.model.run == TaskStatus.FAIL.value) & (cls.model.id.in_(docs_with_non_failed_tasks)))
+            ),
+        )  # including GraphRAG/RAPTOR/Mindmap; re-sync failed docs
         return list(docs.dicts())
 
     @classmethod
@@ -604,8 +625,7 @@ class DocumentService(CommonService):
             )
             if num == 0:
                 logging.error(
-                    "increment_chunk_num: no document matched doc_id=%s kb_id=%s "
-                    "token_num=%s chunk_num=%s duration=%s",
+                    "increment_chunk_num: no document matched doc_id=%s kb_id=%s token_num=%s chunk_num=%s duration=%s",
                     doc_id,
                     kb_id,
                     token_num,
@@ -623,8 +643,7 @@ class DocumentService(CommonService):
             )
             if num == 0:
                 logging.error(
-                    "increment_chunk_num: no knowledgebase matched kb_id=%s for doc_id=%s "
-                    "token_num=%s chunk_num=%s duration=%s",
+                    "increment_chunk_num: no knowledgebase matched kb_id=%s for doc_id=%s token_num=%s chunk_num=%s duration=%s",
                     kb_id,
                     doc_id,
                     token_num,
@@ -660,8 +679,7 @@ class DocumentService(CommonService):
             )
             if num == 0:
                 logging.error(
-                    "decrement_chunk_num: no knowledgebase matched kb_id=%s for doc_id=%s "
-                    "token_num=%s chunk_num=%s duration=%s",
+                    "decrement_chunk_num: no knowledgebase matched kb_id=%s for doc_id=%s token_num=%s chunk_num=%s duration=%s",
                     kb_id,
                     doc_id,
                     token_num,
@@ -1071,7 +1089,7 @@ def queue_raptor_o_graphrag_tasks(sample_doc, ty, priority, fake_doc_id="", doc_
     """
     if doc_ids is None:
         doc_ids = []
-    assert ty in ["graphrag", "raptor", "mindmap"], "type should be graphrag, raptor or mindmap"
+    assert ty in ["graphrag", "raptor", "mindmap", "artifact", "skill"], "type should be graphrag, raptor, mindmap, artifact or skill"
 
     chunking_config = DocumentService.get_chunking_config(sample_doc["id"])
     hasher = xxhash.xxh64()
@@ -1102,6 +1120,51 @@ def queue_raptor_o_graphrag_tasks(sample_doc, ty, priority, fake_doc_id="", doc_
     return task["id"]
 
 
+def queue_per_doc_raptor_task(doc, priority):
+    """Queue a doc-scoped RAPTOR task.
+
+    Distinct from :func:`queue_raptor_o_graphrag_tasks` (which is KB-scoped
+    and uses ``GRAPH_RAPTOR_FAKE_DOC_ID`` as the task's ``doc_id`` so it
+    fans out across the dataset). Here the task's ``doc_id`` is the real
+    document id, so ``TaskHandler._run_raptor`` runs only on this doc's
+    chunks and the RAPTOR summaries it produces are scoped to this doc.
+
+    Triggered automatically at the tail of standard chunking when the
+    doc's ``parser_config["raptor"]["use_raptor"]`` is true. No
+    cross-task dedup — within one chunking-task execution this helper is
+    called at most once, which is the only invariant the caller needs.
+    """
+    chunking_config = DocumentService.get_chunking_config(doc["id"])
+    hasher = xxhash.xxh64()
+    for field in sorted(chunking_config.keys()):
+        hasher.update(str(chunking_config[field]).encode("utf-8"))
+
+    task = {
+        "id": get_uuid(),
+        "doc_id": doc["id"],
+        "from_page": MAXIMUM_TASK_PAGE_NUMBER,
+        "to_page": MAXIMUM_TASK_PAGE_NUMBER,
+        "task_type": "raptor",
+        "progress_msg": datetime.now().strftime("%H:%M:%S") + " created task raptor",
+        "begin_at": datetime.now().strftime("%Y-%m-%d %H:%M:%S"),
+    }
+    for field in ["doc_id", "from_page", "to_page"]:
+        hasher.update(str(task[field]).encode("utf-8"))
+    hasher.update(b"raptor")
+    task["digest"] = hasher.hexdigest()
+    bulk_insert_into_db(Task, [task], True)
+
+    # Redis message carries ``doc_ids`` for downstream consumers
+    # (TaskHandler._run_raptor reads it). Identical to the fake-doc
+    # path's convention so we don't have to special-case the executor.
+    task["doc_ids"] = [doc["id"]]
+    assert REDIS_CONN.queue_product(
+        settings.get_svr_queue_name(priority, "raptor"),
+        message=task,
+    ), "Can't access Redis. Please check the Redis' status."
+    return task["id"]
+
+
 def get_queue_length(priority, suffix="common"):
     group_info = REDIS_CONN.queue_info(settings.get_svr_queue_name(priority, suffix), SVR_CONSUMER_GROUP_NAME)
     if not group_info: