Feat: Add knowledge compilation workflows (#16515)

## Summary
- Add knowledge compilation template APIs, services, and builtin
template seed data
- Add advanced knowledge compile structure/artifact/RAPTOR workflow
support
- Update parsing, dataset/document APIs, and supporting services for
compilation workflows
This commit is contained in:
Kevin Hu
2026-07-02 23:22:07 +08:00
committed by GitHub
parent 7d64a78f83
commit 62f94cd59b
57 changed files with 14587 additions and 3094 deletions

View File

@@ -15,11 +15,13 @@
#
import datetime
import difflib
import hashlib
import json
import logging
from typing import Optional
from api.db.db_models import DB, FileCommit, FileCommitItem, File
from api.db.db_models import DB, FileCommit, FileCommitItem, File, User
from api.db.services.common_service import CommonService
from api.db.services.file_service import FileService
from common import settings
@@ -29,6 +31,149 @@ from common.time_utils import current_timestamp, datetime_format
logger = logging.getLogger(__name__)
# ---------------------------------------------------------------------
# Artifact-commit extension
# ---------------------------------------------------------------------
# Artifact-page saves used to land in the retired ``ArtifactCommit`` table.
# They now flow through :class:`FileCommitService.record_page_edit`, which
# writes one FileCommit + one FileCommitItem per save with the artifact
# columns populated (title/comments on FileCommit; diff/content_after_*/
# slug_kwd/page_type_kwd on FileCommitItem).
#
# ``file_id`` for these commits is a stable content-hash of ``(kb_id, slug)``
# so per-page history queries can filter on it without a real File row —
# no pseudo-File / virtual-folder machinery is created, so the workspace
# UI stays free of ghost entries.
#
# ``folder_id`` is set to ``kb_id`` directly. The datasets URL prefix
# (``/datasets/<kb_id>/commits``) resolves the entity id to itself for
# this scope; workspace file-commit browsing still uses ``/folders/*`` or
# ``/workspace/*`` with the real folder id.
#
# Content storage for ``content_after`` is switched by a module-level
# constant so ops can move blobs between MinIO and the doc-store index
# without touching the schema.
ARTIFACT_CONTENT_STORAGE = "minio" # one of {"minio", "es"}
_ARTIFACT_COMMIT_BUCKET_PREFIX = ".artifact_commits"
_ARTIFACT_ES_KWD = "artifact_commit_content"
def _artifact_file_id(kb_id: str, slug: str) -> str:
"""Deterministic 32-char id for the artifact-page 'file' identity.
Not a real File row — just an index key that groups all commits for
the same page. Hashed so slugs longer than 32 chars still fit.
"""
return hashlib.md5(f"{kb_id}:{slug}".encode("utf-8")).hexdigest()
def _unified_diff(before: str, after: str, slug: str) -> str:
"""Return a unified diff between two markdown strings, or '' if equal."""
if (before or "") == (after or ""):
return ""
return "".join(
difflib.unified_diff(
(before or "").splitlines(keepends=True),
(after or "").splitlines(keepends=True),
fromfile=f"a/{slug}",
tofile=f"b/{slug}",
n=3,
)
)
def _store_content_after(kb_id: str, content: str) -> tuple[str, str]:
"""Persist ``content`` per :data:`ARTIFACT_CONTENT_STORAGE`. Returns
``(storage_kind, location)`` for the row's persistence columns.
Content-addressed by SHA-256 so re-saves with identical bodies share
the same blob.
"""
content_bytes = (content or "").encode("utf-8")
content_hash = hashlib.sha256(content_bytes).hexdigest()
if ARTIFACT_CONTENT_STORAGE == "minio":
location = f"{_ARTIFACT_COMMIT_BUCKET_PREFIX}/{content_hash}"
try:
storage = settings.STORAGE_IMPL
if storage is not None:
storage.put(kb_id, location, content_bytes)
except Exception:
logging.exception(
"record_page_edit: MinIO put failed for kb=%s hash=%s",
kb_id,
content_hash,
)
return "minio", location
if ARTIFACT_CONTENT_STORAGE == "es":
# Store as a single doc-store row so the same connector serves
# reads. The row is not retrievable (available_int=0).
from rag.nlp import search as _rag_search
index = _rag_search.index_name(kb_id) # kb-scoped index namespace
payload = {
"id": content_hash,
"kb_id": kb_id,
"doc_id": kb_id,
"compile_kwd": _ARTIFACT_ES_KWD,
"content_with_weight": content or "",
"available_int": 0,
}
try:
settings.docStoreConn.insert([payload], index, kb_id)
except Exception:
logging.exception(
"record_page_edit: ES insert failed for kb=%s hash=%s",
kb_id,
content_hash,
)
return "es", content_hash
# Unknown storage kind — fall through with empty location; the
# detail path treats missing location as "content not recoverable".
logging.warning(
"record_page_edit: unknown ARTIFACT_CONTENT_STORAGE=%r; content not persisted",
ARTIFACT_CONTENT_STORAGE,
)
return "", ""
def _read_content_after(kb_id: str, storage_kind: str, location: str) -> str:
"""Fetch the previously-stored artifact ``content_after`` blob.
Returns ``""`` when the location is empty (workspace commits) or the
blob is missing.
"""
if not location:
return ""
try:
if storage_kind == "minio":
storage = settings.STORAGE_IMPL
if storage is None:
return ""
raw = storage.get(kb_id, location)
if isinstance(raw, (bytes, bytearray)):
return raw.decode("utf-8", errors="replace")
return str(raw or "")
if storage_kind == "es":
from rag.nlp import search as _rag_search
index = _rag_search.index_name(kb_id)
row = settings.docStoreConn.get(location, index, [kb_id])
if isinstance(row, dict):
return row.get("content_with_weight") or ""
return ""
except Exception:
logging.exception(
"get_page_commit: content read failed kb=%s storage=%s loc=%s",
kb_id,
storage_kind,
location,
)
return ""
def _get_file_parent_id(file_id):
"""Look up a file's parent_id from the File table."""
try:
@@ -156,11 +301,13 @@ class FileCommitService(CommonService):
item["new_location"] = obj_key
# Update file record in DB
File.update({
"location": obj_key,
"size": len(content_bytes),
"update_time": current_timestamp(),
}).where(File.id == file_id).execute()
File.update(
{
"location": obj_key,
"size": len(content_bytes),
"update_time": current_timestamp(),
}
).where(File.id == file_id).execute()
# Update tree state
file_parent = _get_file_parent_id(file_id)
@@ -196,11 +343,13 @@ class FileCommitService(CommonService):
item["new_location"] = obj_key
# Update file record
File.update({
"location": obj_key,
"size": len(content_bytes),
"update_time": current_timestamp(),
}).where(File.id == file_id).execute()
File.update(
{
"location": obj_key,
"size": len(content_bytes),
"update_time": current_timestamp(),
}
).where(File.id == file_id).execute()
# Update tree state
file_parent = _get_file_parent_id(file_id)
@@ -222,9 +371,7 @@ class FileCommitService(CommonService):
item["old_location"] = old_location
# Soft-delete the file record
File.update(status="0", update_time=current_timestamp()).where(
File.id == file_id
).execute()
File.update(status="0", update_time=current_timestamp()).where(File.id == file_id).execute()
# Remove from tree state (mark deleted)
if file_id in tree_state:
@@ -237,9 +384,7 @@ class FileCommitService(CommonService):
item["new_name"] = new_name
# Update the file record name
File.update(name=new_name, update_time=current_timestamp()).where(
File.id == file_id
).execute()
File.update(name=new_name, update_time=current_timestamp()).where(File.id == file_id).execute()
# Update tree state
if file_id in tree_state:
@@ -259,9 +404,7 @@ class FileCommitService(CommonService):
def _get_latest_commit(cls, folder_id):
"""Get the latest (chain head) commit for a folder."""
try:
return cls.model.select().where(
cls.model.folder_id == folder_id
).order_by(cls.model.create_time.desc()).first()
return cls.model.select().where(cls.model.folder_id == folder_id).order_by(cls.model.create_time.desc()).first()
except Exception:
return None
@@ -359,27 +502,31 @@ class FileCommitService(CommonService):
if from_entry is not None and to_entry is None:
# Present in from, absent in to → deleted
diff.append({
"file_id": fid,
"file_name": from_name,
"operation": "delete",
"old_hash": from_hash or (from_item.new_hash if from_item else None),
"old_location": from_entry.get("location", "") if isinstance(from_entry, dict) else None,
"new_hash": None,
"new_location": None,
})
diff.append(
{
"file_id": fid,
"file_name": from_name,
"operation": "delete",
"old_hash": from_hash or (from_item.new_hash if from_item else None),
"old_location": from_entry.get("location", "") if isinstance(from_entry, dict) else None,
"new_hash": None,
"new_location": None,
}
)
elif from_entry is None and to_entry is not None:
# Present in to, absent in from → added
diff.append({
"file_id": fid,
"file_name": to_name,
"operation": "add",
"old_hash": None,
"old_location": None,
"new_hash": to_hash or (to_item.new_hash if to_item else None),
"new_location": to_entry.get("location", "") if isinstance(to_entry, dict) else None,
})
diff.append(
{
"file_id": fid,
"file_name": to_name,
"operation": "add",
"old_hash": None,
"old_location": None,
"new_hash": to_hash or (to_item.new_hash if to_item else None),
"new_location": to_entry.get("location", "") if isinstance(to_entry, dict) else None,
}
)
else:
# Both exist — check for changes
@@ -403,15 +550,17 @@ class FileCommitService(CommonService):
if changed:
old_loc = from_entry.get("location", "") if isinstance(from_entry, dict) else None
new_loc = to_entry.get("location", "") if isinstance(to_entry, dict) else None
diff.append({
"file_id": fid,
"file_name": to_name or from_name,
"operation": operation,
"old_hash": from_hash or (from_item.new_hash if from_item else None),
"old_location": old_loc or (from_item.new_location if from_item else None),
"new_hash": to_hash or (to_item.new_hash if to_item else None),
"new_location": new_loc or (to_item.new_location if to_item else None),
})
diff.append(
{
"file_id": fid,
"file_name": to_name or from_name,
"operation": operation,
"old_hash": from_hash or (from_item.new_hash if from_item else None),
"old_location": old_loc or (from_item.new_location if from_item else None),
"new_hash": to_hash or (to_item.new_hash if to_item else None),
"new_location": new_loc or (to_item.new_location if to_item else None),
}
)
return diff
@@ -449,27 +598,33 @@ class FileCommitService(CommonService):
live_hash = _compute_file_hash(folder_id, fid)
committed_hash = committed_entry.get("hash", "")
if live_hash and live_hash != committed_hash:
changes.append({
"file_id": fid,
"file_name": committed_entry.get("name", ""),
"operation": "modify",
})
changes.append(
{
"file_id": fid,
"file_name": committed_entry.get("name", ""),
"operation": "modify",
}
)
else:
if FileService.get_or_none(id=fid) is None:
changes.append({
"file_id": fid,
"file_name": committed_entry.get("name", ""),
"operation": "delete",
})
changes.append(
{
"file_id": fid,
"file_name": committed_entry.get("name", ""),
"operation": "delete",
}
)
# Check for newly added files
for fid, live_file in current_files.items():
if fid not in processed:
changes.append({
"file_id": fid,
"file_name": live_file.name,
"operation": "add",
})
changes.append(
{
"file_id": fid,
"file_name": live_file.name,
"operation": "add",
}
)
return changes
@@ -520,10 +675,14 @@ class FileCommitService(CommonService):
visited = set()
while current_id and current_id not in visited:
visited.add(current_id)
item = FileCommitItem.select().where(
FileCommitItem.commit_id == current_id,
FileCommitItem.file_id == file_id,
).first()
item = (
FileCommitItem.select()
.where(
FileCommitItem.commit_id == current_id,
FileCommitItem.file_id == file_id,
)
.first()
)
if item and item.new_hash:
obj_path = f".objects/{item.new_hash}"
storage_impl = settings.STORAGE_IMPL
@@ -538,6 +697,215 @@ class FileCommitService(CommonService):
return None
# ------------------------------------------------------------------
# Artifact-page commit surface
# ------------------------------------------------------------------
@classmethod
def record_page_edit(
cls,
*,
tenant_id: str,
kb_id: str,
page_type: str,
slug: str,
content_before: str,
content_after: str,
title: Optional[str] = None,
comments: Optional[str] = None,
user_id: Optional[str] = None,
) -> Optional[str]:
"""Persist one artifact-page edit as a FileCommit + FileCommitItem.
Returns the new commit id, or ``None`` when the diff is empty
(no-op save — skipped per the documented v1 contract).
Bypasses :func:`create_commit` because artifact commits have no
real ``File`` row backing them and don't participate in the
workspace ``tree_state`` snapshot chain.
"""
diff_text = _unified_diff(content_before or "", content_after or "", slug)
if not diff_text:
return None
title_ts = datetime.datetime.now().strftime("%Y%m%d%H%M%S")
final_title = f"{(title or '').strip() or f'{title_ts} {slug}'} "
commit_id = get_uuid()
item_id = get_uuid()
file_id = _artifact_file_id(kb_id, slug)
now_ts = current_timestamp()
now_dt = datetime_format(date_time=datetime.datetime.now())
# Persist the post-save markdown per the configured storage.
# A failure here logs but doesn't block the commit row — the diff
# is still meaningful without content_after.
storage_kind, location = _store_content_after(kb_id, content_after or "")
# Chain to the previous commit for this page so the history stays
# ordered even under concurrent writes (auto-regen + user edit).
parent = (
FileCommit.select(FileCommit.id)
.join(
FileCommitItem,
on=(FileCommitItem.commit_id == FileCommit.id),
)
.where((FileCommit.folder_id == kb_id) & (FileCommitItem.file_id == file_id))
.order_by(FileCommit.create_time.desc())
.first()
)
parent_id = parent.id if parent else None
try:
with DB.atomic():
FileCommit(
id=commit_id,
folder_id=kb_id,
parent_id=parent_id,
# ``message`` stays populated with the same string as
# ``title`` so any generic file-commit consumer still
# renders something sensible.
message=final_title[:512],
author_id=user_id or "",
file_count=1,
tree_state=None,
title=final_title[:255],
comments=comments or "",
create_time=now_ts,
create_date=now_dt,
update_time=now_ts,
update_date=now_dt,
).save(force_insert=True)
FileCommitItem(
id=item_id,
commit_id=commit_id,
file_id=file_id,
operation="modify" if content_before else "add",
diff=diff_text,
content_after_storage=storage_kind or None,
content_after_location=location or None,
slug_kwd=slug,
page_type_kwd=page_type,
create_time=now_ts,
create_date=now_dt,
update_time=now_ts,
update_date=now_dt,
).save(force_insert=True)
except Exception:
logging.exception(
"record_page_edit: insert failed for kb=%s slug=%s",
kb_id,
slug,
)
return None
return commit_id
@classmethod
@DB.connection_context()
def list_page_commits(
cls,
tenant_id: str,
kb_id: str,
slug: str,
page: int = 1,
page_size: int = 50,
) -> tuple[int, list[dict]]:
"""Return (total, items) for one artifact page's history.
Filters by ``FileCommitItem.slug_kwd``; joins User for nickname.
Heavy columns (``diff``, ``content_after``) are excluded — the
detail path fetches them lazily.
"""
page = max(int(page or 1), 1)
page_size = max(min(int(page_size or 50), 200), 1)
file_id = _artifact_file_id(kb_id, slug)
base = (
FileCommit.select(
FileCommit.id,
FileCommit.title,
FileCommit.comments,
FileCommit.author_id,
FileCommit.create_time,
FileCommit.create_date,
)
.join(FileCommitItem, on=(FileCommitItem.commit_id == FileCommit.id))
.where((FileCommit.folder_id == kb_id) & (FileCommitItem.file_id == file_id) & (FileCommitItem.slug_kwd == slug))
)
total = base.count()
rows = list(base.order_by(FileCommit.create_time.desc()).paginate(page, page_size).dicts())
# Preserve the previous response key so callers only re-key once.
for r in rows:
r["user_id"] = r.pop("author_id", None)
user_ids = {r["user_id"] for r in rows if r.get("user_id")}
nickname_by_id: dict[str, str] = {}
if user_ids:
try:
for u in User.select(User.id, User.nickname).where(User.id.in_(list(user_ids))).dicts():
nickname_by_id[u["id"]] = u.get("nickname") or ""
except Exception:
logging.exception(
"list_page_commits: nickname lookup failed",
)
for r in rows:
r["user_nickname"] = nickname_by_id.get(r.get("user_id") or "", "")
return total, rows
@classmethod
@DB.connection_context()
def get_page_commit_detail(
cls,
tenant_id: str,
kb_id: str,
commit_id: str,
) -> Optional[dict]:
"""Return one artifact commit including ``diff`` +
``content_after`` (resolved from storage), or ``None`` when not
found. Scoped by ``folder_id == kb_id`` so a leaked commit id
can't be read cross-tenant.
"""
commit = FileCommit.get_or_none(
(FileCommit.id == commit_id) & (FileCommit.folder_id == kb_id),
)
if commit is None:
return None
item = FileCommitItem.get_or_none(FileCommitItem.commit_id == commit_id)
if item is None:
return None
content_after = _read_content_after(
kb_id,
item.content_after_storage or "",
item.content_after_location or "",
)
nickname = ""
if commit.author_id:
try:
u = User.get_or_none(User.id == commit.author_id)
if u is not None:
nickname = u.nickname or ""
except Exception:
pass
return {
"id": commit.id,
"tenant_id": tenant_id,
"kb_id": kb_id,
"page_type_kwd": item.page_type_kwd,
"slug": item.slug_kwd,
"user_id": commit.author_id or None,
"user_nickname": nickname,
"title": commit.title,
"comments": commit.comments,
"diff": item.diff,
"content_after": content_after,
"create_time": commit.create_time,
"create_date": commit.create_date,
}
@classmethod
@DB.connection_context()
def get_file_version_history(cls, file_id):
@@ -545,20 +913,21 @@ class FileCommitService(CommonService):
Returns list of dicts: [{"commit_id", "operation", "hash", "create_time", "message"}]
"""
items = FileCommitItem.select().where(FileCommitItem.file_id == file_id).order_by(
FileCommitItem.create_time.desc())
items = FileCommitItem.select().where(FileCommitItem.file_id == file_id).order_by(FileCommitItem.create_time.desc())
versions = []
for item in items:
commit = cls.get_commit(item.commit_id)
if commit:
versions.append({
"commit_id": item.commit_id,
"operation": item.operation,
"hash": item.new_hash or item.old_hash or "",
"create_time": item.create_time,
"message": commit.message,
})
versions.append(
{
"commit_id": item.commit_id,
"operation": item.operation,
"hash": item.new_hash or item.old_hash or "",
"create_time": item.create_time,
"message": commit.message,
}
)
return versions
@@ -620,9 +989,7 @@ def _build_hierarchical_tree(tree_state, root_folder_id):
}
# File children
for fid, entry in files_by_parent.get(node_id, []):
fn = {"id": fid, "name": entry.get("name", fid), "type": "file",
"hash": entry.get("hash", ""), "size": entry.get("size", 0),
"status": entry.get("status", "1")}
fn = {"id": fid, "name": entry.get("name", fid), "type": "file", "hash": entry.get("hash", ""), "size": entry.get("size", 0), "status": entry.get("status", "1")}
if entry.get("location"):
fn["location"] = entry["location"]
node["children"].append(fn)