zlei6/ragflow
1
0
Fork 0
mirror of https://github.com/infiniflow/ragflow.git synced 2026-07-25 09:53:29 +08:00
Code Issues Packages Projects Releases Wiki Activity

fix: stabilize knowledge compilation navigation updates (#17345)

Browse Source
This commit is contained in:
buua436
2026-07-24 17:48:27 +08:00
committed by GitHub
parent 1f97823ee3
commit 2ba7ccecaf
12 changed files with 181 additions and 76 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
api/apps/restful_apis/agent_api.py
View File

@@ -1135,6 +1135,9 @@ async def rerun_agent(tenant_id):
if 0 < doc["progress"] < 1:
return get_data_error_result(message=f"`{doc['name']}` is processing...")
from rag.advanced_rag.knowlege_compile.dataset_nav import remove_dataset_nav_doc_sync
remove_dataset_nav_doc_sync(tenant_id, doc["kb_id"], doc["id"])
if settings.docStoreConn.index_exist(search.index_name(tenant_id), doc["kb_id"]):
settings.docStoreConn.delete({"doc_id": doc["id"]}, search.index_name(tenant_id), doc["kb_id"])
doc["progress_msg"] = ""

6
api/apps/restful_apis/document_api.py
View File

@@ -1470,6 +1470,9 @@ def _run_sync(user_id: str, req):
DocumentService.update_by_id(doc_id, info)
if req.get("delete", False):
TaskService.filter_delete([Task.doc_id == doc_id])
from rag.advanced_rag.knowlege_compile.dataset_nav import remove_dataset_nav_doc_sync
remove_dataset_nav_doc_sync(doc_tenant_id, doc.kb_id, doc.id)
if settings.docStoreConn.index_exist(search.index_name(doc_tenant_id), doc.kb_id):
settings.docStoreConn.delete({"doc_id": doc_id}, search.index_name(doc_tenant_id), doc.kb_id)
@@ -1580,6 +1583,9 @@ async def parse_documents(tenant_id, dataset_id):
DocumentService.update_by_id(doc_id, info)
TaskService.filter_delete([Task.doc_id == doc_id])
from rag.advanced_rag.knowlege_compile.dataset_nav import remove_dataset_nav_doc_sync
remove_dataset_nav_doc_sync(tenant_id, doc.kb_id, doc.id)
if settings.docStoreConn.index_exist(search.index_name(tenant_id), doc.kb_id):
settings.docStoreConn.delete({"doc_id": doc_id}, search.index_name(tenant_id), doc.kb_id)

32
api/apps/services/dataset_api_service.py
View File

@@ -2452,7 +2452,10 @@ def _nav_item(row: dict) -> dict:
payload = json.loads(row.get("content_with_weight") or "{}")
except Exception:
payload = {}
is_cluster = (row.get("type_kwd") or payload.get("type")) == "nav_cluster"
row_type = row.get("type_kwd")
if isinstance(row_type, (list, tuple, set)):
row_type = next(iter(row_type), None)
is_cluster = (row_type or payload.get("type")) == "nav_cluster"
return {
"name": row.get("name") or "",
"description": payload.get("description") or "",
@@ -2551,7 +2554,8 @@ async def delete_nav(dataset_id: str, tenant_id: str):
index_nm, _ = pack
try:
deleted = settings.docStoreConn.delete(
deleted = await thread_pool_exec(
settings.docStoreConn.delete,
{"compile_kwd": [_NAV_COMPILE_KWD]},
index_nm,
dataset_id,
@@ -2592,16 +2596,17 @@ async def delete_nav_node(dataset_id: str, tenant_id: str, name: str):
if not frontier:
break
try:
res = settings.docStoreConn.search(
select_fields=["name"],
highlight_fields=[],
condition={"compile_kwd": [_NAV_COMPILE_KWD], "parent_kwd": frontier},
match_expressions=[],
order_by=OrderByExpr(),
offset=0,
limit=10000,
index_names=index_nm,
knowledgebase_ids=[dataset_id],
res = await thread_pool_exec(
settings.docStoreConn.search,
["name"],
[],
{"compile_kwd": [_NAV_COMPILE_KWD], "parent_kwd": frontier},
[],
OrderByExpr(),
0,
10000,
index_nm,
[dataset_id],
)
rows = settings.docStoreConn.get_fields(res, ["name"]) or {}
except Exception:
@@ -2616,7 +2621,8 @@ async def delete_nav_node(dataset_id: str, tenant_id: str, name: str):
frontier = nxt
try:
deleted = settings.docStoreConn.delete(
deleted = await thread_pool_exec(
settings.docStoreConn.delete,
{"compile_kwd": [_NAV_COMPILE_KWD], "name": list(names)},
index_nm,
dataset_id,

28
api/db/services/document_service.py
View File

@@ -492,6 +492,20 @@ class DocumentService(CommonService):
except Exception as e:
logging.warning(f"Failed to delete thumbnail for document {doc.id}: {e}")
# Prune this doc's line from the KB's tree-kind navigation before the
# broad doc_id delete below removes the nav_doc row needed to locate
# and update its parent cluster.
try:
from rag.advanced_rag.knowlege_compile.dataset_nav import (
remove_dataset_nav_doc_sync,
)
remove_dataset_nav_doc_sync(tenant_id, doc.kb_id, doc.id)
except Exception as e:
logging.warning(
f"Failed to prune dataset_nav for document {doc.id}: {e}",
)
# Delete chunks from doc store - this is critical, log errors
try:
settings.docStoreConn.delete({"doc_id": doc.id}, chunk_index_name, doc.kb_id)
@@ -507,20 +521,6 @@ class DocumentService(CommonService):
except Exception as e:
logging.warning(f"Failed to clean up artifact products for document {doc.id}: {e}")
# Prune this doc's line from the KB's tree-kind navigation
# markdown (best-effort — the markdown is a downstream artifact,
# and failure here must not block the document delete).
try:
from rag.advanced_rag.knowlege_compile.dataset_nav import (
remove_dataset_nav_doc_sync,
)
remove_dataset_nav_doc_sync(tenant_id, doc.kb_id, doc.id)
except Exception as e:
logging.warning(
f"Failed to prune dataset_nav for document {doc.id}: {e}",
)
# Delete document metadata (non-critical, log and continue)
try:
DocMetadataService.delete_document_metadata(doc.id, doc.kb_id, tenant_id)

12
common/doc_store/infinity_conn_base.py
View File

@@ -33,7 +33,6 @@ from rag.nlp import is_english
from common import settings
from common.doc_store.doc_store_base import DocStoreConnection, MatchExpr, OrderByExpr
# Concurrent CREATE/DROP TABLE on the same Infinity instance can race on
# Infinity's RocksDB-backed catalog counters (e.g. ``db|1|next_table_id``).
# When two writers touch the counter at the same instant, Infinity surfaces
@@ -321,6 +320,17 @@ class InfinityConnectionBase(DocStoreConnection):
json_conditions.append(f"json_contains({k}, '{literal}')")
if json_conditions:
cond.append("(" + " or ".join(json_conditions) + ")")
elif k in {"compile_kwd", "type_kwd", "parent_kwd"}:
values = v if isinstance(v, list) else [v]
exact_conditions = []
for item in values:
if isinstance(item, str):
item = item.replace("'", "''")
exact_conditions.append(f"{k}='{item}'")
else:
exact_conditions.append(f"{k}={item}")
if exact_conditions:
cond.append("(" + " or ".join(exact_conditions) + ")")
elif self.field_keyword(k):
if isinstance(v, list):
inCond = list()

124
rag/advanced_rag/knowlege_compile/dataset_nav.py
View File

@@ -185,6 +185,7 @@ async def _store_knn(
"content_with_weight",
"name",
"doc_id",
"compile_kwd",
"type_kwd",
"parent_kwd",
"depth_int",
@@ -213,7 +214,13 @@ async def _store_knn(
[kb_id],
)
results = settings.docStoreConn.get_fields(res, fields) if res else {}
return list(results.values())
rows = list(results.values())
if filter_condition and any(not _matches_condition(row, filter_condition) for row in rows):
scanned = await _store_search(tenant_id, kb_id, filter_condition, fields, limit=10000)
rows = [row for row in scanned if _vector_len(row.get(vf)) == vec_dim and _matches_condition(row, filter_condition)]
rows.sort(key=lambda row: _cosine_sim(vec, row.get(vf)), reverse=True)
rows = rows[:top_k]
return rows
async def _store_upsert(tenant_id: str, kb_id: str, doc: dict) -> None:
@@ -388,6 +395,19 @@ def _fine_tokenize(text: str) -> str:
return rag_tokenizer.fine_grained_tokenize(text)
def _matches_condition(row: dict, condition: dict) -> bool:
"""Check the simple equality filters used by dataset navigation."""
for field, expected in condition.items():
if not expected or field == "kb_id":
continue
actual = row.get(field)
actual_values = actual if isinstance(actual, (list, tuple, set)) else [actual]
expected_values = expected if isinstance(expected, (list, tuple, set)) else [expected]
if not any(str(value) == str(item) for value in actual_values for item in expected_values):
return False
return True
# ---------------------------------------------------------------------------
# Incremental clustering core
# ---------------------------------------------------------------------------
@@ -426,6 +446,7 @@ async def _find_best_cluster(
# compute actual similarity to root
stored = best.get(_vec_field(vec_dim))
sim = _cosine_sim(doc_embedding, stored)
visited_names = {best_name}
# Step 2: recursively descend into children
while sim >= _RECURSE_THRESHOLD:
@@ -439,14 +460,18 @@ async def _find_best_cluster(
if not children:
break
child = children[0]
child_name = child.get("name", "")
if not child_name or child_name in visited_names:
break
stored = child.get(_vec_field(vec_dim))
child_sim = _cosine_sim(doc_embedding, stored)
if child_sim < _RECURSE_THRESHOLD:
break
best_name = child.get("name", best_name)
best_name = child_name
best_parent = best.get("parent_kwd", best_parent)
sim = child_sim
best = child
visited_names.add(best_name)
return best_name, best_parent, sim
@@ -568,15 +593,9 @@ async def upsert_dataset_nav_doc(
logging.info("dataset_nav: skipping doc=%s (kb=%s) — no summary", doc_id, kb_id)
return
# 2. Check if this doc already has a nav_doc row
existing_doc = await _store_get(tenant_id, kb_id, _nav_doc_id(doc_id))
if existing_doc:
old_payload = json.loads(existing_doc.get("content_with_weight") or "{}")
if old_payload.get("description") == summary:
logging.info("dataset_nav: doc=%s unchanged, skipping", doc_id)
return
# 3. Embed doc summary
# 2. Embed doc summary before taking the KB lock. The result is
# independent of the nav tree; all tree reads, deletes, and writes below
# are serialized by the same lock.
doc_embedding = await _embed(embd_mdl, summary) if embd_mdl else []
vec_dim = len(doc_embedding)
@@ -592,8 +611,18 @@ async def upsert_dataset_nav_doc(
return
try:
# 3. Check and replace the existing nav_doc while holding the same
# lock as placement. Otherwise concurrent updates can both observe
# the old row and race while deleting/rebuilding its parent cluster.
existing_doc = await _store_get(tenant_id, kb_id, _nav_doc_id(doc_id))
if existing_doc:
old_payload = json.loads(existing_doc.get("content_with_weight") or "{}")
if old_payload.get("description") == summary:
return
await _remove_dataset_nav_doc_locked(tenant_id, kb_id, doc_id)
# 4. Layered KNN search for nearest cluster
if doc_embedding:
if _vector_len(doc_embedding) > 0:
best_name, best_parent, sim = await _find_best_cluster(
tenant_id,
kb_id,
@@ -726,6 +755,45 @@ async def upsert_dataset_nav_doc(
logging.exception("dataset_nav: lock release failed for kb=%s", kb_id)
async def _remove_dataset_nav_doc_locked(
tenant_id: str,
kb_id: str,
doc_id: str,
) -> None:
"""Remove a document nav row; the caller must hold the KB nav lock."""
# 1. Find and delete the nav_doc row
doc_row_id = _nav_doc_id(doc_id)
doc_row = await _store_get(tenant_id, kb_id, doc_row_id)
if not doc_row:
return
parent_name = doc_row.get("parent_kwd", "")
await _store_delete(tenant_id, kb_id, doc_row_id)
# 2. Remove doc_id from the parent cluster's doc_ids_kwd
if parent_name and parent_name != "root":
cluster_id = _nav_cluster_id(kb_id, parent_name)
cluster_row = await _store_get(tenant_id, kb_id, cluster_id)
if cluster_row:
doc_ids = cluster_row.get("doc_ids_kwd") or []
if doc_id in doc_ids:
doc_ids.remove(doc_id)
if not doc_ids:
# Cluster is empty — delete it
await _store_delete(tenant_id, kb_id, cluster_id)
# Recurse: check grandparent
grandparent = cluster_row.get("parent_kwd", "")
if grandparent and grandparent != "root":
await _cleanup_empty_cluster(
tenant_id,
kb_id,
grandparent,
)
else:
cluster_row["doc_ids_kwd"] = doc_ids
cluster_row["doc_count_int"] = len(doc_ids)
await _store_upsert(tenant_id, kb_id, cluster_row)
async def remove_dataset_nav_doc(
tenant_id: str,
kb_id: str,
@@ -751,37 +819,7 @@ async def remove_dataset_nav_doc(
return
try:
# 1. Find and delete the nav_doc row
doc_row_id = _nav_doc_id(doc_id)
doc_row = await _store_get(tenant_id, kb_id, doc_row_id)
if not doc_row:
return
parent_name = doc_row.get("parent_kwd", "")
await _store_delete(tenant_id, kb_id, doc_row_id)
# 2. Remove doc_id from the parent cluster's doc_ids_kwd
if parent_name and parent_name != "root":
cluster_id = _nav_cluster_id(kb_id, parent_name)
cluster_row = await _store_get(tenant_id, kb_id, cluster_id)
if cluster_row:
doc_ids = cluster_row.get("doc_ids_kwd") or []
if doc_id in doc_ids:
doc_ids.remove(doc_id)
if not doc_ids:
# Cluster is empty — delete it
await _store_delete(tenant_id, kb_id, cluster_id)
# Recurse: check grandparent
grandparent = cluster_row.get("parent_kwd", "")
if grandparent and grandparent != "root":
await _cleanup_empty_cluster(
tenant_id,
kb_id,
grandparent,
)
else:
cluster_row["doc_ids_kwd"] = doc_ids
cluster_row["doc_count_int"] = len(doc_ids)
await _store_upsert(tenant_id, kb_id, cluster_row)
await _remove_dataset_nav_doc_locked(tenant_id, kb_id, doc_id)
except Exception:
logging.exception(
"dataset_nav: remove failed for kb=%s doc=%s",

17
rag/advanced_rag/knowlege_compile/runner.py
View File

@@ -197,13 +197,27 @@ async def _upsert_dataset_nav_from_page_index(
if cancel_check():
raise TaskCanceledException("Task was cancelled before dataset navigation update")
try:
# PageIndex rows use their preserved template kind as the
# compile keyword. Older rows were written as ``timeline``
# before compilation kinds were kept distinct, so fall back to
# the legacy keyword when rebuilding an existing graph.
graph = await rebuild_structure_graph_json(
tenant_id,
kb_id,
doc_id,
"timeline",
"page_index",
compilation_template_id=template_id,
)
summary = _page_index_graph_summary(graph)
if not summary:
graph = await rebuild_structure_graph_json(
tenant_id,
kb_id,
doc_id,
"timeline",
compilation_template_id=template_id,
)
summary = _page_index_graph_summary(graph)
except Exception:
logging.exception(
"page_index: failed to rebuild graph summary for dataset_nav doc %s template %s",
@@ -212,7 +226,6 @@ async def _upsert_dataset_nav_from_page_index(
)
continue
summary = _page_index_graph_summary(graph)
if summary:
summaries.append(summary)
chat_mdl = chat_mdl or chat_mdl_by_tid.get(template_id)

9
rag/flow/compiler/compiler.py
View File

@@ -163,7 +163,14 @@ class Compiler(ProcessBase, LLM):
try:
from rag.advanced_rag.knowlege_compile.dataset_nav import upsert_dataset_nav_doc
await upsert_dataset_nav_doc(tenant_id, kb_id, doc_id, tree)
await upsert_dataset_nav_doc(
tenant_id,
kb_id,
doc_id,
tree,
embd_mdl=embedding_model,
chat_mdl=chat_mdl_by_tid[template_id],
)
except Exception:
logging.exception("Compiler: tree-template %s dataset navigation upsert failed for doc %s", template_id, doc_id)

2
rag/svr/task_executor_refactor/chunk_post_processor.py
View File

@@ -1009,6 +1009,8 @@ async def run_tree_templates(
ctx.kb_id,
doc_id,
tree,
embd_mdl=embedding_model,
chat_mdl=chat_mdl_by_tid[template_id],
)
except Exception:
logging.exception(

16
rag/utils/infinity_conn.py
View File

@@ -649,6 +649,22 @@ class InfinityConnection(InfinityConnectionBase):
if k in new_value:
del new_value[k]
# The Infinity Python client inspects value[0] for list values
# while building an update expression. An empty list therefore
# raises IndexError before the request reaches Infinity. Keep
# JSON and keyword-list columns clearable, but do not send empty
# values for other columns (for example, an empty vector returned
# by a partial row read).
for k, v in list(new_value.items()):
if not isinstance(v, list) or v:
continue
if k in _JSON_LIST_FIELDS:
new_value[k] = json.dumps([], ensure_ascii=False)
elif self.field_keyword(k):
new_value[k] = ""
else:
del new_value[k]
remove_opt = {} # "[k,new_value]": [id_to_update, ...]
if removeValue:
col_to_remove = list(removeValue.keys())

6
web/src/components/ui/tree-view.tsx
View File

@@ -23,6 +23,7 @@ export interface TreeDataItem {
selectedIcon?: any;
openIcon?: any;
children?: TreeDataItem[];
hasChildren?: boolean;
actions?: React.ReactNode;
onClick?: () => void;
}
@@ -142,7 +143,8 @@ const TreeItem = React.forwardRef<HTMLDivElement, TreeItemProps>(
<ul>
{data.map((item) => (
<li key={item.id}>
{item.children && item.children.length > 0 ? (
{item.hasChildren ||
(item.children && item.children.length > 0) ? (
<TreeNode
item={item}
selectedItemId={selectedItemId}
@@ -216,7 +218,7 @@ const TreeNode = ({
</AccordionTrigger>
<AccordionContent className="ml-4 pl-1 border-l">
<TreeItem
data={item.children ? item.children : item}
data={item.children ?? []}
selectedItemId={selectedItemId}
handleSelectChange={handleSelectChange}
expandedItemIds={expandedItemIds}

2
web/src/pages/dataset/compilation/utils/nav-tree.ts
View File

@@ -29,6 +29,7 @@ export function buildNavTreeData(
const item: TreeDataItem = {
id: node.name,
name: node.name,
hasChildren: node.has_children,
actions: getActions?.(node, null),
onClick: () => onParentClick(node),
};
@@ -39,6 +40,7 @@ export function buildNavTreeData(
item.children = children.map((child) => ({
id: `${node.name}/${child.name}`,
name: child.name,
hasChildren: child.has_children,
actions: getActions?.(child, node.name),
onClick: () => onChildClick(child, node.name),
}));