mirror of
https://github.com/infiniflow/ragflow.git
synced 2026-07-25 18:03:29 +08:00
Feat: Add graph keyword search and fix dataset synthesizing issue. (#17342)
### Summary Add graph keyword search and fix dataset synthesizing issue.
This commit is contained in:
@@ -53,9 +53,9 @@ _STRUCTURE_INDEX_TYPES = frozenset(_STRUCTURE_INDEX_TYPE_TO_KIND)
|
||||
_VALID_INDEX_TYPES = {"graph", "raptor", "mindmap", "artifact", "skill"} | set(_STRUCTURE_INDEX_TYPES)
|
||||
|
||||
_INDEX_TYPE_TO_TASK_TYPE = {
|
||||
"graph": "graphrag",
|
||||
"graph": "structure_graph",
|
||||
"raptor": "raptor",
|
||||
"mindmap": "mindmap",
|
||||
"mindmap": "structure_mindmap",
|
||||
"artifact": "artifact",
|
||||
"skill": "skill",
|
||||
# Structure merge types carry their own task_type (== index_type) so the
|
||||
@@ -1723,16 +1723,20 @@ def _resolve_dataset_structure_kind(kind) -> str | None:
|
||||
return _DATASET_STRUCTURE_KIND_ALIASES.get(kind.strip().lower().replace("-", "_"))
|
||||
|
||||
|
||||
async def get_dataset_structure(dataset_id: str, tenant_id: str, kind: str):
|
||||
async def get_dataset_structure(dataset_id: str, tenant_id: str, kind: str, keywords: str = ""):
|
||||
"""Load the dataset-scope (KB-wide) structure graph for one ``kind``.
|
||||
|
||||
``kind`` is one of ``graph`` / ``mindmap`` / ``timeline`` /
|
||||
``session_essence`` / ``session_graph``. Reads the
|
||||
``knowledge_graph_kwd="dataset_graph"`` rows written by
|
||||
``rebuild_dataset_structure_graph_json`` (one per template), filters to the
|
||||
requested kind, and returns them grouped by template — mirroring the
|
||||
per-document ``structure/graph`` response so the frontend graph view is
|
||||
reused unchanged.
|
||||
``session_essence`` / ``session_graph``. The ``knowledge_graph_kwd="dataset_graph"``
|
||||
blob rows (one per template, written by ``rebuild_dataset_structure_graph_json``)
|
||||
are used only to DISCOVER the requested kind's template buckets; each bucket's
|
||||
entities/relations are then fetched from the raw KB-wide ``entity`` / ``relation``
|
||||
rows with subgraph sampling — identical to the per-document endpoint but scoped
|
||||
KB-wide (no ``doc_id`` filter), since dataset-merge templates dedup those rows
|
||||
across documents.
|
||||
|
||||
``keywords`` (optional): return the single best-matching entity's 1-hop
|
||||
subgraph (top-1 + neighbors + touching relations) across the kind, via KNN.
|
||||
|
||||
Returns ``(True, {"kind": <kind>, "templates": [...]})`` or
|
||||
``(False, message)`` on auth/validation failure.
|
||||
@@ -1754,30 +1758,10 @@ async def get_dataset_structure(dataset_id: str, tenant_id: str, kind: str):
|
||||
|
||||
from common.doc_store.doc_store_base import OrderByExpr
|
||||
from api.db.services.compilation_template_service import CompilationTemplateService
|
||||
from api.db.services.tenant_llm_service import TenantLLMService
|
||||
from api.apps.services import structure_graph_common as sgc
|
||||
|
||||
select_fields = [
|
||||
"content_with_weight",
|
||||
"compile_kwd",
|
||||
"compilation_template_ids",
|
||||
"compilation_template_kind_kwd",
|
||||
]
|
||||
try:
|
||||
res = await thread_pool_exec(
|
||||
settings.docStoreConn.search,
|
||||
select_fields,
|
||||
[],
|
||||
{"knowledge_graph_kwd": [_DATASET_STRUCTURE_ROW_KWD]},
|
||||
[],
|
||||
OrderByExpr(),
|
||||
0,
|
||||
1000,
|
||||
index_nm,
|
||||
[dataset_id],
|
||||
)
|
||||
rows = settings.docStoreConn.get_fields(res, select_fields) or {}
|
||||
except Exception:
|
||||
logging.exception("get_dataset_structure: docStore search failed for kb=%s", dataset_id)
|
||||
return True, empty
|
||||
keywords = (keywords or "").strip()
|
||||
|
||||
def _row_template_id(row: dict) -> str | None:
|
||||
raw = row.get("compilation_template_ids")
|
||||
@@ -1811,40 +1795,148 @@ async def get_dataset_structure(dataset_id: str, tenant_id: str, kind: str):
|
||||
template_kind_cache[tid] = top_kind
|
||||
return top_kind
|
||||
|
||||
grouped: dict[str, dict] = {}
|
||||
for row in rows.values():
|
||||
def _bucket_meta_for(tid: str, row_kind: str = "") -> dict:
|
||||
if tid not in template_name_cache:
|
||||
_template_meta(tid)
|
||||
return {
|
||||
"template_id": tid,
|
||||
"template_name": template_name_cache.get(tid, tid),
|
||||
"kind": row_kind or template_kind_cache.get(tid) or resolved_kind,
|
||||
}
|
||||
|
||||
# ── Discovery: dataset_graph blob rows, metadata only (no huge content). ──
|
||||
# Keep only rows whose TOP-LEVEL kind matches the request. Raw entity/relation
|
||||
# rows stamp ``compilation_template_kind_kwd`` with config.kind (which folds the
|
||||
# knowledge_graph family), so we scope raw-row queries by template id — resolved
|
||||
# here from the blobs, whose stamp is the top-level kind — not by kind directly.
|
||||
meta_fields = ["compile_kwd", "compilation_template_ids", "compilation_template_kind_kwd"]
|
||||
try:
|
||||
res = await thread_pool_exec(
|
||||
settings.docStoreConn.search,
|
||||
meta_fields,
|
||||
[],
|
||||
{"knowledge_graph_kwd": [_DATASET_STRUCTURE_ROW_KWD]},
|
||||
[],
|
||||
OrderByExpr(),
|
||||
0,
|
||||
1000,
|
||||
index_nm,
|
||||
[dataset_id],
|
||||
)
|
||||
meta_rows = settings.docStoreConn.get_fields(res, meta_fields) or {}
|
||||
except Exception:
|
||||
logging.exception("get_dataset_structure: docStore discovery failed for kb=%s", dataset_id)
|
||||
return True, empty
|
||||
|
||||
kind_template_ids: list[str] = []
|
||||
seen_tid: set[str] = set()
|
||||
has_templateless = False
|
||||
for row in meta_rows.values():
|
||||
tid = _row_template_id(row)
|
||||
stamped_kind = (row.get("compilation_template_kind_kwd") or "").strip()
|
||||
row_kind = stamped_kind or _template_meta(tid) or ""
|
||||
if _resolve_dataset_structure_kind(row_kind) != resolved_kind:
|
||||
continue
|
||||
if tid:
|
||||
if tid not in seen_tid:
|
||||
seen_tid.add(tid)
|
||||
kind_template_ids.append(tid)
|
||||
else:
|
||||
has_templateless = True
|
||||
|
||||
# ── keywords mode: global KNN across the kind → top-1's focused subgraph. ──
|
||||
if keywords:
|
||||
if not kind_template_ids:
|
||||
return True, empty
|
||||
try:
|
||||
graph = json.loads(row.get("content_with_weight") or "{}")
|
||||
model_config = resolve_model_config(kb.tenant_id, LLMType.EMBEDDING.value, kb.embd_id)
|
||||
embd_mdl = TenantLLMService.model_instance(model_config)
|
||||
except Exception:
|
||||
logging.exception("get_dataset_structure: embedding bind failed for kb=%s", dataset_id)
|
||||
return True, empty
|
||||
|
||||
def _scope_for_template(row: dict):
|
||||
tid = _row_template_id(row) or ""
|
||||
stamped = (row.get("compilation_template_kind_kwd") or "").strip()
|
||||
meta = _bucket_meta_for(tid, stamped) if tid else {"template_id": f"kind:{resolved_kind}", "template_name": f"kind:{resolved_kind}", "kind": resolved_kind}
|
||||
return meta, {"compilation_template_ids": [tid]} if tid else {"compilation_template_kind_kwd": [stamped]}
|
||||
|
||||
bucket_meta, kw_entities, kw_relations = await sgc.keyword_subgraph(
|
||||
index_nm,
|
||||
dataset_id,
|
||||
embd_mdl,
|
||||
{"compilation_template_ids": kind_template_ids, "knowledge_graph_kwd": ["entity"]},
|
||||
keywords,
|
||||
_scope_for_template,
|
||||
log_ctx=f"kb={dataset_id}",
|
||||
)
|
||||
if resolved_kind == "mind_map":
|
||||
kw_entities = sgc.filter_entities_with_relations(kw_entities, kw_relations)
|
||||
if not kw_entities:
|
||||
return True, empty
|
||||
if not bucket_meta or (not kw_entities and not kw_relations):
|
||||
return True, empty
|
||||
bucket = dict(bucket_meta)
|
||||
bucket["entities"] = kw_entities
|
||||
bucket["relations"] = kw_relations
|
||||
return True, {"kind": kind, "templates": [bucket]}
|
||||
|
||||
# ── normal mode: per-template subgraph sampling from raw KB-wide rows. ──
|
||||
templates_out: list[dict] = []
|
||||
for tid in kind_template_ids:
|
||||
try:
|
||||
entities, relations = await sgc.build_bucket(index_nm, dataset_id, {"compilation_template_ids": [tid]})
|
||||
except Exception:
|
||||
logging.exception("get_dataset_structure: bucket build failed for kb=%s template=%s", dataset_id, tid)
|
||||
continue
|
||||
if not isinstance(graph, dict):
|
||||
continue
|
||||
entities = graph.get("entities") or []
|
||||
relations = graph.get("relations") or []
|
||||
if resolved_kind == "mind_map":
|
||||
entities = sgc.filter_entities_with_relations(entities, relations)
|
||||
if not entities:
|
||||
continue
|
||||
if not entities and not relations:
|
||||
continue
|
||||
meta = _bucket_meta_for(tid)
|
||||
templates_out.append({**meta, "entities": entities, "relations": relations})
|
||||
|
||||
bucket_id = tid or f"kind:{resolved_kind}"
|
||||
if bucket_id not in grouped:
|
||||
if tid and tid not in template_name_cache:
|
||||
_template_meta(tid)
|
||||
grouped[bucket_id] = {
|
||||
"template_id": bucket_id,
|
||||
"template_name": template_name_cache.get(tid or "", bucket_id),
|
||||
"kind": row_kind or resolved_kind,
|
||||
"entities": [],
|
||||
"relations": [],
|
||||
}
|
||||
grouped[bucket_id]["entities"].extend(entities)
|
||||
grouped[bucket_id]["relations"].extend(relations)
|
||||
# Legacy template-less dataset_graph rows have no template id to scope raw
|
||||
# rows by, so fall back to their blob content directly (no sampling). Rare —
|
||||
# rebuild_dataset_structure_graph_json always stamps a template id.
|
||||
if has_templateless:
|
||||
try:
|
||||
res_l = await thread_pool_exec(
|
||||
settings.docStoreConn.search,
|
||||
["content_with_weight", "compilation_template_kind_kwd"],
|
||||
[],
|
||||
{"knowledge_graph_kwd": [_DATASET_STRUCTURE_ROW_KWD], "must_not": {"exists": "compilation_template_ids"}},
|
||||
[],
|
||||
OrderByExpr(),
|
||||
0,
|
||||
1000,
|
||||
index_nm,
|
||||
[dataset_id],
|
||||
)
|
||||
legacy_rows = settings.docStoreConn.get_fields(res_l, ["content_with_weight", "compilation_template_kind_kwd"]) or {}
|
||||
except Exception:
|
||||
logging.exception("get_dataset_structure: legacy blob fetch failed for kb=%s", dataset_id)
|
||||
legacy_rows = {}
|
||||
legacy_bucket = {"template_id": f"kind:{resolved_kind}", "template_name": f"kind:{resolved_kind}", "kind": resolved_kind, "entities": [], "relations": []}
|
||||
for row in legacy_rows.values():
|
||||
if _resolve_dataset_structure_kind((row.get("compilation_template_kind_kwd") or "").strip()) != resolved_kind:
|
||||
continue
|
||||
try:
|
||||
graph = json.loads(row.get("content_with_weight") or "{}")
|
||||
except Exception:
|
||||
continue
|
||||
if not isinstance(graph, dict):
|
||||
continue
|
||||
legacy_bucket["entities"].extend(graph.get("entities") or [])
|
||||
legacy_bucket["relations"].extend(graph.get("relations") or [])
|
||||
if resolved_kind == "mind_map":
|
||||
legacy_bucket["entities"] = sgc.filter_entities_with_relations(legacy_bucket["entities"], legacy_bucket["relations"])
|
||||
if legacy_bucket["entities"] or legacy_bucket["relations"]:
|
||||
if resolved_kind != "mind_map" or legacy_bucket["entities"]:
|
||||
templates_out.append(legacy_bucket)
|
||||
|
||||
templates_out = [g for g in grouped.values() if g["entities"] or g["relations"]]
|
||||
return True, {"kind": kind, "templates": templates_out}
|
||||
|
||||
|
||||
@@ -2871,15 +2963,16 @@ async def _wiki_search_entity_page(
|
||||
dataset_id: str,
|
||||
offset: int,
|
||||
limit: int,
|
||||
keywords: str = "",
|
||||
):
|
||||
"""One page of artifact_entity rows, ordered by weight_int DESC."""
|
||||
from common.doc_store.doc_store_base import OrderByExpr
|
||||
"""One page of artifact_entity rows.
|
||||
|
||||
order_by = OrderByExpr()
|
||||
try:
|
||||
order_by.desc("weight_int")
|
||||
except Exception:
|
||||
order_by = OrderByExpr()
|
||||
Without ``keywords``: ordered by ``weight_int DESC`` (heaviest nodes first).
|
||||
With ``keywords``: BM25 full-text match over ``content_ltks`` (slug +
|
||||
summary), ordered by relevance. ``artifact_entity`` rows are BM25-only (no
|
||||
embedding vector), so this is a lexical search, not a dense KNN.
|
||||
"""
|
||||
from common.doc_store.doc_store_base import OrderByExpr
|
||||
|
||||
select_fields = [
|
||||
"id",
|
||||
@@ -2888,12 +2981,31 @@ async def _wiki_search_entity_page(
|
||||
"source_chunk_ids",
|
||||
"content_with_weight",
|
||||
]
|
||||
|
||||
keywords = (keywords or "").strip()
|
||||
match_expressions: list = []
|
||||
order_by = OrderByExpr()
|
||||
if keywords:
|
||||
try:
|
||||
match_text, _ = settings.retriever.qryr.question(keywords, min_match=0.1)
|
||||
match_expressions = [match_text]
|
||||
except Exception:
|
||||
logging.exception("get_wiki_graph: failed to build keyword query for kb=%s", dataset_id)
|
||||
match_expressions = []
|
||||
if not match_expressions:
|
||||
# No keywords (or query build failed) → heaviest-weighted first. When a
|
||||
# text match is present the store ranks by BM25 score instead.
|
||||
try:
|
||||
order_by.desc("weight_int")
|
||||
except Exception:
|
||||
order_by = OrderByExpr()
|
||||
|
||||
res = await thread_pool_exec(
|
||||
settings.docStoreConn.search,
|
||||
select_fields,
|
||||
[],
|
||||
{"compile_kwd": [_WIKI_GRAPH_ENTITY_KWD]},
|
||||
[],
|
||||
match_expressions,
|
||||
order_by,
|
||||
offset,
|
||||
limit,
|
||||
@@ -2975,9 +3087,16 @@ async def get_wiki_graph(
|
||||
dataset_id: str,
|
||||
tenant_id: str,
|
||||
node: str | None = None,
|
||||
keywords: str | None = None,
|
||||
top_n: int | None = None,
|
||||
):
|
||||
"""Load the canvas graph payload incrementally from per-row data.
|
||||
|
||||
``top_n`` overrides the entity budget (default ``_WIKI_GRAPH_MAX_LOADING_ENTITY``).
|
||||
``keywords`` (overview mode only; ignored when ``node`` is given) seeds the
|
||||
graph from the best BM25 matches on ``artifact_entity`` rows instead of the
|
||||
heaviest-weighted ones. Only entities referenced by a relation are returned.
|
||||
|
||||
Two modes:
|
||||
|
||||
* **Overview** (``node`` is None) — paginate ``artifact_entity`` rows
|
||||
@@ -3010,7 +3129,18 @@ async def get_wiki_graph(
|
||||
return True, empty
|
||||
index_nm, _ = pack
|
||||
|
||||
cap = _WIKI_GRAPH_MAX_LOADING_ENTITY
|
||||
from api.apps.services import structure_graph_common as sgc
|
||||
|
||||
keywords = (keywords or "").strip()
|
||||
# Entity budget: caller-overridable, clamped to a sane range so a bad param
|
||||
# can neither disable the cap nor blow up the response.
|
||||
if top_n is not None:
|
||||
try:
|
||||
cap = max(1, min(int(top_n), 1024))
|
||||
except (TypeError, ValueError):
|
||||
cap = _WIKI_GRAPH_MAX_LOADING_ENTITY
|
||||
else:
|
||||
cap = _WIKI_GRAPH_MAX_LOADING_ENTITY
|
||||
page_size = _WIKI_GRAPH_ENTITY_PAGE_SIZE
|
||||
|
||||
# ``entities`` preserves first-seen order so the canvas paints the
|
||||
@@ -3111,11 +3241,11 @@ async def get_wiki_graph(
|
||||
to_map = {}
|
||||
for row in (to_map or {}).values():
|
||||
payload = _wiki_entity_payload(row)
|
||||
if payload and len(entities) < cap:
|
||||
if payload and len(entities) < cap * 2:
|
||||
_add_entity(payload)
|
||||
|
||||
return True, {
|
||||
"entities": list(entities.values()),
|
||||
"entities": sgc.filter_entities_with_relations(list(entities.values()), relations),
|
||||
"relations": relations,
|
||||
}
|
||||
|
||||
@@ -3130,6 +3260,7 @@ async def get_wiki_graph(
|
||||
dataset_id,
|
||||
offset,
|
||||
page_size,
|
||||
keywords=keywords,
|
||||
)
|
||||
except Exception:
|
||||
logging.exception(
|
||||
@@ -3223,7 +3354,7 @@ async def get_wiki_graph(
|
||||
page += 1
|
||||
|
||||
return True, {
|
||||
"entities": list(entities.values()),
|
||||
"entities": sgc.filter_entities_with_relations(list(entities.values()), relations),
|
||||
"relations": relations,
|
||||
}
|
||||
|
||||
|
||||
304
api/apps/services/structure_graph_common.py
Normal file
304
api/apps/services/structure_graph_common.py
Normal file
@@ -0,0 +1,304 @@
|
||||
#
|
||||
# Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
#
|
||||
|
||||
"""Shared structure-graph subgraph sampling.
|
||||
|
||||
Both the per-document (``/datasets/<id>/documents/<doc>/structure/graph``) and
|
||||
the dataset-wide (``/datasets/<id>/artifacts_structure``) endpoints render
|
||||
per-template structure graphs. For large graphs we don't return every
|
||||
entity/relation — we fetch a representative subgraph from the raw
|
||||
``knowledge_graph_kwd`` rows (which carry ``mention_count_int`` / ``name_kwd`` /
|
||||
``from_entity_kwd`` / ``to_entity_kwd`` / ``q_<dim>_vec``) so the response — and
|
||||
the frontend render — stay bounded.
|
||||
|
||||
The two endpoints differ only in *scope*: the document endpoint filters raw
|
||||
rows by ``doc_id``; the dataset endpoint queries KB-wide (dataset-merge
|
||||
templates dedup entity/relation rows across documents). That difference lives
|
||||
entirely in the ``scope`` / ``base_entity_condition`` dicts the caller passes —
|
||||
everything else is shared here.
|
||||
"""
|
||||
|
||||
import json
|
||||
import logging
|
||||
|
||||
from common import settings
|
||||
from common.doc_store.doc_store_base import OrderByExpr
|
||||
from common.misc_utils import thread_pool_exec
|
||||
|
||||
|
||||
# Below this combined (entities + relations) count for a bucket, return all rows.
|
||||
GRAPH_FULL_THRESHOLD = 1024
|
||||
# Size of the top-mention entity seed set (set A) for large buckets.
|
||||
GRAPH_TOP_ENTITIES = 256
|
||||
# Upper bound on the relation / neighbor-entity expansion so a hub node can't
|
||||
# blow up the response.
|
||||
GRAPH_EXPANSION_CAP = 4096
|
||||
|
||||
GRAPH_ENTITY_FIELDS = ["id", "content_with_weight", "name_kwd", "mention_count_int", "source_chunk_ids"]
|
||||
GRAPH_RELATION_FIELDS = ["id", "content_with_weight", "from_entity_kwd", "to_entity_kwd"]
|
||||
GRAPH_ALL_FIELDS = ["id", "content_with_weight", "name_kwd", "mention_count_int", "source_chunk_ids", "from_entity_kwd", "to_entity_kwd", "knowledge_graph_kwd"]
|
||||
|
||||
|
||||
async def graph_search(index_name, kb_id, select_fields, condition, order_by, limit, match_expressions=None):
|
||||
"""One raw-row search. Returns ``(field_map, total)`` where ``total`` is the
|
||||
full match count (not the returned slice)."""
|
||||
res = await thread_pool_exec(
|
||||
settings.docStoreConn.search,
|
||||
select_fields,
|
||||
[],
|
||||
condition,
|
||||
match_expressions or [],
|
||||
order_by,
|
||||
0,
|
||||
max(int(limit or 0), 1),
|
||||
index_name,
|
||||
[kb_id],
|
||||
)
|
||||
field_map = settings.docStoreConn.get_fields(res, select_fields) or {}
|
||||
total = settings.docStoreConn.get_total(res)
|
||||
return field_map, int(total or 0)
|
||||
|
||||
|
||||
def project_entity(row: dict) -> dict | None:
|
||||
"""Project a raw ``knowledge_graph_kwd="entity"`` row to the graph-node shape
|
||||
the frontend already consumes, surfacing ``mention_count_int`` as
|
||||
``mention_count``."""
|
||||
from rag.advanced_rag.knowlege_compile.structure import _struct_graph_entity
|
||||
|
||||
try:
|
||||
payload = json.loads(row.get("content_with_weight") or "{}")
|
||||
except Exception:
|
||||
return None
|
||||
if not isinstance(payload, dict):
|
||||
return None
|
||||
node = _struct_graph_entity(payload, row.get("source_chunk_ids"))
|
||||
if not node:
|
||||
return None
|
||||
mc = row.get("mention_count_int")
|
||||
if isinstance(mc, list): # Infinity returns *_int scalars fine, but be defensive
|
||||
mc = mc[0] if mc else None
|
||||
try:
|
||||
if mc is not None:
|
||||
node["mention_count"] = int(mc)
|
||||
except (TypeError, ValueError):
|
||||
pass
|
||||
return node
|
||||
|
||||
|
||||
def project_relation(row: dict) -> dict | None:
|
||||
"""Project a raw ``knowledge_graph_kwd="relation"`` row to the edge shape.
|
||||
Prefers the payload (matching the blob projection); falls back to the
|
||||
authoritative ``*_entity_kwd`` columns."""
|
||||
from rag.advanced_rag.knowlege_compile.structure import _struct_graph_relation
|
||||
|
||||
try:
|
||||
payload = json.loads(row.get("content_with_weight") or "{}")
|
||||
except Exception:
|
||||
payload = {}
|
||||
if isinstance(payload, dict):
|
||||
node = _struct_graph_relation(payload)
|
||||
if node:
|
||||
return node
|
||||
src = str(row.get("from_entity_kwd") or "").strip()
|
||||
tgt = str(row.get("to_entity_kwd") or "").strip()
|
||||
if not src or not tgt:
|
||||
return None
|
||||
typ = payload.get("type") if isinstance(payload, dict) else None
|
||||
return {"from": src, "to": tgt, "type": str(typ).strip() if typ else "related"}
|
||||
|
||||
|
||||
def dedup_entities(entities: list[dict]) -> list[dict]:
|
||||
"""Order-preserving dedup by (lowercased name, type)."""
|
||||
out: list[dict] = []
|
||||
seen: set[tuple[str, str]] = set()
|
||||
for e in entities:
|
||||
key = (str(e.get("name") or "").strip().lower(), str(e.get("type") or "").strip().lower())
|
||||
if not key[0] or key in seen:
|
||||
continue
|
||||
seen.add(key)
|
||||
out.append(e)
|
||||
return out
|
||||
|
||||
|
||||
def filter_entities_with_relations(entities: list[dict], relations: list[dict]) -> list[dict]:
|
||||
"""Keep only entities that are referenced by at least one relation."""
|
||||
if not entities or not relations:
|
||||
return []
|
||||
|
||||
connected: set[str] = set()
|
||||
for relation in relations:
|
||||
if not isinstance(relation, dict):
|
||||
continue
|
||||
for endpoint_key in ("from", "to"):
|
||||
endpoint = relation.get(endpoint_key)
|
||||
if isinstance(endpoint, str):
|
||||
endpoint = endpoint.strip()
|
||||
if endpoint:
|
||||
connected.add(endpoint)
|
||||
|
||||
if not connected:
|
||||
return []
|
||||
|
||||
filtered: list[dict] = []
|
||||
for entity in entities:
|
||||
if not isinstance(entity, dict):
|
||||
continue
|
||||
keys: set[str] = set()
|
||||
# Structure-graph nodes are name-keyed and their relations reference
|
||||
# names; artifact-graph nodes are slug-keyed and their relations
|
||||
# reference slugs. Check all three identity fields so the same filter
|
||||
# serves both callers.
|
||||
for field in ("id", "name", "slug"):
|
||||
value = entity.get(field)
|
||||
if isinstance(value, str):
|
||||
value = value.strip()
|
||||
if value:
|
||||
keys.add(value)
|
||||
if keys & connected:
|
||||
filtered.append(entity)
|
||||
return filtered
|
||||
|
||||
|
||||
async def build_bucket(index_name, kb_id, scope: dict) -> tuple[list[dict], list[dict]]:
|
||||
"""Build one bucket's ``(entities, relations)`` from raw rows.
|
||||
|
||||
``scope`` is the filter WITHOUT ``knowledge_graph_kwd`` — e.g.
|
||||
``{"doc_id":[id], "compilation_template_ids":[tid]}`` (document scope) or
|
||||
``{"compilation_template_ids":[tid]}`` (dataset scope). Small buckets are
|
||||
returned whole; large ones are sampled: top-``GRAPH_TOP_ENTITIES`` entities
|
||||
by ``mention_count_int``, the relations sourced from them, and those
|
||||
relations' target entities.
|
||||
"""
|
||||
both_cond = dict(scope, knowledge_graph_kwd=["entity", "relation"])
|
||||
_, total = await graph_search(index_name, kb_id, ["id"], both_cond, OrderByExpr(), 1)
|
||||
|
||||
if total < GRAPH_FULL_THRESHOLD:
|
||||
field_map, _ = await graph_search(index_name, kb_id, GRAPH_ALL_FIELDS, both_cond, OrderByExpr(), total or 1)
|
||||
entities: list[dict] = []
|
||||
relations: list[dict] = []
|
||||
for row in field_map.values():
|
||||
if row.get("knowledge_graph_kwd") == "relation":
|
||||
edge = project_relation(row)
|
||||
if edge:
|
||||
relations.append(edge)
|
||||
else:
|
||||
node = project_entity(row)
|
||||
if node:
|
||||
entities.append(node)
|
||||
return dedup_entities(entities), relations
|
||||
|
||||
# Large bucket: sample. A = top entities by mention_count_int desc.
|
||||
order_by = OrderByExpr()
|
||||
try:
|
||||
order_by.desc("mention_count_int")
|
||||
except Exception:
|
||||
order_by = OrderByExpr()
|
||||
ent_a_map, _ = await graph_search(index_name, kb_id, GRAPH_ENTITY_FIELDS, dict(scope, knowledge_graph_kwd=["entity"]), order_by, GRAPH_TOP_ENTITIES)
|
||||
set_a = [n for n in (project_entity(r) for r in ent_a_map.values()) if n]
|
||||
a_names = sorted({str(e.get("name") or "").strip() for e in set_a if str(e.get("name") or "").strip()})
|
||||
|
||||
# relations whose source is one of A.
|
||||
relations = []
|
||||
target_names_lower: set[str] = set()
|
||||
if a_names:
|
||||
rel_map, _ = await graph_search(index_name, kb_id, GRAPH_RELATION_FIELDS, dict(scope, knowledge_graph_kwd=["relation"], from_entity_kwd=a_names), OrderByExpr(), GRAPH_EXPANSION_CAP)
|
||||
for row in rel_map.values():
|
||||
edge = project_relation(row)
|
||||
if edge:
|
||||
relations.append(edge)
|
||||
tgt = str(edge.get("to") or "").strip().lower()
|
||||
if tgt:
|
||||
target_names_lower.add(tgt)
|
||||
|
||||
# target entities of those relations (case-insensitive via name_kwd).
|
||||
set_t = []
|
||||
if target_names_lower:
|
||||
tgt_map, _ = await graph_search(index_name, kb_id, GRAPH_ENTITY_FIELDS, dict(scope, knowledge_graph_kwd=["entity"], name_kwd=sorted(target_names_lower)), OrderByExpr(), GRAPH_EXPANSION_CAP)
|
||||
set_t = [n for n in (project_entity(r) for r in tgt_map.values()) if n]
|
||||
|
||||
return dedup_entities(set_a + set_t), relations
|
||||
|
||||
|
||||
async def keyword_subgraph(index_name, kb_id, embd_mdl, base_entity_condition, keywords, scope_for_template, log_ctx="") -> tuple[dict | None, list[dict], list[dict]]:
|
||||
"""KNN the entity rows matching ``base_entity_condition`` for ``keywords``;
|
||||
return ``(top1_bucket_meta, entities, relations)`` for the top-1 entity's
|
||||
1-hop subgraph (top-1 + neighbors + touching relations). ``(None, [], [])``
|
||||
when nothing matches or embedding is unavailable.
|
||||
|
||||
``base_entity_condition`` scopes the KNN (e.g. ``{"doc_id":[id],
|
||||
"knowledge_graph_kwd":["entity"]}`` or ``{"compilation_template_ids":[...],
|
||||
"knowledge_graph_kwd":["entity"]}``). ``scope_for_template(row)`` resolves
|
||||
``(bucket_meta, scope_filter)`` for the matched row (scope WITHOUT
|
||||
``knowledge_graph_kwd``).
|
||||
"""
|
||||
from common.doc_store.doc_store_base import MatchDenseExpr
|
||||
|
||||
try:
|
||||
qv, _ = await thread_pool_exec(embd_mdl.encode_queries, keywords)
|
||||
vec = list(qv)
|
||||
except Exception:
|
||||
logging.exception("structure graph: keyword embedding failed (%s)", log_ctx)
|
||||
return None, [], []
|
||||
if not vec:
|
||||
return None, [], []
|
||||
|
||||
match_expr = MatchDenseExpr(
|
||||
vector_column_name=f"q_{len(vec)}_vec",
|
||||
embedding_data=vec,
|
||||
embedding_data_type="float",
|
||||
distance_type="cosine",
|
||||
topn=1,
|
||||
extra_options={"similarity": 0.0},
|
||||
)
|
||||
top_fields = GRAPH_ENTITY_FIELDS + ["compilation_template_ids", "compile_kwd", "compilation_template_kind_kwd"]
|
||||
top_map, _ = await graph_search(index_name, kb_id, top_fields, base_entity_condition, OrderByExpr(), 1, match_expressions=[match_expr])
|
||||
if not top_map:
|
||||
return None, [], []
|
||||
top_row = next(iter(top_map.values()))
|
||||
top_node = project_entity(top_row)
|
||||
if not top_node:
|
||||
return None, [], []
|
||||
top_name = str(top_node.get("name") or "").strip()
|
||||
if not top_name:
|
||||
return None, [], []
|
||||
bucket_meta, scope = scope_for_template(top_row)
|
||||
|
||||
# Relations where the top-1 entity is source OR target (two term queries).
|
||||
relations: list[dict] = []
|
||||
seen_rel: set[tuple[str, str, str]] = set()
|
||||
neighbor_names_lower: set[str] = set()
|
||||
for field in ("from_entity_kwd", "to_entity_kwd"):
|
||||
rel_map, _ = await graph_search(index_name, kb_id, GRAPH_RELATION_FIELDS, dict(scope, knowledge_graph_kwd=["relation"], **{field: [top_name]}), OrderByExpr(), GRAPH_EXPANSION_CAP)
|
||||
for row in rel_map.values():
|
||||
edge = project_relation(row)
|
||||
if not edge:
|
||||
continue
|
||||
key = (edge.get("from", ""), edge.get("to", ""), edge.get("type", ""))
|
||||
if key in seen_rel:
|
||||
continue
|
||||
seen_rel.add(key)
|
||||
relations.append(edge)
|
||||
for endpoint in (edge.get("from", ""), edge.get("to", "")):
|
||||
endpoint = str(endpoint).strip()
|
||||
if endpoint and endpoint.lower() != top_name.lower():
|
||||
neighbor_names_lower.add(endpoint.lower())
|
||||
|
||||
entities = [top_node]
|
||||
if neighbor_names_lower:
|
||||
nb_map, _ = await graph_search(index_name, kb_id, GRAPH_ENTITY_FIELDS, dict(scope, knowledge_graph_kwd=["entity"], name_kwd=sorted(neighbor_names_lower)), OrderByExpr(), GRAPH_EXPANSION_CAP)
|
||||
entities.extend(n for n in (project_entity(r) for r in nb_map.values()) if n)
|
||||
|
||||
return bucket_meta, dedup_entities(entities), relations
|
||||
Reference in New Issue
Block a user