zlei6/ragflow
1
0
Fork 0
mirror of https://github.com/infiniflow/ragflow.git synced 2026-07-11 06:05:45 +08:00
Code Issues Packages Projects Releases Wiki Activity

Feat: add wiki folder (#16749)

Browse Source 
### Summary

Add wiki folders.
This commit is contained in:
Kevin Hu
2026-07-08 20:08:14 +08:00
committed by GitHub
parent 8e3bbad4da
commit 2c59d07bdb
14 changed files with 893 additions and 264 deletions
Download Patch File Download Diff File Expand all files Collapse all files

246
rag/svr/task_executor_refactor/chunk_post_processor.py
View File

@@ -351,14 +351,10 @@ def count_with_key(docs: List[Dict], key: str) -> int:
import numpy as np # noqa: E402
from typing import Callable, Optional # noqa: E402
from common.exceptions import TaskCanceledException # noqa: E402
from common.misc_utils import thread_pool_exec # noqa: E402
from common.token_utils import num_tokens_from_string # noqa: E402
from rag.nlp import search # noqa: E402
from api.db.services.document_service import DocumentService # noqa: E402
from api.db.services.compilation_template_service import ( # noqa: E402
CompilationTemplateService,
)
from api.db.services.compilation_template_group_service import ( # noqa: E402
CompilationTemplateGroupService,
)
@@ -368,32 +364,20 @@ from api.db.services.task_service import ( # noqa: E402
credit_doc_chunking_task,
is_doc_chunking_aborted,
)
from rag.advanced_rag.knowlege_compile.structure import ( # noqa: E402
CHAIN_KINDS,
compile_structure_from_text,
merge_compiled_structures,
validate_and_correct_chain,
)
# ----- tunables ------------------------------------------------------
# Bound how many source chunks are handed to a single
# ``compile_structure_from_text`` invocation. The call fans them out
# across max_workers internally, so a moderate window keeps memory +
# LLM-context pressure predictable for long docs.
DOC_STRUCTURE_COMPILE_BATCH_CHUNKS = 4
# Bound how many compiled ES-ready docs may accumulate before we flush
# them through ``merge_compiled_structures``. The merger does pairwise
# cosine + LLM duplicate-judging, so it's the more expensive step; we
# cap the per-flush set to keep the local-dedup buckets tractable.
DOC_STRUCTURE_MERGE_MAX_DOCS = 512
# Hard wall on the chain-validator LLM correction step. ``list`` and
# ``timeline`` kinds run this just before each merge flush; anything
# longer than this is treated as a blocked LLM and the uncorrected
# docs are flushed instead.
STRUCTURE_CHAIN_CORRECTION_TIMEOUT_S = 120.0
# The structure-compile batching / merge-flush / chain-correction tunables
# and the non-tree compilation core moved to
# ``rag.advanced_rag.knowlege_compile.runner`` so the ``rag.flow`` Compiler
# component can share them. Re-exported here for backwards compatibility.
from rag.advanced_rag.knowlege_compile.runner import ( # noqa: E402
DOC_STRUCTURE_COMPILE_BATCH_CHUNKS,
DOC_STRUCTURE_MERGE_MAX_DOCS, # noqa: F401
STRUCTURE_CHAIN_CORRECTION_TIMEOUT_S, # noqa: F401
load_active_templates,
run_structure_compile_over_batches,
)
# ----- parser_config helpers -----------------------------------------
@@ -945,27 +929,7 @@ async def run_document_structure_compile(handler, embedding_model: LLMBundle) ->
if not template_ids:
return
active_templates: list[tuple[str, dict]] = []
for template_id in template_ids:
template = CompilationTemplateService.get_saved(template_id, ctx.tenant_id)
if not template:
logging.warning(
"document_structure_compile: template %s not found",
template_id,
)
continue
parser_cfg = template.get("config") or {}
if not isinstance(parser_cfg, dict):
logging.warning(
"document_structure_compile: template %s config is invalid",
template_id,
)
continue
kind = _compilation_template_kind(parser_cfg.get("kind"))
if not kind or kind == "artifacts":
continue
active_templates.append((template_id, parser_cfg))
active_templates = load_active_templates(template_ids, ctx.tenant_id)
if not active_templates:
return
@@ -1018,177 +982,29 @@ async def run_document_structure_compile(handler, embedding_model: LLMBundle) ->
if not non_tree_templates:
return
active_templates = non_tree_templates
progress_cb = ctx.progress_cb
total = len(active_templates)
accumulators: dict[str, list[dict]] = {tid: [] for tid, _ in active_templates}
template_kinds: dict[str, str] = {tid: _compilation_template_kind((cfg or {}).get("kind")) for tid, cfg in active_templates}
agg_infos: dict[str, dict] = {tid: {"inserted": 0, "updated": 0, "duplicates_dropped": 0} for tid, _ in active_templates}
chunks_by_id: dict[str, str] = {}
async def _flush(template_id: str) -> None:
acc = accumulators[template_id]
if not acc:
return
kind = template_kinds.get(template_id, "")
if kind in CHAIN_KINDS:
try:
acc = await asyncio.wait_for(
validate_and_correct_chain(
acc,
chunks_by_id,
chat_mdl_by_tid[template_id],
kind,
callback=progress_cb,
),
timeout=STRUCTURE_CHAIN_CORRECTION_TIMEOUT_S,
)
accumulators[template_id] = acc
except asyncio.TimeoutError:
logging.warning(
"chain validate: timed out after %ss for template %s; using uncorrected docs",
STRUCTURE_CHAIN_CORRECTION_TIMEOUT_S,
template_id,
)
except Exception:
logging.exception(
"chain validate: unexpected failure for template %s; using uncorrected docs",
template_id,
)
info = await merge_compiled_structures(
acc,
chat_mdl_by_tid[template_id],
embedding_model,
async def _stream_doc_batches():
async for batch in handler._load_chunks_for_doc(
ctx.tenant_id,
ctx.kb_id,
compilation_template_id=template_id,
cancel_check=lambda: ctx.has_canceled_func(ctx.id),
)
acc.clear()
if isinstance(info, dict):
agg = agg_infos[template_id]
for k in ("inserted", "updated", "duplicates_dropped"):
agg[k] = agg.get(k, 0) + int(info.get(k, 0) or 0)
ctx.doc_id,
batch_size=DOC_STRUCTURE_COMPILE_BATCH_CHUNKS,
):
yield batch
progress_cb(msg=f"Start document knowledge compilation ({total} template(s)) ...")
batch_no = 0
async for batch in handler._load_chunks_for_doc(
ctx.tenant_id,
ctx.kb_id,
ctx.doc_id,
batch_size=DOC_STRUCTURE_COMPILE_BATCH_CHUNKS,
):
batch_no += 1
for chunk in batch:
cid = chunk.get("id")
if isinstance(cid, str) and cid not in chunks_by_id:
text = chunk.get("content_with_weight") or ""
chunks_by_id[cid] = text if isinstance(text, str) else ""
for idx, (template_id, parser_cfg) in enumerate(active_templates):
progress_cb(msg=f" compile batch {batch_no} ({len(batch)} chunks) for template ({idx + 1}/{total})")
docs = await compile_structure_from_text(
batch,
parser_cfg,
chat_mdl_by_tid[template_id],
embedding_model,
ctx.doc_id,
language=ctx.language,
callback=progress_cb,
compilation_template_id=template_id,
)
if docs:
accumulators[template_id].extend(docs)
if len(accumulators[template_id]) >= DOC_STRUCTURE_MERGE_MAX_DOCS:
progress_cb(msg=f" merge flush ({len(accumulators[template_id])} docs) for template ({idx + 1}/{total})")
await _flush(template_id)
for idx, (template_id, parser_cfg) in enumerate(active_templates):
if ctx.has_canceled_func(ctx.id):
raise TaskCanceledException(f"Task {ctx.id} was cancelled during document knowledge compilation")
await _flush(template_id)
agg = agg_infos[template_id]
ctx.recording_context.record(f"document_structure_compile:{template_id}", agg)
progress_cb(msg=f"Document knowledge compilation done ({idx + 1}/{total}): {agg}")
# ── Synthesis phase ──────────────────────────────────────────────
# If the template has synthesis.enabled, run wiki PLAN+REFINE
# to generate output (wiki page, essence paragraph, etc.).
synthesis_cfg = (parser_cfg or {}).get("synthesis") or {}
if synthesis_cfg.get("enabled"):
example = synthesis_cfg.get("example")
compile_kwd = synthesis_cfg.get("compile_kwd", "artifact_page")
plan_cfg = synthesis_cfg.get("plan") or {}
# Reserved for future wiki_plan_from_reduction extension:
# entity_type_filter, mention_count_threshold, top_n
if plan_cfg:
logging.debug(
"synthesis: template %s plan config %r reserved for future use",
template_id, plan_cfg,
)
if ctx.has_canceled_func(ctx.id):
raise TaskCanceledException(
f"Task {ctx.id} was cancelled before synthesis PLAN"
)
if not example:
logging.warning(
"synthesis: template %s has synthesis.enabled but no example; skipping",
template_id,
)
else:
try:
from rag.advanced_rag.knowlege_compile.wiki import (
wiki_plan_from_reduction,
wiki_refine_from_plan,
)
progress_cb(
msg=f"Synthesis PLAN for template {template_id} (kind={compile_kwd}) ..."
)
plan = await wiki_plan_from_reduction(
chat_mdl=chat_mdl_by_tid[template_id],
embd_mdl=embedding_model,
tenant_id=ctx.tenant_id,
kb_id=ctx.kb_id,
callback=progress_cb,
)
if ctx.has_canceled_func(ctx.id):
raise TaskCanceledException(
f"Task {ctx.id} was cancelled after synthesis PLAN"
)
if not plan or not plan.get("pages"):
progress_cb(
msg=f"Synthesis: no pages planned for template {template_id}."
)
else:
progress_cb(
msg=f"Synthesis REFINE for template {template_id} ({len(plan['pages'])} page(s)) ..."
)
pages = await wiki_refine_from_plan(
chat_mdl=chat_mdl_by_tid[template_id],
embd_mdl=embedding_model,
tenant_id=ctx.tenant_id,
kb_id=ctx.kb_id,
callback=progress_cb,
example=example,
)
# Overwrite compile_kwd on every output page so the
# synthesis type is tracked correctly in ES.
for p in pages or []:
p["compile_kwd"] = compile_kwd
progress_cb(
msg=f"Synthesis done: {len(pages or [])} {compile_kwd} page(s) written."
)
except Exception:
logging.exception(
"synthesis: failed for template %s", template_id,
)
await run_structure_compile_over_batches(
active_templates=non_tree_templates,
chat_mdl_by_tid=chat_mdl_by_tid,
embedding_model=embedding_model,
tenant_id=ctx.tenant_id,
kb_id=ctx.kb_id,
doc_id=ctx.doc_id,
language=ctx.language,
chunk_batches=_stream_doc_batches(),
progress_cb=ctx.progress_cb,
cancel_check=lambda: ctx.has_canceled_func(ctx.id),
record=ctx.recording_context.record,
)
async def run_document_post_chunking_if_last(

9
rag/svr/task_executor_refactor/dataflow_service.py
View File

@@ -109,7 +109,14 @@ class DataflowService:
dataflow_id = corrected_id
# Run pipeline
pipeline = Pipeline(dsl, tenant_id=ctx.tenant_id, doc_id=doc_id, task_id=task_id, flow_id=dataflow_id)
pipeline = Pipeline(
dsl,
tenant_id=ctx.tenant_id,
doc_id=doc_id,
task_id=task_id,
flow_id=dataflow_id,
language=ctx.language,
)
chunks = await pipeline.run(file=ctx.file) if ctx.file else await pipeline.run()
if doc_id == CANVAS_DEBUG_DOC_ID:

117
rag/svr/task_executor_refactor/dataset_wiki_generator.py
View File

@@ -24,9 +24,9 @@ methods and one ``_run_wiki`` orchestrator. The public entry point is
The pipeline runs MAP per (doc, template) — each MAP call resumes from
its own ``artifact_map_extract`` ES rows — then REDUCE / PLAN / REFINE
KB-wide via ``rag.advanced_rag.knowlege_compile.wiki``. Refined pages
land in ES twice: once as searchable ``artifact_page`` rows and once as
``artifact_entity`` / ``artifact_relation`` rows for the dataset
Artifact tab's canvas graph.
land in ES as searchable ``artifact_page`` rows, one
``artifact_page_topic`` row per topic, and ``artifact_entity`` /
``artifact_relation`` rows for the dataset Artifact tab's canvas graph.
Design notes:
@@ -46,6 +46,7 @@ from __future__ import annotations
import asyncio
import json
import logging
import re
from typing import AsyncIterator, Callable, Dict, List, Optional
import xxhash
@@ -83,6 +84,8 @@ WIKI_GRAPH_MAX_CHUNK_IDS_PER_NODE = 64
# path, which carries the user's own title/comments).
WIKI_REGEN_COMMIT_TITLE = "Regenerated by artifact compilation"
WIKI_REGEN_COMMIT_COMMENTS_TEMPLATE = "Auto-update via run_wiki (action={action})"
WIKI_PAGE_COMPILE_KWD = "artifact_page"
WIKI_PAGE_TOPIC_COMPILE_KWD = "artifact_page_topic"
# ----- helpers -------------------------------------------------------
@@ -109,6 +112,95 @@ def _parser_config_compilation_template_ids(parser_config, tenant_id: str) -> li
return template_ids
def _wiki_topic_from_page(page: Dict, fallback: str = "") -> str:
for key in ("topic", "title", "page_type"):
value = page.get(key)
if isinstance(value, str) and value.strip():
return value.strip()
return fallback.strip()
def _wiki_topic_slug(topic: str) -> str:
slug = re.sub(r"[^a-z0-9]+", "-", topic.lower()).strip("-")
if not slug:
slug = xxhash.xxh64(topic.encode("utf-8", "surrogatepass")).hexdigest()[:12]
return f"topic/{slug[:80]}"
async def _ensure_wiki_topic_rows(
ctx: TaskContext,
index: str,
kb_id_str: str,
topics_by_name: dict[str, str],
) -> None:
if not topics_by_name:
return
from common.doc_store.doc_store_base import OrderByExpr
from rag.nlp import rag_tokenizer
topics = list(topics_by_name.keys())
existing: set[str] = set()
try:
res = await thread_pool_exec(
settings.docStoreConn.search,
["id", "topic_kwd"],
[],
{
"compile_kwd": [WIKI_PAGE_TOPIC_COMPILE_KWD],
"topic_kwd": topics,
},
[],
OrderByExpr(),
0,
max(len(topics), 1),
index,
[ctx.kb_id],
)
field_map = settings.docStoreConn.get_fields(res, ["id", "topic_kwd"])
for row in (field_map or {}).values():
raw = row.get("topic_kwd")
if isinstance(raw, list):
existing.update(t for t in raw if isinstance(t, str) and t)
elif isinstance(raw, str) and raw:
existing.add(raw)
except Exception:
logging.exception(
"wiki_persist: topic existence read failed for kb=%s; inserting stable topic ids",
kb_id_str,
)
rows: list[dict] = []
for topic, slug in topics_by_name.items():
if topic in existing:
continue
topic_id = xxhash.xxh64(
f"{kb_id_str}:{WIKI_PAGE_TOPIC_COMPILE_KWD}:{topic}".encode(
"utf-8",
"surrogatepass",
),
).hexdigest()
content_ltks = rag_tokenizer.tokenize(topic)
rows.append(
{
"id": topic_id,
"kb_id": kb_id_str,
"doc_id": kb_id_str,
"compile_kwd": WIKI_PAGE_TOPIC_COMPILE_KWD,
"topic_kwd": topic,
"title_kwd": topic,
"slug_kwd": slug,
"content_with_weight": topic,
"content_ltks": content_ltks,
"content_sm_ltks": rag_tokenizer.fine_grained_tokenize(content_ltks),
"available_int": 1,
}
)
if rows:
await thread_pool_exec(settings.docStoreConn.insert, rows, index, ctx.kb_id)
# ----- persistence ---------------------------------------------------
@@ -125,6 +217,7 @@ async def persist_wiki_pages_to_es(
slug_kwd page.slug
title_kwd page.title
page_type_kwd page.page_type
topic_kwd page.topic
entity_names_kwd page.entity_names
outlinks_kwd page.outlinks
related_kb_pages_kwd page.related_kb_pages
@@ -163,7 +256,7 @@ async def persist_wiki_pages_to_es(
settings.docStoreConn.search,
["id", "slug_kwd", "content_with_weight"],
[],
{"compile_kwd": ["artifact_page"], "slug_kwd": list(target_slugs)},
{"compile_kwd": [WIKI_PAGE_COMPILE_KWD], "slug_kwd": list(target_slugs)},
[],
OrderByExpr(),
0,
@@ -213,11 +306,13 @@ async def persist_wiki_pages_to_es(
return
rows: List[Dict] = []
topics_by_name: dict[str, str] = {}
for page, vec in zip(pages, embeddings):
slug = page.get("slug") or ""
if not slug:
continue
title = page.get("title") or slug
topic = _wiki_topic_from_page(page, title)
summary = page.get("summary") or ""
content_md = page.get("content_md_rendered") or page.get("content_md") or page.get("content_md_raw") or ""
@@ -242,10 +337,11 @@ async def persist_wiki_pages_to_es(
"id": row_id,
"kb_id": kb_id_str,
"doc_id": kb_id_str, # sentinel; KB-scoped row, real provenance in source_doc_ids
"compile_kwd": "artifact_page",
"compile_kwd": WIKI_PAGE_COMPILE_KWD,
"slug_kwd": slug,
"title_kwd": title,
"page_type_kwd": page.get("page_type") or "concept",
"topic_kwd": topic,
"entity_names_kwd": list(page.get("entity_names") or []),
"outlinks_kwd": list(page.get("outlinks") or []),
"outlinks_int": len(list(page.get("outlinks") or [])),
@@ -263,6 +359,8 @@ async def persist_wiki_pages_to_es(
"available_int": 1,
}
)
if topic:
topics_by_name.setdefault(topic, _wiki_topic_slug(topic))
if not rows:
return
@@ -277,6 +375,15 @@ async def persist_wiki_pages_to_es(
)
return
if topics_by_name:
try:
await _ensure_wiki_topic_rows(ctx, index, kb_id_str, topics_by_name)
except Exception:
logging.exception(
"wiki_persist: topic row insert failed for kb=%s",
kb_id_str,
)
# Audit trail: one ArtifactCommit row per page whose rendered
# content actually changed (record_edit silently skips empty diffs).
# Best-effort — commit failures log but don't fail the artifact