zlei6/ragflow
1
0
Fork 0
mirror of https://github.com/infiniflow/ragflow.git synced 2026-07-10 13:45:44 +08:00
Code Issues Packages Projects Releases Wiki Activity

Feat: add wiki folder (#16749)

Browse Source 
### Summary

Add wiki folders.
This commit is contained in:
Kevin Hu
2026-07-08 20:08:14 +08:00
committed by GitHub
parent 8e3bbad4da
commit 2c59d07bdb
14 changed files with 893 additions and 264 deletions
Download Patch File Download Diff File Expand all files Collapse all files

29
rag/flow/extractor/extractor.py
View File

@@ -17,16 +17,9 @@ import logging
import random
from copy import deepcopy
from api.db.services.document_service import DocumentService
from api.db.services.llm_service import LLMBundle
from common.constants import LLMType
import xxhash
from agent.component.llm import LLMParam, LLM
from rag.advanced_rag.knowlege_compile.structure import (
compile_structure_from_text,
merge_compiled_structures,
)
from rag.flow.base import ProcessBase, ProcessParamBase
from rag.prompts.generator import run_toc_from_text
@@ -35,7 +28,6 @@ class ExtractorParam(ProcessParamBase, LLMParam):
def __init__(self):
super().__init__()
self.field_name = ""
self.knowledge_compilation = {}
def check(self):
super().check()
@@ -82,20 +74,6 @@ class Extractor(ProcessBase, LLM):
return d
return None
async def _knowledge_compile(self, docs):
embedding_model = LLMBundle(self._canvas.get_tenant_id(), LLMType.EMBEDDING, max_retries=self._param.max_retries, retry_interval=self._param.delay_after_error)
self.callback(0.2, message="Start to generate table of content ...")
docs = sorted(
docs,
key=lambda d: (
d.get("page_num_int", 0)[0] if isinstance(d.get("page_num_int", 0), list) else d.get("page_num_int", 0),
d.get("top_int", 0)[0] if isinstance(d.get("top_int", 0), list) else d.get("top_int", 0),
),
)
docs = await compile_structure_from_text(docs, self._param.knowledge_compilation, self.chat_mdl, embedding_model, self._canvas._doc_id)
info = await merge_compiled_structures(docs, self.chat_mdl, embedding_model, self._canvas.get_tenant_id(), DocumentService.get_knowledgebase_id(self._canvas._doc_id))
return info
async def _invoke(self, **kwargs):
self.set_output("output_format", "chunks")
self.callback(random.randint(1, 5) / 100.0, "Start to generate.")
@@ -118,13 +96,6 @@ class Extractor(ProcessBase, LLM):
chunks.append(toc)
self.set_output("chunks", chunks)
return
if self._param.field_name in ["set", "list", "graph"]:
for ck in chunks:
ck["doc_id"] = self._canvas._doc_id
ck["id"] = xxhash.xxh64((ck["text"] + str(ck["doc_id"])).encode("utf-8")).hexdigest()
await self._knowledge_compile(chunks)
self.set_output("chunks", chunks)
return
prog = 0
for i, ck in enumerate(chunks):