zlei6/ragflow
1
0
Fork 0
mirror of https://github.com/infiniflow/ragflow.git synced 2026-08-09 00:47:59 +08:00
Code Issues Packages Projects Releases Wiki Activity

Go knowledge compiler with scheduler-driven dataset compilation (#17913)

Browse Source 
Ports dataset knowledge compilation (wiki/graph/tree/mindmap) to the Go
scheduler with a status contract, aligns wiki storage/retrieval with
Python, sizes prompts by content_length, and resolves embedding batch
size from provider capability.
This commit is contained in:
Zhichang Yu
2026-08-06 15:54:00 +08:00
committed by GitHub
parent addc5acdc0
commit 2e37997ab9
116 changed files with 6678 additions and 1942 deletions
Download Patch File Download Diff File Expand all files Collapse all files

20
cmd/ragflow_server.go
View File

@@ -30,6 +30,7 @@ import (
agenttool "ragflow/internal/agent/tool"
"ragflow/internal/channels"
"ragflow/internal/handler"
"ragflow/internal/ingestion/knowledge_compile"
ingestion "ragflow/internal/ingestion/service"
"ragflow/internal/mcp"
"ragflow/internal/router"
@@ -41,6 +42,7 @@ import (
"ragflow/internal/service/file"
"ragflow/internal/service/nav"
"ragflow/internal/service/nlp"
"ragflow/internal/service/wikisearch"
"ragflow/internal/storage"
"ragflow/internal/syncer"
"ragflow/internal/tokenizer"
@@ -554,7 +556,15 @@ func runIngestor(ctx context.Context, cancel context.CancelFunc, args *serverArg
// writes available_int=0 compiled chunks; they just won't be merged until
// the consumer is available.
globalConfig := server.GetConfig()
ingestor := ingestion.NewIngestor(*args.name, 2, []string{"pdf", "docx", "txt"})
ingestorCfg := globalConfig.GetIngestorConfig()
const maxIngestorConcurrency = int32(1<<30 - 1)
if ingestorCfg.MaxConcurrentWorkers > int(maxIngestorConcurrency) {
return fmt.Errorf("ingestor max_concurrent_workers %d exceeds maximum %d", ingestorCfg.MaxConcurrentWorkers, maxIngestorConcurrency)
}
// Apply the configured compiler pool size (no-op when 0; the pool keeps its
// vCPU default, overridable via KC_COMPILE_CONCURRENCY).
knowledge_compile.SetCompilerConcurrency(ingestorCfg.CompilerPoolSize)
ingestor := ingestion.NewIngestor(*args.name, int32(ingestorCfg.MaxConcurrentWorkers), []string{"pdf", "docx", "txt"})
ingestor.SetKnowledgeCompileModelConfig(
globalConfig.GetDefaultChatModel().Name,
globalConfig.GetDefaultEmbeddingModel().Name,
@@ -850,6 +860,14 @@ func startServer(ctx context.Context) {
// on demand so Search/UpsertDoc can embed queries/summaries automatically.
nav.SetNavService(nlp.NewNavService(service.NewNavEmbedder(modelProviderService, "")))
// Install the compiled-wiki search service. It is backed directly by the
// document engine: QueryPages filters the tenant-scoped index to
// compile_kwd="wiki_page" (+ supported kinds) so ordinary source chunks are
// never relabeled as wiki pages, and BackfillChunks fetches original chunks
// by id. When the engine is unavailable the service degrades to empty so the
// agent falls back to hybrid search (no failing call).
wikisearch.SetService(wikisearch.NewEngineService(engine.Get()))
// Initialize router
r := router.NewRouter(authHandler,
userHandler,