zlei6/ragflow
1
0
Fork 0
mirror of https://github.com/infiniflow/ragflow.git synced 2026-08-01 05:23:47 +08:00
Code Issues Packages Projects Releases Wiki Activity

refactor(knowledge_compiler): derive variant from template kind via single template id (#17630)

Browse Source 
## Summary

Refactor the Go `KnowledgeCompilerComponent` so its parameter is a
**single string template id** instead of a DSL-level `variant` (or
plural group id list). The `variant` is no longer in the DSL — it is now
**derived at runtime from the resolved compilation template's `kind`
field**.

This aligns the Go ingestion port with the frontend Compiler operator,
which emits a singular `compilation_template_group_id` and does not
write `variant` into the generated `compiler.json`.
This commit is contained in:
Zhichang Yu
2026-07-31 18:01:55 +08:00
committed by GitHub
parent d1127d5973
commit 29287ef74b
29 changed files with 1714 additions and 342 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
.gitignore vendored
View File

@@ -245,6 +245,7 @@ bin/*
# Local agent tooling state (per-developer; not for commit)
.omc/
.marscode/
.omx/
# Parser test fixtures and python tools
internal/deepdoc/parser/pdf/testdata/

443
agent/templates/compiler.json Normal file
View File

@@ -0,0 +1,443 @@
{
"components": {
"Compiler:CurvySunsLove": {
"downstream": [],
"obj": {
"component_name": "Compiler",
"params": {
"compilation_template_group_id": "c3aa748c8b2111f191f3047c16ec874f",
"outputs": {
"chunks": {
"type": "Array<Object>",
"value": []
}
}
}
},
"upstream": [
"TokenChunker:LovelyPillowsOpen"
]
},
"File": {
"downstream": [
"Parser:HipSignsRhyme"
],
"obj": {
"component_name": "File",
"params": {}
},
"upstream": []
},
"Parser:HipSignsRhyme": {
"downstream": [
"TokenChunker:LovelyPillowsOpen"
],
"obj": {
"component_name": "Parser",
"params": {
"outputs": {
"html": {
"type": "string",
"value": ""
},
"json": {
"type": "Array<object>",
"value": []
},
"markdown": {
"type": "string",
"value": ""
},
"text": {
"type": "string",
"value": ""
}
},
"setups": {
"doc": {
"flatten_media_to_text": false,
"order_index": 7,
"output_format": "json",
"remove_header_footer": false,
"suffix": [
"doc"
],
"vlm": {}
},
"docx": {
"flatten_media_to_text": false,
"order_index": 8,
"output_format": "json",
"remove_header_footer": false,
"suffix": [
"docx"
],
"vlm": {}
},
"email": {
"fields": [
"from",
"to",
"cc",
"bcc",
"date",
"subject",
"body",
"attachments"
],
"order_index": 3,
"output_format": "text",
"suffix": [
"eml",
"msg"
]
},
"html": {
"order_index": 6,
"output_format": "json",
"remove_header_footer": false,
"suffix": [
"htm",
"html"
]
},
"image": {
"order_index": 2,
"output_format": "text",
"parse_method": "ocr",
"suffix": [
"jpg",
"jpeg",
"png",
"gif"
]
},
"markdown": {
"flatten_media_to_text": false,
"order_index": 4,
"output_format": "json",
"suffix": [
"md",
"markdown",
"mdx"
],
"vlm": {}
},
"pdf": {
"flatten_media_to_text": false,
"order_index": 0,
"output_format": "json",
"parse_method": "DeepDOC",
"remove_header_footer": false,
"suffix": [
"pdf"
],
"vlm": {}
},
"slides": {
"order_index": 9,
"output_format": "json",
"parse_method": "DeepDOC",
"suffix": [
"pptx",
"ppt"
]
},
"spreadsheet": {
"flatten_media_to_text": false,
"order_index": 1,
"output_format": "html",
"parse_method": "DeepDOC",
"suffix": [
"xls",
"xlsx",
"csv"
],
"vlm": {}
},
"text&code": {
"order_index": 5,
"output_format": "json",
"suffix": [
"txt",
"py",
"js",
"java",
"c",
"cpp",
"h",
"php",
"go",
"ts",
"sh",
"cs",
"kt",
"sql"
]
}
}
}
},
"upstream": [
"File"
]
},
"TokenChunker:LovelyPillowsOpen": {
"downstream": [
"Compiler:CurvySunsLove"
],
"obj": {
"component_name": "TokenChunker",
"params": {
"children_delimiters": [],
"chunk_token_size": 512,
"delimiter_mode": "token_size",
"delimiters": [],
"image_context_size": 0,
"outputs": {
"chunks": {
"type": "Array<Object>",
"value": []
}
},
"overlapped_percent": 0,
"table_context_size": 0
}
},
"upstream": [
"Parser:HipSignsRhyme"
]
}
},
"globals": {
"sys.history": []
},
"graph": {
"edges": [
{
"id": "xy-edge__Filestart-Parser:HipSignsRhymeend",
"source": "File",
"sourceHandle": "start",
"target": "Parser:HipSignsRhyme",
"targetHandle": "end"
},
{
"data": {
"isHovered": false
},
"id": "xy-edge__Parser:HipSignsRhymestart-TokenChunker:LovelyPillowsOpenend",
"source": "Parser:HipSignsRhyme",
"sourceHandle": "start",
"target": "TokenChunker:LovelyPillowsOpen",
"targetHandle": "end"
},
{
"id": "xy-edge__TokenChunker:LovelyPillowsOpenstart-Compiler:CurvySunsLoveend",
"source": "TokenChunker:LovelyPillowsOpen",
"sourceHandle": "start",
"target": "Compiler:CurvySunsLove",
"targetHandle": "end"
}
],
"nodes": [
{
"data": {
"label": "File",
"name": "File"
},
"id": "File",
"measured": {
"height": 49,
"width": 200
},
"position": {
"x": 50,
"y": 200
},
"sourcePosition": "left",
"targetPosition": "right",
"type": "beginNode"
},
{
"data": {
"form": {
"outputs": {
"html": {
"type": "string",
"value": ""
},
"json": {
"type": "Array<object>",
"value": []
},
"markdown": {
"type": "string",
"value": ""
},
"text": {
"type": "string",
"value": ""
}
},
"setups": [
{
"fileFormat": "pdf",
"flatten_media_to_text": false,
"output_format": "json",
"parse_method": "DeepDOC",
"remove_header_footer": false,
"vlm": {}
},
{
"fileFormat": "spreadsheet",
"flatten_media_to_text": false,
"output_format": "html",
"parse_method": "DeepDOC",
"vlm": {}
},
{
"fileFormat": "image",
"output_format": "text",
"parse_method": "ocr"
},
{
"fields": [
"from",
"to",
"cc",
"bcc",
"date",
"subject",
"body",
"attachments"
],
"fileFormat": "email",
"output_format": "text"
},
{
"fileFormat": "markdown",
"flatten_media_to_text": false,
"output_format": "json",
"vlm": {}
},
{
"fileFormat": "text&code",
"output_format": "json"
},
{
"fileFormat": "html",
"output_format": "json",
"remove_header_footer": false
},
{
"fileFormat": "doc",
"flatten_media_to_text": false,
"output_format": "json",
"remove_header_footer": false,
"vlm": {}
},
{
"fileFormat": "docx",
"flatten_media_to_text": false,
"output_format": "json",
"remove_header_footer": false,
"vlm": {}
},
{
"fileFormat": "slides",
"output_format": "json",
"parse_method": "DeepDOC"
}
]
},
"label": "Parser",
"name": "Parser_0"
},
"dragging": false,
"id": "Parser:HipSignsRhyme",
"measured": {
"height": 197,
"width": 200
},
"position": {
"x": 316.99524094206413,
"y": 195.39629819663406
},
"selected": true,
"sourcePosition": "right",
"targetPosition": "left",
"type": "parserNode"
},
{
"data": {
"form": {
"children_delimiters": [],
"chunk_token_size": 512,
"delimiter_mode": "token_size",
"delimiters": [
{
"value": "\n"
}
],
"image_table_context_window": 0,
"outputs": {
"chunks": {
"type": "Array<Object>",
"value": []
}
},
"overlapped_percent": 0
},
"label": "TokenChunker",
"name": "Token Chunker_0"
},
"id": "TokenChunker:LovelyPillowsOpen",
"measured": {
"height": 73,
"width": 200
},
"position": {
"x": 616.9952409420641,
"y": 195.39629819663406
},
"selected": false,
"sourcePosition": "right",
"targetPosition": "left",
"type": "chunkerNode"
},
{
"data": {
"form": {
"compilation_template_group_id": "c3aa748c8b2111f191f3047c16ec874f",
"outputs": {
"chunks": {
"type": "Array<Object>",
"value": []
}
}
},
"label": "Compiler",
"name": "Compiler_0"
},
"id": "Compiler:CurvySunsLove",
"measured": {
"height": 93,
"width": 200
},
"position": {
"x": 916.9952409420641,
"y": 195.39629819663406
},
"selected": false,
"sourcePosition": "right",
"targetPosition": "left",
"type": "compilationNode"
}
]
},
"history": [],
"messages": [],
"path": [],
"retrieval": [],
"variables": []
}

35
internal/dao/compilation_template.go
View File

@@ -49,21 +49,38 @@ func (dao *CompilationTemplateDAO) ResolveGroupTemplateIDs(ctx context.Context,
db := DB.WithContext(ctx)
// Verify the requested groups exist, are valid, and belong to the tenant.
// Verify the requested groups exist and are valid. The built-in group
// (compiler.json's default) is a global, tenant-agnostic catalogue, so it
// resolves for every tenant; other groups are scoped to the caller's
// tenant.
isBuiltin := make(map[string]bool, len(groupIDs))
for _, gid := range groupIDs {
if gid == BuiltinCompilationTemplateGroupID {
isBuiltin[gid] = true
}
}
var validGroups []entity.CompilationTemplateGroup
if err := db.
Where("id IN ? AND tenant_id = ? AND status = ?", groupIDs, tenantID, string(entity.StatusValid)).
Where("id IN ? AND status = ?", groupIDs, string(entity.StatusValid)).
Find(&validGroups).Error; err != nil {
return nil, fmt.Errorf("resolve compilation template groups: %w", err)
}
valid := make(map[string]struct{}, len(validGroups))
byID := make(map[string]entity.CompilationTemplateGroup, len(validGroups))
for _, g := range validGroups {
valid[g.ID] = struct{}{}
byID[g.ID] = g
}
for _, gid := range groupIDs {
if _, ok := valid[gid]; !ok {
return nil, fmt.Errorf("compilation_template_group %q not found for tenant %q", gid, tenantID)
}
// Non-built-in groups must belong to the requesting tenant.
if !isBuiltin[gid] {
if byID[gid].TenantID != tenantID {
return nil, fmt.Errorf("compilation_template_group %q not found for tenant %q", gid, tenantID)
}
}
}
var templates []entity.CompilationTemplate
@@ -92,3 +109,17 @@ func (dao *CompilationTemplateDAO) ResolveGroupTemplateIDs(ctx context.Context,
}
return out, nil
}
// GetTemplate loads a single, valid compilation template by id for the tenant.
// It backs the KnowledgeCompiler TemplateResolver: the template's kind selects
// the Go compile variant (see common.KindToVariant) and its config is the
// template "content" plumbed to the variant.
func (dao *CompilationTemplateDAO) GetTemplate(ctx context.Context, tenantID, templateID string) (*entity.CompilationTemplate, error) {
var t entity.CompilationTemplate
if err := DB.WithContext(ctx).
Where("id = ? AND tenant_id = ? AND status = ?", templateID, tenantID, string(entity.StatusValid)).
First(&t).Error; err != nil {
return nil, fmt.Errorf("load compilation template %q: %w", templateID, err)
}
return &t, nil
}

110
internal/dao/compilation_template_seed.go Normal file
View File

@@ -0,0 +1,110 @@
//
// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
package dao
import (
"context"
"ragflow/internal/common"
"ragflow/internal/entity"
"go.uber.org/zap"
"gorm.io/gorm"
"gorm.io/gorm/clause"
)
// BuiltinCompilationTemplateGroupID is the id of the system-provided
// compilation template group that compiler.json references as its default
// "create from template" group. It is a global (tenant-agnostic) catalogue so
// any tenant can resolve it without depending on Python-side initialization.
const BuiltinCompilationTemplateGroupID = "c3aa748c8b2111f191f3047c16ec874f"
// builtinCompilationTemplateKinds is the set of variants shipped in the
// built-in group. The Go KnowledgeCompiler derives the variant from each
// template's kind via common.KindToVariant.
var builtinCompilationTemplateKinds = []struct {
Kind string
Name string
}{
{Kind: "tree", Name: "Tree — RAPTOR-based document tree"},
{Kind: "mind_map", Name: "Mind map - Radial concept hierarchy"},
{Kind: "wiki", Name: "Wiki — Graph-based wiki"},
{Kind: "knowledge_graph", Name: "Knowledge graph"},
{Kind: "datasetnav", Name: "Dataset nav — top-down drill-down"},
{Kind: "page_index", Name: "Page index"},
{Kind: "session_essence", Name: "Session essence"},
{Kind: "session_graph", Name: "Session graph"},
{Kind: "timeline", Name: "Timeline"},
}
func strptr(s string) *string { return &s }
// SeedBuiltinCompilationTemplates provisions the built-in compilation template
// group (and its child templates) as a global, tenant-agnostic catalogue, so
// the Go compiler (and the frontend "create from template" catalogue) can
// resolve compiler.json's default group id out of the box. The group row uses
// an empty tenant_id and ResolveGroupTemplateIDs treats this built-in id as
// resolvable for every tenant.
func SeedBuiltinCompilationTemplates(ctx context.Context, db *gorm.DB) error {
if err := SeedBuiltinCompilationTemplatesForTenant(ctx, db, ""); err != nil {
common.Warn("failed to seed built-in compilation templates", zap.Error(err))
return err
}
return nil
}
// SeedBuiltinCompilationTemplatesForTenant upserts the built-in compilation
// template group and its child templates as a global, tenant-agnostic
// catalogue. The tenantID argument is accepted for API symmetry but the
// built-in entries always use an empty tenant_id, because ResolveGroupTemplateIDs
// resolves the built-in group id for every tenant. It is idempotent
// (OnConflict DO NOTHING on id) so it is safe to call repeatedly.
func SeedBuiltinCompilationTemplatesForTenant(ctx context.Context, db *gorm.DB, tenantID string) error {
valid := string(entity.StatusValid)
return db.WithContext(ctx).Transaction(func(tx *gorm.DB) error {
group := &entity.CompilationTemplateGroup{
ID: BuiltinCompilationTemplateGroupID,
TenantID: "", // global built-in catalogue
Name: "Built-in templates",
Status: strptr(valid),
}
if err := tx.WithContext(ctx).Clauses(clause.OnConflict{
Columns: []clause.Column{{Name: "id"}},
DoNothing: true,
}).Create(group).Error; err != nil {
return err
}
for _, t := range builtinCompilationTemplateKinds {
tmpl := &entity.CompilationTemplate{
ID: BuiltinCompilationTemplateGroupID + "-" + t.Kind,
TenantID: nil, // global built-in catalogue
GroupID: strptr(BuiltinCompilationTemplateGroupID),
Name: t.Name,
Kind: t.Kind,
Config: entity.JSONMap{"kind": t.Kind},
Status: strptr(valid),
}
if err := tx.WithContext(ctx).Clauses(clause.OnConflict{
Columns: []clause.Column{{Name: "id"}},
DoNothing: true,
}).Create(tmpl).Error; err != nil {
return err
}
}
return nil
})
}

112
internal/dao/compilation_template_seed_test.go Normal file
View File

@@ -0,0 +1,112 @@
//
// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
package dao
import (
"testing"
"ragflow/internal/entity"
"github.com/glebarez/sqlite"
"gorm.io/gorm"
)
func TestSeedBuiltinCompilationTemplatesForTenant(t *testing.T) {
db, err := gorm.Open(sqlite.Open(":memory:"), &gorm.Config{})
if err != nil {
t.Fatalf("open sqlite: %v", err)
}
if err = db.AutoMigrate(&entity.CompilationTemplateGroup{}, &entity.CompilationTemplate{}); err != nil {
t.Fatalf("migrate: %v", err)
}
// ResolveGroupTemplateIDs reads the package-global DB, so point it at our
// sqlite instance for the duration of the test.
orig := DB
DB = db
t.Cleanup(func() { DB = orig })
ctx := t.Context()
const tenantID = "tenant-1"
// Idempotent across repeated calls, and the built-in group is always a
// global (tenant_id = "") catalogue regardless of the tenantID argument.
for i := 0; i < 2; i++ {
if err = SeedBuiltinCompilationTemplatesForTenant(ctx, db, tenantID); err != nil {
t.Fatalf("seed (iter %d): %v", i, err)
}
}
var groups []entity.CompilationTemplateGroup
if err = db.Find(&groups).Error; err != nil {
t.Fatalf("read groups: %v", err)
}
if len(groups) != 1 {
t.Fatalf("got %d groups, want 1", len(groups))
}
if groups[0].ID != BuiltinCompilationTemplateGroupID || groups[0].TenantID != "" {
t.Fatalf("unexpected group: %#v", groups[0])
}
var templates []entity.CompilationTemplate
if err = db.Find(&templates).Error; err != nil {
t.Fatalf("read templates: %v", err)
}
if len(templates) != len(builtinCompilationTemplateKinds) {
t.Fatalf("got %d templates, want %d", len(templates), len(builtinCompilationTemplateKinds))
}
for _, tmpl := range templates {
// Built-in templates are a global catalogue (tenant_id nil).
if tmpl.TenantID != nil {
t.Errorf("template %s: tenant_id=%v, want nil (global catalogue)", tmpl.ID, tmpl.TenantID)
}
if tmpl.GroupID == nil || *tmpl.GroupID != BuiltinCompilationTemplateGroupID {
t.Errorf("template %s: group_id=%v, want %q", tmpl.ID, tmpl.GroupID, BuiltinCompilationTemplateGroupID)
}
if tmpl.Kind == "" {
t.Errorf("template %s: empty kind", tmpl.ID)
}
if tmpl.Config["kind"] != tmpl.Kind {
t.Errorf("template %s: config.kind=%v, want %q", tmpl.ID, tmpl.Config["kind"], tmpl.Kind)
}
}
// Re-seeding with a different tenant is idempotent for the global group:
// the single built-in group row stays (id is the PK), and templates are
// not duplicated.
if err = SeedBuiltinCompilationTemplatesForTenant(ctx, db, "tenant-2"); err != nil {
t.Fatalf("seed tenant-2: %v", err)
}
var allGroups []entity.CompilationTemplateGroup
if err = db.Find(&allGroups).Error; err != nil {
t.Fatalf("read all groups: %v", err)
}
if len(allGroups) != 1 {
t.Fatalf("got %d groups, want 1 (built-in group is global, single row)", len(allGroups))
}
if allGroups[0].TenantID != "" {
t.Errorf("built-in group tenant_id = %q, want empty (global catalogue)", allGroups[0].TenantID)
}
// ResolveGroupTemplateIDs resolves the built-in group for any tenant.
resolved, err := NewCompilationTemplateDAO().ResolveGroupTemplateIDs(ctx, "some-other-tenant", []string{BuiltinCompilationTemplateGroupID})
if err != nil {
t.Fatalf("resolve built-in group: %v", err)
}
if len(resolved) != len(builtinCompilationTemplateKinds) {
t.Fatalf("resolved %d template ids, want %d", len(resolved), len(builtinCompilationTemplateKinds))
}
}

10
internal/dao/database.go
View File

@@ -158,6 +158,11 @@ func InitDB(ctx context.Context, migrateDB bool) error {
&entity.FileCommit{},
&entity.FileCommitItem{},
&entity.KnowledgeCompileDoc{},
// Knowledge-compile compilation templates and their groups. The Go
// KnowledgeCompilerComponent resolves a compilation_template (or group)
// from these tables at runtime, so the Go side must guarantee they exist.
&entity.CompilationTemplate{},
&entity.CompilationTemplateGroup{},
}
if migrateDB {
@@ -180,6 +185,11 @@ func InitDB(ctx context.Context, migrateDB bool) error {
if err = SeedCanvasTemplates(ctx, DB); err != nil {
common.Warn("Failed to seed canvas templates", zap.Error(err))
}
// Seed the built-in compilation template group (c3aa748c...) for every
// tenant so compiler.json's default group resolves out of the box.
if err = SeedBuiltinCompilationTemplates(ctx, DB); err != nil {
common.Warn("Failed to seed built-in compilation templates", zap.Error(err))
}
common.Info("Database connected and migrated successfully")

8
internal/dao/tenant.go
View File

@@ -105,6 +105,14 @@ func (dao *TenantDAO) Create(ctx context.Context, db *gorm.DB, tenant *entity.Te
return db.WithContext(ctx).Create(tenant).Error
}
// GetAllIDs returns the IDs of all active tenants.
func (dao *TenantDAO) GetAllIDs(ctx context.Context, db *gorm.DB) ([]string, error) {
var ids []string
err := db.WithContext(ctx).Model(&entity.Tenant{}).Where("status = ?", string(entity.StatusValid)).
Pluck("id", &ids).Error
return ids, err
}
// Delete deletes a tenant by ID (soft delete)
func (dao *TenantDAO) Delete(ctx context.Context, db *gorm.DB, id string) error {
return db.WithContext(ctx).Model(&entity.Tenant{}).Where("id = ?", id).Update("status", "0").Error

45
internal/ingestion/component/knowledge_compiler/common/deps.go
View File

@@ -148,7 +148,7 @@ func ResolveDeps(tenantID, llmID, embeddingModel string) (Deps, error) {
// compilation_template ids they contain. It is a DB-backed seam installed by
// production wiring in internal/ingestion/task; the knowledge_compiler package
// itself stays DB-independent. When a component is configured with
// compilation_template_group_ids but no GroupResolver is installed, the
// compilation_template_group_id but no GroupResolver is installed, the
// component fails loudly instead of silently emitting rows that miss
// compilation_template_ids (a data-loss path).
type GroupResolver func(ctx context.Context, tenantID string, groupIDs []string) ([]string, error)
@@ -180,7 +180,48 @@ func ResolveGroupTemplateIDs(ctx context.Context, tenantID string, groupIDs []st
r := groupResolver
groupResolverMu.RUnlock()
if r == nil {
return nil, fmt.Errorf("knowledge_compiler: compilation_template_group_ids provided but no GroupResolver installed (production wiring must call common.SetGroupResolver)")
return nil, fmt.Errorf("knowledge_compiler: compilation_template_group_id provided but no GroupResolver installed (production wiring must call common.SetGroupResolver)")
}
return r(ctx, tenantID, groupIDs)
}
// TemplateResolver loads a single compilation template by id for the tenant.
type TemplateResolver func(ctx context.Context, tenantID, templateID string) (TemplateInfo, error)
// TemplateInfo is a resolved compilation template: its id, the kind that
// selects the Go compiler variant (see KindToVariant), and its config blob (the
// template "content"). It is dependency-light so common can return it without
// importing entity/gorm — production wiring (internal/ingestion/task) converts
// the entity row into this shape.
type TemplateInfo struct {
ID string
Kind string
Config map[string]any
}
var (
templateResolverMu sync.RWMutex
templateResolver TemplateResolver
)
// SetTemplateResolver installs the production (or test) TemplateResolver.
// Production wiring calls this from an init() in internal/ingestion/task; tests
// may inject a stub.
func SetTemplateResolver(r TemplateResolver) {
templateResolverMu.Lock()
defer templateResolverMu.Unlock()
templateResolver = r
}
// ResolveTemplate loads a single compilation template by id via the installed
// resolver. Returns an error when no resolver is installed (compilation_template_id
// provided but unwired) rather than silently compiling with no template config.
func ResolveTemplate(ctx context.Context, tenantID, templateID string) (TemplateInfo, error) {
templateResolverMu.RLock()
r := templateResolver
templateResolverMu.RUnlock()
if r == nil {
return TemplateInfo{}, fmt.Errorf("knowledge_compiler: compilation_template_id provided but no TemplateResolver installed (production wiring must call common.SetTemplateResolver)")
}
return r(ctx, tenantID, templateID)
}

192
internal/ingestion/component/knowledge_compiler/common/types.go
View File

@@ -11,7 +11,6 @@ package common
import (
"errors"
"fmt"
"log"
"strings"
)
@@ -21,7 +20,7 @@ type Variant string
const (
VariantStructure Variant = "structure"
VariantWiki Variant = "wiki"
VariantRaptor Variant = "raptor"
VariantTree Variant = "tree"
VariantMindmap Variant = "mindmap"
VariantDatasetnav Variant = "datasetnav"
)
@@ -32,19 +31,42 @@ var (
ErrNotImplemented = errors.New("knowledge_compiler: variant not yet implemented")
)
// Param is built from the DSL params map. Missing keys fall back to
// Defaults(); variant-name aliases are normalised (see ParseParam).
// Param is built from the DSL params map. The component is driven by a
// compilation template (or template group); the variant is NOT taken from the
// DSL — it is derived from the resolved template's kind at Invoke time (see
// KindToVariant).
type Param struct {
Variant Variant
LLMID string
EmbeddingModel string
Language string
TemplateIDs []string
GroupIDs []string
// CompilationTemplateID selects a single compilation template directly.
// CompilationTemplateGroupID selects a compilation-template group, which
// resolves to one or more templates. When both are non-empty,
// CompilationTemplateID wins (priority: id > group_id).
CompilationTemplateID string
CompilationTemplateGroupID string
LLMID string
EmbeddingModel string
Language string
// SimilarityThreshold defaults to 0.99 when not provided (see Defaults).
SimilarityThreshold float64
MaxWorkers int
EnableHistoricalDedup bool
Extra map[string]any
// Extra carries arbitrary caller-provided overrides merged into the
// resolved template config.
Extra map[string]any
// The following fields are resolved at runtime (not part of the DSL
// surface) and are set by the component before dispatching to a variant:
//
// Variant is derived from the resolved template's kind (see KindToVariant),
// not from the DSL.
Variant Variant
// TemplateID is the resolved compilation template id for the current spec
// (the id path uses it directly; the group path resolves each child). It is
// plumbed to the variant for stamping compilation_template_ids.
TemplateID string
// TemplateConfig is the resolved template's config blob (the template
// "content"), plumbed to the variant for prompt/structure extraction.
TemplateConfig map[string]any
}
// Defaults returns a Param populated with safe production fallbacks.
@@ -83,21 +105,23 @@ type Inputs struct {
HistoricalCandidates []Candidate // optional override path (test/offline)
}
// LogDeprecated emits a one-line deprecation notice for legacy param names.
func LogDeprecated(old, replacement string) {
log.Printf("[knowledge_compiler] deprecated variant name %q; use %q", old, replacement)
}
// Product is one compiled output row (schema_version=1).
type Product struct {
ID string
DocID string
TenantID string
Variant Variant
Content string
Vector []float32
ParentID string
Meta map[string]any
// Kind is the original compilation_template.kind (e.g. "page_index"),
// distinct from Variant (the collapsed Go strategy, e.g. "structure"). It
// is stamped onto compilation_template_kind_kwd so the document-structure
// endpoint can group rows by the true template kind.
Kind string
// TemplateID is the compilation template that produced this row.
TemplateID string
Content string
Vector []float32
ParentID string
Meta map[string]any
}
// Outputs is the result of a variant Run. All compiled products are buffered
@@ -107,15 +131,23 @@ type Outputs struct {
DuplicatesDropped int
}
// ParseParam builds a Param from the DSL params map, applying variant-name
// aliases (mind_map=>mindmap, dataset_nav=>datasetnav) and deprecation logs.
// ParseParam builds a Param from the DSL params map. The variant is NOT taken
// from the DSL — it is derived from the resolved template's kind at Invoke time
// via KindToVariant. At least one of compilation_template_id /
// compilation_template_group_id is required.
func ParseParam(m map[string]any) (Param, error) {
p := Param{}.Defaults()
if m == nil {
return p, fmt.Errorf("knowledge_compiler: params required (variant missing)")
return p, fmt.Errorf("knowledge_compiler: params required (compilation_template_id missing)")
}
if v, ok := m["variant"].(string); ok && v != "" {
p.Variant = normalizeVariant(v)
if v, ok := m["compilation_template_id"].(string); ok && strings.TrimSpace(v) != "" {
p.CompilationTemplateID = strings.TrimSpace(v)
}
if v, ok := m["compilation_template_group_id"].(string); ok && strings.TrimSpace(v) != "" {
p.CompilationTemplateGroupID = strings.TrimSpace(v)
}
if p.CompilationTemplateID == "" && p.CompilationTemplateGroupID == "" {
return p, fmt.Errorf("knowledge_compiler: one of 'compilation_template_id' or 'compilation_template_group_id' is required")
}
if v, ok := m["llm_id"].(string); ok {
p.LLMID = v
@@ -132,46 +164,45 @@ func ParseParam(m map[string]any) (Param, error) {
if v, ok := m["enable_historical_dedup"].(bool); ok {
p.EnableHistoricalDedup = v
}
// compilation_template_ids / compilation_template_group_ids mirror the
// Python parser_config keys (see api/utils/validation_utils.py). Template
// ids are stamped onto every compiled row so the document-structure
// endpoint can group rows by template and the UI renders one tab per
// template. Group ids are resolved to concrete template ids by the caller
// (production wiring resolves them via the compilation-template-group
// service); when passed through raw they are carried as-is on GroupIDs.
p.TemplateIDs = parseStringList(m, "compilation_template_ids", "template_ids")
p.GroupIDs = parseStringList(m, "compilation_template_group_ids", "template_group_ids", "group_ids")
if raw, ok := m["extra"].(map[string]any); ok {
p.Extra = raw
}
if p.Variant == "" {
return p, fmt.Errorf("knowledge_compiler: 'variant' is required")
}
return p, nil
}
// normalizeVariant maps a DSL variant name to the canonical Variant,
// logging a deprecation note for the legacy underscore forms.
func normalizeVariant(s string) Variant {
switch strings.ToLower(strings.TrimSpace(s)) {
case "structure":
return VariantStructure
// KindToVariant maps a compilation_template.kind to the Go compiler Variant.
//
// The Python model uses richer kind values (mind_map, page_index,
// session_essence, session_graph, timeline, knowledge_graph, tree, wiki); the
// Go port collapses them onto its variants. Per the agreed mapping:
//
// tree -> tree
// mind_map -> mindmap
// wiki -> wiki
// page_index / session_essence / -> structure (the graph/knowledge-graph path)
// session_graph / timeline / ->
// knowledge_graph ->
//
// The canonical variant names are also accepted as identity (so a template kind
// may equal the variant, e.g. "datasetnav" which has no Python kind, and the
// internal unit tests can drive each variant through a template id). Unknown
// kinds return ErrUnknownVariant; the caller turns that into a hard failure
// rather than silently emitting uncompiled rows.
func KindToVariant(kind string) (Variant, error) {
switch strings.ToLower(strings.TrimSpace(kind)) {
case "tree":
return VariantTree, nil
case "mind_map", "mindmap":
return VariantMindmap, nil
case "wiki":
return VariantWiki
case "raptor":
return VariantRaptor
case "mindmap", "mind_map":
if strings.Contains(s, "_") {
LogDeprecated("mind_map", "mindmap")
}
return VariantMindmap
return VariantWiki, nil
case "datasetnav", "dataset_nav":
if strings.Contains(s, "_") {
LogDeprecated("dataset_nav", "datasetnav")
}
return VariantDatasetnav
return VariantDatasetnav, nil
case "page_index", "session_essence", "session_graph", "timeline",
"knowledge_graph", "structure", "knowledgegraph", "graph":
return VariantStructure, nil
default:
return Variant(s)
return "", fmt.Errorf("%w: %q", ErrUnknownVariant, kind)
}
}
@@ -185,55 +216,6 @@ func FirstNonEmpty(vals ...string) string {
return ""
}
// parseStringList reads a []string from the params map under any of the given
// keys (first hit wins). Accepts []string, []any (of strings), or a single
// whitespace/comma-separated string. Returns nil when no key is present.
func parseStringList(m map[string]any, keys ...string) []string {
for _, k := range keys {
raw, ok := m[k]
if !ok {
continue
}
switch v := raw.(type) {
case []string:
if len(v) > 0 {
return v
}
case []any:
out := make([]string, 0, len(v))
for _, e := range v {
if s, ok := e.(string); ok {
s = strings.TrimSpace(s)
if s != "" {
out = append(out, s)
}
}
}
if len(out) > 0 {
return out
}
case string:
s := strings.TrimSpace(v)
if s == "" {
continue
}
// Tolerate comma- or whitespace-separated lists.
parts := strings.FieldsFunc(s, func(r rune) bool { return r == ',' || r == ' ' || r == '\t' || r == '\n' })
var out []string
for _, p := range parts {
p = strings.TrimSpace(p)
if p != "" {
out = append(out, p)
}
}
if len(out) > 0 {
return out
}
}
}
return nil
}
// VectorFromChunkMap extracts a pre-computed embedding from a chunk map. The
// upstream pipeline (and the knowledge-compile store) carry the vector under a
// dimension-specific column key "q_<dim>_vec" — set by the tokenizer via

205
internal/ingestion/component/knowledge_compiler/component.go
View File

@@ -1,6 +1,6 @@
// Package knowledge_compiler implements the KnowledgeCompiler ingestion
// component: a single runtime.Component that dispatches to one of the
// knowledge-compile variants (structure / wiki / raptor / mindmap / datasetnav)
// knowledge-compile variants (structure / wiki / tree / mindmap / datasetnav)
// based on the `variant` param. See PORT_PLAN.md for the full design.
package knowledge_compiler
@@ -13,8 +13,8 @@ import (
"ragflow/internal/ingestion/component/knowledge_compiler/common"
"ragflow/internal/ingestion/component/knowledge_compiler/datasetnav"
"ragflow/internal/ingestion/component/knowledge_compiler/mindmap"
"ragflow/internal/ingestion/component/knowledge_compiler/raptor"
"ragflow/internal/ingestion/component/knowledge_compiler/structure"
"ragflow/internal/ingestion/component/knowledge_compiler/tree"
"ragflow/internal/ingestion/component/knowledge_compiler/wiki"
"ragflow/internal/ingestion/component/schema"
"ragflow/internal/tokenizer"
@@ -92,73 +92,83 @@ func (c *KnowledgeCompilerComponent) Invoke(ctx context.Context, db *gorm.DB, in
tenantID, _ := inputs["tenant_id"].(string)
datasetID, _ := inputs["dataset_id"].(string)
// Validate the variant before resolving deps so a bad variant fails fast
// with ErrUnknownVariant rather than a deps-resolution error.
switch param.Variant {
case common.VariantStructure, common.VariantWiki, common.VariantRaptor,
common.VariantMindmap, common.VariantDatasetnav:
// recognised; dispatch below
default:
return nil, fmt.Errorf("%w: %q", common.ErrUnknownVariant, param.Variant)
}
deps, err := common.ResolveDeps(tenantID, param.LLMID, param.EmbeddingModel)
// Resolve the compilation template spec(s). Priority:
// compilation_template_id > compilation_template_group_id. The variant is
// derived from each template's kind (see common.KindToVariant), not from
// the DSL.
specs, err := resolveTemplateSpecs(ctx, tenantID, param)
if err != nil {
return nil, err
}
deps.TenantID = tenantID
deps.DatasetID = datasetID
// Resolve compilation-template-group ids to concrete template ids and merge
// them with any explicitly-passed template ids. Group-only configs that
// cannot be resolved fail loudly rather than silently emitting rows missing
// compilation_template_ids (a data-loss path).
resolvedGroups, err := common.ResolveGroupTemplateIDs(ctx, tenantID, param.GroupIDs)
if err != nil {
return nil, err
}
templateIDs := append(append([]string{}, param.TemplateIDs...), resolvedGroups...)
in, err := buildInputs(inputs, param)
if err != nil {
return nil, err
}
// Stamp the resolved template ids onto every compiled product after the
// variant returns. All products are buffered in out.Products (there is no
// streaming sink path), so the post-Run loop below covers every row (M1).
// Each spec compiles independently; products are stamped with the producing
// template's id and kind. All products are buffered (no streaming sink), so
// the post-run loop below covers every row (M1).
var out common.Outputs
switch param.Variant {
case common.VariantStructure:
out, err = structure.Run(ctx, deps, param, in)
case common.VariantWiki:
out, err = wiki.Run(ctx, deps, param, in)
case common.VariantRaptor:
out, err = raptor.Run(ctx, deps, param, in)
case common.VariantMindmap:
out, err = mindmap.Run(ctx, deps, param, in)
case common.VariantDatasetnav:
out, err = datasetnav.Run(ctx, deps, param, in)
default:
return nil, fmt.Errorf("%w: %q", common.ErrUnknownVariant, param.Variant)
}
if err != nil {
return nil, err
}
// Stamp the compilation template ids onto every product so the chunk
// converter can emit compilation_template_ids (Python stamps one
// template_id per row; here we carry the full resolved list since the Go
// component runs one variant per Invoke and the caller may pass multiple
// template ids from a compilation-template-group). The list is the union of
// explicitly-passed template ids and any resolved from group ids.
if len(templateIDs) > 0 {
for i := range out.Products {
if out.Products[i].Meta == nil {
out.Products[i].Meta = map[string]any{}
}
out.Products[i].Meta["compilation_template_ids"] = templateIDs
for _, spec := range specs {
variant, err := common.KindToVariant(spec.Kind)
if err != nil {
return nil, err
}
// Per-spec Param: variant + resolved template config, with scalar config
// overrides (language, similarity, workers, dedup, llm/embedding) layered
// on top of the DSL params and Extra. This is how the template's stored
// "content" drives the compile without the caller passing variant-specific
// params inline.
specParam := param
specParam.Variant = variant
specParam.TemplateID = spec.ID
specParam.TemplateConfig = spec.Config
overlayTemplateConfig(&specParam, spec.Config)
deps, err := common.ResolveDeps(tenantID, specParam.LLMID, specParam.EmbeddingModel)
if err != nil {
return nil, err
}
deps.TenantID = tenantID
deps.DatasetID = datasetID
specIn := in
if specIn.VariantSpecific == nil {
specIn.VariantSpecific = map[string]any{}
}
specIn.VariantSpecific["config"] = spec.Config
var o common.Outputs
switch variant {
case common.VariantStructure:
o, err = structure.Run(ctx, deps, specParam, specIn)
case common.VariantWiki:
o, err = wiki.Run(ctx, deps, specParam, specIn)
case common.VariantTree:
o, err = tree.Run(ctx, deps, specParam, specIn)
case common.VariantMindmap:
o, err = mindmap.Run(ctx, deps, specParam, specIn)
case common.VariantDatasetnav:
o, err = datasetnav.Run(ctx, deps, specParam, specIn)
default:
return nil, fmt.Errorf("%w: %q", common.ErrUnknownVariant, variant)
}
if err != nil {
return nil, err
}
for i := range o.Products {
if o.Products[i].Meta == nil {
o.Products[i].Meta = map[string]any{}
}
o.Products[i].Meta["compilation_template_ids"] = []string{spec.ID}
o.Products[i].Kind = spec.Kind
o.Products[i].TemplateID = spec.ID
o.Products[i].Variant = variant
}
out.Products = append(out.Products, o.Products...)
}
// Convert the compiled products into chunk-aligned docs (matching
@@ -172,6 +182,79 @@ func (c *KnowledgeCompilerComponent) Invoke(ctx context.Context, db *gorm.DB, in
return mergeChunks(inputs, compiled), nil
}
// resolveTemplateSpecs resolves the configured compilation template spec(s) to
// their TemplateInfo rows. Priority: compilation_template_id >
// compilation_template_group_id. The group path resolves the group to its
// child template ids and then loads each template.
func resolveTemplateSpecs(ctx context.Context, tenantID string, param common.Param) ([]common.TemplateInfo, error) {
if param.CompilationTemplateID != "" {
info, err := common.ResolveTemplate(ctx, tenantID, param.CompilationTemplateID)
if err != nil {
return nil, err
}
return []common.TemplateInfo{info}, nil
}
if param.CompilationTemplateGroupID != "" {
ids, err := common.ResolveGroupTemplateIDs(ctx, tenantID, []string{param.CompilationTemplateGroupID})
if err != nil {
return nil, err
}
specs := make([]common.TemplateInfo, 0, len(ids))
for _, id := range ids {
info, err := common.ResolveTemplate(ctx, tenantID, id)
if err != nil {
return nil, err
}
specs = append(specs, info)
}
return specs, nil
}
return nil, fmt.Errorf("knowledge_compiler: one of compilation_template_id or compilation_template_group_id is required")
}
// overlayTemplateConfig layers scalar fields from the resolved template config
// (the template "content") onto the param. For self-documenting defaults
// (language, similarity_threshold, max_workers, enable_historical_dedup) the
// config value wins when the param is unset. For llm_id / embedding_model, the
// caller's per-call override always wins; the config only fills them in when
// the caller left them empty.
func overlayTemplateConfig(param *common.Param, cfg map[string]any) {
if cfg == nil {
return
}
if v, ok := cfg["language"].(string); ok && v != "" {
param.Language = v
}
if v, ok := cfg["similarity_threshold"].(float64); ok && v > 0 {
param.SimilarityThreshold = v
}
if v, ok := cfg["max_workers"].(float64); ok && int(v) > 0 {
param.MaxWorkers = int(v)
}
if v, ok := cfg["enable_historical_dedup"].(bool); ok {
param.EnableHistoricalDedup = v
}
// llm_id / embedding_model are optional per-call overrides documented on
// Invoke. The template config supplies defaults, so only apply them when
// the caller has not already provided an explicit value (the caller wins).
if v, ok := cfg["llm_id"].(string); ok && v != "" && param.LLMID == "" {
param.LLMID = v
}
if v, ok := cfg["embedding_model"].(string); ok && v != "" && param.EmbeddingModel == "" {
param.EmbeddingModel = v
}
}
// kindOrVariant returns the original template kind when present (the true
// compilation_template.kind, e.g. "page_index"), otherwise the collapsed Go
// variant. It drives the compilation_template_kind_kwd stamp.
func kindOrVariant(p common.Product) string {
if p.Kind != "" {
return p.Kind
}
return string(p.Variant)
}
// variantCompileKWD maps each Go variant to the compile_kwd discriminator value
// Python writes into ES (rag/advanced_rag/knowlege_compile). It is the primary
// key that distinguishes compiled knowledge units from ordinary chunks and
@@ -179,7 +262,7 @@ func (c *KnowledgeCompilerComponent) Invoke(ctx context.Context, db *gorm.DB, in
var variantCompileKWD = map[common.Variant]string{
common.VariantStructure: "structure",
common.VariantWiki: "artifact_page",
common.VariantRaptor: "raptor",
common.VariantTree: "tree",
common.VariantMindmap: "mindmap",
common.VariantDatasetnav: "dataset_nav",
}
@@ -237,7 +320,7 @@ func productsToChunkDocs(products []common.Product) ([]schema.ChunkDoc, error) {
if err := doc.SetExtraValue("compile_kwd", compileKWD); err != nil {
return nil, err
}
if err := doc.SetExtraValue("compilation_template_kind_kwd", string(p.Variant)); err != nil {
if err := doc.SetExtraValue("compilation_template_kind_kwd", kindOrVariant(p)); err != nil {
return nil, err
}
if p.ParentID != "" {
@@ -398,7 +481,7 @@ func applyVariantColumns(doc *schema.ChunkDoc, p common.Product) error {
}
}
case common.VariantRaptor:
case common.VariantTree:
// raptor_kwd tags summary/root nodes; raptor_layer_int records tree depth.
if kind != "" {
if err := doc.SetExtraValue("raptor_kwd", kind); err != nil {

176
internal/ingestion/component/knowledge_compiler/component_test.go
View File

@@ -6,6 +6,7 @@ import (
"errors"
"fmt"
"hash/fnv"
"os"
"regexp"
"strings"
"sync"
@@ -109,7 +110,7 @@ func (mockChat) Chat(_ context.Context, req common.ChatRequest) (*common.ChatRes
}
// proseChat returns generic, non-empty prose for the non-structure variants
// (wiki/raptor/mindmap/datasetnav), which all just need a summary/outline text.
// (wiki/tree/mindmap/datasetnav), which all just need a summary/outline text.
type proseChat struct{}
func (proseChat) Chat(_ context.Context, req common.ChatRequest) (*common.ChatResponse, error) {
@@ -167,9 +168,10 @@ func installMockDeps(t *testing.T) {
func TestKnowledgeCompiler_Structure_EndToEnd(t *testing.T) {
installMockDeps(t)
installVariantTemplateResolver(t, "structure")
c, err := NewKnowledgeCompilerComponent("KnowledgeCompiler", map[string]any{
"variant": "structure", "llm_id": "llm1", "embedding_model": "emb1",
"compilation_template_id": "tpl-structure", "llm_id": "llm1", "embedding_model": "emb1",
})
if err != nil {
t.Fatalf("NewKnowledgeCompilerComponent: %v", err)
@@ -227,7 +229,7 @@ func TestKnowledgeCompiler_Structure_EndToEnd(t *testing.T) {
}
func TestKnowledgeCompiler_UnknownVariant(t *testing.T) {
c, err := NewKnowledgeCompilerComponent("KnowledgeCompiler", map[string]any{"variant": "nope"})
c, err := NewKnowledgeCompilerComponent("KnowledgeCompiler", map[string]any{"compilation_template_id": "nope"})
if err != nil {
t.Fatalf("NewKnowledgeCompilerComponent: %v", err)
}
@@ -242,7 +244,7 @@ func TestKnowledgeCompiler_Alias_Mindmap(t *testing.T) {
// "mind_map" is the deprecated alias for "mindmap"; both resolve to the
// implemented mindmap variant and must run (not ErrUnknownVariant / stub).
c, err := NewKnowledgeCompilerComponent("KnowledgeCompiler", map[string]any{
"variant": "mind_map", "llm_id": "llm1", "embedding_model": "emb1",
"compilation_template_id": "mind_map", "llm_id": "llm1", "embedding_model": "emb1",
})
if err != nil {
t.Fatalf("NewKnowledgeCompilerComponent: %v", err)
@@ -266,7 +268,8 @@ func TestKnowledgeCompiler_Alias_Mindmap(t *testing.T) {
// chunk list (upstream input chunks + compiled knowledge-unit chunks).
func runVariant(t *testing.T, variant string, extra map[string]any) []map[string]any {
t.Helper()
params := map[string]any{"variant": variant, "llm_id": "llm1", "embedding_model": "emb1"}
installVariantTemplateResolver(t, variant)
params := map[string]any{"compilation_template_id": "tpl-" + variant, "llm_id": "llm1", "embedding_model": "emb1"}
for k, v := range extra {
params[k] = v
}
@@ -334,10 +337,10 @@ func TestKnowledgeCompiler_Wiki_EndToEnd(t *testing.T) {
}
}
func TestKnowledgeCompiler_Raptor_EndToEnd(t *testing.T) {
func TestKnowledgeCompiler_Tree_EndToEnd(t *testing.T) {
installProseDeps(t)
// watershed (default tree_order): zero external clustering dependency.
chunks := runVariant(t, "raptor", nil)
chunks := runVariant(t, "tree", nil)
foundRoot := false
for _, c := range chunks {
if kind, _ := c["kc_kind"].(string); kind == "root" {
@@ -345,14 +348,14 @@ func TestKnowledgeCompiler_Raptor_EndToEnd(t *testing.T) {
}
}
if !foundRoot {
t.Fatalf("raptor(default): no 'root' chunk; got %d chunks", len(chunks))
t.Fatalf("tree(default): no 'root' chunk; got %d chunks", len(chunks))
}
// A smaller tree_order (more, smaller clusters) must also run and still
// produce a well-formed tree (root present, chunks non-empty).
chunksCoarse := runVariant(t, "raptor", map[string]any{"extra": map[string]any{"tree_order": 2}})
chunksCoarse := runVariant(t, "tree", map[string]any{"extra": map[string]any{"tree_order": 2}})
if len(chunksCoarse) == 0 {
t.Fatalf("raptor(tree_order=2): produced no chunks")
t.Fatalf("tree(tree_order=2): produced no chunks")
}
foundRootCoarse := false
for _, c := range chunksCoarse {
@@ -361,7 +364,7 @@ func TestKnowledgeCompiler_Raptor_EndToEnd(t *testing.T) {
}
}
if !foundRootCoarse {
t.Fatalf("raptor(tree_order=2): no 'root' chunk; got %d chunks", len(chunksCoarse))
t.Fatalf("tree(tree_order=2): no 'root' chunk; got %d chunks", len(chunksCoarse))
}
}
@@ -405,9 +408,10 @@ func TestKnowledgeCompiler_Datasetnav_EndToEnd(t *testing.T) {
// {output_format:"chunks", chunks:[...]}.
func TestKnowledgeCompiler_EmitsChunks(t *testing.T) {
installMockDeps(t)
installVariantTemplateResolver(t, "structure")
c, err := NewKnowledgeCompilerComponent("KnowledgeCompiler", map[string]any{
"variant": "structure", "llm_id": "llm1", "embedding_model": "emb1",
"compilation_template_id": "tpl-structure", "llm_id": "llm1", "embedding_model": "emb1",
})
if err != nil {
t.Fatalf("NewKnowledgeCompilerComponent: %v", err)
@@ -461,12 +465,12 @@ func TestKnowledgeCompiler_EmitsChunks(t *testing.T) {
// pointing back at their originating input chunks.
func TestKnowledgeCompiler_TemplateIDsAndProvenance(t *testing.T) {
installMockDeps(t)
installVariantTemplateResolver(t, "structure")
c, err := NewKnowledgeCompilerComponent("KnowledgeCompiler", map[string]any{
"variant": "structure",
"llm_id": "llm1",
"embedding_model": "emb1",
"compilation_template_ids": []any{"tpl-A", "tpl-B"},
"compilation_template_id": "tpl-structure",
"llm_id": "llm1",
"embedding_model": "emb1",
})
if err != nil {
t.Fatalf("NewKnowledgeCompilerComponent: %v", err)
@@ -488,7 +492,8 @@ func TestKnowledgeCompiler_TemplateIDsAndProvenance(t *testing.T) {
if ck, _ := cm["compile_kwd"].(string); ck != "hypergraph" {
continue
}
// Every compiled chunk must carry the resolved template ids.
// Every compiled chunk must carry the resolved template id (one per
// template; the component stamps the producing template's id).
var tidsCount int
switch tids := cm["compilation_template_ids"].(type) {
case []any:
@@ -496,8 +501,8 @@ func TestKnowledgeCompiler_TemplateIDsAndProvenance(t *testing.T) {
case []string:
tidsCount = len(tids)
}
if tidsCount != 2 {
t.Fatalf("compiled chunk %v: compilation_template_ids = %v, want 2", cm["id"], cm["compilation_template_ids"])
if tidsCount != 1 {
t.Fatalf("compiled chunk %v: compilation_template_ids = %v, want 1 (the resolved template id)", cm["id"], cm["compilation_template_ids"])
}
// Entity rows must carry source_chunk_ids.
if kg, _ := cm["knowledge_graph_kwd"].(string); kg == "entity" {
@@ -534,8 +539,8 @@ func (m constEmbedder) Encode(_ context.Context, texts []string) ([][]float32, e
return out, nil
}
// TestKnowledgeCompiler_Raptor_DegenerateNoInfiniteLoop is a regression for the
// High-1 bug: RAPTOR could recurse forever when a re-cluster returned a single
// TestKnowledgeCompiler_Tree_DegenerateNoInfiniteLoop is a regression for the
// High-1 bug: the tree builder could recurse forever when a re-cluster returned a single
// label covering all points. The default mockEmbedder emits non-negative
// vectors, so under the watershed default (tree_order=4, ratio 25) every adjacent pair
// has cosine >= 0 and a pathological input can collapse into one cluster; with
@@ -543,10 +548,11 @@ func (m constEmbedder) Encode(_ context.Context, texts []string) ([][]float32, e
// work item at level+1 and hung. This test uses 6 chunks and asserts the run
// terminates with a well-formed root. (If the guard regresses, go test's timeout
// turns the hang into a failure.)
func TestKnowledgeCompiler_Raptor_DegenerateNoInfiniteLoop(t *testing.T) {
func TestKnowledgeCompiler_Tree_DegenerateNoInfiniteLoop(t *testing.T) {
installProseDeps(t)
installVariantTemplateResolver(t, "tree")
c, err := NewKnowledgeCompilerComponent("KnowledgeCompiler", map[string]any{
"variant": "raptor", "llm_id": "llm1", "embedding_model": "emb1",
"compilation_template_id": "tpl-tree", "llm_id": "llm1", "embedding_model": "emb1",
"extra": map[string]any{"tree_order": 4},
})
if err != nil {
@@ -565,7 +571,7 @@ func TestKnowledgeCompiler_Raptor_DegenerateNoInfiniteLoop(t *testing.T) {
"tenant_id": "t1",
})
if err != nil {
t.Fatalf("Invoke hung or errored (RAPTOR infinite recursion?): %v", err)
t.Fatalf("Invoke hung or errored (tree infinite recursion?): %v", err)
}
raw, ok := out["chunks"].([]any)
if !ok {
@@ -580,7 +586,7 @@ func TestKnowledgeCompiler_Raptor_DegenerateNoInfiniteLoop(t *testing.T) {
}
}
if !foundRoot {
t.Fatalf("raptor(degenerate): no root chunk; got %d chunks", len(raw))
t.Fatalf("tree(degenerate): no root chunk; got %d chunks", len(raw))
}
}
@@ -599,8 +605,9 @@ func TestKnowledgeCompiler_Wiki_HistoricalDedupDropsDuplicates(t *testing.T) {
})
t.Cleanup(func() { common.SetDepsResolver(nil) })
installVariantTemplateResolver(t, "wiki")
c, err := NewKnowledgeCompilerComponent("KnowledgeCompiler", map[string]any{
"variant": "wiki", "llm_id": "llm1", "embedding_model": "emb1",
"compilation_template_id": "tpl-wiki", "llm_id": "llm1", "embedding_model": "emb1",
})
if err != nil {
t.Fatalf("NewKnowledgeCompilerComponent: %v", err)
@@ -657,8 +664,9 @@ func TestKnowledgeCompiler_Wiki_UpdateMergesExistingPage(t *testing.T) {
})
t.Cleanup(func() { common.SetDepsResolver(nil) })
installVariantTemplateResolver(t, "wiki")
c, err := NewKnowledgeCompilerComponent("KnowledgeCompiler", map[string]any{
"variant": "wiki", "llm_id": "llm1", "embedding_model": "emb1",
"compilation_template_id": "tpl-wiki", "llm_id": "llm1", "embedding_model": "emb1",
})
if err != nil {
t.Fatalf("NewKnowledgeCompilerComponent: %v", err)
@@ -737,8 +745,9 @@ func TestKnowledgeCompiler_Wiki_HistoricalDedupScopedByDataset(t *testing.T) {
})
t.Cleanup(func() { common.SetDepsResolver(nil) })
installVariantTemplateResolver(t, "wiki")
c, err := NewKnowledgeCompilerComponent("KnowledgeCompiler", map[string]any{
"variant": "wiki", "llm_id": "llm1", "embedding_model": "emb1",
"compilation_template_id": "tpl-wiki", "llm_id": "llm1", "embedding_model": "emb1",
"enable_historical_dedup": true,
})
if err != nil {
@@ -809,8 +818,9 @@ func TestKnowledgeCompiler_Datasetnav_LockKeyedByDataset(t *testing.T) {
})
t.Cleanup(func() { common.SetDepsResolver(nil) })
installVariantTemplateResolver(t, "datasetnav")
c, err := NewKnowledgeCompilerComponent("KnowledgeCompiler", map[string]any{
"variant": "datasetnav", "llm_id": "llm1", "embedding_model": "emb1",
"compilation_template_id": "tpl-datasetnav", "llm_id": "llm1", "embedding_model": "emb1",
})
if err != nil {
t.Fatalf("NewKnowledgeCompilerComponent: %v", err)
@@ -873,8 +883,9 @@ func TestKnowledgeCompiler_Datasetnav_RootIncludesAllSummaries(t *testing.T) {
})
t.Cleanup(func() { common.SetDepsResolver(nil) })
installVariantTemplateResolver(t, "datasetnav")
c, err := NewKnowledgeCompilerComponent("KnowledgeCompiler", map[string]any{
"variant": "datasetnav", "llm_id": "llm1", "embedding_model": "emb1",
"compilation_template_id": "tpl-datasetnav", "llm_id": "llm1", "embedding_model": "emb1",
// nav_radius > 1 guarantees no two chunks group together (cosine <= 1),
// so each chunk becomes its own nav node.
"extra": map[string]any{"nav_radius": 1.1},
@@ -983,8 +994,9 @@ func TestKnowledgeCompiler_Structure_FencedJSONNotDropped(t *testing.T) {
})
t.Cleanup(func() { common.SetDepsResolver(nil) })
installVariantTemplateResolver(t, "structure")
c, err := NewKnowledgeCompilerComponent("KnowledgeCompiler", map[string]any{
"variant": "structure", "llm_id": "llm1", "embedding_model": "emb1",
"compilation_template_id": "tpl-structure", "llm_id": "llm1", "embedding_model": "emb1",
})
if err != nil {
t.Fatalf("NewKnowledgeCompilerComponent: %v", err)
@@ -1022,8 +1034,9 @@ func TestKnowledgeCompiler_Structure_MalformedJSONFailsLoud(t *testing.T) {
})
t.Cleanup(func() { common.SetDepsResolver(nil) })
installVariantTemplateResolver(t, "structure")
c, err := NewKnowledgeCompilerComponent("KnowledgeCompiler", map[string]any{
"variant": "structure", "llm_id": "llm1", "embedding_model": "emb1",
"compilation_template_id": "tpl-structure", "llm_id": "llm1", "embedding_model": "emb1",
})
if err != nil {
t.Fatalf("NewKnowledgeCompilerComponent: %v", err)
@@ -1046,9 +1059,10 @@ func TestKnowledgeCompiler_Structure_MalformedJSONFailsLoud(t *testing.T) {
// pipeline), so they must be forwarded when present.
func TestKnowledgeCompiler_PassThroughEnvelope(t *testing.T) {
installMockDeps(t)
installVariantTemplateResolver(t, "structure")
c, err := NewKnowledgeCompilerComponent("KnowledgeCompiler", map[string]any{
"variant": "structure", "llm_id": "llm1", "embedding_model": "emb1",
"compilation_template_id": "tpl-structure", "llm_id": "llm1", "embedding_model": "emb1",
})
if err != nil {
t.Fatalf("NewKnowledgeCompilerComponent: %v", err)
@@ -1091,21 +1105,34 @@ func groupResolverStub(_ context.Context, _ string, groupIDs []string) ([]string
return out, nil
}
// TestKnowledgeCompiler_GroupIDsResolvedToTemplateIDs is a regression for the
// open question: compilation_template_group_ids were parsed into Param.GroupIDs
// but never resolved or stamped, so group-only configs silently missed
// compilation_template_ids. The component now resolves groups via the installed
// GroupResolver and merges them into the stamped template-id list.
// TestKnowledgeCompiler_GroupIDsResolvedToTemplateIDs is a regression: a
// compilation_template_group_id is resolved via the installed GroupResolver to
// its concrete template ids, and each template is compiled as its own spec and
// stamped with its producing template id. Group-only configs must not silently
// miss compilation_template_ids.
func TestKnowledgeCompiler_GroupIDsResolvedToTemplateIDs(t *testing.T) {
installMockDeps(t)
common.SetGroupResolver(groupResolverStub)
t.Cleanup(func() { common.SetGroupResolver(nil) })
// The group resolves to two concrete template ids; each is compiled as its
// own spec, so the TemplateResolver must return a valid kind for them.
// Override the package stub and restore it afterwards.
common.SetTemplateResolver(func(ctx context.Context, tenantID, templateID string) (common.TemplateInfo, error) {
kind := templateID
if strings.HasPrefix(templateID, "tpl-grp1") {
kind = "structure"
}
return common.TemplateInfo{ID: templateID, Kind: kind, Config: map[string]any{}}, nil
})
t.Cleanup(func() { common.SetTemplateResolver(testTemplateResolver) })
common.SetGroupResolver(groupResolverStub)
t.Cleanup(func() { common.SetGroupResolver(testGroupResolver) })
// compilation_template_group_id (not the obsolete plural list) selects the
// group; compilation_template_id is absent so the group path is taken.
c, err := NewKnowledgeCompilerComponent("KnowledgeCompiler", map[string]any{
"variant": "structure",
"llm_id": "llm1",
"embedding_model": "emb1",
"compilation_template_group_ids": []any{"grp1"},
"compilation_template_group_id": "grp1",
"llm_id": "llm1",
"embedding_model": "emb1",
})
if err != nil {
t.Fatalf("NewKnowledgeCompilerComponent: %v", err)
@@ -1119,6 +1146,7 @@ func TestKnowledgeCompiler_GroupIDsResolvedToTemplateIDs(t *testing.T) {
t.Fatalf("Invoke: %v", err)
}
want := map[string]bool{"tpl-grp1-a": true, "tpl-grp1-b": true}
seen := map[string]bool{}
checked := 0
for _, r := range out["chunks"].([]any) {
cm := r.(map[string]any)
@@ -1137,13 +1165,21 @@ func TestKnowledgeCompiler_GroupIDsResolvedToTemplateIDs(t *testing.T) {
case []string:
got = v
}
if len(got) != 2 || !want[got[0]] || !want[got[1]] {
t.Fatalf("compiled chunk %v: compilation_template_ids = %v, want %v (group ids must resolve to template ids)", cm["id"], got, want)
// Each product is stamped with the single template id that produced it.
if len(got) != 1 || !want[got[0]] {
t.Fatalf("compiled chunk %v: compilation_template_ids = %v, want exactly one of %v", cm["id"], got, want)
}
seen[got[0]] = true
}
if checked == 0 {
t.Fatalf("no compiled chunks inspected (compile_kwd=list expected); assertion was vacuous")
}
// Both resolved template ids must appear across the products.
for id := range want {
if !seen[id] {
t.Fatalf("resolved template id %q was never stamped on any product", id)
}
}
}
// TestKnowledgeCompiler_GroupIDsWithoutResolverFailsLoud is the companion
@@ -1155,10 +1191,9 @@ func TestKnowledgeCompiler_GroupIDsWithoutResolverFailsLoud(t *testing.T) {
common.SetGroupResolver(nil) // ensure no resolver is installed
c, err := NewKnowledgeCompilerComponent("KnowledgeCompiler", map[string]any{
"variant": "structure",
"llm_id": "llm1",
"embedding_model": "emb1",
"compilation_template_group_ids": []any{"grp1"},
"compilation_template_group_id": "grp1",
"llm_id": "llm1",
"embedding_model": "emb1",
})
if err != nil {
t.Fatalf("NewKnowledgeCompilerComponent: %v", err)
@@ -1175,3 +1210,44 @@ func TestKnowledgeCompiler_GroupIDsWithoutResolverFailsLoud(t *testing.T) {
t.Fatalf("error %q does not mention GroupResolver", err.Error())
}
}
// testTemplateResolver / testGroupResolver are the package-wide stub resolvers
// installed by TestMain. The stub maps the template id 1:1 onto the template
// kind, which lets tests resolve arbitrary synthetic ids (e.g. "tpl-structure")
// to a kind. Tests must pass a TEMPLATE ID, never a variant name, as
// compilation_template_id; use installVariantTemplateResolver(t, variant) to
// register a synthetic "tpl-<variant>" id that maps to the desired kind.
// Production wiring installs the real DB-backed resolvers (see
// internal/ingestion/task/knowledge_compiler_wiring.go).
var testTemplateResolver common.TemplateResolver = func(ctx context.Context, tenantID, templateID string) (common.TemplateInfo, error) {
return common.TemplateInfo{ID: templateID, Kind: templateID, Config: map[string]any{}}, nil
}
// installVariantTemplateResolver wires a template resolver so the synthetic
// template id "tpl-<variant>" resolves to Kind <variant>. This lets a test
// select a compiler variant through the production id -> kind -> variant path
// (resolveTemplateSpecs -> KindToVariant) instead of misusing the variant name
// as the compilation_template_id. Any other template id is delegated to the
// package default; the default is restored on cleanup.
func installVariantTemplateResolver(t *testing.T, variant string) {
t.Helper()
prev := testTemplateResolver
common.SetTemplateResolver(func(ctx context.Context, tenantID, templateID string) (common.TemplateInfo, error) {
if templateID == "tpl-"+variant {
return common.TemplateInfo{ID: templateID, Kind: variant, Config: map[string]any{}}, nil
}
return prev(ctx, tenantID, templateID)
})
t.Cleanup(func() { common.SetTemplateResolver(testTemplateResolver) })
}
var testGroupResolver common.GroupResolver = func(ctx context.Context, tenantID string, groupIDs []string) ([]string, error) {
return groupIDs, nil
}
// TestMain installs the stub resolvers for the variant unit tests.
func TestMain(m *testing.M) {
common.SetTemplateResolver(testTemplateResolver)
common.SetGroupResolver(testGroupResolver)
os.Exit(m.Run())
}

2
internal/ingestion/component/knowledge_compiler/golden/corpus.go
View File

@@ -17,7 +17,7 @@ import (
// FixedCorpus is the canonical sample document. It describes a small, coherent
// ecosystem of named entities (Alpha/Beta/Gamma/Delta/Epsilon) with dense
// cross-references so structure extraction, wiki synthesis, and raptor
// cross-references so structure extraction, wiki synthesis, and tree
// clustering all have meaningful signal.
func FixedCorpus() []common.Chunk {
texts := []string{

0
internal/ingestion/component/knowledge_compiler/golden/fixtures/raptor_baseline.json → internal/ingestion/component/knowledge_compiler/golden/fixtures/tree_baseline.json
View File

69
internal/ingestion/component/knowledge_compiler/golden/golden_test.go
View File

@@ -44,27 +44,27 @@ func TestChunksToAny(t *testing.T) {
}
}
func TestAnalyzeRaptorProducts_TreeShape(t *testing.T) {
func TestAnalyzeTreeProducts_TreeShape(t *testing.T) {
vector := json.RawMessage(`[0.1,0.2,0.3]`)
chunks := []schema.ChunkDoc{
{Text: "root summary", Extra: mustExtras(t, map[string]any{
"id": "r1", "doc_id": "d1", "tenant_id": "t1", "compile_kwd": "raptor",
"id": "r1", "doc_id": "d1", "tenant_id": "t1", "compile_kwd": "tree",
"kc_kind": "root", "kc_level": float64(-1), "q_3_vec": vector,
})},
{Text: "leaf A", Extra: mustExtras(t, map[string]any{
"id": "a1", "doc_id": "d1", "tenant_id": "t1", "compile_kwd": "raptor",
"id": "a1", "doc_id": "d1", "tenant_id": "t1", "compile_kwd": "tree",
"kc_kind": "summary", "kc_level": float64(0), "parent_kwd": "r1", "q_3_vec": vector,
})},
{Text: "leaf B", Extra: mustExtras(t, map[string]any{
"id": "b1", "doc_id": "d1", "tenant_id": "t1", "compile_kwd": "raptor",
"id": "b1", "doc_id": "d1", "tenant_id": "t1", "compile_kwd": "tree",
"kc_kind": "summary", "kc_level": float64(0), "parent_kwd": "r1", "q_3_vec": vector,
})},
{Text: "mid A", Extra: mustExtras(t, map[string]any{
"id": "m1", "doc_id": "d1", "tenant_id": "t1", "compile_kwd": "raptor",
"id": "m1", "doc_id": "d1", "tenant_id": "t1", "compile_kwd": "tree",
"kc_kind": "summary", "kc_level": float64(1), "parent_kwd": "r1", "q_3_vec": vector,
})},
}
m := AnalyzeRaptorProducts(chunks)
m := AnalyzeTreeProducts(chunks)
if m.RootCount != 1 {
t.Errorf("RootCount = %d, want 1", m.RootCount)
}
@@ -82,34 +82,73 @@ func TestAnalyzeRaptorProducts_TreeShape(t *testing.T) {
}
}
func TestAnalyzeRaptorProducts_DetectsDanglingParent(t *testing.T) {
func TestAnalyzeTreeProducts_DetectsDanglingParent(t *testing.T) {
chunks := []schema.ChunkDoc{
{Text: "root", Extra: mustExtras(t, map[string]any{
"id": "r1", "doc_id": "d1", "tenant_id": "t1", "compile_kwd": "raptor",
"id": "r1", "doc_id": "d1", "tenant_id": "t1", "compile_kwd": "tree",
"kc_kind": "root",
})},
{Text: "orphan", Extra: mustExtras(t, map[string]any{
"id": "o1", "doc_id": "d1", "tenant_id": "t1", "compile_kwd": "raptor",
"id": "o1", "doc_id": "d1", "tenant_id": "t1", "compile_kwd": "tree",
"kc_kind": "summary", "kc_level": float64(0), "parent_kwd": "missing-parent",
})},
}
m := AnalyzeRaptorProducts(chunks)
m := AnalyzeTreeProducts(chunks)
if m.AllParented {
t.Error("AllParented = true, want false (o1's parent is missing)")
}
}
func TestCoverageFraction_AllParentedOneRoot(t *testing.T) {
m := TreeMetrics{RootCount: 1, LeafClusters: 3, AllParented: true}
func TestCoverageFraction_AllSourcesCovered(t *testing.T) {
// Coverage is now measured from source_chunk_ids of level-0 leaf clusters,
// not structural well-formedness. When every input chunk is referenced, the
// fraction is 1.0 regardless of root/parent structure.
m := TreeMetrics{RootCount: 1, LeafClusters: 3, AllParented: true, CoveredSources: 12}
if cov := m.CoverageFraction(12); cov != 1.0 {
t.Errorf("CoverageFraction = %v, want 1.0", cov)
}
}
func TestCoverageFraction_NoRootIsZero(t *testing.T) {
m := TreeMetrics{LeafClusters: 3, AllParented: true}
func TestCoverageFraction_NoCoveredSourcesIsZero(t *testing.T) {
// A structurally well-formed tree that references no source chunks covers 0.
m := TreeMetrics{RootCount: 1, LeafClusters: 3, AllParented: true, CoveredSources: 0}
if cov := m.CoverageFraction(12); cov != 0.0 {
t.Errorf("CoverageFraction = %v, want 0.0 (no root)", cov)
t.Errorf("CoverageFraction = %v, want 0.0 (no covered sources)", cov)
}
}
func TestCoverageFraction_PartialCoverage(t *testing.T) {
// A tree that drops some source chunks scores below 1.0.
m := TreeMetrics{RootCount: 1, LeafClusters: 3, AllParented: true, CoveredSources: 9}
if cov := m.CoverageFraction(12); cov != 0.75 {
t.Errorf("CoverageFraction = %v, want 0.75", cov)
}
}
// TestAnalyzeTreeProducts_IgnoresUnknownSourceIDs verifies that an unknown
// (leaked/garbage) ID inside source_chunk_ids does not inflate CoveredSources
// past the corpus size, so CoverageFraction stays <= 1.0. A level-0 leaf that
// references 2 valid + 1 unknown ID over an nChunks=2 corpus must still report
// coverage 1.0, not 1.5.
func TestAnalyzeTreeProducts_IgnoresUnknownSourceIDs(t *testing.T) {
vector := json.RawMessage(`[0.1,0.2,0.3]`)
chunks := []schema.ChunkDoc{
{Text: "root", Extra: mustExtras(t, map[string]any{
"id": "r1", "doc_id": "d1", "tenant_id": "t1", "compile_kwd": "tree",
"kc_kind": "root", "kc_level": float64(-1), "q_3_vec": vector,
})},
{Text: "leaf", Extra: mustExtras(t, map[string]any{
"id": "a1", "doc_id": "d1", "tenant_id": "t1", "compile_kwd": "tree",
"kc_kind": "summary", "kc_level": float64(0), "parent_kwd": "r1", "q_3_vec": vector,
"source_chunk_ids": []string{"chunk-01", "chunk-02", "leaked-unknown-id"},
})},
}
m := AnalyzeTreeProducts(chunks, "chunk-01", "chunk-02")
if m.CoveredSources != 2 {
t.Errorf("CoveredSources = %d, want 2 (unknown id excluded)", m.CoveredSources)
}
if cov := m.CoverageFraction(2); cov != 1.0 {
t.Errorf("CoverageFraction = %v, want 1.0 (must not exceed 1.0)", cov)
}
}

65
internal/ingestion/component/knowledge_compiler/golden/metrics.go
View File

@@ -7,7 +7,7 @@ import (
"ragflow/internal/ingestion/component/schema"
)
// TreeMetrics summarizes the structural shape of a raptor product tree.
// TreeMetrics summarizes the structural shape of a tree product tree.
type TreeMetrics struct {
ProductCount int
RootCount int
@@ -16,19 +16,38 @@ type TreeMetrics struct {
AllParented bool
VectorOK bool // every product carries a non-empty vector
SchemaOK bool // every product carries the schema fields
// CoveredSources is the number of distinct source chunk IDs referenced by
// level-0 leaf clusters via their source_chunk_ids meta. It measures how
// completely the input corpus is represented by the tree, independent of
// the tree's structural well-formedness.
CoveredSources int
// covered is the working set of distinct source chunk IDs seen so far.
covered map[string]bool
}
// AnalyzeRaptorProducts validates tree integrity and computes structural
// metrics from a flat chunk list (the compiled raptor output, expressed as
// AnalyzeTreeProducts validates tree integrity and computes structural
// metrics from a flat chunk list (the compiled tree output, expressed as
// schema.ChunkDoc values). Used by the 缺口 C golden gate.
func AnalyzeRaptorProducts(chunks []schema.ChunkDoc) TreeMetrics {
//
// validSourceIDs, when provided, limits coverage counting to source chunk IDs
// that actually belong to the input corpus. This prevents an untrusted
// source_chunk_ids (e.g. a leaked/garbage ID) from inflating CoveredSources
// past nChunks and pushing CoverageFraction above 1.0. When empty, all
// source_chunk_ids are counted (backward compatible for unit tests that build
// synthetic trees).
func AnalyzeTreeProducts(chunks []schema.ChunkDoc, validSourceIDs ...string) TreeMetrics {
ids := make(map[string]bool, len(chunks))
for _, c := range chunks {
if id, ok := c.GetExtraString("id"); ok {
ids[id] = true
}
}
m := TreeMetrics{ProductCount: len(chunks), AllParented: true, VectorOK: true, SchemaOK: true}
validSet := make(map[string]bool, len(validSourceIDs))
for _, id := range validSourceIDs {
validSet[id] = true
}
checkValid := len(validSet) > 0
m := TreeMetrics{ProductCount: len(chunks), AllParented: true, VectorOK: true, SchemaOK: true, covered: make(map[string]bool)}
maxLevel := -1
for _, c := range chunks {
kind, _ := c.GetExtraString("kc_kind")
@@ -42,6 +61,22 @@ func AnalyzeRaptorProducts(chunks []schema.ChunkDoc) TreeMetrics {
case "summary":
if level == 0 {
m.LeafClusters++
// Accumulate the distinct source chunk IDs this leaf cluster
// was built from. Every input chunk is assigned to exactly one
// level-0 cluster in buildTree, so the union of these sets is
// the set of covered source chunks. Only IDs that belong to the
// input corpus count, so an unknown ID cannot inflate coverage.
if src, ok := c.GetExtraStringSlice("source_chunk_ids"); ok {
for _, id := range src {
if checkValid && !validSet[id] {
continue
}
if !m.covered[id] {
m.covered[id] = true
m.CoveredSources++
}
}
}
}
if level > maxLevel {
maxLevel = level
@@ -70,20 +105,24 @@ func AnalyzeRaptorProducts(chunks []schema.ChunkDoc) TreeMetrics {
}
// CoverageFraction reports how completely the input chunks are represented by
// the raptor tree. Every source chunk is assigned to exactly one level-0
// cluster in buildTree, and each such cluster becomes a leaf summary node, so a
// well-formed tree covers 100% of chunks. nChunks is the input chunk count.
// the tree. It is the ratio of distinct source chunk IDs referenced by the
// level-0 leaf clusters (CoveredSources) to the total input chunk count
// (nChunks). This detects dropped source chunks: a structurally well-formed
// tree that silently omits input chunks will score below 1.0.
func (m TreeMetrics) CoverageFraction(nChunks int) float64 {
if nChunks <= 0 {
return 0
}
// Every chunk maps to a level-0 cluster; the number of covered chunks
// equals the total chunk count when at least one leaf cluster exists and
// the tree is well-formed (all parented, single root).
if m.LeafClusters > 0 && m.RootCount == 1 && m.AllParented {
if m.CoveredSources <= 0 {
return 0
}
// Coverage can never exceed 1.0: a source chunk is covered at most once,
// and only corpus IDs are counted, so CoveredSources <= nChunks.
frac := float64(m.CoveredSources) / float64(nChunks)
if frac > 1.0 {
return 1.0
}
return 0.0
return frac
}
// extraFloat reads a numeric Extra value by key.

47
internal/ingestion/component/knowledge_compiler/golden_test.go
View File

@@ -74,7 +74,8 @@ func runVariantChunks(t *testing.T, variant string, extra map[string]any) []sche
// inputs (e.g. parser_config for the structure variant's template shape).
func runVariantChunksWithInputs(t *testing.T, variant string, extra, inputsExtra map[string]any) []schema.ChunkDoc {
t.Helper()
params := map[string]any{"variant": variant, "llm_id": "llm1", "embedding_model": "emb1"}
installVariantTemplateResolver(t, variant)
params := map[string]any{"compilation_template_id": "tpl-" + variant, "llm_id": "llm1", "embedding_model": "emb1"}
for k, v := range extra {
params[k] = v
}
@@ -224,15 +225,15 @@ func TestGolden_Wiki_ProductCount(t *testing.T) {
}
}
// TestGolden_Raptor_Structure is the 缺口 C gate for the raptor variant: on the
// fixed corpus, the watershed builder (raptor.go::watershed) must produce a
// TestGolden_Tree_Structure is the 缺口 C gate for the tree variant: on the
// fixed corpus, the watershed builder (tree/raptor.go::watershed) must produce a
// well-formed tree (single root, fully parented, vectors + schema intact, full
// chunk coverage, bounded depth/cluster count) across several tree_order
// settings. The structural contract is what a regression would violate; the
// exact cluster counts are locked in raptor_baseline.json.
func TestGolden_Raptor_Structure(t *testing.T) {
// exact cluster counts are locked in tree_baseline.json.
func TestGolden_Tree_Structure(t *testing.T) {
installSignedProseDeps(t)
baseline := loadBaseline(t, "raptor_baseline.json")
baseline := loadBaseline(t, "tree_baseline.json")
nChunks := len(golden.FixedCorpus())
cases := []struct {
@@ -248,36 +249,36 @@ func TestGolden_Raptor_Structure(t *testing.T) {
for _, tc := range cases {
var prods []schema.ChunkDoc
if tc.order <= 0 {
prods = runVariantChunks(t, "raptor", nil)
prods = runVariantChunks(t, "tree", nil)
} else {
prods = runVariantChunks(t, "raptor", map[string]any{
prods = runVariantChunks(t, "tree", map[string]any{
"extra": map[string]any{"tree_order": tc.order},
})
}
m := golden.AnalyzeRaptorProducts(prods)
t.Logf("raptor(%s): products=%d root=%d leafClusters=%d maxDepth=%d coverage=%.2f",
m := golden.AnalyzeTreeProducts(prods, goldenCorpusIDs()...)
t.Logf("tree(%s): products=%d root=%d leafClusters=%d maxDepth=%d coverage=%.2f",
tc.name, m.ProductCount, m.RootCount, m.LeafClusters, m.MaxDepth, m.CoverageFraction(nChunks))
if m.RootCount != 1 {
t.Errorf("raptor(%s): rootCount=%d, want 1", tc.name, m.RootCount)
t.Errorf("tree(%s): rootCount=%d, want 1", tc.name, m.RootCount)
}
if !m.AllParented {
t.Errorf("raptor(%s): tree has dangling parent_id references", tc.name)
t.Errorf("tree(%s): tree has dangling parent_id references", tc.name)
}
if !m.VectorOK {
t.Errorf("raptor(%s): some products missing vectors", tc.name)
t.Errorf("tree(%s): some products missing vectors", tc.name)
}
if !m.SchemaOK {
t.Errorf("raptor(%s): some products missing schema fields", tc.name)
t.Errorf("tree(%s): some products missing schema fields", tc.name)
}
if cov := m.CoverageFraction(nChunks); cov < numFloat(baseline, "leaf_coverage_min") {
t.Errorf("raptor(%s): leaf coverage %.2f < baseline %.2f", tc.name, cov, numFloat(baseline, "leaf_coverage_min"))
t.Errorf("tree(%s): leaf coverage %.2f < baseline %.2f", tc.name, cov, numFloat(baseline, "leaf_coverage_min"))
}
if m.LeafClusters < num(baseline, tc.minKey) || m.LeafClusters > num(baseline, tc.maxKey) {
t.Errorf("raptor(%s): leafClusters=%d outside [%d,%d]", tc.name, m.LeafClusters, num(baseline, tc.minKey), num(baseline, tc.maxKey))
t.Errorf("tree(%s): leafClusters=%d outside [%d,%d]", tc.name, m.LeafClusters, num(baseline, tc.minKey), num(baseline, tc.maxKey))
}
if m.MaxDepth > num(baseline, "max_depth_max") {
t.Errorf("raptor(%s): maxDepth=%d > baseline %d", tc.name, m.MaxDepth, num(baseline, "max_depth_max"))
t.Errorf("tree(%s): maxDepth=%d > baseline %d", tc.name, m.MaxDepth, num(baseline, "max_depth_max"))
}
}
}
@@ -291,3 +292,15 @@ func numFloat(m map[string]any, key string) float64 {
}
return 0
}
// goldenCorpusIDs returns the IDs of every chunk in the fixed golden corpus.
// Passed to AnalyzeTreeProducts so coverage only counts source chunk IDs that
// belong to the input corpus (a leaked/unknown ID must not inflate coverage).
func goldenCorpusIDs() []string {
corpus := golden.FixedCorpus()
ids := make([]string, 0, len(corpus))
for _, c := range corpus {
ids = append(ids, c.ID)
}
return ids
}

69
internal/ingestion/component/knowledge_compiler/mindmap/mindmap.go
View File

@@ -12,6 +12,7 @@ package mindmap
import (
"context"
"encoding/json"
"fmt"
"strings"
@@ -33,14 +34,13 @@ func Run(ctx context.Context, deps common.Deps, param common.Param, inputs commo
}
sections := chunkTexts(inputs.Chunks)
if len(sections) == 0 {
return common.Outputs{}, nil
}
// One LLM task per token-budget batch (mirrors __call__'s task fan-out).
batches := packSections(sections, deps.Tokenizer)
results := make([]utility.OMap, len(batches))
// One LLM task per token-budget batch (mirrors __call__'s task fan-out).
// Bounded concurrency. Python fans out with unbounded asyncio.gather plus a
// global chat_limiter semaphore; here we cap with a worker pool sized by
// MaxWorkers (defaulting to 1, i.e. serial).
n := param.MaxWorkers
if n <= 0 {
n = 1
@@ -119,6 +119,12 @@ func Run(ctx context.Context, deps common.Deps, param common.Param, inputs commo
// becomes a "root" product whose content is the serialized {"id","children"}
// tree (Python's MindMapResult.output shape); each inner node becomes a
// "node" product linked via parent_id.
//
// NOTE: the per-node "node" products (content = the node title, plus a
// meta["children"] list of immediate child titles) are a Go component-layer
// contract for streaming the tree into the upstream chunk list. They go
// beyond Python's nested dict output and exist so downstream code can index
// each node independently with parent links.
func treeToProducts(tenantID, docID string, root *utility.Node) []common.Product {
var out []common.Product
rootID := common.StableRowID(tenantID, docID, string(common.VariantMindmap), "root")
@@ -179,42 +185,31 @@ func treeToProducts(tenantID, docID string, root *utility.Node) []common.Product
return out
}
// serializeNode renders the tree in Python's MindMapResult.output shape:
// {"id": ..., "children": [...]}.
func serializeNode(node *utility.Node) string {
var b strings.Builder
b.WriteString(`{"id":`)
b.WriteString(quoteJSON(node.ID))
b.WriteString(`,"children":[`)
for i, c := range node.Children {
if i > 0 {
b.WriteString(",")
}
b.WriteString(serializeNode(c))
}
b.WriteString("]}")
return b.String()
// jsonNode mirrors the {"id","children"} mind-map node shape (Python's
// MindMapResult.output) for JSON serialization.
type jsonNode struct {
ID string `json:"id"`
Children []*jsonNode `json:"children"`
}
func quoteJSON(s string) string {
var b strings.Builder
b.WriteByte('"')
for _, r := range s {
switch r {
case '"':
b.WriteString("\\\"")
case '\\':
b.WriteString("\\\\")
case '\n':
b.WriteString("\\n")
case '\t':
b.WriteString("\\t")
default:
b.WriteRune(r)
}
func toJSONNode(n *utility.Node) *jsonNode {
jn := &jsonNode{ID: n.ID}
for _, c := range n.Children {
jn.Children = append(jn.Children, toJSONNode(c))
}
b.WriteByte('"')
return b.String()
return jn
}
// serializeNode renders the tree in Python's MindMapResult.output shape:
// {"id": ..., "children": [...]}. Uses encoding/json so every control
// character (including \r, \b, \f, etc.) is escaped correctly; the previous
// hand-rolled serializer only escaped ", \, \n and \t.
func serializeNode(node *utility.Node) string {
b, err := json.Marshal(toJSONNode(node))
if err != nil {
return `{"id":"` + node.ID + `","children":[]}`
}
return string(b)
}
func chunkTexts(chunks []common.Chunk) []string {

24
internal/ingestion/component/knowledge_compiler/mindmap/mindmap_test.go
View File

@@ -1,6 +1,7 @@
package mindmap
import (
"encoding/json"
"strings"
"testing"
@@ -78,3 +79,26 @@ func TestSerializeNode_Shape(t *testing.T) {
t.Errorf("serializeNode = %q", js)
}
}
// TestSerializeNode_EscapesControlChars locks that serializeNode uses
// encoding/json: control characters such as CR/LF/TAB/FF are escaped (not
// embedded raw), so the output is always valid JSON.
func TestSerializeNode_EscapesControlChars(t *testing.T) {
root := &utility.Node{ID: "a\rb\tc\fd", Children: []*utility.Node{{ID: "x\ny"}}}
js := serializeNode(root)
var dec struct {
ID string `json:"id"`
Children []struct {
ID string `json:"id"`
} `json:"children"`
}
if err := json.Unmarshal([]byte(js), &dec); err != nil {
t.Fatalf("serializeNode produced invalid JSON: %v (%q)", err, js)
}
if dec.ID != "a\rb\tc\fd" {
t.Errorf("root id round-trip = %q, want %q", dec.ID, "a\rb\tc\fd")
}
if len(dec.Children) != 1 || dec.Children[0].ID != "x\ny" {
t.Errorf("child id round-trip = %+v", dec.Children)
}
}

2
internal/ingestion/component/knowledge_compiler/structure/structure.go
View File

@@ -46,7 +46,7 @@ func Run(ctx context.Context, deps common.Deps, param common.Param, inputs commo
Variant: common.VariantStructure,
Lang: param.Language,
ParserConfig: parserConfig,
TemplateID: common.FirstNonEmpty(param.TemplateIDs...),
TemplateID: param.TemplateID,
}
nodePrompt, edgePromptTmpl := HypergraphPrompts(parserConfig, param.Language)

46
internal/ingestion/component/knowledge_compiler/raptor/raptor.go → internal/ingestion/component/knowledge_compiler/tree/raptor.go
View File

@@ -1,4 +1,4 @@
// Package raptor implements the "raptor" variant of KnowledgeCompiler: a
// Package tree implements the "tree" variant of KnowledgeCompiler: a
// recursive abstractive summarization tree (RAPTOR). It builds a clustering of
// chunk embeddings and summarizes each cluster with the LLM, recursing upward
// until a single root summary remains.
@@ -11,7 +11,7 @@
// defaults to DefaultTreeOrder.
//
// See PORT_PLAN.md §3.3.
package raptor
package tree
import (
"context"
@@ -25,13 +25,13 @@ import (
"ragflow/internal/tokenizer"
)
// Run executes the raptor variant.
// Run executes the tree variant.
func Run(ctx context.Context, deps common.Deps, param common.Param, inputs common.Inputs) (common.Outputs, error) {
if deps.Embed == nil {
return common.Outputs{}, fmt.Errorf("raptor: embedder required")
return common.Outputs{}, fmt.Errorf("tree: embedder required")
}
if deps.Chat == nil {
return common.Outputs{}, fmt.Errorf("raptor: chat model required")
return common.Outputs{}, fmt.Errorf("tree: chat model required")
}
docID := firstNonEmpty(inputs.DocID, deps.DatasetID)
if docID == "" {
@@ -132,7 +132,7 @@ func resolveMaxErrors(param common.Param) int {
// buildTree summarizes each cluster (level 0) and recurses upward: each parent
// node is the LLM summary of its child cluster's texts. The root is a single
// "raptor" product with a stable id. Every node is appended to products (a
// "tree" product with a stable id. Every node is appended to products (a
// caller-owned slice), and only the deepest-level summaries are retained for the
// root synthesis. buildTree reads the source texts, ids, and embeddings directly
// from the passed chunks; when the embeddings are missing or incomplete, it
@@ -171,7 +171,7 @@ func buildTree(ctx context.Context, deps common.Deps, llmID, tenantID, docID str
groups[labels[i]] = append(groups[labels[i]], i)
}
rootID := common.StableRowID(tenantID, docID, string(common.VariantRaptor), "root")
rootID := common.StableRowID(tenantID, docID, string(common.VariantTree), "root")
var (
maxLevel = -1
topLevelTexts []string
@@ -201,12 +201,12 @@ func buildTree(ctx context.Context, deps common.Deps, llmID, tenantID, docID str
if err != nil {
errorCount++
if errorCount >= maxErrors {
return fmt.Errorf("raptor: aborted after %d summarization errors: %w", errorCount, err)
return fmt.Errorf("tree: aborted after %d summarization errors: %w", errorCount, err)
}
log.Printf("raptor: skipping cluster due to summarization error (continuing): %v", err)
log.Printf("tree: skipping cluster due to summarization error (continuing): %v", err)
continue
}
nodeID := common.StableRowID(tenantID, docID, string(common.VariantRaptor),
nodeID := common.StableRowID(tenantID, docID, string(common.VariantTree),
fmt.Sprintf("L%d", task.level), summary)
embedding, err := deps.Embed.Encode(ctx, []string{summary})
if err != nil {
@@ -220,7 +220,7 @@ func buildTree(ctx context.Context, deps common.Deps, llmID, tenantID, docID str
ID: nodeID,
DocID: docID,
TenantID: tenantID,
Variant: common.VariantRaptor,
Variant: common.VariantTree,
Content: summary,
Vector: vec,
ParentID: task.parentID,
@@ -289,7 +289,7 @@ func buildTree(ctx context.Context, deps common.Deps, llmID, tenantID, docID str
// No summaries survived (e.g. every deepest cluster failed while the
// error budget was not yet exhausted). Return the partial tree without a
// root node — Python drops the root in this case rather than crashing.
log.Printf("raptor: no top-level summaries produced, skipping root node")
log.Printf("tree: no top-level summaries produced, skipping root node")
return nil
}
rootContent := buildClusterContent(topLevelTexts, allIndices(len(topLevelTexts)), deps.LLMMaxLength, maxToken)
@@ -299,23 +299,23 @@ func buildTree(ctx context.Context, deps common.Deps, llmID, tenantID, docID str
if err != nil {
// A failed root must not abort the whole tree: Python drops the root
// node and returns the rest of the tree.
log.Printf("raptor: root synthesis failed, skipping root node: %v", err)
log.Printf("tree: root synthesis failed, skipping root node: %v", err)
return nil
}
embedding, err := deps.Embed.Encode(ctx, []string{rootSummary})
if err != nil {
log.Printf("raptor: root embedding failed, skipping root node: %v", err)
log.Printf("tree: root embedding failed, skipping root node: %v", err)
return nil
}
if len(embedding) == 0 {
log.Printf("raptor: root embedding returned no vectors, skipping root node")
log.Printf("tree: root embedding returned no vectors, skipping root node")
return nil
}
*products = append(*products, common.Product{
ID: rootID,
DocID: docID,
TenantID: tenantID,
Variant: common.VariantRaptor,
Variant: common.VariantTree,
Content: rootSummary,
Vector: embedding[0],
ParentID: "",
@@ -540,12 +540,14 @@ func toFloat64Matrix(vecs [][]float32) [][]float64 {
return out
}
// defaultRaptorPrompt is the summary task template. It mirrors Python
// rag/advanced_rag/knowlege_compile/raptor.py, whose _summarize_texts uses
// "Please write a concise summary of the following texts:\n{cluster_content}".
// The {cluster_content} placeholder is filled with the joined cluster text. A
// caller may override it via extra["prompt"] (Python: raptor_cfg["prompt"]).
const defaultRaptorPrompt = "Please write a concise summary of the following texts:\n{cluster_content}"
// defaultRaptorPrompt is the summary task template. It mirrors the production
// prompt from api/db/init_data/compilation_templates/tree.yaml (the tree
// compilation template), NOT the Python fallback in compiler.py:128. The
// {cluster_content} placeholder is filled with the joined cluster text. The
// YAML literal block carries a base indent of 6 spaces before {cluster_content};
// those 6 spaces are part of the prompt and MUST be preserved. A caller may
// override it via extra["prompt"] (Python: raptor_cfg["prompt"]).
const defaultRaptorPrompt = "Please summarize the following paragraphs. Be careful with the numbers, do not make things up. Paragraphs as following:\n {cluster_content}\nThe above is the content you need to summarize."
// raptorSystemHelper mirrors the leading "You're a helpful assistant.\n\nHelp me
// with the following task.\n\n" wrapper Python prepends to the task prompt.

19
internal/ingestion/component/knowledge_compiler/raptor/raptor_test.go → internal/ingestion/component/knowledge_compiler/tree/raptor_test.go
View File

@@ -1,4 +1,4 @@
package raptor
package tree
import (
"context"
@@ -146,3 +146,20 @@ func TestBuildTreeNoPanicWhenAllSummariesFail(t *testing.T) {
t.Fatalf("expected no products when every summary fails, got %d", len(products))
}
}
// TestDefaultRaptorPromptMatchesTreeYAML locks the default summary prompt to the
// production tree.yaml template. It must equal the Python tree compilation
// template prompt (api/db/init_data/compilation_templates/tree.yaml), NOT the
// compiler.py:128 fallback. Critically, the YAML literal block carries a base
// indent of 6 spaces before {cluster_content}; those 6 spaces are part of the
// prompt and must be preserved (Python does self._prompt.format(...), splicing
// the cluster text after the 6-space indent).
func TestDefaultRaptorPromptMatchesTreeYAML(t *testing.T) {
want := "Please summarize the following paragraphs. Be careful with the numbers, do not make things up. Paragraphs as following:\n {cluster_content}\nThe above is the content you need to summarize."
if defaultRaptorPrompt != want {
t.Fatalf("defaultRaptorPrompt drifted from tree.yaml:\n got: %q\nwant: %q", defaultRaptorPrompt, want)
}
if !strings.Contains(defaultRaptorPrompt, "\n {cluster_content}") {
t.Errorf("defaultRaptorPrompt missing the 6-space indent before {cluster_content}: %q", defaultRaptorPrompt)
}
}

4
internal/ingestion/component/knowledge_compiler/raptor/watershed.go → internal/ingestion/component/knowledge_compiler/tree/watershed.go
View File

@@ -6,7 +6,7 @@
// agglomerative hierarchical clustering.
//
// See PORT_PLAN.md §3.3 and the M6 validation gate (缺口 C).
package raptor
package tree
import (
"errors"
@@ -46,7 +46,7 @@ func watershed(embeddings [][]float64, treeOrder int) ([]int, error) {
n := len(embeddings)
switch {
case n == 0:
return nil, errors.New("raptor: watershed requires at least one embedding")
return nil, errors.New("tree: watershed requires at least one embedding")
case n == 1:
return []int{0}, nil
case n == 2:

2
internal/ingestion/component/knowledge_compiler/raptor/watershed_test.go → internal/ingestion/component/knowledge_compiler/tree/watershed_test.go
View File

@@ -1,4 +1,4 @@
package raptor
package tree
import (
"math"

316
internal/ingestion/pipeline/pipeline_knowledge_compiler_dsl_test.go Normal file
View File

@@ -0,0 +1,316 @@
// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package pipeline
import (
"context"
"fmt"
"os"
"path/filepath"
"testing"
// Named import so this file can drive the runtime factory directly.
"ragflow/internal/agent/runtime"
// Blank-imports fire the runtime.MustRegister init hooks that populate
// DefaultRegistry with File/Parser/TokenChunker/KnowledgeCompiler.
_ "ragflow/internal/ingestion/component"
_ "ragflow/internal/ingestion/component/chunker"
_ "ragflow/internal/ingestion/component/knowledge_compiler"
kc "ragflow/internal/ingestion/component/knowledge_compiler/common"
)
// installStubResolvers wires in-memory GroupResolver / TemplateResolver stubs so
// the DSL tests can exercise the template-resolution path without MySQL.
func installStubResolvers(t *testing.T, groupToTemplates map[string][]string, templates map[string]kc.TemplateInfo) {
t.Helper()
kc.SetGroupResolver(func(ctx context.Context, tenantID string, groupIDs []string) ([]string, error) {
var out []string
for _, g := range groupIDs {
out = append(out, groupToTemplates[g]...)
}
return out, nil
})
kc.SetTemplateResolver(func(ctx context.Context, tenantID, templateID string) (kc.TemplateInfo, error) {
info, ok := templates[templateID]
if !ok {
return kc.TemplateInfo{}, fmt.Errorf("no stub template %q", templateID)
}
return info, nil
})
t.Cleanup(func() { kc.SetGroupResolver(nil); kc.SetTemplateResolver(nil) })
}
// TestKnowledgeCompilerDSL_FixtureDecodesAndBindsParams loads the
// frontend-authored pipeline DSL (agent/templates/compiler.json) and verifies
// it decodes through NewPipelineFromDSL and that the Compiler node's authored
// parameters survive intact. The frontend emits the operator as "Compiler" (its
// canvas label) and does NOT set a "variant" — the variant is derived at runtime
// from the compilation_template's kind. The single fixed operator carries a
// singular compilation_template_group_id (see the frontend
// accordion-operators.tsx restriction and the pipeline.tsx default).
func TestKnowledgeCompilerDSL_FixtureDecodesAndBindsParams(t *testing.T) {
raw, err := os.ReadFile(filepath.Join(repoRootFromPipelineTest(t), "agent", "templates", "compiler.json"))
if err != nil {
t.Fatalf("read fixture: %v", err)
}
if _, err := NewPipelineFromDSL(raw, "kc-fixture"); err != nil {
t.Fatalf("NewPipelineFromDSL: %v", err)
}
schemas, err := ExtractAllComponentParams(raw)
if err != nil {
t.Fatalf("ExtractAllComponentParams: %v", err)
}
var compiler *ComponentParamsSchema
for i := range schemas {
if schemas[i].ComponentName == "Compiler" {
compiler = &schemas[i]
break
}
}
if compiler == nil {
t.Fatal("fixture has no component with component_name \"Compiler\"")
}
params := compiler.ParamsDefaults
// The frontend Compiler DSL omits "variant"; the Go component derives it
// from the template kind, so absence is expected.
if _, ok := params["variant"]; ok {
t.Errorf("fixture unexpectedly sets variant; frontend Compiler DSL omits it")
}
// Authored as a single string group id in the frontend form.
gid, ok := params["compilation_template_group_id"].(string)
if !ok || gid != "c3aa748c8b2111f191f3047c16ec874f" {
t.Fatalf("compilation_template_group_id = %v, want single string id", params["compilation_template_group_id"])
}
}
// TestKnowledgeCompilerDSL_FrontendDSLDecodesAndConstructs documents the
// contract between the frontend-authored Compiler DSL and the Go component: as
// emitted, the DSL has no "variant" but carries compilation_template_group_id,
// so ParseParam (and thus the runtime factory) succeeds without any server-side
// variant injection. What the component does need at Invoke time is the
// TemplateResolver seam — without it, resolving the template fails loudly
// rather than compiling with no template config.
func TestKnowledgeCompilerDSL_FrontendDSLDecodesAndConstructs(t *testing.T) {
raw, err := os.ReadFile(filepath.Join(repoRootFromPipelineTest(t), "agent", "templates", "compiler.json"))
if err != nil {
t.Fatalf("read fixture: %v", err)
}
schemas, err := ExtractAllComponentParams(raw)
if err != nil {
t.Fatalf("ExtractAllComponentParams: %v", err)
}
var compiler *ComponentParamsSchema
for i := range schemas {
if schemas[i].ComponentName == "Compiler" {
compiler = &schemas[i]
break
}
}
if compiler == nil {
t.Fatal("fixture has no component with component_name \"Compiler\"")
}
params := compiler.ParamsDefaults
runtime.InstallDefaultRegistryFactory()
f := runtime.DefaultFactory()
if f == nil {
t.Fatal("default runtime factory not installed")
}
// The fixture params (single group id, no variant) construct fine.
comp, err := f("KnowledgeCompiler", params)
if err != nil {
t.Fatalf("construct from fixture params: %v", err)
}
if comp == nil {
t.Fatal("construct from fixture params: nil component")
}
// Without an installed TemplateResolver, the resolution seam fails loud.
kc.SetTemplateResolver(nil)
if _, err := kc.ResolveTemplate(context.Background(), "tenant", "t1"); err == nil {
t.Fatal("ResolveTemplate without resolver: expected error")
}
}
// TestKnowledgeCompilerDSL_RegisteredAndConstructible confirms the Go runtime
// registers the component under the canonical name "KnowledgeCompiler" and that
// the runtime factory can build a component instance from a DSL params map that
// carries either compilation_template_id or compilation_template_group_id (the
// variant is no longer part of the DSL surface). The frontend label "Compiler"
// (see the fixture tests) maps to this runtime name at the API/canvas layer,
// not inside the pipeline DSL decoder.
func TestKnowledgeCompilerDSL_RegisteredAndConstructible(t *testing.T) {
runtime.InstallDefaultRegistryFactory()
if _, _, _, ok := runtime.DefaultRegistry.Lookup("KnowledgeCompiler"); !ok {
t.Fatal("KnowledgeCompiler not registered in the runtime factory")
}
f := runtime.DefaultFactory()
if f == nil {
t.Fatal("default runtime factory not installed")
}
cases := []map[string]any{
{"compilation_template_id": "t1"},
{"compilation_template_group_id": "g1"},
// Both present: compilation_template_id wins (priority id > group_id).
{"compilation_template_id": "t1", "compilation_template_group_id": "g1"},
}
for i, params := range cases {
comp, err := f("KnowledgeCompiler", params)
if err != nil {
t.Fatalf("case %d construct: %v", i, err)
}
if comp == nil {
t.Fatalf("case %d: nil component", i)
}
}
// Param map with neither id resolves to a parse error.
if _, err := f("KnowledgeCompiler", map[string]any{}); err == nil {
t.Fatal("construct with no template spec: expected error")
}
}
// TestKnowledgeCompilerDSL_ParamBinding exercises the DSL params map ->
// common.Param translation directly, including the id > group_id priority, the
// scalar/extra/default parsing that the pipeline wires from the canvas node
// params, and that the variant is intentionally NOT part of the DSL surface.
func TestKnowledgeCompilerDSL_ParamBinding(t *testing.T) {
// compilation_template_id wins over compilation_template_group_id.
p, err := kc.ParseParam(map[string]any{
"compilation_template_id": "t1",
"compilation_template_group_id": "g1",
"llm_id": "llm-1",
"embedding_model": "emb-1",
"language": "Chinese",
"extra": map[string]any{"prompt": "summarize"},
})
if err != nil {
t.Fatalf("ParseParam: %v", err)
}
if p.CompilationTemplateID != "t1" {
t.Errorf("CompilationTemplateID = %q, want t1", p.CompilationTemplateID)
}
if p.CompilationTemplateGroupID != "g1" {
t.Errorf("CompilationTemplateGroupID = %q, want g1", p.CompilationTemplateGroupID)
}
if p.Variant != "" {
t.Errorf("Variant should be empty after ParseParam (derived from kind later), got %q", p.Variant)
}
if p.LLMID != "llm-1" || p.EmbeddingModel != "emb-1" || p.Language != "Chinese" {
t.Errorf("scalar fields = %+v", p)
}
if p.Extra["prompt"] != "summarize" {
t.Errorf("Extra = %v, want prompt=summarize", p.Extra)
}
// Defaults must be applied for fields the DSL omits.
if p.MaxWorkers != 4 {
t.Errorf("MaxWorkers default = %d, want 4", p.MaxWorkers)
}
if p.SimilarityThreshold != 0.99 {
t.Errorf("SimilarityThreshold default = %v, want 0.99", p.SimilarityThreshold)
}
if p.EnableHistoricalDedup {
t.Errorf("EnableHistoricalDedup default = true, want false")
}
}
// TestKnowledgeCompilerDSL_KindToVariant locks the compilation_template.kind ->
// Go Variant mapping. tree->tree, mind_map->mindmap, wiki->wiki; the
// graph-family kinds (page_index / session_essence / session_graph / timeline /
// knowledge_graph) all collapse onto the structure (graph) variant; unknown
// kinds return ErrUnknownVariant.
func TestKnowledgeCompilerDSL_KindToVariant(t *testing.T) {
cases := []struct {
kind string
want kc.Variant
}{
{"tree", kc.VariantTree},
{"mind_map", kc.VariantMindmap},
{"wiki", kc.VariantWiki},
{"page_index", kc.VariantStructure},
{"session_essence", kc.VariantStructure},
{"session_graph", kc.VariantStructure},
{"timeline", kc.VariantStructure},
{"knowledge_graph", kc.VariantStructure},
}
for _, c := range cases {
got, err := kc.KindToVariant(c.kind)
if err != nil {
t.Fatalf("KindToVariant(%q): %v", c.kind, err)
}
if got != c.want {
t.Errorf("KindToVariant(%q) = %q, want %q", c.kind, got, c.want)
}
}
if _, err := kc.KindToVariant("not-a-real-kind"); err == nil {
t.Fatal("KindToVariant(unknown): expected ErrUnknownVariant")
}
}
// TestKnowledgeCompilerDSL_VariantValidation documents the parse-time validation
// boundary reachable without a running LLM/embedder: at least one of
// compilation_template_id / compilation_template_group_id is required. The
// variant is not parsed from the DSL at all (it is derived from the resolved
// template's kind at Invoke time).
func TestKnowledgeCompilerDSL_VariantValidation(t *testing.T) {
if _, err := kc.ParseParam(map[string]any{}); err == nil {
t.Fatal("ParseParam with no template spec: expected error")
}
if _, err := kc.ParseParam(map[string]any{"compilation_template_group_id": ""}); err == nil {
t.Fatal("ParseParam with empty group id: expected error")
}
if _, err := kc.ParseParam(map[string]any{"compilation_template_id": "t1"}); err != nil {
t.Fatalf("ParseParam with id only: unexpected error %v", err)
}
}
// TestKnowledgeCompilerDSL_ResolveTemplateSeam verifies the group->template
// resolution path uses the installed seams: compilation_template_group_id
// resolves to child template ids (GroupResolver) and each is loaded by
// TemplateResolver, which yields the kind that selects the variant.
func TestKnowledgeCompilerDSL_ResolveTemplateSeam(t *testing.T) {
installStubResolvers(t, map[string][]string{
"g1": {"t1", "t2"},
}, map[string]kc.TemplateInfo{
"t1": {ID: "t1", Kind: "mind_map", Config: map[string]any{"language": "English"}},
"t2": {ID: "t2", Kind: "tree", Config: map[string]any{}},
})
ids, err := kc.ResolveGroupTemplateIDs(context.Background(), "tenant", []string{"g1"})
if err != nil {
t.Fatalf("ResolveGroupTemplateIDs: %v", err)
}
if len(ids) != 2 || ids[0] != "t1" || ids[1] != "t2" {
t.Fatalf("group resolved to %v, want [t1 t2]", ids)
}
info, err := kc.ResolveTemplate(context.Background(), "tenant", "t1")
if err != nil {
t.Fatalf("ResolveTemplate t1: %v", err)
}
if info.Kind != "mind_map" {
t.Errorf("template kind = %q, want mind_map", info.Kind)
}
if v, _ := kc.KindToVariant(info.Kind); v != kc.VariantMindmap {
t.Errorf("kind %q -> variant %q, want mindmap", info.Kind, v)
}
}

23
internal/ingestion/task/knowledge_compiler_wiring.go
View File

@@ -47,11 +47,12 @@ import (
func init() {
kc.SetDepsResolver(newKnowledgeCompilerDepsResolver())
kc.SetGroupResolver(newKnowledgeCompilerGroupResolver())
kc.SetTemplateResolver(newKnowledgeCompilerTemplateResolver())
}
// newKnowledgeCompilerGroupResolver builds the production GroupResolver backed by
// the compilation_template DAO. Without it, any config carrying
// compilation_template_group_ids would fail loud at runtime (the component
// compilation_template_group_id would fail loud at runtime (the component
// refuses to silently drop the compilation_template_ids stamp). It resolves each
// group id to its child template ids so group-based configs stamp the full set
// on every compiled unit.
@@ -62,6 +63,26 @@ func newKnowledgeCompilerGroupResolver() kc.GroupResolver {
}
}
// newKnowledgeCompilerTemplateResolver builds the production TemplateResolver
// backed by the compilation_template DAO. It loads a single template by id and
// returns its id, kind (which selects the Go variant via common.KindToVariant),
// and config (the template "content"). Without it, any config carrying
// compilation_template_id would fail loudly at runtime.
func newKnowledgeCompilerTemplateResolver() kc.TemplateResolver {
tmplDAO := dao.NewCompilationTemplateDAO()
return func(ctx context.Context, tenantID, templateID string) (kc.TemplateInfo, error) {
t, err := tmplDAO.GetTemplate(ctx, tenantID, templateID)
if err != nil {
return kc.TemplateInfo{}, err
}
return kc.TemplateInfo{
ID: t.ID,
Kind: t.Kind,
Config: map[string]any(t.Config),
}, nil
}
}
// newKnowledgeCompilerDepsResolver builds the production DepsResolver. Each call
// yields a fresh Deps whose ChatInvoker / Embedder are bound to the resolved
// tenant + model ids (captured in the closure), mirroring how the Tokenizer

23
rag/flow/compiler/compiler.py
View File

@@ -43,20 +43,29 @@ class CompilerParam(ProcessParamBase, LLMParam):
"""Parameters for the knowledge-Compiler flow component.
Same LLM-backed shape as the Extractor, but instead of a single inline
``knowledge_compilation`` config it drives compilation from one or more
saved **compilation-template groups** (``compilation_template_group_ids``).
Each group resolves to a set of templates, each of which carries its own
structure-compilation config (kind, fields, synthesis, ...).
``knowledge_compilation`` config it drives compilation from a saved
**compilation-template group** (``compilation_template_group_id``). The group
resolves to a set of templates, each of which carries its own
structure-compilation config (kind, fields, synthesis, ...). The legacy
plural ``compilation_template_group_ids`` key is still accepted as a
fallback so existing pipelines keep working.
"""
def __init__(self):
super().__init__()
self.compilation_template_group_id = ""
self.compilation_template_group_ids = []
def check(self):
self.check_empty(self.compilation_template_group_ids, "Compilation Template Groups")
if isinstance(self.compilation_template_group_ids, str):
self.compilation_template_group_ids = [self.compilation_template_group_ids]
# Prefer the singular group id emitted by the frontend; fall back to the
# legacy plural key for backward compatibility. check() normalises the
# chosen value into self.compilation_template_group_ids (a list) so the
# rest of the Compiler only ever reads that.
group = self.compilation_template_group_id or self.compilation_template_group_ids
self.check_empty(group, "Compilation Template Group")
if isinstance(group, str):
group = [group]
self.compilation_template_group_ids = group
class Compiler(ProcessBase, LLM):

2
web/src/pages/agent/canvas/node/compilation-node.tsx
View File

@@ -10,7 +10,7 @@ export function CompilationNode({ ...props }: NodeProps<IRagNode>) {
const { data } = props;
const { t } = useTranslation();
const options = useCompilationTemplateGroupOptions();
const groupId = get(data, 'form.compilation_template_group_ids');
const groupId = get(data, 'form.compilation_template_group_id');
const groupName =
options.find((option) => option.value === groupId)?.label ?? groupId;

2
web/src/pages/agent/constant/pipeline.tsx
View File

@@ -362,7 +362,7 @@ export const initialExtractorValues = {
};
export const initialCompilationValues = {
compilation_template_group_ids: [],
compilation_template_group_id: '',
outputs: {
chunks: { type: 'Array<Object>', value: [] },
},

4
web/src/pages/agent/form/compilation-form/index.tsx
View File

@@ -13,7 +13,7 @@ import { FormWrapper } from '../components/form-wrapper';
import { Output } from '../components/output';
export const FormSchema = z.object({
compilation_template_group_ids: z.string().optional(),
compilation_template_group_id: z.string().optional(),
});
export type CompilationFormSchemaType = z.infer<typeof FormSchema>;
@@ -33,7 +33,7 @@ const CompilationForm = ({ node }: INextOperatorForm) => {
return (
<Form {...form}>
<FormWrapper>
<CompilationTemplateFormField name="compilation_template_group_ids"></CompilationTemplateFormField>
<CompilationTemplateFormField name="compilation_template_group_id"></CompilationTemplateFormField>
<Output list={outputList}></Output>
</FormWrapper>
</Form>