diff --git a/cmd/ragflow_server.go b/cmd/ragflow_server.go index 75898b31f9..e7c6fab1db 100644 --- a/cmd/ragflow_server.go +++ b/cmd/ragflow_server.go @@ -569,6 +569,14 @@ func runIngestor(ctx context.Context, cancel context.CancelFunc, args *serverArg globalConfig.GetDefaultChatModel().Name, globalConfig.GetDefaultEmbeddingModel().Name, ) + // The dataset-level knowledge-compile consumer (tree/structure products) upserts + // into the dataset-nav tree, so the Ingestor must install the same ES-backed + // NavService the API server installs. Without this, nav.GetNavService() returns + // nil and tree/structure products are dropped (the consumer logs "nav service + // unavailable, skipping dataset-nav upsert"), leaving the dataset tree empty. + // The embedder resolves the tenant's embedding model on demand, so both + // Search and UpsertDoc can embed queries/summaries automatically. + nav.SetNavService(nlp.NewNavService(service.NewNavEmbedder(service.NewModelProviderService(), ""))) // Memory extraction runs on the Ingestor's shared NATS consumer + worker // pool (task_type="memory" dispatched by processMessage -> executeMemoryTask), // so there is no longer a dedicated Redis memory consumer to start. diff --git a/internal/dao/compilation_template_group.go b/internal/dao/compilation_template_group.go index 76826d6b9d..2075a292b6 100644 --- a/internal/dao/compilation_template_group.go +++ b/internal/dao/compilation_template_group.go @@ -59,6 +59,33 @@ func (dao *CompilationTemplateGroupDAO) ListSaved(ctx context.Context, db *gorm. return groups, nil } +// ListOwnedSaved returns only the tenant's own valid groups (built-in groups +// with empty tenant_id are excluded), mirroring the Python list_saved() query +// (cls.model.tenant_id == tenant_id). The merged /agents list uses this so +// built-in catalogue groups do not leak into a tenant's canvas list. +func (dao *CompilationTemplateGroupDAO) ListOwnedSaved(ctx context.Context, db *gorm.DB, tenantID, keywords, scope, orderby string, desc bool) ([]*entity.CompilationTemplateGroup, error) { + q := db.WithContext(ctx). + Where("tenant_id = ? AND status = ?", tenantID, string(entity.StatusValid)) + if keywords != "" { + q = q.Where("name LIKE ?", "%"+keywords+"%") + } + if scope != "" { + q = q.Where("scope = ?", scope) + } + if orderby != "name" && orderby != "scope" && orderby != "create_time" && orderby != "update_time" { + orderby = "create_time" + } + dir := "asc" + if desc { + dir = "desc" + } + var groups []*entity.CompilationTemplateGroup + if err := q.Order(orderby + " " + dir).Find(&groups).Error; err != nil { + return nil, err + } + return groups, nil +} + // CountSavedByTenant counts the tenant's own valid groups; built-in groups // (empty tenant_id) are excluded, mirroring the group_count query in Python // get_owner_filter / get_category_filter. diff --git a/internal/handler/agent_test.go b/internal/handler/agent_test.go index da88e819c4..1873868473 100644 --- a/internal/handler/agent_test.go +++ b/internal/handler/agent_test.go @@ -393,9 +393,12 @@ func setupAgentRouter(svc agentServiceIface) *gin.Engine { func TestListAgents_Success(t *testing.T) { title := "My Agent" + agentJSON, _ := json.Marshal(service.AgentItem{ + ID: "canvas-1", Title: &title, Permission: "me", CanvasCategory: "agent_canvas", + }) svc := &fakeAgentService{ result: &service.ListAgentsResponse{ - Canvas: []*service.AgentItem{{ID: "canvas-1", Title: &title, Permission: "me", CanvasCategory: "agent_canvas"}}, + Canvas: []json.RawMessage{agentJSON}, Total: 1, }, code: common.CodeSuccess, diff --git a/internal/handler/dataset_artifact.go b/internal/handler/dataset_artifact.go index dcddfddda7..3ac47a86f2 100644 --- a/internal/handler/dataset_artifact.go +++ b/internal/handler/dataset_artifact.go @@ -339,7 +339,9 @@ func (h *DatasetArtifactHandler) ListNavigation(c *gin.Context) { common.ErrorWithCode(c, common.CodeDataError, err.Error()) return } - common.SuccessWithData(c, gin.H{"total": total, "nav": items}, "success") + // Response key is "items" (not "nav") — the frontend DatasetNavList reads + // data.items and Python list_nav_clusters/_nav_search return {"total","items"}. + common.SuccessWithData(c, gin.H{"total": total, "items": items}, "success") } // DeleteNavigation handles DELETE /navigation — delete all navigation clusters. @@ -381,7 +383,9 @@ func (h *DatasetArtifactHandler) ListNavigationChildren(c *gin.Context) { common.ErrorWithCode(c, common.CodeDataError, err.Error()) return } - common.SuccessWithData(c, gin.H{"total": total, "children": items}, "success") + // Same contract as the top-level nav list: Python list_nav_children returns + // {"total","items"} and the frontend reads data.items for child expansion. + common.SuccessWithData(c, gin.H{"total": total, "items": items}, "success") } // GetSkillTree handles GET /skills — skill tree. @@ -453,13 +457,20 @@ func (h *DatasetArtifactHandler) GetDocumentGraph(c *gin.Context) { } datasetID := c.Param("dataset_id") documentID := c.Param("document_id") - graphType := c.Query("graph_type") - items, total, err := h.svc.GetDocumentGraph(c.Request.Context(), tenantID, datasetID, documentID, graphType) + resp, err := h.svc.GetDocumentGraph(c.Request.Context(), service.DocumentStructureGraphInput{ + TenantID: tenantID, + DatasetID: datasetID, + DocumentID: documentID, + Keywords: c.Query("keywords"), + }) if err != nil { common.ErrorWithCode(c, common.CodeDataError, err.Error()) return } - common.SuccessWithData(c, gin.H{"total": total, "graph": items}, "success") + if resp == nil { + resp = &service.DocumentStructureGraphResponse{Templates: []service.DocumentStructureGraphTemplate{}} + } + common.SuccessWithData(c, resp, "success") } // DeleteDocumentGraph handles DELETE /documents//structure/graph — delete document structure graph. diff --git a/internal/ingestion/component/knowledge_compiler/component.go b/internal/ingestion/component/knowledge_compiler/component.go index aa0b23357b..8b29cbebb1 100644 --- a/internal/ingestion/component/knowledge_compiler/component.go +++ b/internal/ingestion/component/knowledge_compiler/component.go @@ -449,44 +449,7 @@ func applyVariantColumns(doc *schema.ChunkDoc, p common.Product) error { switch p.Variant { case common.VariantStructure: // knowledge_graph_kwd: "entity" | "relation" | "graph". - if kind != "" { - if err := doc.SetExtraValue("knowledge_graph_kwd", kind); err != nil { - return err - } - } - // Relations carry from/to entity endpoints (from_entity_kwd / to_entity_kwd). - if kind == "relation" { - if v := metaString(p.Meta, "from"); v != "" { - if err := doc.SetExtraValue("from_entity_kwd", v); err != nil { - return err - } - } - if v := metaString(p.Meta, "to"); v != "" { - if err := doc.SetExtraValue("to_entity_kwd", v); err != nil { - return err - } - } - } - // Entities carry their canonical name on name_kwd (lowercased, mirroring - // Python's _struct_to_doc_storage_doc; the structure-graph endpoints - // filter/sort on it) plus entity_type_kwd and mention_count_int. - if kind == "entity" { - if v := metaString(p.Meta, "name"); v != "" { - if err := doc.SetExtraValue("name_kwd", strings.ToLower(v)); err != nil { - return err - } - } - if v := metaString(p.Meta, "entity_type"); v != "" { - if err := doc.SetExtraValue("entity_type_kwd", v); err != nil { - return err - } - } - } - if v, ok := metaInt(p.Meta, "mention_count"); ok { - if err := doc.SetExtraValue("mention_count_int", v); err != nil { - return err - } - } + return applyStructureGraphColumns(doc, p, kind) case common.VariantWiki: // One artifact_page row per wiki page; section rows reuse the same @@ -568,23 +531,35 @@ func applyVariantColumns(doc *schema.ChunkDoc, p common.Product) error { } case common.VariantTree: - // raptor_kwd tags summary/root nodes; raptor_layer_int records tree depth. - if kind != "" { - if err := doc.SetExtraValue("raptor_kwd", kind); err != nil { - return err + switch kind { + case "entity", "relation", "graph": + // The tree is also projected onto the structure-graph shape (Python + // raptor_tree_to_graph + _struct_upsert_tree_graph_rows): entity / + // relation rows carry knowledge_graph_kwd and the compact graph blob + // (kind "graph") is the /structure/graph discovery row. This is the + // same storage contract as the structure variant, so both share + // applyStructureGraphColumns. + return applyStructureGraphColumns(doc, p, kind) + default: + // RAPTOR summary/root rows: raptor_kwd tags the node kind; + // raptor_layer_int records tree depth. + if kind != "" { + if err := doc.SetExtraValue("raptor_kwd", kind); err != nil { + return err + } } - } - if v, ok := metaInt(p.Meta, "level"); ok { - if err := doc.SetExtraValue("raptor_layer_int", v); err != nil { - return err + if v, ok := metaInt(p.Meta, "level"); ok { + if err := doc.SetExtraValue("raptor_layer_int", v); err != nil { + return err + } + if err := doc.SetExtraValue("depth_int", v); err != nil { + return err + } } - if err := doc.SetExtraValue("depth_int", v); err != nil { - return err - } - } - if v := metaStringSlice(p.Meta, "children"); len(v) > 0 { - if err := doc.SetExtraValue("children_kwd", v); err != nil { - return err + if v := metaStringSlice(p.Meta, "children"); len(v) > 0 { + if err := doc.SetExtraValue("children_kwd", v); err != nil { + return err + } } } @@ -611,6 +586,53 @@ func applyVariantColumns(doc *schema.ChunkDoc, p common.Product) error { return nil } +// applyStructureGraphColumns emits the structure-graph row columns shared by the +// structure and tree variants (Python _struct_to_doc_storage_doc contract): +// - knowledge_graph_kwd: "entity" | "relation" | "graph" +// - relations: from_entity_kwd / to_entity_kwd +// - entities: name_kwd (lowercased) / entity_type_kwd +// - mention_count_int +// +// Keeping this in one helper prevents the two variants' storage contracts from +// diverging (review Major). +func applyStructureGraphColumns(doc *schema.ChunkDoc, p common.Product, kind string) error { + if kind != "" { + if err := doc.SetExtraValue("knowledge_graph_kwd", kind); err != nil { + return err + } + } + if kind == "relation" { + if v := metaString(p.Meta, "from"); v != "" { + if err := doc.SetExtraValue("from_entity_kwd", v); err != nil { + return err + } + } + if v := metaString(p.Meta, "to"); v != "" { + if err := doc.SetExtraValue("to_entity_kwd", v); err != nil { + return err + } + } + } + if kind == "entity" { + if v := metaString(p.Meta, "name"); v != "" { + if err := doc.SetExtraValue("name_kwd", strings.ToLower(v)); err != nil { + return err + } + } + if v := metaString(p.Meta, "entity_type"); v != "" { + if err := doc.SetExtraValue("entity_type_kwd", v); err != nil { + return err + } + } + } + if v, ok := metaInt(p.Meta, "mention_count"); ok { + if err := doc.SetExtraValue("mention_count_int", v); err != nil { + return err + } + } + return nil +} + // metaString reads a string-valued Product.Meta key. func metaString(m map[string]any, key string) string { v, _ := m[key].(string) diff --git a/internal/ingestion/component/knowledge_compiler/golden/metrics.go b/internal/ingestion/component/knowledge_compiler/golden/metrics.go index 753cfa8ff2..27224bb628 100644 --- a/internal/ingestion/component/knowledge_compiler/golden/metrics.go +++ b/internal/ingestion/component/knowledge_compiler/golden/metrics.go @@ -47,10 +47,18 @@ func AnalyzeTreeProducts(chunks []schema.ChunkDoc, validSourceIDs ...string) Tre validSet[id] = true } checkValid := len(validSet) > 0 - m := TreeMetrics{ProductCount: len(chunks), AllParented: true, VectorOK: true, SchemaOK: true, covered: make(map[string]bool)} + m := TreeMetrics{AllParented: true, VectorOK: true, SchemaOK: true, covered: make(map[string]bool)} maxLevel := -1 for _, c := range chunks { kind, _ := c.GetExtraString("kc_kind") + // Auxiliary tree-graph rows (kind entity/relation/graph, from the + // structure-graph projection) are not part of the RAPTOR tree structure, + // so they are excluded from the structural metrics (ProductCount, + // AllParented, SchemaOK, ...). The tree itself is the root/summary set. + if kind != "root" && kind != "summary" { + continue + } + m.ProductCount++ level := 0 if lf, ok := extraFloat(c, "kc_level"); ok { level = int(lf) diff --git a/internal/ingestion/component/knowledge_compiler/tree/graph.go b/internal/ingestion/component/knowledge_compiler/tree/graph.go new file mode 100644 index 0000000000..a6b3a678ac --- /dev/null +++ b/internal/ingestion/component/knowledge_compiler/tree/graph.go @@ -0,0 +1,356 @@ +package tree + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "sort" + "strings" + + "ragflow/internal/ingestion/component/knowledge_compiler/common" +) + +// stringsContains reports whether s contains substring sub. +func stringsContains(s, sub string) bool { + return strings.Contains(s, sub) +} + +// strSliceContains reports whether s contains v. +func strSliceContains(s []string, v string) bool { + for _, e := range s { + if e == v { + return true + } + } + return false +} + +// stringsJoinNonEmpty joins the non-empty parts with sep. +func stringsJoinNonEmpty(parts []string, sep string) string { + var kept []string + for _, p := range parts { + if p != "" { + kept = append(kept, p) + } + } + return strings.Join(kept, sep) +} + +// stringMetaSlice coerces a Product.Meta value ([]string or []any of strings) +// into a []string. +func stringMetaSlice(v any) []string { + switch x := v.(type) { + case []string: + return x + case []any: + out := make([]string, 0, len(x)) + for _, e := range x { + if s, ok := e.(string); ok && s != "" { + out = append(out, s) + } + } + return out + } + return nil +} + +// payloadChunkIDs extracts the source_chunk_ids from a tree-graph payload. +func payloadChunkIDs(payload map[string]any) []string { + switch v := payload["source_chunk_ids"].(type) { + case []string: + return v + case []any: + var out []string + for _, e := range v { + if s, ok := e.(string); ok && s != "" { + out = append(out, s) + } + } + return out + } + return nil +} + +// payloadDescription is the embedding input for a tree-graph entity/relation: +// the concatenated string values of every field except description (lists +// flattened), matching Python _struct_payload_description. +func payloadDescription(payload map[string]any) string { + keys := make([]string, 0, len(payload)) + for k := range payload { + if k != "description" { + keys = append(keys, k) + } + } + sort.Strings(keys) + var parts []string + for _, k := range keys { + switch v := payload[k].(type) { + case string: + if v != "" { + parts = append(parts, v) + } + case []string: + for _, e := range v { + if e != "" { + parts = append(parts, e) + } + } + case []any: + for _, e := range v { + if s, ok := e.(string); ok && s != "" { + parts = append(parts, s) + } + } + } + } + return strings.Join(parts, " ") +} + +// payloadJSON serialises a payload the way Python's json.dumps(ensure_ascii= +// False) does (no HTML escaping), with alphabetically sorted keys for a +// canonical, hash-stable form. +func payloadJSON(payload map[string]any) string { + var b bytes.Buffer + enc := json.NewEncoder(&b) + enc.SetEscapeHTML(false) + if err := enc.Encode(payload); err != nil { + return "{}" + } + return strings.TrimSpace(b.String()) +} + +// graphNode mirrors Python's RAPTOR tree node dict (title/description/children/ +// source_chunk_ids) reconstructed from the flat products emitted by buildTree, +// so the tree can be projected to a {entities, relations} graph exactly like +// Python's raptor_tree_to_graph (chunk_post_processor.py:470). +type graphNode struct { + title string + description string + sourceChunkIDs []string + children []*graphNode +} + +// collapseUnary merges a node that wraps exactly one child into that child, +// mirroring Python raptor_tree_to_graph._collapse_unary: the parent's and the +// child's descriptions/source-chunk-ids are concatenated (dedup'd), then the +// collapsed node adopts the child's children. +func collapseUnary(node *graphNode) *graphNode { + collapsed := &graphNode{ + title: node.title, + description: node.description, + sourceChunkIDs: node.sourceChunkIDs, + } + for _, c := range node.children { + collapsed.children = append(collapsed.children, collapseUnary(c)) + } + for len(collapsed.children) == 1 { + child := collapsed.children[0] + parentTitle := collapsed.title + childTitle := child.title + parentDesc := collapsed.description + if parentDesc == "" { + parentDesc = parentTitle + } + childDesc := child.description + if childDesc == "" { + childDesc = childTitle + } + + var descriptions []string + descriptions = append(descriptions, parentDesc) + if childTitle != "" && childTitle != parentTitle && + !stringsContains(childDesc, childTitle) { + descriptions = append(descriptions, childTitle) + } + if childDesc != "" && !strSliceContains(descriptions, childDesc) { + descriptions = append(descriptions, childDesc) + } + + sourceChunkIDs := append([]string{}, collapsed.sourceChunkIDs...) + for _, id := range child.sourceChunkIDs { + if id != "" && !strSliceContains(sourceChunkIDs, id) { + sourceChunkIDs = append(sourceChunkIDs, id) + } + } + + collapsed.description = stringsJoinNonEmpty(descriptions, "\n\n") + collapsed.sourceChunkIDs = sourceChunkIDs + collapsed.children = child.children + } + return collapsed +} + +// raptorTreeToGraph projects a RAPTOR tree onto {entities, relations}, matching +// Python raptor_tree_to_graph: every node becomes an entity of type "tree_node"; +// every parent→child edge (that is not a self-loop) becomes a "child" relation. +func raptorTreeToGraph(root *graphNode) ([]map[string]any, []map[string]any) { + var entities []map[string]any + var relations []map[string]any + var walk func(node *graphNode, parentTitle string) + walk = func(node *graphNode, parentTitle string) { + if node == nil { + return + } + title := node.title + ent := map[string]any{ + "name": title, + "type": "tree_node", + "description": firstNonEmpty(node.description, title), + "mention_count": 1, + } + if len(node.sourceChunkIDs) > 0 { + ent["source_chunk_ids"] = node.sourceChunkIDs + } + entities = append(entities, ent) + if parentTitle != "" && parentTitle != title { + relations = append(relations, map[string]any{ + "from": parentTitle, + "to": title, + "type": "child", + }) + } + for _, child := range node.children { + walk(child, title) + } + } + walk(root, "") + return entities, relations +} + +// buildTreeGraph reconstructs the tree from the flat summary products and +// produces the entity/relation/graph products Python writes for a tree variant +// (_struct_upsert_tree_graph_rows + _struct_upsert_graph_json): +// - one entity product per tree node (kind "entity", knowledge_graph_kwd via +// the writer), type "tree_node"; +// - one relation product per parent→child edge (kind "relation"); +// - one compact graph blob product (kind "graph") carrying the whole +// {entities, relations} projection, which is the /structure/graph discovery +// row (Python scans knowledge_graph_kwd="graph"). +// +// templateID is stamped into each row so the document-structure endpoint can +// group by template id; compileKWD is "tree". +func buildTreeGraph(ctx context.Context, deps common.Deps, docID string, products []common.Product) ([]common.Product, error) { + if deps.Embed == nil { + return nil, fmt.Errorf("tree: embedder required to build the tree graph") + } + root := reconstructTree(products) + if root == nil { + // No root summary survived; there is no tree to project. + return nil, nil + } + root = collapseUnary(root) + entities, relations := raptorTreeToGraph(root) + + var out []common.Product + var descs []string + var payloads []map[string]any + var kinds []string + for _, ent := range entities { + descs = append(descs, payloadDescription(ent)) + payloads = append(payloads, ent) + kinds = append(kinds, "entity") + } + for _, rel := range relations { + descs = append(descs, payloadDescription(rel)) + payloads = append(payloads, rel) + kinds = append(kinds, "relation") + } + vecs, err := deps.Embed.Encode(ctx, descs) + if err != nil { + return nil, err + } + for i, payload := range payloads { + kind := kinds[i] + var vec []float32 + if i < len(vecs) { + vec = vecs[i] + } + meta := map[string]any{ + "kind": kind, + "compile_kwd": "tree", + "source_chunk_ids": payloadChunkIDs(payload), + "mention_count": 1, + } + if kind == "entity" { + if name, ok := payload["name"].(string); ok && name != "" { + meta["name"] = name + } + if typ, ok := payload["type"].(string); ok && typ != "" { + meta["entity_type"] = typ + } else { + meta["entity_type"] = "other" + } + } else { + if from, ok := payload["from"].(string); ok { + meta["from"] = from + } + if to, ok := payload["to"].(string); ok { + meta["to"] = to + } + } + out = append(out, common.Product{ + ID: common.StableRowID(payloadJSON(payload), docID), + DocID: docID, + TenantID: deps.TenantID, + Variant: common.VariantTree, + Content: payloadJSON(payload), + Vector: vec, + Meta: meta, + }) + } + + // Compact graph blob discovery row (knowledge_graph_kwd="graph"). + graph := map[string]any{"entities": entities, "relations": relations} + graphContent := payloadJSON(graph) + graphVecs, err := deps.Embed.Encode(ctx, []string{graphContent}) + if err != nil { + return nil, err + } + var gv []float32 + if len(graphVecs) > 0 { + gv = graphVecs[0] + } + out = append(out, common.Product{ + ID: common.StableRowID(docID, "tree", "structure_graph"), + DocID: docID, + TenantID: deps.TenantID, + Variant: common.VariantTree, + Content: graphContent, + Vector: gv, + Meta: map[string]any{ + "kind": "graph", + "compile_kwd": "tree", + }, + }) + return out, nil +} + +// reconstructTree assembles a graphNode tree from the flat summary products: +// the root has Meta.kind=="root"; every other node's parent is the product with +// ID == node.ParentID. Node title comes from Meta.title, description from +// Content, source chunk ids from Meta.source_chunk_ids. +func reconstructTree(products []common.Product) *graphNode { + byID := make(map[string]*graphNode, len(products)) + for _, p := range products { + title, _ := p.Meta["title"].(string) + byID[p.ID] = &graphNode{ + title: title, + description: p.Content, + sourceChunkIDs: stringMetaSlice(p.Meta["source_chunk_ids"]), + } + } + var root *graphNode + for _, p := range products { + kind, _ := p.Meta["kind"].(string) + node := byID[p.ID] + if kind == "root" { + root = node + continue + } + if parent := byID[p.ParentID]; parent != nil { + parent.children = append(parent.children, node) + } + } + return root +} diff --git a/internal/ingestion/component/knowledge_compiler/tree/graph_test.go b/internal/ingestion/component/knowledge_compiler/tree/graph_test.go new file mode 100644 index 0000000000..5aa56797ac --- /dev/null +++ b/internal/ingestion/component/knowledge_compiler/tree/graph_test.go @@ -0,0 +1,107 @@ +package tree + +import ( + "reflect" + "testing" + + "ragflow/internal/ingestion/component/knowledge_compiler/common" +) + +// TestRaptorTreeToGraph_CollapsesUnaryAndProjects verifies the tree→graph +// projection matches Python raptor_tree_to_graph: unary chains are collapsed +// (descriptions concatenated), each node becomes an entity of type tree_node, +// and each parent→child edge (skipping self-loops) becomes a child relation. +func TestRaptorTreeToGraph_CollapsesUnaryAndProjects(t *testing.T) { + // Build a tree: root → [X, A → [B]]. A is a unary wrapper (only child B); the + // collapse merges A into B, while root (two children) and B (no children) + // stay. Entities: root, X, B. Relations: root→X, root→B. + root := &graphNode{ + title: "root", + description: "root desc", + children: []*graphNode{ + {title: "X", description: "X desc"}, + { + title: "A", + description: "A desc", + children: []*graphNode{ + {title: "B", description: "B desc", sourceChunkIDs: []string{"c1"}}, + }, + }, + }, + } + root = collapseUnary(root) + entities, relations := raptorTreeToGraph(root) + + // The unary node A survives but the child B is folded into it (it keeps A's + // title, concatenated descriptions and B's source chunk ids), matching + // Python _collapse_unary. Entities: root, X, A. + var names []string + for _, e := range entities { + names = append(names, e["name"].(string)) + } + if !reflect.DeepEqual(names, []string{"root", "X", "A"}) { + t.Fatalf("entities after unary collapse = %v, want [root X A]", names) + } + // A must have B folded in: concatenated descriptions + the source chunk id. + byName := map[string]map[string]any{} + for _, e := range entities { + byName[e["name"].(string)] = e + } + a := byName["A"] + if a["description"] != "A desc\n\nB desc" { + t.Errorf("collapsed A description = %q, want %q", a["description"], "A desc\n\nB desc") + } + if !reflect.DeepEqual(a["source_chunk_ids"], []string{"c1"}) { + t.Errorf("collapsed A source_chunk_ids = %v, want [c1]", a["source_chunk_ids"]) + } + if a["type"] != "tree_node" { + t.Errorf("entity type = %v, want tree_node", a["type"]) + } + // Two relations: root → X and root → A (A is root's child after collapse). + if len(relations) != 2 { + t.Fatalf("relations = %v, want exactly [root->X, root->A]", relations) + } + wantRels := []struct{ from, to string }{{"root", "X"}, {"root", "A"}} + gotRels := []struct{ from, to string }{ + {relations[0]["from"].(string), relations[0]["to"].(string)}, + {relations[1]["from"].(string), relations[1]["to"].(string)}, + } + if !reflect.DeepEqual(gotRels, wantRels) { + t.Errorf("relations = %v, want %v", gotRels, wantRels) + } +} + +// TestRaptorTreeToGraph_SkipsSelfLoop verifies a parent whose child has the same +// title does not produce a self-loop relation (Python's guard). +func TestRaptorTreeToGraph_SkipsSelfLoop(t *testing.T) { + root := &graphNode{ + title: "same", + children: []*graphNode{ + {title: "same", description: "child"}, + }, + } + _, relations := raptorTreeToGraph(root) + if len(relations) != 0 { + t.Fatalf("self-loop relation must be skipped, got %v", relations) + } +} + +// TestReconstructTree_FromFlatProducts verifies the flat summary products +// (root + ParentID chains) are reassembled into a nested tree. +func TestReconstructTree_FromFlatProducts(t *testing.T) { + products := []common.Product{ + {ID: "root", Meta: map[string]any{"kind": "root", "title": "root"}}, + {ID: "n1", ParentID: "root", Meta: map[string]any{"kind": "summary", "title": "N1"}, Content: "n1 desc"}, + {ID: "n2", ParentID: "n1", Meta: map[string]any{"kind": "summary", "title": "N2"}, Content: "n2 desc"}, + } + root := reconstructTree(products) + if root == nil || root.title != "root" { + t.Fatalf("reconstructed root = %+v, want title root", root) + } + if len(root.children) != 1 || root.children[0].title != "N1" { + t.Fatalf("root children = %+v, want [N1]", root.children) + } + if len(root.children[0].children) != 1 || root.children[0].children[0].title != "N2" { + t.Fatalf("N1 children = %+v, want [N2]", root.children[0].children) + } +} diff --git a/internal/ingestion/component/knowledge_compiler/tree/raptor.go b/internal/ingestion/component/knowledge_compiler/tree/raptor.go index 5813e2e460..486d2bdb2f 100644 --- a/internal/ingestion/component/knowledge_compiler/tree/raptor.go +++ b/internal/ingestion/component/knowledge_compiler/tree/raptor.go @@ -53,6 +53,18 @@ func Run(ctx context.Context, deps common.Deps, param common.Param, inputs commo return common.Outputs{}, err } + // Project the RAPTOR tree onto the {entities, relations} structure-graph + // shape (Python raptor_tree_to_graph) and persist it as entity/relation rows + // plus a compact graph blob (knowledge_graph_kwd="graph"), so the + // document-structure /structure/graph endpoint can serve the tree. A failure + // here must not abort the whole tree compile — the summary nodes are already + // valid on their own — so it is best-effort and surfaced as a log. + if graphProds, err := buildTreeGraph(ctx, deps, docID, products); err != nil { + log.Printf("tree: graph projection failed (best-effort, continuing): %v", err) + } else { + products = append(products, graphProds...) + } + out := common.Outputs{ Products: products, } diff --git a/internal/service/agent.go b/internal/service/agent.go index 1d9910ce5c..7e4dabe513 100644 --- a/internal/service/agent.go +++ b/internal/service/agent.go @@ -310,6 +310,7 @@ type AgentService struct { versionDAO *dao.UserCanvasVersionDAO api4ConversationDAO *dao.API4ConversationDAO compilationTemplateGroupDAO *dao.CompilationTemplateGroupDAO + compilationTemplateDAO *dao.CompilationTemplateDAO // driver is the per-process runner that drives canvas // invocations and produces SSE events. V1 persistence is @@ -383,6 +384,7 @@ func NewAgentServiceWithOptions( versionDAO: dao.NewUserCanvasVersionDAO(), api4ConversationDAO: dao.NewAPI4ConversationDAO(), compilationTemplateGroupDAO: dao.NewCompilationTemplateGroupDAO(), + compilationTemplateDAO: dao.NewCompilationTemplateDAO(), runner: canvas.NewRunner(), activeSessions: make(map[string]*activeAgentRun), checkpointStore: cp, @@ -415,14 +417,28 @@ type AgentItem struct { CreateTime *int64 `json:"create_time,omitempty"` UpdateTime *int64 `json:"update_time,omitempty"` ReleaseTime *int64 `json:"release_time,omitempty"` + // Type discriminates agent vs compilation-template-group items in the merged + // /agents response. It is set only for merged items (agent => "agent"). + Type string `json:"type,omitempty"` } -// ListAgentsResponse is the response body for GET /api/v1/agents. +// ListAgentsResponse is the response body for GET /api/v1/agents. Canvas holds +// pre-marshalled JSON items because a canvas entry is either an agent (AgentItem +// shape) or a compilation template group (group shape with a "type" of +// "compilation_template_group"); a single typed slice cannot express both. type ListAgentsResponse struct { - Canvas []*AgentItem `json:"canvas"` - Total int64 `json:"total"` + Canvas []json.RawMessage `json:"canvas"` + Total int64 `json:"total"` } +// AgentItemType discriminates the two kinds of canvas items in the merged +// /agents response, mirroring Python _COMPILATION_TEMPLATE_GROUP_CATEGORY and +// the frontend AgentListItem union. +const ( + AgentItemTypeAgent = "agent" + AgentItemTypeGroup = CompilationTemplateGroupCategory // "compilation_template_group" +) + // CompilationTemplateGroupCategory is the synthetic canvas_category the // frontend uses to filter compilation template groups through the merged // /agents endpoint. Mirrors Python _COMPILATION_TEMPLATE_GROUP_CATEGORY. @@ -592,17 +608,46 @@ func (s *AgentService) ListAgents(ctx context.Context, userID string, keywords s } } + // A canvas entry is either an agent (user_canvas) or a compilation template + // group. Python splits canvas_category on commas and, when the tenant is the + // sole effective owner, merges the caller's template groups into the list. + categories := splitCategoryList(canvasCategory) + wantsGroups := sliceContains(categories, CompilationTemplateGroupCategory) + agentCategories := filterCategory(categories, CompilationTemplateGroupCategory) + // Merge mode mirrors Python: no category, no canvas_type, no tags -> the + // caller's template groups are interleaved with agents by update_time. + mergeMode := len(categories) == 0 && canvasType == "" && len(tags) == 0 + + // Groups-only mode: canvas_category is exactly ["compilation_template_group"]. + // Template groups are always the caller's own, so they are only visible when + // the caller is an effective owner; otherwise (e.g. owner_ids names another + // user) return an empty list (review Major). + if len(categories) == 1 && wantsGroups { + if !sliceContains(effectiveOwnerIDs, userID) { + return &ListAgentsResponse{Canvas: []json.RawMessage{}, Total: 0}, common.CodeSuccess, nil + } + return s.listAgentsGroupsOnly(ctx, userID, keywords, orderBy, desc, page, pageSize) + } + + // Fetch agents. In merge/mixed modes we disable SQL pagination (page=0) and + // paginate in Go after interleaving with groups, matching Python. + listPage, listSize := page, pageSize + agentCategoryFilter := canvasCategory + if mergeMode || (wantsGroups && len(agentCategories) > 0) { + listPage, listSize = 0, 0 + agentCategoryFilter = strings.Join(agentCategories, ",") + } canvases, total, err := s.canvasDAO.ListByTenantIDs( ctx, dao.DB, effectiveOwnerIDs, userID, - page, - pageSize, + listPage, + listSize, orderBy, desc, keywords, - canvasCategory, + agentCategoryFilter, canvasType, tags, ) @@ -610,31 +655,252 @@ func (s *AgentService) ListAgents(ctx context.Context, userID string, keywords s return nil, common.CodeServerError, fmt.Errorf("failed to list agents: %w", err) } - items := make([]*AgentItem, len(canvases)) + agentItems := make([]*AgentItem, len(canvases)) for i, c := range canvases { - items[i] = toAgentItem(c) + agentItems[i] = toAgentItem(c) + } + s.attachReleaseTimes(ctx, agentItems) + + // Groups are owner-only (no team sharing) and scoped to the caller, so they + // are merged only when the caller's own tenant is an effective owner + // (Python include_template_groups). + includeGroups := sliceContains(effectiveOwnerIDs, userID) + if includeGroups && (mergeMode || wantsGroups) { + return s.mergeAgentsAndGroups(ctx, userID, agentItems, keywords, orderBy, desc, page, pageSize) } - // Attach the latest release time per canvas so agent cards can render - // the "published at" line (Python UserCanvasService.get_list parity). - if len(items) > 0 { - canvasIDs := make([]string, 0, len(items)) - for _, item := range items { - canvasIDs = append(canvasIDs, item.ID) - } - var releaseTimes map[string]int64 - releaseTimes, err = s.versionDAO.GetLatestReleaseTimes(ctx, dao.DB, canvasIDs) + raw := make([]json.RawMessage, len(agentItems)) + for i, item := range agentItems { + item.Type = AgentItemTypeAgent + raw[i] = marshalAgentItem(item) + } + return &ListAgentsResponse{Canvas: raw, Total: total}, common.CodeSuccess, nil +} + +// listAgentsGroupsOnly returns only the caller's compilation template groups +// (Python canvas_category == ["compilation_template_group"] branch). +func (s *AgentService) listAgentsGroupsOnly(ctx context.Context, userID, keywords, orderBy string, desc bool, page, pageSize int) (*ListAgentsResponse, common.ErrorCode, error) { + groups, err := s.compilationTemplateGroupDAO.ListOwnedSaved(ctx, dao.DB, userID, keywords, "", orderBy, desc) + if err != nil { + return nil, common.CodeServerError, fmt.Errorf("failed to list compilation template groups: %w", err) + } + total := int64(len(groups)) + groups = slicePage(groups, page, pageSize) + raw := make([]json.RawMessage, 0, len(groups)) + for _, g := range groups { + item, err := s.marshalMergeGroupItem(ctx, userID, g) if err != nil { - return nil, common.CodeServerError, fmt.Errorf("failed to get release times: %w", err) + return nil, common.CodeServerError, fmt.Errorf("failed to build group item: %w", err) } - for _, item := range items { - if t, ok := releaseTimes[item.ID]; ok { - item.ReleaseTime = &t - } + raw = append(raw, item) + } + return &ListAgentsResponse{Canvas: raw, Total: total}, common.CodeSuccess, nil +} + +// mergeAgentsAndGroups combines agents and the caller's compilation template +// groups into a single list ordered by (canvas_category, name) ascending, then +// pages in Go. desc is accepted for API signature parity but the ordering is +// intentionally category/name based, not chronological. +func (s *AgentService) mergeAgentsAndGroups(ctx context.Context, userID string, agentItems []*AgentItem, keywords, orderBy string, desc bool, page, pageSize int) (*ListAgentsResponse, common.ErrorCode, error) { + groups, err := s.compilationTemplateGroupDAO.ListOwnedSaved(ctx, dao.DB, userID, keywords, "", orderBy, desc) + if err != nil { + return nil, common.CodeServerError, fmt.Errorf("failed to list compilation template groups: %w", err) + } + merged := make([]mergeCanvasItem, 0, len(agentItems)+len(groups)) + for _, item := range agentItems { + item.Type = AgentItemTypeAgent + merged = append(merged, mergeCanvasItem{ + item: item, + category: item.CanvasCategory, + name: derefString(item.Title), + }) + } + for _, g := range groups { + raw, err := s.marshalMergeGroupItem(ctx, userID, g) + if err != nil { + return nil, common.CodeServerError, fmt.Errorf("failed to build group item: %w", err) + } + merged = append(merged, mergeCanvasItem{ + raw: raw, + category: CompilationTemplateGroupCategory, + name: g.Name, + }) + } + // Order the merged list by (category, name) ascending (A-Z): agents and + // compilation template groups are grouped by flow category, then sorted by + // display name within each category. The category sort key uses the raw + // canvas_category string so the natural order is agent_canvas < + // compilation_template_group < dataflow_canvas. + sort.SliceStable(merged, func(i, j int) bool { + if merged[i].category != merged[j].category { + return merged[i].category < merged[j].category + } + return strings.ToLower(merged[i].name) < strings.ToLower(merged[j].name) + }) + total := int64(len(merged)) + merged = slicePage(merged, page, pageSize) + raw := make([]json.RawMessage, 0, len(merged)) + for _, m := range merged { + if m.raw != nil { + raw = append(raw, m.raw) + } else if m.item != nil { + raw = append(raw, marshalAgentItem(m.item)) } } + return &ListAgentsResponse{Canvas: raw, Total: total}, common.CodeSuccess, nil +} - return &ListAgentsResponse{Canvas: items, Total: total}, common.CodeSuccess, nil +// marshalMergeGroupItem renders a compilation template group as a merged /agents +// canvas item: the group shape (ICompilationTemplateGroup) plus the "type" and +// "title" discriminators the frontend union expects (Python _group_to_dict). +func (s *AgentService) marshalMergeGroupItem(ctx context.Context, userID string, g *entity.CompilationTemplateGroup) (json.RawMessage, error) { + children, err := s.compilationTemplateDAO.ListByGroup(ctx, dao.DB, g.ID) + if err != nil { + return nil, err + } + templates := make([]json.RawMessage, 0, len(children)) + for _, c := range children { + templates = append(templates, marshalGroupTemplate(c)) + } + item := map[string]interface{}{ + "id": g.ID, + "name": g.Name, + "title": g.Name, + "description": derefString(g.Description), + "scope": g.Scope, + "create_time": intValuePtr(g.CreateTime), + "update_time": intValuePtr(g.UpdateTime), + "templates": templates, + "type": AgentItemTypeGroup, + } + b, err := json.Marshal(item) + if err != nil { + return nil, err + } + return b, nil +} + +// mergeCanvasItem is a decoded entry in the merged /agents list: exactly one of +// item (an agent) or raw (a marshalled group) is set. category is the flow +// classification (canvas_category, or "compilation_template_group" for groups) +// and name is the display title; the merged list is ordered by (category, name). +type mergeCanvasItem struct { + item *AgentItem + raw json.RawMessage + category string + name string +} + +// attachReleaseTimes populates ReleaseTime for each agent item from the latest +// published version (Python UserCanvasService.get_list parity). +func (s *AgentService) attachReleaseTimes(ctx context.Context, items []*AgentItem) { + if len(items) == 0 { + return + } + canvasIDs := make([]string, 0, len(items)) + for _, item := range items { + canvasIDs = append(canvasIDs, item.ID) + } + releaseTimes, err := s.versionDAO.GetLatestReleaseTimes(ctx, dao.DB, canvasIDs) + if err != nil { + return + } + for _, item := range items { + if t, ok := releaseTimes[item.ID]; ok { + item.ReleaseTime = &t + } + } +} + +// marshalAgentItem renders an agent item to its JSON representation. +func marshalAgentItem(item *AgentItem) json.RawMessage { + b, err := json.Marshal(item) + if err != nil { + return json.RawMessage("null") + } + return b +} + +// marshalGroupTemplate renders a compilation template child to the read-side +// shape the frontend group card expects (mirrors Python _to_saved_dict). +func marshalGroupTemplate(c *entity.CompilationTemplate) json.RawMessage { + item := map[string]interface{}{ + "id": c.ID, + "name": c.Name, + "description": derefString(c.Description), + "kind": c.Kind, + "config": c.Config, + "create_time": intValuePtr(c.CreateTime), + "update_time": intValuePtr(c.UpdateTime), + } + b, err := json.Marshal(item) + if err != nil { + return json.RawMessage("null") + } + return b +} + +// splitCategoryList splits a comma-separated canvas_category query into the +// non-empty categories, mirroring Python +// request.args.get("canvas_category", "").strip().split(","). +func splitCategoryList(s string) []string { + var out []string + for _, part := range strings.Split(s, ",") { + if p := strings.TrimSpace(part); p != "" { + out = append(out, p) + } + } + return out +} + +// filterCategory returns the categories in src that are not equal to drop. +func filterCategory(src []string, drop string) []string { + var out []string + for _, c := range src { + if c != drop { + out = append(out, c) + } + } + return out +} + +// sliceContains reports whether v is present in s. +func sliceContains[T comparable](s []T, v T) bool { + for _, e := range s { + if e == v { + return true + } + } + return false +} + +// slicePage returns a shallow copy of s bounded to the requested page window. +// A page <= 0 or pageSize <= 0 returns s unchanged (caller did not ask to page). +// The page bound is checked arithmetically before computing the offset so an +// unbounded positive page/page_size can never overflow to a negative start and +// panic (review Critical). +func slicePage[T any](s []T, page, pageSize int) []T { + if page <= 0 || pageSize <= 0 || len(s) == 0 { + return s + } + // page-1 must be <= (len(s)-1)/pageSize, i.e. start must be < len(s). + if page-1 > (len(s)-1)/pageSize { + return nil + } + start := (page - 1) * pageSize + if pageSize >= len(s)-start { + return s[start:] + } + return s[start : start+pageSize] +} + +// intValuePtr dereferences a *int64 to a plain int64 (0 when nil), used for the +// group/template create_time & update_time epoch fields in merged items. +func intValuePtr(p *int64) int64 { + if p == nil { + return 0 + } + return *p } // CreateAgentRequest is the input shape for CreateAgent. diff --git a/internal/service/agent_test.go b/internal/service/agent_test.go index aea52e6cc5..e5f2a53f05 100644 --- a/internal/service/agent_test.go +++ b/internal/service/agent_test.go @@ -834,6 +834,10 @@ func setupAgentSessionServiceTest(t *testing.T) { &entity.UserCanvasVersion{}, &entity.UserTenant{}, &entity.API4Conversation{}, + // The merged /agents list reads compilation template groups (and their + // children) alongside canvases, so those tables must exist too. + &entity.CompilationTemplateGroup{}, + &entity.CompilationTemplate{}, ); err != nil { t.Fatalf("failed to migrate: %v", err) } @@ -1886,16 +1890,20 @@ func TestListAgentsIncludesReleaseTime(t *testing.T) { } var released, draft *AgentItem - for _, item := range resp.Canvas { + for _, raw := range resp.Canvas { + var item AgentItem + if err := json.Unmarshal(raw, &item); err != nil { + t.Fatalf("unmarshal canvas item: %v", err) + } switch item.ID { case "canvas-listed-released": - released = item + released = &item case "canvas-listed-draft": - draft = item + draft = &item } } if released == nil || draft == nil { - t.Fatalf("expected both canvases in list, got %#v", resp.Canvas) + t.Fatalf("expected both canvases in list, got %s", resp.Canvas) } if released.ReleaseTime == nil || *released.ReleaseTime != releaseTime { t.Fatalf("ReleaseTime = %v, want %d", released.ReleaseTime, releaseTime) @@ -1905,6 +1913,151 @@ func TestListAgentsIncludesReleaseTime(t *testing.T) { } } +// TestListAgents_MergesCompilationTemplateGroups verifies that a compilation +// template group owned by the caller appears in the merged /agents list +// (no canvas_category filter), carrying the "compilation_template_group" type +// discriminator and its title = name. Built-in catalogue groups (empty tenant) +// must NOT leak in. +func TestListAgents_MergesCompilationTemplateGroups(t *testing.T) { + setupAgentSessionServiceTest(t) + + base := time.Now().UnixMilli() + groupUpdate := base + 1000 // group updated most recently + canvasUpdate := base + 5000 // agent updated most recently + if err := dao.DB.Create(&entity.User{ID: "user-1", Nickname: "owner", Email: "owner@test.com"}).Error; err != nil { + t.Fatalf("failed to seed user: %v", err) + } + // Agent updated most recently (canvasUpdate > groupUpdate). + if err := dao.DB.Create(&entity.UserCanvas{ + ID: "canvas-1", UserID: "user-1", Title: sptr("Agent canvas"), + CanvasCategory: "agent_canvas", + BaseModel: entity.BaseModel{UpdateTime: &canvasUpdate}, + }).Error; err != nil { + t.Fatalf("failed to seed canvas: %v", err) + } + // The caller's own group (must appear), updated before the canvas. + createAgentSessionTestCompilationGroup(t, "group-own", "user-1", groupUpdate) + // A built-in catalogue group with empty tenant_id (must NOT appear). + if err := dao.DB.Create(&entity.CompilationTemplateGroup{ + ID: "group-builtin", TenantID: "", Name: "Built-in templates", + Scope: "file", BaseModel: entity.BaseModel{CreateTime: &base}, + }).Error; err != nil { + t.Fatalf("failed to seed builtin group: %v", err) + } + + resp, code, err := NewAgentService().ListAgents(t.Context(), "user-1", "", 1, 30, "create_time", true, nil, "", "", nil) + if err != nil || code != common.CodeSuccess { + t.Fatalf("ListAgents failed: code=%v err=%v", code, err) + } + + var sawAgent, sawOwnGroup, sawBuiltin bool + var prevCat, prevName string + orderOK := true + for i, raw := range resp.Canvas { + var item map[string]interface{} + if err := json.Unmarshal(raw, &item); err != nil { + t.Fatalf("unmarshal canvas item: %v", err) + } + // Sort category: agents carry canvas_category; groups are classified by + // the "compilation_template_group" type discriminator (mirroring the + // frontend AgentListItem union). + cat := "agent" + if item["type"] == AgentItemTypeGroup { + cat = CompilationTemplateGroupCategory + } else if cc, ok := item["canvas_category"].(string); ok { + cat = cc + } + name, _ := item["title"].(string) + if i > 0 { + if cat < prevCat || (cat == prevCat && strings.ToLower(name) < strings.ToLower(prevName)) { + orderOK = false + } + } + prevCat, prevName = cat, name + switch item["type"] { + case AgentItemTypeAgent: + if item["id"] == "canvas-1" { + sawAgent = true + } + case AgentItemTypeGroup: + if item["id"] == "group-own" { + sawOwnGroup = true + if item["title"] != "own-group" || item["name"] != "own-group" { + t.Errorf("group title/name = %v/%v, want own-group", item["title"], item["name"]) + } + if item["templates"] == nil { + t.Errorf("group item missing templates") + } + } + if item["id"] == "group-builtin" { + sawBuiltin = true + } + } + } + if !sawAgent { + t.Fatal("agent canvas-1 not in merged list") + } + if !sawOwnGroup { + t.Fatal("caller's compilation template group not in merged list") + } + if sawBuiltin { + t.Fatal("built-in catalogue group leaked into merged list") + } + if !orderOK { + t.Fatal("merged list is not ordered by (canvas_category, name) ascending") + } + // agent_canvas < compilation_template_group, so the canvas must be first. + var first map[string]interface{} + if err := json.Unmarshal(resp.Canvas[0], &first); err != nil { + t.Fatalf("unmarshal first item: %v", err) + } + if first["id"] != "canvas-1" { + t.Fatalf("first merged item = %v, want canvas-1 (agent_canvas category)", first["id"]) + } +} + +// TestSlicePage_NoOverflowPanic guards the review-Critical integer overflow: +// an unbounded positive page/page_size must never overflow (page-1)*pageSize to +// a negative start and panic; out-of-range pages return nil. +func TestSlicePage_NoOverflowPanic(t *testing.T) { + s := []int{1, 2, 3, 4, 5} + // A huge page must return nil, not panic on an overflowing offset. + if got := slicePage(s, 1<<40, 1<<40); got != nil { + t.Fatalf("huge page/pageSize = %v, want nil", got) + } + if got := slicePage(s, 2, 3); len(got) != 2 || got[0] != 4 || got[1] != 5 { + t.Fatalf("page2/size3 = %v, want [4 5]", got) + } + if got := slicePage(s, 3, 2); len(got) != 1 || got[0] != 5 { + t.Fatalf("page3/size2 (start 4) = %v, want [5]", got) + } + if got := slicePage(s, 4, 2); got != nil { + t.Fatalf("page4/size2 (start 6 >= len 5) = %v, want nil", got) + } + // Empty/no-paging semantics preserved (empty input returns the empty slice, + // never a panic). + if got := slicePage([]int{}, 1, 10); len(got) != 0 { + t.Fatalf("empty slice = %v, want empty", got) + } +} + +func createAgentSessionTestCompilationGroup(t *testing.T, id, tenantID string, now int64) { + t.Helper() + if err := dao.DB.Create(&entity.CompilationTemplateGroup{ + ID: id, TenantID: tenantID, Name: "own-group", Scope: "file", + BaseModel: entity.BaseModel{CreateTime: &now, UpdateTime: &now}, + }).Error; err != nil { + t.Fatalf("failed to seed group %s: %v", id, err) + } + if err := dao.DB.Create(&entity.CompilationTemplate{ + ID: id + "-tpl", GroupID: &id, Name: "tree tpl", Kind: "tree", + Config: entity.JSONMap{}, + BaseModel: entity.BaseModel{CreateTime: &now}, + }).Error; err != nil { + t.Fatalf("failed to seed template for %s: %v", id, err) + } +} + func TestPublishAgentUpdatesCanvasAndReleasedVersion(t *testing.T) { setupAgentSessionServiceTest(t) diff --git a/internal/service/compilation_template_service.go b/internal/service/compilation_template_service.go index 8b55809c09..ecd02c66f3 100644 --- a/internal/service/compilation_template_service.go +++ b/internal/service/compilation_template_service.go @@ -210,8 +210,17 @@ func ValidateTemplatePayload(req map[string]interface{}, requireAll bool) error } config, hasConfig := req["config"] if hasConfig { - configMap, ok := config.(map[string]interface{}) - if !ok { + // config may arrive as map[string]interface{} (raw payloads) or as + // entity.JSONMap (payloads built from a typed GroupTemplate, whose Config + // field is JSONMap). A bare type assertion on the former would reject the + // latter because JSONMap is a distinct named type, so normalize both. + var configMap map[string]interface{} + switch c := config.(type) { + case map[string]interface{}: + configMap = c + case entity.JSONMap: + configMap = map[string]interface{}(c) + default: return errors.New("invalid template config") } if len(fmt.Sprint(configMap["global_rules"])) > 4096 { diff --git a/internal/service/compilation_template_service_test.go b/internal/service/compilation_template_service_test.go new file mode 100644 index 0000000000..fb9392d810 --- /dev/null +++ b/internal/service/compilation_template_service_test.go @@ -0,0 +1,69 @@ +package service + +import ( + "testing" + + "ragflow/internal/entity" +) + +// TestValidateTemplatePayload_AcceptsJSONMapConfig covers the create-from-UI +// regression: GroupTemplate.Config is an entity.JSONMap (a named map type), and +// ValidateTemplatePayload used a bare config.(map[string]interface{}) assertion +// that silently rejected it, surfacing as "102 invalid template config". Both +// the raw map and the JSONMap forms must validate identically. +func TestValidateTemplatePayload_AcceptsJSONMapConfig(t *testing.T) { + cfg := map[string]interface{}{ + "kind": "graph", + "entity": map[string]interface{}{ + "fields": []interface{}{ + map[string]interface{}{ + "type": "organization", + "description": "company", + }, + }, + }, + "relation": map[string]interface{}{ + "fields": []interface{}{ + map[string]interface{}{ + "type": "acquisition", + "description": "took over", + }, + }, + }, + } + base := map[string]interface{}{ + "name": "tpl", + "kind": "graph", + } + + // 1) Raw map[string]interface{} payload (older path) must pass. + raw := map[string]interface{}{} + for k, v := range base { + raw[k] = v + } + raw["config"] = cfg + if err := ValidateTemplatePayload(raw, true); err != nil { + t.Fatalf("raw map config rejected: %v", err) + } + + // 2) entity.JSONMap payload (what the group create path builds from + // GroupTemplate.Config) must also pass — this is the regression. + jm := map[string]interface{}{} + for k, v := range base { + jm[k] = v + } + jm["config"] = entity.JSONMap(cfg) + if err := ValidateTemplatePayload(jm, true); err != nil { + t.Fatalf("entity.JSONMap config rejected: %v", err) + } + + // 3) A non-map config is still an error. + bad := map[string]interface{}{} + for k, v := range base { + bad[k] = v + } + bad["config"] = "not-a-map" + if err := ValidateTemplatePayload(bad, true); err == nil { + t.Fatal("non-map config should be rejected") + } +} diff --git a/internal/service/dataset_artifact_service.go b/internal/service/dataset_artifact_service.go index c43f925af5..0ef4ca3744 100644 --- a/internal/service/dataset_artifact_service.go +++ b/internal/service/dataset_artifact_service.go @@ -49,7 +49,6 @@ const ( FieldStructureIndexType = "structure_index_type" FieldStructureKind = "structure_kind" FieldPageID = "page_id" - FieldGraphType = "graph_type" ) // DatasetArtifactService reads knowledge-compilation artifacts (wiki pages, @@ -620,38 +619,6 @@ func (s *DatasetArtifactService) DeleteStructures(ctx context.Context, tenantID, return len(ids), nil } -// DocGraphItem is a single node/edge entry in a document's structure graph. -type DocGraphItem struct { - ID string `json:"id"` - Content string `json:"content"` - SourceID string `json:"source_id"` -} - -// GetDocumentGraph returns the structure graph of a single document. -func (s *DatasetArtifactService) GetDocumentGraph(ctx context.Context, tenantID, datasetID, documentID, graphType string) ([]DocGraphItem, int64, error) { - filter := map[string]interface{}{ - "doc_id": []string{documentID}, - "compiled_graph_kwd": []string{"graph"}, - } - if graphType != "" { - filter[FieldGraphType] = []string{graphType} - } - chunks, total, err := s.searchCompiled(ctx, tenantID, datasetID, filter, - []string{"id", "content_with_weight", "source_id"}, 0, 10000, nil) - if err != nil { - return nil, 0, err - } - items := make([]DocGraphItem, 0, len(chunks)) - for _, c := range chunks { - items = append(items, DocGraphItem{ - ID: firstStringValue(c["id"]), - Content: firstStringValue(c["content_with_weight"]), - SourceID: firstStringValue(c["source_id"]), - }) - } - return items, total, nil -} - // DeleteDocumentGraph deletes the structure graph of a single document. func (s *DatasetArtifactService) DeleteDocumentGraph(ctx context.Context, tenantID, datasetID, documentID string) (int, error) { docEngine := engine.Get() @@ -682,28 +649,12 @@ func (s *DatasetArtifactService) DeleteDocumentGraph(ctx context.Context, tenant return len(ids), nil } -// NavigationItem is a single navigation cluster (REST response shape, kept -// stable for frontend compatibility). -type NavigationItem struct { - Name string `json:"name"` - Title string `json:"title"` - Count int `json:"count"` -} - -// NavChildItem is a single child entry under a navigation cluster (REST -// response shape, kept stable for frontend compatibility). -type NavChildItem struct { - Name string `json:"name"` - Title string `json:"title"` - Count int `json:"count"` -} - -// ListNavClusters returns the navigation clusters of a dataset. It is DEPRECATED -// and now delegates to the ES-backed NavService (internal/service datasetnav): -// the previous implementation queried nav_cluster_kwd/count_int fields that -// Python never writes, so it could never read the real nav tree. Do not add -// field-level patches here — route everything through NavService. -func (s *DatasetArtifactService) ListNavClusters(ctx context.Context, tenantID, datasetID string) ([]NavigationItem, int64, error) { +// ListNavClusters returns the navigation clusters of a dataset. It delegates to +// the ES-backed NavService and returns the frontend DatasetNavNode shape +// (snake_case NavNode JSON), matching Python GET /navigation exactly. The old +// NavigationItem{name,title,count} shape did not match the frontend interface +// and has been removed. +func (s *DatasetArtifactService) ListNavClusters(ctx context.Context, tenantID, datasetID string) ([]nav.NavNode, int64, error) { ns := nav.GetNavService() if ns == nil { return nil, 0, fmt.Errorf("datasetnav: NavService not initialized (SetNavService must be called at bootstrap)") @@ -712,16 +663,12 @@ func (s *DatasetArtifactService) ListNavClusters(ctx context.Context, tenantID, if err != nil { return nil, 0, err } - items := make([]NavigationItem, 0, len(nodes)) - for _, n := range nodes { - items = append(items, NavigationItem{Name: n.Name, Title: n.Description, Count: n.DocCount}) - } - return items, total, nil + return nodes, total, nil } -// ListNavChildren returns the children of a navigation cluster. DEPRECATED — -// delegates to NavService.ListChildren. -func (s *DatasetArtifactService) ListNavChildren(ctx context.Context, tenantID, datasetID, name string) ([]NavChildItem, int64, error) { +// ListNavChildren returns the children of a navigation cluster in the frontend +// DatasetNavNode shape. +func (s *DatasetArtifactService) ListNavChildren(ctx context.Context, tenantID, datasetID, name string) ([]nav.NavNode, int64, error) { ns := nav.GetNavService() if ns == nil { return nil, 0, fmt.Errorf("datasetnav: NavService not initialized (SetNavService must be called at bootstrap)") @@ -730,11 +677,7 @@ func (s *DatasetArtifactService) ListNavChildren(ctx context.Context, tenantID, if err != nil { return nil, 0, err } - items := make([]NavChildItem, 0, len(nodes)) - for _, n := range nodes { - items = append(items, NavChildItem{Name: n.Name, Title: n.Description, Count: n.DocCount}) - } - return items, total, nil + return nodes, total, nil } // DeleteNav removes the direct nav_doc children of every root cluster of a diff --git a/internal/service/dataset_structure_graph.go b/internal/service/dataset_structure_graph.go new file mode 100644 index 0000000000..77ed8f448a --- /dev/null +++ b/internal/service/dataset_structure_graph.go @@ -0,0 +1,1190 @@ +// +// Copyright 2026 The InfiniFlow Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// + +package service + +import ( + "context" + "encoding/json" + "fmt" + "regexp" + "sort" + "strings" + + "ragflow/internal/dao" + "ragflow/internal/engine" + "ragflow/internal/engine/types" +) + +// Structure-graph sampling constants (mirror structure_graph_common.py). +const ( + graphFullThreshold = 1024 // below this combined count, return all rows + graphTopEntities = 256 // seed set A size for large buckets + graphKeywordCandidates = 16 // keyword/KNN candidate rows + graphExpansionCap = 4096 // hub-node expansion cap +) + +var graphEntityFields = []string{"id", "content_with_weight", "name_kwd", "mention_count_int", "source_chunk_ids", "doc_id", "doc_ids_kwd", "source_doc_ids"} +var graphRelationFields = []string{"id", "content_with_weight", "from_entity_kwd", "to_entity_kwd", "doc_id", "doc_ids_kwd", "source_doc_ids"} +var graphAllFields = []string{ + "id", "content_with_weight", "name_kwd", "mention_count_int", "source_chunk_ids", + "from_entity_kwd", "to_entity_kwd", "knowledge_graph_kwd", "doc_id", "doc_ids_kwd", "source_doc_ids", +} + +// StructureGraphNode is a projected entity in the structure graph response. +type StructureGraphNode map[string]interface{} + +// StructureGraphRelation is a projected relation in the structure graph response. +type StructureGraphRelation map[string]interface{} + +// DocumentStructureGraphTemplate is one per-template bucket in the response. +type DocumentStructureGraphTemplate struct { + TemplateID string `json:"template_id"` + TemplateName string `json:"template_name"` + Kind string `json:"kind"` + Entities []StructureGraphNode `json:"entities"` + Relations []StructureGraphRelation `json:"relations"` +} + +// DocumentStructureGraphResponse mirrors Python's {"templates": [...]}. +type DocumentStructureGraphResponse struct { + Templates []DocumentStructureGraphTemplate `json:"templates"` +} + +// graphRowSearch runs one raw-row search over the tenant's document index. +func graphRowSearch(ctx context.Context, tenantID, datasetID string, selectFields []string, filter map[string]interface{}, orderBy *types.OrderByExpr, offset, limit int, matchExprs []interface{}) (map[string]map[string]interface{}, int64, error) { + docEngine := engine.Get() + if docEngine == nil { + return nil, 0, fmt.Errorf("document engine is not initialized") + } + merged := make(map[string]interface{}, len(filter)+1) + for k, v := range filter { + merged[k] = v + } + merged["kb_id"] = []string{datasetID} + if limit < 1 { + limit = 1 + } + res, err := docEngine.Search(ctx, &types.SearchRequest{ + IndexNames: []string{fmt.Sprintf("ragflow_%s", tenantID)}, + KbIDs: []string{datasetID}, + Offset: offset, + Limit: limit, + SelectFields: selectFields, + Filter: merged, + OrderBy: orderBy, + MatchExprs: matchExprs, + }) + if err != nil { + return nil, 0, err + } + if res == nil { + return nil, 0, nil + } + byID := make(map[string]map[string]interface{}, len(res.Chunks)) + for _, c := range res.Chunks { + id := firstStringValue(c["id"]) + if id != "" { + byID[id] = c + } + } + return byID, res.Total, nil +} + +// graphLoadPayload parses content_with_weight into a dict. +func graphLoadPayload(row map[string]interface{}) map[string]interface{} { + raw := firstStringValue(row["content_with_weight"]) + if raw == "" { + return nil + } + var m map[string]interface{} + if err := json.Unmarshal([]byte(raw), &m); err != nil { + return nil + } + return m +} + +func graphIsInvalidSentinel(v string) bool { + if v == "" { + return true + } + lv := strings.ToLower(v) + for _, bad := range []string{"unknown", "none", "null", "nan", "n/a", "undefined", "other"} { + if lv == bad { + return true + } + } + return false +} + +// projectEntity mirrors _struct_graph_entity + project_entity. +func projectEntity(row map[string]interface{}) StructureGraphNode { + payload := graphLoadPayload(row) + if payload == nil { + return nil + } + name := "" + for _, k := range []string{"name", "text", "term", "title"} { + if v, ok := payload[k].(string); ok && strings.TrimSpace(v) != "" { + name = strings.TrimSpace(v) + break + } + } + if name == "" || graphIsInvalidSentinel(name) { + return nil + } + typ := "other" + if v, ok := payload["type"].(string); ok && strings.TrimSpace(v) != "" { + typ = strings.TrimSpace(v) + } + var aliases []string + switch a := payload["aliases"].(type) { + case string: + if strings.TrimSpace(a) != "" { + aliases = []string{strings.TrimSpace(a)} + } + case []interface{}: + for _, e := range a { + if s, ok := e.(string); ok && strings.TrimSpace(s) != "" { + aliases = append(aliases, strings.TrimSpace(s)) + } + } + case []string: + for _, s := range a { + if strings.TrimSpace(s) != "" { + aliases = append(aliases, strings.TrimSpace(s)) + } + } + } + desc := "" + for _, k := range []string{"description", "definition_excerpt"} { + if v, ok := payload[k].(string); ok { + desc = strings.TrimSpace(v) + if desc != "" { + break + } + } + } + chunkIDs := graphSourceChunkIDs(payload, row) + node := StructureGraphNode{ + "aliases": aliases, + "mention_count": 1, + "name": name, + "source_chunk_ids": chunkIDs, + "type": typ, + "description": desc, + } + if mc, ok := graphMentionCount(row); ok { + node["mention_count"] = mc + } + return node +} + +func graphSourceChunkIDs(payload, row map[string]interface{}) []string { + var out []string + add := func(v interface{}) { + switch x := v.(type) { + case []string: + for _, s := range x { + if s != "" { + out = append(out, s) + } + } + case []interface{}: + for _, e := range x { + if s, ok := e.(string); ok && s != "" { + out = append(out, s) + } + } + case string: + if x != "" { + out = append(out, x) + } + } + } + if raw, ok := payload["source_chunk_ids"]; ok { + add(raw) + } else { + add(row["source_chunk_ids"]) + } + // dedup order-preserving + seen := map[string]bool{} + dedup := out[:0] + for _, s := range out { + if !seen[s] { + seen[s] = true + dedup = append(dedup, s) + } + } + return dedup +} + +func graphMentionCount(row map[string]interface{}) (int, bool) { + v := row["mention_count_int"] + if l, ok := v.([]interface{}); ok { + if len(l) > 0 { + v = l[0] + } else { + return 0, false + } + } + switch n := v.(type) { + case int: + return n, true + case int64: + return int(n), true + case float64: + return int(n), true + case json.Number: + if i, err := n.Int64(); err == nil { + return int(i), true + } + } + return 0, false +} + +// projectRelation mirrors _struct_graph_relation + project_relation. +func projectRelation(row map[string]interface{}) StructureGraphRelation { + payload := graphLoadPayload(row) + src, tgt := "", "" + if payload != nil { + for _, k := range []string{"source", "src", "from"} { + if v, ok := payload[k].(string); ok { + src = strings.TrimSpace(v) + if src != "" { + break + } + } + } + for _, k := range []string{"target", "tgt", "to"} { + if v, ok := payload[k].(string); ok { + tgt = strings.TrimSpace(v) + if tgt != "" { + break + } + } + } + } + typ := "related" + if payload != nil { + if v, ok := payload["type"].(string); ok && strings.TrimSpace(v) != "" { + typ = strings.TrimSpace(v) + } + } + if src == "" || tgt == "" || graphIsInvalidSentinel(src) || graphIsInvalidSentinel(tgt) { + // Fall back to the authoritative *_entity_kwd columns. + fallbackSrc := strings.TrimSpace(firstStringValue(row["from_entity_kwd"])) + fallbackTgt := strings.TrimSpace(firstStringValue(row["to_entity_kwd"])) + if fallbackSrc != "" && fallbackTgt != "" { + return StructureGraphRelation{"from": fallbackSrc, "to": fallbackTgt, "type": typ} + } + return nil + } + return StructureGraphRelation{"from": src, "to": tgt, "type": typ} +} + +// dedupEntities order-preserving by (lowercased name, type). +func dedupEntities(entities []StructureGraphNode) []StructureGraphNode { + var out []StructureGraphNode + seen := map[string]bool{} + for _, e := range entities { + name := strings.ToLower(strings.TrimSpace(graphStr(e["name"]))) + typ := strings.ToLower(strings.TrimSpace(graphStr(e["type"]))) + key := name + "\x00" + typ + if name == "" || seen[key] { + continue + } + seen[key] = true + out = append(out, e) + } + return out +} + +func entityResponseID(entity StructureGraphNode) string { + for _, f := range []string{"id", "name", "slug"} { + if v, ok := entity[f].(string); ok && strings.TrimSpace(v) != "" { + return strings.TrimSpace(v) + } + } + return "" +} + +func endpointTerms(value string) []string { + value = strings.TrimSpace(value) + if value == "" { + return nil + } + return sortedUnique([]string{value, strings.ToLower(value)}) +} + +func sortedUnique(in []string) []string { + seen := map[string]bool{} + var out []string + for _, s := range in { + if s != "" && !seen[s] { + seen[s] = true + out = append(out, s) + } + } + sort.Strings(out) + return out +} + +// normalizeRelationEndpoints aligns relation endpoints to returned entity ids. +func normalizeRelationEndpoints(entities []StructureGraphNode, relations []StructureGraphRelation) []StructureGraphRelation { + if len(entities) == 0 || len(relations) == 0 { + return relations + } + lookup := map[string]string{} + ambiguous := map[string]bool{} + for _, entity := range entities { + respID := entityResponseID(entity) + if respID == "" { + continue + } + for _, f := range []string{"id", "name", "slug"} { + v, ok := entity[f].(string) + if !ok || strings.TrimSpace(v) == "" { + continue + } + key := strings.ToLower(strings.TrimSpace(v)) + if cur, exists := lookup[key]; exists && cur != respID { + ambiguous[key] = true + continue + } + lookup[key] = respID + } + } + for k := range ambiguous { + delete(lookup, k) + } + normalized := make([]StructureGraphRelation, 0, len(relations)) + for _, rel := range relations { + item := make(StructureGraphRelation, len(rel)+2) + for k, v := range rel { + item[k] = v + } + for _, f := range []string{"from", "to"} { + if v, ok := item[f].(string); ok { + if mapped, found := lookup[strings.ToLower(strings.TrimSpace(v))]; found { + item[f] = mapped + } + } + } + normalized = append(normalized, item) + } + return normalized +} + +// rowHasEnabledSource mirrors _row_has_enabled_source. +func rowHasEnabledSource(row map[string]interface{}, excludedDocIDs map[string]bool) bool { + if len(excludedDocIDs) == 0 { + return true + } + sourceIDs := map[string]bool{} + flatten := func(v interface{}) { + var walk func(interface{}) + walk = func(x interface{}) { + switch val := x.(type) { + case string: + t := strings.TrimSpace(val) + if t == "" { + return + } + var parsed []interface{} + if json.Unmarshal([]byte(t), &parsed) == nil { + for _, p := range parsed { + walk(p) + } + return + } + sourceIDs[t] = true + case []interface{}: + for _, e := range val { + walk(e) + } + case []string: + for _, s := range val { + walk(s) + } + default: + if s, ok := x.(string); ok && s != "" { + sourceIDs[s] = true + } + } + } + walk(v) + } + for _, f := range []string{"doc_ids_kwd", "source_doc_ids"} { + if v, ok := row[f]; ok && v != nil { + flatten(v) + } + } + if len(sourceIDs) > 0 { + for id := range sourceIDs { + if !excludedDocIDs[id] { + return true + } + } + return false + } + flatten(row["doc_id"]) + if len(sourceIDs) > 0 { + for id := range sourceIDs { + if !excludedDocIDs[id] { + return true + } + } + return false + } + return true +} + +func graphStr(v interface{}) string { + if v == nil { + return "" + } + if s, ok := v.(string); ok { + return s + } + return fmt.Sprintf("%v", v) +} + +// buildBucket mirrors sgc.build_bucket: small buckets whole, large sampled. +func (s *DatasetArtifactService) buildBucket(ctx context.Context, tenantID, datasetID string, scope map[string]interface{}, excludedDocIDs map[string]bool) ([]StructureGraphNode, []StructureGraphRelation, error) { + excludedDocIDs = excludedDocIDsOrEmpty(excludedDocIDs) + bothCond := copyFilter(scope) + bothCond["knowledge_graph_kwd"] = []string{"entity", "relation"} + _, total, err := graphRowSearch(ctx, tenantID, datasetID, []string{"id"}, bothCond, nil, 0, 1, nil) + if err != nil { + return nil, nil, err + } + if total < graphFullThreshold { + fieldMap, _, err := graphRowSearch(ctx, tenantID, datasetID, graphAllFields, bothCond, nil, 0, int(total), nil) + if err != nil { + return nil, nil, err + } + var entities []StructureGraphNode + var relations []StructureGraphRelation + for _, row := range fieldMap { + if !rowHasEnabledSource(row, excludedDocIDs) { + continue + } + kg := firstStringValue(row["knowledge_graph_kwd"]) + if kg == "relation" { + if edge := projectRelation(row); edge != nil { + relations = append(relations, edge) + } + } else { + if node := projectEntity(row); node != nil { + entities = append(entities, node) + } + } + } + entities = dedupEntities(entities) + return entities, normalizeRelationEndpoints(entities, relations), nil + } + + // Large bucket: sample. A = top entities by mention_count_int desc. + orderBy := (&types.OrderByExpr{}).Desc("mention_count_int") + var setA []StructureGraphNode + entityOffset := 0 + var entityTotal int64 = -1 + for len(setA) < graphTopEntities && (entityTotal == -1 || int64(entityOffset) < entityTotal) { + cond := copyFilter(scope) + cond["knowledge_graph_kwd"] = []string{"entity"} + entAMap, entTotal, err := graphRowSearch(ctx, tenantID, datasetID, graphEntityFields, cond, orderBy, entityOffset, graphTopEntities, nil) + if err != nil { + return nil, nil, err + } + entityTotal = entTotal + if len(entAMap) == 0 { + break + } + for _, row := range entAMap { + if !rowHasEnabledSource(row, excludedDocIDs) { + continue + } + if n := projectEntity(row); n != nil { + setA = append(setA, n) + } + } + entityOffset += len(entAMap) + } + if len(setA) > graphTopEntities { + setA = setA[:graphTopEntities] + } + var aNames []string + for _, e := range setA { + if n := strings.TrimSpace(graphStr(e["name"])); n != "" { + aNames = append(aNames, n) + } + } + var aNameTerms []string + for _, name := range aNames { + aNameTerms = append(aNameTerms, endpointTerms(name)...) + } + aNameTerms = sortedUnique(aNameTerms) + + var relations []StructureGraphRelation + targetNamesLower := map[string]bool{} + if len(aNameTerms) > 0 { + cond := copyFilter(scope) + cond["knowledge_graph_kwd"] = []string{"relation"} + cond["from_entity_kwd"] = aNameTerms + relMap, _, err := graphRowSearch(ctx, tenantID, datasetID, graphRelationFields, cond, nil, 0, graphExpansionCap, nil) + if err != nil { + return nil, nil, err + } + for _, row := range relMap { + if !rowHasEnabledSource(row, excludedDocIDs) { + continue + } + if edge := projectRelation(row); edge != nil { + relations = append(relations, edge) + if tgt := strings.ToLower(strings.TrimSpace(graphStr(edge["to"]))); tgt != "" { + targetNamesLower[tgt] = true + } + } + } + } + var setT []StructureGraphNode + if len(targetNamesLower) > 0 { + cond := copyFilter(scope) + cond["knowledge_graph_kwd"] = []string{"entity"} + cond["name_kwd"] = sortedKeys(targetNamesLower) + tgtMap, _, err := graphRowSearch(ctx, tenantID, datasetID, graphEntityFields, cond, nil, 0, graphExpansionCap, nil) + if err != nil { + return nil, nil, err + } + for _, row := range tgtMap { + if !rowHasEnabledSource(row, excludedDocIDs) { + continue + } + if n := projectEntity(row); n != nil { + setT = append(setT, n) + } + } + } + entities := dedupEntities(append(setA, setT...)) + return entities, normalizeRelationEndpoints(entities, relations), nil +} + +func copyFilter(in map[string]interface{}) map[string]interface{} { + out := make(map[string]interface{}, len(in)) + for k, v := range in { + out[k] = v + } + return out +} + +func excludedDocIDsOrEmpty(in map[string]bool) map[string]bool { + if in == nil { + return map[string]bool{} + } + return in +} + +func sortedKeys(m map[string]bool) []string { + out := make([]string, 0, len(m)) + for k := range m { + out = append(out, k) + } + sort.Strings(out) + return out +} + +// compilationTemplateKind normalizes a template kind (mirror Python +// _compilation_template_kind). +func compilationTemplateKind(kind string) string { + k := strings.ToLower(strings.TrimSpace(kind)) + switch k { + case "page_index", "pageindex": + return "page_index" + case "wiki", "wiki_page": + return "wiki" + case "raptor": + return "raptor" + } + return k +} + +// structureKindForBucket returns the normalized kind used for tree/page_index +// hierarchy handling. +func structureKindForBucket(kind string) string { + k := compilationTemplateKind(kind) + if k == "page_index" { + return "page_index" + } + return strings.ReplaceAll(k, "-", "_") +} + +// loadDocumentTemplateMeta resolves the doc's configured template group into +// configured_ids + template_meta maps, mirroring the Python resolution. +func (s *DatasetArtifactService) loadDocumentTemplateMeta(ctx context.Context, tenantID, documentID string) ([]string, map[string]map[string]interface{}, error) { + empty := map[string]map[string]interface{}{} + doc, err := dao.NewDocumentDAO().GetByID(ctx, dao.DB, documentID) + if err != nil || doc == nil { + return nil, empty, err + } + groupID := "" + if doc.ParserConfig != nil { + if gid, ok := doc.ParserConfig["compilation_template_group_id"].(string); ok && gid != "" { + groupID = gid + } else if ext, ok := doc.ParserConfig["ext"].(map[string]interface{}); ok { + if gid, ok := ext["compilation_template_group_id"].(string); ok && gid != "" { + groupID = gid + } + } + } + if groupID == "" { + return nil, empty, nil + } + group, err := NewCompilationTemplateGroupService().GetSaved(ctx, tenantID, groupID) + if err != nil { + return nil, empty, err + } + configuredIDs := []string{} + meta := map[string]map[string]interface{}{} + seen := map[string]bool{} + if group != nil { + for _, t := range group.Templates { + tid := t.ID + if tid == "" || seen[tid] { + continue + } + rawKind := "" + if t.Config != nil { + if k, ok := t.Config["kind"].(string); ok { + rawKind = k + } + } + if rawKind == "" { + rawKind = t.Kind + } + kindNorm := compilationTemplateKind(rawKind) + if kindNorm == "wiki" { + continue + } + seen[tid] = true + configuredIDs = append(configuredIDs, tid) + meta[tid] = map[string]interface{}{ + "template_id": tid, + "template_name": t.Name, + "kind": rawKind, + } + } + } + return configuredIDs, meta, nil +} + +// DocumentStructureGraphInput is the parsed request for GetDocumentGraph. +type DocumentStructureGraphInput struct { + TenantID string + DatasetID string + DocumentID string + GraphType string + Keywords string +} + +// GetDocumentGraph returns the per-template structure graph of a document, +// mirroring Python get_document_structure_graph (normal + keywords modes). +func (s *DatasetArtifactService) GetDocumentGraph(ctx context.Context, in DocumentStructureGraphInput) (*DocumentStructureGraphResponse, error) { + configuredIDs, templateMeta, err := s.loadDocumentTemplateMeta(ctx, in.TenantID, in.DocumentID) + if err != nil { + return nil, err + } + + resp := &DocumentStructureGraphResponse{Templates: []DocumentStructureGraphTemplate{}} + + // keywords mode: name matching/KNN → matched entities' subgraph. + if in.Keywords != "" { + bucketMeta, entities, relations, err := s.keywordSubgraph(ctx, in.TenantID, in.DatasetID, in.DocumentID, in.Keywords, templateMeta) + if err != nil { + return nil, err + } + if bucketMeta == nil || (len(entities) == 0 && len(relations) == 0) { + return resp, nil + } + resp.Templates = append(resp.Templates, DocumentStructureGraphTemplate{ + TemplateID: graphStr(bucketMeta["template_id"]), + TemplateName: graphStr(bucketMeta["template_name"]), + Kind: graphStr(bucketMeta["kind"]), + Entities: entities, + Relations: relations, + }) + return resp, nil + } + + // normal mode: discover buckets from per-doc graph blob rows. + metaFields := []string{"compile_kwd", "compilation_template_ids", "compilation_template_kind_kwd"} + metaRows, _, err := graphRowSearch(ctx, in.TenantID, in.DatasetID, metaFields, + map[string]interface{}{"doc_id": []string{in.DocumentID}, "knowledge_graph_kwd": []string{"graph"}}, nil, 0, 1000, nil) + if err != nil { + return nil, err + } + + bucketMetas := map[string]map[string]interface{}{} + bucketScopes := map[string]map[string]interface{}{} + for _, row := range metaRows { + meta, scope := resolveGraphBucket(row, templateMeta, in.DocumentID) + bid := graphStr(meta["template_id"]) + if _, ok := bucketMetas[bid]; !ok { + bucketMetas[bid] = meta + bucketScopes[bid] = scope + } + } + + grouped := map[string]DocumentStructureGraphTemplate{} + for bid, meta := range bucketMetas { + entities, relations, err := s.buildBucket(ctx, in.TenantID, in.DatasetID, bucketScopes[bid], nil) + if err != nil { + return nil, err + } + if len(entities) == 0 && len(relations) == 0 { + continue + } + grouped[bid] = DocumentStructureGraphTemplate{ + TemplateID: graphStr(meta["template_id"]), + TemplateName: graphStr(meta["template_name"]), + Kind: graphStr(meta["kind"]), + Entities: entities, + Relations: relations, + } + } + + // RAPTOR summary graph blob (compile_kwd = raptor_graph). + s.appendRaptorBlob(ctx, in.TenantID, in.DatasetID, in.DocumentID, grouped) + + // Order: configured templates first, then discovered. + orderedIDs := []string{} + for _, tid := range configuredIDs { + if _, ok := grouped[tid]; ok && !containsStr(orderedIDs, tid) { + orderedIDs = append(orderedIDs, tid) + } + } + for bid := range grouped { + if !containsStr(orderedIDs, bid) { + orderedIDs = append(orderedIDs, bid) + } + } + for _, bid := range orderedIDs { + if g, ok := grouped[bid]; ok && (len(g.Entities) > 0 || len(g.Relations) > 0) { + resp.Templates = append(resp.Templates, g) + } + } + return resp, nil +} + +func containsStr(list []string, v string) bool { + for _, e := range list { + if e == v { + return true + } + } + return false +} + +// resolveGraphBucket mirrors Python _resolve_bucket. +func resolveGraphBucket(row map[string]interface{}, templateMeta map[string]map[string]interface{}, documentID string) (map[string]interface{}, map[string]interface{}) { + compileKwd := firstStringValue(row["compile_kwd"]) + kindVal := firstStringValue(row["compilation_template_kind_kwd"]) + if kindVal == "" { + kindVal = compileKwd + } + tid := rowTemplateID(row) + if tid != "" { + kindNorm := compilationTemplateKind(kindVal) + meta := templateMeta[tid] + // Only a unique kind match can substitute a missing template meta; an + // empty kindNorm must never match (review Major — the previous nil + // round-trip stored a nil entry that the kind-only loop then matched). + if meta == nil && kindNorm != "" { + kindMatches := []map[string]interface{}{} + for _, m := range templateMeta { + if compilationTemplateKind(graphStr(m["kind"])) == kindNorm { + kindMatches = append(kindMatches, m) + } + } + if len(kindMatches) == 1 { + meta = kindMatches[0] + } + } + bucketName := graphStr(meta["template_name"]) + if bucketName == "" { + bucketName = tid + } + bucketKind := graphStr(meta["kind"]) + if bucketKind == "" { + bucketKind = kindVal + } + return map[string]interface{}{ + "template_id": tid, + "template_name": bucketName, + "kind": bucketKind, + }, map[string]interface{}{ + "doc_id": []string{documentID}, + "compilation_template_ids": []string{tid}, + } + } + bucketID := "legacy:" + compileKwd + return map[string]interface{}{ + "template_id": bucketID, + "template_name": "Legacy (" + compileKwd + ")", + "kind": kindVal, + }, map[string]interface{}{ + "doc_id": []string{documentID}, + "compile_kwd": []string{compileKwd}, + "must_not": map[string]interface{}{"exists": "compilation_template_ids"}, + } +} + +func rowTemplateID(row map[string]interface{}) string { + switch v := row["compilation_template_ids"].(type) { + case []interface{}: + for _, e := range v { + if s, ok := e.(string); ok && strings.TrimSpace(s) != "" { + return strings.TrimSpace(s) + } + } + case []string: + for _, s := range v { + if strings.TrimSpace(s) != "" { + return strings.TrimSpace(s) + } + } + case string: + if strings.TrimSpace(v) != "" { + return strings.TrimSpace(v) + } + } + return "" +} + +func (s *DatasetArtifactService) appendRaptorBlob(ctx context.Context, tenantID, datasetID, documentID string, grouped map[string]DocumentStructureGraphTemplate) { + rows, _, err := graphRowSearch(ctx, tenantID, datasetID, []string{"content_with_weight", "compile_kwd"}, + map[string]interface{}{"doc_id": []string{documentID}, "compile_kwd": []string{"raptor_graph"}}, nil, 0, 16, nil) + if err != nil { + return + } + for _, row := range rows { + payload := graphLoadPayload(row) + if payload == nil { + continue + } + rEntities, _ := payload["entities"].([]interface{}) + rRelations, _ := payload["relations"].([]interface{}) + if len(rEntities) == 0 && len(rRelations) == 0 { + continue + } + rb, ok := grouped["raptor"] + if !ok { + rb = DocumentStructureGraphTemplate{TemplateID: "raptor", TemplateName: "RAPTOR Summary", Kind: "raptor"} + grouped["raptor"] = rb + } + rb.Entities = append(rb.Entities, toNodeSlice(rEntities)...) + rb.Relations = append(rb.Relations, toRelationSlice(rRelations)...) + grouped["raptor"] = rb + } +} + +func toNodeSlice(in []interface{}) []StructureGraphNode { + var out []StructureGraphNode + for _, e := range in { + if m, ok := e.(map[string]interface{}); ok { + out = append(out, StructureGraphNode(m)) + } + } + return out +} + +func toRelationSlice(in []interface{}) []StructureGraphRelation { + var out []StructureGraphRelation + for _, e := range in { + if m, ok := e.(map[string]interface{}); ok { + out = append(out, StructureGraphRelation(m)) + } + } + return out +} + +var graphTextQueryRE = regexp.MustCompile(`[ :|\r\n\t,，。？?/\x60!！&^%()\[\]{}<>*~'"\\=]+`) + +func nameMatchesQuery(node StructureGraphNode, query string) bool { + name := strings.ToLower(strings.TrimSpace(graphStr(node["name"]))) + q := strings.ToLower(query) + if name == "" || q == "" { + return false + } + if strings.Contains(name, q) { + return true + } + terms := strings.Fields(q) + if len(terms) == 0 { + return false + } + for _, t := range terms { + if !strings.Contains(name, t) { + return false + } + } + return true +} + +// keywordSubgraph mirrors sgc.keyword_subgraph (BM25 + KNN fallback + ancestor walk). +func (s *DatasetArtifactService) keywordSubgraph(ctx context.Context, tenantID, datasetID, documentID, keywords string, templateMeta map[string]map[string]interface{}) (map[string]interface{}, []StructureGraphNode, []StructureGraphRelation, error) { + baseEntityCond := map[string]interface{}{ + "doc_id": []string{documentID}, + "knowledge_graph_kwd": []string{"entity"}, + } + topFields := append(append([]string{}, graphEntityFields...), "compilation_template_ids", "compile_kwd", "compilation_template_kind_kwd") + + textQuery := strings.TrimSpace(graphTextQueryRE.ReplaceAllString(keywords, " ")) + var candidates []struct { + row map[string]interface{} + node StructureGraphNode + } + validTop := func(rows map[string]map[string]interface{}) []struct { + row map[string]interface{} + node StructureGraphNode + } { + var out []struct { + row map[string]interface{} + node StructureGraphNode + } + for _, row := range rows { + if !rowHasEnabledSource(row, map[string]bool{}) { + continue + } + if node := projectEntity(row); node != nil && strings.TrimSpace(graphStr(node["name"])) != "" { + out = append(out, struct { + row map[string]interface{} + node StructureGraphNode + }{row, node}) + } + } + return out + } + if textQuery != "" { + textExpr := &types.MatchTextExpr{ + Fields: []string{"content_ltks^10", "content_sm_ltks"}, + MatchingText: textQuery, + TopN: graphKeywordCandidates, + ExtraOptions: map[string]interface{}{"original_query": keywords}, + } + topMap, _, err := graphRowSearch(ctx, tenantID, datasetID, topFields, baseEntityCond, nil, 0, graphKeywordCandidates, []interface{}{textExpr}) + if err != nil { + return nil, nil, nil, err + } + for _, c := range validTop(topMap) { + if nameMatchesQuery(c.node, textQuery) { + candidates = append(candidates, c) + } + } + } + // Prefer detail entities over title-typed ancestors. + detailCandidates := candidates[:0] + for _, c := range candidates { + if strings.ToLower(strings.TrimSpace(graphStr(c.node["type"]))) != "title" { + detailCandidates = append(detailCandidates, c) + } + } + if len(detailCandidates) > 0 { + candidates = detailCandidates + } + + // Semantic fallback via KNN. + if len(candidates) == 0 { + vec, err := embedQuery(ctx, tenantID, datasetID, keywords) + if err != nil || len(vec) == 0 { + return nil, nil, nil, nil + } + denseExpr := &types.MatchDenseExpr{ + VectorColumnName: fmt.Sprintf("q_%d_vec", len(vec)), + EmbeddingData: vec, + EmbeddingDataType: "float", + DistanceType: "cosine", + TopN: graphKeywordCandidates, + ExtraOptions: map[string]interface{}{"similarity": 0.3}, + } + topMap, _, err := graphRowSearch(ctx, tenantID, datasetID, topFields, baseEntityCond, nil, 0, graphKeywordCandidates, []interface{}{denseExpr}) + if err != nil { + return nil, nil, nil, err + } + candidates = validTop(topMap) + } + if len(candidates) == 0 { + return nil, nil, nil, nil + } + + scopeForTemplate := func(row map[string]interface{}) (map[string]interface{}, map[string]interface{}) { + return resolveGraphBucket(row, templateMeta, documentID) + } + bucketMeta, scope := scopeForTemplate(candidates[0].row) + bucketID := graphStr(bucketMeta["template_id"]) + + matchedNodes := []StructureGraphNode{} + for _, c := range candidates { + cMeta, _ := scopeForTemplate(c.row) + if graphStr(cMeta["template_id"]) == bucketID { + matchedNodes = append(matchedNodes, c.node) + } + } + if len(matchedNodes) == 0 { + return nil, nil, nil, nil + } + structureKind := structureKindForBucket(graphStr(bucketMeta["kind"])) + + var relations []StructureGraphRelation + seenRel := map[string]bool{} + neighborNamesLower := map[string]bool{} + matchedNames := map[string]bool{} + for _, n := range matchedNodes { + if name := strings.ToLower(strings.TrimSpace(graphStr(n["name"]))); name != "" { + matchedNames[name] = true + } + } + if structureKind != "tree" && structureKind != "page_index" { + for _, matchedNode := range matchedNodes { + matchedName := strings.TrimSpace(graphStr(matchedNode["name"])) + terms := endpointTerms(matchedName) + for _, field := range []string{"from_entity_kwd", "to_entity_kwd"} { + cond := copyFilter(scope) + cond["knowledge_graph_kwd"] = []string{"relation"} + cond[field] = terms + relMap, _, err := graphRowSearch(ctx, tenantID, datasetID, graphRelationFields, cond, nil, 0, graphExpansionCap, nil) + if err != nil { + return nil, nil, nil, err + } + for _, row := range relMap { + if !rowHasEnabledSource(row, map[string]bool{}) { + continue + } + if edge := projectRelation(row); edge != nil { + key := graphStr(edge["from"]) + "\x00" + graphStr(edge["to"]) + "\x00" + graphStr(edge["type"]) + if seenRel[key] { + continue + } + seenRel[key] = true + relations = append(relations, edge) + for _, endpoint := range []string{graphStr(edge["from"]), graphStr(edge["to"])} { + if ep := strings.TrimSpace(endpoint); ep != "" && !matchedNames[strings.ToLower(ep)] { + neighborNamesLower[strings.ToLower(ep)] = true + } + } + } + if len(relations) >= graphExpansionCap { + break + } + } + if len(relations) >= graphExpansionCap { + break + } + } + if len(relations) >= graphExpansionCap { + break + } + } + } + + // Ancestor walk for tree/page_index. + if (structureKind == "tree" || structureKind == "page_index") && len(relations) < graphExpansionCap { + ancestorFrontier := map[string]bool{} + seenAncestors := map[string]bool{} + for n := range matchedNames { + ancestorFrontier[n] = true + seenAncestors[n] = true + } + for len(ancestorFrontier) > 0 && len(relations) < graphExpansionCap { + nextFrontier := map[string]bool{} + cond := copyFilter(scope) + cond["knowledge_graph_kwd"] = []string{"relation"} + cond["to_entity_kwd"] = sortedKeys(ancestorFrontier) + relMap, _, err := graphRowSearch(ctx, tenantID, datasetID, graphRelationFields, cond, nil, 0, graphExpansionCap-len(relations), nil) + if err != nil { + return nil, nil, nil, err + } + for _, row := range relMap { + if !rowHasEnabledSource(row, map[string]bool{}) { + continue + } + if edge := projectRelation(row); edge != nil { + key := graphStr(edge["from"]) + "\x00" + graphStr(edge["to"]) + "\x00" + graphStr(edge["type"]) + if seenRel[key] { + continue + } + seenRel[key] = true + relations = append(relations, edge) + parent := strings.ToLower(strings.TrimSpace(graphStr(edge["from"]))) + if parent != "" && !seenAncestors[parent] { + seenAncestors[parent] = true + nextFrontier[parent] = true + } + if len(relations) >= graphExpansionCap { + break + } + } + } + ancestorFrontier = nextFrontier + for n := range nextFrontier { + neighborNamesLower[n] = true + } + } + } + + entities := append([]StructureGraphNode{}, matchedNodes...) + if len(neighborNamesLower) > 0 { + cond := copyFilter(scope) + cond["knowledge_graph_kwd"] = []string{"entity"} + cond["name_kwd"] = sortedKeys(neighborNamesLower) + nbMap, _, err := graphRowSearch(ctx, tenantID, datasetID, graphEntityFields, cond, nil, 0, graphExpansionCap, nil) + if err != nil { + return nil, nil, nil, err + } + for _, row := range nbMap { + if !rowHasEnabledSource(row, map[string]bool{}) { + continue + } + if n := projectEntity(row); n != nil { + entities = append(entities, n) + } + } + } + entities = dedupEntities(entities) + if structureKind == "tree" || structureKind == "page_index" { + entityNames := map[string]bool{} + for _, e := range entities { + if n := strings.ToLower(strings.TrimSpace(graphStr(e["name"]))); n != "" { + entityNames[n] = true + } + } + var filtered []StructureGraphRelation + for _, r := range relations { + if entityNames[strings.ToLower(strings.TrimSpace(graphStr(r["from"])))] && entityNames[strings.ToLower(strings.TrimSpace(graphStr(r["to"])))] { + filtered = append(filtered, r) + } + } + relations = filtered + } + return bucketMeta, entities, normalizeRelationEndpoints(entities, relations), nil +} + +// embedQuery embeds a query via the nav embedder (best-effort). The embedder +// resolves the tenant's embedding model on demand; an empty result (or error) is +// treated as "no semantic candidates", matching Python's fallback behavior. +func embedQuery(ctx context.Context, tenantID, datasetID, query string) ([]float64, error) { + embedder := NewNavEmbedder(NewModelProviderService(), "") + vecs, err := embedder.Encode(ctx, tenantID, []string{query}) + if err != nil || len(vecs) == 0 || len(vecs[0]) == 0 { + return nil, fmt.Errorf("embed query failed") + } + f := make([]float64, len(vecs[0])) + for i, v := range vecs[0] { + f[i] = float64(v) + } + return f, nil +} diff --git a/internal/service/dataset_structure_graph_test.go b/internal/service/dataset_structure_graph_test.go new file mode 100644 index 0000000000..74a767b9e2 --- /dev/null +++ b/internal/service/dataset_structure_graph_test.go @@ -0,0 +1,107 @@ +package service + +import ( + "reflect" + "testing" +) + +// TestProjectEntity_FromPayload verifies projectEntity maps the tree-node +// payload shape to the graph-node shape (mirroring _struct_graph_entity). +func TestProjectEntity_FromPayload(t *testing.T) { + row := map[string]interface{}{ + "content_with_weight": `{"name":"NVIDIA","type":"tree_node","description":"chip maker","source_chunk_ids":["c1","c2"]}`, + "source_chunk_ids": []string{"c1"}, + "mention_count_int": 3, + } + n := projectEntity(row) + if n == nil { + t.Fatal("projectEntity returned nil") + } + if n["name"] != "NVIDIA" || n["type"] != "tree_node" { + t.Errorf("name/type = %v/%v, want NVIDIA/tree_node", n["name"], n["type"]) + } + if n["mention_count"] != 3 { + t.Errorf("mention_count = %v, want 3", n["mention_count"]) + } + chunks, _ := n["source_chunk_ids"].([]string) + if !reflect.DeepEqual(chunks, []string{"c1", "c2"}) { + t.Errorf("source_chunk_ids = %v, want [c1 c2]", chunks) + } +} + +// TestProjectEntity_RejectsInvalidName verifies sentinel/empty names are dropped. +func TestProjectEntity_RejectsInvalidName(t *testing.T) { + for _, payload := range []string{`{"name":""}`, `{"name":"unknown"}`} { + if n := projectEntity(map[string]interface{}{"content_with_weight": payload}); n != nil { + t.Errorf("expected nil for payload %s, got %v", payload, n) + } + } +} + +// TestProjectRelation_FromPayload verifies the relation projection. +func TestProjectRelation_FromPayload(t *testing.T) { + row := map[string]interface{}{"content_with_weight": `{"from":"NVIDIA","to":"GPU","type":"child"}`} + r := projectRelation(row) + if r == nil { + t.Fatal("projectRelation returned nil") + } + if r["from"] != "NVIDIA" || r["to"] != "GPU" || r["type"] != "child" { + t.Errorf("relation = %v, want {NVIDIA GPU child}", r) + } +} + +// TestProjectRelation_FallsBackToKwdColumns verifies the authoritative *_entity_kwd +// columns are used when the payload has no from/to. +func TestProjectRelation_FallsBackToKwdColumns(t *testing.T) { + row := map[string]interface{}{ + "content_with_weight": `{"type":"related"}`, + "from_entity_kwd": "NVIDIA", + "to_entity_kwd": "GPU", + } + r := projectRelation(row) + if r == nil || r["from"] != "NVIDIA" || r["to"] != "GPU" || r["type"] != "related" { + t.Errorf("relation = %v, want {NVIDIA GPU related}", r) + } +} + +// TestDedupEntities_OrderPreserving verifies dedup by (lowercased name, type). +func TestDedupEntities_OrderPreserving(t *testing.T) { + in := []StructureGraphNode{ + {"name": "A", "type": "x"}, + {"name": "a", "type": "x"}, // dup (case-insensitive) + {"name": "B", "type": "y"}, + {"name": ""}, // dropped (empty name) + } + out := dedupEntities(in) + if len(out) != 2 || out[0]["name"] != "A" || out[1]["name"] != "B" { + t.Errorf("dedupEntities = %v, want [A B]", out) + } +} + +// TestNormalizeRelationEndpoints aligns relation endpoints to entity ids/names. +func TestNormalizeRelationEndpoints(t *testing.T) { + entities := []StructureGraphNode{{"name": "NVIDIA", "type": "x"}} + relations := []StructureGraphRelation{{"from": "nvidia", "to": "GPU", "type": "child"}} + out := normalizeRelationEndpoints(entities, relations) + if out[0]["from"] != "NVIDIA" { + t.Errorf("normalized from = %v, want NVIDIA (matched to entity name)", out[0]["from"]) + } +} + +// TestCompilationTemplateKind_Normalization covers kind normalization. +func TestCompilationTemplateKind_Normalization(t *testing.T) { + if compilationTemplateKind("Page_Index") != "page_index" { + t.Errorf("Page_Index => %q, want page_index", compilationTemplateKind("Page_Index")) + } + if compilationTemplateKind("tree") != "tree" { + t.Errorf("tree => %q, want tree", compilationTemplateKind("tree")) + } +} + +// TestRowTemplateID_FromList covers compilation_template_ids extraction. +func TestRowTemplateID_FromList(t *testing.T) { + row := map[string]interface{}{"compilation_template_ids": []interface{}{"", "tid1", "tid2"}} + if got := rowTemplateID(row); got != "tid1" { + t.Errorf("rowTemplateID = %q, want tid1", got) + } +} diff --git a/internal/service/nav/nav.go b/internal/service/nav/nav.go index a87fcfd8c9..243ea0ea0a 100644 --- a/internal/service/nav/nav.go +++ b/internal/service/nav/nav.go @@ -27,14 +27,16 @@ import ( "sync" ) -// NavNode mirrors Python's _nav_item (dataset_api_service.py). +// NavNode mirrors Python's _nav_item (dataset_api_service.py). The JSON field +// names are snake_case to match the frontend DatasetNavNode contract and the +// Python GET /navigation payload exactly. type NavNode struct { - Name string // name - Description string // content_with_weight payload description - DocCount int // doc_count_int (cluster) or 1 (leaf) - Type string // "cluster" | "doc" - DocID string // leaf doc_id; empty for cluster - HasChildren bool // is_cluster + Name string `json:"name"` + Description string `json:"description"` + DocCount int `json:"doc_count"` + Type string `json:"type"` + DocID string `json:"doc_id,omitempty"` + HasChildren bool `json:"has_children"` } // NavHit is one KNN hit on a nav row. diff --git a/internal/service/nlp/datasetnav.go b/internal/service/nlp/datasetnav.go index cf057f8795..4959b1b4d2 100644 --- a/internal/service/nlp/datasetnav.go +++ b/internal/service/nlp/datasetnav.go @@ -177,8 +177,23 @@ func (s *NavService) ListChildren(ctx context.Context, tenantID, kbID, name stri // nodeFromRow converts an engine row into a NavNode. func (s *NavService) nodeFromRow(row map[string]interface{}, fallbackType string) nav.NavNode { + name := firstStringValue(row["title_kwd"]) + // Prefer an explicit readable "name" column (Python rows carry one); fall + // back to title_kwd. + if n := firstStringValue(row["name"]); n != "" { + name = n + } + // A raw id (doc_id or "cluster_") is not a human-readable name; fall + // back to a title derived from the payload description. Cluster names are the + // child-lookup key (parent_kwd references them verbatim), so they must stay + // intact; only non-cluster (leaf) rows get the readable fallback, otherwise + // GET /navigation/{cluster}/children would no longer match (review Major). + isCluster := firstStringValue(row["type_kwd"]) == "nav_cluster" + if !isCluster && graphIsRawID(name) { + name = "" + } node := nav.NavNode{ - Name: firstStringValue(row["title_kwd"]), + Name: name, DocCount: intValue(row["doc_count_int"]), Type: fallbackType, DocID: firstStringValue(row["doc_id"]), @@ -196,6 +211,13 @@ func (s *NavService) nodeFromRow(row map[string]interface{}, fallbackType string if d, ok := m["description"].(string); ok { node.Description = d } + if node.Name == "" { + if t, ok := m["title"].(string); ok && strings.TrimSpace(t) != "" { + node.Name = cleanTitle(t) + } else if d, ok := m["description"].(string); ok && strings.TrimSpace(d) != "" { + node.Name = cleanTitle(fallbackTitle(d)) + } + } } } node.HasChildren = node.Type == "cluster" @@ -205,6 +227,31 @@ func (s *NavService) nodeFromRow(row map[string]interface{}, fallbackType string return node } +// graphIsRawID reports whether a name is a meaningless internal id (a doc id or +// a "cluster_" key) rather than a human-readable title. +func graphIsRawID(name string) bool { + name = strings.TrimSpace(name) + if name == "" { + return true + } + if strings.HasPrefix(name, "cluster_") && len(name) == len("cluster_")+8 { + return true + } + if len(name) == 32 && isHexString(name) { + return true + } + return false +} + +func isHexString(s string) bool { + for _, r := range s { + if !((r >= '0' && r <= '9') || (r >= 'a' && r <= 'f') || (r >= 'A' && r <= 'F')) { + return false + } + } + return true +} + // Search runs query KNN over nav rows and returns routed doc ids. func (s *NavService) Search(ctx context.Context, tenantID, kbID, query string, embd []float32, topK int) ([]nav.NavHit, error) { if topK <= 0 { @@ -343,8 +390,11 @@ func (s *NavService) UpsertDoc(ctx context.Context, in nav.UpsertDocInput) error "compile_kwd": navCompileKwd, "available_int": 0, "type_kwd": "nav_doc", - "title_kwd": in.DocID, - "parent_kwd": parent, + // The nav_doc's display name is a readable title derived from the summary + // (Python _clean_title/_fallback_title) — NOT the raw doc id, which is + // meaningless in the UI. The doc_id is still stored for lookups. + "title_kwd": cleanTitle(fallbackTitle(in.Summary)), + "parent_kwd": parent, // The nav_doc sits one level below its (possibly nested) parent // cluster, so its depth is parentDepth+1 — not a hard-coded 1. "depth_int": bestDepth + 1, @@ -376,6 +426,7 @@ func (s *NavService) UpsertDoc(ctx context.Context, in nav.UpsertDocInput) error } _, err = de.InsertChunks(ctx, []map[string]interface{}{{ "id": navClusterID(in.TenantID, in.KbID, name), + "doc_id": in.KbID, // cluster rows carry the kb as doc_id (Python _build_nav_cluster_row), so ES InsertChunks does not skip them "compile_kwd": navCompileKwd, "available_int": 0, "type_kwd": "nav_cluster", @@ -387,6 +438,27 @@ func (s *NavService) UpsertDoc(ctx context.Context, in nav.UpsertDocInput) error "content_with_weight": payloadJSONNav(map[string]interface{}{"type": "nav_cluster", "description": summary}), "q_" + fmt.Sprintf("%d", len(vec)) + "_vec": f32ToF64Slice(vec), }}, idx, in.KbID) + if err != nil { + return err + } + // Always emit a nav_doc leaf under the (new) cluster, mirroring Python + // upsert_dataset_nav_doc (the merge branch does the same at its parent). A + // new root cluster that only folds the doc into doc_ids_kwd would leave the + // nav tree with a single cluster and no nav_doc child, so /children returns + // empty. The nav_doc carries a readable title + parent_kwd = the cluster name. + _, err = de.InsertChunks(ctx, []map[string]interface{}{{ + "id": navDocID(in.TenantID, in.KbID, in.DocID), + "compile_kwd": navCompileKwd, + "available_int": 0, + "type_kwd": "nav_doc", + "title_kwd": cleanTitle(fallbackTitle(in.Summary)), + "parent_kwd": name, + "depth_int": depth + 1, + "doc_id": in.DocID, + "doc_count_int": 1, + "content_with_weight": payloadJSONNav(map[string]interface{}{"type": "nav_doc", "description": in.Summary}), + "q_" + fmt.Sprintf("%d", len(vec)) + "_vec": f32ToF64Slice(vec), + }}, idx, in.KbID) return err } @@ -775,16 +847,54 @@ func (s *NavService) llmMergeDescription(ctx context.Context, tenantID string, t return strings.Join(texts, "\n") } +// cleanTitle normalizes a raw title/summary into a one-line, length-capped +// display name (mirroring Python dataset_nav._clean_title). +func cleanTitle(title string) string { + return truncateString(strings.Join(strings.Fields(title), " "), 48) +} + +// fallbackTitle derives a short readable title from a summary by taking the +// first non-empty line and stripping Markdown emphasis/heading markers. The +// tree-root summaries carry a one-line Markdown title ("**Title**" or +// "## Title") on the first line, so this yields a clean, URL-friendly label — +// far more readable than taking the first whitespace words of the body. +func fallbackTitle(summary string) string { + line := summary + if idx := strings.IndexAny(line, "\n\r"); idx >= 0 { + line = line[:idx] + } + line = strings.TrimSpace(line) + // Strip Markdown emphasis and heading markers. + line = strings.Trim(line, "*# \t") + if line == "" { + return "Cluster" + } + return line +} + +// readableClusterName returns a readable yet unique nav-cluster key of the form +// " <8-hex>" (mirroring Python dataset_nav._readable_cluster_name): the +// title keeps the node name human-readable, the short hash of the seed keeps the +// per-KB uniqueness that the tree keying relies on. +func readableClusterName(title, seed string) string { + t := cleanTitle(title) + if t == "" { + t = "Cluster" + } + return t + " " + shortHash8(seed) +} + // llmCreateSummary builds a short cluster name + summary for a source text via // the optional LLM (mirroring Python _llm_create_summary); without an LLM it -// falls back to a deterministic name from a content hash and the text itself. +// falls back to a readable name derived from the summary's first words plus a +// short hash, so nav clusters are human-readable instead of "cluster_<hash>". func (s *NavService) llmCreateSummary(ctx context.Context, tenantID, text string) (name, summary string) { if s.llm != nil { if n, sm, err := s.llm.CreateSummary(ctx, tenantID, text); err == nil && n != "" { return n, sm } } - return "cluster_" + contentHash8(text), text + return readableClusterName(fallbackTitle(text), text), text } // appendDocToCluster appends a doc id to a cluster's doc_ids_kwd and bumps its @@ -1015,6 +1125,20 @@ func payloadJSONNav(v map[string]interface{}) string { return string(b) } +// truncateString caps s to n runes (not bytes). +func truncateString(s string, n int) string { + runes := []rune(s) + if len(runes) <= n { + return s + } + return string(runes[:n]) +} + +// shortHash8 is a stable 8-char hash suffix for readable nav names. +func shortHash8(s string) string { + return contentHash8(s) +} + // contentHash8 is a stable 8-char hash. func contentHash8(s string) string { h := uint32(2166136261) diff --git a/internal/service/nlp/datasetnav_test.go b/internal/service/nlp/datasetnav_test.go index fbf4620704..19523e9733 100644 --- a/internal/service/nlp/datasetnav_test.go +++ b/internal/service/nlp/datasetnav_test.go @@ -2,8 +2,10 @@ package nlp import ( "context" + "encoding/json" "fmt" "math" + "strings" "testing" "ragflow/internal/engine/types" @@ -303,6 +305,101 @@ func TestNavService_ListClusters_FiltersRoot(t *testing.T) { } } +// TestNavNode_JSONShape_SnakeCase locks the REST contract: the frontend +// DatasetNavNode and Python GET /navigation both use snake_case keys. If the +// Go field names leaked (Name/Description/DocCount...) the frontend tree would +// read undefined fields and render empty. +func TestNavNode_JSONShape_SnakeCase(t *testing.T) { + n := nav.NavNode{Name: "cluster_x", Description: "d", DocCount: 3, Type: "cluster", HasChildren: true} + b, err := json.Marshal(n) + if err != nil { + t.Fatal(err) + } + var m map[string]interface{} + if err := json.Unmarshal(b, &m); err != nil { + t.Fatal(err) + } + for _, want := range []string{"name", "description", "doc_count", "type", "has_children"} { + if _, ok := m[want]; !ok { + t.Errorf("NavNode JSON missing snake_case key %q; got %s", want, b) + } + } + for _, bad := range []string{"Name", "Description", "DocCount", "HasChildren"} { + if _, ok := m[bad]; ok { + t.Errorf("NavNode JSON leaked PascalCase key %q; got %s", bad, b) + } + } +} + +// TestNavNamingHelpers_Readable verifies the nav display-name helpers produce +// human-readable names (mirroring Python _clean_title/_fallback_title/ +// _readable_cluster_name) instead of raw ids. +func TestNavNamingHelpers_Readable(t *testing.T) { + if cleanTitle(" hello world ") != "hello world" { + t.Errorf("cleanTitle = %q, want %q", cleanTitle(" hello world "), "hello world") + } + // fallbackTitle takes the first non-empty line and strips Markdown markers. + if got := fallbackTitle("a b c d e f g h"); got != "a b c d e f g h" { + t.Errorf("fallbackTitle = %q, want the first line", got) + } + if got := fallbackTitle("**何进诛阉与董后之废**\n\nbody text"); got != "何进诛阉与董后之废" { + t.Errorf("fallbackTitle = %q, want the stripped markdown title", got) + } + if got := fallbackTitle(""); got != "Cluster" { + t.Errorf("fallbackTitle('') = %q, want Cluster", got) + } + name := readableClusterName("何进诛阉", "seed-text") + if !strings.HasPrefix(name, "何进诛阉 ") { + t.Errorf("readableClusterName = %q, want prefix %q", name, "何进诛阉 ") + } +} + +// TestNavNamingHelpers_RawIDDetection verifies meaningless ids are detected so +// nodeFromRow can fall back to a readable title. +func TestNavNamingHelpers_RawIDDetection(t *testing.T) { + for _, raw := range []string{"d3778ef9c0f5495fa4bdadc00a5bf15c", "cluster_abc12345", ""} { + if !graphIsRawID(raw) { + t.Errorf("graphIsRawID(%q) = false, want true", raw) + } + } + for _, ok := range []string{"何进诛阉", "NVIDIA financial performance 8738e200"} { + if graphIsRawID(ok) { + t.Errorf("graphIsRawID(%q) = true, want false", ok) + } + } +} + +// TestNodeFromRow_ReadableName verifies nodeFromRow falls back to a readable +// title derived from the payload description when title_kwd is a raw id. +func TestNodeFromRow_ReadableName(t *testing.T) { + ns := &NavService{} + row := map[string]interface{}{ + "title_kwd": "d3778ef9c0f5495fa4bdadc00a5bf15c", + "doc_id": "d3778ef9c0f5495fa4bdadc00a5bf15c", + "type_kwd": "nav_doc", + "doc_count_int": 1, + "content_with_weight": `{"type":"nav_doc","description":"刘备三战黄巾军与何进诛阉之议\nsecond line"}`, + } + node := ns.nodeFromRow(row, "doc") + if node.Name == "" || node.Name == "d3778ef9c0f5495fa4bdadc00a5bf15c" { + t.Fatalf("nodeFromRow name = %q, want readable fallback", node.Name) + } + if node.DocID != "d3778ef9c0f5495fa4bdadc00a5bf15c" { + t.Errorf("nodeFromRow DocID = %q, want the raw doc id preserved", node.DocID) + } + // A cluster row keeps its stored key verbatim (it is the child-lookup + // parent_kwd), even when it looks like a raw id (review Major). + clusterRow := map[string]interface{}{ + "title_kwd": "cluster_abc12345", + "type_kwd": "nav_cluster", + "doc_count_int": 2, + } + cluster := ns.nodeFromRow(clusterRow, "cluster") + if cluster.Name != "cluster_abc12345" { + t.Errorf("cluster key = %q, want it kept verbatim as the parent_kwd lookup key", cluster.Name) + } +} + // TestNavService_Search_ReturnsHit asserts acceptance #5. func TestNavService_Search_ReturnsHit(t *testing.T) { eng := newMemNavEngine() @@ -353,15 +450,21 @@ func TestNavService_Acceptance4_ListChildren(t *testing.T) { if err != nil { t.Fatal(err) } - // Exactly one nav_doc (for d2) sits under the cluster; d1 is the cluster. - if total != 1 || len(children) != 1 { - t.Fatalf("expected 1 child under cluster, got total=%d len=%d", total, len(children)) + // Every doc upserted under the cluster emits a nav_doc leaf (Python + // upsert_dataset_nav_doc), so d1 (which created the cluster) and d2 (which + // merged in) both sit under it: two nav_docs total. + if total != 2 || len(children) != 2 { + t.Fatalf("expected 2 children under cluster, got total=%d len=%d", total, len(children)) } - if children[0].DocID != "d2" { - t.Errorf("child doc_id = %q, want d2", children[0].DocID) + gotIDs := map[string]bool{} + for _, c := range children { + gotIDs[c.DocID] = true + if c.Type != "doc" { + t.Errorf("child type = %q, want doc", c.Type) + } } - if children[0].Type != "doc" { - t.Errorf("child type = %q, want doc", children[0].Type) + if !gotIDs["d1"] || !gotIDs["d2"] { + t.Errorf("expected nav_docs for both d1 and d2, got %v", gotIDs) } }