Port dataset-level structure merge for timeline/graph/mindmap (#18201)

Ports Python dataset-level structure aggregation (timeline, graph, mindmap) to Go. Mindmap emits entity/relation rows and merges like graph. Adds dataset_merge guard, engine gate, resolveDatasetStructureKind, kind-required structure graph GET/DELETE API, per-index task-id fields.
This commit is contained in:
Zhichang Yu
2026-08-13 18:37:47 +08:00
committed by GitHub
parent a6f2eac496
commit 0784bef5b0
16 changed files with 958 additions and 307 deletions

View File

@@ -124,7 +124,19 @@ type Knowledgebase struct {
WikiTaskFinishAt *time.Time `gorm:"column:wiki_task_finish_at" json:"wiki_task_finish_at,omitempty"`
SkillTaskID *string `gorm:"column:skill_task_id;size:32;index" json:"skill_task_id,omitempty"`
SkillTaskFinishAt *time.Time `gorm:"column:skill_task_finish_at" json:"skill_task_finish_at,omitempty"`
Status *string `gorm:"column:status;size:1;index;default:'1'" json:"status,omitempty"`
// Dataset-level structure merge task state (plan §8, mirror Python
// _INDEX_TYPE_TO_TASK_ID_FIELD's per-structure-index entries).
StructureGraphTaskID *string `gorm:"column:structure_graph_task_id;size:32;index" json:"structure_graph_task_id,omitempty"`
StructureGraphTaskFinishAt *time.Time `gorm:"column:structure_graph_task_finish_at" json:"structure_graph_task_finish_at,omitempty"`
StructureMindmapTaskID *string `gorm:"column:structure_mindmap_task_id;size:32;index" json:"structure_mindmap_task_id,omitempty"`
StructureMindmapTaskFinishAt *time.Time `gorm:"column:structure_mindmap_task_finish_at" json:"structure_mindmap_task_finish_at,omitempty"`
TimelineTaskID *string `gorm:"column:timeline_task_id;size:32;index" json:"timeline_task_id,omitempty"`
TimelineTaskFinishAt *time.Time `gorm:"column:timeline_task_finish_at" json:"timeline_task_finish_at,omitempty"`
SessionGraphTaskID *string `gorm:"column:session_graph_task_id;size:32;index" json:"session_graph_task_id,omitempty"`
SessionGraphTaskFinishAt *time.Time `gorm:"column:session_graph_task_finish_at" json:"session_graph_task_finish_at,omitempty"`
SessionEssenceTaskID *string `gorm:"column:session_essence_task_id;size:32;index" json:"session_essence_task_id,omitempty"`
SessionEssenceTaskFinishAt *time.Time `gorm:"column:session_essence_task_finish_at" json:"session_essence_task_finish_at,omitempty"`
Status *string `gorm:"column:status;size:1;index;default:'1'" json:"status,omitempty"`
BaseModel
}

View File

@@ -14,6 +14,7 @@
package handler
import (
"errors"
"net/http"
"strconv"
@@ -276,35 +277,56 @@ func (h *DatasetArtifactHandler) GetArtifactGraph(c *gin.Context) {
common.SuccessWithData(c, graph, "success")
}
// ListStructures handles GET /artifacts/structure — list compiled structures of a dataset.
// ListStructures handles GET /artifacts/structure?kind=<kind> — the dataset-scope
// structure graph for a resolved kind (mirrors Python get_dataset_structure).
// kind is REQUIRED: missing or invalid → 400 ARGUMENT_ERROR.
func (h *DatasetArtifactHandler) ListStructures(c *gin.Context) {
_, tenantID, _ := h.datasetOwner(c, c.Param("dataset_id"))
if tenantID == "" {
return
}
datasetID := c.Param("dataset_id")
structureKind := c.Query("structure_kind")
structureIndexType := c.Query("structure_index_type")
items, total, err := h.svc.ListStructures(c.Request.Context(), tenantID, datasetID, structureKind, structureIndexType)
if err != nil {
common.ErrorWithCode(c, common.CodeDataError, err.Error())
kind := c.Query("kind")
if kind == "" {
common.ErrorWithCode(c, common.CodeArgumentError, "kind is required")
return
}
common.SuccessWithData(c, gin.H{"total": total, "structures": items}, "success")
in := service.DatasetStructureGraphInput{TenantID: tenantID, DatasetID: datasetID, Kind: kind}
resp, err := h.svc.GetDatasetStructure(c.Request.Context(), in)
if err != nil {
if errors.Is(err, service.ErrInvalidStructureKind) {
common.ErrorWithCode(c, common.CodeArgumentError, err.Error())
} else {
common.ErrorWithCode(c, common.CodeServerError, err.Error())
}
return
}
common.SuccessWithData(c, resp, "success")
}
// DeleteStructures handles DELETE /artifacts/structure — delete compiled structures of a dataset.
// DeleteStructures handles DELETE /artifacts/structure?kind=<kind>&wipe=<bool> —
// cancel the kind's task (wipe=false) or delete its dataset rows (wipe=true).
// kind is REQUIRED.
func (h *DatasetArtifactHandler) DeleteStructures(c *gin.Context) {
_, tenantID, _ := h.datasetOwner(c, c.Param("dataset_id"))
if tenantID == "" {
return
}
datasetID := c.Param("dataset_id")
structureKind := c.Query("structure_kind")
structureIndexType := c.Query("structure_index_type")
n, err := h.svc.DeleteStructures(c.Request.Context(), tenantID, datasetID, structureKind, structureIndexType)
kind := c.Query("kind")
if kind == "" {
common.ErrorWithCode(c, common.CodeArgumentError, "kind is required")
return
}
wipe := c.Query("wipe") == "true"
in := service.DatasetStructureGraphInput{TenantID: tenantID, DatasetID: datasetID, Kind: kind, Wipe: wipe}
n, err := h.svc.DeleteDatasetStructure(c.Request.Context(), in)
if err != nil {
common.ErrorWithCode(c, common.CodeDataError, err.Error())
if errors.Is(err, service.ErrInvalidStructureKind) {
common.ErrorWithCode(c, common.CodeArgumentError, err.Error())
} else {
common.ErrorWithCode(c, common.CodeServerError, err.Error())
}
return
}
common.SuccessWithData(c, gin.H{"deleted": n}, "success")

View File

@@ -157,11 +157,31 @@ func (c *KnowledgeCompilerComponent) Invoke(ctx context.Context, db *gorm.DB, in
deps.TenantID = tenantID
deps.DatasetID = datasetID
// Per-spec Inputs copy: each spec must get its own VariantSpecific map,
// otherwise specIn.VariantSpecific below would mutate the shared map and
// let a later template inherit the previous template's parser_config
// (a template with an empty config would then run with the wrong parser
// behavior). Copy the map (and preserve an empty map when nil).
specIn := in
if specIn.VariantSpecific == nil {
specIn.VariantSpecific = map[string]any{}
specIn.VariantSpecific = make(map[string]any, len(in.VariantSpecific)+1)
for k, v := range in.VariantSpecific {
specIn.VariantSpecific[k] = v
}
// The template config (flat: kind/entity/relation/plan/…) is delivered to
// the structure and wiki variants under the "parser_config" key — the SAME
// key those variants read (structure.Run / wikiPipeline.mapBatch do
// VariantSpecific["parser_config"]). Storing it as "config" left the
// variants with a nil config, so InferType saw no "kind" and fell back to
// "list" (breaking timeline: its compile_kwd became "list" instead of
// "timeline", so dropIsolatedTimelineEntities never ran and the timeline
// rendered every entity isolated).
//
// Only overwrite when the template actually carries a config: an empty
// template config (e.g. a resolver stub) must not clobber a parser_config
// the caller already supplied on the inputs.
if len(spec.Config) > 0 {
specIn.VariantSpecific["parser_config"] = spec.Config
}
specIn.VariantSpecific["config"] = spec.Config
var o common.Outputs
switch variant {
@@ -564,23 +584,14 @@ func applyVariantColumns(doc *schema.ChunkDoc, p common.Product) error {
}
case common.VariantMindmap:
// Tree nodes: depth_int records the outline level.
if v, ok := metaInt(p.Meta, "level"); ok {
if err := doc.SetExtraValue("depth_int", v); err != nil {
return err
}
}
if v := metaString(p.Meta, "name"); v != "" {
if err := doc.SetExtraValue("title_kwd", v); err != nil {
return err
}
setTitleTokens(doc, v)
}
if v := metaStringSlice(p.Meta, "children"); len(v) > 0 {
if err := doc.SetExtraValue("children_kwd", v); err != nil {
return err
}
}
// Mindmap now emits entity/relation rows (plan §1.2) so it participates in
// dataset-level merge exactly like graph/timeline: each node is an entity,
// each parent→child edge is a relation. Reuse the shared structure-graph
// column contract (knowledge_graph_kwd + from/to_entity_kwd + name_kwd +
// entity_type_kwd + mention_count_int). The relation type lives in the
// content_with_weight payload ({"from","to","type"}), matching Python —
// NOT a dedicated relation_type_kwd column.
return applyStructureGraphColumns(doc, p, kind)
}
return nil

View File

@@ -388,21 +388,27 @@ func TestKnowledgeCompiler_Tree_EndToEnd(t *testing.T) {
func TestKnowledgeCompiler_Mindmap_EndToEnd(t *testing.T) {
installProseDeps(t)
// The proseChat reply is flat text; parseOutline still yields a root + the
// reply as a child node, so chunks are non-empty and parent-linked.
// The proseChat reply is flat text; ShapeTree yields a bare root (no parsed
// children). Mindmap now emits entity/relation rows (plan §1.2): every node
// is an entity, every parent→child edge a relation. With a flat reply there
// is at least one entity (the root) and it must carry name_kwd +
// knowledge_graph_kwd="entity" (the structure-graph storage contract).
chunks := runVariant(t, "mindmap", nil)
// Root chunk must have empty parent_kwd and kind "root".
var root map[string]any
entityCount := 0
for _, c := range chunks {
if kind, _ := c["kc_kind"].(string); kind == "root" {
root = c
kind, _ := c["kc_kind"].(string)
if kind == "entity" {
entityCount++
if _, ok := c["name_kwd"]; !ok {
t.Fatalf("mindmap entity chunk missing name_kwd: %+v", c)
}
if kg, _ := c["knowledge_graph_kwd"].(string); kg != "entity" {
t.Fatalf("mindmap entity chunk knowledge_graph_kwd = %q, want entity", kg)
}
}
}
if root == nil {
t.Fatalf("mindmap: no root chunk")
}
if pid, _ := root["parent_kwd"].(string); pid != "" {
t.Fatalf("mindmap: root parent_kwd = %q, want empty", pid)
if entityCount == 0 {
t.Fatalf("mindmap: no entity chunks; got %d chunks", len(chunks))
}
}

View File

@@ -12,7 +12,6 @@ package mindmap
import (
"context"
"encoding/json"
"fmt"
"strings"
@@ -132,38 +131,47 @@ func Run(ctx context.Context, deps common.Deps, param common.Param, inputs commo
return out, nil
}
// treeToProducts flattens the shaped mind-map tree into Products. The root
// becomes a "root" product whose content is the serialized {"id","children"}
// tree (Python's MindMapResult.output shape); each inner node becomes a
// "node" product linked via parent_id.
// treeToProducts flattens the shaped mind-map tree into entity/relation Products
// so mindmap participates in dataset-level merge exactly like graph (plan §1.2,
// aligning with Python's dataset_structure_merger which merges
// knowledge_graph_kwd IN {entity,relation} rows for structure_mindmap too).
//
// NOTE: the per-node "node" products (content = the node title, plus a
// meta["children"] list of immediate child titles) are a Go component-layer
// contract for streaming the tree into the upstream chunk list. They go
// beyond Python's nested dict output and exist so downstream code can index
// each node independently with parent links.
// Mapping:
// - each node (including the root) → an entity product (kind="entity",
// name = node id, type = "mindmap").
// - each parent→child edge → a relation product (kind="relation",
// from = parent id, to = child id, type = "related" — Python's default).
//
// The entity/relation discriminator is carried in Meta["kind"] so the consumer's
// mergeStructureDataset buckets entities by (name,type) and relations by
// (from,type,to), matching graph/timeline.
func treeToProducts(tenantID, docID string, root *utility.Node) []common.Product {
var out []common.Product
rootID := common.StableRowID(tenantID, docID, string(common.VariantMindmap), "root")
if root == nil || root.ID == "" {
return out
}
seen := map[string]bool{}
// Entity: root node.
out = append(out, common.Product{
ID: rootID,
ID: common.StableRowID(tenantID, docID, string(common.VariantMindmap), "entity", root.ID),
DocID: docID,
TenantID: tenantID,
Variant: common.VariantMindmap,
Content: serializeNode(root),
Content: root.ID,
Meta: map[string]any{
"kind": "root",
"level": 0,
"name": root.ID,
"kind": "entity",
"name": root.ID,
"entity_type": "mindmap",
"compile_kwd": "mindmap",
},
})
seen[root.ID] = true
type pending struct {
node *utility.Node
parentID string
level int
node *utility.Node
parent string
}
queue := []pending{{root, rootID, 0}}
queue := []pending{{root, root.ID}}
for len(queue) > 0 {
p := queue[0]
queue = queue[1:]
@@ -171,64 +179,45 @@ func treeToProducts(tenantID, docID string, root *utility.Node) []common.Product
if child.ID == "" {
continue
}
level := p.level + 1
id := common.StableRowID(tenantID, docID, string(common.VariantMindmap), "node", child.ID)
childTitles := make([]string, 0, len(child.Children))
for _, gc := range child.Children {
if gc.ID != "" {
childTitles = append(childTitles, gc.ID)
}
}
meta := map[string]any{
"kind": "node",
"level": level,
"name": child.ID,
}
if len(childTitles) > 0 {
meta["children"] = childTitles
// Entity: child node (dedup by id so a DAG-shaped tree does not emit
// the same node twice).
if !seen[child.ID] {
seen[child.ID] = true
out = append(out, common.Product{
ID: common.StableRowID(tenantID, docID, string(common.VariantMindmap), "entity", child.ID),
DocID: docID,
TenantID: tenantID,
Variant: common.VariantMindmap,
Content: child.ID,
Meta: map[string]any{
"kind": "entity",
"name": child.ID,
"entity_type": "mindmap",
"compile_kwd": "mindmap",
},
})
}
// Relation: parent → child edge (type = "related", Python default).
out = append(out, common.Product{
ID: id,
ID: common.StableRowID(tenantID, docID, string(common.VariantMindmap), "relation", p.parent, child.ID),
DocID: docID,
TenantID: tenantID,
Variant: common.VariantMindmap,
Content: child.ID,
ParentID: p.parentID,
Meta: meta,
Content: p.parent + " related " + child.ID,
Meta: map[string]any{
"kind": "relation",
"from": p.parent,
"to": child.ID,
"relation_type": "related",
"compile_kwd": "mindmap",
},
})
queue = append(queue, pending{child, id, level})
queue = append(queue, pending{child, child.ID})
}
}
return out
}
// jsonNode mirrors the {"id","children"} mind-map node shape (Python's
// MindMapResult.output) for JSON serialization.
type jsonNode struct {
ID string `json:"id"`
Children []*jsonNode `json:"children"`
}
func toJSONNode(n *utility.Node) *jsonNode {
jn := &jsonNode{ID: n.ID}
for _, c := range n.Children {
jn.Children = append(jn.Children, toJSONNode(c))
}
return jn
}
// serializeNode renders the tree in Python's MindMapResult.output shape:
// {"id": ..., "children": [...]}. Uses encoding/json so every control
// character (including \r, \b, \f, etc.) is escaped correctly; the previous
// hand-rolled serializer only escaped ", \, \n and \t.
func serializeNode(node *utility.Node) string {
b, err := json.Marshal(toJSONNode(node))
if err != nil {
return `{"id":"` + node.ID + `","children":[]}`
}
return string(b)
}
func chunkTexts(chunks []common.Chunk) []string {
var out []string
for _, c := range chunks {

View File

@@ -1,7 +1,6 @@
package mindmap
import (
"encoding/json"
"strings"
"testing"
@@ -48,57 +47,50 @@ func TestTreeToProducts_ParentLinks(t *testing.T) {
{ID: "B"},
}}
products := treeToProducts("t1", "d1", root)
if len(products) != 5 {
t.Fatalf("products = %d, want 5 (root+A+A1+A2+B)", len(products))
// 5 entities (root, A, A1, A2, B) + 4 relations (root→A, A→A1, A→A2, root→B).
if len(products) != 9 {
t.Fatalf("products = %d, want 9 (5 entities + 4 relations)", len(products))
}
if products[0].Meta["kind"] != "root" || products[0].ParentID != "" {
t.Errorf("root product malformed: %+v", products[0].Meta)
}
if products[1].ParentID != products[0].ID {
t.Errorf("A parent link = %q, want root id", products[1].ParentID)
}
var aID string
entCount, relCount := 0, 0
fromTo := map[string]bool{}
for _, p := range products {
if p.Meta["name"] == "A" {
aID = p.ID
}
}
for _, p := range products {
if n, _ := p.Meta["name"].(string); n == "A1" || n == "A2" {
if p.ParentID != aID {
t.Errorf("%s parent = %q, want A id", n, p.ParentID)
kind, _ := p.Meta["kind"].(string)
switch kind {
case "entity":
entCount++
if p.Meta["entity_type"] != "mindmap" {
t.Errorf("entity %v type = %v, want mindmap", p.Meta["name"], p.Meta["entity_type"])
}
if p.Meta["compile_kwd"] != "mindmap" {
t.Errorf("entity %v compile_kwd = %v, want mindmap", p.Meta["name"], p.Meta["compile_kwd"])
}
case "relation":
relCount++
from, _ := p.Meta["from"].(string)
to, _ := p.Meta["to"].(string)
fromTo[from+"->"+to] = true
if p.Meta["relation_type"] != "related" {
t.Errorf("relation %v->%v type = %v, want related", from, to, p.Meta["relation_type"])
}
default:
t.Errorf("unexpected kind %q", kind)
}
}
if entCount != 5 || relCount != 4 {
t.Errorf("entities=%d relations=%d, want 5/4", entCount, relCount)
}
for _, edge := range []string{"root->A", "A->A1", "A->A2", "root->B"} {
if !fromTo[edge] {
t.Errorf("missing relation %s", edge)
}
}
}
func TestSerializeNode_Shape(t *testing.T) {
root := &utility.Node{ID: "Top \"quoted\"", Children: []*utility.Node{{ID: "child"}}}
js := serializeNode(root)
if !strings.HasPrefix(js, `{"id":"Top \"quoted\""`) {
t.Errorf("serializeNode = %q", js)
}
}
// TestSerializeNode_EscapesControlChars locks that serializeNode uses
// encoding/json: control characters such as CR/LF/TAB/FF are escaped (not
// embedded raw), so the output is always valid JSON.
func TestSerializeNode_EscapesControlChars(t *testing.T) {
root := &utility.Node{ID: "a\rb\tc\fd", Children: []*utility.Node{{ID: "x\ny"}}}
js := serializeNode(root)
var dec struct {
ID string `json:"id"`
Children []struct {
ID string `json:"id"`
} `json:"children"`
}
if err := json.Unmarshal([]byte(js), &dec); err != nil {
t.Fatalf("serializeNode produced invalid JSON: %v (%q)", err, js)
}
if dec.ID != "a\rb\tc\fd" {
t.Errorf("root id round-trip = %q, want %q", dec.ID, "a\rb\tc\fd")
}
if len(dec.Children) != 1 || dec.Children[0].ID != "x\ny" {
t.Errorf("child id round-trip = %+v", dec.Children)
func TestTreeToProducts_EmptyAndNil(t *testing.T) {
if got := treeToProducts("t1", "d1", nil); len(got) != 0 {
t.Errorf("nil root produced %d products", len(got))
}
if got := treeToProducts("t1", "d1", &utility.Node{ID: ""}); len(got) != 0 {
t.Errorf("empty root id produced %d products", len(got))
}
}

View File

@@ -1026,17 +1026,40 @@ func pageIndexSummary(graphJSON string) string {
// dataset_structure_merger._merge_bucket but is a self-contained from-scratch
// path (B3: the legacy unified merge never aggregated structure by name/type).
func (c *Consumer) mergeStructureDataset(ctx context.Context, tenant, kb string, products []kccommon.Product) error {
// Bucket by (lower(name), type) for entities and (lower(from), lower(to))
// for relations — relations have no "name" and must not be silently dropped
// (review issue 3).
type ekey struct{ name, typ string }
type rkey struct{ from, to string }
common.Info("knowledge_compile: mergeStructureDataset entry",
zap.String("kb_id", kb),
zap.Int("products", len(products)))
// Bucket by (lower(name), type, compile kind) for entities and
// (lower(from), lower(type), lower(to), compile kind) for relations —
// relations have no "name" and must not be silently dropped (review issue 3),
// and the relation type + compile kind keep distinct structure kinds and
// distinct relation types from collapsing into one dataset row.
type ekey struct{ name, typ, ckwd string }
type rkey struct{ from, typ, to, ckwd string }
entByKey := make(map[ekey]*StructureBucket, 16)
relByKey := make(map[rkey]*StructureBucket, 16)
// The authoritative Python (dataset_structure_merger._do_build) merges EVERY
// structure-kind doc row unconditionally — dataset scope is driven by the
// task type (structure_graph/timeline/structure_mindmap), NOT by a template
// Config["dataset_merge"] flag (that was a stale runner.py concept). So
// structure/mindmap entity/relation products always enter the dataset merge.
for _, p := range products {
if p.Variant != kccommon.VariantStructure {
if p.Variant != kccommon.VariantStructure && p.Variant != kccommon.VariantMindmap {
continue
}
// Dataset rows carry the SAME compile_kwd as the doc rows (the inferred
// compile type / autotype: "hypergraph"/"timeline"/"mindmap"/"list"/…),
// mirroring Python dataset_structure_merger._do_build which passes the doc
// row's compile_kwd through verbatim. The template kind (p.Kind, restored
// from compilation_template_kind_kwd) is stored on the SEPARATE
// compilation_template_kind_kwd field and is what read/delete paths match
// on — NOT compile_kwd. Do not rewrite compile_kwd to the template kind:
// that would diverge from Python (which never does) and split doc vs
// dataset rows.
ckwd := metaString(p.Meta, "compile_kwd")
if ckwd == "" {
ckwd = compileKwdForVariant(p.Variant)
}
kind := metaString(p.Meta, "kind")
from := metaString(p.Meta, "from")
to := metaString(p.Meta, "to")
@@ -1047,10 +1070,17 @@ func (c *Consumer) mergeStructureDataset(ctx context.Context, tenant, kb string,
if from == "" || to == "" {
continue
}
k := rkey{from: strings.ToLower(from), to: strings.ToLower(to)}
relType := metaString(p.Meta, "relation_type")
if relType == "" {
relType = metaString(p.Meta, "type")
}
if relType == "" {
relType = "related"
}
k := rkey{from: strings.ToLower(from), typ: strings.ToLower(relType), to: strings.ToLower(to), ckwd: ckwd}
b := relByKey[k]
if b == nil {
b = &StructureBucket{Name: from + " -> " + to, Type: "relation", FromEntity: from, ToEntity: to}
b = &StructureBucket{Name: from + " -> " + to, Type: "relation", FromEntity: from, ToEntity: to, CompileKwd: ckwd, TemplateID: p.TemplateID, TemplateKind: p.Kind, RelationType: relType}
relByKey[k] = b
}
appendBucket(b, p)
@@ -1065,10 +1095,10 @@ func (c *Consumer) mergeStructureDataset(ctx context.Context, tenant, kb string,
if name == "" {
continue
}
k := ekey{name: strings.ToLower(name), typ: typ}
k := ekey{name: strings.ToLower(name), typ: typ, ckwd: ckwd}
b := entByKey[k]
if b == nil {
b = &StructureBucket{Name: name, Type: typ}
b = &StructureBucket{Name: name, Type: typ, CompileKwd: ckwd, TemplateID: p.TemplateID, TemplateKind: p.Kind}
entByKey[k] = b
}
appendBucket(b, p)
@@ -1080,6 +1110,11 @@ func (c *Consumer) mergeStructureDataset(ctx context.Context, tenant, kb string,
for _, b := range relByKey {
buckets = append(buckets, *b)
}
common.Info("knowledge_compile: mergeStructureDataset buckets",
zap.String("kb_id", kb),
zap.Int("entities", len(entByKey)),
zap.Int("relations", len(relByKey)),
zap.Int("total_buckets", len(buckets)))
if len(buckets) == 0 {
return nil
}
@@ -1097,14 +1132,28 @@ func (c *Consumer) mergeStructureDataset(ctx context.Context, tenant, kb string,
// element-wise running mean (review issue 13). VecCount tracks how many vectors
// have been folded so the mean is order-independent.
func appendBucket(b *StructureBucket, p kccommon.Product) {
if strings.TrimSpace(p.Content) != "" {
// The product's Content is the doc row's full content_with_weight JSON
// (e.g. {"category":"Person","description":"…","name":"…","type":"entity"}),
// NOT the plain-text description. Extract the "description" field so the
// dataset row's description stays a plain-text string (mirror Python
// _struct_merge_graph_entities, which folds the entity description, not the
// whole payload). Fall back to Content only when it is not a JSON object.
if desc := structureProductDescription(p.Content); desc != "" {
if b.Description != "" {
b.Description += "\n"
}
b.Description += strings.TrimSpace(p.Content)
b.Description += desc
}
b.SourceDocIDs = appendUnique(b.SourceDocIDs, []string{p.DocID})
b.SourceChunkIDs = appendUnique(b.SourceChunkIDs, metaStringSlice(p.Meta, "source_chunk_ids"))
// mention_count_int: sum the per-entity mention counts (Python
// _struct_merge_graph_entities sums mention_count across the merged entities).
// Each entity product carries mention_count ≥ 1 (structure compile.go L231);
// mindmap entities carry none, so they contribute a default of 0 and are
// simply not stamped (mention_count_int is omitted when 0).
if mc, ok := metaInt(p.Meta, "mention_count"); ok && mc > 0 {
b.MentionCount += int(mc)
}
if len(p.Vector) > 0 {
b.VecCount++
if len(b.Vector) == 0 {
@@ -1122,6 +1171,33 @@ func appendBucket(b *StructureBucket, p kccommon.Product) {
}
}
// structureProductDescription extracts the plain-text description from a structure
// product's Content. In production Content is the doc row's content_with_weight
// JSON ({"description":"…","name":"…","type":"…"}), so we parse it and return the
// "description" field — the whole payload must NOT leak into the dataset row's
// description (that is the bug: projectEntity then renders the raw JSON object).
// When Content is NOT a JSON object (plain text, legacy/unit-test rows), it is
// returned verbatim so existing folded-description behavior is preserved.
func structureProductDescription(content string) string {
content = strings.TrimSpace(content)
if content == "" {
return ""
}
if content[0] == '{' {
var m map[string]any
if err := json.Unmarshal([]byte(content), &m); err == nil {
if d, ok := m["description"].(string); ok {
if s := strings.TrimSpace(d); s != "" {
return s
}
}
// JSON object without a plain-text description: nothing to fold.
return ""
}
}
return content
}
// appendUnique appends only values not already present, preserving order.
func appendUnique(dst, values []string) []string {
seen := make(map[string]bool, len(dst)+len(values))

View File

@@ -79,12 +79,16 @@ var compiledSelectFields = []string{
"name_kwd", "entity_type_kwd", "from_entity_kwd", "to_entity_kwd",
"slug_kwd", "type",
"kc_kind", "create_timestamp_flt", "create_time",
"compilation_template_kind_kwd",
"compilation_template_kind_kwd", "compilation_template_ids",
// The product kind discriminator for structure/tree: the component stores it
// under knowledge_graph_kwd (structure: graph/entity/relation) / raptor_kwd
// (tree: root/summary). Without these the reader cannot restore Meta["kind"]
// and the dataset-nav dispatch would skip structure/tree products (B2).
"knowledge_graph_kwd", "raptor_kwd",
// mention_count_int round-trips the entity mention count for reprojection.
// (relation type lives in the content_with_weight payload, matching Python —
// there is NO relation_type_kwd column.)
"mention_count_int",
}
// wikiSelectFields are the additional columns a wiki page carries (beyond
@@ -208,9 +212,21 @@ func productFromChunkMap(c map[string]interface{}, tenant string, expect kccommo
if err != nil || mapped != expect {
return kccommon.Product{}, false
}
// Round-trip the raw compile_kwd (the inferred compile type / autotype:
// list/set/hypergraph/timeline/mindmap/…) so the dataset-level merge can
// stamp the SAME value on the dataset row as the doc row (Python _do_build
// carries the doc row's compile_kwd through verbatim). Without this, the
// merge falls back to compileKwdForVariant ("structure"/"mindmap"), which
// diverges from the doc row's autotype ("hypergraph"/"timeline").
merged := isAvailable(c["available_int"])
meta := map[string]any{}
// Preserve the raw compile_kwd (autotype) for the dataset merge (see the
// round-trip note above). A non-string scalar from the engine is normalized
// via asString, matching the variant reverse-map at the top of this func.
if v := asString(c["compile_kwd"]); v != "" {
meta["compile_kwd"] = v
}
if v, ok := c["name_kwd"].(string); ok && v != "" {
meta["name"] = v
}
@@ -225,6 +241,9 @@ func productFromChunkMap(c map[string]interface{}, tenant string, expect kccommo
meta["to"] = v
meta["kind"] = "relation"
}
if v, ok := metaInt(c, "mention_count_int"); ok {
meta["mention_count"] = v
}
if v, ok := c["slug_kwd"].(string); ok && v != "" {
// slug_kwd is the full "<page_type>/<slug>" form (Python writer
// contract); reconstruct it verbatim so the round-trip stays full-form.
@@ -310,16 +329,26 @@ func productFromChunkMap(c map[string]interface{}, tenant string, expect kccommo
// so callers (e.g. RebuildDataset's variant recovery, B1a) can map it back
// via KindToVariant instead of re-deriving from the ambiguous compile_kwd.
kind := asString(c["compilation_template_kind_kwd"])
// Restore the compilation template id (from compilation_template_ids) so the
// dataset merge can bucket structure rows per template and read/delete paths
// can filter by it. A row should carry exactly one template id; if it carries
// more than one the first is used (multi-template rows are a config error
// surfaced elsewhere).
templateID := ""
if ids := metaStringSlice(c, "compilation_template_ids"); len(ids) > 0 {
templateID = ids[0]
}
return kccommon.Product{
ID: id,
DocID: docID,
TenantID: tenant,
Variant: expect,
Kind: kind,
Content: content,
Vector: vec,
Meta: meta,
Merged: merged,
ID: id,
DocID: docID,
TenantID: tenant,
Variant: expect,
Kind: kind,
TemplateID: templateID,
Content: content,
Vector: vec,
Meta: meta,
Merged: merged,
}, true
}

View File

@@ -114,17 +114,102 @@ func TestMergeStructureDataset_RelationsNotDropped(t *testing.T) {
}
}
// TestMergeStructureDataset_DescriptionIsPlainText covers the description bug:
// the product Content is the doc row's content_with_weight JSON, so the dataset
// row's folded description must be the plain-text "description" field, NOT the raw
// JSON object (which would leak the whole entity payload into the rendered graph).
func TestMergeStructureDataset_DescriptionIsPlainText(t *testing.T) {
c := &Consumer{writer: &fakeWriter{}}
products := []kccommon.Product{
{Variant: kccommon.VariantStructure, DocID: "d1",
Content: `{"category":"Person","description":"汉桓帝,禁锢善类","name":"桓帝","type":"entity"}`,
Meta: map[string]any{"name": "桓帝", "entity_type": "Person", "compile_kwd": "list", "source_chunk_ids": []string{"c1"}}},
}
if err := c.mergeStructureDataset(context.Background(), "t1", "kb1", products); err != nil {
t.Fatalf("mergeStructureDataset: %v", err)
}
fw := c.writer.(*fakeWriter)
if len(fw.buckets) != 1 {
t.Fatalf("want 1 bucket, got %d", len(fw.buckets))
}
got := fw.buckets[0].Description
if got != "汉桓帝,禁锢善类" {
t.Errorf("description = %q, want plain-text \"汉桓帝,禁锢善类\" (not the raw JSON payload)", got)
}
if strings.Contains(got, "{") || strings.Contains(got, "category") {
t.Errorf("description leaked the JSON payload: %q", got)
}
}
// TestMergeStructureDataset_CompileKwdIsAutotypeNotTemplateKind covers the
// option-A alignment: the dataset row's compile_kwd is the doc row's inferred
// compile type (autotype, e.g. "hypergraph"/"mindmap"), NOT the template kind.
// The template kind travels on the separate TemplateKind field (stamped to
// compilation_template_kind_kwd by WriteMergedStructure), which is what read/
// delete paths match on. This mirrors Python _do_build (compile_kwd passes
// through verbatim) + get_dataset_structure (matches template kind).
func TestMergeStructureDataset_CompileKwdIsAutotypeNotTemplateKind(t *testing.T) {
c := &Consumer{writer: &fakeWriter{}}
products := []kccommon.Product{
{Variant: kccommon.VariantStructure, DocID: "d1", Kind: "knowledge_graph", TemplateID: "tpl-graph",
Content: "Engine desc",
Meta: map[string]any{"name": "Engine", "entity_type": "component", "compile_kwd": "hypergraph", "source_chunk_ids": []string{"c1"}}},
{Variant: kccommon.VariantMindmap, DocID: "d2", Kind: "mind_map", TemplateID: "tpl-mindmap",
Content: "Fuel desc",
Meta: map[string]any{"name": "Fuel", "entity_type": "substance", "compile_kwd": "mindmap", "source_chunk_ids": []string{"c2"}}},
}
if err := c.mergeStructureDataset(context.Background(), "t1", "kb1", products); err != nil {
t.Fatalf("mergeStructureDataset: %v", err)
}
fw := c.writer.(*fakeWriter)
if len(fw.buckets) != 2 {
t.Fatalf("want 2 buckets, got %d", len(fw.buckets))
}
for i := range fw.buckets {
b := fw.buckets[i]
switch b.Name {
case "Engine":
if b.CompileKwd != "hypergraph" {
t.Errorf("Engine compile_kwd = %q, want autotype \"hypergraph\" (not template kind)", b.CompileKwd)
}
if b.TemplateKind != "knowledge_graph" {
t.Errorf("Engine TemplateKind = %q, want \"knowledge_graph\"", b.TemplateKind)
}
if b.TemplateID != "tpl-graph" {
t.Errorf("Engine TemplateID = %q, want \"tpl-graph\"", b.TemplateID)
}
case "Fuel":
if b.CompileKwd != "mindmap" {
t.Errorf("Fuel compile_kwd = %q, want autotype \"mindmap\" (not template kind)", b.CompileKwd)
}
if b.TemplateKind != "mind_map" {
t.Errorf("Fuel TemplateKind = %q, want \"mind_map\"", b.TemplateKind)
}
if b.TemplateID != "tpl-mindmap" {
t.Errorf("Fuel TemplateID = %q, want \"tpl-mindmap\"", b.TemplateID)
}
}
}
}
// TestDatasetLevelStructureID_StableAndCaseInsensitive covers G1: the id is
// deterministic and case-insensitive on name so merges hit the same row.
func TestDatasetLevelStructureID_StableAndCaseInsensitive(t *testing.T) {
a := datasetLevelStructureID("t1", "kb1", "Engine", "component")
b := datasetLevelStructureID("t1", "kb1", "engine", "component")
a := datasetLevelStructureID("t1", "kb1", "Engine", "component", "timeline", "")
b := datasetLevelStructureID("t1", "kb1", "engine", "component", "timeline", "")
if a != b {
t.Errorf("structure id should be case-insensitive on name: %q vs %q", a, b)
}
if a == datasetLevelStructureID("t1", "kb1", "Fuel", "component") {
if a == datasetLevelStructureID("t1", "kb1", "Fuel", "component", "timeline", "") {
t.Errorf("different names must yield different ids")
}
if a == datasetLevelStructureID("t1", "kb1", "Engine", "component", "mindmap", "") {
t.Errorf("different compile kinds must yield different ids")
}
if datasetLevelStructureID("t1", "kb1", "A -> B", "relation", "graph", "causes") ==
datasetLevelStructureID("t1", "kb1", "A -> B", "relation", "graph", "contradicts") {
t.Errorf("different relation types between the same endpoints must yield different ids")
}
}
// TestKwdToVariant_MapsStructureSubKinds covers B1a: structure doc-level products
@@ -182,6 +267,11 @@ func TestProductFromChunkMap_RestoresStructureTreeKind(t *testing.T) {
if kind, _ := sp.Meta["kind"].(string); kind != "graph" {
t.Errorf("structure Meta.kind = %q, want graph (so nav dispatch can pick it)", kind)
}
// Round-trip: the raw compile_kwd (autotype) must be preserved in Meta so the
// dataset merge stamps the SAME value on the dataset row as the doc row.
if ckwd, _ := sp.Meta["compile_kwd"].(string); ckwd != "list" {
t.Errorf("structure Meta.compile_kwd = %q, want autotype \"list\" (not \"structure\")", ckwd)
}
// tree root product: kind stored in raptor_kwd.
treeRow := map[string]interface{}{

View File

@@ -101,6 +101,28 @@ type StructureBucket struct {
VecCount int // number of vectors folded into Vector (for true mean)
FromEntity string // relation only
ToEntity string // relation only
// CompileKwd is the raw compile keyword (the inferred compile type / autotype,
// e.g. "hypergraph", "timeline", "mindmap", "list") that produced this bucket.
// It is stamped on the stored row's compile_kwd so distinct structure kinds
// never collide in the same dataset namespace (plan §1.1). It matches the doc
// row's own compile_kwd — Python _do_build carries the doc row's compile_kwd
// verbatim onto the dataset row, it does NOT rewrite it to the template kind.
CompileKwd string
// TemplateID is the compilation template id (compilation_template_ids[0]) that
// produced this bucket. Stamped on the dataset row so read/delete paths can
// filter per template (mirror Python get_dataset_structure).
TemplateID string
// TemplateKind is the authoritative template kind (compilation_template_kind_kwd,
// e.g. "knowledge_graph", "mind_map", "timeline"). This — NOT compile_kwd — is
// the field read/delete paths match on for the dataset-structure kind
// (mirror Python get_dataset_structure._discover_scope_templates, which
// matches _resolve_dataset_structure_kind against compilation_template_kind_kwd).
TemplateKind string
// RelationType is the relation type (Python default "related"), persisted as
// relation_type_kwd and part of the relation bucket/ID identity.
RelationType string
// MentionCount is the entity mention count (mention_count_int); 0 when unset.
MentionCount int
}
// engineWriter persists dataset-level merged products through the global
@@ -111,6 +133,33 @@ type engineWriter struct {
eng engine.DocEngine
}
// datasetStructureSupported reports whether the running doc engine can filter
// the dataset-structure fields (knowledge_graph_kwd + scope_kwd + raw
// compile_kwd) this feature writes/deletes. Only infinity and elasticsearch
// support them today; OceanBase/SeekDB/SereneDB have explicit schemas that lack
// these filter keys and would reject the query or silently drop unknown fields.
// The guard is checked at every dataset-structure write/delete/rebuild entry so
// those engines never leave partial state or hit an unknown-filter error (plan §5).
func datasetStructureSupported() bool {
switch engine.GetEngineType() {
case "infinity", "elasticsearch":
return true
case "":
// Engine not initialized (unit tests inject a fake engine directly via
// engineWriter.eng). The guard is only meaningful against a real engine
// type, so an empty type is treated as supported.
return true
default:
return false
}
}
// errDatasetStructureUnsupported returns an error (not a silent no-op) naming
// the engine when a dataset-structure operation runs on an unsupported engine.
func errDatasetStructureUnsupported() error {
return fmt.Errorf("dataset structure graph unsupported on doc engine %q", engine.GetEngineType())
}
// writeMergedBatchSize bounds how many rows each parallel InsertChunks call
// carries, so the DocEngine write fan-out stays granular under the shared pool.
const writeMergedBatchSize = 200
@@ -202,13 +251,18 @@ func (w engineWriter) WriteMerged(ctx context.Context, tenant, kb string, produc
// WriteMergedStructure writes the dataset-level structure merged rows for a KB
// (G1/G4). Each StructureBucket is a scope_kwd="dataset" row with a stable
// dataset-level id keyed on (name, type), the folded description, the union of
// source docs/chunks, and the bucket vector. Rows are available_int=1 so the
// dataset-level structure index is searchable, and compile_kwd="structure".
// dataset-level id keyed on (name, type, raw compile kind), the folded
// description, the union of source docs/chunks, and the bucket vector. Rows are
// available_int=1 so the dataset-level structure index is searchable; the raw
// compile kind (timeline/graph/mindmap) is stamped on compile_kwd and the
// entity/relation discriminator on knowledge_graph_kwd.
func (w engineWriter) WriteMergedStructure(ctx context.Context, tenant, kb string, buckets []StructureBucket) error {
if len(buckets) == 0 {
return nil
}
if !datasetStructureSupported() {
return errDatasetStructureUnsupported()
}
eng := w.eng
if eng == nil {
eng = engine.Get()
@@ -229,7 +283,7 @@ func (w engineWriter) WriteMergedStructure(ctx context.Context, tenant, kb strin
if b.Name == "" {
continue
}
ids = append(ids, datasetLevelStructureID(tenant, kb, b.Name, b.Type))
ids = append(ids, datasetLevelStructureID(tenant, kb, b.Name, b.Type, b.CompileKwd, b.RelationType))
}
if len(ids) > 0 {
res, err := eng.Search(ctx, &types.SearchRequest{
@@ -263,43 +317,128 @@ func (w engineWriter) WriteMergedStructure(ctx context.Context, tenant, kb strin
if b.Name == "" {
continue
}
bid := datasetLevelStructureID(tenant, kb, b.Name, b.Type)
bid := datasetLevelStructureID(tenant, kb, b.Name, b.Type, b.CompileKwd, b.RelationType)
// Union the current batch's sources with any already-accumulated ones.
if prev, ok := existing[bid]; ok {
b.SourceDocIDs = appendUnique(b.SourceDocIDs, prev.SourceDocIDs)
b.SourceChunkIDs = appendUnique(b.SourceChunkIDs, prev.SourceChunkIDs)
}
ckwd := b.CompileKwd
if ckwd == "" {
ckwd = compileKwdStructure
}
// content_with_weight is a JSON payload, matching Python (and the existing
// wiki projection writer below): the structured fields (from/to/type for
// relations, name/type for entities) live INSIDE the payload, while
// from/to are also copied to from_entity_kwd/to_entity_kwd top-level
// columns for filtering. There is NO relation_type_kwd column — the
// relation type is carried only in the payload, exactly as Python does.
var payloadMap map[string]any
if b.FromEntity != "" || b.ToEntity != "" {
relType := b.RelationType
if relType == "" {
relType = "related"
}
payloadMap = map[string]any{
"from": b.FromEntity,
"to": b.ToEntity,
"type": relType,
"description": desc,
}
} else {
typ := b.Type
if typ == "" {
typ = "other"
}
payloadMap = map[string]any{
"name": b.Name,
"type": typ,
"description": desc,
}
}
payloadBytes, err := json.Marshal(payloadMap)
if err != nil {
return err
}
payload := string(payloadBytes)
row := map[string]interface{}{
// Stable dataset-level id keyed on the (name, type) or (from, to) bucket.
// Stable dataset-level id keyed on the (name, type) or (from, to)
// bucket plus the raw compile kind.
"id": bid,
"doc_id": kb,
"tenant_id": tenant,
"kb_id": kb,
"available_int": 1,
"compile_kwd": compileKwdStructure,
"compile_kwd": ckwd,
"scope_kwd": "dataset",
"content_with_weight": desc,
"kc_payload": desc,
"content_with_weight": payload,
"kc_payload": payload,
"source_doc_ids": b.SourceDocIDs,
"source_chunk_ids": b.SourceChunkIDs,
"create_time": now.Format("2006-01-02 15:04:05"),
"create_timestamp_flt": float64(now.Unix()),
}
// Stamp the authoritative template identity so read/delete paths can match
// the dataset-structure kind by compilation_template_kind_kwd (mirror Python
// get_dataset_structure._discover_scope_templates), independent of the
// autotype-valued compile_kwd above.
if b.TemplateKind != "" {
row["compilation_template_kind_kwd"] = b.TemplateKind
}
if b.TemplateID != "" {
row["compilation_template_ids"] = []string{b.TemplateID}
}
if b.FromEntity != "" || b.ToEntity != "" {
// relation row: carries from/to entities; kind=relation, no name_kwd.
row["knowledge_graph_kwd"] = "relation"
row["type_kwd"] = "relation"
row["from_entity_kwd"] = b.FromEntity
row["to_entity_kwd"] = b.ToEntity
} else {
row["knowledge_graph_kwd"] = "entity"
row["type_kwd"] = "entity"
row["name_kwd"] = b.Name
row["entity_type_kwd"] = b.Type
if b.MentionCount > 0 {
row["mention_count_int"] = b.MentionCount
}
}
if len(b.Vector) > 0 {
row["q_"+fmt.Sprintf("%d", len(b.Vector))+"_vec"] = f32ToF64Slice(b.Vector)
}
rows = append(rows, row)
}
// Write one kg_build_meta build-marker row per raw compile kind (the bucket
// model is keyed by raw kind; template-id granularity is a follow-up when the
// bucket key gains a template dimension). The marker is available_int=0 and
// carries create_timestamp_flt as its timestamp (build_timestamp_flt is a
// Python _META_ROW_KWD legacy field NOT present in the Infinity/ES/OS schemas,
// so writing it would fail with an undefined-column error — see review). It is
// a write/delete-side marker — GET discovery does NOT read it (it scans
// knowledge_graph_kwd=["entity"] rows instead, per §6).
seenKwd := map[string]bool{}
for _, b := range buckets {
ckwd := b.CompileKwd
if ckwd == "" {
ckwd = compileKwdStructure
}
if seenKwd[ckwd] {
continue
}
seenKwd[ckwd] = true
rows = append(rows, map[string]interface{}{
"id": "dataset_build_meta_" + hashStr(tenant+"\x00"+kb+"\x00"+ckwd),
"doc_id": kb,
"tenant_id": tenant,
"kb_id": kb,
"available_int": 0,
"compile_kwd": ckwd,
"scope_kwd": "dataset",
"knowledge_graph_kwd": "kg_build_meta",
"create_time": now.Format("2006-01-02 15:04:05"),
"create_timestamp_flt": float64(now.Unix()),
})
}
if len(rows) == 0 {
return nil
}
@@ -329,6 +468,9 @@ func (w engineWriter) DeleteStructureForDocs(ctx context.Context, tenant, kb str
if len(deletedDocIDs) == 0 {
return nil
}
if !datasetStructureSupported() {
return errDatasetStructureUnsupported()
}
eng := w.eng
if eng == nil {
eng = engine.Get()
@@ -346,14 +488,25 @@ func (w engineWriter) DeleteStructureForDocs(ctx context.Context, tenant, kb str
// surviving silently (review Minor).
const pageSize = 500
var ghostIDs []string
// Raw compile kinds whose entity/relation rows were declared ghosts; used to
// drop a now-empty kind's kg_build_meta build marker after cleanup.
affectedKinds := map[string]bool{}
for offset := 0; ; offset += pageSize {
res, err := eng.Search(ctx, &types.SearchRequest{
IndexNames: []string{baseName},
KbIDs: []string{kb},
SelectFields: []string{"id", "source_doc_ids"},
Filter: map[string]interface{}{"kb_id": kb, "scope_kwd": "dataset", "compile_kwd": compileKwdStructure},
Offset: offset,
Limit: pageSize,
SelectFields: []string{"id", "source_doc_ids", "compile_kwd"},
// Match dataset-scope entity/relation rows across ALL structure kinds
// (timeline/graph/session_graph/mindmap), which now each stamp their
// raw compile_kwd; knowledge_graph_kwd ∈ {entity,relation} + scope_kwd
// =dataset is the kind-agnostic predicate (plan §1, §4.2).
Filter: map[string]interface{}{
"kb_id": kb,
"scope_kwd": "dataset",
"knowledge_graph_kwd": []string{"entity", "relation"},
},
Offset: offset,
Limit: pageSize,
})
if err != nil {
return fmt.Errorf("structure ghost scan: %w", err)
@@ -379,6 +532,9 @@ func (w engineWriter) DeleteStructureForDocs(ctx context.Context, tenant, kb str
}
if allGone {
ghostIDs = append(ghostIDs, id)
if ckwd, _ := c["compile_kwd"].(string); ckwd != "" {
affectedKinds[ckwd] = true
}
}
}
if len(res.Chunks) < pageSize {
@@ -388,18 +544,51 @@ func (w engineWriter) DeleteStructureForDocs(ctx context.Context, tenant, kb str
if len(ghostIDs) == 0 {
return nil
}
_, err := eng.DeleteChunks(ctx, map[string]interface{}{"id": ghostIDs, "kb_id": kb}, baseName, kb)
if err != nil {
if _, err := eng.DeleteChunks(ctx, map[string]interface{}{"id": ghostIDs, "kb_id": kb}, baseName, kb); err != nil {
return fmt.Errorf("structure ghost cleanup: %w", err)
}
// Drop the kg_build_meta build marker for any raw kind that no longer has an
// entity/relation row (plan §3.1.1 step 4).
for ckwd := range affectedKinds {
res, err := eng.Search(ctx, &types.SearchRequest{
IndexNames: []string{baseName},
KbIDs: []string{kb},
SelectFields: []string{"id"},
Filter: map[string]interface{}{
"kb_id": kb,
"scope_kwd": "dataset",
"compile_kwd": ckwd,
"knowledge_graph_kwd": []string{"entity", "relation"},
},
Limit: 1,
})
if err != nil {
return fmt.Errorf("structure meta check (kind %s): %w", ckwd, err)
}
if len(res.Chunks) > 0 {
continue // still has entity/relation rows; keep the marker
}
if _, err := eng.DeleteChunks(ctx, map[string]interface{}{
"kb_id": kb,
"scope_kwd": "dataset",
"compile_kwd": ckwd,
"knowledge_graph_kwd": "kg_build_meta",
}, baseName, kb); err != nil {
return fmt.Errorf("structure meta cleanup (kind %s): %w", ckwd, err)
}
}
return nil
}
// datasetLevelStructureID builds the stable dataset-level id for a structure
// bucket, keyed on (name, type). It must be deterministic so an incremental
// merge read-modify-writes the same row (and a rebuild clean removes it).
func datasetLevelStructureID(tenant, kb, name, typ string) string {
return "dataset_structure_" + hashStr(tenant+"\x00"+kb+"\x00"+strings.ToLower(name)+"\x00"+typ)
// bucket, keyed on (name, type, compile kind, relation type). It must be
// deterministic so an incremental merge read-modify-writes the same row (and a
// rebuild clean removes it). Including the raw compile kind keeps
// timeline/graph/mindmap buckets from colliding in the same dataset namespace;
// including the relation type keeps two relation types between the same endpoints
// (e.g. "causes" vs "contradicts") from colliding into one id (review fix).
func datasetLevelStructureID(tenant, kb, name, typ, compileKwd, relationType string) string {
return "dataset_structure_" + hashStr(tenant+"\x00"+kb+"\x00"+strings.ToLower(name)+"\x00"+typ+"\x00"+compileKwd+"\x00"+strings.ToLower(relationType))
}
// f32ToF64Slice converts a float32 vector to float64 for the engine's dense
@@ -937,7 +1126,13 @@ func (w engineWriter) DeleteMergedForVariant(ctx context.Context, tenant, kb str
// Full-set clean (B1b): everything the consumer manages. Structure
// dataset rows are tagged scope_kwd="dataset" (not a fixed compile_kwd),
// so the full clean deletes by kb_id + compile_kwd IN (the fixed-kwd
// buckets) OR scope_kwd="dataset".
// buckets) OR scope_kwd="dataset". The scope_kwd="dataset" sweep requires
// the dataset-structure filter keys, so it is guarded like the per-variant
// structure branch (review fix: this is the highest-impact destructive
// path and must fail loudly on unsupported engines).
if !datasetStructureSupported() {
return errDatasetStructureUnsupported()
}
_, err := eng.DeleteChunks(ctx, map[string]interface{}{
"kb_id": kb,
"compile_kwd": []string{
@@ -979,17 +1174,23 @@ func (w engineWriter) DeleteMergedForVariant(ctx context.Context, tenant, kb str
}, baseName, kb); err != nil {
return fmt.Errorf("delete merged (wiki): %w", err)
}
case kccommon.VariantStructure:
// structure dataset rows are scope_kwd="dataset" + compile_kwd=
// "structure". The compile_kwd is required: wiki merged rows ALSO carry
// scope_kwd="dataset" (W5), so a scope-only sweep would wrongly delete
// wiki merged rows too (review Major).
case kccommon.VariantStructure, kccommon.VariantMindmap:
if !datasetStructureSupported() {
return errDatasetStructureUnsupported()
}
// structure/mindmap dataset rows are scope_kwd="dataset" +
// knowledge_graph_kwd ∈ {entity,relation,kg_build_meta} with a raw
// compile_kwd (timeline/graph/session_graph/mindmap). knowledge_graph_kwd
// is required: wiki merged rows ALSO carry scope_kwd="dataset" (W5), so
// a scope-only sweep would wrongly delete wiki merged rows too (review
// Major). kg_build_meta (the build marker) is deleted together with the
// entity/relation rows.
if _, err := eng.DeleteChunks(ctx, map[string]interface{}{
"kb_id": kb,
"scope_kwd": "dataset",
"compile_kwd": []string{compileKwdStructure},
"kb_id": kb,
"scope_kwd": "dataset",
"knowledge_graph_kwd": []string{"entity", "relation", "kg_build_meta"},
}, baseName, kb); err != nil {
return fmt.Errorf("delete merged (structure): %w", err)
return fmt.Errorf("delete merged (structure/mindmap): %w", err)
}
case kccommon.VariantTree:
// tree/nav rows are compile_kwd="dataset_nav" with available_int=0;
@@ -1000,8 +1201,6 @@ func (w engineWriter) DeleteMergedForVariant(ctx context.Context, tenant, kb str
}, baseName, kb); err != nil {
return fmt.Errorf("delete merged (nav): %w", err)
}
case kccommon.VariantMindmap:
// mindmap has no dataset-level merged rows in the consumer path
}
}
return nil

View File

@@ -39,16 +39,6 @@ const (
CompileKwdSkillAll = "skill_all"
CompileKwdDatasetNav = "dataset_nav"
CompileKwdRaptorGraph = "raptor_graph"
CompileKwdStructure = "structure"
CompileKwdStructureIndex = "structureIndex"
CompileKwdStructureEntity = "structureEntity"
CompileKwdStructureRelation = "structureRelation"
CompileKwdStructureCommunity = "structureCommunity"
FieldStructureIndexType = "structure_index_type"
FieldStructureKind = "structure_kind"
FieldPageID = "page_id"
)
// DatasetArtifactService reads knowledge-compilation artifacts (wiki pages,
@@ -550,75 +540,6 @@ func (s *DatasetArtifactService) ClearWiki(ctx context.Context, tenantID, datase
return deleted, nil
}
// StructureItem is a single compiled structure entry for a dataset.
type StructureItem struct {
PageID string `json:"page_id"`
StructureKind string `json:"structure_kind"`
StructureIndexType string `json:"structure_index_type"`
Data string `json:"data"`
}
// ListStructures returns the compiled structures of a dataset, filtered by
// optional structure_kind and structure_index_type.
func (s *DatasetArtifactService) ListStructures(ctx context.Context, tenantID, datasetID, structureKind, structureIndexType string) ([]StructureItem, int64, error) {
filter := map[string]interface{}{"compile_kwd": []string{CompileKwdStructure}}
if structureKind != "" {
filter[FieldStructureKind] = []string{structureKind}
}
if structureIndexType != "" {
filter[FieldStructureIndexType] = []string{structureIndexType}
}
chunks, total, err := s.searchCompiled(ctx, tenantID, datasetID, filter,
[]string{FieldPageID, FieldStructureKind, FieldStructureIndexType, "content_with_weight"}, 0, 10000, nil)
if err != nil {
return nil, 0, err
}
items := make([]StructureItem, 0, len(chunks))
for _, c := range chunks {
items = append(items, StructureItem{
PageID: firstStringValue(c[FieldPageID]),
StructureKind: firstStringValue(c[FieldStructureKind]),
StructureIndexType: firstStringValue(c[FieldStructureIndexType]),
Data: firstStringValue(c["content_with_weight"]),
})
}
return items, total, nil
}
// DeleteStructures deletes the compiled structures of a dataset, optionally
// scoped by structure_kind and structure_index_type.
func (s *DatasetArtifactService) DeleteStructures(ctx context.Context, tenantID, datasetID, structureKind, structureIndexType string) (int, error) {
docEngine := engine.Get()
if docEngine == nil {
return 0, fmt.Errorf("document engine is not initialized")
}
filter := map[string]interface{}{"compile_kwd": []string{CompileKwdStructure}}
if structureKind != "" {
filter[FieldStructureKind] = []string{structureKind}
}
if structureIndexType != "" {
filter[FieldStructureIndexType] = []string{structureIndexType}
}
chunks, _, err := s.searchCompiled(ctx, tenantID, datasetID, filter, []string{"id"}, 0, 10000, nil)
if err != nil {
return 0, err
}
if len(chunks) == 0 {
return 0, nil
}
ids := make([]string, 0, len(chunks))
for _, c := range chunks {
if id, ok := c["id"].(string); ok {
ids = append(ids, id)
}
}
cond := map[string]interface{}{"id": ids, "kb_id": datasetID}
if _, err := docEngine.DeleteChunks(ctx, cond, wikiIndexName(tenantID), datasetID); err != nil {
return 0, err
}
return len(ids), nil
}
// DeleteDocumentGraph deletes the structure graph of a single document.
func (s *DatasetArtifactService) DeleteDocumentGraph(ctx context.Context, tenantID, datasetID, documentID string) (int, error) {
docEngine := engine.Get()

View File

@@ -10,11 +10,13 @@ package service
import (
"context"
"encoding/json"
"errors"
"fmt"
"regexp"
"sort"
"strings"
"gorm.io/gorm"
"ragflow/internal/dao"
"ragflow/internal/engine"
"ragflow/internal/engine/types"
@@ -468,8 +470,12 @@ func (s *DatasetArtifactService) buildBucket(ctx context.Context, tenantID, data
if err != nil {
return nil, nil, err
}
var entities []StructureGraphNode
var relations []StructureGraphRelation
// Initialize as empty (non-nil) slices so a bucket with entities but no
// relations serializes "relations": [] instead of null — the frontend
// adapters (adaptKnowledgeGraphToForceGraph et al.) call .filter() on
// relations without a null guard (mirror Python, which always returns []).
entities := make([]StructureGraphNode, 0)
relations := make([]StructureGraphRelation, 0)
for _, row := range fieldMap {
if !rowHasEnabledSource(row, excludedDocIDs) {
continue
@@ -530,7 +536,7 @@ func (s *DatasetArtifactService) buildBucket(ctx context.Context, tenantID, data
}
aNameTerms = sortedUnique(aNameTerms)
var relations []StructureGraphRelation
relations := make([]StructureGraphRelation, 0)
targetNamesLower := map[string]bool{}
if len(aNameTerms) > 0 {
cond := copyFilter(scope)
@@ -613,6 +619,59 @@ func compilationTemplateKind(kind string) string {
return k
}
// ErrInvalidStructureKind is returned when the request kind does not resolve to a
// supported dataset-structure kind. Handlers use errors.Is to map it to
// CodeArgumentError (400) while treating every other service error as a server
// fault (500).
var ErrInvalidStructureKind = errors.New("invalid structure kind")
// datasetStructureSupported mirrors the writer's engine guard: only Infinity and
// Elasticsearch resolve the dataset-structure filter keys (scope_kwd +
// knowledge_graph_kwd). The writer keeps a private copy in the knowledge_compile
// package; this service-side copy keeps the API delete path in agreement without
// exporting an ingestion-internal helper across package boundaries.
func datasetStructureSupported() bool {
switch engine.GetEngineType() {
case "infinity", "elasticsearch", "":
return true
default:
return false
}
}
// errDatasetStructureUnsupported returns a visible error naming the engine.
func errDatasetStructureUnsupported() error {
return fmt.Errorf("dataset structure graph unsupported on doc engine %q", engine.GetEngineType())
}
// resolveDatasetStructureKind maps a user-facing dataset-structure kind to the
// stored top-level kind (mirror Python _DATASET_STRUCTURE_KIND_ALIASES). It is
// DELIBERATELY separate from compilationTemplateKind: that helper folds
// knowledge_graph→timeline, which would merge distinct dataset kinds, and it
// lacks the graph→knowledge_graph / mindmap→mind_map aliases (Python's
// dataset_api_service.py L1803-1805 documents exactly why it does not reuse the
// general normalizer). Returns "" for an invalid kind (caller maps that to
// 400 ARGUMENT_ERROR, never "return all").
func resolveDatasetStructureKind(kind string) string {
k := strings.ToLower(strings.TrimSpace(strings.ReplaceAll(kind, "-", "_")))
switch k {
case "graph", "knowledge_graph":
return "knowledge_graph"
case "mindmap", "mind_map":
// Align with Python _DATASET_STRUCTURE_KIND_ALIASES: resolve the
// user-facing "mindmap" to the stored compilation_template_kind_kwd value
// "mind_map" (the template kind, which is what read/delete paths match on).
return "mind_map"
case "timeline":
return "timeline"
case "session_essence":
return "session_essence"
case "session_graph":
return "session_graph"
}
return ""
}
// structureKindForBucket returns the normalized kind used for tree/page_index
// hierarchy handling.
func structureKindForBucket(kind string) string {
@@ -720,8 +779,10 @@ func (s *DatasetArtifactService) GetDocumentGraph(ctx context.Context, in Docume
return resp, nil
}
// normal mode: discover buckets from per-doc graph blob rows.
metaFields := []string{"compile_kwd", "compilation_template_ids", "compilation_template_kind_kwd"}
// normal mode: discover buckets from per-doc graph blob rows. "id" is
// required for the same reason as dataset discovery (Infinity only projects
// listed fields; graphRowSearch keys by id).
metaFields := []string{"id", "compile_kwd", "compilation_template_ids", "compilation_template_kind_kwd"}
metaRows, _, err := graphRowSearch(ctx, in.TenantID, in.DatasetID, metaFields,
map[string]interface{}{"doc_id": []string{in.DocumentID}, "knowledge_graph_kwd": []string{"graph"}}, nil, 0, 1000, nil)
if err != nil {
@@ -789,6 +850,197 @@ func containsStr(list []string, v string) bool {
return false
}
// DatasetStructureGraphInput is the parsed request for the dataset-scope
// structure graph endpoint (GET/DELETE /datasets/:id/artifacts/structure).
// Kind is REQUIRED (mirrors Python dataset_api.py:715): missing or invalid →
// 400 ARGUMENT_ERROR. Wipe applies to DELETE: true deletes the dataset rows,
// false only cancels the task (rows are left for the next rebuild to clean).
type DatasetStructureGraphInput struct {
TenantID string
DatasetID string
Kind string
Wipe bool
}
// DatasetStructureGraphResponse mirrors Python get_dataset_structure's
// {"kind": ..., "templates": [...]}.
type DatasetStructureGraphResponse struct {
Kind string `json:"kind"`
Templates []DocumentStructureGraphTemplate `json:"templates"`
}
// GetDatasetStructure returns the dataset-scope structure graph for a resolved
// kind, mirroring Python get_dataset_structure (dataset_api_service.py). Discovery
// scans knowledge_graph_kwd=["entity"] dataset rows (scope_kwd="dataset") and
// matches the resolved kind against the stamped compilation_template_kind_kwd —
// NOT compile_kwd (compile_kwd holds the autotype "hypergraph"/"list"/"mindmap",
// which is never the kind discriminator; Python _discover_scope_templates matches
// _resolve_dataset_structure_kind against compilation_template_kind_kwd the same
// way). It collects distinct template ids, then reads each template's dataset
// entity/relation rows via buildBucket. It does NOT read kg_build_meta (write/
// delete-side only).
func (s *DatasetArtifactService) GetDatasetStructure(ctx context.Context, in DatasetStructureGraphInput) (*DatasetStructureGraphResponse, error) {
resolved := resolveDatasetStructureKind(in.Kind)
if resolved == "" {
return nil, fmt.Errorf("%w: %q", ErrInvalidStructureKind, in.Kind)
}
if !datasetStructureSupported() {
return nil, errDatasetStructureUnsupported()
}
resp := &DatasetStructureGraphResponse{Kind: resolved, Templates: []DocumentStructureGraphTemplate{}}
// Discover distinct template ids whose stamped template kind resolves to the
// requested kind, scanning dataset-scope entity rows only. scope_kwd="dataset"
// is required here (unlike the legacy doc_graph fallback) because dataset rows
// are the only ones carrying compilation_template_kind_kwd we can trust for the
// dataset-scope kind match.
templateIDs := map[string]struct{}{}
// "id" must be projected: graphRowSearch keys its result map by the row id,
// and Infinity only returns fields listed in SelectFields (it does not
// synthesize id), so omitting it silently drops every row (review Major).
// The resolved kind is pushed into the filter so the engine applies the
// predicate instead of scanning all entity rows and discarding them in Go.
metaFields := []string{"id", "compilation_template_kind_kwd", "compilation_template_ids"}
for offset := 0; ; offset += 1000 {
rows, total, err := graphRowSearch(ctx, in.TenantID, in.DatasetID, metaFields,
map[string]interface{}{
"knowledge_graph_kwd": []string{"entity"},
"scope_kwd": []string{"dataset"},
"compilation_template_kind_kwd": []string{resolved},
}, nil, offset, 1000, nil)
if err != nil {
return nil, err
}
for _, row := range rows {
tkind := firstStringValue(row["compilation_template_kind_kwd"])
if tkind == "" || resolveDatasetStructureKind(tkind) != resolved {
continue
}
tid := rowTemplateID(row)
if tid != "" {
templateIDs[tid] = struct{}{}
}
}
if int64(offset+1000) >= total || len(rows) == 0 {
break
}
}
// Read each template's dataset entity/relation rows.
for tid := range templateIDs {
scope := map[string]interface{}{
"scope_kwd": []string{"dataset"},
"compilation_template_ids": []string{tid},
"compilation_template_kind_kwd": []string{resolved},
}
entities, relations, err := s.buildBucket(ctx, in.TenantID, in.DatasetID, scope, nil)
if err != nil {
return nil, err
}
if len(entities) == 0 && len(relations) == 0 {
continue
}
resp.Templates = append(resp.Templates, DocumentStructureGraphTemplate{
TemplateID: tid,
TemplateName: tid,
Kind: resolved,
Entities: entities,
Relations: relations,
})
}
return resp, nil
}
// DeleteDatasetStructure handles DELETE /datasets/:id/artifacts/structure?kind=&wipe=.
// It validates kind like GET. wipe=false cancels the kind's task (via the task-id
// field) without deleting rows; wipe=true deletes the kind's kg_build_meta marker
// + dataset entity/relation rows (document-scope rows are never touched).
func (s *DatasetArtifactService) DeleteDatasetStructure(ctx context.Context, in DatasetStructureGraphInput) (int, error) {
resolved := resolveDatasetStructureKind(in.Kind)
if resolved == "" {
return 0, fmt.Errorf("%w: %q", ErrInvalidStructureKind, in.Kind)
}
if !in.Wipe {
// Cancel the task without deleting rows. Task cancellation is task-id
// granular (mirrors Python delete_index REDIS set "{task_id}-cancel"); the
// actual row cleanup happens on the next rebuild. Rows are preserved.
return s.cancelDatasetStructureTask(ctx, in.TenantID, in.DatasetID, resolved)
}
if !datasetStructureSupported() {
return 0, errDatasetStructureUnsupported()
}
docEngine := engine.Get()
if docEngine == nil {
return 0, fmt.Errorf("document engine is not initialized")
}
indexName := fmt.Sprintf("ragflow_%s", in.TenantID)
cond := map[string]interface{}{
"kb_id": in.DatasetID,
"scope_kwd": "dataset",
"compilation_template_kind_kwd": resolved,
"knowledge_graph_kwd": []string{"entity", "relation", "kg_build_meta"},
}
n, err := docEngine.DeleteChunks(ctx, cond, indexName, in.DatasetID)
if err != nil {
return 0, err
}
return int(n), nil
}
// cancelDatasetStructureTask cancels the running dataset-structure task for a
// resolved kind by clearing both its per-index task-id and finish-at fields.
// This mirrors Python delete_index (dataset_api_service.py), which clears
// {task_id_field: "", task_finish_at_field: None} so the task state is fully
// reset (a stale finish-at would otherwise leave the kind looking "done" after
// cancel). The row cleanup itself is driven by the next rebuild; Go executes the
// merge synchronously inside the ingestor (no independent task/marker), so there
// is no Redis "{task_id}-cancel" marker to publish here.
func (s *DatasetArtifactService) cancelDatasetStructureTask(ctx context.Context, tenantID, datasetID, resolvedKind string) (int, error) {
field := datasetStructureTaskIDField(resolvedKind)
if field == "" {
return 0, nil
}
updates := map[string]interface{}{
field: "",
// gorm.Updates ignores nil values, so use an explicit NULL expression to
// actually clear the finish-at timestamp (mirrors Python None).
datasetStructureTaskFinishAtField(field): gorm.Expr("NULL"),
}
if err := dao.NewKnowledgebaseDAO().UpdateByID(ctx, dao.DB, datasetID, updates); err != nil {
return 0, err
}
return 0, nil
}
// datasetStructureTaskFinishAtField maps a task-id field to its sibling finish-at
// field, mirroring Python f"{task_id_field.replace('_task_id', '_task_finish_at')}".
func datasetStructureTaskFinishAtField(taskIDField string) string {
return strings.Replace(taskIDField, "_task_id", "_task_finish_at", 1)
}
// datasetStructureTaskIDField maps a resolved dataset-structure kind to its kb
// task-id field name. It mirrors Python _INDEX_TYPE_TO_TASK_ID_FIELD: each
// dataset-merge kind carries its own "<index_type>_task_id" field (structure_graph,
// structure_mindmap, timeline, session_graph, session_essence), NOT the legacy
// doc-level graphrag_task_id/mindmap_task_id. Empty means "no task-id field".
func datasetStructureTaskIDField(resolvedKind string) string {
switch resolvedKind {
case "knowledge_graph":
// "graph" is already normalized to "knowledge_graph" by
// resolveDatasetStructureKind, so no separate "graph" case is needed.
return "structure_graph_task_id"
case "mindmap", "mind_map":
return "structure_mindmap_task_id"
case "timeline":
return "timeline_task_id"
case "session_graph":
return "session_graph_task_id"
case "session_essence":
return "session_essence_task_id"
}
return ""
}
// resolveGraphBucket mirrors Python _resolve_bucket.
func resolveGraphBucket(row map[string]interface{}, templateMeta map[string]map[string]interface{}, documentID string) (map[string]interface{}, map[string]interface{}) {
compileKwd := firstStringValue(row["compile_kwd"])
@@ -866,7 +1118,7 @@ func rowTemplateID(row map[string]interface{}) string {
}
func (s *DatasetArtifactService) appendRaptorBlob(ctx context.Context, tenantID, datasetID, documentID string, grouped map[string]DocumentStructureGraphTemplate) {
rows, _, err := graphRowSearch(ctx, tenantID, datasetID, []string{"content_with_weight", "compile_kwd"},
rows, _, err := graphRowSearch(ctx, tenantID, datasetID, []string{"id", "content_with_weight", "compile_kwd"},
map[string]interface{}{"doc_id": []string{documentID}, "compile_kwd": []string{"raptor_graph"}}, nil, 0, 16, nil)
if err != nil {
return

View File

@@ -875,6 +875,45 @@ func TestIngestionTaskServiceMarkStoppedIdempotentOnAlreadyStopped(t *testing.T)
}
}
// TestIngestionTaskServiceRequestStopNeverReopensTerminalTask locks the invariant
// that a terminal task (COMPLETED/STOPPED/FAILED) can never be moved back to
// STOPPING. A terminal task whose message was already acked must not regress to
// the in-flight STOPPING state (which has no settled worker and would be stuck
// forever). RequestStop is a no-op for terminal states.
func TestIngestionTaskServiceRequestStopNeverReopensTerminalTask(t *testing.T) {
for _, status := range []string{common.COMPLETED, common.STOPPED, common.FAILED} {
t.Run(status, func(t *testing.T) {
db := setupServiceTestDB(t)
pushServiceDB(t, db)
insertTestIngestionTaskWithStatus(t, "task-1", "user-1", "doc-1", "kb-1", status)
ctx := t.Context()
svc := NewIngestionTaskService()
task, err := svc.RequestStop(ctx, "task-1")
if err != nil {
t.Fatalf("RequestStop on %s task should be a no-op, got: %v", status, err)
}
if task.Status != status {
t.Fatalf("RequestStop moved %s task to %q, must stay %s", status, task.Status, status)
}
})
}
}
// TestValidateTransitionRejectsTerminalToStopping locks the state-machine
// invariant directly: STOPPING is only reachable from RUNNING, never from a
// terminal state (COMPLETED/STOPPED/FAILED) or from STOPPING itself.
func TestValidateTransitionRejectsTerminalToStopping(t *testing.T) {
for _, from := range []string{common.COMPLETED, common.STOPPED, common.FAILED, common.STOPPING, common.CREATED} {
if err := validateTransition(from, common.STOPPING); err == nil {
t.Errorf("validateTransition(%s -> STOPPING) = nil, want error", from)
}
}
if err := validateTransition(common.RUNNING, common.STOPPING); err != nil {
t.Errorf("validateTransition(RUNNING -> STOPPING) = %v, want nil", err)
}
}
func TestIngestionTaskServiceMarkFailedIdempotentOnAlreadyTerminal(t *testing.T) {
db := setupServiceTestDB(t)
pushServiceDB(t, db)

View File

@@ -20,6 +20,8 @@ import (
"context"
"fmt"
"strings"
"ragflow/internal/entity"
)
// NavEmbedder is the production implementation of nlp.NavEmbedder. It resolves
@@ -45,7 +47,17 @@ func (e *NavEmbedder) Encode(ctx context.Context, tenantID string, texts []strin
}
name := e.embdModelName
if name == "" {
name = tenantID // composite name falls back to tenant default resolution
// Resolve the tenant's default embedding model composite reference
// ("<model>@<instance>@<provider>") — NOT the tenant id. Passing the
// tenant id as the model ref makes ResolveModelConfig parse it as a
// "model@provider" key, which fails with "provider name missing in model
// name: <tenant_id>". Mirrors knowledge_compiler_wiring.go's chat-model
// default resolution.
ref, err := e.modelSvc.GetTenantDefaultModelRef(ctx, tenantID, entity.ModelTypeEmbedding)
if err != nil {
return nil, fmt.Errorf("datasetnav: resolve embedding model for tenant %s: %w", tenantID, err)
}
name = ref
}
model, err := e.modelSvc.GetEmbeddingModel(ctx, tenantID, name)
if err != nil {