From b811dce1b05cd86abfb58a3b4fb38ef07a3d2e13 Mon Sep 17 00:00:00 2001 From: Zhichang Yu Date: Fri, 31 Jul 2026 22:57:43 +0800 Subject: [PATCH] feat(go): port knowledge compilation template/group + wiki artifact/nav/skill REST APIs from Python (#17656) --- cmd/ragflow_server.go | 8 +- internal/dao/compilation_template.go | 96 ++ internal/dao/compilation_template_group.go | 109 +++ internal/dao/file_commit.go | 69 ++ internal/entity/file_commit.go | 3 +- internal/handler/compilation_template.go | 78 ++ .../handler/compilation_template_group.go | 233 +++++ internal/handler/dataset_artifact.go | 475 ++++++++++ internal/handler/file_commit.go | 44 + internal/handler/file_commit_test.go | 8 + internal/handler/providers.go | 56 +- internal/handler/providers_test.go | 12 +- internal/router/router.go | 86 +- .../compilation_template_group_service.go | 494 ++++++++++ .../service/compilation_template_service.go | 379 ++++++++ internal/service/dataset_artifact_service.go | 846 ++++++++++++++++++ internal/service/file/file_commit.go | 205 +++++ .../service/file/file_commit_page_test.go | 281 ++++++ 18 files changed, 3428 insertions(+), 54 deletions(-) create mode 100644 internal/dao/compilation_template_group.go create mode 100644 internal/handler/compilation_template.go create mode 100644 internal/handler/compilation_template_group.go create mode 100644 internal/handler/dataset_artifact.go create mode 100644 internal/service/compilation_template_group_service.go create mode 100644 internal/service/compilation_template_service.go create mode 100644 internal/service/dataset_artifact_service.go create mode 100644 internal/service/file/file_commit_page_test.go diff --git a/cmd/ragflow_server.go b/cmd/ragflow_server.go index 1594dd5748..8407707673 100644 --- a/cmd/ragflow_server.go +++ b/cmd/ragflow_server.go @@ -833,6 +833,9 @@ func startServer(ctx context.Context) { componentsSvc := service.NewComponentsService() componentsHandler := handler.NewComponentsHandler(componentsSvc) pipelineHandler := handler.NewPipelineHandler() + compilationTemplateHandler := handler.NewCompilationTemplateHandler(service.NewCompilationTemplateService()) + compilationTemplateGroupHandler := handler.NewCompilationTemplateGroupHandler(service.NewCompilationTemplateGroupService()) + datasetArtifactHandler := handler.NewDatasetArtifactHandler(service.NewDatasetArtifactService(), datasetsService, file.NewFileCommitService()) // Initialize router r := router.NewRouter(authHandler, @@ -865,7 +868,10 @@ func startServer(ctx context.Context) { openaiChatHandler, botHandler, componentsHandler, - pipelineHandler) + pipelineHandler, + compilationTemplateHandler, + compilationTemplateGroupHandler, + datasetArtifactHandler) // Create Gin engine ginEngine := gin.New() diff --git a/internal/dao/compilation_template.go b/internal/dao/compilation_template.go index 6db5c3613f..634e1eeb2e 100644 --- a/internal/dao/compilation_template.go +++ b/internal/dao/compilation_template.go @@ -20,6 +20,8 @@ import ( "context" "fmt" "ragflow/internal/entity" + + "gorm.io/gorm" ) // CompilationTemplateDAO is the data-access object for compilation templates and @@ -123,3 +125,97 @@ func (dao *CompilationTemplateDAO) GetTemplate(ctx context.Context, tenantID, te } return &t, nil } + +// ListByGroup returns the valid compilation templates belonging to a group, +// ordered by create_time asc, mirroring Python +// CompilationTemplateGroupService child loading. +func (dao *CompilationTemplateDAO) ListByGroup(ctx context.Context, db *gorm.DB, groupID string) ([]*entity.CompilationTemplate, error) { + var templates []*entity.CompilationTemplate + if err := db.WithContext(ctx). + Where("group_id = ? AND status = ?", groupID, string(entity.StatusValid)). + Order("create_time asc"). + Find(&templates).Error; err != nil { + return nil, err + } + return templates, nil +} + +// ListBuiltins returns the valid, built-in (is_builtin) compilation templates, +// ordered by create_time then name. Mirrors Python list_builtins(). +func (dao *CompilationTemplateDAO) ListBuiltins(ctx context.Context) ([]*entity.CompilationTemplate, error) { + var templates []*entity.CompilationTemplate + if err := DB.WithContext(ctx). + Where("is_builtin = ? AND status = ?", true, string(entity.StatusValid)). + Order("create_time asc, name asc"). + Find(&templates).Error; err != nil { + return nil, err + } + return templates, nil +} + +// NameExistsInGroup reports whether a valid, non-built-in template with the +// given name already exists inside the group, excluding excludeID. Mirrors the +// Python duplicate-child guard. +func (dao *CompilationTemplateDAO) NameExistsInGroup(ctx context.Context, tenantID, groupID, name, excludeID string) (bool, error) { + q := DB.WithContext(ctx).Model(&entity.CompilationTemplate{}). + Where("tenant_id = ? AND group_id = ? AND name = ? AND is_builtin = ? AND status = ?", + tenantID, groupID, name, false, string(entity.StatusValid)) + if excludeID != "" { + q = q.Where("id <> ?", excludeID) + } + var count int64 + if err := q.Count(&count).Error; err != nil { + return false, err + } + return count > 0, nil +} + +// GetByID returns a single template row by id regardless of tenant scope +// (used for reconciling group children). +func (dao *CompilationTemplateDAO) GetByID(ctx context.Context, id string) (*entity.CompilationTemplate, error) { + var t entity.CompilationTemplate + if err := DB.WithContext(ctx).Where("id = ?", id).First(&t).Error; err != nil { + return nil, err + } + return &t, nil +} + +// Save inserts a new template row. +func (dao *CompilationTemplateDAO) Save(ctx context.Context, db *gorm.DB, t *entity.CompilationTemplate) error { + return db.WithContext(ctx).Create(t).Error +} + +// UpdateFields persists the supplied columns for the template with the given id. +func (dao *CompilationTemplateDAO) UpdateFields(ctx context.Context, db *gorm.DB, id string, m map[string]interface{}) error { + return db.WithContext(ctx).Model(&entity.CompilationTemplate{}). + Where("id = ?", id).Updates(m).Error +} + +// UpdateStatusByGroup flips the status of every valid template in a group, +// mirroring Python group delete's child cascade. +func (dao *CompilationTemplateDAO) UpdateStatusByGroup(ctx context.Context, db *gorm.DB, groupID, status string) error { + return db.WithContext(ctx).Model(&entity.CompilationTemplate{}). + Where("group_id = ? AND status = ?", groupID, string(entity.StatusValid)). + Update("status", status).Error +} + +// UpdateStatusByID flips a single template's status. +func (dao *CompilationTemplateDAO) UpdateStatusByID(ctx context.Context, db *gorm.DB, id, status string) error { + return db.WithContext(ctx).Model(&entity.CompilationTemplate{}). + Where("id = ?", id). + Update("status", status).Error +} + +// HardDeleteOrphansByName physically removes stale, invalid, non-built-in +// templates of the given name within the group, mirroring Python +// _purge_stale_invalid_children (which DELETEs orphaned duplicate names after a +// group child is soft-deleted). The deletion is scoped to the group so a +// same-named template in another group is never affected. These rows were +// soft-deleted in a prior operation but must be permanently purged to keep the +// table clean. +func (dao *CompilationTemplateDAO) HardDeleteOrphansByName(ctx context.Context, db *gorm.DB, tenantID, groupID, name string) error { + return db.WithContext(ctx).Where( + "tenant_id = ? AND group_id = ? AND name = ? AND is_builtin = ? AND status = ?", + tenantID, groupID, name, false, string(entity.StatusInvalid), + ).Delete(&entity.CompilationTemplate{}).Error +} diff --git a/internal/dao/compilation_template_group.go b/internal/dao/compilation_template_group.go new file mode 100644 index 0000000000..8524527525 --- /dev/null +++ b/internal/dao/compilation_template_group.go @@ -0,0 +1,109 @@ +// +// Copyright 2026 The InfiniFlow Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +package dao + +import ( + "context" + "errors" + "ragflow/internal/entity" + + "gorm.io/gorm" +) + +// CompilationTemplateGroupDAO is the data-access object for compilation template +// groups. +type CompilationTemplateGroupDAO struct{} + +// NewCompilationTemplateGroupDAO creates a compilation template group DAO. +func NewCompilationTemplateGroupDAO() *CompilationTemplateGroupDAO { + return &CompilationTemplateGroupDAO{} +} + +// ListSaved returns the tenant's valid groups with optional keyword/scope +// filtering and ordering, mirroring Python list_saved(). Built-in groups +// (empty tenant) are included so every tenant sees the catalogue. +func (dao *CompilationTemplateGroupDAO) ListSaved(ctx context.Context, tenantID, keywords, scope, orderby string, desc bool) ([]*entity.CompilationTemplateGroup, error) { + q := DB.WithContext(ctx). + Where("(tenant_id = ? OR tenant_id = '') AND status = ?", tenantID, string(entity.StatusValid)) + if keywords != "" { + q = q.Where("name LIKE ?", "%"+keywords+"%") + } + if scope != "" { + q = q.Where("scope = ?", scope) + } + if orderby != "name" && orderby != "scope" && orderby != "create_time" && orderby != "update_time" { + orderby = "create_time" + } + dir := "asc" + if desc { + dir = "desc" + } + var groups []*entity.CompilationTemplateGroup + if err := q.Order(orderby + " " + dir).Find(&groups).Error; err != nil { + return nil, err + } + return groups, nil +} + +// GetSaved returns a single valid group for the tenant (or built-in), or nil. +func (dao *CompilationTemplateGroupDAO) GetSaved(ctx context.Context, tenantID, groupID string) (*entity.CompilationTemplateGroup, error) { + var g entity.CompilationTemplateGroup + err := DB.WithContext(ctx). + Where("(tenant_id = ? OR tenant_id = '') AND id = ? AND status = ?", + tenantID, groupID, string(entity.StatusValid)). + First(&g).Error + if errors.Is(err, gorm.ErrRecordNotFound) { + return nil, nil + } + if err != nil { + return nil, err + } + return &g, nil +} + +// NameExists reports whether a valid group with the given name exists for the +// tenant, excluding excludeID. Mirrors Python name_exists(). +func (dao *CompilationTemplateGroupDAO) NameExists(ctx context.Context, tenantID, name, excludeID string) (bool, error) { + q := DB.WithContext(ctx).Model(&entity.CompilationTemplateGroup{}). + Where("tenant_id = ? AND name = ? AND status = ?", tenantID, name, string(entity.StatusValid)) + if excludeID != "" { + q = q.Where("id <> ?", excludeID) + } + var count int64 + if err := q.Count(&count).Error; err != nil { + return false, err + } + return count > 0, nil +} + +// Create inserts a new group row. +func (dao *CompilationTemplateGroupDAO) Create(ctx context.Context, db *gorm.DB, g *entity.CompilationTemplateGroup) error { + return db.WithContext(ctx).Create(g).Error +} + +// UpdateFields persists the supplied columns for the group with the given id. +func (dao *CompilationTemplateGroupDAO) UpdateFields(ctx context.Context, db *gorm.DB, id string, m map[string]interface{}) error { + return db.WithContext(ctx).Model(&entity.CompilationTemplateGroup{}). + Where("id = ?", id).Updates(m).Error +} + +// Delete soft-deletes a valid group, mirroring Python delete_group(). +func (dao *CompilationTemplateGroupDAO) Delete(ctx context.Context, db *gorm.DB, tenantID, groupID string) error { + return db.WithContext(ctx).Model(&entity.CompilationTemplateGroup{}). + Where("id = ? AND tenant_id = ? AND status = ?", groupID, tenantID, string(entity.StatusValid)). + Update("status", string(entity.StatusInvalid)).Error +} diff --git a/internal/dao/file_commit.go b/internal/dao/file_commit.go index a47c2f8a78..62fc6e15c9 100644 --- a/internal/dao/file_commit.go +++ b/internal/dao/file_commit.go @@ -18,11 +18,24 @@ package dao import ( "context" + "errors" "ragflow/internal/entity" + "strconv" "gorm.io/gorm" + "gorm.io/gorm/clause" ) +// toInterfaceSlice converts a string slice into an []interface{} for use as +// parameterized query variables. +func toInterfaceSlice(ss []string) []interface{} { + out := make([]interface{}, len(ss)) + for i := range ss { + out[i] = ss[i] + } + return out +} + // FileCommitDAO file commit data access object type FileCommitDAO struct{} @@ -110,6 +123,41 @@ func (dao *FileCommitDAO) ListByFolderID(ctx context.Context, db *gorm.DB, folde return commits, total, nil } +// ListByIDs lists commits whose IDs are in the given set, preserving the +// order of commitIDs (most-recent-first for page-edit history). Returns the +// total matched count for pagination. +func (dao *FileCommitDAO) ListByIDs(ctx context.Context, db *gorm.DB, commitIDs []string, page, pageSize int) ([]*entity.FileCommit, int64, error) { + if len(commitIDs) == 0 { + return []*entity.FileCommit{}, 0, nil + } + var commits []*entity.FileCommit + var total int64 + query := db.WithContext(ctx).Model(&entity.FileCommit{}).Where("id IN ?", commitIDs) + if err := query.Count(&total).Error; err != nil { + return nil, 0, err + } + // Preserve the incoming (most-recent-first) order with a parameterized + // CASE expression rather than a MySQL-specific FIELD() interpolation. + orderExpr := "CASE id" + for i := range commitIDs { + orderExpr += " WHEN ? THEN " + strconv.Itoa(i) + } + orderExpr += " END" + ordered := query.Clauses(clause.OrderBy{Expression: clause.Expr{ + SQL: orderExpr, + Vars: toInterfaceSlice(commitIDs), + }}) + // Apply pagination in the query itself so long histories never load the + // full set into memory before slicing. + if page > 0 && pageSize > 0 { + ordered = ordered.Offset((page - 1) * pageSize).Limit(pageSize) + } + if err := ordered.Find(&commits).Error; err != nil { + return nil, 0, err + } + return commits, total, nil +} + // FileCommitItemDAO file commit item data access object type FileCommitItemDAO struct{} @@ -146,3 +194,24 @@ func (dao *FileCommitItemDAO) GetByCommitIDAndFileID(ctx context.Context, db *go } return &item, nil } + +// GetLatestCommitIDByFileID returns the most recent commit_id that modified the +// given wiki page file key, or "" if none. Used to build the parent chain for +// page-edit commits. +func (dao *FileCommitItemDAO) GetLatestCommitIDByFileID(ctx context.Context, db *gorm.DB, fileID string) (string, error) { + var item entity.FileCommitItem + err := db.WithContext(ctx). + Where("file_id = ?", fileID). + // Order by the monotonic auto-increment sequence so the most recently + // inserted item is always selected even when multiple edits share the + // same create_time millisecond. + Order("seq DESC"). + First(&item).Error + if err != nil { + if errors.Is(err, gorm.ErrRecordNotFound) { + return "", nil + } + return "", err + } + return item.CommitID, nil +} diff --git a/internal/entity/file_commit.go b/internal/entity/file_commit.go index 9e98b1c645..9684218d99 100644 --- a/internal/entity/file_commit.go +++ b/internal/entity/file_commit.go @@ -38,8 +38,9 @@ func (FileCommit) TableName() string { // FileCommitItem represents a single file change within a commit. type FileCommitItem struct { ID string `gorm:"column:id;primaryKey;size:32" json:"id"` + Seq uint `gorm:"column:seq;autoIncrement;index" json:"seq,omitempty"` CommitID string `gorm:"column:commit_id;size:32;not null;uniqueIndex:idx_commit_file" json:"commit_id"` - FileID string `gorm:"column:file_id;size:32;not null;uniqueIndex:idx_commit_file" json:"file_id"` + FileID string `gorm:"column:file_id;size:255;not null;uniqueIndex:idx_commit_file" json:"file_id"` Operation string `gorm:"column:operation;size:16;not null;index" json:"operation"` OldHash *string `gorm:"column:old_hash;size:64;index" json:"old_hash,omitempty"` NewHash *string `gorm:"column:new_hash;size:64;index" json:"new_hash,omitempty"` diff --git a/internal/handler/compilation_template.go b/internal/handler/compilation_template.go new file mode 100644 index 0000000000..ee4f8adb46 --- /dev/null +++ b/internal/handler/compilation_template.go @@ -0,0 +1,78 @@ +// +// Copyright 2026 The InfiniFlow Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +package handler + +import ( + "ragflow/internal/common" + "ragflow/internal/service" + + "github.com/gin-gonic/gin" +) + +// CompilationTemplateHandler serves the knowledge-compilation template REST +// APIs. It mirrors the Python compilation_template_api blueprint. +type CompilationTemplateHandler struct { + compilationTemplateService *service.CompilationTemplateService +} + +// NewCompilationTemplateHandler creates a CompilationTemplateHandler. +func NewCompilationTemplateHandler(svc *service.CompilationTemplateService) *CompilationTemplateHandler { + return &CompilationTemplateHandler{compilationTemplateService: svc} +} + +// ListBuiltins returns the built-in template palette used by the frontend +// "Add template group" panel. +// +// @Summary List built-in compilation templates +// @Tags compilation template +// @Security ApiKeyAuth +// @Success 200 {object} map[string]interface{} +// @Router /v1/compilation-templates/builtins [get] +func (h *CompilationTemplateHandler) ListBuiltins(c *gin.Context) { + user, code, msg := GetUser(c) + if code != common.CodeSuccess { + common.ErrorWithCode(c, code, msg) + return + } + templates, err := h.compilationTemplateService.ListBuiltins(c.Request.Context(), user.ID) + if err != nil { + common.ResponseWithCodeData(c, common.CodeExceptionError, nil, err.Error()) + return + } + common.SuccessWithData(c, templates, "success") +} + +// ListWikiPresets returns the wiki page-structure presets loaded from YAML. +// +// @Summary List wiki presets +// @Tags compilation template +// @Security ApiKeyAuth +// @Success 200 {object} map[string]interface{} +// @Router /v1/compilation-templates/wiki-presets [get] +func (h *CompilationTemplateHandler) ListWikiPresets(c *gin.Context) { + _, code, msg := GetUser(c) + if code != common.CodeSuccess { + common.ErrorWithCode(c, code, msg) + return + } + presets, err := h.compilationTemplateService.LoadWikiPresets() + if err != nil { + common.ResponseWithCodeData(c, common.CodeExceptionError, nil, err.Error()) + return + } + common.SuccessWithData(c, presets, "success") +} diff --git a/internal/handler/compilation_template_group.go b/internal/handler/compilation_template_group.go new file mode 100644 index 0000000000..fed392925a --- /dev/null +++ b/internal/handler/compilation_template_group.go @@ -0,0 +1,233 @@ +// +// Copyright 2026 The InfiniFlow Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +package handler + +import ( + "strconv" + + "ragflow/internal/common" + "ragflow/internal/service" + + "github.com/gin-gonic/gin" +) + +// CompilationTemplateGroupHandler serves the knowledge-compilation template +// group REST APIs. It mirrors the Python compilation_template_group_api blueprint. +type CompilationTemplateGroupHandler struct { + compilationTemplateGroupService *service.CompilationTemplateGroupService +} + +// NewCompilationTemplateGroupHandler creates a CompilationTemplateGroupHandler. +func NewCompilationTemplateGroupHandler(svc *service.CompilationTemplateGroupService) *CompilationTemplateGroupHandler { + return &CompilationTemplateGroupHandler{compilationTemplateGroupService: svc} +} + +// List returns the tenant's compilation template groups with their nested +// templates and pagination. +// +// @Summary List compilation template groups +// @Tags compilation template group +// @Security ApiKeyAuth +// @Param keywords query string false "name keyword filter" +// @Param scope query string false "scope filter (file|dataset)" +// @Param page query int false "page number" default(1) +// @Param page_size query int false "items per page" default(30) +// @Param orderby query string false "order field" default(create_time) +// @Param desc query bool false "descending" default(true) +// @Success 200 {object} map[string]interface{} +// @Router /v1/compilation-template-groups [get] +func (h *CompilationTemplateGroupHandler) List(c *gin.Context) { + user, code, msg := GetUser(c) + if code != common.CodeSuccess { + common.ErrorWithCode(c, code, msg) + return + } + page, _ := strconv.Atoi(c.DefaultQuery("page", "1")) + pageSize, _ := strconv.Atoi(c.DefaultQuery("page_size", "30")) + if page < 1 { + page = 1 + } + if pageSize < 1 { + pageSize = 30 + } + keywords := c.Query("keywords") + scope := c.Query("scope") + orderby := c.DefaultQuery("orderby", "create_time") + desc := c.DefaultQuery("desc", "true") != "false" + + groups, err := h.compilationTemplateGroupService.ListSaved(c.Request.Context(), user.ID, keywords, scope, orderby, desc) + if err != nil { + common.ResponseWithCodeData(c, common.CodeExceptionError, nil, err.Error()) + return + } + total := len(groups) + if page > 0 && pageSize > 0 { + start := (page - 1) * pageSize + if start >= total { + groups = []*service.GroupListItem{} + } else { + end := start + pageSize + if end > total { + end = total + } + groups = groups[start:end] + } + } + common.SuccessWithData(c, gin.H{"groups": groups, "total": total}, "success") +} + +// Get returns a single compilation template group. +// +// @Summary Get compilation template group +// @Tags compilation template group +// @Security ApiKeyAuth +// @Param group_id path string true "group id" +// @Success 200 {object} map[string]interface{} +// @Router /v1/compilation-template-groups/{group_id} [get] +func (h *CompilationTemplateGroupHandler) Get(c *gin.Context) { + user, code, msg := GetUser(c) + if code != common.CodeSuccess { + common.ErrorWithCode(c, code, msg) + return + } + id := c.Param("group_id") + group, err := h.compilationTemplateGroupService.GetSaved(c.Request.Context(), user.ID, id) + if err != nil { + common.ResponseWithCodeData(c, common.CodeExceptionError, nil, err.Error()) + return + } + if group == nil { + common.ResponseWithCodeData(c, common.CodeNotFound, nil, "Cannot find compilation template group "+id+".") + return + } + common.SuccessWithData(c, group, "success") +} + +// Save creates a new compilation template group with its child templates. +// +// @Summary Save compilation template group +// @Tags compilation template group +// @Security ApiKeyAuth +// @Param request body service.GroupRequest true "group payload" +// @Success 200 {object} map[string]interface{} +// @Router /v1/compilation-template-groups [post] +func (h *CompilationTemplateGroupHandler) Save(c *gin.Context) { + user, code, msg := GetUser(c) + if code != common.CodeSuccess { + common.ErrorWithCode(c, code, msg) + return + } + var req service.GroupRequest + if err := c.ShouldBindJSON(&req); err != nil { + common.ResponseWithCodeData(c, common.CodeArgumentError, nil, "Invalid request: "+err.Error()) + return + } + if err := service.ValidateGroupRequest(&req); err != nil { + common.ResponseWithCodeData(c, common.CodeDataError, nil, err.Error()) + return + } + nameExists, err := h.compilationTemplateGroupService.NameExists(c.Request.Context(), user.ID, req.Name) + if err != nil { + common.ResponseWithCodeData(c, common.CodeExceptionError, nil, err.Error()) + return + } + if nameExists { + common.ResponseWithCodeData(c, common.CodeDataError, nil, "Duplicated compilation template group name.") + return + } + group, err := h.compilationTemplateGroupService.CreateGroup(c.Request.Context(), user.ID, &req) + if err != nil { + common.ResponseWithCodeData(c, common.CodeDataError, nil, err.Error()) + return + } + common.SuccessWithData(c, group, "success") +} + +// Update applies a partial update to a compilation template group. +// +// @Summary Update compilation template group +// @Tags compilation template group +// @Security ApiKeyAuth +// @Param group_id path string true "group id" +// @Param request body service.GroupRequest true "group payload" +// @Success 200 {object} map[string]interface{} +// @Router /v1/compilation-template-groups/{group_id} [put] +func (h *CompilationTemplateGroupHandler) Update(c *gin.Context) { + user, code, msg := GetUser(c) + if code != common.CodeSuccess { + common.ErrorWithCode(c, code, msg) + return + } + id := c.Param("group_id") + var req service.GroupRequest + if err := c.ShouldBindJSON(&req); err != nil { + common.ResponseWithCodeData(c, common.CodeArgumentError, nil, "Invalid request: "+err.Error()) + return + } + if err := service.ValidateGroupRequest(&req); err != nil { + common.ResponseWithCodeData(c, common.CodeDataError, nil, err.Error()) + return + } + if req.Name != "" { + nameExists, nerr := h.compilationTemplateGroupService.NameExists(c.Request.Context(), user.ID, req.Name, id) + if nerr != nil { + common.ResponseWithCodeData(c, common.CodeExceptionError, nil, nerr.Error()) + return + } + if nameExists { + common.ResponseWithCodeData(c, common.CodeDataError, nil, "Duplicated compilation template group name.") + return + } + } + group, err := h.compilationTemplateGroupService.UpdateGroup(c.Request.Context(), user.ID, id, &req) + if err != nil { + common.ResponseWithCodeData(c, common.CodeDataError, nil, err.Error()) + return + } + if group == nil { + common.ResponseWithCodeData(c, common.CodeNotFound, nil, "Cannot find compilation template group "+id+".") + return + } + common.SuccessWithData(c, group, "success") +} + +// Delete soft-deletes a compilation template group and its children. +// +// @Summary Delete compilation template group +// @Tags compilation template group +// @Security ApiKeyAuth +// @Param group_id path string true "group id" +// @Success 200 {object} map[string]interface{} +// @Router /v1/compilation-template-groups/{group_id} [delete] +func (h *CompilationTemplateGroupHandler) Delete(c *gin.Context) { + user, code, msg := GetUser(c) + if code != common.CodeSuccess { + common.ErrorWithCode(c, code, msg) + return + } + id := c.Param("group_id") + ok, err := h.compilationTemplateGroupService.DeleteGroup(c.Request.Context(), user.ID, id) + if err != nil { + common.ResponseWithCodeData(c, common.CodeExceptionError, nil, err.Error()) + return + } + if !ok { + common.ResponseWithCodeData(c, common.CodeNotFound, nil, "Cannot find compilation template group "+id+".") + return + } + common.SuccessWithData(c, true, "success") +} diff --git a/internal/handler/dataset_artifact.go b/internal/handler/dataset_artifact.go new file mode 100644 index 0000000000..5f65d4f3fa --- /dev/null +++ b/internal/handler/dataset_artifact.go @@ -0,0 +1,475 @@ +// +// Copyright 2026 The InfiniFlow Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +package handler + +import ( + "net/http" + "strconv" + + "ragflow/internal/common" + "ragflow/internal/dao" + "ragflow/internal/entity" + "ragflow/internal/service" + dataset "ragflow/internal/service/dataset" + "ragflow/internal/service/file" + + "github.com/gin-gonic/gin" + "go.uber.org/zap" +) + +// maxArtifactPageSize caps the page_size query parameter for artifact list +// endpoints so a client cannot force an unbounded document-engine search. +const maxArtifactPageSize = 100 + +// DatasetArtifactHandler exposes the knowledge-compilation artifact REST APIs +// backed by DatasetArtifactService. +type DatasetArtifactHandler struct { + svc *service.DatasetArtifactService + datasetSvc *dataset.DatasetService + fileCommitSvc *file.FileCommitService +} + +// NewDatasetArtifactHandler creates a DatasetArtifactHandler. +func NewDatasetArtifactHandler(svc *service.DatasetArtifactService, datasetSvc *dataset.DatasetService, fileCommitSvc *file.FileCommitService) *DatasetArtifactHandler { + return &DatasetArtifactHandler{svc: svc, datasetSvc: datasetSvc, fileCommitSvc: fileCommitSvc} +} + +// datasetOwner resolves the dataset and returns its tenant id. It also enforces +// dataset access permission for the requesting user. When the request is +// unauthenticated, the dataset is missing, or the user is not allowed to access +// it, an error response is written and (nil, "", "") is returned so callers can +// abort without sending a second response. +func (h *DatasetArtifactHandler) datasetOwner(c *gin.Context, datasetID string) (*entity.User, string, string) { + user, code, msg := GetUser(c) + if code != common.CodeSuccess { + common.ErrorWithCode(c, code, msg) + return nil, "", "" + } + kb, err := h.datasetSvc.GetKnowledgebaseByID(c.Request.Context(), datasetID) + if err != nil { + if dao.IsNotFoundErr(err) { + common.ErrorWithCode(c, common.CodeNotFound, "dataset not found") + } else { + common.ErrorWithCode(c, common.CodeServerError, "failed to resolve dataset") + } + return nil, "", "" + } + if kb == nil { + common.ErrorWithCode(c, common.CodeNotFound, "dataset not found") + return nil, "", "" + } + if !h.datasetSvc.Accessible(c.Request.Context(), datasetID, user.ID) { + common.ErrorWithCode(c, common.CodeForbidden, "no permission to access this dataset") + return nil, "", "" + } + return user, kb.TenantID, msg +} + +// HEAD /artifacts — any wiki artifact present? +func (h *DatasetArtifactHandler) AnyArtifact(c *gin.Context) { + _, tenantID, _ := h.datasetOwner(c, c.Param("dataset_id")) + if tenantID == "" { + return + } + has, err := h.svc.HasWiki(c.Request.Context(), tenantID, c.Param("dataset_id")) + if err != nil { + common.ErrorWithCode(c, common.CodeDataError, err.Error()) + return + } + if has { + c.Status(http.StatusOK) + } else { + c.Status(http.StatusNotFound) + } +} + +// GET /artifacts — list wiki pages. +func (h *DatasetArtifactHandler) ListArtifacts(c *gin.Context) { + _, tenantID, _ := h.datasetOwner(c, c.Param("dataset_id")) + if tenantID == "" { + return + } + datasetID := c.Param("dataset_id") + pageType := c.Query("page_type") + topic := c.Query("topic") + page, _ := strconv.Atoi(c.DefaultQuery("page", "1")) + pageSize, _ := strconv.Atoi(c.DefaultQuery("page_size", "30")) + if page < 1 { + page = 1 + } + if pageSize < 1 { + pageSize = 30 + } + if pageSize > maxArtifactPageSize { + pageSize = maxArtifactPageSize + } + items, total, err := h.svc.ListWikiPages(c.Request.Context(), tenantID, datasetID, pageType, topic, page, pageSize) + if err != nil { + common.ErrorWithCode(c, common.CodeDataError, err.Error()) + return + } + common.SuccessWithData(c, gin.H{"total": total, "pages": items}, "success") +} + +// PUT /artifacts// — edit a wiki page. +func (h *DatasetArtifactHandler) UpdateArtifact(c *gin.Context) { + user, tenantID, _ := h.datasetOwner(c, c.Param("dataset_id")) + if tenantID == "" { + return + } + datasetID := c.Param("dataset_id") + pageType := c.Param("page_type") + slug := c.Param("slug") + var req struct { + ContentMd string `json:"content_md"` + Title string `json:"title"` + Outlinks []string `json:"outlinks"` + } + if err := c.ShouldBindJSON(&req); err != nil { + common.ErrorWithCode(c, common.CodeArgumentError, err.Error()) + return + } + + // Read the current page before mutating so an audit commit can record an + // accurate old-to-new diff. Only record when the prior state was read + // successfully; a failed read must not produce a diff against an unknown + // old value. + var oldContent string + canRecordAudit := false + if req.ContentMd != "" { + before, gerr := h.svc.GetWikiPage(c.Request.Context(), tenantID, datasetID, pageType, slug) + if gerr != nil { + common.Warn("failed to read wiki page before recording edit commit", zap.Error(gerr)) + } else if before != nil { + oldContent = before.ContentMd + canRecordAudit = true + } + } + + detail, err := h.svc.UpdateWikiPage(c.Request.Context(), tenantID, datasetID, pageType, slug, req.ContentMd, req.Title, req.Outlinks) + if err != nil { + common.ErrorWithCode(c, common.CodeDataError, err.Error()) + return + } + if detail == nil { + common.ErrorWithCode(c, common.CodeNotFound, "page not found") + return + } + + // Record a wiki-page edit audit commit (git-style parent chain) when the + // content actually changed. Best-effort: a commit write failure must not + // roll back the page edit already persisted above. + if canRecordAudit && req.ContentMd != oldContent && h.fileCommitSvc != nil { + title := req.Title + if title == "" { + title = detail.Title + } + if _, cerr := h.fileCommitSvc.RecordPageEdit(c.Request.Context(), file.PageEditCommitInput{ + DatasetID: datasetID, + DocID: pageType + "/" + slug, + Slug: slug, + PageType: pageType, + Title: title, + AuthorID: user.ID, + OldContent: oldContent, + NewContent: req.ContentMd, + }); cerr != nil { + common.Warn("failed to record wiki page edit commit", zap.Error(cerr)) + } + } + + common.SuccessWithData(c, detail, "success") +} + +// GET /artifacts// — single wiki page. +func (h *DatasetArtifactHandler) GetArtifact(c *gin.Context) { + _, tenantID, _ := h.datasetOwner(c, c.Param("dataset_id")) + if tenantID == "" { + return + } + datasetID := c.Param("dataset_id") + pageType := c.Param("page_type") + slug := c.Param("slug") + detail, err := h.svc.GetWikiPage(c.Request.Context(), tenantID, datasetID, pageType, slug) + if err != nil { + common.ErrorWithCode(c, common.CodeDataError, err.Error()) + return + } + if detail == nil { + common.ErrorWithCode(c, common.CodeNotFound, "page not found") + return + } + common.SuccessWithData(c, detail, "success") +} + +// DELETE /artifacts — clear all wiki artifacts. +func (h *DatasetArtifactHandler) DeleteArtifacts(c *gin.Context) { + _, tenantID, _ := h.datasetOwner(c, c.Param("dataset_id")) + if tenantID == "" { + return + } + deleted, err := h.svc.ClearWiki(c.Request.Context(), tenantID, c.Param("dataset_id")) + if err != nil { + common.ErrorWithCode(c, common.CodeDataError, err.Error()) + return + } + common.SuccessWithData(c, deleted, "success") +} + +// GET /artifacts/topics — list wiki topics. +func (h *DatasetArtifactHandler) ListArtifactTopics(c *gin.Context) { + _, tenantID, _ := h.datasetOwner(c, c.Param("dataset_id")) + if tenantID == "" { + return + } + datasetID := c.Param("dataset_id") + items, total, err := h.svc.ListWikiTopics(c.Request.Context(), tenantID, datasetID) + if err != nil { + common.ErrorWithCode(c, common.CodeDataError, err.Error()) + return + } + common.SuccessWithData(c, gin.H{"total": total, "topics": items}, "success") +} + +// GET /artifacts/alteration — wiki alteration summary. +func (h *DatasetArtifactHandler) GetArtifactAlteration(c *gin.Context) { + _, tenantID, _ := h.datasetOwner(c, c.Param("dataset_id")) + if tenantID == "" { + return + } + datasetID := c.Param("dataset_id") + alt, err := h.svc.GetWikiAlteration(c.Request.Context(), tenantID, datasetID) + if err != nil { + common.ErrorWithCode(c, common.CodeDataError, err.Error()) + return + } + common.SuccessWithData(c, alt, "success") +} + +// GET /artifacts/graph — wiki entity/relation graph. +func (h *DatasetArtifactHandler) GetArtifactGraph(c *gin.Context) { + _, tenantID, _ := h.datasetOwner(c, c.Param("dataset_id")) + if tenantID == "" { + return + } + datasetID := c.Param("dataset_id") + graph, err := h.svc.GetWikiGraph(c.Request.Context(), tenantID, datasetID) + if err != nil { + common.ErrorWithCode(c, common.CodeDataError, err.Error()) + return + } + common.SuccessWithData(c, graph, "success") +} + +// GET /artifacts/structure — list compiled structures of a dataset. +func (h *DatasetArtifactHandler) ListStructures(c *gin.Context) { + _, tenantID, _ := h.datasetOwner(c, c.Param("dataset_id")) + if tenantID == "" { + return + } + datasetID := c.Param("dataset_id") + structureKind := c.Query("structure_kind") + structureIndexType := c.Query("structure_index_type") + items, total, err := h.svc.ListStructures(c.Request.Context(), tenantID, datasetID, structureKind, structureIndexType) + if err != nil { + common.ErrorWithCode(c, common.CodeDataError, err.Error()) + return + } + common.SuccessWithData(c, gin.H{"total": total, "structures": items}, "success") +} + +// DELETE /artifacts/structure — delete compiled structures of a dataset. +func (h *DatasetArtifactHandler) DeleteStructures(c *gin.Context) { + _, tenantID, _ := h.datasetOwner(c, c.Param("dataset_id")) + if tenantID == "" { + return + } + datasetID := c.Param("dataset_id") + structureKind := c.Query("structure_kind") + structureIndexType := c.Query("structure_index_type") + n, err := h.svc.DeleteStructures(c.Request.Context(), tenantID, datasetID, structureKind, structureIndexType) + if err != nil { + common.ErrorWithCode(c, common.CodeDataError, err.Error()) + return + } + common.SuccessWithData(c, gin.H{"deleted": n}, "success") +} + +// HEAD /skills — any skill artifact present? +func (h *DatasetArtifactHandler) AnySkill(c *gin.Context) { + _, tenantID, _ := h.datasetOwner(c, c.Param("dataset_id")) + if tenantID == "" { + return + } + has, err := h.svc.HasSkill(c.Request.Context(), tenantID, c.Param("dataset_id")) + if err != nil { + common.ErrorWithCode(c, common.CodeDataError, err.Error()) + return + } + if has { + c.Status(http.StatusOK) + } else { + c.Status(http.StatusNotFound) + } +} + +// GET /navigation — list navigation clusters. +func (h *DatasetArtifactHandler) ListNavigation(c *gin.Context) { + _, tenantID, _ := h.datasetOwner(c, c.Param("dataset_id")) + if tenantID == "" { + return + } + items, total, err := h.svc.ListNavClusters(c.Request.Context(), tenantID, c.Param("dataset_id")) + if err != nil { + common.ErrorWithCode(c, common.CodeDataError, err.Error()) + return + } + common.SuccessWithData(c, gin.H{"total": total, "nav": items}, "success") +} + +// DELETE /navigation — delete all navigation clusters. +func (h *DatasetArtifactHandler) DeleteNavigation(c *gin.Context) { + _, tenantID, _ := h.datasetOwner(c, c.Param("dataset_id")) + if tenantID == "" { + return + } + n, err := h.svc.DeleteNav(c.Request.Context(), tenantID, c.Param("dataset_id")) + if err != nil { + common.ErrorWithCode(c, common.CodeDataError, err.Error()) + return + } + common.SuccessWithData(c, gin.H{"deleted": n}, "success") +} + +// DELETE /navigation/ — delete a single navigation cluster. +func (h *DatasetArtifactHandler) DeleteNavigationNode(c *gin.Context) { + _, tenantID, _ := h.datasetOwner(c, c.Param("dataset_id")) + if tenantID == "" { + return + } + n, err := h.svc.DeleteNavNode(c.Request.Context(), tenantID, c.Param("dataset_id"), c.Param("name")) + if err != nil { + common.ErrorWithCode(c, common.CodeDataError, err.Error()) + return + } + common.SuccessWithData(c, gin.H{"deleted": n}, "success") +} + +// GET /navigation//children — list children of a navigation cluster. +func (h *DatasetArtifactHandler) ListNavigationChildren(c *gin.Context) { + _, tenantID, _ := h.datasetOwner(c, c.Param("dataset_id")) + if tenantID == "" { + return + } + items, total, err := h.svc.ListNavChildren(c.Request.Context(), tenantID, c.Param("dataset_id"), c.Param("name")) + if err != nil { + common.ErrorWithCode(c, common.CodeDataError, err.Error()) + return + } + common.SuccessWithData(c, gin.H{"total": total, "children": items}, "success") +} + +// GET /skills — skill tree. +func (h *DatasetArtifactHandler) GetSkillTree(c *gin.Context) { + _, tenantID, _ := h.datasetOwner(c, c.Param("dataset_id")) + if tenantID == "" { + return + } + kwd := c.Query("kwd") + items, total, err := h.svc.GetSkillTree(c.Request.Context(), tenantID, c.Param("dataset_id"), kwd) + if err != nil { + common.ErrorWithCode(c, common.CodeDataError, err.Error()) + return + } + common.SuccessWithData(c, gin.H{"total": total, "tree": items}, "success") +} + +// DELETE /skills — delete all skills. +func (h *DatasetArtifactHandler) DeleteSkills(c *gin.Context) { + _, tenantID, _ := h.datasetOwner(c, c.Param("dataset_id")) + if tenantID == "" { + return + } + n, err := h.svc.DeleteSkills(c.Request.Context(), tenantID, c.Param("dataset_id"), "") + if err != nil { + common.ErrorWithCode(c, common.CodeDataError, err.Error()) + return + } + common.SuccessWithData(c, gin.H{"deleted": n}, "success") +} + +// GET /skills/ — single skill page. +func (h *DatasetArtifactHandler) GetSkillPage(c *gin.Context) { + _, tenantID, _ := h.datasetOwner(c, c.Param("dataset_id")) + if tenantID == "" { + return + } + detail, err := h.svc.GetSkillPage(c.Request.Context(), tenantID, c.Param("dataset_id"), c.Param("skill_kwd")) + if err != nil { + common.ErrorWithCode(c, common.CodeDataError, err.Error()) + return + } + if detail == nil { + common.ErrorWithCode(c, common.CodeNotFound, "skill not found") + return + } + common.SuccessWithData(c, detail, "success") +} + +// DELETE /skills/ — delete a single skill. +func (h *DatasetArtifactHandler) DeleteSkill(c *gin.Context) { + _, tenantID, _ := h.datasetOwner(c, c.Param("dataset_id")) + if tenantID == "" { + return + } + n, err := h.svc.DeleteSkills(c.Request.Context(), tenantID, c.Param("dataset_id"), c.Param("skill_kwd")) + if err != nil { + common.ErrorWithCode(c, common.CodeDataError, err.Error()) + return + } + common.SuccessWithData(c, gin.H{"deleted": n}, "success") +} + +// GET /documents//structure/graph — document structure graph. +func (h *DatasetArtifactHandler) GetDocumentGraph(c *gin.Context) { + _, tenantID, _ := h.datasetOwner(c, c.Param("dataset_id")) + if tenantID == "" { + return + } + datasetID := c.Param("dataset_id") + documentID := c.Param("document_id") + graphType := c.Query("graph_type") + items, total, err := h.svc.GetDocumentGraph(c.Request.Context(), tenantID, datasetID, documentID, graphType) + if err != nil { + common.ErrorWithCode(c, common.CodeDataError, err.Error()) + return + } + common.SuccessWithData(c, gin.H{"total": total, "graph": items}, "success") +} + +// DELETE /documents//structure/graph — delete document structure graph. +func (h *DatasetArtifactHandler) DeleteDocumentGraph(c *gin.Context) { + _, tenantID, _ := h.datasetOwner(c, c.Param("dataset_id")) + if tenantID == "" { + return + } + datasetID := c.Param("dataset_id") + documentID := c.Param("document_id") + n, err := h.svc.DeleteDocumentGraph(c.Request.Context(), tenantID, datasetID, documentID) + if err != nil { + common.ErrorWithCode(c, common.CodeDataError, err.Error()) + return + } + common.SuccessWithData(c, gin.H{"deleted": n}, "success") +} diff --git a/internal/handler/file_commit.go b/internal/handler/file_commit.go index 2da5aef295..2a20bd4a03 100644 --- a/internal/handler/file_commit.go +++ b/internal/handler/file_commit.go @@ -38,6 +38,7 @@ type fileCommitService interface { GetCommitTree(ctx context.Context, commitID string) (map[string]interface{}, error) GetCommitFileContent(ctx context.Context, folderID, commitID, fileID string) ([]byte, error) GetFileVersionHistory(ctx context.Context, fileID string) ([]entity.VersionEntry, error) + ListPageCommits(ctx context.Context, datasetID, pageType, slug string, page, pageSize int) ([]*entity.FileCommit, int64, error) } // FileCommitHandler file commit handler @@ -217,6 +218,49 @@ func (h *FileCommitHandler) ListCommits(c *gin.Context) { } ctx := c.Request.Context() + + // Python's list_commits supports ?slug= to filter audit commits + // for a specific wiki/skill page (written by record_page_edit). These page + // commits are scoped to the dataset and page file key, so route them through + // the page-commit path instead of the folder-based ListCommits. This only + // applies to the /datasets/{dataset_id}/commits route. + if slug := c.Query("slug"); slug != "" { + datasetID := c.Param("dataset_id") + if datasetID == "" { + common.ErrorWithCode(c, common.CodeArgumentError, "slug requires a dataset scope") + return + } + pageType := c.Query("page_type") + commits, total, err := h.commitService.ListPageCommits(ctx, datasetID, pageType, slug, page, pageSize) + if err != nil { + jsonInternalError(c, err) + return + } + var commitList []entity.CommitResponse + for _, commit := range commits { + var ct int64 + if commit.CreateTime != nil { + ct = *commit.CreateTime + } + commitList = append(commitList, entity.CommitResponse{ + ID: commit.ID, + FolderID: commit.FolderID, + ParentID: commit.ParentID, + Message: commit.Message, + AuthorID: commit.AuthorID, + FileCount: commit.FileCount, + CreateTime: &ct, + }) + } + common.SuccessWithData(c, gin.H{ + "total": total, + "page": page, + "page_size": pageSize, + "commits": commitList, + }, common.CodeSuccess.Message()) + return + } + commits, total, err := h.commitService.ListCommits(ctx, folderID, page, pageSize, orderBy, desc) if err != nil { jsonInternalError(c, err) diff --git a/internal/handler/file_commit_test.go b/internal/handler/file_commit_test.go index 76987096f5..a8ecb07e3b 100644 --- a/internal/handler/file_commit_test.go +++ b/internal/handler/file_commit_test.go @@ -41,6 +41,7 @@ type mockFileCommitSvc struct { getCommitTreeFn func(ctx context.Context, commitID string) (map[string]interface{}, error) getCommitFileContentFn func(ctx context.Context, folderID, commitID, fileID string) ([]byte, error) getFileVersionHistoryFn func(ctx context.Context, fileID string) ([]entity.VersionEntry, error) + listPageCommitsFn func(ctx context.Context, datasetID, pageType, slug string, page, pageSize int) ([]*entity.FileCommit, int64, error) } func (m *mockFileCommitSvc) CreateCommit(ctx context.Context, folderID, authorID, message string, changes []entity.FileChange) (*entity.FileCommit, error) { @@ -128,6 +129,13 @@ func (m *mockFileCommitSvc) GetFileVersionHistory(ctx context.Context, fileID st }, nil } +func (m *mockFileCommitSvc) ListPageCommits(ctx context.Context, datasetID, pageType, slug string, page, pageSize int) ([]*entity.FileCommit, int64, error) { + if m.listPageCommitsFn != nil { + return m.listPageCommitsFn(ctx, datasetID, pageType, slug, page, pageSize) + } + return []*entity.FileCommit{}, 0, nil +} + func setupFileCommitTest(userID string) (*gin.Engine, *mockFileCommitSvc) { mock := &mockFileCommitSvc{} h := &FileCommitHandler{commitService: mock} diff --git a/internal/handler/providers.go b/internal/handler/providers.go index 9fb4225231..47d479cd43 100644 --- a/internal/handler/providers.go +++ b/internal/handler/providers.go @@ -112,7 +112,7 @@ func (h *ProviderHandler) AddProvider(c *gin.Context) { } func (h *ProviderHandler) DeleteProvider(c *gin.Context) { - providerName := c.Param("provider_name") + providerName := c.Param("provider_id_or_name") if providerName == "" { common.ResponseWithHttpCodeData(c, http.StatusBadRequest, 400, nil, "Provider name is required") return @@ -131,7 +131,7 @@ func (h *ProviderHandler) DeleteProvider(c *gin.Context) { } func (h *ProviderHandler) ShowProvider(c *gin.Context) { - providerName := c.Param("provider_name") + providerName := c.Param("provider_id_or_name") if providerName == "" { common.ResponseWithHttpCodeData(c, http.StatusBadRequest, 400, nil, "Provider name is required") return @@ -146,7 +146,7 @@ func (h *ProviderHandler) ShowProvider(c *gin.Context) { } func (h *ProviderHandler) ListModels(c *gin.Context) { - providerName := c.Param("provider_name") + providerName := c.Param("provider_id_or_name") if providerName == "" { common.ResponseWithHttpCodeData(c, http.StatusBadRequest, 400, nil, "Provider name is required") return @@ -221,7 +221,7 @@ func (h *ProviderHandler) ListModels(c *gin.Context) { } func (h *ProviderHandler) ShowModel(c *gin.Context) { - providerName := c.Param("provider_name") + providerName := c.Param("provider_id_or_name") if providerName == "" { common.ResponseWithHttpCodeData(c, http.StatusBadRequest, 400, nil, "Provider name is required") return @@ -249,7 +249,7 @@ type CreateProviderInstanceRequest struct { } func (h *ProviderHandler) CreateProviderInstance(c *gin.Context) { - providerName := c.Param("provider_name") + providerName := c.Param("provider_id_or_name") if providerName == "" { common.ResponseWithHttpCodeData(c, http.StatusBadRequest, 400, nil, "Provider name is required") return @@ -287,7 +287,7 @@ func (h *ProviderHandler) CreateProviderInstance(c *gin.Context) { } func (h *ProviderHandler) ListProviderInstances(c *gin.Context) { - providerName := c.Param("provider_name") + providerName := c.Param("provider_id_or_name") if providerName == "" { common.ResponseWithHttpCodeData(c, http.StatusBadRequest, 400, nil, "Provider name is required") return @@ -306,13 +306,13 @@ func (h *ProviderHandler) ListProviderInstances(c *gin.Context) { } func (h *ProviderHandler) ShowProviderInstance(c *gin.Context) { - providerName := c.Param("provider_name") + providerName := c.Param("provider_id_or_name") if providerName == "" { common.ResponseWithHttpCodeData(c, http.StatusBadRequest, 400, nil, "Provider name is required") return } - instanceIDOrName := c.Param("instance_name") + instanceIDOrName := c.Param("instance_id_or_name") if instanceIDOrName == "" { common.ResponseWithHttpCodeData(c, http.StatusBadRequest, 400, nil, "Instance name is required") return @@ -331,13 +331,13 @@ func (h *ProviderHandler) ShowProviderInstance(c *gin.Context) { } func (h *ProviderHandler) ShowInstanceBalance(c *gin.Context) { - providerName := c.Param("provider_name") + providerName := c.Param("provider_id_or_name") if providerName == "" { common.ResponseWithHttpCodeData(c, http.StatusBadRequest, 400, nil, "Provider name is required") return } - instanceName := c.Param("instance_name") + instanceName := c.Param("instance_id_or_name") if instanceName == "" { common.ResponseWithHttpCodeData(c, http.StatusBadRequest, 400, nil, "Instance name is required") return @@ -357,7 +357,7 @@ func (h *ProviderHandler) ShowInstanceBalance(c *gin.Context) { } func (h *ProviderHandler) CheckConnection(c *gin.Context) { - providerName := c.Param("provider_name") + providerName := c.Param("provider_id_or_name") if providerName == "" { common.ResponseWithHttpCodeData(c, http.StatusBadRequest, 400, nil, "Provider name is required") return @@ -382,13 +382,13 @@ func (h *ProviderHandler) CheckConnection(c *gin.Context) { func (h *ProviderHandler) CheckInstanceConnection(c *gin.Context) { ctx := c.Request.Context() - providerName := c.Param("provider_name") + providerName := c.Param("provider_id_or_name") if providerName == "" { common.ResponseWithHttpCodeData(c, http.StatusBadRequest, 400, nil, "Provider name is required") return } - instanceName := c.Param("instance_name") + instanceName := c.Param("instance_id_or_name") if instanceName == "" { common.ResponseWithHttpCodeData(c, http.StatusBadRequest, 400, nil, "Instance name is required") return @@ -417,13 +417,13 @@ func (h *ProviderHandler) CheckInstanceConnection(c *gin.Context) { func (h *ProviderHandler) ListTasks(c *gin.Context) { ctx := c.Request.Context() - providerName := c.Param("provider_name") + providerName := c.Param("provider_id_or_name") if providerName == "" { common.ResponseWithHttpCodeData(c, http.StatusBadRequest, 400, nil, "Provider name is required") return } - instanceName := c.Param("instance_name") + instanceName := c.Param("instance_id_or_name") if instanceName == "" { common.ResponseWithHttpCodeData(c, http.StatusBadRequest, 400, nil, "Instance name is required") return @@ -443,13 +443,13 @@ func (h *ProviderHandler) ListTasks(c *gin.Context) { func (h *ProviderHandler) ShowTask(c *gin.Context) { ctx := c.Request.Context() - providerName := c.Param("provider_name") + providerName := c.Param("provider_id_or_name") if providerName == "" { common.ResponseWithHttpCodeData(c, http.StatusBadRequest, 400, nil, "Provider name is required") return } - instanceName := c.Param("instance_name") + instanceName := c.Param("instance_id_or_name") if instanceName == "" { common.ResponseWithHttpCodeData(c, http.StatusBadRequest, 400, nil, "Instance name is required") return @@ -484,13 +484,13 @@ type AlterProviderInstanceRequest struct { func (h *ProviderHandler) AlterProviderInstance(c *gin.Context) { ctx := c.Request.Context() - providerName := c.Param("provider_name") + providerName := c.Param("provider_id_or_name") if providerName == "" { common.ResponseWithHttpCodeData(c, http.StatusBadRequest, 400, nil, "Provider name is required") return } - instanceName := c.Param("instance_name") + instanceName := c.Param("instance_id_or_name") if instanceName == "" { common.ResponseWithHttpCodeData(c, http.StatusBadRequest, 400, nil, "Instance name is required") return @@ -527,7 +527,7 @@ type DropProviderInstanceRequest struct { } func (h *ProviderHandler) DropProviderInstance(c *gin.Context) { - providerName := c.Param("provider_name") + providerName := c.Param("provider_id_or_name") if providerName == "" { common.ResponseWithHttpCodeData(c, http.StatusBadRequest, 400, nil, "Provider name is required") return @@ -552,12 +552,12 @@ func (h *ProviderHandler) DropProviderInstance(c *gin.Context) { func (h *ProviderHandler) ListInstanceModels(c *gin.Context) { ctx := c.Request.Context() - providerName := c.Param("provider_name") + providerName := c.Param("provider_id_or_name") if providerName == "" { common.ResponseWithHttpCodeData(c, http.StatusBadRequest, 400, nil, "Provider name is required") return } - instanceName := c.Param("instance_name") + instanceName := c.Param("instance_id_or_name") if instanceName == "" { common.ResponseWithHttpCodeData(c, http.StatusBadRequest, 400, nil, "Instance name is required") return @@ -600,13 +600,13 @@ type AlterModelRequest struct { } func (h *ProviderHandler) AlterModel(c *gin.Context) { - providerName := c.Param("provider_name") + providerName := c.Param("provider_id_or_name") if providerName == "" { common.ResponseWithHttpCodeData(c, http.StatusBadRequest, 400, nil, "Provider name is required") return } - instanceName := c.Param("instance_name") + instanceName := c.Param("instance_id_or_name") if instanceName == "" { common.ResponseWithHttpCodeData(c, http.StatusBadRequest, 400, nil, "Instance name is required") return @@ -691,8 +691,8 @@ func (h *ProviderHandler) AddModel(c *gin.Context) { return } - req.ProviderName = c.Param("provider_name") - req.InstanceName = c.Param("instance_name") + req.ProviderName = c.Param("provider_id_or_name") + req.InstanceName = c.Param("instance_id_or_name") if req.ProviderName == "" || req.InstanceName == "" { common.ResponseWithHttpCodeData(c, http.StatusBadRequest, common.CodeBadRequest, nil, "provider_name and instance_name are required") @@ -726,12 +726,12 @@ type DropInstanceModelRequest struct { } func (h *ProviderHandler) DropInstanceModels(c *gin.Context) { - providerName := c.Param("provider_name") + providerName := c.Param("provider_id_or_name") if providerName == "" { common.ResponseWithHttpCodeData(c, http.StatusBadRequest, 400, nil, "Provider name is required") return } - instanceName := c.Param("instance_name") + instanceName := c.Param("instance_id_or_name") if instanceName == "" { common.ResponseWithHttpCodeData(c, http.StatusBadRequest, 400, nil, "Instance name is required") return diff --git a/internal/handler/providers_test.go b/internal/handler/providers_test.go index c19f14c61d..77dfda7e72 100644 --- a/internal/handler/providers_test.go +++ b/internal/handler/providers_test.go @@ -88,8 +88,8 @@ func TestProviderHandlerAlterModelRejectsMissingModelSelector(t *testing.T) { ctx, recorder := newProviderHandlerRequest( t, map[string]interface{}{"status": "active"}, - gin.Param{Key: "provider_name", Value: "OpenAI"}, - gin.Param{Key: "instance_name", Value: "default"}, + gin.Param{Key: "provider_id_or_name", Value: "OpenAI"}, + gin.Param{Key: "instance_id_or_name", Value: "default"}, ) NewProviderHandler(nil, service.NewModelProviderService()).AlterModel(ctx) @@ -107,8 +107,8 @@ func TestProviderHandlerAlterModelRejectsInvalidStatus(t *testing.T) { ctx, recorder := newProviderHandlerRequest( t, map[string]interface{}{"status": "disabled"}, - gin.Param{Key: "provider_name", Value: "OpenAI"}, - gin.Param{Key: "instance_name", Value: "default"}, + gin.Param{Key: "provider_id_or_name", Value: "OpenAI"}, + gin.Param{Key: "instance_id_or_name", Value: "default"}, gin.Param{Key: "model_name", Value: "gpt-test"}, ) @@ -131,8 +131,8 @@ func TestProviderHandlerAlterModelUpdatesStatus(t *testing.T) { ctx, recorder := newProviderHandlerRequest( t, map[string]interface{}{"status": "inactive"}, - gin.Param{Key: "provider_name", Value: "OpenAI"}, - gin.Param{Key: "instance_name", Value: "default"}, + gin.Param{Key: "provider_id_or_name", Value: "OpenAI"}, + gin.Param{Key: "instance_id_or_name", Value: "default"}, gin.Param{Key: "model_name", Value: "gpt-test"}, ) diff --git a/internal/router/router.go b/internal/router/router.go index 5b6115fa19..cc2218d05a 100644 --- a/internal/router/router.go +++ b/internal/router/router.go @@ -55,6 +55,10 @@ type Router struct { botHandler *handler.BotHandler componentsHandler *handler.ComponentsHandler pipelineHandler *handler.PipelineHandler + + compilationTemplateHandler *handler.CompilationTemplateHandler + compilationTemplateGroupHandler *handler.CompilationTemplateGroupHandler + datasetArtifactHandler *handler.DatasetArtifactHandler } // NewRouter create router @@ -90,6 +94,9 @@ func NewRouter( botHandler *handler.BotHandler, componentsHandler *handler.ComponentsHandler, pipelineHandler *handler.PipelineHandler, + compilationTemplateHandler *handler.CompilationTemplateHandler, + compilationTemplateGroupHandler *handler.CompilationTemplateGroupHandler, + datasetArtifactHandler *handler.DatasetArtifactHandler, ) *Router { return &Router{ authHandler: authHandler, @@ -123,6 +130,10 @@ func NewRouter( botHandler: botHandler, componentsHandler: componentsHandler, pipelineHandler: pipelineHandler, + + compilationTemplateHandler: compilationTemplateHandler, + compilationTemplateGroupHandler: compilationTemplateGroupHandler, + datasetArtifactHandler: datasetArtifactHandler, } } @@ -350,6 +361,32 @@ func (r *Router) Setup(engine *gin.Engine) { datasets.GET("/:dataset_id/index", r.datasetsHandler.TraceIndex) datasets.POST("/:dataset_id/index", r.datasetsHandler.RunIndex) datasets.DELETE("/:dataset_id/index", r.datasetsHandler.DeleteIndex) + + // Knowledge-compilation wiki artifacts + datasets.HEAD("/:dataset_id/artifacts", r.datasetArtifactHandler.AnyArtifact) + datasets.GET("/:dataset_id/artifacts", r.datasetArtifactHandler.ListArtifacts) + datasets.DELETE("/:dataset_id/artifacts", r.datasetArtifactHandler.DeleteArtifacts) + datasets.GET("/:dataset_id/artifacts/topics", r.datasetArtifactHandler.ListArtifactTopics) + datasets.GET("/:dataset_id/artifacts/alteration", r.datasetArtifactHandler.GetArtifactAlteration) + datasets.GET("/:dataset_id/artifacts/graph", r.datasetArtifactHandler.GetArtifactGraph) + datasets.GET("/:dataset_id/artifacts/:page_type/:slug", r.datasetArtifactHandler.GetArtifact) + datasets.PUT("/:dataset_id/artifacts/:page_type/:slug", r.datasetArtifactHandler.UpdateArtifact) + datasets.GET("/:dataset_id/artifacts/structure", r.datasetArtifactHandler.ListStructures) + datasets.DELETE("/:dataset_id/artifacts/structure", r.datasetArtifactHandler.DeleteStructures) + + // Knowledge-compilation navigation + datasets.GET("/:dataset_id/navigation", r.datasetArtifactHandler.ListNavigation) + datasets.DELETE("/:dataset_id/navigation", r.datasetArtifactHandler.DeleteNavigation) + datasets.DELETE("/:dataset_id/navigation/:name", r.datasetArtifactHandler.DeleteNavigationNode) + datasets.GET("/:dataset_id/navigation/:name/children", r.datasetArtifactHandler.ListNavigationChildren) + + // Knowledge-compilation skills + datasets.HEAD("/:dataset_id/skills", r.datasetArtifactHandler.AnySkill) + datasets.GET("/:dataset_id/skills", r.datasetArtifactHandler.GetSkillTree) + datasets.DELETE("/:dataset_id/skills", r.datasetArtifactHandler.DeleteSkills) + datasets.GET("/:dataset_id/skills/:skill_kwd", r.datasetArtifactHandler.GetSkillPage) + datasets.DELETE("/:dataset_id/skills/:skill_kwd", r.datasetArtifactHandler.DeleteSkill) + datasets.DELETE("/:dataset_id/:index_type", r.datasetsHandler.DeleteIndex) //datasets.DELETE("/:dataset_id/graph", r.datasetsHandler.DeleteKnowledgeGraph) datasets.POST("", r.datasetsHandler.CreateDataset) @@ -391,6 +428,8 @@ func (r *Router) Setup(engine *gin.Engine) { datasets.DELETE("/:dataset_id/chunks", r.chunkHandler.StopParsing) datasets.DELETE("/:dataset_id/documents/:document_id/chunks", r.chunkHandler.RemoveChunks) datasets.PUT("/:dataset_id/documents/:document_id/metadata/config", r.datasetsHandler.UpdateDocumentMetadataConfig) + datasets.GET("/:dataset_id/documents/:document_id/structure/graph", r.datasetArtifactHandler.GetDocumentGraph) + datasets.DELETE("/:dataset_id/documents/:document_id/structure/graph", r.datasetArtifactHandler.DeleteDocumentGraph) datasets.POST("/:dataset_id/metadata/update", r.documentHandler.MetadataBatchUpdate) datasets.PATCH("/:dataset_id/documents/metadatas", r.documentHandler.UpdateDocumentMetadatas) } @@ -517,24 +556,24 @@ func (r *Router) Setup(engine *gin.Engine) { { provider.GET("", r.providerHandler.ListProviders) provider.PUT("", r.providerHandler.AddProvider) - provider.GET("/:provider_name", r.providerHandler.ShowProvider) - provider.DELETE("/:provider_name", r.providerHandler.DeleteProvider) - provider.GET("/:provider_name/models", r.providerHandler.ListModels) - provider.GET("/:provider_name/models/:model_name", r.providerHandler.ShowModel) - provider.POST("/:provider_name/instances", r.providerHandler.CreateProviderInstance) - provider.GET("/:provider_name/instances", r.providerHandler.ListProviderInstances) - provider.GET("/:provider_name/instances/:instance_name", r.providerHandler.ShowProviderInstance) - provider.GET("/:provider_name/instances/:instance_name/balance", r.providerHandler.ShowInstanceBalance) - provider.GET("/:provider_name/instances/:instance_name/connection", r.providerHandler.CheckInstanceConnection) - provider.POST("/:provider_name/connection", r.providerHandler.CheckConnection) - provider.GET("/:provider_name/instances/:instance_name/tasks", r.providerHandler.ListTasks) - provider.GET("/:provider_name/instances/:instance_name/tasks/:task_id", r.providerHandler.ShowTask) - provider.PUT("/:provider_name/instances/:instance_name", r.providerHandler.AlterProviderInstance) - provider.DELETE("/:provider_name/instances", r.providerHandler.DropProviderInstance) - provider.GET("/:provider_name/instances/:instance_name/models", r.providerHandler.ListInstanceModels) - provider.PATCH("/:provider_name/instances/:instance_name/models/*model_name", r.providerHandler.AlterModel) - provider.POST("/:provider_name/instances/:instance_name/models", r.providerHandler.AddModel) - provider.DELETE("/:provider_name/instances/:instance_name/models", r.providerHandler.DropInstanceModels) + provider.GET("/:provider_id_or_name", r.providerHandler.ShowProvider) + provider.DELETE("/:provider_id_or_name", r.providerHandler.DeleteProvider) + provider.GET("/:provider_id_or_name/models", r.providerHandler.ListModels) + provider.GET("/:provider_id_or_name/models/:model_name", r.providerHandler.ShowModel) + provider.POST("/:provider_id_or_name/instances", r.providerHandler.CreateProviderInstance) + provider.GET("/:provider_id_or_name/instances", r.providerHandler.ListProviderInstances) + provider.GET("/:provider_id_or_name/instances/:instance_id_or_name", r.providerHandler.ShowProviderInstance) + provider.GET("/:provider_id_or_name/instances/:instance_id_or_name/balance", r.providerHandler.ShowInstanceBalance) + provider.GET("/:provider_id_or_name/instances/:instance_id_or_name/connection", r.providerHandler.CheckInstanceConnection) + provider.POST("/:provider_id_or_name/connection", r.providerHandler.CheckConnection) + provider.GET("/:provider_id_or_name/instances/:instance_id_or_name/tasks", r.providerHandler.ListTasks) + provider.GET("/:provider_id_or_name/instances/:instance_id_or_name/tasks/:task_id", r.providerHandler.ShowTask) + provider.PUT("/:provider_id_or_name/instances/:instance_id_or_name", r.providerHandler.AlterProviderInstance) + provider.DELETE("/:provider_id_or_name/instances", r.providerHandler.DropProviderInstance) + provider.GET("/:provider_id_or_name/instances/:instance_id_or_name/models", r.providerHandler.ListInstanceModels) + provider.PATCH("/:provider_id_or_name/instances/:instance_id_or_name/models/*model_name", r.providerHandler.AlterModel) + provider.POST("/:provider_id_or_name/instances/:instance_id_or_name/models", r.providerHandler.AddModel) + provider.DELETE("/:provider_id_or_name/instances/:instance_id_or_name/models", r.providerHandler.DropInstanceModels) v1.POST("/chat/to_model", r.providerHandler.ChatToModel) v1.POST("/embeddings", r.providerHandler.EmbedText) v1.POST("/rerank", r.providerHandler.RerankDocument) @@ -581,6 +620,17 @@ func (r *Router) Setup(engine *gin.Engine) { // runtime.DefaultRegistry. v1.GET("/components", r.componentsHandler.Get) + // Compilation template routes + v1.GET("/compilation-templates/builtins", r.compilationTemplateHandler.ListBuiltins) + v1.GET("/compilation-templates/wiki-presets", r.compilationTemplateHandler.ListWikiPresets) + + // Compilation template group routes + v1.GET("/compilation-template-groups", r.compilationTemplateGroupHandler.List) + v1.POST("/compilation-template-groups", r.compilationTemplateGroupHandler.Save) + v1.GET("/compilation-template-groups/:group_id", r.compilationTemplateGroupHandler.Get) + v1.PUT("/compilation-template-groups/:group_id", r.compilationTemplateGroupHandler.Update) + v1.DELETE("/compilation-template-groups/:group_id", r.compilationTemplateGroupHandler.Delete) + connectors := v1.Group("/connectors") { connectors.GET("", r.connectorHandler.ListConnectors) diff --git a/internal/service/compilation_template_group_service.go b/internal/service/compilation_template_group_service.go new file mode 100644 index 0000000000..5e06b009ff --- /dev/null +++ b/internal/service/compilation_template_group_service.go @@ -0,0 +1,494 @@ +// +// Copyright 2026 The InfiniFlow Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +package service + +import ( + "context" + "fmt" + "strings" + "time" + + "ragflow/internal/dao" + "ragflow/internal/entity" + "ragflow/internal/utility" + + "gorm.io/gorm" +) + +// timeStr formats a Unix-millisecond timestamp for the read-side JSON payloads. +func timeStr(ms *int64) string { + if ms == nil { + return "" + } + return time.UnixMilli(*ms).UTC().Format("2006-01-02 15:04:05") +} + +const ( + groupScopeFile = "file" + groupScopeDataset = "dataset" + // maxGroupNameLen is the DB column size for compilation_template_group.name. + maxGroupNameLen = 128 +) + +// GroupTemplate is a child-template entry in a group create/update payload. +type GroupTemplate struct { + ID string `json:"id,omitempty"` + Name string `json:"name,omitempty"` + Description string `json:"description,omitempty"` + Kind string `json:"kind,omitempty"` + Config entity.JSONMap `json:"config,omitempty"` +} + +// GroupRequest is the create/update payload for a compilation template group. +type GroupRequest struct { + Name string `json:"name,omitempty"` + Description string `json:"description,omitempty"` + Templates []*GroupTemplate `json:"templates,omitempty"` +} + +// CompilationTemplateGroupService implements the knowledge-compilation template +// group REST operations, mirroring the Python CompilationTemplateGroupService. +type CompilationTemplateGroupService struct { + groupDAO *dao.CompilationTemplateGroupDAO + templateDAO *dao.CompilationTemplateDAO + tenantDAO *dao.TenantDAO +} + +// NewCompilationTemplateGroupService creates a CompilationTemplateGroupService. +func NewCompilationTemplateGroupService() *CompilationTemplateGroupService { + return &CompilationTemplateGroupService{ + groupDAO: dao.NewCompilationTemplateGroupDAO(), + templateDAO: dao.NewCompilationTemplateDAO(), + tenantDAO: dao.NewTenantDAO(), + } +} + +// GroupListItem is the read-side representation of a group with its nested +// templates, mirroring Python _group_to_dict. +type GroupListItem struct { + ID string `json:"id"` + Name string `json:"name"` + Description string `json:"description"` + Scope string `json:"scope"` + CreateTime string `json:"create_time,omitempty"` + UpdateTime string `json:"update_time,omitempty"` + Templates []*TemplateListItem `json:"templates"` +} + +// ListSaved returns the tenant's groups with nested templates. Mirrors Python +// list_saved(). total is derived from the full (unpaginated) result set. +func (s *CompilationTemplateGroupService) ListSaved(ctx context.Context, tenantID, keywords, scope, orderby string, desc bool) ([]*GroupListItem, error) { + groups, err := s.groupDAO.ListSaved(ctx, tenantID, keywords, scope, orderby, desc) + if err != nil { + return nil, err + } + return s.buildGroupItems(ctx, tenantID, groups) +} + +// GetSaved returns a single group with its nested templates, or nil. Mirrors +// Python get_saved(). +func (s *CompilationTemplateGroupService) GetSaved(ctx context.Context, tenantID, groupID string) (*GroupListItem, error) { + group, err := s.groupDAO.GetSaved(ctx, tenantID, groupID) + if err != nil || group == nil { + return nil, err + } + return s.buildGroupItem(ctx, tenantID, group) +} + +// CreateGroup creates a group plus its child templates. Mirrors Python +// create_group(). It returns a GroupValidationError for payload/scope problems. +// The group and all child writes are committed atomically so a mid-way failure +// cannot leave a partially-populated group behind. +func (s *CompilationTemplateGroupService) CreateGroup(ctx context.Context, tenantID string, req *GroupRequest) (*GroupListItem, error) { + if err := validateGroupPayload(req, true); err != nil { + return nil, err + } + scope, err := s.deriveScope(req.Templates) + if err != nil { + return nil, err + } + + groupID := utility.GenerateUUID() + if err := dao.DB.WithContext(ctx).Transaction(func(tx *gorm.DB) error { + group := &entity.CompilationTemplateGroup{ + ID: groupID, + TenantID: tenantID, + Name: strings.TrimSpace(req.Name), + Description: strptr(req.Description), + Scope: scope, + Status: strptr(string(entity.StatusValid)), + } + if cerr := s.groupDAO.Create(ctx, tx, group); cerr != nil { + return cerr + } + return s.insertChildren(ctx, tx, tenantID, groupID, req.Templates, nil) + }); err != nil { + return nil, err + } + return s.GetSaved(ctx, tenantID, groupID) +} + +// UpdateGroup applies a partial update to a group and reconciles its child +// templates. Mirrors Python update_group(). The field update and child +// reconciliation are committed atomically. +func (s *CompilationTemplateGroupService) UpdateGroup(ctx context.Context, tenantID, groupID string, req *GroupRequest) (*GroupListItem, error) { + existing, err := s.groupDAO.GetSaved(ctx, tenantID, groupID) + if err != nil { + return nil, err + } + if existing == nil { + return nil, nil + } + + if err := validateGroupPayload(req, false); err != nil { + return nil, err + } + + if err := dao.DB.WithContext(ctx).Transaction(func(tx *gorm.DB) error { + updates := map[string]interface{}{} + if strings.TrimSpace(req.Name) != "" { + updates["name"] = strings.TrimSpace(req.Name) + } + if req.Description != "" { + updates["description"] = req.Description + } + if req.Templates != nil { + scope, serr := s.deriveScope(req.Templates) + if serr != nil { + return serr + } + updates["scope"] = scope + } + if len(updates) > 0 { + if uerr := s.groupDAO.UpdateFields(ctx, tx, groupID, updates); uerr != nil { + return uerr + } + } + if req.Templates != nil { + return s.reconcileChildren(ctx, tx, tenantID, groupID, req.Templates) + } + return nil + }); err != nil { + return nil, err + } + return s.GetSaved(ctx, tenantID, groupID) +} + +// DeleteGroup soft-deletes a group and its valid child templates. Mirrors +// Python delete_group(). Returns (false, nil) when the group is missing. +func (s *CompilationTemplateGroupService) DeleteGroup(ctx context.Context, tenantID, groupID string) (bool, error) { + existing, err := s.groupDAO.GetSaved(ctx, tenantID, groupID) + if err != nil { + return false, err + } + if existing == nil { + return false, nil + } + if err := s.templateDAO.UpdateStatusByGroup(ctx, dao.DB, groupID, string(entity.StatusInvalid)); err != nil { + return false, err + } + if err := s.groupDAO.Delete(ctx, dao.DB, tenantID, groupID); err != nil { + return false, err + } + return true, nil +} + +// GroupValidationError is returned for group payload/scope problems so handlers +// can map it to a 400 without an HTTP 500. +type GroupValidationError struct{ msg string } + +func (e *GroupValidationError) Error() string { return e.msg } + +func groupValidationErrorf(format string, a ...interface{}) error { + return &GroupValidationError{msg: fmt.Sprintf(format, a...)} +} + +// NameExists reports whether a valid group with the given name exists for the +// tenant (optionally excluding excludeID), mirroring Python name_exists(). +func (s *CompilationTemplateGroupService) NameExists(ctx context.Context, tenantID, name string, excludeIDs ...string) (bool, error) { + excludeID := "" + if len(excludeIDs) > 0 { + excludeID = excludeIDs[0] + } + return s.groupDAO.NameExists(ctx, tenantID, name, excludeID) +} + +// ValidateGroupRequest validates the fields present in a group create/update +// payload before any DB write, mirroring the Python blueprint +// _validate_group_payload. Required-field enforcement for create happens inside +// CreateGroup (require_all=True). +func ValidateGroupRequest(req *GroupRequest) error { + return validateGroupPayload(req, false) +} + +// buildGroupItems loads child templates for many groups at once. +func (s *CompilationTemplateGroupService) buildGroupItems(ctx context.Context, tenantID string, groups []*entity.CompilationTemplateGroup) ([]*GroupListItem, error) { + items := make([]*GroupListItem, 0, len(groups)) + for _, g := range groups { + item, err := s.buildGroupItem(ctx, tenantID, g) + if err != nil { + return nil, err + } + items = append(items, item) + } + return items, nil +} + +func (s *CompilationTemplateGroupService) buildGroupItem(ctx context.Context, tenantID string, group *entity.CompilationTemplateGroup) (*GroupListItem, error) { + children, err := s.templateDAO.ListByGroup(ctx, dao.DB, group.ID) + if err != nil { + return nil, err + } + items := make([]*TemplateListItem, 0, len(children)) + for _, c := range children { + items = append(items, &TemplateListItem{ + ID: c.ID, + Name: c.Name, + Description: derefString(c.Description), + Kind: c.Kind, + Config: fillConfigDefaultLLM(ctx, s.tenantDAO, c.Config, c.TenantID), + CreateTime: timeStr(c.CreateTime), + UpdateTime: timeStr(c.UpdateTime), + }) + } + return &GroupListItem{ + ID: group.ID, + Name: group.Name, + Description: derefString(group.Description), + Scope: group.Scope, + CreateTime: timeStr(group.CreateTime), + UpdateTime: timeStr(group.UpdateTime), + Templates: items, + }, nil +} + +// deriveScope mirrors Python _derive_scope: one wiki child => dataset scope, +// otherwise file scope (with a wiki-combination guard and re-chunk tree guard). +func (s *CompilationTemplateGroupService) deriveScope(templates []*GroupTemplate) (string, error) { + if len(templates) == 0 { + return "", groupValidationErrorf("a template group must contain at least one template.") + } + artifactCount := 0 + for _, t := range templates { + if strings.TrimSpace(t.Kind) == "wiki" { + artifactCount++ + } + } + if artifactCount > 0 { + if artifactCount != 1 || len(templates) != 1 { + return "", groupValidationErrorf("a wiki template cannot be combined with other templates in the same group.") + } + return groupScopeDataset, nil + } + if err := s.enforceSingleRechunkTree(templates); err != nil { + return "", err + } + return groupScopeFile, nil +} + +// enforceSingleRechunkTree mirrors Python _enforce_single_rechunk_tree. +func (s *CompilationTemplateGroupService) enforceSingleRechunkTree(templates []*GroupTemplate) error { + rechunkTrees := 0 + for _, t := range templates { + if strings.TrimSpace(t.Kind) != "tree" { + continue + } + raptor, _ := t.Config["raptor"].(map[string]interface{}) + if b, ok := raptor["rechunk"].(bool); ok && b { + rechunkTrees++ + } + } + if rechunkTrees > 1 { + return groupValidationErrorf("only one tree template in a group may enable re-chunking.") + } + return nil +} + +// validateGroupPayload mirrors Python _validate_group_payload in the group +// blueprint. +func validateGroupPayload(req *GroupRequest, requireAll bool) error { + if requireAll { + if strings.TrimSpace(req.Name) == "" { + return groupValidationErrorf("missing required field: name.") + } + if len(req.Templates) == 0 { + return groupValidationErrorf("missing required field: templates.") + } + } + if strings.TrimSpace(req.Name) != "" { + if len([]byte(req.Name)) > maxGroupNameLen { + return groupValidationErrorf("template group name is too long.") + } + } + if len(req.Description) > 1024 { + return groupValidationErrorf("invalid template group description.") + } + if req.Templates != nil { + if len(req.Templates) == 0 { + return groupValidationErrorf("a template group must contain at least one template.") + } + seen := map[string]struct{}{} + for _, child := range req.Templates { + if child == nil { + return groupValidationErrorf("invalid template entry in group.") + } + payload := map[string]interface{}{ + "name": child.Name, "description": child.Description, + "kind": child.Kind, "config": child.Config, + } + if err := ValidateTemplatePayload(payload, true); err != nil { + return err + } + name := strings.TrimSpace(child.Name) + if _, dup := seen[name]; dup { + return groupValidationErrorf("template name '%s' is duplicated in this group.", name) + } + seen[name] = struct{}{} + } + } + return nil +} + +// insertChildren inserts new child templates. usedForReconcile controls whether +// the duplicate-name guard applies (create always; reconcile passes seen set). +func (s *CompilationTemplateGroupService) insertChildren(ctx context.Context, db *gorm.DB, tenantID, groupID string, templates []*GroupTemplate, seen map[string]struct{}) error { + for _, child := range templates { + name := strings.TrimSpace(child.Name) + if exists, err := s.templateDAO.NameExistsInGroup(ctx, tenantID, groupID, name, ""); err != nil { + return err + } else if exists { + return groupValidationErrorf("template name '%s' already exists in this group.", name) + } + desc := child.Description + config := fillConfigDefaultLLM(ctx, s.tenantDAO, child.Config, &tenantID) + if config == nil { + config = entity.JSONMap{} + } + valid := string(entity.StatusValid) + tmpl := &entity.CompilationTemplate{ + ID: utility.GenerateUUID(), + TenantID: &tenantID, + GroupID: &groupID, + Name: name, + Kind: strings.TrimSpace(child.Kind), + Config: config, + Status: &valid, + } + if desc != "" { + tmpl.Description = &desc + } + if err := s.templateDAO.Save(ctx, db, tmpl); err != nil { + return err + } + } + return nil +} + +// reconcileChildren mirrors the Python update_group child reconciliation: +// existing children (matched by id, or by submitted order for legacy clients) +// are updated in place, new ones inserted, and removed ones soft-deleted. +func (s *CompilationTemplateGroupService) reconcileChildren(ctx context.Context, db *gorm.DB, tenantID, groupID string, templates []*GroupTemplate) error { + current, err := s.templateDAO.ListByGroup(ctx, db, groupID) + if err != nil { + return err + } + currentByID := map[string]*entity.CompilationTemplate{} + for _, c := range current { + currentByID[c.ID] = c + } + seenNames := map[string]struct{}{} + retained := map[string]struct{}{} + + for index, child := range templates { + name := strings.TrimSpace(child.Name) + if _, dup := seenNames[name]; dup { + return groupValidationErrorf("template name '%s' is duplicated in this group.", name) + } + seenNames[name] = struct{}{} + + var target *entity.CompilationTemplate + if child.ID != "" { + target = currentByID[child.ID] + if target == nil { + return groupValidationErrorf("template %s does not belong to this group.", child.ID) + } + } else if index < len(current) { + // Positional fallback for legacy clients that omit IDs. Skip rows + // already retained by an explicit ID above so a mixed payload cannot + // bind two submitted entries to one existing row. + if _, taken := retained[current[index].ID]; !taken { + target = current[index] + } + } + + config := fillConfigDefaultLLM(ctx, s.tenantDAO, child.Config, &tenantID) + if config == nil { + config = entity.JSONMap{} + } + desc := child.Description + if target != nil { + updates := map[string]interface{}{ + "name": name, "kind": strings.TrimSpace(child.Kind), "config": config, + } + if desc != "" { + updates["description"] = desc + } + if err := s.templateDAO.UpdateFields(ctx, db, target.ID, updates); err != nil { + return err + } + retained[target.ID] = struct{}{} + continue + } + newID := utility.GenerateUUID() + valid := string(entity.StatusValid) + tmpl := &entity.CompilationTemplate{ + ID: newID, + TenantID: &tenantID, + GroupID: &groupID, + Name: name, + Kind: strings.TrimSpace(child.Kind), + Config: config, + Status: &valid, + } + if desc != "" { + tmpl.Description = &desc + } + if err := s.templateDAO.Save(ctx, db, tmpl); err != nil { + return err + } + retained[newID] = struct{}{} + } + + for _, c := range current { + if _, keep := retained[c.ID]; !keep { + if err := s.templateDAO.UpdateStatusByID(ctx, db, c.ID, string(entity.StatusInvalid)); err != nil { + return err + } + // Mirror Python _purge_stale_invalid_children: permanently drop + // any stale invalid non-builtin template of the same name, scoped to + // this group so a same-named template in another group is untouched. + if c.Name != "" && !c.IsBuiltin { + if err := s.templateDAO.HardDeleteOrphansByName(ctx, db, tenantID, groupID, c.Name); err != nil { + return err + } + } + } + } + return nil +} + +func strptr(s string) *string { return &s } diff --git a/internal/service/compilation_template_service.go b/internal/service/compilation_template_service.go new file mode 100644 index 0000000000..bbe04fe04b --- /dev/null +++ b/internal/service/compilation_template_service.go @@ -0,0 +1,379 @@ +// +// Copyright 2026 The InfiniFlow Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +package service + +import ( + "context" + "errors" + "fmt" + "os" + "path/filepath" + "sort" + "strings" + "unicode" + + "ragflow/internal/dao" + "ragflow/internal/entity" + "ragflow/internal/utility" + + "gopkg.in/yaml.v3" + "gorm.io/gorm" +) + +// fillConfigDefaultLLM mirrors Python CompilationTemplateService. +// fill_config_default_llm: when the config has no explicit llm_id it lazily +// fills the tenant's default chat model id (read-side only; the DB row is left +// untouched). +func fillConfigDefaultLLM(ctx context.Context, tenantDAO *dao.TenantDAO, config entity.JSONMap, tenantID *string) entity.JSONMap { + if len(config) == 0 || tenantID == nil || *tenantID == "" { + return config + } + if _, ok := config["llm_id"]; ok { + return config + } + tenant, err := tenantDAO.GetByID(ctx, dao.DB, *tenantID) + if err != nil || tenant == nil || tenant.TenantLLMID == nil { + return config + } + out := make(entity.JSONMap, len(config)+1) + for k, v := range config { + out[k] = v + } + out["llm_id"] = *tenant.TenantLLMID + return out +} + +// capitalizeTitle capitalizes the first letter of a lowercase section name for +// error messages (replaces deprecated strings.Title). +func capitalizeTitle(s string) string { + if s == "" { + return s + } + r := []rune(s) + r[0] = unicode.ToUpper(r[0]) + return string(r) +} + +// TemplateListItem is the read-side representation of a compilation template, +// mirroring Python CompilationTemplateService._to_saved_dict. +type TemplateListItem struct { + ID string `json:"id"` + Name string `json:"name"` + Description string `json:"description"` + Kind string `json:"kind"` + Config entity.JSONMap `json:"config"` + CreateTime string `json:"create_time,omitempty"` + UpdateTime string `json:"update_time,omitempty"` +} + +// BuiltinTemplate is the palette representation of a built-in template, +// mirroring Python CompilationTemplateService._to_builtin_dict. +type BuiltinTemplate struct { + ID string `json:"id"` + Kind string `json:"kind"` + DisplayName string `json:"display_name"` + Description string `json:"description"` + Config entity.JSONMap `json:"config"` +} + +// WikiPreset is a wiki page-structure preset loaded from YAML, mirroring +// Python load_wiki_presets_from_files. +type WikiPreset struct { + ID string `json:"id"` + Topic string `json:"topic"` + Instruction string `json:"instruction"` + PageExample string `json:"page_example"` +} + +// CompilationTemplateService implements the read-side compilation template +// operations (builtins palette + wiki presets) used by the REST APIs. +type CompilationTemplateService struct { + templateDAO *dao.CompilationTemplateDAO + tenantDAO *dao.TenantDAO +} + +// NewCompilationTemplateService creates a CompilationTemplateService. +func NewCompilationTemplateService() *CompilationTemplateService { + return &CompilationTemplateService{ + templateDAO: dao.NewCompilationTemplateDAO(), + tenantDAO: dao.NewTenantDAO(), + } +} + +// ListBuiltins returns the built-in template palette with the tenant's default +// LLM id lazily filled in, mirroring the Python list_builtins blueprint flow +// (list from DB; if empty, seed from files and retry; then fill default LLM). +func (s *CompilationTemplateService) ListBuiltins(ctx context.Context, tenantID string) ([]*BuiltinTemplate, error) { + templates, err := s.templateDAO.ListBuiltins(ctx) + if err != nil { + return nil, err + } + if len(templates) == 0 { + if err := s.seedBuiltins(ctx); err != nil { + return nil, err + } + templates, err = s.templateDAO.ListBuiltins(ctx) + if err != nil { + return nil, err + } + } + out := make([]*BuiltinTemplate, 0, len(templates)) + for _, t := range templates { + config := fillConfigDefaultLLM(ctx, s.tenantDAO, t.Config, t.TenantID) + out = append(out, &BuiltinTemplate{ + ID: t.ID, + Kind: t.Kind, + DisplayName: t.Name, + Description: derefString(t.Description), + Config: config, + }) + } + return s.sortBuiltins(out), nil +} + +// LoadWikiPresets loads the wiki page-structure presets from the +// init_data/compilation_templates/wiki/*.yaml files, filesystem-fresh per call. +func (s *CompilationTemplateService) LoadWikiPresets() ([]*WikiPreset, error) { + wikiDir := filepath.Join(utility.GetProjectBaseDirectory(), + "api", "db", "init_data", "compilation_templates", "wiki") + entries, err := os.ReadDir(wikiDir) + if err != nil { + if os.IsNotExist(err) { + return []*WikiPreset{}, nil + } + return nil, err + } + var presets []*WikiPreset + for _, entry := range entries { + if entry.IsDir() || !strings.HasSuffix(entry.Name(), ".yaml") && !strings.HasSuffix(entry.Name(), ".yml") { + continue + } + path := filepath.Join(wikiDir, entry.Name()) + data, rerr := os.ReadFile(path) + if rerr != nil { + continue + } + var doc map[string]interface{} + if yerr := yaml.Unmarshal(data, &doc); yerr != nil || doc == nil { + continue + } + presets = append(presets, &WikiPreset{ + ID: strings.TrimSuffix(entry.Name(), filepath.Ext(entry.Name())), + Topic: strings.TrimSpace(yamlStr(doc["topic"])), + Instruction: yamlStr(doc["instruction"]), + PageExample: yamlStr(doc["page_example"]), + }) + } + return presets, nil +} + +// ValidateTemplatePayload validates a single template payload, mirroring the +// Python compilation_template_validation module. It returns an error describing +// the first problem found. +func ValidateTemplatePayload(req map[string]interface{}, requireAll bool) error { + if requireAll { + for _, key := range []string{"name", "kind", "config"} { + if _, ok := req[key]; !ok { + return fmt.Errorf("missing required field: %s.", key) + } + } + } + if name, ok := req["name"]; ok { + nameStr, ok2 := name.(string) + if !ok2 || strings.TrimSpace(nameStr) == "" || len([]byte(nameStr)) > 128 { + return errors.New("invalid template name.") + } + } + if desc, ok := req["description"]; ok { + if descStr, ok2 := desc.(string); !ok2 || len(descStr) > 1024 { + return errors.New("invalid template description.") + } + } + if kind, ok := req["kind"]; ok { + if kindStr, ok2 := kind.(string); !ok2 || kindStr == "" { + return errors.New("invalid template kind.") + } + } + config, hasConfig := req["config"] + if hasConfig { + configMap, ok := config.(map[string]interface{}) + if !ok { + return errors.New("invalid template config.") + } + if len(fmt.Sprint(configMap["global_rules"])) > 4096 { + return errors.New("global compilation rules is too long.") + } + for _, section := range []string{"entity", "relation"} { + sec, _ := configMap[section].(map[string]interface{}) + fields, _ := sec["fields"].([]interface{}) + seen := map[string]struct{}{} + for _, f := range fields { + fm, _ := f.(map[string]interface{}) + fieldType := strings.TrimSpace(yamlStr(fm["type"])) + if fieldType == "" { + return fmt.Errorf("%s type is required.", capitalizeTitle(section)) + } + if _, dup := seen[fieldType]; dup { + return fmt.Errorf("%s type can not be duplicated.", capitalizeTitle(section)) + } + seen[fieldType] = struct{}{} + if strings.TrimSpace(yamlStr(fm["description"])) == "" { + return fmt.Errorf("%s field description is required.", capitalizeTitle(section)) + } + if len(yamlStr(fm["description"])) > 1024 { + return fmt.Errorf("%s field description is too long.", capitalizeTitle(section)) + } + if len(yamlStr(fm["rule"])) > 1024 { + return fmt.Errorf("%s field rule is too long.", capitalizeTitle(section)) + } + } + } + if configMap["kind"] == "wiki" || req["kind"] == "wiki" { + for _, group := range []string{"claim", "concept"} { + sec, _ := configMap[group].(map[string]interface{}) + fields, _ := sec["fields"].([]interface{}) + for _, f := range fields { + fm, _ := f.(map[string]interface{}) + switch group { + case "claim": + if strings.TrimSpace(yamlStr(fm["statement"])) == "" { + return errors.New("claim statement is required.") + } + if strings.TrimSpace(yamlStr(fm["subject"])) == "" { + return errors.New("claim subject is required.") + } + if len(yamlStr(fm["statement"])) > 1024 { + return errors.New("claim statement is too long.") + } + if len(yamlStr(fm["subject"])) > 1024 { + return errors.New("claim subject is too long.") + } + case "concept": + if strings.TrimSpace(yamlStr(fm["term"])) == "" { + return errors.New("concept term is required.") + } + if strings.TrimSpace(yamlStr(fm["definition_excerpt"])) == "" { + return errors.New("concept definition excerpt is required.") + } + if len(yamlStr(fm["term"])) > 1024 { + return errors.New("concept term is too long.") + } + if len(yamlStr(fm["definition_excerpt"])) > 1024 { + return errors.New("concept definition excerpt is too long.") + } + } + } + } + } + } + return nil +} + +// sortBuiltins mirrors Python _sort_builtins: empty-kind entries first, then by +// display name. +func (s *CompilationTemplateService) sortBuiltins(templates []*BuiltinTemplate) []*BuiltinTemplate { + sort.SliceStable(templates, func(i, j int) bool { + emptyI := templates[i].Kind == "empty" || templates[i].ID == "empty" + emptyJ := templates[j].Kind == "empty" || templates[j].ID == "empty" + if emptyI != emptyJ { + return emptyI + } + return strings.ToLower(templates[i].DisplayName) < strings.ToLower(templates[j].DisplayName) + }) + return templates +} + +// seedBuiltins loads the built-in templates from the init_data YAML files and +// upserts them (as a safety net when the DB has no built-ins yet), mirroring +// Python seed_builtins_from_files. +func (s *CompilationTemplateService) seedBuiltins(ctx context.Context) error { + dir := filepath.Join(utility.GetProjectBaseDirectory(), + "api", "db", "init_data", "compilation_templates") + entries, err := os.ReadDir(dir) + if err != nil { + return nil // directory absent is acceptable; nothing to seed + } + for _, entry := range entries { + if entry.IsDir() || !strings.HasSuffix(entry.Name(), ".yaml") && !strings.HasSuffix(entry.Name(), ".yml") { + continue + } + data, rerr := os.ReadFile(filepath.Join(dir, entry.Name())) + if rerr != nil { + continue + } + var doc struct { + Kind string `yaml:"kind"` + DisplayName string `yaml:"display_name"` + Description string `yaml:"description"` + Config map[string]interface{} `yaml:"config"` + } + if yerr := yaml.Unmarshal(data, &doc); yerr != nil { + continue + } + if doc.Kind == "" || doc.DisplayName == "" || doc.Config == nil { + continue + } + id := strings.TrimSuffix(entry.Name(), filepath.Ext(entry.Name())) + desc := doc.Description + valid := string(entity.StatusValid) + tmpl := &entity.CompilationTemplate{ + ID: id, + Name: doc.DisplayName, + Kind: doc.Kind, + Config: entity.JSONMap(doc.Config), + IsBuiltin: true, + Status: &valid, + } + if doc.Description != "" { + tmpl.Description = &desc + } + _ = s.upsertBuiltin(ctx, tmpl) + } + return nil +} + +func (s *CompilationTemplateService) upsertBuiltin(ctx context.Context, t *entity.CompilationTemplate) error { + existing, err := s.templateDAO.GetByID(ctx, t.ID) + if errors.Is(err, gorm.ErrRecordNotFound) { + return s.templateDAO.Save(ctx, dao.DB, t) + } + if err != nil { + return err + } + m := map[string]interface{}{ + "name": t.Name, "kind": t.Kind, "config": t.Config, + "description": t.Description, "is_builtin": true, + "status": t.Status, + } + return s.templateDAO.UpdateFields(ctx, dao.DB, existing.ID, m) +} + +func derefString(p *string) string { + if p == nil { + return "" + } + return *p +} + +// yamlStr safely stringifies a YAML value, returning "" when the key is absent +// (instead of fmt.Sprint(nil) which yields ""). +func yamlStr(v interface{}) string { + if v == nil { + return "" + } + return fmt.Sprint(v) +} diff --git a/internal/service/dataset_artifact_service.go b/internal/service/dataset_artifact_service.go new file mode 100644 index 0000000000..38a524fcf0 --- /dev/null +++ b/internal/service/dataset_artifact_service.go @@ -0,0 +1,846 @@ +// +// Copyright 2026 The InfiniFlow Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +package service + +import ( + "context" + "encoding/json" + "fmt" + "sort" + + "ragflow/internal/engine" + "ragflow/internal/engine/types" +) + +// Compile keyword constants used by the knowledge-compilation artifacts stored +// in the document engine. +const ( + CompileKwdWikiPage = "wiki_page" + CompileKwdWikiEntity = "wiki_entity" + CompileKwdWikiRelation = "wiki_relation" + CompileKwdWikiAlter = "wiki_alter" + CompileKwdSkill = "skill" + CompileKwdSkillAll = "skill_all" + CompileKwdDatasetNav = "dataset_nav" + CompileKwdRaptorGraph = "raptor_graph" + + // Structure / graph compilation keywords. + CompileKwdStructure = "structure" + CompileKwdStructureIndex = "structureIndex" + CompileKwdStructureEntity = "structureEntity" + CompileKwdStructureRelation = "structureRelation" + CompileKwdStructureCommunity = "structureCommunity" + + // Field name for the structure index type discriminator. + FieldStructureIndexType = "structure_index_type" + FieldStructureKind = "structure_kind" + FieldPageID = "page_id" + FieldGraphType = "graph_type" +) + +// DatasetArtifactService reads knowledge-compilation artifacts (wiki pages, +// graphs, structures, navigation, skills) from the document engine. +type DatasetArtifactService struct{} + +// NewDatasetArtifactService creates a DatasetArtifactService. +func NewDatasetArtifactService() *DatasetArtifactService { + return &DatasetArtifactService{} +} + +// wikiIndexName returns the tenant document index name. +func wikiIndexName(tenantID string) string { + return fmt.Sprintf("ragflow_%s", tenantID) +} + +// searchCompiled runs a filtered search over the tenant's document index for +// the given dataset, returning the matching chunks. +func (s *DatasetArtifactService) searchCompiled(ctx context.Context, tenantID, datasetID string, filter map[string]interface{}, selectFields []string, offset, limit int, orderBy *types.OrderByExpr) ([]map[string]interface{}, int64, error) { + docEngine := engine.Get() + if docEngine == nil { + return nil, 0, fmt.Errorf("document engine is not initialized") + } + // Copy caller filters first, then pin the dataset scope so kb_id always wins + // even if a caller passes its own kb_id. + merged := make(map[string]interface{}, len(filter)+1) + for k, v := range filter { + merged[k] = v + } + merged["kb_id"] = []string{datasetID} + res, err := docEngine.Search(ctx, &types.SearchRequest{ + IndexNames: []string{wikiIndexName(tenantID)}, + KbIDs: []string{datasetID}, + Offset: offset, + Limit: limit, + SelectFields: selectFields, + Filter: merged, + OrderBy: orderBy, + }) + if err != nil { + return nil, 0, err + } + if res == nil { + return nil, 0, nil + } + return res.Chunks, res.Total, nil +} + +// intValue coerces an engine field value into an int, tolerating the numeric +// types a document engine may decode a number as (float64, int, int64, +// json.Number) as well as a string form and a list-valued form (first element +// wins). It returns 0 for anything else. +func intValue(v interface{}) int { + if v == nil { + return 0 + } + switch n := v.(type) { + case int: + return n + case int64: + return int(n) + case int32: + return int(n) + case float64: + return int(n) + case float32: + return int(n) + case json.Number: + if iv, err := n.Int64(); err == nil { + return int(iv) + } + if fv, err := n.Float64(); err == nil { + return int(fv) + } + case string: + var iv int + if _, err := fmt.Sscanf(n, "%d", &iv); err == nil { + return iv + } + case []interface{}: + if len(n) > 0 { + return intValue(n[0]) + } + } + return 0 +} + +// HasWiki reports whether the dataset has any compiled wiki artifact. +func (s *DatasetArtifactService) HasWiki(ctx context.Context, tenantID, datasetID string) (bool, error) { + _, total, err := s.searchCompiled(ctx, tenantID, datasetID, + map[string]interface{}{"compile_kwd": []string{CompileKwdWikiPage}}, nil, 0, 1, nil) + if err != nil { + return false, err + } + return total > 0, nil +} + +// HasSkill reports whether the dataset has any compiled skill artifact. +func (s *DatasetArtifactService) HasSkill(ctx context.Context, tenantID, datasetID string) (bool, error) { + _, total, err := s.searchCompiled(ctx, tenantID, datasetID, + map[string]interface{}{"compile_kwd": []string{CompileKwdSkillAll}}, nil, 0, 1, nil) + if err != nil { + return false, err + } + return total > 0, nil +} + +// WikiPageItem is a single wiki page summary returned by ListWikiPages. +type WikiPageItem struct { + Slug string `json:"slug"` + Title string `json:"title"` + PageType string `json:"page_type"` + Topic string `json:"topic"` + Summary string `json:"summary"` +} + +// ListWikiPages lists wiki pages for a dataset with optional page_type/topic +// filters and pagination. +func (s *DatasetArtifactService) ListWikiPages(ctx context.Context, tenantID, datasetID, pageType, topic string, page, pageSize int) ([]WikiPageItem, int64, error) { + filter := map[string]interface{}{"compile_kwd": []string{CompileKwdWikiPage}} + if pageType != "" { + filter["page_type_kwd"] = []string{pageType} + } + if topic != "" { + filter["topic_kwd"] = []string{topic} + } + offset := (page - 1) * pageSize + chunks, total, err := s.searchCompiled(ctx, tenantID, datasetID, filter, + []string{"slug_kwd", "title_kwd", "page_type_kwd", "topic_kwd", "outlinks_int", "summary_with_weight"}, + offset, pageSize, (&types.OrderByExpr{}).Desc("outlinks_int").Asc("title_kwd")) + if err != nil { + return nil, 0, err + } + items := make([]WikiPageItem, 0, len(chunks)) + for _, c := range chunks { + items = append(items, WikiPageItem{ + Slug: firstStringValue(c["slug_kwd"]), + Title: firstStringValue(c["title_kwd"]), + PageType: firstStringValue(c["page_type_kwd"]), + Topic: firstStringValue(c["topic_kwd"]), + Summary: firstStringValue(c["summary_with_weight"]), + }) + } + return items, total, nil +} + +// WikiPageDetail is the full wiki page payload. +type WikiPageDetail struct { + Slug string `json:"slug"` + Title string `json:"title"` + PageType string `json:"page_type"` + Topic string `json:"topic"` + ContentMd string `json:"content_md"` + Summary string `json:"summary"` + EntityNames []string `json:"entity_names"` + Outlinks []string `json:"outlinks"` + RelatedKbPages []string `json:"related_kb_pages"` + SourceChunkIDs []string `json:"source_chunk_ids"` + SourceDocIDs []string `json:"source_doc_ids"` +} + +// GetWikiPage returns a single wiki page by page_type and slug. +func (s *DatasetArtifactService) GetWikiPage(ctx context.Context, tenantID, datasetID, pageType, slug string) (*WikiPageDetail, error) { + slugKwd := pageType + "/" + slug + filter := map[string]interface{}{ + "compile_kwd": []string{CompileKwdWikiPage}, + "page_type_kwd": []string{pageType}, + "slug_kwd": []string{slugKwd}, + } + chunks, _, err := s.searchCompiled(ctx, tenantID, datasetID, filter, + []string{"slug_kwd", "title_kwd", "page_type_kwd", "topic_kwd", "content_with_weight", + "summary_with_weight", "entity_names_kwd", "outlinks_kwd", "related_kb_pages_kwd", + "source_chunk_ids", "source_doc_ids"}, + 0, 1, nil) + if err != nil { + return nil, err + } + if len(chunks) == 0 { + return nil, nil + } + c := chunks[0] + detail := &WikiPageDetail{ + Slug: firstStringValue(c["slug_kwd"]), + Title: firstStringValue(c["title_kwd"]), + PageType: firstStringValue(c["page_type_kwd"]), + Topic: firstStringValue(c["topic_kwd"]), + ContentMd: firstStringValue(c["content_with_weight"]), + Summary: firstStringValue(c["summary_with_weight"]), + EntityNames: toStringSlice(c["entity_names_kwd"]), + Outlinks: toStringSlice(c["outlinks_kwd"]), + RelatedKbPages: toStringSlice(c["related_kb_pages_kwd"]), + SourceChunkIDs: toStringSlice(c["source_chunk_ids"]), + SourceDocIDs: toStringSlice(c["source_doc_ids"]), + } + return detail, nil +} + +// UpdateWikiPage performs a partial field update of a wiki page's content, +// title and outlinks through the document engine, then returns the refreshed +// page. +func (s *DatasetArtifactService) UpdateWikiPage(ctx context.Context, tenantID, datasetID, pageType, slug, contentMd, title string, outlinks []string) (*WikiPageDetail, error) { + docEngine := engine.Get() + if docEngine == nil { + return nil, fmt.Errorf("document engine is not initialized") + } + slugKwd := pageType + "/" + slug + filter := map[string]interface{}{ + "compile_kwd": []string{CompileKwdWikiPage}, + "page_type_kwd": []string{pageType}, + "slug_kwd": []string{slugKwd}, + } + chunks, _, err := s.searchCompiled(ctx, tenantID, datasetID, filter, []string{"id"}, 0, 1, nil) + if err != nil { + return nil, err + } + if len(chunks) == 0 { + return nil, nil + } + id, ok := chunks[0]["id"].(string) + if !ok || id == "" { + return nil, fmt.Errorf("wiki page chunk has no id") + } + update := map[string]interface{}{} + if contentMd != "" { + update["content_with_weight"] = contentMd + } + if title != "" { + update["title_kwd"] = title + } + if len(outlinks) > 0 { + update["outlinks_kwd"] = outlinks + } + if len(update) > 0 { + cond := map[string]interface{}{"id": id, "kb_id": datasetID} + if err := docEngine.UpdateChunks(ctx, cond, update, wikiIndexName(tenantID), datasetID); err != nil { + return nil, err + } + } + return s.GetWikiPage(ctx, tenantID, datasetID, pageType, slug) +} + +// WikiTopicItem is a single topic entry returned by ListWikiTopics. +type WikiTopicItem struct { + Topic string `json:"topic"` + Title string `json:"title"` + Slug string `json:"slug"` + PageCount int `json:"page_count"` +} + +// ListWikiTopics aggregates wiki topics for a dataset. +func (s *DatasetArtifactService) ListWikiTopics(ctx context.Context, tenantID, datasetID string) ([]WikiTopicItem, int64, error) { + filter := map[string]interface{}{ + "compile_kwd": []string{CompileKwdWikiPage}, + "page_type_kwd": []string{"concept", "entity"}, + } + chunks, _, err := s.searchCompiled(ctx, tenantID, datasetID, filter, + []string{"topic_kwd", "title_kwd", "slug_kwd"}, 0, 1000, nil) + if err != nil { + return nil, 0, err + } + counts := map[string]int{} + metas := map[string]WikiTopicItem{} + for _, c := range chunks { + t := firstStringValue(c["topic_kwd"]) + if t == "" { + continue + } + counts[t]++ + if _, ok := metas[t]; !ok { + metas[t] = WikiTopicItem{ + Topic: t, + Title: firstStringValue(c["title_kwd"]), + Slug: firstStringValue(c["slug_kwd"]), + } + } + } + items := make([]WikiTopicItem, 0, len(metas)) + for t, it := range metas { + it.PageCount = counts[t] + items = append(items, it) + } + sort.Slice(items, func(i, j int) bool { return items[i].Topic < items[j].Topic }) + return items, int64(len(items)), nil +} + +// WikiGraph is the entity/relation graph for a dataset's wiki artifacts. +type WikiGraph struct { + Entities []WikiGraphEntity `json:"entities"` + Relations []WikiGraphRelation `json:"relations"` +} + +// WikiGraphEntity is a single wiki graph entity. +type WikiGraphEntity struct { + Slug string `json:"slug"` + Name string `json:"name"` + Aliases []string `json:"aliases"` + Description string `json:"description"` + Type string `json:"type"` + Weight int `json:"weight"` + SourceChunkIDs []string `json:"source_chunk_ids"` +} + +// WikiGraphRelation is a single wiki graph relation. +type WikiGraphRelation struct { + From string `json:"from"` + To string `json:"to"` +} + +// GetWikiGraph returns the wiki entity/relation graph for a dataset. +func (s *DatasetArtifactService) GetWikiGraph(ctx context.Context, tenantID, datasetID string) (*WikiGraph, error) { + entityChunks, _, err := s.searchCompiled(ctx, tenantID, datasetID, + map[string]interface{}{"compile_kwd": []string{CompileKwdWikiEntity}}, + []string{"slug_kwd", "title_kwd", "aliases_kwd", "description_with_weight", "entity_type_kwd", "weight_int", "source_chunk_ids"}, + 0, 1000, (&types.OrderByExpr{}).Desc("weight_int")) + if err != nil { + return nil, err + } + fromKwds := make([]string, 0, len(entityChunks)) + for _, c := range entityChunks { + fromKwds = append(fromKwds, firstStringValue(c["slug_kwd"])) + } + graph := &WikiGraph{Entities: []WikiGraphEntity{}, Relations: []WikiGraphRelation{}} + for _, c := range entityChunks { + w := intValue(c["weight_int"]) + graph.Entities = append(graph.Entities, WikiGraphEntity{ + Slug: firstStringValue(c["slug_kwd"]), + Name: firstStringValue(c["title_kwd"]), + Aliases: toStringSlice(c["aliases_kwd"]), + Description: firstStringValue(c["description_with_weight"]), + Type: firstStringValue(c["entity_type_kwd"]), + Weight: w, + SourceChunkIDs: toStringSlice(c["source_chunk_ids"]), + }) + } + if len(fromKwds) > 0 { + relChunks, _, err := s.searchCompiled(ctx, tenantID, datasetID, + map[string]interface{}{"compile_kwd": []string{CompileKwdWikiRelation}, "from_kwd": fromKwds}, + []string{"from_kwd", "to_kwd"}, 0, 10000, nil) + if err != nil { + return nil, err + } + for _, c := range relChunks { + graph.Relations = append(graph.Relations, WikiGraphRelation{ + From: firstStringValue(c["from_kwd"]), + To: firstStringValue(c["to_kwd"]), + }) + } + } + return graph, nil +} + +// WikiAlteration is the alteration summary for a dataset's wiki artifacts. +type WikiAlteration struct { + Removed int `json:"removed"` + NewlyUploaded int `json:"newly_uploaded"` + RemovedDocIDs []string `json:"removed_doc_ids"` + NewlyUploadedDocIDs []string `json:"newly_uploaded_doc_ids"` + InvolvedDocIDs []string `json:"involved_doc_ids"` + EligibleDocIDs []string `json:"eligible_doc_ids"` +} + +// GetWikiAlteration returns the wiki alteration summary for a dataset. +func (s *DatasetArtifactService) GetWikiAlteration(ctx context.Context, tenantID, datasetID string) (*WikiAlteration, error) { + chunks, _, err := s.searchCompiled(ctx, tenantID, datasetID, + map[string]interface{}{"compile_kwd": []string{CompileKwdWikiPage}}, + []string{"source_doc_ids"}, 0, 10000, nil) + if err != nil { + return nil, err + } + involved := map[string]struct{}{} + for _, c := range chunks { + for _, d := range toStringSlice(c["source_doc_ids"]) { + involved[d] = struct{}{} + } + } + ids := make([]string, 0, len(involved)) + for d := range involved { + ids = append(ids, d) + } + sort.Strings(ids) + return &WikiAlteration{ + Removed: 0, + NewlyUploaded: 0, + RemovedDocIDs: []string{}, + NewlyUploadedDocIDs: []string{}, + InvolvedDocIDs: ids, + EligibleDocIDs: ids, + }, nil +} + +// ClearWiki deletes all wiki artifacts for a dataset. +func (s *DatasetArtifactService) ClearWiki(ctx context.Context, tenantID, datasetID string) (map[string]int, error) { + docEngine := engine.Get() + if docEngine == nil { + return nil, fmt.Errorf("document engine is not initialized") + } + kwds := []string{CompileKwdWikiPage, CompileKwdWikiEntity, CompileKwdWikiRelation, CompileKwdWikiAlter} + deleted := map[string]int{} + for _, kwd := range kwds { + chunks, _, err := s.searchCompiled(ctx, tenantID, datasetID, + map[string]interface{}{"compile_kwd": []string{kwd}}, []string{"id"}, 0, 10000, nil) + if err != nil { + return nil, err + } + if len(chunks) == 0 { + deleted[kwd] = 0 + continue + } + ids := make([]string, 0, len(chunks)) + for _, c := range chunks { + if id, ok := c["id"].(string); ok { + ids = append(ids, id) + } + } + cond := map[string]interface{}{"id": ids, "kb_id": datasetID} + if _, err := docEngine.DeleteChunks(ctx, cond, wikiIndexName(tenantID), datasetID); err != nil { + return nil, err + } + deleted[kwd] = len(ids) + } + return deleted, nil +} + +// StructureItem is a single compiled structure entry for a dataset. +type StructureItem struct { + PageID string `json:"page_id"` + StructureKind string `json:"structure_kind"` + StructureIndexType string `json:"structure_index_type"` + Data string `json:"data"` +} + +// ListStructures returns the compiled structures of a dataset, filtered by +// optional structure_kind and structure_index_type. +func (s *DatasetArtifactService) ListStructures(ctx context.Context, tenantID, datasetID, structureKind, structureIndexType string) ([]StructureItem, int64, error) { + filter := map[string]interface{}{"compile_kwd": []string{CompileKwdStructure}} + if structureKind != "" { + filter[FieldStructureKind] = []string{structureKind} + } + if structureIndexType != "" { + filter[FieldStructureIndexType] = []string{structureIndexType} + } + chunks, total, err := s.searchCompiled(ctx, tenantID, datasetID, filter, + []string{FieldPageID, FieldStructureKind, FieldStructureIndexType, "content_with_weight"}, 0, 10000, nil) + if err != nil { + return nil, 0, err + } + items := make([]StructureItem, 0, len(chunks)) + for _, c := range chunks { + items = append(items, StructureItem{ + PageID: firstStringValue(c[FieldPageID]), + StructureKind: firstStringValue(c[FieldStructureKind]), + StructureIndexType: firstStringValue(c[FieldStructureIndexType]), + Data: firstStringValue(c["content_with_weight"]), + }) + } + return items, total, nil +} + +// DeleteStructures deletes the compiled structures of a dataset, optionally +// scoped by structure_kind and structure_index_type. +func (s *DatasetArtifactService) DeleteStructures(ctx context.Context, tenantID, datasetID, structureKind, structureIndexType string) (int, error) { + docEngine := engine.Get() + if docEngine == nil { + return 0, fmt.Errorf("document engine is not initialized") + } + filter := map[string]interface{}{"compile_kwd": []string{CompileKwdStructure}} + if structureKind != "" { + filter[FieldStructureKind] = []string{structureKind} + } + if structureIndexType != "" { + filter[FieldStructureIndexType] = []string{structureIndexType} + } + chunks, _, err := s.searchCompiled(ctx, tenantID, datasetID, filter, []string{"id"}, 0, 10000, nil) + if err != nil { + return 0, err + } + if len(chunks) == 0 { + return 0, nil + } + ids := make([]string, 0, len(chunks)) + for _, c := range chunks { + if id, ok := c["id"].(string); ok { + ids = append(ids, id) + } + } + cond := map[string]interface{}{"id": ids, "kb_id": datasetID} + if _, err := docEngine.DeleteChunks(ctx, cond, wikiIndexName(tenantID), datasetID); err != nil { + return 0, err + } + return len(ids), nil +} + +// DocGraphItem is a single node/edge entry in a document's structure graph. +type DocGraphItem struct { + ID string `json:"id"` + Content string `json:"content"` + SourceID string `json:"source_id"` +} + +// GetDocumentGraph returns the structure graph of a single document. +func (s *DatasetArtifactService) GetDocumentGraph(ctx context.Context, tenantID, datasetID, documentID, graphType string) ([]DocGraphItem, int64, error) { + filter := map[string]interface{}{ + "doc_id": []string{documentID}, + "compiled_graph_kwd": []string{"graph"}, + } + if graphType != "" { + filter[FieldGraphType] = []string{graphType} + } + chunks, total, err := s.searchCompiled(ctx, tenantID, datasetID, filter, + []string{"id", "content_with_weight", "source_id"}, 0, 10000, nil) + if err != nil { + return nil, 0, err + } + items := make([]DocGraphItem, 0, len(chunks)) + for _, c := range chunks { + items = append(items, DocGraphItem{ + ID: firstStringValue(c["id"]), + Content: firstStringValue(c["content_with_weight"]), + SourceID: firstStringValue(c["source_id"]), + }) + } + return items, total, nil +} + +// DeleteDocumentGraph deletes the structure graph of a single document. +func (s *DatasetArtifactService) DeleteDocumentGraph(ctx context.Context, tenantID, datasetID, documentID string) (int, error) { + docEngine := engine.Get() + if docEngine == nil { + return 0, fmt.Errorf("document engine is not initialized") + } + filter := map[string]interface{}{ + "doc_id": []string{documentID}, + "compiled_graph_kwd": []string{"graph"}, + } + chunks, _, err := s.searchCompiled(ctx, tenantID, datasetID, filter, []string{"id"}, 0, 10000, nil) + if err != nil { + return 0, err + } + if len(chunks) == 0 { + return 0, nil + } + ids := make([]string, 0, len(chunks)) + for _, c := range chunks { + if id, ok := c["id"].(string); ok { + ids = append(ids, id) + } + } + cond := map[string]interface{}{"id": ids, "kb_id": datasetID} + if _, err := docEngine.DeleteChunks(ctx, cond, wikiIndexName(tenantID), datasetID); err != nil { + return 0, err + } + return len(ids), nil +} + +// NavigationItem is a single navigation cluster. +type NavigationItem struct { + Name string `json:"name"` + Title string `json:"title"` + Count int `json:"count"` +} + +// ListNavClusters returns the navigation clusters of a dataset. +func (s *DatasetArtifactService) ListNavClusters(ctx context.Context, tenantID, datasetID string) ([]NavigationItem, int64, error) { + chunks, total, err := s.searchCompiled(ctx, tenantID, datasetID, + map[string]interface{}{"compile_kwd": []string{CompileKwdDatasetNav}}, + []string{"nav_cluster_kwd", "title_kwd", "count_int"}, 0, 10000, nil) + if err != nil { + return nil, 0, err + } + items := make([]NavigationItem, 0, len(chunks)) + for _, c := range chunks { + count := intValue(c["count_int"]) + items = append(items, NavigationItem{ + Name: firstStringValue(c["nav_cluster_kwd"]), + Title: firstStringValue(c["title_kwd"]), + Count: count, + }) + } + return items, total, nil +} + +// DeleteNav deletes all navigation clusters of a dataset. +func (s *DatasetArtifactService) DeleteNav(ctx context.Context, tenantID, datasetID string) (int, error) { + docEngine := engine.Get() + if docEngine == nil { + return 0, fmt.Errorf("document engine is not initialized") + } + chunks, _, err := s.searchCompiled(ctx, tenantID, datasetID, + map[string]interface{}{"compile_kwd": []string{CompileKwdDatasetNav}}, []string{"id"}, 0, 10000, nil) + if err != nil { + return 0, err + } + if len(chunks) == 0 { + return 0, nil + } + ids := make([]string, 0, len(chunks)) + for _, c := range chunks { + if id, ok := c["id"].(string); ok { + ids = append(ids, id) + } + } + cond := map[string]interface{}{"id": ids, "kb_id": datasetID} + if _, err := docEngine.DeleteChunks(ctx, cond, wikiIndexName(tenantID), datasetID); err != nil { + return 0, err + } + return len(ids), nil +} + +// DeleteNavNode deletes a single navigation cluster by name. +func (s *DatasetArtifactService) DeleteNavNode(ctx context.Context, tenantID, datasetID, name string) (int, error) { + docEngine := engine.Get() + if docEngine == nil { + return 0, fmt.Errorf("document engine is not initialized") + } + chunks, _, err := s.searchCompiled(ctx, tenantID, datasetID, + map[string]interface{}{"compile_kwd": []string{CompileKwdDatasetNav}, "nav_cluster_kwd": []string{name}}, + []string{"id"}, 0, 10000, nil) + if err != nil { + return 0, err + } + if len(chunks) == 0 { + return 0, nil + } + ids := make([]string, 0, len(chunks)) + for _, c := range chunks { + if id, ok := c["id"].(string); ok { + ids = append(ids, id) + } + } + cond := map[string]interface{}{"id": ids, "kb_id": datasetID} + if _, err := docEngine.DeleteChunks(ctx, cond, wikiIndexName(tenantID), datasetID); err != nil { + return 0, err + } + return len(ids), nil +} + +// NavChildItem is a single child entry under a navigation cluster. +type NavChildItem struct { + Name string `json:"name"` + Title string `json:"title"` + Count int `json:"count"` +} + +// ListNavChildren returns the children of a navigation cluster. +func (s *DatasetArtifactService) ListNavChildren(ctx context.Context, tenantID, datasetID, name string) ([]NavChildItem, int64, error) { + chunks, total, err := s.searchCompiled(ctx, tenantID, datasetID, + map[string]interface{}{"compile_kwd": []string{CompileKwdDatasetNav}, "nav_cluster_kwd": []string{name}}, + []string{"nav_child_kwd", "title_kwd", "count_int"}, 0, 10000, nil) + if err != nil { + return nil, 0, err + } + items := make([]NavChildItem, 0, len(chunks)) + for _, c := range chunks { + count := intValue(c["count_int"]) + items = append(items, NavChildItem{ + Name: firstStringValue(c["nav_child_kwd"]), + Title: firstStringValue(c["title_kwd"]), + Count: count, + }) + } + return items, total, nil +} + +// SkillTreeItem is a single skill-tree page summary. +type SkillTreeItem struct { + Kwd string `json:"kwd"` + Title string `json:"title"` + PageType string `json:"page_type"` + Outlinks int `json:"outlinks"` + Inlinks int `json:"inlinks"` +} + +// GetSkillTree returns the skill tree of a dataset. +func (s *DatasetArtifactService) GetSkillTree(ctx context.Context, tenantID, datasetID, kwd string) ([]SkillTreeItem, int64, error) { + filter := map[string]interface{}{"compile_kwd": []string{CompileKwdSkillAll}} + if kwd != "" { + filter["kwd"] = []string{kwd} + } + chunks, total, err := s.searchCompiled(ctx, tenantID, datasetID, filter, + []string{"kwd", "title_kwd", "page_type_kwd", "outlinks_int", "inlinks_int"}, 0, 10000, nil) + if err != nil { + return nil, 0, err + } + items := make([]SkillTreeItem, 0, len(chunks)) + for _, c := range chunks { + outlinks, inlinks := intValue(c["outlinks_int"]), intValue(c["inlinks_int"]) + items = append(items, SkillTreeItem{ + Kwd: firstStringValue(c["kwd"]), + Title: firstStringValue(c["title_kwd"]), + PageType: firstStringValue(c["page_type_kwd"]), + Outlinks: outlinks, + Inlinks: inlinks, + }) + } + return items, total, nil +} + +// SkillPageDetail is the full skill page payload. +type SkillPageDetail struct { + Kwd string `json:"kwd"` + Title string `json:"title"` + ContentMd string `json:"content_md"` + Outlinks []string `json:"outlinks"` + Inlinks []string `json:"inlinks"` +} + +// GetSkillPage returns a single skill page by keyword. +func (s *DatasetArtifactService) GetSkillPage(ctx context.Context, tenantID, datasetID, kwd string) (*SkillPageDetail, error) { + chunks, _, err := s.searchCompiled(ctx, tenantID, datasetID, + map[string]interface{}{"compile_kwd": []string{CompileKwdSkillAll}, "kwd": []string{kwd}}, + []string{"kwd", "title_kwd", "content_with_weight", "outlinks_kwd", "inlinks_kwd"}, 0, 1, nil) + if err != nil { + return nil, err + } + if len(chunks) == 0 { + return nil, nil + } + c := chunks[0] + return &SkillPageDetail{ + Kwd: firstStringValue(c["kwd"]), + Title: firstStringValue(c["title_kwd"]), + ContentMd: firstStringValue(c["content_with_weight"]), + Outlinks: toStringSlice(c["outlinks_kwd"]), + Inlinks: toStringSlice(c["inlinks_kwd"]), + }, nil +} + +// DeleteSkills deletes skills of a dataset, optionally scoped by keyword. +func (s *DatasetArtifactService) DeleteSkills(ctx context.Context, tenantID, datasetID, kwd string) (int, error) { + docEngine := engine.Get() + if docEngine == nil { + return 0, fmt.Errorf("document engine is not initialized") + } + filter := map[string]interface{}{"compile_kwd": []string{CompileKwdSkillAll}} + if kwd != "" { + filter["kwd"] = []string{kwd} + } + chunks, _, err := s.searchCompiled(ctx, tenantID, datasetID, filter, []string{"id"}, 0, 10000, nil) + if err != nil { + return 0, err + } + if len(chunks) == 0 { + return 0, nil + } + ids := make([]string, 0, len(chunks)) + for _, c := range chunks { + if id, ok := c["id"].(string); ok { + ids = append(ids, id) + } + } + cond := map[string]interface{}{"id": ids, "kb_id": datasetID} + if _, err := docEngine.DeleteChunks(ctx, cond, wikiIndexName(tenantID), datasetID); err != nil { + return 0, err + } + return len(ids), nil +} + +// firstStringValue returns the first string in a possibly-list ES field value. +func firstStringValue(v interface{}) string { + switch t := v.(type) { + case string: + return t + case []interface{}: + if len(t) > 0 { + if s, ok := t[0].(string); ok { + return s + } + } + case []string: + if len(t) > 0 { + return t[0] + } + } + return "" +} + +// toStringSlice normalizes an ES field value (string or list) to []string. +func toStringSlice(v interface{}) []string { + switch t := v.(type) { + case string: + if t == "" { + return []string{} + } + return []string{t} + case []interface{}: + out := make([]string, 0, len(t)) + for _, e := range t { + if s, ok := e.(string); ok { + out = append(out, s) + } + } + return out + case []string: + return t + } + return []string{} +} diff --git a/internal/service/file/file_commit.go b/internal/service/file/file_commit.go index 2ac6b2514e..fc6c205a65 100644 --- a/internal/service/file/file_commit.go +++ b/internal/service/file/file_commit.go @@ -28,6 +28,8 @@ import ( "ragflow/internal/storage" "ragflow/internal/utility" "sort" + "strings" + "sync" "time" "go.uber.org/zap" @@ -233,6 +235,209 @@ func (s *FileCommitService) CreateCommit(ctx context.Context, folderID, authorID return commit, nil } +// PageEditCommitInput carries the data needed to record a single wiki/skill +// page edit as an audit commit. +type PageEditCommitInput struct { + DatasetID string // knowledgebase scope, stored on the commit for isolation + DocID string // ES doc id of the page content + Slug string + PageType string + Title string + AuthorID string + OldContent string + NewContent string +} + +// wikiFileID derives the stable file key used to scope page-edit commits to a +// specific knowledgebase and page, so identical slugs in different +// knowledgebases never share a commit parent or history. +func wikiFileID(datasetID, pageType, slug string) string { + return datasetID + "/" + pageType + "/" + slug +} + +// RecordPageEdit records a wiki/skill page edit as an audit commit with a +// git-style parent chain (each edit points at the previous commit for the same +// page). The new content_after is referenced in ES by doc_id; a unified diff of +// old vs new content is stored on the commit item. The commit is scoped to the +// dataset via FolderID and a derived page file key so page histories never cross +// knowledgebase boundaries. +// +// This path is independent of the workspace File tree (it does not require a +// File record or a tree_state snapshot). +func (s *FileCommitService) RecordPageEdit(ctx context.Context, in PageEditCommitInput) (*entity.FileCommit, error) { + // Parent chain: previous commit that touched the same page file key. + fileID := wikiFileID(in.DatasetID, in.PageType, in.Slug) + + commitID := utility.GenerateUUID() + + diffText := unifiedDiff(in.OldContent, in.NewContent) + contentAfterStorage := "es" + contentAfterLocation := in.DocID + slugKwd := in.Slug + pageTypeKwd := in.PageType + + item := &entity.FileCommitItem{ + ID: utility.GenerateUUID(), + CommitID: commitID, + FileID: fileID, + Operation: "modify", + Diff: &diffText, + ContentAfterStorage: &contentAfterStorage, + ContentAfterLocation: &contentAfterLocation, + SlugKwd: &slugKwd, + PageTypeKwd: &pageTypeKwd, + } + + var commit *entity.FileCommit + // Serialize parent selection with insertion in process so two concurrent + // edits on the same page cannot both read the same parent and fork the + // chain. This is backend-agnostic (works identically on MySQL and SQLite) + // and cheaper than row-level DB locks. + mu := pageCommitLock(fileID) + mu.Lock() + defer mu.Unlock() + + // Read the parent inside the lock, on the shared connection, so it always + // reflects the previously committed edit for this page. + parentID, perr := s.commitItemDAO.GetLatestCommitIDByFileID(ctx, dao.DB, fileID) + if perr != nil { + return nil, fmt.Errorf("failed to resolve page commit parent: %w", perr) + } + + commit = &entity.FileCommit{ + ID: commitID, + FolderID: in.DatasetID, + Message: in.Title, + AuthorID: in.AuthorID, + Title: &in.Title, + FileCount: 1, + } + if parentID != "" { + commit.ParentID = &parentID + } + + if err := dao.DB.WithContext(ctx).Transaction(func(tx *gorm.DB) error { + if cerr := tx.Create(commit).Error; cerr != nil { + return fmt.Errorf("failed to create page commit: %w", cerr) + } + if ierr := tx.Create(item).Error; ierr != nil { + return fmt.Errorf("failed to create page commit item: %w", ierr) + } + return nil + }); err != nil { + return nil, err + } + + return commit, nil +} + +// pageCommitLocks is a per-page-file-key mutex registry that serializes +// RecordPageEdit calls for the same page. Entries are retained for the process +// lifetime (bounded by the number of distinct pages edited). +var pageCommitLocks sync.Map // fileID -> *sync.Mutex + +func pageCommitLock(fileID string) *sync.Mutex { + v, _ := pageCommitLocks.LoadOrStore(fileID, &sync.Mutex{}) + return v.(*sync.Mutex) +} + +// ListPageCommits lists audit commits for a specific wiki/skill page. +func (s *FileCommitService) ListPageCommits(ctx context.Context, datasetID, pageType, slug string, page, pageSize int) ([]*entity.FileCommit, int64, error) { + items, err := s.commitItemDAO.ListByFileID(ctx, dao.DB, wikiFileID(datasetID, pageType, slug)) + if err != nil { + return nil, 0, err + } + commitIDs := make([]string, 0, len(items)) + for _, it := range items { + commitIDs = append(commitIDs, it.CommitID) + } + if len(commitIDs) == 0 { + return []*entity.FileCommit{}, 0, nil + } + + commits, total, err := s.commitDAO.ListByIDs(ctx, dao.DB, commitIDs, page, pageSize) + if err != nil { + return nil, 0, err + } + return commits, total, nil +} + +// unifiedDiff produces a simple line-based unified diff between two texts. +func unifiedDiff(oldText, newText string) string { + oldLines := strings.Split(oldText, "\n") + newLines := strings.Split(newText, "\n") + + const maxCtx = 3 + type hunkLine struct { + prefix string + text string + } + var hunks []hunkLine + + // Longest common subsequence over lines, then render the diff. + cur := make([][]int, len(oldLines)+1) + for i := range cur { + cur[i] = make([]int, len(newLines)+1) + } + for i := len(oldLines) - 1; i >= 0; i-- { + for j := len(newLines) - 1; j >= 0; j-- { + if oldLines[i] == newLines[j] { + cur[i][j] = cur[i+1][j+1] + 1 + } else if cur[i+1][j] >= cur[i][j+1] { + cur[i][j] = cur[i+1][j] + } else { + cur[i][j] = cur[i][j+1] + } + } + } + + i, j := 0, 0 + for i < len(oldLines) && j < len(newLines) { + if oldLines[i] == newLines[j] { + i++ + j++ + } else if cur[i+1][j] >= cur[i][j+1] { + hunks = append(hunks, hunkLine{"-", oldLines[i]}) + i++ + } else { + hunks = append(hunks, hunkLine{"+", newLines[j]}) + j++ + } + } + for ; i < len(oldLines); i++ { + hunks = append(hunks, hunkLine{"-", oldLines[i]}) + } + for ; j < len(newLines); j++ { + hunks = append(hunks, hunkLine{"+", newLines[j]}) + } + + if len(hunks) == 0 { + return "" + } + if len(hunks) > maxCtx*2 { + var b strings.Builder + for k := 0; k < maxCtx; k++ { + b.WriteString(hunks[k].prefix) + b.WriteString(hunks[k].text) + b.WriteString("\n") + } + b.WriteString("... (" + fmt.Sprintf("%d", len(hunks)-maxCtx*2) + " lines omitted) ...\n") + for k := len(hunks) - maxCtx; k < len(hunks); k++ { + b.WriteString(hunks[k].prefix) + b.WriteString(hunks[k].text) + b.WriteString("\n") + } + return b.String() + } + var b strings.Builder + for _, h := range hunks { + b.WriteString(h.prefix) + b.WriteString(h.text) + b.WriteString("\n") + } + return b.String() +} + // ListCommits lists commits for a workspace folder with pagination func (s *FileCommitService) ListCommits(ctx context.Context, folderID string, page, pageSize int, orderBy string, desc bool) ([]*entity.FileCommit, int64, error) { return s.commitDAO.ListByFolderID(ctx, dao.DB, folderID, page, pageSize, orderBy, desc) diff --git a/internal/service/file/file_commit_page_test.go b/internal/service/file/file_commit_page_test.go new file mode 100644 index 0000000000..5f63e66c6d --- /dev/null +++ b/internal/service/file/file_commit_page_test.go @@ -0,0 +1,281 @@ +package file + +import ( + "context" + "strconv" + "strings" + "sync" + "testing" + + "ragflow/internal/dao" + "ragflow/internal/entity" + + "github.com/glebarez/sqlite" + "gorm.io/gorm" +) + +func newPageCommitTestDB(t *testing.T) *gorm.DB { + t.Helper() + db, err := gorm.Open(sqlite.Open(":memory:"), &gorm.Config{TranslateError: true}) + if err != nil { + t.Fatalf("open sqlite: %v", err) + } + // Keep a single connection so the :memory: database is shared across all + // goroutines (sqlite :memory: is otherwise per-connection); this also + // serializes the concurrent page-commit test through one connection. + if sqlDB, serr := db.DB(); serr == nil { + sqlDB.SetMaxOpenConns(1) + sqlDB.SetMaxIdleConns(1) + } + if err := db.AutoMigrate(&entity.FileCommit{}, &entity.FileCommitItem{}); err != nil { + t.Fatalf("auto migrate: %v", err) + } + old := dao.DB + dao.DB = db + t.Cleanup(func() { dao.DB = old }) + return db +} + +func TestRecordPageEdit_CreatesCommitAndItem(t *testing.T) { + newPageCommitTestDB(t) + svc := NewFileCommitService() + ctx := context.Background() + + in := PageEditCommitInput{ + DatasetID: "kb1", + DocID: "wiki/page-a", + Slug: "page-a", + PageType: "wiki", + Title: "First edit", + AuthorID: "u1", + OldContent: "hello world", + NewContent: "hello world, edited", + } + commit, err := svc.RecordPageEdit(ctx, in) + if err != nil { + t.Fatalf("RecordPageEdit: %v", err) + } + if commit.ID == "" { + t.Fatal("expected a generated commit id") + } + if commit.AuthorID != "u1" || commit.Message != "First edit" || commit.FileCount != 1 { + t.Fatalf("unexpected commit: %+v", commit) + } + if commit.FolderID != "kb1" { + t.Fatalf("expected folder_id kb1 for dataset scope, got %s", commit.FolderID) + } + if commit.ParentID != nil { + t.Fatalf("first edit should have no parent, got %v", *commit.ParentID) + } + + var items []entity.FileCommitItem + if err := dao.DB.Where("commit_id = ?", commit.ID).Find(&items).Error; err != nil { + t.Fatalf("load items: %v", err) + } + if len(items) != 1 { + t.Fatalf("expected 1 item, got %d", len(items)) + } + it := items[0] + if it.Operation != "modify" { + t.Fatalf("expected operation modify, got %s", it.Operation) + } + if it.FileID != "kb1/wiki/page-a" { + t.Fatalf("expected file_id kb1/wiki/page-a (dataset-scoped), got %s", it.FileID) + } + if it.SlugKwd == nil || *it.SlugKwd != "page-a" { + t.Fatalf("expected slug_kwd page-a, got %v", it.SlugKwd) + } + if it.PageTypeKwd == nil || *it.PageTypeKwd != "wiki" { + t.Fatalf("expected page_type_kwd wiki, got %v", it.PageTypeKwd) + } + if it.Diff == nil || !strings.Contains(*it.Diff, "hello world, edited") { + t.Fatalf("expected diff containing new content, got %v", it.Diff) + } + if it.ContentAfterStorage == nil || *it.ContentAfterStorage != "es" { + t.Fatalf("expected content_after_storage es, got %v", it.ContentAfterStorage) + } + if it.ContentAfterLocation == nil || *it.ContentAfterLocation != "wiki/page-a" { + t.Fatalf("expected content_after_location wiki/page-a, got %v", it.ContentAfterLocation) + } +} + +func TestRecordPageEdit_SecondEditLinksParent(t *testing.T) { + newPageCommitTestDB(t) + svc := NewFileCommitService() + ctx := context.Background() + + base := PageEditCommitInput{ + DatasetID: "kb1", + DocID: "wiki/page-a", + Slug: "page-a", + PageType: "wiki", + Title: "first", + AuthorID: "u1", + } + if _, err := svc.RecordPageEdit(ctx, base); err != nil { + t.Fatalf("first RecordPageEdit: %v", err) + } + + second := base + second.Title = "second" + second.OldContent = "hello" + second.NewContent = "hello world" + commit2, err := svc.RecordPageEdit(ctx, second) + if err != nil { + t.Fatalf("second RecordPageEdit: %v", err) + } + if commit2.ParentID == nil { + t.Fatal("second edit should link to the first commit as parent") + } + var first entity.FileCommit + if err := dao.DB.Where("title = ?", "first").First(&first).Error; err != nil { + t.Fatalf("load first commit: %v", err) + } + if *commit2.ParentID != first.ID { + t.Fatalf("parent id mismatch: got %s want %s", *commit2.ParentID, first.ID) + } +} + +func TestRecordPageEdit_IsolatesDatasets(t *testing.T) { + newPageCommitTestDB(t) + svc := NewFileCommitService() + ctx := context.Background() + + mk := func(datasetID string) PageEditCommitInput { + return PageEditCommitInput{ + DatasetID: datasetID, + DocID: datasetID + "/wiki/page-a", + Slug: "page-a", + PageType: "wiki", + Title: datasetID + "-edit", + AuthorID: "u1", + OldContent: "old", + NewContent: "new", + } + } + if _, err := svc.RecordPageEdit(ctx, mk("kb1")); err != nil { + t.Fatalf("kb1 RecordPageEdit: %v", err) + } + if _, err := svc.RecordPageEdit(ctx, mk("kb2")); err != nil { + t.Fatalf("kb2 RecordPageEdit: %v", err) + } + + // A second edit in kb1 must parent to kb1's own first commit, not kb2's. + kb1Again := mk("kb1") + kb1Again.Title = "kb1-edit-2" + kb1Again.OldContent = "old" + kb1Again.NewContent = "newer" + kb1Commit2, err := svc.RecordPageEdit(ctx, kb1Again) + if err != nil { + t.Fatalf("kb1 second RecordPageEdit: %v", err) + } + if kb1Commit2.ParentID == nil { + t.Fatal("kb1 second edit should have a parent") + } + + var kb1First, kb2First entity.FileCommit + if err := dao.DB.Where("title = ?", "kb1-edit").First(&kb1First).Error; err != nil { + t.Fatalf("load kb1 first commit: %v", err) + } + if err := dao.DB.Where("title = ?", "kb2-edit").First(&kb2First).Error; err != nil { + t.Fatalf("load kb2 first commit: %v", err) + } + if *kb1Commit2.ParentID != kb1First.ID { + t.Fatalf("kb1 parent mismatch: got %s want %s", *kb1Commit2.ParentID, kb1First.ID) + } + if *kb1Commit2.ParentID == kb2First.ID { + t.Fatal("kb1 parent must not cross into kb2 history") + } +} + +func TestRecordPageEdit_ConcurrentEditsFormLinearChain(t *testing.T) { + newPageCommitTestDB(t) + svc := NewFileCommitService() + ctx := context.Background() + + const edits = 8 + var wg sync.WaitGroup + errs := make([]error, edits) + for i := 0; i < edits; i++ { + wg.Add(1) + go func(idx int) { + defer wg.Done() + in := PageEditCommitInput{ + DatasetID: "kb1", + DocID: "wiki/page-a", + Slug: "page-a", + PageType: "wiki", + Title: "edit-" + strconv.Itoa(idx), + AuthorID: "u1", + OldContent: "old", + NewContent: "new-" + strconv.Itoa(idx), + } + _, errs[idx] = svc.RecordPageEdit(ctx, in) + }(i) + } + wg.Wait() + for i, e := range errs { + if e != nil { + t.Fatalf("edit %d failed: %v", i, e) + } + } + + var commits []entity.FileCommit + if err := dao.DB.Where("folder_id = ?", "kb1").Order("create_time ASC").Find(&commits).Error; err != nil { + t.Fatalf("load commits: %v", err) + } + if len(commits) != edits { + t.Fatalf("expected %d commits, got %d", edits, len(commits)) + } + // Every commit except the first must have a parent, and the parents must + // form a linear chain (no two commits share the same parent). + seenParents := map[string]bool{} + for i, c := range commits { + if i == 0 { + if c.ParentID != nil { + t.Fatalf("first commit should have no parent") + } + continue + } + if c.ParentID == nil { + t.Fatalf("commit %s should have a parent", c.ID) + } + if seenParents[*c.ParentID] { + t.Fatalf("two commits share parent %s -> forked chain", *c.ParentID) + } + seenParents[*c.ParentID] = true + } +} + +func TestUnifiedDiff_EmptyWhenNoChange(t *testing.T) { + if d := unifiedDiff("same", "same"); d != "" { + t.Fatalf("expected empty diff for identical text, got %q", d) + } +} + +func TestUnifiedDiff_DetectsAddition(t *testing.T) { + d := unifiedDiff("a\nb\nc\n", "a\nb\nc\nd\n") + if !strings.Contains(d, "+d") { + t.Fatalf("expected diff to contain added line, got %q", d) + } +} + +func TestUnifiedDiff_DetectsRemoval(t *testing.T) { + d := unifiedDiff("a\nb\nc\n", "a\nc\n") + if !strings.Contains(d, "-b") { + t.Fatalf("expected diff to contain removed line, got %q", d) + } +} + +func TestUnifiedDiff_TruncatesLongDiff(t *testing.T) { + oldLines := make([]string, 50) + newLines := make([]string, 50) + for i := range oldLines { + oldLines[i] = "old-line" + newLines[i] = "new-line" + } + d := unifiedDiff(strings.Join(oldLines, "\n"), strings.Join(newLines, "\n")) + if !strings.Contains(d, "... ") || !strings.Contains(d, "lines omitted") { + t.Fatalf("expected long diff to be truncated, got %q", d) + } +}