feat(go): port knowledge compilation template/group + wiki artifact/nav/skill REST APIs from Python (#17656)

This commit is contained in:
Zhichang Yu
2026-07-31 22:57:43 +08:00
committed by GitHub
parent d67d14e4c6
commit b811dce1b0
18 changed files with 3428 additions and 54 deletions

View File

@@ -833,6 +833,9 @@ func startServer(ctx context.Context) {
componentsSvc := service.NewComponentsService()
componentsHandler := handler.NewComponentsHandler(componentsSvc)
pipelineHandler := handler.NewPipelineHandler()
compilationTemplateHandler := handler.NewCompilationTemplateHandler(service.NewCompilationTemplateService())
compilationTemplateGroupHandler := handler.NewCompilationTemplateGroupHandler(service.NewCompilationTemplateGroupService())
datasetArtifactHandler := handler.NewDatasetArtifactHandler(service.NewDatasetArtifactService(), datasetsService, file.NewFileCommitService())
// Initialize router
r := router.NewRouter(authHandler,
@@ -865,7 +868,10 @@ func startServer(ctx context.Context) {
openaiChatHandler,
botHandler,
componentsHandler,
pipelineHandler)
pipelineHandler,
compilationTemplateHandler,
compilationTemplateGroupHandler,
datasetArtifactHandler)
// Create Gin engine
ginEngine := gin.New()

View File

@@ -20,6 +20,8 @@ import (
"context"
"fmt"
"ragflow/internal/entity"
"gorm.io/gorm"
)
// CompilationTemplateDAO is the data-access object for compilation templates and
@@ -123,3 +125,97 @@ func (dao *CompilationTemplateDAO) GetTemplate(ctx context.Context, tenantID, te
}
return &t, nil
}
// ListByGroup returns the valid compilation templates belonging to a group,
// ordered by create_time asc, mirroring Python
// CompilationTemplateGroupService child loading.
func (dao *CompilationTemplateDAO) ListByGroup(ctx context.Context, db *gorm.DB, groupID string) ([]*entity.CompilationTemplate, error) {
var templates []*entity.CompilationTemplate
if err := db.WithContext(ctx).
Where("group_id = ? AND status = ?", groupID, string(entity.StatusValid)).
Order("create_time asc").
Find(&templates).Error; err != nil {
return nil, err
}
return templates, nil
}
// ListBuiltins returns the valid, built-in (is_builtin) compilation templates,
// ordered by create_time then name. Mirrors Python list_builtins().
func (dao *CompilationTemplateDAO) ListBuiltins(ctx context.Context) ([]*entity.CompilationTemplate, error) {
var templates []*entity.CompilationTemplate
if err := DB.WithContext(ctx).
Where("is_builtin = ? AND status = ?", true, string(entity.StatusValid)).
Order("create_time asc, name asc").
Find(&templates).Error; err != nil {
return nil, err
}
return templates, nil
}
// NameExistsInGroup reports whether a valid, non-built-in template with the
// given name already exists inside the group, excluding excludeID. Mirrors the
// Python duplicate-child guard.
func (dao *CompilationTemplateDAO) NameExistsInGroup(ctx context.Context, tenantID, groupID, name, excludeID string) (bool, error) {
q := DB.WithContext(ctx).Model(&entity.CompilationTemplate{}).
Where("tenant_id = ? AND group_id = ? AND name = ? AND is_builtin = ? AND status = ?",
tenantID, groupID, name, false, string(entity.StatusValid))
if excludeID != "" {
q = q.Where("id <> ?", excludeID)
}
var count int64
if err := q.Count(&count).Error; err != nil {
return false, err
}
return count > 0, nil
}
// GetByID returns a single template row by id regardless of tenant scope
// (used for reconciling group children).
func (dao *CompilationTemplateDAO) GetByID(ctx context.Context, id string) (*entity.CompilationTemplate, error) {
var t entity.CompilationTemplate
if err := DB.WithContext(ctx).Where("id = ?", id).First(&t).Error; err != nil {
return nil, err
}
return &t, nil
}
// Save inserts a new template row.
func (dao *CompilationTemplateDAO) Save(ctx context.Context, db *gorm.DB, t *entity.CompilationTemplate) error {
return db.WithContext(ctx).Create(t).Error
}
// UpdateFields persists the supplied columns for the template with the given id.
func (dao *CompilationTemplateDAO) UpdateFields(ctx context.Context, db *gorm.DB, id string, m map[string]interface{}) error {
return db.WithContext(ctx).Model(&entity.CompilationTemplate{}).
Where("id = ?", id).Updates(m).Error
}
// UpdateStatusByGroup flips the status of every valid template in a group,
// mirroring Python group delete's child cascade.
func (dao *CompilationTemplateDAO) UpdateStatusByGroup(ctx context.Context, db *gorm.DB, groupID, status string) error {
return db.WithContext(ctx).Model(&entity.CompilationTemplate{}).
Where("group_id = ? AND status = ?", groupID, string(entity.StatusValid)).
Update("status", status).Error
}
// UpdateStatusByID flips a single template's status.
func (dao *CompilationTemplateDAO) UpdateStatusByID(ctx context.Context, db *gorm.DB, id, status string) error {
return db.WithContext(ctx).Model(&entity.CompilationTemplate{}).
Where("id = ?", id).
Update("status", status).Error
}
// HardDeleteOrphansByName physically removes stale, invalid, non-built-in
// templates of the given name within the group, mirroring Python
// _purge_stale_invalid_children (which DELETEs orphaned duplicate names after a
// group child is soft-deleted). The deletion is scoped to the group so a
// same-named template in another group is never affected. These rows were
// soft-deleted in a prior operation but must be permanently purged to keep the
// table clean.
func (dao *CompilationTemplateDAO) HardDeleteOrphansByName(ctx context.Context, db *gorm.DB, tenantID, groupID, name string) error {
return db.WithContext(ctx).Where(
"tenant_id = ? AND group_id = ? AND name = ? AND is_builtin = ? AND status = ?",
tenantID, groupID, name, false, string(entity.StatusInvalid),
).Delete(&entity.CompilationTemplate{}).Error
}

View File

@@ -0,0 +1,109 @@
//
// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
package dao
import (
"context"
"errors"
"ragflow/internal/entity"
"gorm.io/gorm"
)
// CompilationTemplateGroupDAO is the data-access object for compilation template
// groups.
type CompilationTemplateGroupDAO struct{}
// NewCompilationTemplateGroupDAO creates a compilation template group DAO.
func NewCompilationTemplateGroupDAO() *CompilationTemplateGroupDAO {
return &CompilationTemplateGroupDAO{}
}
// ListSaved returns the tenant's valid groups with optional keyword/scope
// filtering and ordering, mirroring Python list_saved(). Built-in groups
// (empty tenant) are included so every tenant sees the catalogue.
func (dao *CompilationTemplateGroupDAO) ListSaved(ctx context.Context, tenantID, keywords, scope, orderby string, desc bool) ([]*entity.CompilationTemplateGroup, error) {
q := DB.WithContext(ctx).
Where("(tenant_id = ? OR tenant_id = '') AND status = ?", tenantID, string(entity.StatusValid))
if keywords != "" {
q = q.Where("name LIKE ?", "%"+keywords+"%")
}
if scope != "" {
q = q.Where("scope = ?", scope)
}
if orderby != "name" && orderby != "scope" && orderby != "create_time" && orderby != "update_time" {
orderby = "create_time"
}
dir := "asc"
if desc {
dir = "desc"
}
var groups []*entity.CompilationTemplateGroup
if err := q.Order(orderby + " " + dir).Find(&groups).Error; err != nil {
return nil, err
}
return groups, nil
}
// GetSaved returns a single valid group for the tenant (or built-in), or nil.
func (dao *CompilationTemplateGroupDAO) GetSaved(ctx context.Context, tenantID, groupID string) (*entity.CompilationTemplateGroup, error) {
var g entity.CompilationTemplateGroup
err := DB.WithContext(ctx).
Where("(tenant_id = ? OR tenant_id = '') AND id = ? AND status = ?",
tenantID, groupID, string(entity.StatusValid)).
First(&g).Error
if errors.Is(err, gorm.ErrRecordNotFound) {
return nil, nil
}
if err != nil {
return nil, err
}
return &g, nil
}
// NameExists reports whether a valid group with the given name exists for the
// tenant, excluding excludeID. Mirrors Python name_exists().
func (dao *CompilationTemplateGroupDAO) NameExists(ctx context.Context, tenantID, name, excludeID string) (bool, error) {
q := DB.WithContext(ctx).Model(&entity.CompilationTemplateGroup{}).
Where("tenant_id = ? AND name = ? AND status = ?", tenantID, name, string(entity.StatusValid))
if excludeID != "" {
q = q.Where("id <> ?", excludeID)
}
var count int64
if err := q.Count(&count).Error; err != nil {
return false, err
}
return count > 0, nil
}
// Create inserts a new group row.
func (dao *CompilationTemplateGroupDAO) Create(ctx context.Context, db *gorm.DB, g *entity.CompilationTemplateGroup) error {
return db.WithContext(ctx).Create(g).Error
}
// UpdateFields persists the supplied columns for the group with the given id.
func (dao *CompilationTemplateGroupDAO) UpdateFields(ctx context.Context, db *gorm.DB, id string, m map[string]interface{}) error {
return db.WithContext(ctx).Model(&entity.CompilationTemplateGroup{}).
Where("id = ?", id).Updates(m).Error
}
// Delete soft-deletes a valid group, mirroring Python delete_group().
func (dao *CompilationTemplateGroupDAO) Delete(ctx context.Context, db *gorm.DB, tenantID, groupID string) error {
return db.WithContext(ctx).Model(&entity.CompilationTemplateGroup{}).
Where("id = ? AND tenant_id = ? AND status = ?", groupID, tenantID, string(entity.StatusValid)).
Update("status", string(entity.StatusInvalid)).Error
}

View File

@@ -18,11 +18,24 @@ package dao
import (
"context"
"errors"
"ragflow/internal/entity"
"strconv"
"gorm.io/gorm"
"gorm.io/gorm/clause"
)
// toInterfaceSlice converts a string slice into an []interface{} for use as
// parameterized query variables.
func toInterfaceSlice(ss []string) []interface{} {
out := make([]interface{}, len(ss))
for i := range ss {
out[i] = ss[i]
}
return out
}
// FileCommitDAO file commit data access object
type FileCommitDAO struct{}
@@ -110,6 +123,41 @@ func (dao *FileCommitDAO) ListByFolderID(ctx context.Context, db *gorm.DB, folde
return commits, total, nil
}
// ListByIDs lists commits whose IDs are in the given set, preserving the
// order of commitIDs (most-recent-first for page-edit history). Returns the
// total matched count for pagination.
func (dao *FileCommitDAO) ListByIDs(ctx context.Context, db *gorm.DB, commitIDs []string, page, pageSize int) ([]*entity.FileCommit, int64, error) {
if len(commitIDs) == 0 {
return []*entity.FileCommit{}, 0, nil
}
var commits []*entity.FileCommit
var total int64
query := db.WithContext(ctx).Model(&entity.FileCommit{}).Where("id IN ?", commitIDs)
if err := query.Count(&total).Error; err != nil {
return nil, 0, err
}
// Preserve the incoming (most-recent-first) order with a parameterized
// CASE expression rather than a MySQL-specific FIELD() interpolation.
orderExpr := "CASE id"
for i := range commitIDs {
orderExpr += " WHEN ? THEN " + strconv.Itoa(i)
}
orderExpr += " END"
ordered := query.Clauses(clause.OrderBy{Expression: clause.Expr{
SQL: orderExpr,
Vars: toInterfaceSlice(commitIDs),
}})
// Apply pagination in the query itself so long histories never load the
// full set into memory before slicing.
if page > 0 && pageSize > 0 {
ordered = ordered.Offset((page - 1) * pageSize).Limit(pageSize)
}
if err := ordered.Find(&commits).Error; err != nil {
return nil, 0, err
}
return commits, total, nil
}
// FileCommitItemDAO file commit item data access object
type FileCommitItemDAO struct{}
@@ -146,3 +194,24 @@ func (dao *FileCommitItemDAO) GetByCommitIDAndFileID(ctx context.Context, db *go
}
return &item, nil
}
// GetLatestCommitIDByFileID returns the most recent commit_id that modified the
// given wiki page file key, or "" if none. Used to build the parent chain for
// page-edit commits.
func (dao *FileCommitItemDAO) GetLatestCommitIDByFileID(ctx context.Context, db *gorm.DB, fileID string) (string, error) {
var item entity.FileCommitItem
err := db.WithContext(ctx).
Where("file_id = ?", fileID).
// Order by the monotonic auto-increment sequence so the most recently
// inserted item is always selected even when multiple edits share the
// same create_time millisecond.
Order("seq DESC").
First(&item).Error
if err != nil {
if errors.Is(err, gorm.ErrRecordNotFound) {
return "", nil
}
return "", err
}
return item.CommitID, nil
}

View File

@@ -38,8 +38,9 @@ func (FileCommit) TableName() string {
// FileCommitItem represents a single file change within a commit.
type FileCommitItem struct {
ID string `gorm:"column:id;primaryKey;size:32" json:"id"`
Seq uint `gorm:"column:seq;autoIncrement;index" json:"seq,omitempty"`
CommitID string `gorm:"column:commit_id;size:32;not null;uniqueIndex:idx_commit_file" json:"commit_id"`
FileID string `gorm:"column:file_id;size:32;not null;uniqueIndex:idx_commit_file" json:"file_id"`
FileID string `gorm:"column:file_id;size:255;not null;uniqueIndex:idx_commit_file" json:"file_id"`
Operation string `gorm:"column:operation;size:16;not null;index" json:"operation"`
OldHash *string `gorm:"column:old_hash;size:64;index" json:"old_hash,omitempty"`
NewHash *string `gorm:"column:new_hash;size:64;index" json:"new_hash,omitempty"`

View File

@@ -0,0 +1,78 @@
//
// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
package handler
import (
"ragflow/internal/common"
"ragflow/internal/service"
"github.com/gin-gonic/gin"
)
// CompilationTemplateHandler serves the knowledge-compilation template REST
// APIs. It mirrors the Python compilation_template_api blueprint.
type CompilationTemplateHandler struct {
compilationTemplateService *service.CompilationTemplateService
}
// NewCompilationTemplateHandler creates a CompilationTemplateHandler.
func NewCompilationTemplateHandler(svc *service.CompilationTemplateService) *CompilationTemplateHandler {
return &CompilationTemplateHandler{compilationTemplateService: svc}
}
// ListBuiltins returns the built-in template palette used by the frontend
// "Add template group" panel.
//
// @Summary List built-in compilation templates
// @Tags compilation template
// @Security ApiKeyAuth
// @Success 200 {object} map[string]interface{}
// @Router /v1/compilation-templates/builtins [get]
func (h *CompilationTemplateHandler) ListBuiltins(c *gin.Context) {
user, code, msg := GetUser(c)
if code != common.CodeSuccess {
common.ErrorWithCode(c, code, msg)
return
}
templates, err := h.compilationTemplateService.ListBuiltins(c.Request.Context(), user.ID)
if err != nil {
common.ResponseWithCodeData(c, common.CodeExceptionError, nil, err.Error())
return
}
common.SuccessWithData(c, templates, "success")
}
// ListWikiPresets returns the wiki page-structure presets loaded from YAML.
//
// @Summary List wiki presets
// @Tags compilation template
// @Security ApiKeyAuth
// @Success 200 {object} map[string]interface{}
// @Router /v1/compilation-templates/wiki-presets [get]
func (h *CompilationTemplateHandler) ListWikiPresets(c *gin.Context) {
_, code, msg := GetUser(c)
if code != common.CodeSuccess {
common.ErrorWithCode(c, code, msg)
return
}
presets, err := h.compilationTemplateService.LoadWikiPresets()
if err != nil {
common.ResponseWithCodeData(c, common.CodeExceptionError, nil, err.Error())
return
}
common.SuccessWithData(c, presets, "success")
}

View File

@@ -0,0 +1,233 @@
//
// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
package handler
import (
"strconv"
"ragflow/internal/common"
"ragflow/internal/service"
"github.com/gin-gonic/gin"
)
// CompilationTemplateGroupHandler serves the knowledge-compilation template
// group REST APIs. It mirrors the Python compilation_template_group_api blueprint.
type CompilationTemplateGroupHandler struct {
compilationTemplateGroupService *service.CompilationTemplateGroupService
}
// NewCompilationTemplateGroupHandler creates a CompilationTemplateGroupHandler.
func NewCompilationTemplateGroupHandler(svc *service.CompilationTemplateGroupService) *CompilationTemplateGroupHandler {
return &CompilationTemplateGroupHandler{compilationTemplateGroupService: svc}
}
// List returns the tenant's compilation template groups with their nested
// templates and pagination.
//
// @Summary List compilation template groups
// @Tags compilation template group
// @Security ApiKeyAuth
// @Param keywords query string false "name keyword filter"
// @Param scope query string false "scope filter (file|dataset)"
// @Param page query int false "page number" default(1)
// @Param page_size query int false "items per page" default(30)
// @Param orderby query string false "order field" default(create_time)
// @Param desc query bool false "descending" default(true)
// @Success 200 {object} map[string]interface{}
// @Router /v1/compilation-template-groups [get]
func (h *CompilationTemplateGroupHandler) List(c *gin.Context) {
user, code, msg := GetUser(c)
if code != common.CodeSuccess {
common.ErrorWithCode(c, code, msg)
return
}
page, _ := strconv.Atoi(c.DefaultQuery("page", "1"))
pageSize, _ := strconv.Atoi(c.DefaultQuery("page_size", "30"))
if page < 1 {
page = 1
}
if pageSize < 1 {
pageSize = 30
}
keywords := c.Query("keywords")
scope := c.Query("scope")
orderby := c.DefaultQuery("orderby", "create_time")
desc := c.DefaultQuery("desc", "true") != "false"
groups, err := h.compilationTemplateGroupService.ListSaved(c.Request.Context(), user.ID, keywords, scope, orderby, desc)
if err != nil {
common.ResponseWithCodeData(c, common.CodeExceptionError, nil, err.Error())
return
}
total := len(groups)
if page > 0 && pageSize > 0 {
start := (page - 1) * pageSize
if start >= total {
groups = []*service.GroupListItem{}
} else {
end := start + pageSize
if end > total {
end = total
}
groups = groups[start:end]
}
}
common.SuccessWithData(c, gin.H{"groups": groups, "total": total}, "success")
}
// Get returns a single compilation template group.
//
// @Summary Get compilation template group
// @Tags compilation template group
// @Security ApiKeyAuth
// @Param group_id path string true "group id"
// @Success 200 {object} map[string]interface{}
// @Router /v1/compilation-template-groups/{group_id} [get]
func (h *CompilationTemplateGroupHandler) Get(c *gin.Context) {
user, code, msg := GetUser(c)
if code != common.CodeSuccess {
common.ErrorWithCode(c, code, msg)
return
}
id := c.Param("group_id")
group, err := h.compilationTemplateGroupService.GetSaved(c.Request.Context(), user.ID, id)
if err != nil {
common.ResponseWithCodeData(c, common.CodeExceptionError, nil, err.Error())
return
}
if group == nil {
common.ResponseWithCodeData(c, common.CodeNotFound, nil, "Cannot find compilation template group "+id+".")
return
}
common.SuccessWithData(c, group, "success")
}
// Save creates a new compilation template group with its child templates.
//
// @Summary Save compilation template group
// @Tags compilation template group
// @Security ApiKeyAuth
// @Param request body service.GroupRequest true "group payload"
// @Success 200 {object} map[string]interface{}
// @Router /v1/compilation-template-groups [post]
func (h *CompilationTemplateGroupHandler) Save(c *gin.Context) {
user, code, msg := GetUser(c)
if code != common.CodeSuccess {
common.ErrorWithCode(c, code, msg)
return
}
var req service.GroupRequest
if err := c.ShouldBindJSON(&req); err != nil {
common.ResponseWithCodeData(c, common.CodeArgumentError, nil, "Invalid request: "+err.Error())
return
}
if err := service.ValidateGroupRequest(&req); err != nil {
common.ResponseWithCodeData(c, common.CodeDataError, nil, err.Error())
return
}
nameExists, err := h.compilationTemplateGroupService.NameExists(c.Request.Context(), user.ID, req.Name)
if err != nil {
common.ResponseWithCodeData(c, common.CodeExceptionError, nil, err.Error())
return
}
if nameExists {
common.ResponseWithCodeData(c, common.CodeDataError, nil, "Duplicated compilation template group name.")
return
}
group, err := h.compilationTemplateGroupService.CreateGroup(c.Request.Context(), user.ID, &req)
if err != nil {
common.ResponseWithCodeData(c, common.CodeDataError, nil, err.Error())
return
}
common.SuccessWithData(c, group, "success")
}
// Update applies a partial update to a compilation template group.
//
// @Summary Update compilation template group
// @Tags compilation template group
// @Security ApiKeyAuth
// @Param group_id path string true "group id"
// @Param request body service.GroupRequest true "group payload"
// @Success 200 {object} map[string]interface{}
// @Router /v1/compilation-template-groups/{group_id} [put]
func (h *CompilationTemplateGroupHandler) Update(c *gin.Context) {
user, code, msg := GetUser(c)
if code != common.CodeSuccess {
common.ErrorWithCode(c, code, msg)
return
}
id := c.Param("group_id")
var req service.GroupRequest
if err := c.ShouldBindJSON(&req); err != nil {
common.ResponseWithCodeData(c, common.CodeArgumentError, nil, "Invalid request: "+err.Error())
return
}
if err := service.ValidateGroupRequest(&req); err != nil {
common.ResponseWithCodeData(c, common.CodeDataError, nil, err.Error())
return
}
if req.Name != "" {
nameExists, nerr := h.compilationTemplateGroupService.NameExists(c.Request.Context(), user.ID, req.Name, id)
if nerr != nil {
common.ResponseWithCodeData(c, common.CodeExceptionError, nil, nerr.Error())
return
}
if nameExists {
common.ResponseWithCodeData(c, common.CodeDataError, nil, "Duplicated compilation template group name.")
return
}
}
group, err := h.compilationTemplateGroupService.UpdateGroup(c.Request.Context(), user.ID, id, &req)
if err != nil {
common.ResponseWithCodeData(c, common.CodeDataError, nil, err.Error())
return
}
if group == nil {
common.ResponseWithCodeData(c, common.CodeNotFound, nil, "Cannot find compilation template group "+id+".")
return
}
common.SuccessWithData(c, group, "success")
}
// Delete soft-deletes a compilation template group and its children.
//
// @Summary Delete compilation template group
// @Tags compilation template group
// @Security ApiKeyAuth
// @Param group_id path string true "group id"
// @Success 200 {object} map[string]interface{}
// @Router /v1/compilation-template-groups/{group_id} [delete]
func (h *CompilationTemplateGroupHandler) Delete(c *gin.Context) {
user, code, msg := GetUser(c)
if code != common.CodeSuccess {
common.ErrorWithCode(c, code, msg)
return
}
id := c.Param("group_id")
ok, err := h.compilationTemplateGroupService.DeleteGroup(c.Request.Context(), user.ID, id)
if err != nil {
common.ResponseWithCodeData(c, common.CodeExceptionError, nil, err.Error())
return
}
if !ok {
common.ResponseWithCodeData(c, common.CodeNotFound, nil, "Cannot find compilation template group "+id+".")
return
}
common.SuccessWithData(c, true, "success")
}

View File

@@ -0,0 +1,475 @@
//
// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
package handler
import (
"net/http"
"strconv"
"ragflow/internal/common"
"ragflow/internal/dao"
"ragflow/internal/entity"
"ragflow/internal/service"
dataset "ragflow/internal/service/dataset"
"ragflow/internal/service/file"
"github.com/gin-gonic/gin"
"go.uber.org/zap"
)
// maxArtifactPageSize caps the page_size query parameter for artifact list
// endpoints so a client cannot force an unbounded document-engine search.
const maxArtifactPageSize = 100
// DatasetArtifactHandler exposes the knowledge-compilation artifact REST APIs
// backed by DatasetArtifactService.
type DatasetArtifactHandler struct {
svc *service.DatasetArtifactService
datasetSvc *dataset.DatasetService
fileCommitSvc *file.FileCommitService
}
// NewDatasetArtifactHandler creates a DatasetArtifactHandler.
func NewDatasetArtifactHandler(svc *service.DatasetArtifactService, datasetSvc *dataset.DatasetService, fileCommitSvc *file.FileCommitService) *DatasetArtifactHandler {
return &DatasetArtifactHandler{svc: svc, datasetSvc: datasetSvc, fileCommitSvc: fileCommitSvc}
}
// datasetOwner resolves the dataset and returns its tenant id. It also enforces
// dataset access permission for the requesting user. When the request is
// unauthenticated, the dataset is missing, or the user is not allowed to access
// it, an error response is written and (nil, "", "") is returned so callers can
// abort without sending a second response.
func (h *DatasetArtifactHandler) datasetOwner(c *gin.Context, datasetID string) (*entity.User, string, string) {
user, code, msg := GetUser(c)
if code != common.CodeSuccess {
common.ErrorWithCode(c, code, msg)
return nil, "", ""
}
kb, err := h.datasetSvc.GetKnowledgebaseByID(c.Request.Context(), datasetID)
if err != nil {
if dao.IsNotFoundErr(err) {
common.ErrorWithCode(c, common.CodeNotFound, "dataset not found")
} else {
common.ErrorWithCode(c, common.CodeServerError, "failed to resolve dataset")
}
return nil, "", ""
}
if kb == nil {
common.ErrorWithCode(c, common.CodeNotFound, "dataset not found")
return nil, "", ""
}
if !h.datasetSvc.Accessible(c.Request.Context(), datasetID, user.ID) {
common.ErrorWithCode(c, common.CodeForbidden, "no permission to access this dataset")
return nil, "", ""
}
return user, kb.TenantID, msg
}
// HEAD /artifacts — any wiki artifact present?
func (h *DatasetArtifactHandler) AnyArtifact(c *gin.Context) {
_, tenantID, _ := h.datasetOwner(c, c.Param("dataset_id"))
if tenantID == "" {
return
}
has, err := h.svc.HasWiki(c.Request.Context(), tenantID, c.Param("dataset_id"))
if err != nil {
common.ErrorWithCode(c, common.CodeDataError, err.Error())
return
}
if has {
c.Status(http.StatusOK)
} else {
c.Status(http.StatusNotFound)
}
}
// GET /artifacts — list wiki pages.
func (h *DatasetArtifactHandler) ListArtifacts(c *gin.Context) {
_, tenantID, _ := h.datasetOwner(c, c.Param("dataset_id"))
if tenantID == "" {
return
}
datasetID := c.Param("dataset_id")
pageType := c.Query("page_type")
topic := c.Query("topic")
page, _ := strconv.Atoi(c.DefaultQuery("page", "1"))
pageSize, _ := strconv.Atoi(c.DefaultQuery("page_size", "30"))
if page < 1 {
page = 1
}
if pageSize < 1 {
pageSize = 30
}
if pageSize > maxArtifactPageSize {
pageSize = maxArtifactPageSize
}
items, total, err := h.svc.ListWikiPages(c.Request.Context(), tenantID, datasetID, pageType, topic, page, pageSize)
if err != nil {
common.ErrorWithCode(c, common.CodeDataError, err.Error())
return
}
common.SuccessWithData(c, gin.H{"total": total, "pages": items}, "success")
}
// PUT /artifacts/<page_type>/<slug> — edit a wiki page.
func (h *DatasetArtifactHandler) UpdateArtifact(c *gin.Context) {
user, tenantID, _ := h.datasetOwner(c, c.Param("dataset_id"))
if tenantID == "" {
return
}
datasetID := c.Param("dataset_id")
pageType := c.Param("page_type")
slug := c.Param("slug")
var req struct {
ContentMd string `json:"content_md"`
Title string `json:"title"`
Outlinks []string `json:"outlinks"`
}
if err := c.ShouldBindJSON(&req); err != nil {
common.ErrorWithCode(c, common.CodeArgumentError, err.Error())
return
}
// Read the current page before mutating so an audit commit can record an
// accurate old-to-new diff. Only record when the prior state was read
// successfully; a failed read must not produce a diff against an unknown
// old value.
var oldContent string
canRecordAudit := false
if req.ContentMd != "" {
before, gerr := h.svc.GetWikiPage(c.Request.Context(), tenantID, datasetID, pageType, slug)
if gerr != nil {
common.Warn("failed to read wiki page before recording edit commit", zap.Error(gerr))
} else if before != nil {
oldContent = before.ContentMd
canRecordAudit = true
}
}
detail, err := h.svc.UpdateWikiPage(c.Request.Context(), tenantID, datasetID, pageType, slug, req.ContentMd, req.Title, req.Outlinks)
if err != nil {
common.ErrorWithCode(c, common.CodeDataError, err.Error())
return
}
if detail == nil {
common.ErrorWithCode(c, common.CodeNotFound, "page not found")
return
}
// Record a wiki-page edit audit commit (git-style parent chain) when the
// content actually changed. Best-effort: a commit write failure must not
// roll back the page edit already persisted above.
if canRecordAudit && req.ContentMd != oldContent && h.fileCommitSvc != nil {
title := req.Title
if title == "" {
title = detail.Title
}
if _, cerr := h.fileCommitSvc.RecordPageEdit(c.Request.Context(), file.PageEditCommitInput{
DatasetID: datasetID,
DocID: pageType + "/" + slug,
Slug: slug,
PageType: pageType,
Title: title,
AuthorID: user.ID,
OldContent: oldContent,
NewContent: req.ContentMd,
}); cerr != nil {
common.Warn("failed to record wiki page edit commit", zap.Error(cerr))
}
}
common.SuccessWithData(c, detail, "success")
}
// GET /artifacts/<page_type>/<slug> — single wiki page.
func (h *DatasetArtifactHandler) GetArtifact(c *gin.Context) {
_, tenantID, _ := h.datasetOwner(c, c.Param("dataset_id"))
if tenantID == "" {
return
}
datasetID := c.Param("dataset_id")
pageType := c.Param("page_type")
slug := c.Param("slug")
detail, err := h.svc.GetWikiPage(c.Request.Context(), tenantID, datasetID, pageType, slug)
if err != nil {
common.ErrorWithCode(c, common.CodeDataError, err.Error())
return
}
if detail == nil {
common.ErrorWithCode(c, common.CodeNotFound, "page not found")
return
}
common.SuccessWithData(c, detail, "success")
}
// DELETE /artifacts — clear all wiki artifacts.
func (h *DatasetArtifactHandler) DeleteArtifacts(c *gin.Context) {
_, tenantID, _ := h.datasetOwner(c, c.Param("dataset_id"))
if tenantID == "" {
return
}
deleted, err := h.svc.ClearWiki(c.Request.Context(), tenantID, c.Param("dataset_id"))
if err != nil {
common.ErrorWithCode(c, common.CodeDataError, err.Error())
return
}
common.SuccessWithData(c, deleted, "success")
}
// GET /artifacts/topics — list wiki topics.
func (h *DatasetArtifactHandler) ListArtifactTopics(c *gin.Context) {
_, tenantID, _ := h.datasetOwner(c, c.Param("dataset_id"))
if tenantID == "" {
return
}
datasetID := c.Param("dataset_id")
items, total, err := h.svc.ListWikiTopics(c.Request.Context(), tenantID, datasetID)
if err != nil {
common.ErrorWithCode(c, common.CodeDataError, err.Error())
return
}
common.SuccessWithData(c, gin.H{"total": total, "topics": items}, "success")
}
// GET /artifacts/alteration — wiki alteration summary.
func (h *DatasetArtifactHandler) GetArtifactAlteration(c *gin.Context) {
_, tenantID, _ := h.datasetOwner(c, c.Param("dataset_id"))
if tenantID == "" {
return
}
datasetID := c.Param("dataset_id")
alt, err := h.svc.GetWikiAlteration(c.Request.Context(), tenantID, datasetID)
if err != nil {
common.ErrorWithCode(c, common.CodeDataError, err.Error())
return
}
common.SuccessWithData(c, alt, "success")
}
// GET /artifacts/graph — wiki entity/relation graph.
func (h *DatasetArtifactHandler) GetArtifactGraph(c *gin.Context) {
_, tenantID, _ := h.datasetOwner(c, c.Param("dataset_id"))
if tenantID == "" {
return
}
datasetID := c.Param("dataset_id")
graph, err := h.svc.GetWikiGraph(c.Request.Context(), tenantID, datasetID)
if err != nil {
common.ErrorWithCode(c, common.CodeDataError, err.Error())
return
}
common.SuccessWithData(c, graph, "success")
}
// GET /artifacts/structure — list compiled structures of a dataset.
func (h *DatasetArtifactHandler) ListStructures(c *gin.Context) {
_, tenantID, _ := h.datasetOwner(c, c.Param("dataset_id"))
if tenantID == "" {
return
}
datasetID := c.Param("dataset_id")
structureKind := c.Query("structure_kind")
structureIndexType := c.Query("structure_index_type")
items, total, err := h.svc.ListStructures(c.Request.Context(), tenantID, datasetID, structureKind, structureIndexType)
if err != nil {
common.ErrorWithCode(c, common.CodeDataError, err.Error())
return
}
common.SuccessWithData(c, gin.H{"total": total, "structures": items}, "success")
}
// DELETE /artifacts/structure — delete compiled structures of a dataset.
func (h *DatasetArtifactHandler) DeleteStructures(c *gin.Context) {
_, tenantID, _ := h.datasetOwner(c, c.Param("dataset_id"))
if tenantID == "" {
return
}
datasetID := c.Param("dataset_id")
structureKind := c.Query("structure_kind")
structureIndexType := c.Query("structure_index_type")
n, err := h.svc.DeleteStructures(c.Request.Context(), tenantID, datasetID, structureKind, structureIndexType)
if err != nil {
common.ErrorWithCode(c, common.CodeDataError, err.Error())
return
}
common.SuccessWithData(c, gin.H{"deleted": n}, "success")
}
// HEAD /skills — any skill artifact present?
func (h *DatasetArtifactHandler) AnySkill(c *gin.Context) {
_, tenantID, _ := h.datasetOwner(c, c.Param("dataset_id"))
if tenantID == "" {
return
}
has, err := h.svc.HasSkill(c.Request.Context(), tenantID, c.Param("dataset_id"))
if err != nil {
common.ErrorWithCode(c, common.CodeDataError, err.Error())
return
}
if has {
c.Status(http.StatusOK)
} else {
c.Status(http.StatusNotFound)
}
}
// GET /navigation — list navigation clusters.
func (h *DatasetArtifactHandler) ListNavigation(c *gin.Context) {
_, tenantID, _ := h.datasetOwner(c, c.Param("dataset_id"))
if tenantID == "" {
return
}
items, total, err := h.svc.ListNavClusters(c.Request.Context(), tenantID, c.Param("dataset_id"))
if err != nil {
common.ErrorWithCode(c, common.CodeDataError, err.Error())
return
}
common.SuccessWithData(c, gin.H{"total": total, "nav": items}, "success")
}
// DELETE /navigation — delete all navigation clusters.
func (h *DatasetArtifactHandler) DeleteNavigation(c *gin.Context) {
_, tenantID, _ := h.datasetOwner(c, c.Param("dataset_id"))
if tenantID == "" {
return
}
n, err := h.svc.DeleteNav(c.Request.Context(), tenantID, c.Param("dataset_id"))
if err != nil {
common.ErrorWithCode(c, common.CodeDataError, err.Error())
return
}
common.SuccessWithData(c, gin.H{"deleted": n}, "success")
}
// DELETE /navigation/<name> — delete a single navigation cluster.
func (h *DatasetArtifactHandler) DeleteNavigationNode(c *gin.Context) {
_, tenantID, _ := h.datasetOwner(c, c.Param("dataset_id"))
if tenantID == "" {
return
}
n, err := h.svc.DeleteNavNode(c.Request.Context(), tenantID, c.Param("dataset_id"), c.Param("name"))
if err != nil {
common.ErrorWithCode(c, common.CodeDataError, err.Error())
return
}
common.SuccessWithData(c, gin.H{"deleted": n}, "success")
}
// GET /navigation/<name>/children — list children of a navigation cluster.
func (h *DatasetArtifactHandler) ListNavigationChildren(c *gin.Context) {
_, tenantID, _ := h.datasetOwner(c, c.Param("dataset_id"))
if tenantID == "" {
return
}
items, total, err := h.svc.ListNavChildren(c.Request.Context(), tenantID, c.Param("dataset_id"), c.Param("name"))
if err != nil {
common.ErrorWithCode(c, common.CodeDataError, err.Error())
return
}
common.SuccessWithData(c, gin.H{"total": total, "children": items}, "success")
}
// GET /skills — skill tree.
func (h *DatasetArtifactHandler) GetSkillTree(c *gin.Context) {
_, tenantID, _ := h.datasetOwner(c, c.Param("dataset_id"))
if tenantID == "" {
return
}
kwd := c.Query("kwd")
items, total, err := h.svc.GetSkillTree(c.Request.Context(), tenantID, c.Param("dataset_id"), kwd)
if err != nil {
common.ErrorWithCode(c, common.CodeDataError, err.Error())
return
}
common.SuccessWithData(c, gin.H{"total": total, "tree": items}, "success")
}
// DELETE /skills — delete all skills.
func (h *DatasetArtifactHandler) DeleteSkills(c *gin.Context) {
_, tenantID, _ := h.datasetOwner(c, c.Param("dataset_id"))
if tenantID == "" {
return
}
n, err := h.svc.DeleteSkills(c.Request.Context(), tenantID, c.Param("dataset_id"), "")
if err != nil {
common.ErrorWithCode(c, common.CodeDataError, err.Error())
return
}
common.SuccessWithData(c, gin.H{"deleted": n}, "success")
}
// GET /skills/<skill_kwd> — single skill page.
func (h *DatasetArtifactHandler) GetSkillPage(c *gin.Context) {
_, tenantID, _ := h.datasetOwner(c, c.Param("dataset_id"))
if tenantID == "" {
return
}
detail, err := h.svc.GetSkillPage(c.Request.Context(), tenantID, c.Param("dataset_id"), c.Param("skill_kwd"))
if err != nil {
common.ErrorWithCode(c, common.CodeDataError, err.Error())
return
}
if detail == nil {
common.ErrorWithCode(c, common.CodeNotFound, "skill not found")
return
}
common.SuccessWithData(c, detail, "success")
}
// DELETE /skills/<skill_kwd> — delete a single skill.
func (h *DatasetArtifactHandler) DeleteSkill(c *gin.Context) {
_, tenantID, _ := h.datasetOwner(c, c.Param("dataset_id"))
if tenantID == "" {
return
}
n, err := h.svc.DeleteSkills(c.Request.Context(), tenantID, c.Param("dataset_id"), c.Param("skill_kwd"))
if err != nil {
common.ErrorWithCode(c, common.CodeDataError, err.Error())
return
}
common.SuccessWithData(c, gin.H{"deleted": n}, "success")
}
// GET /documents/<document_id>/structure/graph — document structure graph.
func (h *DatasetArtifactHandler) GetDocumentGraph(c *gin.Context) {
_, tenantID, _ := h.datasetOwner(c, c.Param("dataset_id"))
if tenantID == "" {
return
}
datasetID := c.Param("dataset_id")
documentID := c.Param("document_id")
graphType := c.Query("graph_type")
items, total, err := h.svc.GetDocumentGraph(c.Request.Context(), tenantID, datasetID, documentID, graphType)
if err != nil {
common.ErrorWithCode(c, common.CodeDataError, err.Error())
return
}
common.SuccessWithData(c, gin.H{"total": total, "graph": items}, "success")
}
// DELETE /documents/<document_id>/structure/graph — delete document structure graph.
func (h *DatasetArtifactHandler) DeleteDocumentGraph(c *gin.Context) {
_, tenantID, _ := h.datasetOwner(c, c.Param("dataset_id"))
if tenantID == "" {
return
}
datasetID := c.Param("dataset_id")
documentID := c.Param("document_id")
n, err := h.svc.DeleteDocumentGraph(c.Request.Context(), tenantID, datasetID, documentID)
if err != nil {
common.ErrorWithCode(c, common.CodeDataError, err.Error())
return
}
common.SuccessWithData(c, gin.H{"deleted": n}, "success")
}

View File

@@ -38,6 +38,7 @@ type fileCommitService interface {
GetCommitTree(ctx context.Context, commitID string) (map[string]interface{}, error)
GetCommitFileContent(ctx context.Context, folderID, commitID, fileID string) ([]byte, error)
GetFileVersionHistory(ctx context.Context, fileID string) ([]entity.VersionEntry, error)
ListPageCommits(ctx context.Context, datasetID, pageType, slug string, page, pageSize int) ([]*entity.FileCommit, int64, error)
}
// FileCommitHandler file commit handler
@@ -217,6 +218,49 @@ func (h *FileCommitHandler) ListCommits(c *gin.Context) {
}
ctx := c.Request.Context()
// Python's list_commits supports ?slug=<page_slug> to filter audit commits
// for a specific wiki/skill page (written by record_page_edit). These page
// commits are scoped to the dataset and page file key, so route them through
// the page-commit path instead of the folder-based ListCommits. This only
// applies to the /datasets/{dataset_id}/commits route.
if slug := c.Query("slug"); slug != "" {
datasetID := c.Param("dataset_id")
if datasetID == "" {
common.ErrorWithCode(c, common.CodeArgumentError, "slug requires a dataset scope")
return
}
pageType := c.Query("page_type")
commits, total, err := h.commitService.ListPageCommits(ctx, datasetID, pageType, slug, page, pageSize)
if err != nil {
jsonInternalError(c, err)
return
}
var commitList []entity.CommitResponse
for _, commit := range commits {
var ct int64
if commit.CreateTime != nil {
ct = *commit.CreateTime
}
commitList = append(commitList, entity.CommitResponse{
ID: commit.ID,
FolderID: commit.FolderID,
ParentID: commit.ParentID,
Message: commit.Message,
AuthorID: commit.AuthorID,
FileCount: commit.FileCount,
CreateTime: &ct,
})
}
common.SuccessWithData(c, gin.H{
"total": total,
"page": page,
"page_size": pageSize,
"commits": commitList,
}, common.CodeSuccess.Message())
return
}
commits, total, err := h.commitService.ListCommits(ctx, folderID, page, pageSize, orderBy, desc)
if err != nil {
jsonInternalError(c, err)

View File

@@ -41,6 +41,7 @@ type mockFileCommitSvc struct {
getCommitTreeFn func(ctx context.Context, commitID string) (map[string]interface{}, error)
getCommitFileContentFn func(ctx context.Context, folderID, commitID, fileID string) ([]byte, error)
getFileVersionHistoryFn func(ctx context.Context, fileID string) ([]entity.VersionEntry, error)
listPageCommitsFn func(ctx context.Context, datasetID, pageType, slug string, page, pageSize int) ([]*entity.FileCommit, int64, error)
}
func (m *mockFileCommitSvc) CreateCommit(ctx context.Context, folderID, authorID, message string, changes []entity.FileChange) (*entity.FileCommit, error) {
@@ -128,6 +129,13 @@ func (m *mockFileCommitSvc) GetFileVersionHistory(ctx context.Context, fileID st
}, nil
}
func (m *mockFileCommitSvc) ListPageCommits(ctx context.Context, datasetID, pageType, slug string, page, pageSize int) ([]*entity.FileCommit, int64, error) {
if m.listPageCommitsFn != nil {
return m.listPageCommitsFn(ctx, datasetID, pageType, slug, page, pageSize)
}
return []*entity.FileCommit{}, 0, nil
}
func setupFileCommitTest(userID string) (*gin.Engine, *mockFileCommitSvc) {
mock := &mockFileCommitSvc{}
h := &FileCommitHandler{commitService: mock}

View File

@@ -112,7 +112,7 @@ func (h *ProviderHandler) AddProvider(c *gin.Context) {
}
func (h *ProviderHandler) DeleteProvider(c *gin.Context) {
providerName := c.Param("provider_name")
providerName := c.Param("provider_id_or_name")
if providerName == "" {
common.ResponseWithHttpCodeData(c, http.StatusBadRequest, 400, nil, "Provider name is required")
return
@@ -131,7 +131,7 @@ func (h *ProviderHandler) DeleteProvider(c *gin.Context) {
}
func (h *ProviderHandler) ShowProvider(c *gin.Context) {
providerName := c.Param("provider_name")
providerName := c.Param("provider_id_or_name")
if providerName == "" {
common.ResponseWithHttpCodeData(c, http.StatusBadRequest, 400, nil, "Provider name is required")
return
@@ -146,7 +146,7 @@ func (h *ProviderHandler) ShowProvider(c *gin.Context) {
}
func (h *ProviderHandler) ListModels(c *gin.Context) {
providerName := c.Param("provider_name")
providerName := c.Param("provider_id_or_name")
if providerName == "" {
common.ResponseWithHttpCodeData(c, http.StatusBadRequest, 400, nil, "Provider name is required")
return
@@ -221,7 +221,7 @@ func (h *ProviderHandler) ListModels(c *gin.Context) {
}
func (h *ProviderHandler) ShowModel(c *gin.Context) {
providerName := c.Param("provider_name")
providerName := c.Param("provider_id_or_name")
if providerName == "" {
common.ResponseWithHttpCodeData(c, http.StatusBadRequest, 400, nil, "Provider name is required")
return
@@ -249,7 +249,7 @@ type CreateProviderInstanceRequest struct {
}
func (h *ProviderHandler) CreateProviderInstance(c *gin.Context) {
providerName := c.Param("provider_name")
providerName := c.Param("provider_id_or_name")
if providerName == "" {
common.ResponseWithHttpCodeData(c, http.StatusBadRequest, 400, nil, "Provider name is required")
return
@@ -287,7 +287,7 @@ func (h *ProviderHandler) CreateProviderInstance(c *gin.Context) {
}
func (h *ProviderHandler) ListProviderInstances(c *gin.Context) {
providerName := c.Param("provider_name")
providerName := c.Param("provider_id_or_name")
if providerName == "" {
common.ResponseWithHttpCodeData(c, http.StatusBadRequest, 400, nil, "Provider name is required")
return
@@ -306,13 +306,13 @@ func (h *ProviderHandler) ListProviderInstances(c *gin.Context) {
}
func (h *ProviderHandler) ShowProviderInstance(c *gin.Context) {
providerName := c.Param("provider_name")
providerName := c.Param("provider_id_or_name")
if providerName == "" {
common.ResponseWithHttpCodeData(c, http.StatusBadRequest, 400, nil, "Provider name is required")
return
}
instanceIDOrName := c.Param("instance_name")
instanceIDOrName := c.Param("instance_id_or_name")
if instanceIDOrName == "" {
common.ResponseWithHttpCodeData(c, http.StatusBadRequest, 400, nil, "Instance name is required")
return
@@ -331,13 +331,13 @@ func (h *ProviderHandler) ShowProviderInstance(c *gin.Context) {
}
func (h *ProviderHandler) ShowInstanceBalance(c *gin.Context) {
providerName := c.Param("provider_name")
providerName := c.Param("provider_id_or_name")
if providerName == "" {
common.ResponseWithHttpCodeData(c, http.StatusBadRequest, 400, nil, "Provider name is required")
return
}
instanceName := c.Param("instance_name")
instanceName := c.Param("instance_id_or_name")
if instanceName == "" {
common.ResponseWithHttpCodeData(c, http.StatusBadRequest, 400, nil, "Instance name is required")
return
@@ -357,7 +357,7 @@ func (h *ProviderHandler) ShowInstanceBalance(c *gin.Context) {
}
func (h *ProviderHandler) CheckConnection(c *gin.Context) {
providerName := c.Param("provider_name")
providerName := c.Param("provider_id_or_name")
if providerName == "" {
common.ResponseWithHttpCodeData(c, http.StatusBadRequest, 400, nil, "Provider name is required")
return
@@ -382,13 +382,13 @@ func (h *ProviderHandler) CheckConnection(c *gin.Context) {
func (h *ProviderHandler) CheckInstanceConnection(c *gin.Context) {
ctx := c.Request.Context()
providerName := c.Param("provider_name")
providerName := c.Param("provider_id_or_name")
if providerName == "" {
common.ResponseWithHttpCodeData(c, http.StatusBadRequest, 400, nil, "Provider name is required")
return
}
instanceName := c.Param("instance_name")
instanceName := c.Param("instance_id_or_name")
if instanceName == "" {
common.ResponseWithHttpCodeData(c, http.StatusBadRequest, 400, nil, "Instance name is required")
return
@@ -417,13 +417,13 @@ func (h *ProviderHandler) CheckInstanceConnection(c *gin.Context) {
func (h *ProviderHandler) ListTasks(c *gin.Context) {
ctx := c.Request.Context()
providerName := c.Param("provider_name")
providerName := c.Param("provider_id_or_name")
if providerName == "" {
common.ResponseWithHttpCodeData(c, http.StatusBadRequest, 400, nil, "Provider name is required")
return
}
instanceName := c.Param("instance_name")
instanceName := c.Param("instance_id_or_name")
if instanceName == "" {
common.ResponseWithHttpCodeData(c, http.StatusBadRequest, 400, nil, "Instance name is required")
return
@@ -443,13 +443,13 @@ func (h *ProviderHandler) ListTasks(c *gin.Context) {
func (h *ProviderHandler) ShowTask(c *gin.Context) {
ctx := c.Request.Context()
providerName := c.Param("provider_name")
providerName := c.Param("provider_id_or_name")
if providerName == "" {
common.ResponseWithHttpCodeData(c, http.StatusBadRequest, 400, nil, "Provider name is required")
return
}
instanceName := c.Param("instance_name")
instanceName := c.Param("instance_id_or_name")
if instanceName == "" {
common.ResponseWithHttpCodeData(c, http.StatusBadRequest, 400, nil, "Instance name is required")
return
@@ -484,13 +484,13 @@ type AlterProviderInstanceRequest struct {
func (h *ProviderHandler) AlterProviderInstance(c *gin.Context) {
ctx := c.Request.Context()
providerName := c.Param("provider_name")
providerName := c.Param("provider_id_or_name")
if providerName == "" {
common.ResponseWithHttpCodeData(c, http.StatusBadRequest, 400, nil, "Provider name is required")
return
}
instanceName := c.Param("instance_name")
instanceName := c.Param("instance_id_or_name")
if instanceName == "" {
common.ResponseWithHttpCodeData(c, http.StatusBadRequest, 400, nil, "Instance name is required")
return
@@ -527,7 +527,7 @@ type DropProviderInstanceRequest struct {
}
func (h *ProviderHandler) DropProviderInstance(c *gin.Context) {
providerName := c.Param("provider_name")
providerName := c.Param("provider_id_or_name")
if providerName == "" {
common.ResponseWithHttpCodeData(c, http.StatusBadRequest, 400, nil, "Provider name is required")
return
@@ -552,12 +552,12 @@ func (h *ProviderHandler) DropProviderInstance(c *gin.Context) {
func (h *ProviderHandler) ListInstanceModels(c *gin.Context) {
ctx := c.Request.Context()
providerName := c.Param("provider_name")
providerName := c.Param("provider_id_or_name")
if providerName == "" {
common.ResponseWithHttpCodeData(c, http.StatusBadRequest, 400, nil, "Provider name is required")
return
}
instanceName := c.Param("instance_name")
instanceName := c.Param("instance_id_or_name")
if instanceName == "" {
common.ResponseWithHttpCodeData(c, http.StatusBadRequest, 400, nil, "Instance name is required")
return
@@ -600,13 +600,13 @@ type AlterModelRequest struct {
}
func (h *ProviderHandler) AlterModel(c *gin.Context) {
providerName := c.Param("provider_name")
providerName := c.Param("provider_id_or_name")
if providerName == "" {
common.ResponseWithHttpCodeData(c, http.StatusBadRequest, 400, nil, "Provider name is required")
return
}
instanceName := c.Param("instance_name")
instanceName := c.Param("instance_id_or_name")
if instanceName == "" {
common.ResponseWithHttpCodeData(c, http.StatusBadRequest, 400, nil, "Instance name is required")
return
@@ -691,8 +691,8 @@ func (h *ProviderHandler) AddModel(c *gin.Context) {
return
}
req.ProviderName = c.Param("provider_name")
req.InstanceName = c.Param("instance_name")
req.ProviderName = c.Param("provider_id_or_name")
req.InstanceName = c.Param("instance_id_or_name")
if req.ProviderName == "" || req.InstanceName == "" {
common.ResponseWithHttpCodeData(c, http.StatusBadRequest, common.CodeBadRequest, nil, "provider_name and instance_name are required")
@@ -726,12 +726,12 @@ type DropInstanceModelRequest struct {
}
func (h *ProviderHandler) DropInstanceModels(c *gin.Context) {
providerName := c.Param("provider_name")
providerName := c.Param("provider_id_or_name")
if providerName == "" {
common.ResponseWithHttpCodeData(c, http.StatusBadRequest, 400, nil, "Provider name is required")
return
}
instanceName := c.Param("instance_name")
instanceName := c.Param("instance_id_or_name")
if instanceName == "" {
common.ResponseWithHttpCodeData(c, http.StatusBadRequest, 400, nil, "Instance name is required")
return

View File

@@ -88,8 +88,8 @@ func TestProviderHandlerAlterModelRejectsMissingModelSelector(t *testing.T) {
ctx, recorder := newProviderHandlerRequest(
t,
map[string]interface{}{"status": "active"},
gin.Param{Key: "provider_name", Value: "OpenAI"},
gin.Param{Key: "instance_name", Value: "default"},
gin.Param{Key: "provider_id_or_name", Value: "OpenAI"},
gin.Param{Key: "instance_id_or_name", Value: "default"},
)
NewProviderHandler(nil, service.NewModelProviderService()).AlterModel(ctx)
@@ -107,8 +107,8 @@ func TestProviderHandlerAlterModelRejectsInvalidStatus(t *testing.T) {
ctx, recorder := newProviderHandlerRequest(
t,
map[string]interface{}{"status": "disabled"},
gin.Param{Key: "provider_name", Value: "OpenAI"},
gin.Param{Key: "instance_name", Value: "default"},
gin.Param{Key: "provider_id_or_name", Value: "OpenAI"},
gin.Param{Key: "instance_id_or_name", Value: "default"},
gin.Param{Key: "model_name", Value: "gpt-test"},
)
@@ -131,8 +131,8 @@ func TestProviderHandlerAlterModelUpdatesStatus(t *testing.T) {
ctx, recorder := newProviderHandlerRequest(
t,
map[string]interface{}{"status": "inactive"},
gin.Param{Key: "provider_name", Value: "OpenAI"},
gin.Param{Key: "instance_name", Value: "default"},
gin.Param{Key: "provider_id_or_name", Value: "OpenAI"},
gin.Param{Key: "instance_id_or_name", Value: "default"},
gin.Param{Key: "model_name", Value: "gpt-test"},
)

View File

@@ -55,6 +55,10 @@ type Router struct {
botHandler *handler.BotHandler
componentsHandler *handler.ComponentsHandler
pipelineHandler *handler.PipelineHandler
compilationTemplateHandler *handler.CompilationTemplateHandler
compilationTemplateGroupHandler *handler.CompilationTemplateGroupHandler
datasetArtifactHandler *handler.DatasetArtifactHandler
}
// NewRouter create router
@@ -90,6 +94,9 @@ func NewRouter(
botHandler *handler.BotHandler,
componentsHandler *handler.ComponentsHandler,
pipelineHandler *handler.PipelineHandler,
compilationTemplateHandler *handler.CompilationTemplateHandler,
compilationTemplateGroupHandler *handler.CompilationTemplateGroupHandler,
datasetArtifactHandler *handler.DatasetArtifactHandler,
) *Router {
return &Router{
authHandler: authHandler,
@@ -123,6 +130,10 @@ func NewRouter(
botHandler: botHandler,
componentsHandler: componentsHandler,
pipelineHandler: pipelineHandler,
compilationTemplateHandler: compilationTemplateHandler,
compilationTemplateGroupHandler: compilationTemplateGroupHandler,
datasetArtifactHandler: datasetArtifactHandler,
}
}
@@ -350,6 +361,32 @@ func (r *Router) Setup(engine *gin.Engine) {
datasets.GET("/:dataset_id/index", r.datasetsHandler.TraceIndex)
datasets.POST("/:dataset_id/index", r.datasetsHandler.RunIndex)
datasets.DELETE("/:dataset_id/index", r.datasetsHandler.DeleteIndex)
// Knowledge-compilation wiki artifacts
datasets.HEAD("/:dataset_id/artifacts", r.datasetArtifactHandler.AnyArtifact)
datasets.GET("/:dataset_id/artifacts", r.datasetArtifactHandler.ListArtifacts)
datasets.DELETE("/:dataset_id/artifacts", r.datasetArtifactHandler.DeleteArtifacts)
datasets.GET("/:dataset_id/artifacts/topics", r.datasetArtifactHandler.ListArtifactTopics)
datasets.GET("/:dataset_id/artifacts/alteration", r.datasetArtifactHandler.GetArtifactAlteration)
datasets.GET("/:dataset_id/artifacts/graph", r.datasetArtifactHandler.GetArtifactGraph)
datasets.GET("/:dataset_id/artifacts/:page_type/:slug", r.datasetArtifactHandler.GetArtifact)
datasets.PUT("/:dataset_id/artifacts/:page_type/:slug", r.datasetArtifactHandler.UpdateArtifact)
datasets.GET("/:dataset_id/artifacts/structure", r.datasetArtifactHandler.ListStructures)
datasets.DELETE("/:dataset_id/artifacts/structure", r.datasetArtifactHandler.DeleteStructures)
// Knowledge-compilation navigation
datasets.GET("/:dataset_id/navigation", r.datasetArtifactHandler.ListNavigation)
datasets.DELETE("/:dataset_id/navigation", r.datasetArtifactHandler.DeleteNavigation)
datasets.DELETE("/:dataset_id/navigation/:name", r.datasetArtifactHandler.DeleteNavigationNode)
datasets.GET("/:dataset_id/navigation/:name/children", r.datasetArtifactHandler.ListNavigationChildren)
// Knowledge-compilation skills
datasets.HEAD("/:dataset_id/skills", r.datasetArtifactHandler.AnySkill)
datasets.GET("/:dataset_id/skills", r.datasetArtifactHandler.GetSkillTree)
datasets.DELETE("/:dataset_id/skills", r.datasetArtifactHandler.DeleteSkills)
datasets.GET("/:dataset_id/skills/:skill_kwd", r.datasetArtifactHandler.GetSkillPage)
datasets.DELETE("/:dataset_id/skills/:skill_kwd", r.datasetArtifactHandler.DeleteSkill)
datasets.DELETE("/:dataset_id/:index_type", r.datasetsHandler.DeleteIndex)
//datasets.DELETE("/:dataset_id/graph", r.datasetsHandler.DeleteKnowledgeGraph)
datasets.POST("", r.datasetsHandler.CreateDataset)
@@ -391,6 +428,8 @@ func (r *Router) Setup(engine *gin.Engine) {
datasets.DELETE("/:dataset_id/chunks", r.chunkHandler.StopParsing)
datasets.DELETE("/:dataset_id/documents/:document_id/chunks", r.chunkHandler.RemoveChunks)
datasets.PUT("/:dataset_id/documents/:document_id/metadata/config", r.datasetsHandler.UpdateDocumentMetadataConfig)
datasets.GET("/:dataset_id/documents/:document_id/structure/graph", r.datasetArtifactHandler.GetDocumentGraph)
datasets.DELETE("/:dataset_id/documents/:document_id/structure/graph", r.datasetArtifactHandler.DeleteDocumentGraph)
datasets.POST("/:dataset_id/metadata/update", r.documentHandler.MetadataBatchUpdate)
datasets.PATCH("/:dataset_id/documents/metadatas", r.documentHandler.UpdateDocumentMetadatas)
}
@@ -517,24 +556,24 @@ func (r *Router) Setup(engine *gin.Engine) {
{
provider.GET("", r.providerHandler.ListProviders)
provider.PUT("", r.providerHandler.AddProvider)
provider.GET("/:provider_name", r.providerHandler.ShowProvider)
provider.DELETE("/:provider_name", r.providerHandler.DeleteProvider)
provider.GET("/:provider_name/models", r.providerHandler.ListModels)
provider.GET("/:provider_name/models/:model_name", r.providerHandler.ShowModel)
provider.POST("/:provider_name/instances", r.providerHandler.CreateProviderInstance)
provider.GET("/:provider_name/instances", r.providerHandler.ListProviderInstances)
provider.GET("/:provider_name/instances/:instance_name", r.providerHandler.ShowProviderInstance)
provider.GET("/:provider_name/instances/:instance_name/balance", r.providerHandler.ShowInstanceBalance)
provider.GET("/:provider_name/instances/:instance_name/connection", r.providerHandler.CheckInstanceConnection)
provider.POST("/:provider_name/connection", r.providerHandler.CheckConnection)
provider.GET("/:provider_name/instances/:instance_name/tasks", r.providerHandler.ListTasks)
provider.GET("/:provider_name/instances/:instance_name/tasks/:task_id", r.providerHandler.ShowTask)
provider.PUT("/:provider_name/instances/:instance_name", r.providerHandler.AlterProviderInstance)
provider.DELETE("/:provider_name/instances", r.providerHandler.DropProviderInstance)
provider.GET("/:provider_name/instances/:instance_name/models", r.providerHandler.ListInstanceModels)
provider.PATCH("/:provider_name/instances/:instance_name/models/*model_name", r.providerHandler.AlterModel)
provider.POST("/:provider_name/instances/:instance_name/models", r.providerHandler.AddModel)
provider.DELETE("/:provider_name/instances/:instance_name/models", r.providerHandler.DropInstanceModels)
provider.GET("/:provider_id_or_name", r.providerHandler.ShowProvider)
provider.DELETE("/:provider_id_or_name", r.providerHandler.DeleteProvider)
provider.GET("/:provider_id_or_name/models", r.providerHandler.ListModels)
provider.GET("/:provider_id_or_name/models/:model_name", r.providerHandler.ShowModel)
provider.POST("/:provider_id_or_name/instances", r.providerHandler.CreateProviderInstance)
provider.GET("/:provider_id_or_name/instances", r.providerHandler.ListProviderInstances)
provider.GET("/:provider_id_or_name/instances/:instance_id_or_name", r.providerHandler.ShowProviderInstance)
provider.GET("/:provider_id_or_name/instances/:instance_id_or_name/balance", r.providerHandler.ShowInstanceBalance)
provider.GET("/:provider_id_or_name/instances/:instance_id_or_name/connection", r.providerHandler.CheckInstanceConnection)
provider.POST("/:provider_id_or_name/connection", r.providerHandler.CheckConnection)
provider.GET("/:provider_id_or_name/instances/:instance_id_or_name/tasks", r.providerHandler.ListTasks)
provider.GET("/:provider_id_or_name/instances/:instance_id_or_name/tasks/:task_id", r.providerHandler.ShowTask)
provider.PUT("/:provider_id_or_name/instances/:instance_id_or_name", r.providerHandler.AlterProviderInstance)
provider.DELETE("/:provider_id_or_name/instances", r.providerHandler.DropProviderInstance)
provider.GET("/:provider_id_or_name/instances/:instance_id_or_name/models", r.providerHandler.ListInstanceModels)
provider.PATCH("/:provider_id_or_name/instances/:instance_id_or_name/models/*model_name", r.providerHandler.AlterModel)
provider.POST("/:provider_id_or_name/instances/:instance_id_or_name/models", r.providerHandler.AddModel)
provider.DELETE("/:provider_id_or_name/instances/:instance_id_or_name/models", r.providerHandler.DropInstanceModels)
v1.POST("/chat/to_model", r.providerHandler.ChatToModel)
v1.POST("/embeddings", r.providerHandler.EmbedText)
v1.POST("/rerank", r.providerHandler.RerankDocument)
@@ -581,6 +620,17 @@ func (r *Router) Setup(engine *gin.Engine) {
// runtime.DefaultRegistry.
v1.GET("/components", r.componentsHandler.Get)
// Compilation template routes
v1.GET("/compilation-templates/builtins", r.compilationTemplateHandler.ListBuiltins)
v1.GET("/compilation-templates/wiki-presets", r.compilationTemplateHandler.ListWikiPresets)
// Compilation template group routes
v1.GET("/compilation-template-groups", r.compilationTemplateGroupHandler.List)
v1.POST("/compilation-template-groups", r.compilationTemplateGroupHandler.Save)
v1.GET("/compilation-template-groups/:group_id", r.compilationTemplateGroupHandler.Get)
v1.PUT("/compilation-template-groups/:group_id", r.compilationTemplateGroupHandler.Update)
v1.DELETE("/compilation-template-groups/:group_id", r.compilationTemplateGroupHandler.Delete)
connectors := v1.Group("/connectors")
{
connectors.GET("", r.connectorHandler.ListConnectors)

View File

@@ -0,0 +1,494 @@
//
// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
package service
import (
"context"
"fmt"
"strings"
"time"
"ragflow/internal/dao"
"ragflow/internal/entity"
"ragflow/internal/utility"
"gorm.io/gorm"
)
// timeStr formats a Unix-millisecond timestamp for the read-side JSON payloads.
func timeStr(ms *int64) string {
if ms == nil {
return ""
}
return time.UnixMilli(*ms).UTC().Format("2006-01-02 15:04:05")
}
const (
groupScopeFile = "file"
groupScopeDataset = "dataset"
// maxGroupNameLen is the DB column size for compilation_template_group.name.
maxGroupNameLen = 128
)
// GroupTemplate is a child-template entry in a group create/update payload.
type GroupTemplate struct {
ID string `json:"id,omitempty"`
Name string `json:"name,omitempty"`
Description string `json:"description,omitempty"`
Kind string `json:"kind,omitempty"`
Config entity.JSONMap `json:"config,omitempty"`
}
// GroupRequest is the create/update payload for a compilation template group.
type GroupRequest struct {
Name string `json:"name,omitempty"`
Description string `json:"description,omitempty"`
Templates []*GroupTemplate `json:"templates,omitempty"`
}
// CompilationTemplateGroupService implements the knowledge-compilation template
// group REST operations, mirroring the Python CompilationTemplateGroupService.
type CompilationTemplateGroupService struct {
groupDAO *dao.CompilationTemplateGroupDAO
templateDAO *dao.CompilationTemplateDAO
tenantDAO *dao.TenantDAO
}
// NewCompilationTemplateGroupService creates a CompilationTemplateGroupService.
func NewCompilationTemplateGroupService() *CompilationTemplateGroupService {
return &CompilationTemplateGroupService{
groupDAO: dao.NewCompilationTemplateGroupDAO(),
templateDAO: dao.NewCompilationTemplateDAO(),
tenantDAO: dao.NewTenantDAO(),
}
}
// GroupListItem is the read-side representation of a group with its nested
// templates, mirroring Python _group_to_dict.
type GroupListItem struct {
ID string `json:"id"`
Name string `json:"name"`
Description string `json:"description"`
Scope string `json:"scope"`
CreateTime string `json:"create_time,omitempty"`
UpdateTime string `json:"update_time,omitempty"`
Templates []*TemplateListItem `json:"templates"`
}
// ListSaved returns the tenant's groups with nested templates. Mirrors Python
// list_saved(). total is derived from the full (unpaginated) result set.
func (s *CompilationTemplateGroupService) ListSaved(ctx context.Context, tenantID, keywords, scope, orderby string, desc bool) ([]*GroupListItem, error) {
groups, err := s.groupDAO.ListSaved(ctx, tenantID, keywords, scope, orderby, desc)
if err != nil {
return nil, err
}
return s.buildGroupItems(ctx, tenantID, groups)
}
// GetSaved returns a single group with its nested templates, or nil. Mirrors
// Python get_saved().
func (s *CompilationTemplateGroupService) GetSaved(ctx context.Context, tenantID, groupID string) (*GroupListItem, error) {
group, err := s.groupDAO.GetSaved(ctx, tenantID, groupID)
if err != nil || group == nil {
return nil, err
}
return s.buildGroupItem(ctx, tenantID, group)
}
// CreateGroup creates a group plus its child templates. Mirrors Python
// create_group(). It returns a GroupValidationError for payload/scope problems.
// The group and all child writes are committed atomically so a mid-way failure
// cannot leave a partially-populated group behind.
func (s *CompilationTemplateGroupService) CreateGroup(ctx context.Context, tenantID string, req *GroupRequest) (*GroupListItem, error) {
if err := validateGroupPayload(req, true); err != nil {
return nil, err
}
scope, err := s.deriveScope(req.Templates)
if err != nil {
return nil, err
}
groupID := utility.GenerateUUID()
if err := dao.DB.WithContext(ctx).Transaction(func(tx *gorm.DB) error {
group := &entity.CompilationTemplateGroup{
ID: groupID,
TenantID: tenantID,
Name: strings.TrimSpace(req.Name),
Description: strptr(req.Description),
Scope: scope,
Status: strptr(string(entity.StatusValid)),
}
if cerr := s.groupDAO.Create(ctx, tx, group); cerr != nil {
return cerr
}
return s.insertChildren(ctx, tx, tenantID, groupID, req.Templates, nil)
}); err != nil {
return nil, err
}
return s.GetSaved(ctx, tenantID, groupID)
}
// UpdateGroup applies a partial update to a group and reconciles its child
// templates. Mirrors Python update_group(). The field update and child
// reconciliation are committed atomically.
func (s *CompilationTemplateGroupService) UpdateGroup(ctx context.Context, tenantID, groupID string, req *GroupRequest) (*GroupListItem, error) {
existing, err := s.groupDAO.GetSaved(ctx, tenantID, groupID)
if err != nil {
return nil, err
}
if existing == nil {
return nil, nil
}
if err := validateGroupPayload(req, false); err != nil {
return nil, err
}
if err := dao.DB.WithContext(ctx).Transaction(func(tx *gorm.DB) error {
updates := map[string]interface{}{}
if strings.TrimSpace(req.Name) != "" {
updates["name"] = strings.TrimSpace(req.Name)
}
if req.Description != "" {
updates["description"] = req.Description
}
if req.Templates != nil {
scope, serr := s.deriveScope(req.Templates)
if serr != nil {
return serr
}
updates["scope"] = scope
}
if len(updates) > 0 {
if uerr := s.groupDAO.UpdateFields(ctx, tx, groupID, updates); uerr != nil {
return uerr
}
}
if req.Templates != nil {
return s.reconcileChildren(ctx, tx, tenantID, groupID, req.Templates)
}
return nil
}); err != nil {
return nil, err
}
return s.GetSaved(ctx, tenantID, groupID)
}
// DeleteGroup soft-deletes a group and its valid child templates. Mirrors
// Python delete_group(). Returns (false, nil) when the group is missing.
func (s *CompilationTemplateGroupService) DeleteGroup(ctx context.Context, tenantID, groupID string) (bool, error) {
existing, err := s.groupDAO.GetSaved(ctx, tenantID, groupID)
if err != nil {
return false, err
}
if existing == nil {
return false, nil
}
if err := s.templateDAO.UpdateStatusByGroup(ctx, dao.DB, groupID, string(entity.StatusInvalid)); err != nil {
return false, err
}
if err := s.groupDAO.Delete(ctx, dao.DB, tenantID, groupID); err != nil {
return false, err
}
return true, nil
}
// GroupValidationError is returned for group payload/scope problems so handlers
// can map it to a 400 without an HTTP 500.
type GroupValidationError struct{ msg string }
func (e *GroupValidationError) Error() string { return e.msg }
func groupValidationErrorf(format string, a ...interface{}) error {
return &GroupValidationError{msg: fmt.Sprintf(format, a...)}
}
// NameExists reports whether a valid group with the given name exists for the
// tenant (optionally excluding excludeID), mirroring Python name_exists().
func (s *CompilationTemplateGroupService) NameExists(ctx context.Context, tenantID, name string, excludeIDs ...string) (bool, error) {
excludeID := ""
if len(excludeIDs) > 0 {
excludeID = excludeIDs[0]
}
return s.groupDAO.NameExists(ctx, tenantID, name, excludeID)
}
// ValidateGroupRequest validates the fields present in a group create/update
// payload before any DB write, mirroring the Python blueprint
// _validate_group_payload. Required-field enforcement for create happens inside
// CreateGroup (require_all=True).
func ValidateGroupRequest(req *GroupRequest) error {
return validateGroupPayload(req, false)
}
// buildGroupItems loads child templates for many groups at once.
func (s *CompilationTemplateGroupService) buildGroupItems(ctx context.Context, tenantID string, groups []*entity.CompilationTemplateGroup) ([]*GroupListItem, error) {
items := make([]*GroupListItem, 0, len(groups))
for _, g := range groups {
item, err := s.buildGroupItem(ctx, tenantID, g)
if err != nil {
return nil, err
}
items = append(items, item)
}
return items, nil
}
func (s *CompilationTemplateGroupService) buildGroupItem(ctx context.Context, tenantID string, group *entity.CompilationTemplateGroup) (*GroupListItem, error) {
children, err := s.templateDAO.ListByGroup(ctx, dao.DB, group.ID)
if err != nil {
return nil, err
}
items := make([]*TemplateListItem, 0, len(children))
for _, c := range children {
items = append(items, &TemplateListItem{
ID: c.ID,
Name: c.Name,
Description: derefString(c.Description),
Kind: c.Kind,
Config: fillConfigDefaultLLM(ctx, s.tenantDAO, c.Config, c.TenantID),
CreateTime: timeStr(c.CreateTime),
UpdateTime: timeStr(c.UpdateTime),
})
}
return &GroupListItem{
ID: group.ID,
Name: group.Name,
Description: derefString(group.Description),
Scope: group.Scope,
CreateTime: timeStr(group.CreateTime),
UpdateTime: timeStr(group.UpdateTime),
Templates: items,
}, nil
}
// deriveScope mirrors Python _derive_scope: one wiki child => dataset scope,
// otherwise file scope (with a wiki-combination guard and re-chunk tree guard).
func (s *CompilationTemplateGroupService) deriveScope(templates []*GroupTemplate) (string, error) {
if len(templates) == 0 {
return "", groupValidationErrorf("a template group must contain at least one template.")
}
artifactCount := 0
for _, t := range templates {
if strings.TrimSpace(t.Kind) == "wiki" {
artifactCount++
}
}
if artifactCount > 0 {
if artifactCount != 1 || len(templates) != 1 {
return "", groupValidationErrorf("a wiki template cannot be combined with other templates in the same group.")
}
return groupScopeDataset, nil
}
if err := s.enforceSingleRechunkTree(templates); err != nil {
return "", err
}
return groupScopeFile, nil
}
// enforceSingleRechunkTree mirrors Python _enforce_single_rechunk_tree.
func (s *CompilationTemplateGroupService) enforceSingleRechunkTree(templates []*GroupTemplate) error {
rechunkTrees := 0
for _, t := range templates {
if strings.TrimSpace(t.Kind) != "tree" {
continue
}
raptor, _ := t.Config["raptor"].(map[string]interface{})
if b, ok := raptor["rechunk"].(bool); ok && b {
rechunkTrees++
}
}
if rechunkTrees > 1 {
return groupValidationErrorf("only one tree template in a group may enable re-chunking.")
}
return nil
}
// validateGroupPayload mirrors Python _validate_group_payload in the group
// blueprint.
func validateGroupPayload(req *GroupRequest, requireAll bool) error {
if requireAll {
if strings.TrimSpace(req.Name) == "" {
return groupValidationErrorf("missing required field: name.")
}
if len(req.Templates) == 0 {
return groupValidationErrorf("missing required field: templates.")
}
}
if strings.TrimSpace(req.Name) != "" {
if len([]byte(req.Name)) > maxGroupNameLen {
return groupValidationErrorf("template group name is too long.")
}
}
if len(req.Description) > 1024 {
return groupValidationErrorf("invalid template group description.")
}
if req.Templates != nil {
if len(req.Templates) == 0 {
return groupValidationErrorf("a template group must contain at least one template.")
}
seen := map[string]struct{}{}
for _, child := range req.Templates {
if child == nil {
return groupValidationErrorf("invalid template entry in group.")
}
payload := map[string]interface{}{
"name": child.Name, "description": child.Description,
"kind": child.Kind, "config": child.Config,
}
if err := ValidateTemplatePayload(payload, true); err != nil {
return err
}
name := strings.TrimSpace(child.Name)
if _, dup := seen[name]; dup {
return groupValidationErrorf("template name '%s' is duplicated in this group.", name)
}
seen[name] = struct{}{}
}
}
return nil
}
// insertChildren inserts new child templates. usedForReconcile controls whether
// the duplicate-name guard applies (create always; reconcile passes seen set).
func (s *CompilationTemplateGroupService) insertChildren(ctx context.Context, db *gorm.DB, tenantID, groupID string, templates []*GroupTemplate, seen map[string]struct{}) error {
for _, child := range templates {
name := strings.TrimSpace(child.Name)
if exists, err := s.templateDAO.NameExistsInGroup(ctx, tenantID, groupID, name, ""); err != nil {
return err
} else if exists {
return groupValidationErrorf("template name '%s' already exists in this group.", name)
}
desc := child.Description
config := fillConfigDefaultLLM(ctx, s.tenantDAO, child.Config, &tenantID)
if config == nil {
config = entity.JSONMap{}
}
valid := string(entity.StatusValid)
tmpl := &entity.CompilationTemplate{
ID: utility.GenerateUUID(),
TenantID: &tenantID,
GroupID: &groupID,
Name: name,
Kind: strings.TrimSpace(child.Kind),
Config: config,
Status: &valid,
}
if desc != "" {
tmpl.Description = &desc
}
if err := s.templateDAO.Save(ctx, db, tmpl); err != nil {
return err
}
}
return nil
}
// reconcileChildren mirrors the Python update_group child reconciliation:
// existing children (matched by id, or by submitted order for legacy clients)
// are updated in place, new ones inserted, and removed ones soft-deleted.
func (s *CompilationTemplateGroupService) reconcileChildren(ctx context.Context, db *gorm.DB, tenantID, groupID string, templates []*GroupTemplate) error {
current, err := s.templateDAO.ListByGroup(ctx, db, groupID)
if err != nil {
return err
}
currentByID := map[string]*entity.CompilationTemplate{}
for _, c := range current {
currentByID[c.ID] = c
}
seenNames := map[string]struct{}{}
retained := map[string]struct{}{}
for index, child := range templates {
name := strings.TrimSpace(child.Name)
if _, dup := seenNames[name]; dup {
return groupValidationErrorf("template name '%s' is duplicated in this group.", name)
}
seenNames[name] = struct{}{}
var target *entity.CompilationTemplate
if child.ID != "" {
target = currentByID[child.ID]
if target == nil {
return groupValidationErrorf("template %s does not belong to this group.", child.ID)
}
} else if index < len(current) {
// Positional fallback for legacy clients that omit IDs. Skip rows
// already retained by an explicit ID above so a mixed payload cannot
// bind two submitted entries to one existing row.
if _, taken := retained[current[index].ID]; !taken {
target = current[index]
}
}
config := fillConfigDefaultLLM(ctx, s.tenantDAO, child.Config, &tenantID)
if config == nil {
config = entity.JSONMap{}
}
desc := child.Description
if target != nil {
updates := map[string]interface{}{
"name": name, "kind": strings.TrimSpace(child.Kind), "config": config,
}
if desc != "" {
updates["description"] = desc
}
if err := s.templateDAO.UpdateFields(ctx, db, target.ID, updates); err != nil {
return err
}
retained[target.ID] = struct{}{}
continue
}
newID := utility.GenerateUUID()
valid := string(entity.StatusValid)
tmpl := &entity.CompilationTemplate{
ID: newID,
TenantID: &tenantID,
GroupID: &groupID,
Name: name,
Kind: strings.TrimSpace(child.Kind),
Config: config,
Status: &valid,
}
if desc != "" {
tmpl.Description = &desc
}
if err := s.templateDAO.Save(ctx, db, tmpl); err != nil {
return err
}
retained[newID] = struct{}{}
}
for _, c := range current {
if _, keep := retained[c.ID]; !keep {
if err := s.templateDAO.UpdateStatusByID(ctx, db, c.ID, string(entity.StatusInvalid)); err != nil {
return err
}
// Mirror Python _purge_stale_invalid_children: permanently drop
// any stale invalid non-builtin template of the same name, scoped to
// this group so a same-named template in another group is untouched.
if c.Name != "" && !c.IsBuiltin {
if err := s.templateDAO.HardDeleteOrphansByName(ctx, db, tenantID, groupID, c.Name); err != nil {
return err
}
}
}
}
return nil
}
func strptr(s string) *string { return &s }

View File

@@ -0,0 +1,379 @@
//
// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
package service
import (
"context"
"errors"
"fmt"
"os"
"path/filepath"
"sort"
"strings"
"unicode"
"ragflow/internal/dao"
"ragflow/internal/entity"
"ragflow/internal/utility"
"gopkg.in/yaml.v3"
"gorm.io/gorm"
)
// fillConfigDefaultLLM mirrors Python CompilationTemplateService.
// fill_config_default_llm: when the config has no explicit llm_id it lazily
// fills the tenant's default chat model id (read-side only; the DB row is left
// untouched).
func fillConfigDefaultLLM(ctx context.Context, tenantDAO *dao.TenantDAO, config entity.JSONMap, tenantID *string) entity.JSONMap {
if len(config) == 0 || tenantID == nil || *tenantID == "" {
return config
}
if _, ok := config["llm_id"]; ok {
return config
}
tenant, err := tenantDAO.GetByID(ctx, dao.DB, *tenantID)
if err != nil || tenant == nil || tenant.TenantLLMID == nil {
return config
}
out := make(entity.JSONMap, len(config)+1)
for k, v := range config {
out[k] = v
}
out["llm_id"] = *tenant.TenantLLMID
return out
}
// capitalizeTitle capitalizes the first letter of a lowercase section name for
// error messages (replaces deprecated strings.Title).
func capitalizeTitle(s string) string {
if s == "" {
return s
}
r := []rune(s)
r[0] = unicode.ToUpper(r[0])
return string(r)
}
// TemplateListItem is the read-side representation of a compilation template,
// mirroring Python CompilationTemplateService._to_saved_dict.
type TemplateListItem struct {
ID string `json:"id"`
Name string `json:"name"`
Description string `json:"description"`
Kind string `json:"kind"`
Config entity.JSONMap `json:"config"`
CreateTime string `json:"create_time,omitempty"`
UpdateTime string `json:"update_time,omitempty"`
}
// BuiltinTemplate is the palette representation of a built-in template,
// mirroring Python CompilationTemplateService._to_builtin_dict.
type BuiltinTemplate struct {
ID string `json:"id"`
Kind string `json:"kind"`
DisplayName string `json:"display_name"`
Description string `json:"description"`
Config entity.JSONMap `json:"config"`
}
// WikiPreset is a wiki page-structure preset loaded from YAML, mirroring
// Python load_wiki_presets_from_files.
type WikiPreset struct {
ID string `json:"id"`
Topic string `json:"topic"`
Instruction string `json:"instruction"`
PageExample string `json:"page_example"`
}
// CompilationTemplateService implements the read-side compilation template
// operations (builtins palette + wiki presets) used by the REST APIs.
type CompilationTemplateService struct {
templateDAO *dao.CompilationTemplateDAO
tenantDAO *dao.TenantDAO
}
// NewCompilationTemplateService creates a CompilationTemplateService.
func NewCompilationTemplateService() *CompilationTemplateService {
return &CompilationTemplateService{
templateDAO: dao.NewCompilationTemplateDAO(),
tenantDAO: dao.NewTenantDAO(),
}
}
// ListBuiltins returns the built-in template palette with the tenant's default
// LLM id lazily filled in, mirroring the Python list_builtins blueprint flow
// (list from DB; if empty, seed from files and retry; then fill default LLM).
func (s *CompilationTemplateService) ListBuiltins(ctx context.Context, tenantID string) ([]*BuiltinTemplate, error) {
templates, err := s.templateDAO.ListBuiltins(ctx)
if err != nil {
return nil, err
}
if len(templates) == 0 {
if err := s.seedBuiltins(ctx); err != nil {
return nil, err
}
templates, err = s.templateDAO.ListBuiltins(ctx)
if err != nil {
return nil, err
}
}
out := make([]*BuiltinTemplate, 0, len(templates))
for _, t := range templates {
config := fillConfigDefaultLLM(ctx, s.tenantDAO, t.Config, t.TenantID)
out = append(out, &BuiltinTemplate{
ID: t.ID,
Kind: t.Kind,
DisplayName: t.Name,
Description: derefString(t.Description),
Config: config,
})
}
return s.sortBuiltins(out), nil
}
// LoadWikiPresets loads the wiki page-structure presets from the
// init_data/compilation_templates/wiki/*.yaml files, filesystem-fresh per call.
func (s *CompilationTemplateService) LoadWikiPresets() ([]*WikiPreset, error) {
wikiDir := filepath.Join(utility.GetProjectBaseDirectory(),
"api", "db", "init_data", "compilation_templates", "wiki")
entries, err := os.ReadDir(wikiDir)
if err != nil {
if os.IsNotExist(err) {
return []*WikiPreset{}, nil
}
return nil, err
}
var presets []*WikiPreset
for _, entry := range entries {
if entry.IsDir() || !strings.HasSuffix(entry.Name(), ".yaml") && !strings.HasSuffix(entry.Name(), ".yml") {
continue
}
path := filepath.Join(wikiDir, entry.Name())
data, rerr := os.ReadFile(path)
if rerr != nil {
continue
}
var doc map[string]interface{}
if yerr := yaml.Unmarshal(data, &doc); yerr != nil || doc == nil {
continue
}
presets = append(presets, &WikiPreset{
ID: strings.TrimSuffix(entry.Name(), filepath.Ext(entry.Name())),
Topic: strings.TrimSpace(yamlStr(doc["topic"])),
Instruction: yamlStr(doc["instruction"]),
PageExample: yamlStr(doc["page_example"]),
})
}
return presets, nil
}
// ValidateTemplatePayload validates a single template payload, mirroring the
// Python compilation_template_validation module. It returns an error describing
// the first problem found.
func ValidateTemplatePayload(req map[string]interface{}, requireAll bool) error {
if requireAll {
for _, key := range []string{"name", "kind", "config"} {
if _, ok := req[key]; !ok {
return fmt.Errorf("missing required field: %s.", key)
}
}
}
if name, ok := req["name"]; ok {
nameStr, ok2 := name.(string)
if !ok2 || strings.TrimSpace(nameStr) == "" || len([]byte(nameStr)) > 128 {
return errors.New("invalid template name.")
}
}
if desc, ok := req["description"]; ok {
if descStr, ok2 := desc.(string); !ok2 || len(descStr) > 1024 {
return errors.New("invalid template description.")
}
}
if kind, ok := req["kind"]; ok {
if kindStr, ok2 := kind.(string); !ok2 || kindStr == "" {
return errors.New("invalid template kind.")
}
}
config, hasConfig := req["config"]
if hasConfig {
configMap, ok := config.(map[string]interface{})
if !ok {
return errors.New("invalid template config.")
}
if len(fmt.Sprint(configMap["global_rules"])) > 4096 {
return errors.New("global compilation rules is too long.")
}
for _, section := range []string{"entity", "relation"} {
sec, _ := configMap[section].(map[string]interface{})
fields, _ := sec["fields"].([]interface{})
seen := map[string]struct{}{}
for _, f := range fields {
fm, _ := f.(map[string]interface{})
fieldType := strings.TrimSpace(yamlStr(fm["type"]))
if fieldType == "" {
return fmt.Errorf("%s type is required.", capitalizeTitle(section))
}
if _, dup := seen[fieldType]; dup {
return fmt.Errorf("%s type can not be duplicated.", capitalizeTitle(section))
}
seen[fieldType] = struct{}{}
if strings.TrimSpace(yamlStr(fm["description"])) == "" {
return fmt.Errorf("%s field description is required.", capitalizeTitle(section))
}
if len(yamlStr(fm["description"])) > 1024 {
return fmt.Errorf("%s field description is too long.", capitalizeTitle(section))
}
if len(yamlStr(fm["rule"])) > 1024 {
return fmt.Errorf("%s field rule is too long.", capitalizeTitle(section))
}
}
}
if configMap["kind"] == "wiki" || req["kind"] == "wiki" {
for _, group := range []string{"claim", "concept"} {
sec, _ := configMap[group].(map[string]interface{})
fields, _ := sec["fields"].([]interface{})
for _, f := range fields {
fm, _ := f.(map[string]interface{})
switch group {
case "claim":
if strings.TrimSpace(yamlStr(fm["statement"])) == "" {
return errors.New("claim statement is required.")
}
if strings.TrimSpace(yamlStr(fm["subject"])) == "" {
return errors.New("claim subject is required.")
}
if len(yamlStr(fm["statement"])) > 1024 {
return errors.New("claim statement is too long.")
}
if len(yamlStr(fm["subject"])) > 1024 {
return errors.New("claim subject is too long.")
}
case "concept":
if strings.TrimSpace(yamlStr(fm["term"])) == "" {
return errors.New("concept term is required.")
}
if strings.TrimSpace(yamlStr(fm["definition_excerpt"])) == "" {
return errors.New("concept definition excerpt is required.")
}
if len(yamlStr(fm["term"])) > 1024 {
return errors.New("concept term is too long.")
}
if len(yamlStr(fm["definition_excerpt"])) > 1024 {
return errors.New("concept definition excerpt is too long.")
}
}
}
}
}
}
return nil
}
// sortBuiltins mirrors Python _sort_builtins: empty-kind entries first, then by
// display name.
func (s *CompilationTemplateService) sortBuiltins(templates []*BuiltinTemplate) []*BuiltinTemplate {
sort.SliceStable(templates, func(i, j int) bool {
emptyI := templates[i].Kind == "empty" || templates[i].ID == "empty"
emptyJ := templates[j].Kind == "empty" || templates[j].ID == "empty"
if emptyI != emptyJ {
return emptyI
}
return strings.ToLower(templates[i].DisplayName) < strings.ToLower(templates[j].DisplayName)
})
return templates
}
// seedBuiltins loads the built-in templates from the init_data YAML files and
// upserts them (as a safety net when the DB has no built-ins yet), mirroring
// Python seed_builtins_from_files.
func (s *CompilationTemplateService) seedBuiltins(ctx context.Context) error {
dir := filepath.Join(utility.GetProjectBaseDirectory(),
"api", "db", "init_data", "compilation_templates")
entries, err := os.ReadDir(dir)
if err != nil {
return nil // directory absent is acceptable; nothing to seed
}
for _, entry := range entries {
if entry.IsDir() || !strings.HasSuffix(entry.Name(), ".yaml") && !strings.HasSuffix(entry.Name(), ".yml") {
continue
}
data, rerr := os.ReadFile(filepath.Join(dir, entry.Name()))
if rerr != nil {
continue
}
var doc struct {
Kind string `yaml:"kind"`
DisplayName string `yaml:"display_name"`
Description string `yaml:"description"`
Config map[string]interface{} `yaml:"config"`
}
if yerr := yaml.Unmarshal(data, &doc); yerr != nil {
continue
}
if doc.Kind == "" || doc.DisplayName == "" || doc.Config == nil {
continue
}
id := strings.TrimSuffix(entry.Name(), filepath.Ext(entry.Name()))
desc := doc.Description
valid := string(entity.StatusValid)
tmpl := &entity.CompilationTemplate{
ID: id,
Name: doc.DisplayName,
Kind: doc.Kind,
Config: entity.JSONMap(doc.Config),
IsBuiltin: true,
Status: &valid,
}
if doc.Description != "" {
tmpl.Description = &desc
}
_ = s.upsertBuiltin(ctx, tmpl)
}
return nil
}
func (s *CompilationTemplateService) upsertBuiltin(ctx context.Context, t *entity.CompilationTemplate) error {
existing, err := s.templateDAO.GetByID(ctx, t.ID)
if errors.Is(err, gorm.ErrRecordNotFound) {
return s.templateDAO.Save(ctx, dao.DB, t)
}
if err != nil {
return err
}
m := map[string]interface{}{
"name": t.Name, "kind": t.Kind, "config": t.Config,
"description": t.Description, "is_builtin": true,
"status": t.Status,
}
return s.templateDAO.UpdateFields(ctx, dao.DB, existing.ID, m)
}
func derefString(p *string) string {
if p == nil {
return ""
}
return *p
}
// yamlStr safely stringifies a YAML value, returning "" when the key is absent
// (instead of fmt.Sprint(nil) which yields "<nil>").
func yamlStr(v interface{}) string {
if v == nil {
return ""
}
return fmt.Sprint(v)
}

View File

@@ -0,0 +1,846 @@
//
// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
package service
import (
"context"
"encoding/json"
"fmt"
"sort"
"ragflow/internal/engine"
"ragflow/internal/engine/types"
)
// Compile keyword constants used by the knowledge-compilation artifacts stored
// in the document engine.
const (
CompileKwdWikiPage = "wiki_page"
CompileKwdWikiEntity = "wiki_entity"
CompileKwdWikiRelation = "wiki_relation"
CompileKwdWikiAlter = "wiki_alter"
CompileKwdSkill = "skill"
CompileKwdSkillAll = "skill_all"
CompileKwdDatasetNav = "dataset_nav"
CompileKwdRaptorGraph = "raptor_graph"
// Structure / graph compilation keywords.
CompileKwdStructure = "structure"
CompileKwdStructureIndex = "structureIndex"
CompileKwdStructureEntity = "structureEntity"
CompileKwdStructureRelation = "structureRelation"
CompileKwdStructureCommunity = "structureCommunity"
// Field name for the structure index type discriminator.
FieldStructureIndexType = "structure_index_type"
FieldStructureKind = "structure_kind"
FieldPageID = "page_id"
FieldGraphType = "graph_type"
)
// DatasetArtifactService reads knowledge-compilation artifacts (wiki pages,
// graphs, structures, navigation, skills) from the document engine.
type DatasetArtifactService struct{}
// NewDatasetArtifactService creates a DatasetArtifactService.
func NewDatasetArtifactService() *DatasetArtifactService {
return &DatasetArtifactService{}
}
// wikiIndexName returns the tenant document index name.
func wikiIndexName(tenantID string) string {
return fmt.Sprintf("ragflow_%s", tenantID)
}
// searchCompiled runs a filtered search over the tenant's document index for
// the given dataset, returning the matching chunks.
func (s *DatasetArtifactService) searchCompiled(ctx context.Context, tenantID, datasetID string, filter map[string]interface{}, selectFields []string, offset, limit int, orderBy *types.OrderByExpr) ([]map[string]interface{}, int64, error) {
docEngine := engine.Get()
if docEngine == nil {
return nil, 0, fmt.Errorf("document engine is not initialized")
}
// Copy caller filters first, then pin the dataset scope so kb_id always wins
// even if a caller passes its own kb_id.
merged := make(map[string]interface{}, len(filter)+1)
for k, v := range filter {
merged[k] = v
}
merged["kb_id"] = []string{datasetID}
res, err := docEngine.Search(ctx, &types.SearchRequest{
IndexNames: []string{wikiIndexName(tenantID)},
KbIDs: []string{datasetID},
Offset: offset,
Limit: limit,
SelectFields: selectFields,
Filter: merged,
OrderBy: orderBy,
})
if err != nil {
return nil, 0, err
}
if res == nil {
return nil, 0, nil
}
return res.Chunks, res.Total, nil
}
// intValue coerces an engine field value into an int, tolerating the numeric
// types a document engine may decode a number as (float64, int, int64,
// json.Number) as well as a string form and a list-valued form (first element
// wins). It returns 0 for anything else.
func intValue(v interface{}) int {
if v == nil {
return 0
}
switch n := v.(type) {
case int:
return n
case int64:
return int(n)
case int32:
return int(n)
case float64:
return int(n)
case float32:
return int(n)
case json.Number:
if iv, err := n.Int64(); err == nil {
return int(iv)
}
if fv, err := n.Float64(); err == nil {
return int(fv)
}
case string:
var iv int
if _, err := fmt.Sscanf(n, "%d", &iv); err == nil {
return iv
}
case []interface{}:
if len(n) > 0 {
return intValue(n[0])
}
}
return 0
}
// HasWiki reports whether the dataset has any compiled wiki artifact.
func (s *DatasetArtifactService) HasWiki(ctx context.Context, tenantID, datasetID string) (bool, error) {
_, total, err := s.searchCompiled(ctx, tenantID, datasetID,
map[string]interface{}{"compile_kwd": []string{CompileKwdWikiPage}}, nil, 0, 1, nil)
if err != nil {
return false, err
}
return total > 0, nil
}
// HasSkill reports whether the dataset has any compiled skill artifact.
func (s *DatasetArtifactService) HasSkill(ctx context.Context, tenantID, datasetID string) (bool, error) {
_, total, err := s.searchCompiled(ctx, tenantID, datasetID,
map[string]interface{}{"compile_kwd": []string{CompileKwdSkillAll}}, nil, 0, 1, nil)
if err != nil {
return false, err
}
return total > 0, nil
}
// WikiPageItem is a single wiki page summary returned by ListWikiPages.
type WikiPageItem struct {
Slug string `json:"slug"`
Title string `json:"title"`
PageType string `json:"page_type"`
Topic string `json:"topic"`
Summary string `json:"summary"`
}
// ListWikiPages lists wiki pages for a dataset with optional page_type/topic
// filters and pagination.
func (s *DatasetArtifactService) ListWikiPages(ctx context.Context, tenantID, datasetID, pageType, topic string, page, pageSize int) ([]WikiPageItem, int64, error) {
filter := map[string]interface{}{"compile_kwd": []string{CompileKwdWikiPage}}
if pageType != "" {
filter["page_type_kwd"] = []string{pageType}
}
if topic != "" {
filter["topic_kwd"] = []string{topic}
}
offset := (page - 1) * pageSize
chunks, total, err := s.searchCompiled(ctx, tenantID, datasetID, filter,
[]string{"slug_kwd", "title_kwd", "page_type_kwd", "topic_kwd", "outlinks_int", "summary_with_weight"},
offset, pageSize, (&types.OrderByExpr{}).Desc("outlinks_int").Asc("title_kwd"))
if err != nil {
return nil, 0, err
}
items := make([]WikiPageItem, 0, len(chunks))
for _, c := range chunks {
items = append(items, WikiPageItem{
Slug: firstStringValue(c["slug_kwd"]),
Title: firstStringValue(c["title_kwd"]),
PageType: firstStringValue(c["page_type_kwd"]),
Topic: firstStringValue(c["topic_kwd"]),
Summary: firstStringValue(c["summary_with_weight"]),
})
}
return items, total, nil
}
// WikiPageDetail is the full wiki page payload.
type WikiPageDetail struct {
Slug string `json:"slug"`
Title string `json:"title"`
PageType string `json:"page_type"`
Topic string `json:"topic"`
ContentMd string `json:"content_md"`
Summary string `json:"summary"`
EntityNames []string `json:"entity_names"`
Outlinks []string `json:"outlinks"`
RelatedKbPages []string `json:"related_kb_pages"`
SourceChunkIDs []string `json:"source_chunk_ids"`
SourceDocIDs []string `json:"source_doc_ids"`
}
// GetWikiPage returns a single wiki page by page_type and slug.
func (s *DatasetArtifactService) GetWikiPage(ctx context.Context, tenantID, datasetID, pageType, slug string) (*WikiPageDetail, error) {
slugKwd := pageType + "/" + slug
filter := map[string]interface{}{
"compile_kwd": []string{CompileKwdWikiPage},
"page_type_kwd": []string{pageType},
"slug_kwd": []string{slugKwd},
}
chunks, _, err := s.searchCompiled(ctx, tenantID, datasetID, filter,
[]string{"slug_kwd", "title_kwd", "page_type_kwd", "topic_kwd", "content_with_weight",
"summary_with_weight", "entity_names_kwd", "outlinks_kwd", "related_kb_pages_kwd",
"source_chunk_ids", "source_doc_ids"},
0, 1, nil)
if err != nil {
return nil, err
}
if len(chunks) == 0 {
return nil, nil
}
c := chunks[0]
detail := &WikiPageDetail{
Slug: firstStringValue(c["slug_kwd"]),
Title: firstStringValue(c["title_kwd"]),
PageType: firstStringValue(c["page_type_kwd"]),
Topic: firstStringValue(c["topic_kwd"]),
ContentMd: firstStringValue(c["content_with_weight"]),
Summary: firstStringValue(c["summary_with_weight"]),
EntityNames: toStringSlice(c["entity_names_kwd"]),
Outlinks: toStringSlice(c["outlinks_kwd"]),
RelatedKbPages: toStringSlice(c["related_kb_pages_kwd"]),
SourceChunkIDs: toStringSlice(c["source_chunk_ids"]),
SourceDocIDs: toStringSlice(c["source_doc_ids"]),
}
return detail, nil
}
// UpdateWikiPage performs a partial field update of a wiki page's content,
// title and outlinks through the document engine, then returns the refreshed
// page.
func (s *DatasetArtifactService) UpdateWikiPage(ctx context.Context, tenantID, datasetID, pageType, slug, contentMd, title string, outlinks []string) (*WikiPageDetail, error) {
docEngine := engine.Get()
if docEngine == nil {
return nil, fmt.Errorf("document engine is not initialized")
}
slugKwd := pageType + "/" + slug
filter := map[string]interface{}{
"compile_kwd": []string{CompileKwdWikiPage},
"page_type_kwd": []string{pageType},
"slug_kwd": []string{slugKwd},
}
chunks, _, err := s.searchCompiled(ctx, tenantID, datasetID, filter, []string{"id"}, 0, 1, nil)
if err != nil {
return nil, err
}
if len(chunks) == 0 {
return nil, nil
}
id, ok := chunks[0]["id"].(string)
if !ok || id == "" {
return nil, fmt.Errorf("wiki page chunk has no id")
}
update := map[string]interface{}{}
if contentMd != "" {
update["content_with_weight"] = contentMd
}
if title != "" {
update["title_kwd"] = title
}
if len(outlinks) > 0 {
update["outlinks_kwd"] = outlinks
}
if len(update) > 0 {
cond := map[string]interface{}{"id": id, "kb_id": datasetID}
if err := docEngine.UpdateChunks(ctx, cond, update, wikiIndexName(tenantID), datasetID); err != nil {
return nil, err
}
}
return s.GetWikiPage(ctx, tenantID, datasetID, pageType, slug)
}
// WikiTopicItem is a single topic entry returned by ListWikiTopics.
type WikiTopicItem struct {
Topic string `json:"topic"`
Title string `json:"title"`
Slug string `json:"slug"`
PageCount int `json:"page_count"`
}
// ListWikiTopics aggregates wiki topics for a dataset.
func (s *DatasetArtifactService) ListWikiTopics(ctx context.Context, tenantID, datasetID string) ([]WikiTopicItem, int64, error) {
filter := map[string]interface{}{
"compile_kwd": []string{CompileKwdWikiPage},
"page_type_kwd": []string{"concept", "entity"},
}
chunks, _, err := s.searchCompiled(ctx, tenantID, datasetID, filter,
[]string{"topic_kwd", "title_kwd", "slug_kwd"}, 0, 1000, nil)
if err != nil {
return nil, 0, err
}
counts := map[string]int{}
metas := map[string]WikiTopicItem{}
for _, c := range chunks {
t := firstStringValue(c["topic_kwd"])
if t == "" {
continue
}
counts[t]++
if _, ok := metas[t]; !ok {
metas[t] = WikiTopicItem{
Topic: t,
Title: firstStringValue(c["title_kwd"]),
Slug: firstStringValue(c["slug_kwd"]),
}
}
}
items := make([]WikiTopicItem, 0, len(metas))
for t, it := range metas {
it.PageCount = counts[t]
items = append(items, it)
}
sort.Slice(items, func(i, j int) bool { return items[i].Topic < items[j].Topic })
return items, int64(len(items)), nil
}
// WikiGraph is the entity/relation graph for a dataset's wiki artifacts.
type WikiGraph struct {
Entities []WikiGraphEntity `json:"entities"`
Relations []WikiGraphRelation `json:"relations"`
}
// WikiGraphEntity is a single wiki graph entity.
type WikiGraphEntity struct {
Slug string `json:"slug"`
Name string `json:"name"`
Aliases []string `json:"aliases"`
Description string `json:"description"`
Type string `json:"type"`
Weight int `json:"weight"`
SourceChunkIDs []string `json:"source_chunk_ids"`
}
// WikiGraphRelation is a single wiki graph relation.
type WikiGraphRelation struct {
From string `json:"from"`
To string `json:"to"`
}
// GetWikiGraph returns the wiki entity/relation graph for a dataset.
func (s *DatasetArtifactService) GetWikiGraph(ctx context.Context, tenantID, datasetID string) (*WikiGraph, error) {
entityChunks, _, err := s.searchCompiled(ctx, tenantID, datasetID,
map[string]interface{}{"compile_kwd": []string{CompileKwdWikiEntity}},
[]string{"slug_kwd", "title_kwd", "aliases_kwd", "description_with_weight", "entity_type_kwd", "weight_int", "source_chunk_ids"},
0, 1000, (&types.OrderByExpr{}).Desc("weight_int"))
if err != nil {
return nil, err
}
fromKwds := make([]string, 0, len(entityChunks))
for _, c := range entityChunks {
fromKwds = append(fromKwds, firstStringValue(c["slug_kwd"]))
}
graph := &WikiGraph{Entities: []WikiGraphEntity{}, Relations: []WikiGraphRelation{}}
for _, c := range entityChunks {
w := intValue(c["weight_int"])
graph.Entities = append(graph.Entities, WikiGraphEntity{
Slug: firstStringValue(c["slug_kwd"]),
Name: firstStringValue(c["title_kwd"]),
Aliases: toStringSlice(c["aliases_kwd"]),
Description: firstStringValue(c["description_with_weight"]),
Type: firstStringValue(c["entity_type_kwd"]),
Weight: w,
SourceChunkIDs: toStringSlice(c["source_chunk_ids"]),
})
}
if len(fromKwds) > 0 {
relChunks, _, err := s.searchCompiled(ctx, tenantID, datasetID,
map[string]interface{}{"compile_kwd": []string{CompileKwdWikiRelation}, "from_kwd": fromKwds},
[]string{"from_kwd", "to_kwd"}, 0, 10000, nil)
if err != nil {
return nil, err
}
for _, c := range relChunks {
graph.Relations = append(graph.Relations, WikiGraphRelation{
From: firstStringValue(c["from_kwd"]),
To: firstStringValue(c["to_kwd"]),
})
}
}
return graph, nil
}
// WikiAlteration is the alteration summary for a dataset's wiki artifacts.
type WikiAlteration struct {
Removed int `json:"removed"`
NewlyUploaded int `json:"newly_uploaded"`
RemovedDocIDs []string `json:"removed_doc_ids"`
NewlyUploadedDocIDs []string `json:"newly_uploaded_doc_ids"`
InvolvedDocIDs []string `json:"involved_doc_ids"`
EligibleDocIDs []string `json:"eligible_doc_ids"`
}
// GetWikiAlteration returns the wiki alteration summary for a dataset.
func (s *DatasetArtifactService) GetWikiAlteration(ctx context.Context, tenantID, datasetID string) (*WikiAlteration, error) {
chunks, _, err := s.searchCompiled(ctx, tenantID, datasetID,
map[string]interface{}{"compile_kwd": []string{CompileKwdWikiPage}},
[]string{"source_doc_ids"}, 0, 10000, nil)
if err != nil {
return nil, err
}
involved := map[string]struct{}{}
for _, c := range chunks {
for _, d := range toStringSlice(c["source_doc_ids"]) {
involved[d] = struct{}{}
}
}
ids := make([]string, 0, len(involved))
for d := range involved {
ids = append(ids, d)
}
sort.Strings(ids)
return &WikiAlteration{
Removed: 0,
NewlyUploaded: 0,
RemovedDocIDs: []string{},
NewlyUploadedDocIDs: []string{},
InvolvedDocIDs: ids,
EligibleDocIDs: ids,
}, nil
}
// ClearWiki deletes all wiki artifacts for a dataset.
func (s *DatasetArtifactService) ClearWiki(ctx context.Context, tenantID, datasetID string) (map[string]int, error) {
docEngine := engine.Get()
if docEngine == nil {
return nil, fmt.Errorf("document engine is not initialized")
}
kwds := []string{CompileKwdWikiPage, CompileKwdWikiEntity, CompileKwdWikiRelation, CompileKwdWikiAlter}
deleted := map[string]int{}
for _, kwd := range kwds {
chunks, _, err := s.searchCompiled(ctx, tenantID, datasetID,
map[string]interface{}{"compile_kwd": []string{kwd}}, []string{"id"}, 0, 10000, nil)
if err != nil {
return nil, err
}
if len(chunks) == 0 {
deleted[kwd] = 0
continue
}
ids := make([]string, 0, len(chunks))
for _, c := range chunks {
if id, ok := c["id"].(string); ok {
ids = append(ids, id)
}
}
cond := map[string]interface{}{"id": ids, "kb_id": datasetID}
if _, err := docEngine.DeleteChunks(ctx, cond, wikiIndexName(tenantID), datasetID); err != nil {
return nil, err
}
deleted[kwd] = len(ids)
}
return deleted, nil
}
// StructureItem is a single compiled structure entry for a dataset.
type StructureItem struct {
PageID string `json:"page_id"`
StructureKind string `json:"structure_kind"`
StructureIndexType string `json:"structure_index_type"`
Data string `json:"data"`
}
// ListStructures returns the compiled structures of a dataset, filtered by
// optional structure_kind and structure_index_type.
func (s *DatasetArtifactService) ListStructures(ctx context.Context, tenantID, datasetID, structureKind, structureIndexType string) ([]StructureItem, int64, error) {
filter := map[string]interface{}{"compile_kwd": []string{CompileKwdStructure}}
if structureKind != "" {
filter[FieldStructureKind] = []string{structureKind}
}
if structureIndexType != "" {
filter[FieldStructureIndexType] = []string{structureIndexType}
}
chunks, total, err := s.searchCompiled(ctx, tenantID, datasetID, filter,
[]string{FieldPageID, FieldStructureKind, FieldStructureIndexType, "content_with_weight"}, 0, 10000, nil)
if err != nil {
return nil, 0, err
}
items := make([]StructureItem, 0, len(chunks))
for _, c := range chunks {
items = append(items, StructureItem{
PageID: firstStringValue(c[FieldPageID]),
StructureKind: firstStringValue(c[FieldStructureKind]),
StructureIndexType: firstStringValue(c[FieldStructureIndexType]),
Data: firstStringValue(c["content_with_weight"]),
})
}
return items, total, nil
}
// DeleteStructures deletes the compiled structures of a dataset, optionally
// scoped by structure_kind and structure_index_type.
func (s *DatasetArtifactService) DeleteStructures(ctx context.Context, tenantID, datasetID, structureKind, structureIndexType string) (int, error) {
docEngine := engine.Get()
if docEngine == nil {
return 0, fmt.Errorf("document engine is not initialized")
}
filter := map[string]interface{}{"compile_kwd": []string{CompileKwdStructure}}
if structureKind != "" {
filter[FieldStructureKind] = []string{structureKind}
}
if structureIndexType != "" {
filter[FieldStructureIndexType] = []string{structureIndexType}
}
chunks, _, err := s.searchCompiled(ctx, tenantID, datasetID, filter, []string{"id"}, 0, 10000, nil)
if err != nil {
return 0, err
}
if len(chunks) == 0 {
return 0, nil
}
ids := make([]string, 0, len(chunks))
for _, c := range chunks {
if id, ok := c["id"].(string); ok {
ids = append(ids, id)
}
}
cond := map[string]interface{}{"id": ids, "kb_id": datasetID}
if _, err := docEngine.DeleteChunks(ctx, cond, wikiIndexName(tenantID), datasetID); err != nil {
return 0, err
}
return len(ids), nil
}
// DocGraphItem is a single node/edge entry in a document's structure graph.
type DocGraphItem struct {
ID string `json:"id"`
Content string `json:"content"`
SourceID string `json:"source_id"`
}
// GetDocumentGraph returns the structure graph of a single document.
func (s *DatasetArtifactService) GetDocumentGraph(ctx context.Context, tenantID, datasetID, documentID, graphType string) ([]DocGraphItem, int64, error) {
filter := map[string]interface{}{
"doc_id": []string{documentID},
"compiled_graph_kwd": []string{"graph"},
}
if graphType != "" {
filter[FieldGraphType] = []string{graphType}
}
chunks, total, err := s.searchCompiled(ctx, tenantID, datasetID, filter,
[]string{"id", "content_with_weight", "source_id"}, 0, 10000, nil)
if err != nil {
return nil, 0, err
}
items := make([]DocGraphItem, 0, len(chunks))
for _, c := range chunks {
items = append(items, DocGraphItem{
ID: firstStringValue(c["id"]),
Content: firstStringValue(c["content_with_weight"]),
SourceID: firstStringValue(c["source_id"]),
})
}
return items, total, nil
}
// DeleteDocumentGraph deletes the structure graph of a single document.
func (s *DatasetArtifactService) DeleteDocumentGraph(ctx context.Context, tenantID, datasetID, documentID string) (int, error) {
docEngine := engine.Get()
if docEngine == nil {
return 0, fmt.Errorf("document engine is not initialized")
}
filter := map[string]interface{}{
"doc_id": []string{documentID},
"compiled_graph_kwd": []string{"graph"},
}
chunks, _, err := s.searchCompiled(ctx, tenantID, datasetID, filter, []string{"id"}, 0, 10000, nil)
if err != nil {
return 0, err
}
if len(chunks) == 0 {
return 0, nil
}
ids := make([]string, 0, len(chunks))
for _, c := range chunks {
if id, ok := c["id"].(string); ok {
ids = append(ids, id)
}
}
cond := map[string]interface{}{"id": ids, "kb_id": datasetID}
if _, err := docEngine.DeleteChunks(ctx, cond, wikiIndexName(tenantID), datasetID); err != nil {
return 0, err
}
return len(ids), nil
}
// NavigationItem is a single navigation cluster.
type NavigationItem struct {
Name string `json:"name"`
Title string `json:"title"`
Count int `json:"count"`
}
// ListNavClusters returns the navigation clusters of a dataset.
func (s *DatasetArtifactService) ListNavClusters(ctx context.Context, tenantID, datasetID string) ([]NavigationItem, int64, error) {
chunks, total, err := s.searchCompiled(ctx, tenantID, datasetID,
map[string]interface{}{"compile_kwd": []string{CompileKwdDatasetNav}},
[]string{"nav_cluster_kwd", "title_kwd", "count_int"}, 0, 10000, nil)
if err != nil {
return nil, 0, err
}
items := make([]NavigationItem, 0, len(chunks))
for _, c := range chunks {
count := intValue(c["count_int"])
items = append(items, NavigationItem{
Name: firstStringValue(c["nav_cluster_kwd"]),
Title: firstStringValue(c["title_kwd"]),
Count: count,
})
}
return items, total, nil
}
// DeleteNav deletes all navigation clusters of a dataset.
func (s *DatasetArtifactService) DeleteNav(ctx context.Context, tenantID, datasetID string) (int, error) {
docEngine := engine.Get()
if docEngine == nil {
return 0, fmt.Errorf("document engine is not initialized")
}
chunks, _, err := s.searchCompiled(ctx, tenantID, datasetID,
map[string]interface{}{"compile_kwd": []string{CompileKwdDatasetNav}}, []string{"id"}, 0, 10000, nil)
if err != nil {
return 0, err
}
if len(chunks) == 0 {
return 0, nil
}
ids := make([]string, 0, len(chunks))
for _, c := range chunks {
if id, ok := c["id"].(string); ok {
ids = append(ids, id)
}
}
cond := map[string]interface{}{"id": ids, "kb_id": datasetID}
if _, err := docEngine.DeleteChunks(ctx, cond, wikiIndexName(tenantID), datasetID); err != nil {
return 0, err
}
return len(ids), nil
}
// DeleteNavNode deletes a single navigation cluster by name.
func (s *DatasetArtifactService) DeleteNavNode(ctx context.Context, tenantID, datasetID, name string) (int, error) {
docEngine := engine.Get()
if docEngine == nil {
return 0, fmt.Errorf("document engine is not initialized")
}
chunks, _, err := s.searchCompiled(ctx, tenantID, datasetID,
map[string]interface{}{"compile_kwd": []string{CompileKwdDatasetNav}, "nav_cluster_kwd": []string{name}},
[]string{"id"}, 0, 10000, nil)
if err != nil {
return 0, err
}
if len(chunks) == 0 {
return 0, nil
}
ids := make([]string, 0, len(chunks))
for _, c := range chunks {
if id, ok := c["id"].(string); ok {
ids = append(ids, id)
}
}
cond := map[string]interface{}{"id": ids, "kb_id": datasetID}
if _, err := docEngine.DeleteChunks(ctx, cond, wikiIndexName(tenantID), datasetID); err != nil {
return 0, err
}
return len(ids), nil
}
// NavChildItem is a single child entry under a navigation cluster.
type NavChildItem struct {
Name string `json:"name"`
Title string `json:"title"`
Count int `json:"count"`
}
// ListNavChildren returns the children of a navigation cluster.
func (s *DatasetArtifactService) ListNavChildren(ctx context.Context, tenantID, datasetID, name string) ([]NavChildItem, int64, error) {
chunks, total, err := s.searchCompiled(ctx, tenantID, datasetID,
map[string]interface{}{"compile_kwd": []string{CompileKwdDatasetNav}, "nav_cluster_kwd": []string{name}},
[]string{"nav_child_kwd", "title_kwd", "count_int"}, 0, 10000, nil)
if err != nil {
return nil, 0, err
}
items := make([]NavChildItem, 0, len(chunks))
for _, c := range chunks {
count := intValue(c["count_int"])
items = append(items, NavChildItem{
Name: firstStringValue(c["nav_child_kwd"]),
Title: firstStringValue(c["title_kwd"]),
Count: count,
})
}
return items, total, nil
}
// SkillTreeItem is a single skill-tree page summary.
type SkillTreeItem struct {
Kwd string `json:"kwd"`
Title string `json:"title"`
PageType string `json:"page_type"`
Outlinks int `json:"outlinks"`
Inlinks int `json:"inlinks"`
}
// GetSkillTree returns the skill tree of a dataset.
func (s *DatasetArtifactService) GetSkillTree(ctx context.Context, tenantID, datasetID, kwd string) ([]SkillTreeItem, int64, error) {
filter := map[string]interface{}{"compile_kwd": []string{CompileKwdSkillAll}}
if kwd != "" {
filter["kwd"] = []string{kwd}
}
chunks, total, err := s.searchCompiled(ctx, tenantID, datasetID, filter,
[]string{"kwd", "title_kwd", "page_type_kwd", "outlinks_int", "inlinks_int"}, 0, 10000, nil)
if err != nil {
return nil, 0, err
}
items := make([]SkillTreeItem, 0, len(chunks))
for _, c := range chunks {
outlinks, inlinks := intValue(c["outlinks_int"]), intValue(c["inlinks_int"])
items = append(items, SkillTreeItem{
Kwd: firstStringValue(c["kwd"]),
Title: firstStringValue(c["title_kwd"]),
PageType: firstStringValue(c["page_type_kwd"]),
Outlinks: outlinks,
Inlinks: inlinks,
})
}
return items, total, nil
}
// SkillPageDetail is the full skill page payload.
type SkillPageDetail struct {
Kwd string `json:"kwd"`
Title string `json:"title"`
ContentMd string `json:"content_md"`
Outlinks []string `json:"outlinks"`
Inlinks []string `json:"inlinks"`
}
// GetSkillPage returns a single skill page by keyword.
func (s *DatasetArtifactService) GetSkillPage(ctx context.Context, tenantID, datasetID, kwd string) (*SkillPageDetail, error) {
chunks, _, err := s.searchCompiled(ctx, tenantID, datasetID,
map[string]interface{}{"compile_kwd": []string{CompileKwdSkillAll}, "kwd": []string{kwd}},
[]string{"kwd", "title_kwd", "content_with_weight", "outlinks_kwd", "inlinks_kwd"}, 0, 1, nil)
if err != nil {
return nil, err
}
if len(chunks) == 0 {
return nil, nil
}
c := chunks[0]
return &SkillPageDetail{
Kwd: firstStringValue(c["kwd"]),
Title: firstStringValue(c["title_kwd"]),
ContentMd: firstStringValue(c["content_with_weight"]),
Outlinks: toStringSlice(c["outlinks_kwd"]),
Inlinks: toStringSlice(c["inlinks_kwd"]),
}, nil
}
// DeleteSkills deletes skills of a dataset, optionally scoped by keyword.
func (s *DatasetArtifactService) DeleteSkills(ctx context.Context, tenantID, datasetID, kwd string) (int, error) {
docEngine := engine.Get()
if docEngine == nil {
return 0, fmt.Errorf("document engine is not initialized")
}
filter := map[string]interface{}{"compile_kwd": []string{CompileKwdSkillAll}}
if kwd != "" {
filter["kwd"] = []string{kwd}
}
chunks, _, err := s.searchCompiled(ctx, tenantID, datasetID, filter, []string{"id"}, 0, 10000, nil)
if err != nil {
return 0, err
}
if len(chunks) == 0 {
return 0, nil
}
ids := make([]string, 0, len(chunks))
for _, c := range chunks {
if id, ok := c["id"].(string); ok {
ids = append(ids, id)
}
}
cond := map[string]interface{}{"id": ids, "kb_id": datasetID}
if _, err := docEngine.DeleteChunks(ctx, cond, wikiIndexName(tenantID), datasetID); err != nil {
return 0, err
}
return len(ids), nil
}
// firstStringValue returns the first string in a possibly-list ES field value.
func firstStringValue(v interface{}) string {
switch t := v.(type) {
case string:
return t
case []interface{}:
if len(t) > 0 {
if s, ok := t[0].(string); ok {
return s
}
}
case []string:
if len(t) > 0 {
return t[0]
}
}
return ""
}
// toStringSlice normalizes an ES field value (string or list) to []string.
func toStringSlice(v interface{}) []string {
switch t := v.(type) {
case string:
if t == "" {
return []string{}
}
return []string{t}
case []interface{}:
out := make([]string, 0, len(t))
for _, e := range t {
if s, ok := e.(string); ok {
out = append(out, s)
}
}
return out
case []string:
return t
}
return []string{}
}

View File

@@ -28,6 +28,8 @@ import (
"ragflow/internal/storage"
"ragflow/internal/utility"
"sort"
"strings"
"sync"
"time"
"go.uber.org/zap"
@@ -233,6 +235,209 @@ func (s *FileCommitService) CreateCommit(ctx context.Context, folderID, authorID
return commit, nil
}
// PageEditCommitInput carries the data needed to record a single wiki/skill
// page edit as an audit commit.
type PageEditCommitInput struct {
DatasetID string // knowledgebase scope, stored on the commit for isolation
DocID string // ES doc id of the page content
Slug string
PageType string
Title string
AuthorID string
OldContent string
NewContent string
}
// wikiFileID derives the stable file key used to scope page-edit commits to a
// specific knowledgebase and page, so identical slugs in different
// knowledgebases never share a commit parent or history.
func wikiFileID(datasetID, pageType, slug string) string {
return datasetID + "/" + pageType + "/" + slug
}
// RecordPageEdit records a wiki/skill page edit as an audit commit with a
// git-style parent chain (each edit points at the previous commit for the same
// page). The new content_after is referenced in ES by doc_id; a unified diff of
// old vs new content is stored on the commit item. The commit is scoped to the
// dataset via FolderID and a derived page file key so page histories never cross
// knowledgebase boundaries.
//
// This path is independent of the workspace File tree (it does not require a
// File record or a tree_state snapshot).
func (s *FileCommitService) RecordPageEdit(ctx context.Context, in PageEditCommitInput) (*entity.FileCommit, error) {
// Parent chain: previous commit that touched the same page file key.
fileID := wikiFileID(in.DatasetID, in.PageType, in.Slug)
commitID := utility.GenerateUUID()
diffText := unifiedDiff(in.OldContent, in.NewContent)
contentAfterStorage := "es"
contentAfterLocation := in.DocID
slugKwd := in.Slug
pageTypeKwd := in.PageType
item := &entity.FileCommitItem{
ID: utility.GenerateUUID(),
CommitID: commitID,
FileID: fileID,
Operation: "modify",
Diff: &diffText,
ContentAfterStorage: &contentAfterStorage,
ContentAfterLocation: &contentAfterLocation,
SlugKwd: &slugKwd,
PageTypeKwd: &pageTypeKwd,
}
var commit *entity.FileCommit
// Serialize parent selection with insertion in process so two concurrent
// edits on the same page cannot both read the same parent and fork the
// chain. This is backend-agnostic (works identically on MySQL and SQLite)
// and cheaper than row-level DB locks.
mu := pageCommitLock(fileID)
mu.Lock()
defer mu.Unlock()
// Read the parent inside the lock, on the shared connection, so it always
// reflects the previously committed edit for this page.
parentID, perr := s.commitItemDAO.GetLatestCommitIDByFileID(ctx, dao.DB, fileID)
if perr != nil {
return nil, fmt.Errorf("failed to resolve page commit parent: %w", perr)
}
commit = &entity.FileCommit{
ID: commitID,
FolderID: in.DatasetID,
Message: in.Title,
AuthorID: in.AuthorID,
Title: &in.Title,
FileCount: 1,
}
if parentID != "" {
commit.ParentID = &parentID
}
if err := dao.DB.WithContext(ctx).Transaction(func(tx *gorm.DB) error {
if cerr := tx.Create(commit).Error; cerr != nil {
return fmt.Errorf("failed to create page commit: %w", cerr)
}
if ierr := tx.Create(item).Error; ierr != nil {
return fmt.Errorf("failed to create page commit item: %w", ierr)
}
return nil
}); err != nil {
return nil, err
}
return commit, nil
}
// pageCommitLocks is a per-page-file-key mutex registry that serializes
// RecordPageEdit calls for the same page. Entries are retained for the process
// lifetime (bounded by the number of distinct pages edited).
var pageCommitLocks sync.Map // fileID -> *sync.Mutex
func pageCommitLock(fileID string) *sync.Mutex {
v, _ := pageCommitLocks.LoadOrStore(fileID, &sync.Mutex{})
return v.(*sync.Mutex)
}
// ListPageCommits lists audit commits for a specific wiki/skill page.
func (s *FileCommitService) ListPageCommits(ctx context.Context, datasetID, pageType, slug string, page, pageSize int) ([]*entity.FileCommit, int64, error) {
items, err := s.commitItemDAO.ListByFileID(ctx, dao.DB, wikiFileID(datasetID, pageType, slug))
if err != nil {
return nil, 0, err
}
commitIDs := make([]string, 0, len(items))
for _, it := range items {
commitIDs = append(commitIDs, it.CommitID)
}
if len(commitIDs) == 0 {
return []*entity.FileCommit{}, 0, nil
}
commits, total, err := s.commitDAO.ListByIDs(ctx, dao.DB, commitIDs, page, pageSize)
if err != nil {
return nil, 0, err
}
return commits, total, nil
}
// unifiedDiff produces a simple line-based unified diff between two texts.
func unifiedDiff(oldText, newText string) string {
oldLines := strings.Split(oldText, "\n")
newLines := strings.Split(newText, "\n")
const maxCtx = 3
type hunkLine struct {
prefix string
text string
}
var hunks []hunkLine
// Longest common subsequence over lines, then render the diff.
cur := make([][]int, len(oldLines)+1)
for i := range cur {
cur[i] = make([]int, len(newLines)+1)
}
for i := len(oldLines) - 1; i >= 0; i-- {
for j := len(newLines) - 1; j >= 0; j-- {
if oldLines[i] == newLines[j] {
cur[i][j] = cur[i+1][j+1] + 1
} else if cur[i+1][j] >= cur[i][j+1] {
cur[i][j] = cur[i+1][j]
} else {
cur[i][j] = cur[i][j+1]
}
}
}
i, j := 0, 0
for i < len(oldLines) && j < len(newLines) {
if oldLines[i] == newLines[j] {
i++
j++
} else if cur[i+1][j] >= cur[i][j+1] {
hunks = append(hunks, hunkLine{"-", oldLines[i]})
i++
} else {
hunks = append(hunks, hunkLine{"+", newLines[j]})
j++
}
}
for ; i < len(oldLines); i++ {
hunks = append(hunks, hunkLine{"-", oldLines[i]})
}
for ; j < len(newLines); j++ {
hunks = append(hunks, hunkLine{"+", newLines[j]})
}
if len(hunks) == 0 {
return ""
}
if len(hunks) > maxCtx*2 {
var b strings.Builder
for k := 0; k < maxCtx; k++ {
b.WriteString(hunks[k].prefix)
b.WriteString(hunks[k].text)
b.WriteString("\n")
}
b.WriteString("... (" + fmt.Sprintf("%d", len(hunks)-maxCtx*2) + " lines omitted) ...\n")
for k := len(hunks) - maxCtx; k < len(hunks); k++ {
b.WriteString(hunks[k].prefix)
b.WriteString(hunks[k].text)
b.WriteString("\n")
}
return b.String()
}
var b strings.Builder
for _, h := range hunks {
b.WriteString(h.prefix)
b.WriteString(h.text)
b.WriteString("\n")
}
return b.String()
}
// ListCommits lists commits for a workspace folder with pagination
func (s *FileCommitService) ListCommits(ctx context.Context, folderID string, page, pageSize int, orderBy string, desc bool) ([]*entity.FileCommit, int64, error) {
return s.commitDAO.ListByFolderID(ctx, dao.DB, folderID, page, pageSize, orderBy, desc)

View File

@@ -0,0 +1,281 @@
package file
import (
"context"
"strconv"
"strings"
"sync"
"testing"
"ragflow/internal/dao"
"ragflow/internal/entity"
"github.com/glebarez/sqlite"
"gorm.io/gorm"
)
func newPageCommitTestDB(t *testing.T) *gorm.DB {
t.Helper()
db, err := gorm.Open(sqlite.Open(":memory:"), &gorm.Config{TranslateError: true})
if err != nil {
t.Fatalf("open sqlite: %v", err)
}
// Keep a single connection so the :memory: database is shared across all
// goroutines (sqlite :memory: is otherwise per-connection); this also
// serializes the concurrent page-commit test through one connection.
if sqlDB, serr := db.DB(); serr == nil {
sqlDB.SetMaxOpenConns(1)
sqlDB.SetMaxIdleConns(1)
}
if err := db.AutoMigrate(&entity.FileCommit{}, &entity.FileCommitItem{}); err != nil {
t.Fatalf("auto migrate: %v", err)
}
old := dao.DB
dao.DB = db
t.Cleanup(func() { dao.DB = old })
return db
}
func TestRecordPageEdit_CreatesCommitAndItem(t *testing.T) {
newPageCommitTestDB(t)
svc := NewFileCommitService()
ctx := context.Background()
in := PageEditCommitInput{
DatasetID: "kb1",
DocID: "wiki/page-a",
Slug: "page-a",
PageType: "wiki",
Title: "First edit",
AuthorID: "u1",
OldContent: "hello world",
NewContent: "hello world, edited",
}
commit, err := svc.RecordPageEdit(ctx, in)
if err != nil {
t.Fatalf("RecordPageEdit: %v", err)
}
if commit.ID == "" {
t.Fatal("expected a generated commit id")
}
if commit.AuthorID != "u1" || commit.Message != "First edit" || commit.FileCount != 1 {
t.Fatalf("unexpected commit: %+v", commit)
}
if commit.FolderID != "kb1" {
t.Fatalf("expected folder_id kb1 for dataset scope, got %s", commit.FolderID)
}
if commit.ParentID != nil {
t.Fatalf("first edit should have no parent, got %v", *commit.ParentID)
}
var items []entity.FileCommitItem
if err := dao.DB.Where("commit_id = ?", commit.ID).Find(&items).Error; err != nil {
t.Fatalf("load items: %v", err)
}
if len(items) != 1 {
t.Fatalf("expected 1 item, got %d", len(items))
}
it := items[0]
if it.Operation != "modify" {
t.Fatalf("expected operation modify, got %s", it.Operation)
}
if it.FileID != "kb1/wiki/page-a" {
t.Fatalf("expected file_id kb1/wiki/page-a (dataset-scoped), got %s", it.FileID)
}
if it.SlugKwd == nil || *it.SlugKwd != "page-a" {
t.Fatalf("expected slug_kwd page-a, got %v", it.SlugKwd)
}
if it.PageTypeKwd == nil || *it.PageTypeKwd != "wiki" {
t.Fatalf("expected page_type_kwd wiki, got %v", it.PageTypeKwd)
}
if it.Diff == nil || !strings.Contains(*it.Diff, "hello world, edited") {
t.Fatalf("expected diff containing new content, got %v", it.Diff)
}
if it.ContentAfterStorage == nil || *it.ContentAfterStorage != "es" {
t.Fatalf("expected content_after_storage es, got %v", it.ContentAfterStorage)
}
if it.ContentAfterLocation == nil || *it.ContentAfterLocation != "wiki/page-a" {
t.Fatalf("expected content_after_location wiki/page-a, got %v", it.ContentAfterLocation)
}
}
func TestRecordPageEdit_SecondEditLinksParent(t *testing.T) {
newPageCommitTestDB(t)
svc := NewFileCommitService()
ctx := context.Background()
base := PageEditCommitInput{
DatasetID: "kb1",
DocID: "wiki/page-a",
Slug: "page-a",
PageType: "wiki",
Title: "first",
AuthorID: "u1",
}
if _, err := svc.RecordPageEdit(ctx, base); err != nil {
t.Fatalf("first RecordPageEdit: %v", err)
}
second := base
second.Title = "second"
second.OldContent = "hello"
second.NewContent = "hello world"
commit2, err := svc.RecordPageEdit(ctx, second)
if err != nil {
t.Fatalf("second RecordPageEdit: %v", err)
}
if commit2.ParentID == nil {
t.Fatal("second edit should link to the first commit as parent")
}
var first entity.FileCommit
if err := dao.DB.Where("title = ?", "first").First(&first).Error; err != nil {
t.Fatalf("load first commit: %v", err)
}
if *commit2.ParentID != first.ID {
t.Fatalf("parent id mismatch: got %s want %s", *commit2.ParentID, first.ID)
}
}
func TestRecordPageEdit_IsolatesDatasets(t *testing.T) {
newPageCommitTestDB(t)
svc := NewFileCommitService()
ctx := context.Background()
mk := func(datasetID string) PageEditCommitInput {
return PageEditCommitInput{
DatasetID: datasetID,
DocID: datasetID + "/wiki/page-a",
Slug: "page-a",
PageType: "wiki",
Title: datasetID + "-edit",
AuthorID: "u1",
OldContent: "old",
NewContent: "new",
}
}
if _, err := svc.RecordPageEdit(ctx, mk("kb1")); err != nil {
t.Fatalf("kb1 RecordPageEdit: %v", err)
}
if _, err := svc.RecordPageEdit(ctx, mk("kb2")); err != nil {
t.Fatalf("kb2 RecordPageEdit: %v", err)
}
// A second edit in kb1 must parent to kb1's own first commit, not kb2's.
kb1Again := mk("kb1")
kb1Again.Title = "kb1-edit-2"
kb1Again.OldContent = "old"
kb1Again.NewContent = "newer"
kb1Commit2, err := svc.RecordPageEdit(ctx, kb1Again)
if err != nil {
t.Fatalf("kb1 second RecordPageEdit: %v", err)
}
if kb1Commit2.ParentID == nil {
t.Fatal("kb1 second edit should have a parent")
}
var kb1First, kb2First entity.FileCommit
if err := dao.DB.Where("title = ?", "kb1-edit").First(&kb1First).Error; err != nil {
t.Fatalf("load kb1 first commit: %v", err)
}
if err := dao.DB.Where("title = ?", "kb2-edit").First(&kb2First).Error; err != nil {
t.Fatalf("load kb2 first commit: %v", err)
}
if *kb1Commit2.ParentID != kb1First.ID {
t.Fatalf("kb1 parent mismatch: got %s want %s", *kb1Commit2.ParentID, kb1First.ID)
}
if *kb1Commit2.ParentID == kb2First.ID {
t.Fatal("kb1 parent must not cross into kb2 history")
}
}
func TestRecordPageEdit_ConcurrentEditsFormLinearChain(t *testing.T) {
newPageCommitTestDB(t)
svc := NewFileCommitService()
ctx := context.Background()
const edits = 8
var wg sync.WaitGroup
errs := make([]error, edits)
for i := 0; i < edits; i++ {
wg.Add(1)
go func(idx int) {
defer wg.Done()
in := PageEditCommitInput{
DatasetID: "kb1",
DocID: "wiki/page-a",
Slug: "page-a",
PageType: "wiki",
Title: "edit-" + strconv.Itoa(idx),
AuthorID: "u1",
OldContent: "old",
NewContent: "new-" + strconv.Itoa(idx),
}
_, errs[idx] = svc.RecordPageEdit(ctx, in)
}(i)
}
wg.Wait()
for i, e := range errs {
if e != nil {
t.Fatalf("edit %d failed: %v", i, e)
}
}
var commits []entity.FileCommit
if err := dao.DB.Where("folder_id = ?", "kb1").Order("create_time ASC").Find(&commits).Error; err != nil {
t.Fatalf("load commits: %v", err)
}
if len(commits) != edits {
t.Fatalf("expected %d commits, got %d", edits, len(commits))
}
// Every commit except the first must have a parent, and the parents must
// form a linear chain (no two commits share the same parent).
seenParents := map[string]bool{}
for i, c := range commits {
if i == 0 {
if c.ParentID != nil {
t.Fatalf("first commit should have no parent")
}
continue
}
if c.ParentID == nil {
t.Fatalf("commit %s should have a parent", c.ID)
}
if seenParents[*c.ParentID] {
t.Fatalf("two commits share parent %s -> forked chain", *c.ParentID)
}
seenParents[*c.ParentID] = true
}
}
func TestUnifiedDiff_EmptyWhenNoChange(t *testing.T) {
if d := unifiedDiff("same", "same"); d != "" {
t.Fatalf("expected empty diff for identical text, got %q", d)
}
}
func TestUnifiedDiff_DetectsAddition(t *testing.T) {
d := unifiedDiff("a\nb\nc\n", "a\nb\nc\nd\n")
if !strings.Contains(d, "+d") {
t.Fatalf("expected diff to contain added line, got %q", d)
}
}
func TestUnifiedDiff_DetectsRemoval(t *testing.T) {
d := unifiedDiff("a\nb\nc\n", "a\nc\n")
if !strings.Contains(d, "-b") {
t.Fatalf("expected diff to contain removed line, got %q", d)
}
}
func TestUnifiedDiff_TruncatesLongDiff(t *testing.T) {
oldLines := make([]string, 50)
newLines := make([]string, 50)
for i := range oldLines {
oldLines[i] = "old-line"
newLines[i] = "new-line"
}
d := unifiedDiff(strings.Join(oldLines, "\n"), strings.Join(newLines, "\n"))
if !strings.Contains(d, "... ") || !strings.Contains(d, "lines omitted") {
t.Fatalf("expected long diff to be truncated, got %q", d)
}
}