mirror of
https://github.com/infiniflow/ragflow.git
synced 2026-08-01 13:33:48 +08:00
feat(go): port knowledge compilation template/group + wiki artifact/nav/skill REST APIs from Python (#17656)
This commit is contained in:
@@ -833,6 +833,9 @@ func startServer(ctx context.Context) {
|
||||
componentsSvc := service.NewComponentsService()
|
||||
componentsHandler := handler.NewComponentsHandler(componentsSvc)
|
||||
pipelineHandler := handler.NewPipelineHandler()
|
||||
compilationTemplateHandler := handler.NewCompilationTemplateHandler(service.NewCompilationTemplateService())
|
||||
compilationTemplateGroupHandler := handler.NewCompilationTemplateGroupHandler(service.NewCompilationTemplateGroupService())
|
||||
datasetArtifactHandler := handler.NewDatasetArtifactHandler(service.NewDatasetArtifactService(), datasetsService, file.NewFileCommitService())
|
||||
|
||||
// Initialize router
|
||||
r := router.NewRouter(authHandler,
|
||||
@@ -865,7 +868,10 @@ func startServer(ctx context.Context) {
|
||||
openaiChatHandler,
|
||||
botHandler,
|
||||
componentsHandler,
|
||||
pipelineHandler)
|
||||
pipelineHandler,
|
||||
compilationTemplateHandler,
|
||||
compilationTemplateGroupHandler,
|
||||
datasetArtifactHandler)
|
||||
|
||||
// Create Gin engine
|
||||
ginEngine := gin.New()
|
||||
|
||||
@@ -20,6 +20,8 @@ import (
|
||||
"context"
|
||||
"fmt"
|
||||
"ragflow/internal/entity"
|
||||
|
||||
"gorm.io/gorm"
|
||||
)
|
||||
|
||||
// CompilationTemplateDAO is the data-access object for compilation templates and
|
||||
@@ -123,3 +125,97 @@ func (dao *CompilationTemplateDAO) GetTemplate(ctx context.Context, tenantID, te
|
||||
}
|
||||
return &t, nil
|
||||
}
|
||||
|
||||
// ListByGroup returns the valid compilation templates belonging to a group,
|
||||
// ordered by create_time asc, mirroring Python
|
||||
// CompilationTemplateGroupService child loading.
|
||||
func (dao *CompilationTemplateDAO) ListByGroup(ctx context.Context, db *gorm.DB, groupID string) ([]*entity.CompilationTemplate, error) {
|
||||
var templates []*entity.CompilationTemplate
|
||||
if err := db.WithContext(ctx).
|
||||
Where("group_id = ? AND status = ?", groupID, string(entity.StatusValid)).
|
||||
Order("create_time asc").
|
||||
Find(&templates).Error; err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return templates, nil
|
||||
}
|
||||
|
||||
// ListBuiltins returns the valid, built-in (is_builtin) compilation templates,
|
||||
// ordered by create_time then name. Mirrors Python list_builtins().
|
||||
func (dao *CompilationTemplateDAO) ListBuiltins(ctx context.Context) ([]*entity.CompilationTemplate, error) {
|
||||
var templates []*entity.CompilationTemplate
|
||||
if err := DB.WithContext(ctx).
|
||||
Where("is_builtin = ? AND status = ?", true, string(entity.StatusValid)).
|
||||
Order("create_time asc, name asc").
|
||||
Find(&templates).Error; err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return templates, nil
|
||||
}
|
||||
|
||||
// NameExistsInGroup reports whether a valid, non-built-in template with the
|
||||
// given name already exists inside the group, excluding excludeID. Mirrors the
|
||||
// Python duplicate-child guard.
|
||||
func (dao *CompilationTemplateDAO) NameExistsInGroup(ctx context.Context, tenantID, groupID, name, excludeID string) (bool, error) {
|
||||
q := DB.WithContext(ctx).Model(&entity.CompilationTemplate{}).
|
||||
Where("tenant_id = ? AND group_id = ? AND name = ? AND is_builtin = ? AND status = ?",
|
||||
tenantID, groupID, name, false, string(entity.StatusValid))
|
||||
if excludeID != "" {
|
||||
q = q.Where("id <> ?", excludeID)
|
||||
}
|
||||
var count int64
|
||||
if err := q.Count(&count).Error; err != nil {
|
||||
return false, err
|
||||
}
|
||||
return count > 0, nil
|
||||
}
|
||||
|
||||
// GetByID returns a single template row by id regardless of tenant scope
|
||||
// (used for reconciling group children).
|
||||
func (dao *CompilationTemplateDAO) GetByID(ctx context.Context, id string) (*entity.CompilationTemplate, error) {
|
||||
var t entity.CompilationTemplate
|
||||
if err := DB.WithContext(ctx).Where("id = ?", id).First(&t).Error; err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &t, nil
|
||||
}
|
||||
|
||||
// Save inserts a new template row.
|
||||
func (dao *CompilationTemplateDAO) Save(ctx context.Context, db *gorm.DB, t *entity.CompilationTemplate) error {
|
||||
return db.WithContext(ctx).Create(t).Error
|
||||
}
|
||||
|
||||
// UpdateFields persists the supplied columns for the template with the given id.
|
||||
func (dao *CompilationTemplateDAO) UpdateFields(ctx context.Context, db *gorm.DB, id string, m map[string]interface{}) error {
|
||||
return db.WithContext(ctx).Model(&entity.CompilationTemplate{}).
|
||||
Where("id = ?", id).Updates(m).Error
|
||||
}
|
||||
|
||||
// UpdateStatusByGroup flips the status of every valid template in a group,
|
||||
// mirroring Python group delete's child cascade.
|
||||
func (dao *CompilationTemplateDAO) UpdateStatusByGroup(ctx context.Context, db *gorm.DB, groupID, status string) error {
|
||||
return db.WithContext(ctx).Model(&entity.CompilationTemplate{}).
|
||||
Where("group_id = ? AND status = ?", groupID, string(entity.StatusValid)).
|
||||
Update("status", status).Error
|
||||
}
|
||||
|
||||
// UpdateStatusByID flips a single template's status.
|
||||
func (dao *CompilationTemplateDAO) UpdateStatusByID(ctx context.Context, db *gorm.DB, id, status string) error {
|
||||
return db.WithContext(ctx).Model(&entity.CompilationTemplate{}).
|
||||
Where("id = ?", id).
|
||||
Update("status", status).Error
|
||||
}
|
||||
|
||||
// HardDeleteOrphansByName physically removes stale, invalid, non-built-in
|
||||
// templates of the given name within the group, mirroring Python
|
||||
// _purge_stale_invalid_children (which DELETEs orphaned duplicate names after a
|
||||
// group child is soft-deleted). The deletion is scoped to the group so a
|
||||
// same-named template in another group is never affected. These rows were
|
||||
// soft-deleted in a prior operation but must be permanently purged to keep the
|
||||
// table clean.
|
||||
func (dao *CompilationTemplateDAO) HardDeleteOrphansByName(ctx context.Context, db *gorm.DB, tenantID, groupID, name string) error {
|
||||
return db.WithContext(ctx).Where(
|
||||
"tenant_id = ? AND group_id = ? AND name = ? AND is_builtin = ? AND status = ?",
|
||||
tenantID, groupID, name, false, string(entity.StatusInvalid),
|
||||
).Delete(&entity.CompilationTemplate{}).Error
|
||||
}
|
||||
|
||||
109
internal/dao/compilation_template_group.go
Normal file
109
internal/dao/compilation_template_group.go
Normal file
@@ -0,0 +1,109 @@
|
||||
//
|
||||
// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
//
|
||||
|
||||
package dao
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"ragflow/internal/entity"
|
||||
|
||||
"gorm.io/gorm"
|
||||
)
|
||||
|
||||
// CompilationTemplateGroupDAO is the data-access object for compilation template
|
||||
// groups.
|
||||
type CompilationTemplateGroupDAO struct{}
|
||||
|
||||
// NewCompilationTemplateGroupDAO creates a compilation template group DAO.
|
||||
func NewCompilationTemplateGroupDAO() *CompilationTemplateGroupDAO {
|
||||
return &CompilationTemplateGroupDAO{}
|
||||
}
|
||||
|
||||
// ListSaved returns the tenant's valid groups with optional keyword/scope
|
||||
// filtering and ordering, mirroring Python list_saved(). Built-in groups
|
||||
// (empty tenant) are included so every tenant sees the catalogue.
|
||||
func (dao *CompilationTemplateGroupDAO) ListSaved(ctx context.Context, tenantID, keywords, scope, orderby string, desc bool) ([]*entity.CompilationTemplateGroup, error) {
|
||||
q := DB.WithContext(ctx).
|
||||
Where("(tenant_id = ? OR tenant_id = '') AND status = ?", tenantID, string(entity.StatusValid))
|
||||
if keywords != "" {
|
||||
q = q.Where("name LIKE ?", "%"+keywords+"%")
|
||||
}
|
||||
if scope != "" {
|
||||
q = q.Where("scope = ?", scope)
|
||||
}
|
||||
if orderby != "name" && orderby != "scope" && orderby != "create_time" && orderby != "update_time" {
|
||||
orderby = "create_time"
|
||||
}
|
||||
dir := "asc"
|
||||
if desc {
|
||||
dir = "desc"
|
||||
}
|
||||
var groups []*entity.CompilationTemplateGroup
|
||||
if err := q.Order(orderby + " " + dir).Find(&groups).Error; err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return groups, nil
|
||||
}
|
||||
|
||||
// GetSaved returns a single valid group for the tenant (or built-in), or nil.
|
||||
func (dao *CompilationTemplateGroupDAO) GetSaved(ctx context.Context, tenantID, groupID string) (*entity.CompilationTemplateGroup, error) {
|
||||
var g entity.CompilationTemplateGroup
|
||||
err := DB.WithContext(ctx).
|
||||
Where("(tenant_id = ? OR tenant_id = '') AND id = ? AND status = ?",
|
||||
tenantID, groupID, string(entity.StatusValid)).
|
||||
First(&g).Error
|
||||
if errors.Is(err, gorm.ErrRecordNotFound) {
|
||||
return nil, nil
|
||||
}
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &g, nil
|
||||
}
|
||||
|
||||
// NameExists reports whether a valid group with the given name exists for the
|
||||
// tenant, excluding excludeID. Mirrors Python name_exists().
|
||||
func (dao *CompilationTemplateGroupDAO) NameExists(ctx context.Context, tenantID, name, excludeID string) (bool, error) {
|
||||
q := DB.WithContext(ctx).Model(&entity.CompilationTemplateGroup{}).
|
||||
Where("tenant_id = ? AND name = ? AND status = ?", tenantID, name, string(entity.StatusValid))
|
||||
if excludeID != "" {
|
||||
q = q.Where("id <> ?", excludeID)
|
||||
}
|
||||
var count int64
|
||||
if err := q.Count(&count).Error; err != nil {
|
||||
return false, err
|
||||
}
|
||||
return count > 0, nil
|
||||
}
|
||||
|
||||
// Create inserts a new group row.
|
||||
func (dao *CompilationTemplateGroupDAO) Create(ctx context.Context, db *gorm.DB, g *entity.CompilationTemplateGroup) error {
|
||||
return db.WithContext(ctx).Create(g).Error
|
||||
}
|
||||
|
||||
// UpdateFields persists the supplied columns for the group with the given id.
|
||||
func (dao *CompilationTemplateGroupDAO) UpdateFields(ctx context.Context, db *gorm.DB, id string, m map[string]interface{}) error {
|
||||
return db.WithContext(ctx).Model(&entity.CompilationTemplateGroup{}).
|
||||
Where("id = ?", id).Updates(m).Error
|
||||
}
|
||||
|
||||
// Delete soft-deletes a valid group, mirroring Python delete_group().
|
||||
func (dao *CompilationTemplateGroupDAO) Delete(ctx context.Context, db *gorm.DB, tenantID, groupID string) error {
|
||||
return db.WithContext(ctx).Model(&entity.CompilationTemplateGroup{}).
|
||||
Where("id = ? AND tenant_id = ? AND status = ?", groupID, tenantID, string(entity.StatusValid)).
|
||||
Update("status", string(entity.StatusInvalid)).Error
|
||||
}
|
||||
@@ -18,11 +18,24 @@ package dao
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"ragflow/internal/entity"
|
||||
"strconv"
|
||||
|
||||
"gorm.io/gorm"
|
||||
"gorm.io/gorm/clause"
|
||||
)
|
||||
|
||||
// toInterfaceSlice converts a string slice into an []interface{} for use as
|
||||
// parameterized query variables.
|
||||
func toInterfaceSlice(ss []string) []interface{} {
|
||||
out := make([]interface{}, len(ss))
|
||||
for i := range ss {
|
||||
out[i] = ss[i]
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// FileCommitDAO file commit data access object
|
||||
type FileCommitDAO struct{}
|
||||
|
||||
@@ -110,6 +123,41 @@ func (dao *FileCommitDAO) ListByFolderID(ctx context.Context, db *gorm.DB, folde
|
||||
return commits, total, nil
|
||||
}
|
||||
|
||||
// ListByIDs lists commits whose IDs are in the given set, preserving the
|
||||
// order of commitIDs (most-recent-first for page-edit history). Returns the
|
||||
// total matched count for pagination.
|
||||
func (dao *FileCommitDAO) ListByIDs(ctx context.Context, db *gorm.DB, commitIDs []string, page, pageSize int) ([]*entity.FileCommit, int64, error) {
|
||||
if len(commitIDs) == 0 {
|
||||
return []*entity.FileCommit{}, 0, nil
|
||||
}
|
||||
var commits []*entity.FileCommit
|
||||
var total int64
|
||||
query := db.WithContext(ctx).Model(&entity.FileCommit{}).Where("id IN ?", commitIDs)
|
||||
if err := query.Count(&total).Error; err != nil {
|
||||
return nil, 0, err
|
||||
}
|
||||
// Preserve the incoming (most-recent-first) order with a parameterized
|
||||
// CASE expression rather than a MySQL-specific FIELD() interpolation.
|
||||
orderExpr := "CASE id"
|
||||
for i := range commitIDs {
|
||||
orderExpr += " WHEN ? THEN " + strconv.Itoa(i)
|
||||
}
|
||||
orderExpr += " END"
|
||||
ordered := query.Clauses(clause.OrderBy{Expression: clause.Expr{
|
||||
SQL: orderExpr,
|
||||
Vars: toInterfaceSlice(commitIDs),
|
||||
}})
|
||||
// Apply pagination in the query itself so long histories never load the
|
||||
// full set into memory before slicing.
|
||||
if page > 0 && pageSize > 0 {
|
||||
ordered = ordered.Offset((page - 1) * pageSize).Limit(pageSize)
|
||||
}
|
||||
if err := ordered.Find(&commits).Error; err != nil {
|
||||
return nil, 0, err
|
||||
}
|
||||
return commits, total, nil
|
||||
}
|
||||
|
||||
// FileCommitItemDAO file commit item data access object
|
||||
type FileCommitItemDAO struct{}
|
||||
|
||||
@@ -146,3 +194,24 @@ func (dao *FileCommitItemDAO) GetByCommitIDAndFileID(ctx context.Context, db *go
|
||||
}
|
||||
return &item, nil
|
||||
}
|
||||
|
||||
// GetLatestCommitIDByFileID returns the most recent commit_id that modified the
|
||||
// given wiki page file key, or "" if none. Used to build the parent chain for
|
||||
// page-edit commits.
|
||||
func (dao *FileCommitItemDAO) GetLatestCommitIDByFileID(ctx context.Context, db *gorm.DB, fileID string) (string, error) {
|
||||
var item entity.FileCommitItem
|
||||
err := db.WithContext(ctx).
|
||||
Where("file_id = ?", fileID).
|
||||
// Order by the monotonic auto-increment sequence so the most recently
|
||||
// inserted item is always selected even when multiple edits share the
|
||||
// same create_time millisecond.
|
||||
Order("seq DESC").
|
||||
First(&item).Error
|
||||
if err != nil {
|
||||
if errors.Is(err, gorm.ErrRecordNotFound) {
|
||||
return "", nil
|
||||
}
|
||||
return "", err
|
||||
}
|
||||
return item.CommitID, nil
|
||||
}
|
||||
|
||||
@@ -38,8 +38,9 @@ func (FileCommit) TableName() string {
|
||||
// FileCommitItem represents a single file change within a commit.
|
||||
type FileCommitItem struct {
|
||||
ID string `gorm:"column:id;primaryKey;size:32" json:"id"`
|
||||
Seq uint `gorm:"column:seq;autoIncrement;index" json:"seq,omitempty"`
|
||||
CommitID string `gorm:"column:commit_id;size:32;not null;uniqueIndex:idx_commit_file" json:"commit_id"`
|
||||
FileID string `gorm:"column:file_id;size:32;not null;uniqueIndex:idx_commit_file" json:"file_id"`
|
||||
FileID string `gorm:"column:file_id;size:255;not null;uniqueIndex:idx_commit_file" json:"file_id"`
|
||||
Operation string `gorm:"column:operation;size:16;not null;index" json:"operation"`
|
||||
OldHash *string `gorm:"column:old_hash;size:64;index" json:"old_hash,omitempty"`
|
||||
NewHash *string `gorm:"column:new_hash;size:64;index" json:"new_hash,omitempty"`
|
||||
|
||||
78
internal/handler/compilation_template.go
Normal file
78
internal/handler/compilation_template.go
Normal file
@@ -0,0 +1,78 @@
|
||||
//
|
||||
// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
//
|
||||
|
||||
package handler
|
||||
|
||||
import (
|
||||
"ragflow/internal/common"
|
||||
"ragflow/internal/service"
|
||||
|
||||
"github.com/gin-gonic/gin"
|
||||
)
|
||||
|
||||
// CompilationTemplateHandler serves the knowledge-compilation template REST
|
||||
// APIs. It mirrors the Python compilation_template_api blueprint.
|
||||
type CompilationTemplateHandler struct {
|
||||
compilationTemplateService *service.CompilationTemplateService
|
||||
}
|
||||
|
||||
// NewCompilationTemplateHandler creates a CompilationTemplateHandler.
|
||||
func NewCompilationTemplateHandler(svc *service.CompilationTemplateService) *CompilationTemplateHandler {
|
||||
return &CompilationTemplateHandler{compilationTemplateService: svc}
|
||||
}
|
||||
|
||||
// ListBuiltins returns the built-in template palette used by the frontend
|
||||
// "Add template group" panel.
|
||||
//
|
||||
// @Summary List built-in compilation templates
|
||||
// @Tags compilation template
|
||||
// @Security ApiKeyAuth
|
||||
// @Success 200 {object} map[string]interface{}
|
||||
// @Router /v1/compilation-templates/builtins [get]
|
||||
func (h *CompilationTemplateHandler) ListBuiltins(c *gin.Context) {
|
||||
user, code, msg := GetUser(c)
|
||||
if code != common.CodeSuccess {
|
||||
common.ErrorWithCode(c, code, msg)
|
||||
return
|
||||
}
|
||||
templates, err := h.compilationTemplateService.ListBuiltins(c.Request.Context(), user.ID)
|
||||
if err != nil {
|
||||
common.ResponseWithCodeData(c, common.CodeExceptionError, nil, err.Error())
|
||||
return
|
||||
}
|
||||
common.SuccessWithData(c, templates, "success")
|
||||
}
|
||||
|
||||
// ListWikiPresets returns the wiki page-structure presets loaded from YAML.
|
||||
//
|
||||
// @Summary List wiki presets
|
||||
// @Tags compilation template
|
||||
// @Security ApiKeyAuth
|
||||
// @Success 200 {object} map[string]interface{}
|
||||
// @Router /v1/compilation-templates/wiki-presets [get]
|
||||
func (h *CompilationTemplateHandler) ListWikiPresets(c *gin.Context) {
|
||||
_, code, msg := GetUser(c)
|
||||
if code != common.CodeSuccess {
|
||||
common.ErrorWithCode(c, code, msg)
|
||||
return
|
||||
}
|
||||
presets, err := h.compilationTemplateService.LoadWikiPresets()
|
||||
if err != nil {
|
||||
common.ResponseWithCodeData(c, common.CodeExceptionError, nil, err.Error())
|
||||
return
|
||||
}
|
||||
common.SuccessWithData(c, presets, "success")
|
||||
}
|
||||
233
internal/handler/compilation_template_group.go
Normal file
233
internal/handler/compilation_template_group.go
Normal file
@@ -0,0 +1,233 @@
|
||||
//
|
||||
// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
//
|
||||
|
||||
package handler
|
||||
|
||||
import (
|
||||
"strconv"
|
||||
|
||||
"ragflow/internal/common"
|
||||
"ragflow/internal/service"
|
||||
|
||||
"github.com/gin-gonic/gin"
|
||||
)
|
||||
|
||||
// CompilationTemplateGroupHandler serves the knowledge-compilation template
|
||||
// group REST APIs. It mirrors the Python compilation_template_group_api blueprint.
|
||||
type CompilationTemplateGroupHandler struct {
|
||||
compilationTemplateGroupService *service.CompilationTemplateGroupService
|
||||
}
|
||||
|
||||
// NewCompilationTemplateGroupHandler creates a CompilationTemplateGroupHandler.
|
||||
func NewCompilationTemplateGroupHandler(svc *service.CompilationTemplateGroupService) *CompilationTemplateGroupHandler {
|
||||
return &CompilationTemplateGroupHandler{compilationTemplateGroupService: svc}
|
||||
}
|
||||
|
||||
// List returns the tenant's compilation template groups with their nested
|
||||
// templates and pagination.
|
||||
//
|
||||
// @Summary List compilation template groups
|
||||
// @Tags compilation template group
|
||||
// @Security ApiKeyAuth
|
||||
// @Param keywords query string false "name keyword filter"
|
||||
// @Param scope query string false "scope filter (file|dataset)"
|
||||
// @Param page query int false "page number" default(1)
|
||||
// @Param page_size query int false "items per page" default(30)
|
||||
// @Param orderby query string false "order field" default(create_time)
|
||||
// @Param desc query bool false "descending" default(true)
|
||||
// @Success 200 {object} map[string]interface{}
|
||||
// @Router /v1/compilation-template-groups [get]
|
||||
func (h *CompilationTemplateGroupHandler) List(c *gin.Context) {
|
||||
user, code, msg := GetUser(c)
|
||||
if code != common.CodeSuccess {
|
||||
common.ErrorWithCode(c, code, msg)
|
||||
return
|
||||
}
|
||||
page, _ := strconv.Atoi(c.DefaultQuery("page", "1"))
|
||||
pageSize, _ := strconv.Atoi(c.DefaultQuery("page_size", "30"))
|
||||
if page < 1 {
|
||||
page = 1
|
||||
}
|
||||
if pageSize < 1 {
|
||||
pageSize = 30
|
||||
}
|
||||
keywords := c.Query("keywords")
|
||||
scope := c.Query("scope")
|
||||
orderby := c.DefaultQuery("orderby", "create_time")
|
||||
desc := c.DefaultQuery("desc", "true") != "false"
|
||||
|
||||
groups, err := h.compilationTemplateGroupService.ListSaved(c.Request.Context(), user.ID, keywords, scope, orderby, desc)
|
||||
if err != nil {
|
||||
common.ResponseWithCodeData(c, common.CodeExceptionError, nil, err.Error())
|
||||
return
|
||||
}
|
||||
total := len(groups)
|
||||
if page > 0 && pageSize > 0 {
|
||||
start := (page - 1) * pageSize
|
||||
if start >= total {
|
||||
groups = []*service.GroupListItem{}
|
||||
} else {
|
||||
end := start + pageSize
|
||||
if end > total {
|
||||
end = total
|
||||
}
|
||||
groups = groups[start:end]
|
||||
}
|
||||
}
|
||||
common.SuccessWithData(c, gin.H{"groups": groups, "total": total}, "success")
|
||||
}
|
||||
|
||||
// Get returns a single compilation template group.
|
||||
//
|
||||
// @Summary Get compilation template group
|
||||
// @Tags compilation template group
|
||||
// @Security ApiKeyAuth
|
||||
// @Param group_id path string true "group id"
|
||||
// @Success 200 {object} map[string]interface{}
|
||||
// @Router /v1/compilation-template-groups/{group_id} [get]
|
||||
func (h *CompilationTemplateGroupHandler) Get(c *gin.Context) {
|
||||
user, code, msg := GetUser(c)
|
||||
if code != common.CodeSuccess {
|
||||
common.ErrorWithCode(c, code, msg)
|
||||
return
|
||||
}
|
||||
id := c.Param("group_id")
|
||||
group, err := h.compilationTemplateGroupService.GetSaved(c.Request.Context(), user.ID, id)
|
||||
if err != nil {
|
||||
common.ResponseWithCodeData(c, common.CodeExceptionError, nil, err.Error())
|
||||
return
|
||||
}
|
||||
if group == nil {
|
||||
common.ResponseWithCodeData(c, common.CodeNotFound, nil, "Cannot find compilation template group "+id+".")
|
||||
return
|
||||
}
|
||||
common.SuccessWithData(c, group, "success")
|
||||
}
|
||||
|
||||
// Save creates a new compilation template group with its child templates.
|
||||
//
|
||||
// @Summary Save compilation template group
|
||||
// @Tags compilation template group
|
||||
// @Security ApiKeyAuth
|
||||
// @Param request body service.GroupRequest true "group payload"
|
||||
// @Success 200 {object} map[string]interface{}
|
||||
// @Router /v1/compilation-template-groups [post]
|
||||
func (h *CompilationTemplateGroupHandler) Save(c *gin.Context) {
|
||||
user, code, msg := GetUser(c)
|
||||
if code != common.CodeSuccess {
|
||||
common.ErrorWithCode(c, code, msg)
|
||||
return
|
||||
}
|
||||
var req service.GroupRequest
|
||||
if err := c.ShouldBindJSON(&req); err != nil {
|
||||
common.ResponseWithCodeData(c, common.CodeArgumentError, nil, "Invalid request: "+err.Error())
|
||||
return
|
||||
}
|
||||
if err := service.ValidateGroupRequest(&req); err != nil {
|
||||
common.ResponseWithCodeData(c, common.CodeDataError, nil, err.Error())
|
||||
return
|
||||
}
|
||||
nameExists, err := h.compilationTemplateGroupService.NameExists(c.Request.Context(), user.ID, req.Name)
|
||||
if err != nil {
|
||||
common.ResponseWithCodeData(c, common.CodeExceptionError, nil, err.Error())
|
||||
return
|
||||
}
|
||||
if nameExists {
|
||||
common.ResponseWithCodeData(c, common.CodeDataError, nil, "Duplicated compilation template group name.")
|
||||
return
|
||||
}
|
||||
group, err := h.compilationTemplateGroupService.CreateGroup(c.Request.Context(), user.ID, &req)
|
||||
if err != nil {
|
||||
common.ResponseWithCodeData(c, common.CodeDataError, nil, err.Error())
|
||||
return
|
||||
}
|
||||
common.SuccessWithData(c, group, "success")
|
||||
}
|
||||
|
||||
// Update applies a partial update to a compilation template group.
|
||||
//
|
||||
// @Summary Update compilation template group
|
||||
// @Tags compilation template group
|
||||
// @Security ApiKeyAuth
|
||||
// @Param group_id path string true "group id"
|
||||
// @Param request body service.GroupRequest true "group payload"
|
||||
// @Success 200 {object} map[string]interface{}
|
||||
// @Router /v1/compilation-template-groups/{group_id} [put]
|
||||
func (h *CompilationTemplateGroupHandler) Update(c *gin.Context) {
|
||||
user, code, msg := GetUser(c)
|
||||
if code != common.CodeSuccess {
|
||||
common.ErrorWithCode(c, code, msg)
|
||||
return
|
||||
}
|
||||
id := c.Param("group_id")
|
||||
var req service.GroupRequest
|
||||
if err := c.ShouldBindJSON(&req); err != nil {
|
||||
common.ResponseWithCodeData(c, common.CodeArgumentError, nil, "Invalid request: "+err.Error())
|
||||
return
|
||||
}
|
||||
if err := service.ValidateGroupRequest(&req); err != nil {
|
||||
common.ResponseWithCodeData(c, common.CodeDataError, nil, err.Error())
|
||||
return
|
||||
}
|
||||
if req.Name != "" {
|
||||
nameExists, nerr := h.compilationTemplateGroupService.NameExists(c.Request.Context(), user.ID, req.Name, id)
|
||||
if nerr != nil {
|
||||
common.ResponseWithCodeData(c, common.CodeExceptionError, nil, nerr.Error())
|
||||
return
|
||||
}
|
||||
if nameExists {
|
||||
common.ResponseWithCodeData(c, common.CodeDataError, nil, "Duplicated compilation template group name.")
|
||||
return
|
||||
}
|
||||
}
|
||||
group, err := h.compilationTemplateGroupService.UpdateGroup(c.Request.Context(), user.ID, id, &req)
|
||||
if err != nil {
|
||||
common.ResponseWithCodeData(c, common.CodeDataError, nil, err.Error())
|
||||
return
|
||||
}
|
||||
if group == nil {
|
||||
common.ResponseWithCodeData(c, common.CodeNotFound, nil, "Cannot find compilation template group "+id+".")
|
||||
return
|
||||
}
|
||||
common.SuccessWithData(c, group, "success")
|
||||
}
|
||||
|
||||
// Delete soft-deletes a compilation template group and its children.
|
||||
//
|
||||
// @Summary Delete compilation template group
|
||||
// @Tags compilation template group
|
||||
// @Security ApiKeyAuth
|
||||
// @Param group_id path string true "group id"
|
||||
// @Success 200 {object} map[string]interface{}
|
||||
// @Router /v1/compilation-template-groups/{group_id} [delete]
|
||||
func (h *CompilationTemplateGroupHandler) Delete(c *gin.Context) {
|
||||
user, code, msg := GetUser(c)
|
||||
if code != common.CodeSuccess {
|
||||
common.ErrorWithCode(c, code, msg)
|
||||
return
|
||||
}
|
||||
id := c.Param("group_id")
|
||||
ok, err := h.compilationTemplateGroupService.DeleteGroup(c.Request.Context(), user.ID, id)
|
||||
if err != nil {
|
||||
common.ResponseWithCodeData(c, common.CodeExceptionError, nil, err.Error())
|
||||
return
|
||||
}
|
||||
if !ok {
|
||||
common.ResponseWithCodeData(c, common.CodeNotFound, nil, "Cannot find compilation template group "+id+".")
|
||||
return
|
||||
}
|
||||
common.SuccessWithData(c, true, "success")
|
||||
}
|
||||
475
internal/handler/dataset_artifact.go
Normal file
475
internal/handler/dataset_artifact.go
Normal file
@@ -0,0 +1,475 @@
|
||||
//
|
||||
// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
//
|
||||
|
||||
package handler
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
"strconv"
|
||||
|
||||
"ragflow/internal/common"
|
||||
"ragflow/internal/dao"
|
||||
"ragflow/internal/entity"
|
||||
"ragflow/internal/service"
|
||||
dataset "ragflow/internal/service/dataset"
|
||||
"ragflow/internal/service/file"
|
||||
|
||||
"github.com/gin-gonic/gin"
|
||||
"go.uber.org/zap"
|
||||
)
|
||||
|
||||
// maxArtifactPageSize caps the page_size query parameter for artifact list
|
||||
// endpoints so a client cannot force an unbounded document-engine search.
|
||||
const maxArtifactPageSize = 100
|
||||
|
||||
// DatasetArtifactHandler exposes the knowledge-compilation artifact REST APIs
|
||||
// backed by DatasetArtifactService.
|
||||
type DatasetArtifactHandler struct {
|
||||
svc *service.DatasetArtifactService
|
||||
datasetSvc *dataset.DatasetService
|
||||
fileCommitSvc *file.FileCommitService
|
||||
}
|
||||
|
||||
// NewDatasetArtifactHandler creates a DatasetArtifactHandler.
|
||||
func NewDatasetArtifactHandler(svc *service.DatasetArtifactService, datasetSvc *dataset.DatasetService, fileCommitSvc *file.FileCommitService) *DatasetArtifactHandler {
|
||||
return &DatasetArtifactHandler{svc: svc, datasetSvc: datasetSvc, fileCommitSvc: fileCommitSvc}
|
||||
}
|
||||
|
||||
// datasetOwner resolves the dataset and returns its tenant id. It also enforces
|
||||
// dataset access permission for the requesting user. When the request is
|
||||
// unauthenticated, the dataset is missing, or the user is not allowed to access
|
||||
// it, an error response is written and (nil, "", "") is returned so callers can
|
||||
// abort without sending a second response.
|
||||
func (h *DatasetArtifactHandler) datasetOwner(c *gin.Context, datasetID string) (*entity.User, string, string) {
|
||||
user, code, msg := GetUser(c)
|
||||
if code != common.CodeSuccess {
|
||||
common.ErrorWithCode(c, code, msg)
|
||||
return nil, "", ""
|
||||
}
|
||||
kb, err := h.datasetSvc.GetKnowledgebaseByID(c.Request.Context(), datasetID)
|
||||
if err != nil {
|
||||
if dao.IsNotFoundErr(err) {
|
||||
common.ErrorWithCode(c, common.CodeNotFound, "dataset not found")
|
||||
} else {
|
||||
common.ErrorWithCode(c, common.CodeServerError, "failed to resolve dataset")
|
||||
}
|
||||
return nil, "", ""
|
||||
}
|
||||
if kb == nil {
|
||||
common.ErrorWithCode(c, common.CodeNotFound, "dataset not found")
|
||||
return nil, "", ""
|
||||
}
|
||||
if !h.datasetSvc.Accessible(c.Request.Context(), datasetID, user.ID) {
|
||||
common.ErrorWithCode(c, common.CodeForbidden, "no permission to access this dataset")
|
||||
return nil, "", ""
|
||||
}
|
||||
return user, kb.TenantID, msg
|
||||
}
|
||||
|
||||
// HEAD /artifacts — any wiki artifact present?
|
||||
func (h *DatasetArtifactHandler) AnyArtifact(c *gin.Context) {
|
||||
_, tenantID, _ := h.datasetOwner(c, c.Param("dataset_id"))
|
||||
if tenantID == "" {
|
||||
return
|
||||
}
|
||||
has, err := h.svc.HasWiki(c.Request.Context(), tenantID, c.Param("dataset_id"))
|
||||
if err != nil {
|
||||
common.ErrorWithCode(c, common.CodeDataError, err.Error())
|
||||
return
|
||||
}
|
||||
if has {
|
||||
c.Status(http.StatusOK)
|
||||
} else {
|
||||
c.Status(http.StatusNotFound)
|
||||
}
|
||||
}
|
||||
|
||||
// GET /artifacts — list wiki pages.
|
||||
func (h *DatasetArtifactHandler) ListArtifacts(c *gin.Context) {
|
||||
_, tenantID, _ := h.datasetOwner(c, c.Param("dataset_id"))
|
||||
if tenantID == "" {
|
||||
return
|
||||
}
|
||||
datasetID := c.Param("dataset_id")
|
||||
pageType := c.Query("page_type")
|
||||
topic := c.Query("topic")
|
||||
page, _ := strconv.Atoi(c.DefaultQuery("page", "1"))
|
||||
pageSize, _ := strconv.Atoi(c.DefaultQuery("page_size", "30"))
|
||||
if page < 1 {
|
||||
page = 1
|
||||
}
|
||||
if pageSize < 1 {
|
||||
pageSize = 30
|
||||
}
|
||||
if pageSize > maxArtifactPageSize {
|
||||
pageSize = maxArtifactPageSize
|
||||
}
|
||||
items, total, err := h.svc.ListWikiPages(c.Request.Context(), tenantID, datasetID, pageType, topic, page, pageSize)
|
||||
if err != nil {
|
||||
common.ErrorWithCode(c, common.CodeDataError, err.Error())
|
||||
return
|
||||
}
|
||||
common.SuccessWithData(c, gin.H{"total": total, "pages": items}, "success")
|
||||
}
|
||||
|
||||
// PUT /artifacts/<page_type>/<slug> — edit a wiki page.
|
||||
func (h *DatasetArtifactHandler) UpdateArtifact(c *gin.Context) {
|
||||
user, tenantID, _ := h.datasetOwner(c, c.Param("dataset_id"))
|
||||
if tenantID == "" {
|
||||
return
|
||||
}
|
||||
datasetID := c.Param("dataset_id")
|
||||
pageType := c.Param("page_type")
|
||||
slug := c.Param("slug")
|
||||
var req struct {
|
||||
ContentMd string `json:"content_md"`
|
||||
Title string `json:"title"`
|
||||
Outlinks []string `json:"outlinks"`
|
||||
}
|
||||
if err := c.ShouldBindJSON(&req); err != nil {
|
||||
common.ErrorWithCode(c, common.CodeArgumentError, err.Error())
|
||||
return
|
||||
}
|
||||
|
||||
// Read the current page before mutating so an audit commit can record an
|
||||
// accurate old-to-new diff. Only record when the prior state was read
|
||||
// successfully; a failed read must not produce a diff against an unknown
|
||||
// old value.
|
||||
var oldContent string
|
||||
canRecordAudit := false
|
||||
if req.ContentMd != "" {
|
||||
before, gerr := h.svc.GetWikiPage(c.Request.Context(), tenantID, datasetID, pageType, slug)
|
||||
if gerr != nil {
|
||||
common.Warn("failed to read wiki page before recording edit commit", zap.Error(gerr))
|
||||
} else if before != nil {
|
||||
oldContent = before.ContentMd
|
||||
canRecordAudit = true
|
||||
}
|
||||
}
|
||||
|
||||
detail, err := h.svc.UpdateWikiPage(c.Request.Context(), tenantID, datasetID, pageType, slug, req.ContentMd, req.Title, req.Outlinks)
|
||||
if err != nil {
|
||||
common.ErrorWithCode(c, common.CodeDataError, err.Error())
|
||||
return
|
||||
}
|
||||
if detail == nil {
|
||||
common.ErrorWithCode(c, common.CodeNotFound, "page not found")
|
||||
return
|
||||
}
|
||||
|
||||
// Record a wiki-page edit audit commit (git-style parent chain) when the
|
||||
// content actually changed. Best-effort: a commit write failure must not
|
||||
// roll back the page edit already persisted above.
|
||||
if canRecordAudit && req.ContentMd != oldContent && h.fileCommitSvc != nil {
|
||||
title := req.Title
|
||||
if title == "" {
|
||||
title = detail.Title
|
||||
}
|
||||
if _, cerr := h.fileCommitSvc.RecordPageEdit(c.Request.Context(), file.PageEditCommitInput{
|
||||
DatasetID: datasetID,
|
||||
DocID: pageType + "/" + slug,
|
||||
Slug: slug,
|
||||
PageType: pageType,
|
||||
Title: title,
|
||||
AuthorID: user.ID,
|
||||
OldContent: oldContent,
|
||||
NewContent: req.ContentMd,
|
||||
}); cerr != nil {
|
||||
common.Warn("failed to record wiki page edit commit", zap.Error(cerr))
|
||||
}
|
||||
}
|
||||
|
||||
common.SuccessWithData(c, detail, "success")
|
||||
}
|
||||
|
||||
// GET /artifacts/<page_type>/<slug> — single wiki page.
|
||||
func (h *DatasetArtifactHandler) GetArtifact(c *gin.Context) {
|
||||
_, tenantID, _ := h.datasetOwner(c, c.Param("dataset_id"))
|
||||
if tenantID == "" {
|
||||
return
|
||||
}
|
||||
datasetID := c.Param("dataset_id")
|
||||
pageType := c.Param("page_type")
|
||||
slug := c.Param("slug")
|
||||
detail, err := h.svc.GetWikiPage(c.Request.Context(), tenantID, datasetID, pageType, slug)
|
||||
if err != nil {
|
||||
common.ErrorWithCode(c, common.CodeDataError, err.Error())
|
||||
return
|
||||
}
|
||||
if detail == nil {
|
||||
common.ErrorWithCode(c, common.CodeNotFound, "page not found")
|
||||
return
|
||||
}
|
||||
common.SuccessWithData(c, detail, "success")
|
||||
}
|
||||
|
||||
// DELETE /artifacts — clear all wiki artifacts.
|
||||
func (h *DatasetArtifactHandler) DeleteArtifacts(c *gin.Context) {
|
||||
_, tenantID, _ := h.datasetOwner(c, c.Param("dataset_id"))
|
||||
if tenantID == "" {
|
||||
return
|
||||
}
|
||||
deleted, err := h.svc.ClearWiki(c.Request.Context(), tenantID, c.Param("dataset_id"))
|
||||
if err != nil {
|
||||
common.ErrorWithCode(c, common.CodeDataError, err.Error())
|
||||
return
|
||||
}
|
||||
common.SuccessWithData(c, deleted, "success")
|
||||
}
|
||||
|
||||
// GET /artifacts/topics — list wiki topics.
|
||||
func (h *DatasetArtifactHandler) ListArtifactTopics(c *gin.Context) {
|
||||
_, tenantID, _ := h.datasetOwner(c, c.Param("dataset_id"))
|
||||
if tenantID == "" {
|
||||
return
|
||||
}
|
||||
datasetID := c.Param("dataset_id")
|
||||
items, total, err := h.svc.ListWikiTopics(c.Request.Context(), tenantID, datasetID)
|
||||
if err != nil {
|
||||
common.ErrorWithCode(c, common.CodeDataError, err.Error())
|
||||
return
|
||||
}
|
||||
common.SuccessWithData(c, gin.H{"total": total, "topics": items}, "success")
|
||||
}
|
||||
|
||||
// GET /artifacts/alteration — wiki alteration summary.
|
||||
func (h *DatasetArtifactHandler) GetArtifactAlteration(c *gin.Context) {
|
||||
_, tenantID, _ := h.datasetOwner(c, c.Param("dataset_id"))
|
||||
if tenantID == "" {
|
||||
return
|
||||
}
|
||||
datasetID := c.Param("dataset_id")
|
||||
alt, err := h.svc.GetWikiAlteration(c.Request.Context(), tenantID, datasetID)
|
||||
if err != nil {
|
||||
common.ErrorWithCode(c, common.CodeDataError, err.Error())
|
||||
return
|
||||
}
|
||||
common.SuccessWithData(c, alt, "success")
|
||||
}
|
||||
|
||||
// GET /artifacts/graph — wiki entity/relation graph.
|
||||
func (h *DatasetArtifactHandler) GetArtifactGraph(c *gin.Context) {
|
||||
_, tenantID, _ := h.datasetOwner(c, c.Param("dataset_id"))
|
||||
if tenantID == "" {
|
||||
return
|
||||
}
|
||||
datasetID := c.Param("dataset_id")
|
||||
graph, err := h.svc.GetWikiGraph(c.Request.Context(), tenantID, datasetID)
|
||||
if err != nil {
|
||||
common.ErrorWithCode(c, common.CodeDataError, err.Error())
|
||||
return
|
||||
}
|
||||
common.SuccessWithData(c, graph, "success")
|
||||
}
|
||||
|
||||
// GET /artifacts/structure — list compiled structures of a dataset.
|
||||
func (h *DatasetArtifactHandler) ListStructures(c *gin.Context) {
|
||||
_, tenantID, _ := h.datasetOwner(c, c.Param("dataset_id"))
|
||||
if tenantID == "" {
|
||||
return
|
||||
}
|
||||
datasetID := c.Param("dataset_id")
|
||||
structureKind := c.Query("structure_kind")
|
||||
structureIndexType := c.Query("structure_index_type")
|
||||
items, total, err := h.svc.ListStructures(c.Request.Context(), tenantID, datasetID, structureKind, structureIndexType)
|
||||
if err != nil {
|
||||
common.ErrorWithCode(c, common.CodeDataError, err.Error())
|
||||
return
|
||||
}
|
||||
common.SuccessWithData(c, gin.H{"total": total, "structures": items}, "success")
|
||||
}
|
||||
|
||||
// DELETE /artifacts/structure — delete compiled structures of a dataset.
|
||||
func (h *DatasetArtifactHandler) DeleteStructures(c *gin.Context) {
|
||||
_, tenantID, _ := h.datasetOwner(c, c.Param("dataset_id"))
|
||||
if tenantID == "" {
|
||||
return
|
||||
}
|
||||
datasetID := c.Param("dataset_id")
|
||||
structureKind := c.Query("structure_kind")
|
||||
structureIndexType := c.Query("structure_index_type")
|
||||
n, err := h.svc.DeleteStructures(c.Request.Context(), tenantID, datasetID, structureKind, structureIndexType)
|
||||
if err != nil {
|
||||
common.ErrorWithCode(c, common.CodeDataError, err.Error())
|
||||
return
|
||||
}
|
||||
common.SuccessWithData(c, gin.H{"deleted": n}, "success")
|
||||
}
|
||||
|
||||
// HEAD /skills — any skill artifact present?
|
||||
func (h *DatasetArtifactHandler) AnySkill(c *gin.Context) {
|
||||
_, tenantID, _ := h.datasetOwner(c, c.Param("dataset_id"))
|
||||
if tenantID == "" {
|
||||
return
|
||||
}
|
||||
has, err := h.svc.HasSkill(c.Request.Context(), tenantID, c.Param("dataset_id"))
|
||||
if err != nil {
|
||||
common.ErrorWithCode(c, common.CodeDataError, err.Error())
|
||||
return
|
||||
}
|
||||
if has {
|
||||
c.Status(http.StatusOK)
|
||||
} else {
|
||||
c.Status(http.StatusNotFound)
|
||||
}
|
||||
}
|
||||
|
||||
// GET /navigation — list navigation clusters.
|
||||
func (h *DatasetArtifactHandler) ListNavigation(c *gin.Context) {
|
||||
_, tenantID, _ := h.datasetOwner(c, c.Param("dataset_id"))
|
||||
if tenantID == "" {
|
||||
return
|
||||
}
|
||||
items, total, err := h.svc.ListNavClusters(c.Request.Context(), tenantID, c.Param("dataset_id"))
|
||||
if err != nil {
|
||||
common.ErrorWithCode(c, common.CodeDataError, err.Error())
|
||||
return
|
||||
}
|
||||
common.SuccessWithData(c, gin.H{"total": total, "nav": items}, "success")
|
||||
}
|
||||
|
||||
// DELETE /navigation — delete all navigation clusters.
|
||||
func (h *DatasetArtifactHandler) DeleteNavigation(c *gin.Context) {
|
||||
_, tenantID, _ := h.datasetOwner(c, c.Param("dataset_id"))
|
||||
if tenantID == "" {
|
||||
return
|
||||
}
|
||||
n, err := h.svc.DeleteNav(c.Request.Context(), tenantID, c.Param("dataset_id"))
|
||||
if err != nil {
|
||||
common.ErrorWithCode(c, common.CodeDataError, err.Error())
|
||||
return
|
||||
}
|
||||
common.SuccessWithData(c, gin.H{"deleted": n}, "success")
|
||||
}
|
||||
|
||||
// DELETE /navigation/<name> — delete a single navigation cluster.
|
||||
func (h *DatasetArtifactHandler) DeleteNavigationNode(c *gin.Context) {
|
||||
_, tenantID, _ := h.datasetOwner(c, c.Param("dataset_id"))
|
||||
if tenantID == "" {
|
||||
return
|
||||
}
|
||||
n, err := h.svc.DeleteNavNode(c.Request.Context(), tenantID, c.Param("dataset_id"), c.Param("name"))
|
||||
if err != nil {
|
||||
common.ErrorWithCode(c, common.CodeDataError, err.Error())
|
||||
return
|
||||
}
|
||||
common.SuccessWithData(c, gin.H{"deleted": n}, "success")
|
||||
}
|
||||
|
||||
// GET /navigation/<name>/children — list children of a navigation cluster.
|
||||
func (h *DatasetArtifactHandler) ListNavigationChildren(c *gin.Context) {
|
||||
_, tenantID, _ := h.datasetOwner(c, c.Param("dataset_id"))
|
||||
if tenantID == "" {
|
||||
return
|
||||
}
|
||||
items, total, err := h.svc.ListNavChildren(c.Request.Context(), tenantID, c.Param("dataset_id"), c.Param("name"))
|
||||
if err != nil {
|
||||
common.ErrorWithCode(c, common.CodeDataError, err.Error())
|
||||
return
|
||||
}
|
||||
common.SuccessWithData(c, gin.H{"total": total, "children": items}, "success")
|
||||
}
|
||||
|
||||
// GET /skills — skill tree.
|
||||
func (h *DatasetArtifactHandler) GetSkillTree(c *gin.Context) {
|
||||
_, tenantID, _ := h.datasetOwner(c, c.Param("dataset_id"))
|
||||
if tenantID == "" {
|
||||
return
|
||||
}
|
||||
kwd := c.Query("kwd")
|
||||
items, total, err := h.svc.GetSkillTree(c.Request.Context(), tenantID, c.Param("dataset_id"), kwd)
|
||||
if err != nil {
|
||||
common.ErrorWithCode(c, common.CodeDataError, err.Error())
|
||||
return
|
||||
}
|
||||
common.SuccessWithData(c, gin.H{"total": total, "tree": items}, "success")
|
||||
}
|
||||
|
||||
// DELETE /skills — delete all skills.
|
||||
func (h *DatasetArtifactHandler) DeleteSkills(c *gin.Context) {
|
||||
_, tenantID, _ := h.datasetOwner(c, c.Param("dataset_id"))
|
||||
if tenantID == "" {
|
||||
return
|
||||
}
|
||||
n, err := h.svc.DeleteSkills(c.Request.Context(), tenantID, c.Param("dataset_id"), "")
|
||||
if err != nil {
|
||||
common.ErrorWithCode(c, common.CodeDataError, err.Error())
|
||||
return
|
||||
}
|
||||
common.SuccessWithData(c, gin.H{"deleted": n}, "success")
|
||||
}
|
||||
|
||||
// GET /skills/<skill_kwd> — single skill page.
|
||||
func (h *DatasetArtifactHandler) GetSkillPage(c *gin.Context) {
|
||||
_, tenantID, _ := h.datasetOwner(c, c.Param("dataset_id"))
|
||||
if tenantID == "" {
|
||||
return
|
||||
}
|
||||
detail, err := h.svc.GetSkillPage(c.Request.Context(), tenantID, c.Param("dataset_id"), c.Param("skill_kwd"))
|
||||
if err != nil {
|
||||
common.ErrorWithCode(c, common.CodeDataError, err.Error())
|
||||
return
|
||||
}
|
||||
if detail == nil {
|
||||
common.ErrorWithCode(c, common.CodeNotFound, "skill not found")
|
||||
return
|
||||
}
|
||||
common.SuccessWithData(c, detail, "success")
|
||||
}
|
||||
|
||||
// DELETE /skills/<skill_kwd> — delete a single skill.
|
||||
func (h *DatasetArtifactHandler) DeleteSkill(c *gin.Context) {
|
||||
_, tenantID, _ := h.datasetOwner(c, c.Param("dataset_id"))
|
||||
if tenantID == "" {
|
||||
return
|
||||
}
|
||||
n, err := h.svc.DeleteSkills(c.Request.Context(), tenantID, c.Param("dataset_id"), c.Param("skill_kwd"))
|
||||
if err != nil {
|
||||
common.ErrorWithCode(c, common.CodeDataError, err.Error())
|
||||
return
|
||||
}
|
||||
common.SuccessWithData(c, gin.H{"deleted": n}, "success")
|
||||
}
|
||||
|
||||
// GET /documents/<document_id>/structure/graph — document structure graph.
|
||||
func (h *DatasetArtifactHandler) GetDocumentGraph(c *gin.Context) {
|
||||
_, tenantID, _ := h.datasetOwner(c, c.Param("dataset_id"))
|
||||
if tenantID == "" {
|
||||
return
|
||||
}
|
||||
datasetID := c.Param("dataset_id")
|
||||
documentID := c.Param("document_id")
|
||||
graphType := c.Query("graph_type")
|
||||
items, total, err := h.svc.GetDocumentGraph(c.Request.Context(), tenantID, datasetID, documentID, graphType)
|
||||
if err != nil {
|
||||
common.ErrorWithCode(c, common.CodeDataError, err.Error())
|
||||
return
|
||||
}
|
||||
common.SuccessWithData(c, gin.H{"total": total, "graph": items}, "success")
|
||||
}
|
||||
|
||||
// DELETE /documents/<document_id>/structure/graph — delete document structure graph.
|
||||
func (h *DatasetArtifactHandler) DeleteDocumentGraph(c *gin.Context) {
|
||||
_, tenantID, _ := h.datasetOwner(c, c.Param("dataset_id"))
|
||||
if tenantID == "" {
|
||||
return
|
||||
}
|
||||
datasetID := c.Param("dataset_id")
|
||||
documentID := c.Param("document_id")
|
||||
n, err := h.svc.DeleteDocumentGraph(c.Request.Context(), tenantID, datasetID, documentID)
|
||||
if err != nil {
|
||||
common.ErrorWithCode(c, common.CodeDataError, err.Error())
|
||||
return
|
||||
}
|
||||
common.SuccessWithData(c, gin.H{"deleted": n}, "success")
|
||||
}
|
||||
@@ -38,6 +38,7 @@ type fileCommitService interface {
|
||||
GetCommitTree(ctx context.Context, commitID string) (map[string]interface{}, error)
|
||||
GetCommitFileContent(ctx context.Context, folderID, commitID, fileID string) ([]byte, error)
|
||||
GetFileVersionHistory(ctx context.Context, fileID string) ([]entity.VersionEntry, error)
|
||||
ListPageCommits(ctx context.Context, datasetID, pageType, slug string, page, pageSize int) ([]*entity.FileCommit, int64, error)
|
||||
}
|
||||
|
||||
// FileCommitHandler file commit handler
|
||||
@@ -217,6 +218,49 @@ func (h *FileCommitHandler) ListCommits(c *gin.Context) {
|
||||
}
|
||||
|
||||
ctx := c.Request.Context()
|
||||
|
||||
// Python's list_commits supports ?slug=<page_slug> to filter audit commits
|
||||
// for a specific wiki/skill page (written by record_page_edit). These page
|
||||
// commits are scoped to the dataset and page file key, so route them through
|
||||
// the page-commit path instead of the folder-based ListCommits. This only
|
||||
// applies to the /datasets/{dataset_id}/commits route.
|
||||
if slug := c.Query("slug"); slug != "" {
|
||||
datasetID := c.Param("dataset_id")
|
||||
if datasetID == "" {
|
||||
common.ErrorWithCode(c, common.CodeArgumentError, "slug requires a dataset scope")
|
||||
return
|
||||
}
|
||||
pageType := c.Query("page_type")
|
||||
commits, total, err := h.commitService.ListPageCommits(ctx, datasetID, pageType, slug, page, pageSize)
|
||||
if err != nil {
|
||||
jsonInternalError(c, err)
|
||||
return
|
||||
}
|
||||
var commitList []entity.CommitResponse
|
||||
for _, commit := range commits {
|
||||
var ct int64
|
||||
if commit.CreateTime != nil {
|
||||
ct = *commit.CreateTime
|
||||
}
|
||||
commitList = append(commitList, entity.CommitResponse{
|
||||
ID: commit.ID,
|
||||
FolderID: commit.FolderID,
|
||||
ParentID: commit.ParentID,
|
||||
Message: commit.Message,
|
||||
AuthorID: commit.AuthorID,
|
||||
FileCount: commit.FileCount,
|
||||
CreateTime: &ct,
|
||||
})
|
||||
}
|
||||
common.SuccessWithData(c, gin.H{
|
||||
"total": total,
|
||||
"page": page,
|
||||
"page_size": pageSize,
|
||||
"commits": commitList,
|
||||
}, common.CodeSuccess.Message())
|
||||
return
|
||||
}
|
||||
|
||||
commits, total, err := h.commitService.ListCommits(ctx, folderID, page, pageSize, orderBy, desc)
|
||||
if err != nil {
|
||||
jsonInternalError(c, err)
|
||||
|
||||
@@ -41,6 +41,7 @@ type mockFileCommitSvc struct {
|
||||
getCommitTreeFn func(ctx context.Context, commitID string) (map[string]interface{}, error)
|
||||
getCommitFileContentFn func(ctx context.Context, folderID, commitID, fileID string) ([]byte, error)
|
||||
getFileVersionHistoryFn func(ctx context.Context, fileID string) ([]entity.VersionEntry, error)
|
||||
listPageCommitsFn func(ctx context.Context, datasetID, pageType, slug string, page, pageSize int) ([]*entity.FileCommit, int64, error)
|
||||
}
|
||||
|
||||
func (m *mockFileCommitSvc) CreateCommit(ctx context.Context, folderID, authorID, message string, changes []entity.FileChange) (*entity.FileCommit, error) {
|
||||
@@ -128,6 +129,13 @@ func (m *mockFileCommitSvc) GetFileVersionHistory(ctx context.Context, fileID st
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (m *mockFileCommitSvc) ListPageCommits(ctx context.Context, datasetID, pageType, slug string, page, pageSize int) ([]*entity.FileCommit, int64, error) {
|
||||
if m.listPageCommitsFn != nil {
|
||||
return m.listPageCommitsFn(ctx, datasetID, pageType, slug, page, pageSize)
|
||||
}
|
||||
return []*entity.FileCommit{}, 0, nil
|
||||
}
|
||||
|
||||
func setupFileCommitTest(userID string) (*gin.Engine, *mockFileCommitSvc) {
|
||||
mock := &mockFileCommitSvc{}
|
||||
h := &FileCommitHandler{commitService: mock}
|
||||
|
||||
@@ -112,7 +112,7 @@ func (h *ProviderHandler) AddProvider(c *gin.Context) {
|
||||
}
|
||||
|
||||
func (h *ProviderHandler) DeleteProvider(c *gin.Context) {
|
||||
providerName := c.Param("provider_name")
|
||||
providerName := c.Param("provider_id_or_name")
|
||||
if providerName == "" {
|
||||
common.ResponseWithHttpCodeData(c, http.StatusBadRequest, 400, nil, "Provider name is required")
|
||||
return
|
||||
@@ -131,7 +131,7 @@ func (h *ProviderHandler) DeleteProvider(c *gin.Context) {
|
||||
}
|
||||
|
||||
func (h *ProviderHandler) ShowProvider(c *gin.Context) {
|
||||
providerName := c.Param("provider_name")
|
||||
providerName := c.Param("provider_id_or_name")
|
||||
if providerName == "" {
|
||||
common.ResponseWithHttpCodeData(c, http.StatusBadRequest, 400, nil, "Provider name is required")
|
||||
return
|
||||
@@ -146,7 +146,7 @@ func (h *ProviderHandler) ShowProvider(c *gin.Context) {
|
||||
}
|
||||
|
||||
func (h *ProviderHandler) ListModels(c *gin.Context) {
|
||||
providerName := c.Param("provider_name")
|
||||
providerName := c.Param("provider_id_or_name")
|
||||
if providerName == "" {
|
||||
common.ResponseWithHttpCodeData(c, http.StatusBadRequest, 400, nil, "Provider name is required")
|
||||
return
|
||||
@@ -221,7 +221,7 @@ func (h *ProviderHandler) ListModels(c *gin.Context) {
|
||||
}
|
||||
|
||||
func (h *ProviderHandler) ShowModel(c *gin.Context) {
|
||||
providerName := c.Param("provider_name")
|
||||
providerName := c.Param("provider_id_or_name")
|
||||
if providerName == "" {
|
||||
common.ResponseWithHttpCodeData(c, http.StatusBadRequest, 400, nil, "Provider name is required")
|
||||
return
|
||||
@@ -249,7 +249,7 @@ type CreateProviderInstanceRequest struct {
|
||||
}
|
||||
|
||||
func (h *ProviderHandler) CreateProviderInstance(c *gin.Context) {
|
||||
providerName := c.Param("provider_name")
|
||||
providerName := c.Param("provider_id_or_name")
|
||||
if providerName == "" {
|
||||
common.ResponseWithHttpCodeData(c, http.StatusBadRequest, 400, nil, "Provider name is required")
|
||||
return
|
||||
@@ -287,7 +287,7 @@ func (h *ProviderHandler) CreateProviderInstance(c *gin.Context) {
|
||||
}
|
||||
|
||||
func (h *ProviderHandler) ListProviderInstances(c *gin.Context) {
|
||||
providerName := c.Param("provider_name")
|
||||
providerName := c.Param("provider_id_or_name")
|
||||
if providerName == "" {
|
||||
common.ResponseWithHttpCodeData(c, http.StatusBadRequest, 400, nil, "Provider name is required")
|
||||
return
|
||||
@@ -306,13 +306,13 @@ func (h *ProviderHandler) ListProviderInstances(c *gin.Context) {
|
||||
}
|
||||
|
||||
func (h *ProviderHandler) ShowProviderInstance(c *gin.Context) {
|
||||
providerName := c.Param("provider_name")
|
||||
providerName := c.Param("provider_id_or_name")
|
||||
if providerName == "" {
|
||||
common.ResponseWithHttpCodeData(c, http.StatusBadRequest, 400, nil, "Provider name is required")
|
||||
return
|
||||
}
|
||||
|
||||
instanceIDOrName := c.Param("instance_name")
|
||||
instanceIDOrName := c.Param("instance_id_or_name")
|
||||
if instanceIDOrName == "" {
|
||||
common.ResponseWithHttpCodeData(c, http.StatusBadRequest, 400, nil, "Instance name is required")
|
||||
return
|
||||
@@ -331,13 +331,13 @@ func (h *ProviderHandler) ShowProviderInstance(c *gin.Context) {
|
||||
}
|
||||
|
||||
func (h *ProviderHandler) ShowInstanceBalance(c *gin.Context) {
|
||||
providerName := c.Param("provider_name")
|
||||
providerName := c.Param("provider_id_or_name")
|
||||
if providerName == "" {
|
||||
common.ResponseWithHttpCodeData(c, http.StatusBadRequest, 400, nil, "Provider name is required")
|
||||
return
|
||||
}
|
||||
|
||||
instanceName := c.Param("instance_name")
|
||||
instanceName := c.Param("instance_id_or_name")
|
||||
if instanceName == "" {
|
||||
common.ResponseWithHttpCodeData(c, http.StatusBadRequest, 400, nil, "Instance name is required")
|
||||
return
|
||||
@@ -357,7 +357,7 @@ func (h *ProviderHandler) ShowInstanceBalance(c *gin.Context) {
|
||||
}
|
||||
|
||||
func (h *ProviderHandler) CheckConnection(c *gin.Context) {
|
||||
providerName := c.Param("provider_name")
|
||||
providerName := c.Param("provider_id_or_name")
|
||||
if providerName == "" {
|
||||
common.ResponseWithHttpCodeData(c, http.StatusBadRequest, 400, nil, "Provider name is required")
|
||||
return
|
||||
@@ -382,13 +382,13 @@ func (h *ProviderHandler) CheckConnection(c *gin.Context) {
|
||||
|
||||
func (h *ProviderHandler) CheckInstanceConnection(c *gin.Context) {
|
||||
ctx := c.Request.Context()
|
||||
providerName := c.Param("provider_name")
|
||||
providerName := c.Param("provider_id_or_name")
|
||||
if providerName == "" {
|
||||
common.ResponseWithHttpCodeData(c, http.StatusBadRequest, 400, nil, "Provider name is required")
|
||||
return
|
||||
}
|
||||
|
||||
instanceName := c.Param("instance_name")
|
||||
instanceName := c.Param("instance_id_or_name")
|
||||
if instanceName == "" {
|
||||
common.ResponseWithHttpCodeData(c, http.StatusBadRequest, 400, nil, "Instance name is required")
|
||||
return
|
||||
@@ -417,13 +417,13 @@ func (h *ProviderHandler) CheckInstanceConnection(c *gin.Context) {
|
||||
|
||||
func (h *ProviderHandler) ListTasks(c *gin.Context) {
|
||||
ctx := c.Request.Context()
|
||||
providerName := c.Param("provider_name")
|
||||
providerName := c.Param("provider_id_or_name")
|
||||
if providerName == "" {
|
||||
common.ResponseWithHttpCodeData(c, http.StatusBadRequest, 400, nil, "Provider name is required")
|
||||
return
|
||||
}
|
||||
|
||||
instanceName := c.Param("instance_name")
|
||||
instanceName := c.Param("instance_id_or_name")
|
||||
if instanceName == "" {
|
||||
common.ResponseWithHttpCodeData(c, http.StatusBadRequest, 400, nil, "Instance name is required")
|
||||
return
|
||||
@@ -443,13 +443,13 @@ func (h *ProviderHandler) ListTasks(c *gin.Context) {
|
||||
|
||||
func (h *ProviderHandler) ShowTask(c *gin.Context) {
|
||||
ctx := c.Request.Context()
|
||||
providerName := c.Param("provider_name")
|
||||
providerName := c.Param("provider_id_or_name")
|
||||
if providerName == "" {
|
||||
common.ResponseWithHttpCodeData(c, http.StatusBadRequest, 400, nil, "Provider name is required")
|
||||
return
|
||||
}
|
||||
|
||||
instanceName := c.Param("instance_name")
|
||||
instanceName := c.Param("instance_id_or_name")
|
||||
if instanceName == "" {
|
||||
common.ResponseWithHttpCodeData(c, http.StatusBadRequest, 400, nil, "Instance name is required")
|
||||
return
|
||||
@@ -484,13 +484,13 @@ type AlterProviderInstanceRequest struct {
|
||||
|
||||
func (h *ProviderHandler) AlterProviderInstance(c *gin.Context) {
|
||||
ctx := c.Request.Context()
|
||||
providerName := c.Param("provider_name")
|
||||
providerName := c.Param("provider_id_or_name")
|
||||
if providerName == "" {
|
||||
common.ResponseWithHttpCodeData(c, http.StatusBadRequest, 400, nil, "Provider name is required")
|
||||
return
|
||||
}
|
||||
|
||||
instanceName := c.Param("instance_name")
|
||||
instanceName := c.Param("instance_id_or_name")
|
||||
if instanceName == "" {
|
||||
common.ResponseWithHttpCodeData(c, http.StatusBadRequest, 400, nil, "Instance name is required")
|
||||
return
|
||||
@@ -527,7 +527,7 @@ type DropProviderInstanceRequest struct {
|
||||
}
|
||||
|
||||
func (h *ProviderHandler) DropProviderInstance(c *gin.Context) {
|
||||
providerName := c.Param("provider_name")
|
||||
providerName := c.Param("provider_id_or_name")
|
||||
if providerName == "" {
|
||||
common.ResponseWithHttpCodeData(c, http.StatusBadRequest, 400, nil, "Provider name is required")
|
||||
return
|
||||
@@ -552,12 +552,12 @@ func (h *ProviderHandler) DropProviderInstance(c *gin.Context) {
|
||||
|
||||
func (h *ProviderHandler) ListInstanceModels(c *gin.Context) {
|
||||
ctx := c.Request.Context()
|
||||
providerName := c.Param("provider_name")
|
||||
providerName := c.Param("provider_id_or_name")
|
||||
if providerName == "" {
|
||||
common.ResponseWithHttpCodeData(c, http.StatusBadRequest, 400, nil, "Provider name is required")
|
||||
return
|
||||
}
|
||||
instanceName := c.Param("instance_name")
|
||||
instanceName := c.Param("instance_id_or_name")
|
||||
if instanceName == "" {
|
||||
common.ResponseWithHttpCodeData(c, http.StatusBadRequest, 400, nil, "Instance name is required")
|
||||
return
|
||||
@@ -600,13 +600,13 @@ type AlterModelRequest struct {
|
||||
}
|
||||
|
||||
func (h *ProviderHandler) AlterModel(c *gin.Context) {
|
||||
providerName := c.Param("provider_name")
|
||||
providerName := c.Param("provider_id_or_name")
|
||||
if providerName == "" {
|
||||
common.ResponseWithHttpCodeData(c, http.StatusBadRequest, 400, nil, "Provider name is required")
|
||||
return
|
||||
}
|
||||
|
||||
instanceName := c.Param("instance_name")
|
||||
instanceName := c.Param("instance_id_or_name")
|
||||
if instanceName == "" {
|
||||
common.ResponseWithHttpCodeData(c, http.StatusBadRequest, 400, nil, "Instance name is required")
|
||||
return
|
||||
@@ -691,8 +691,8 @@ func (h *ProviderHandler) AddModel(c *gin.Context) {
|
||||
return
|
||||
}
|
||||
|
||||
req.ProviderName = c.Param("provider_name")
|
||||
req.InstanceName = c.Param("instance_name")
|
||||
req.ProviderName = c.Param("provider_id_or_name")
|
||||
req.InstanceName = c.Param("instance_id_or_name")
|
||||
|
||||
if req.ProviderName == "" || req.InstanceName == "" {
|
||||
common.ResponseWithHttpCodeData(c, http.StatusBadRequest, common.CodeBadRequest, nil, "provider_name and instance_name are required")
|
||||
@@ -726,12 +726,12 @@ type DropInstanceModelRequest struct {
|
||||
}
|
||||
|
||||
func (h *ProviderHandler) DropInstanceModels(c *gin.Context) {
|
||||
providerName := c.Param("provider_name")
|
||||
providerName := c.Param("provider_id_or_name")
|
||||
if providerName == "" {
|
||||
common.ResponseWithHttpCodeData(c, http.StatusBadRequest, 400, nil, "Provider name is required")
|
||||
return
|
||||
}
|
||||
instanceName := c.Param("instance_name")
|
||||
instanceName := c.Param("instance_id_or_name")
|
||||
if instanceName == "" {
|
||||
common.ResponseWithHttpCodeData(c, http.StatusBadRequest, 400, nil, "Instance name is required")
|
||||
return
|
||||
|
||||
@@ -88,8 +88,8 @@ func TestProviderHandlerAlterModelRejectsMissingModelSelector(t *testing.T) {
|
||||
ctx, recorder := newProviderHandlerRequest(
|
||||
t,
|
||||
map[string]interface{}{"status": "active"},
|
||||
gin.Param{Key: "provider_name", Value: "OpenAI"},
|
||||
gin.Param{Key: "instance_name", Value: "default"},
|
||||
gin.Param{Key: "provider_id_or_name", Value: "OpenAI"},
|
||||
gin.Param{Key: "instance_id_or_name", Value: "default"},
|
||||
)
|
||||
|
||||
NewProviderHandler(nil, service.NewModelProviderService()).AlterModel(ctx)
|
||||
@@ -107,8 +107,8 @@ func TestProviderHandlerAlterModelRejectsInvalidStatus(t *testing.T) {
|
||||
ctx, recorder := newProviderHandlerRequest(
|
||||
t,
|
||||
map[string]interface{}{"status": "disabled"},
|
||||
gin.Param{Key: "provider_name", Value: "OpenAI"},
|
||||
gin.Param{Key: "instance_name", Value: "default"},
|
||||
gin.Param{Key: "provider_id_or_name", Value: "OpenAI"},
|
||||
gin.Param{Key: "instance_id_or_name", Value: "default"},
|
||||
gin.Param{Key: "model_name", Value: "gpt-test"},
|
||||
)
|
||||
|
||||
@@ -131,8 +131,8 @@ func TestProviderHandlerAlterModelUpdatesStatus(t *testing.T) {
|
||||
ctx, recorder := newProviderHandlerRequest(
|
||||
t,
|
||||
map[string]interface{}{"status": "inactive"},
|
||||
gin.Param{Key: "provider_name", Value: "OpenAI"},
|
||||
gin.Param{Key: "instance_name", Value: "default"},
|
||||
gin.Param{Key: "provider_id_or_name", Value: "OpenAI"},
|
||||
gin.Param{Key: "instance_id_or_name", Value: "default"},
|
||||
gin.Param{Key: "model_name", Value: "gpt-test"},
|
||||
)
|
||||
|
||||
|
||||
@@ -55,6 +55,10 @@ type Router struct {
|
||||
botHandler *handler.BotHandler
|
||||
componentsHandler *handler.ComponentsHandler
|
||||
pipelineHandler *handler.PipelineHandler
|
||||
|
||||
compilationTemplateHandler *handler.CompilationTemplateHandler
|
||||
compilationTemplateGroupHandler *handler.CompilationTemplateGroupHandler
|
||||
datasetArtifactHandler *handler.DatasetArtifactHandler
|
||||
}
|
||||
|
||||
// NewRouter create router
|
||||
@@ -90,6 +94,9 @@ func NewRouter(
|
||||
botHandler *handler.BotHandler,
|
||||
componentsHandler *handler.ComponentsHandler,
|
||||
pipelineHandler *handler.PipelineHandler,
|
||||
compilationTemplateHandler *handler.CompilationTemplateHandler,
|
||||
compilationTemplateGroupHandler *handler.CompilationTemplateGroupHandler,
|
||||
datasetArtifactHandler *handler.DatasetArtifactHandler,
|
||||
) *Router {
|
||||
return &Router{
|
||||
authHandler: authHandler,
|
||||
@@ -123,6 +130,10 @@ func NewRouter(
|
||||
botHandler: botHandler,
|
||||
componentsHandler: componentsHandler,
|
||||
pipelineHandler: pipelineHandler,
|
||||
|
||||
compilationTemplateHandler: compilationTemplateHandler,
|
||||
compilationTemplateGroupHandler: compilationTemplateGroupHandler,
|
||||
datasetArtifactHandler: datasetArtifactHandler,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -350,6 +361,32 @@ func (r *Router) Setup(engine *gin.Engine) {
|
||||
datasets.GET("/:dataset_id/index", r.datasetsHandler.TraceIndex)
|
||||
datasets.POST("/:dataset_id/index", r.datasetsHandler.RunIndex)
|
||||
datasets.DELETE("/:dataset_id/index", r.datasetsHandler.DeleteIndex)
|
||||
|
||||
// Knowledge-compilation wiki artifacts
|
||||
datasets.HEAD("/:dataset_id/artifacts", r.datasetArtifactHandler.AnyArtifact)
|
||||
datasets.GET("/:dataset_id/artifacts", r.datasetArtifactHandler.ListArtifacts)
|
||||
datasets.DELETE("/:dataset_id/artifacts", r.datasetArtifactHandler.DeleteArtifacts)
|
||||
datasets.GET("/:dataset_id/artifacts/topics", r.datasetArtifactHandler.ListArtifactTopics)
|
||||
datasets.GET("/:dataset_id/artifacts/alteration", r.datasetArtifactHandler.GetArtifactAlteration)
|
||||
datasets.GET("/:dataset_id/artifacts/graph", r.datasetArtifactHandler.GetArtifactGraph)
|
||||
datasets.GET("/:dataset_id/artifacts/:page_type/:slug", r.datasetArtifactHandler.GetArtifact)
|
||||
datasets.PUT("/:dataset_id/artifacts/:page_type/:slug", r.datasetArtifactHandler.UpdateArtifact)
|
||||
datasets.GET("/:dataset_id/artifacts/structure", r.datasetArtifactHandler.ListStructures)
|
||||
datasets.DELETE("/:dataset_id/artifacts/structure", r.datasetArtifactHandler.DeleteStructures)
|
||||
|
||||
// Knowledge-compilation navigation
|
||||
datasets.GET("/:dataset_id/navigation", r.datasetArtifactHandler.ListNavigation)
|
||||
datasets.DELETE("/:dataset_id/navigation", r.datasetArtifactHandler.DeleteNavigation)
|
||||
datasets.DELETE("/:dataset_id/navigation/:name", r.datasetArtifactHandler.DeleteNavigationNode)
|
||||
datasets.GET("/:dataset_id/navigation/:name/children", r.datasetArtifactHandler.ListNavigationChildren)
|
||||
|
||||
// Knowledge-compilation skills
|
||||
datasets.HEAD("/:dataset_id/skills", r.datasetArtifactHandler.AnySkill)
|
||||
datasets.GET("/:dataset_id/skills", r.datasetArtifactHandler.GetSkillTree)
|
||||
datasets.DELETE("/:dataset_id/skills", r.datasetArtifactHandler.DeleteSkills)
|
||||
datasets.GET("/:dataset_id/skills/:skill_kwd", r.datasetArtifactHandler.GetSkillPage)
|
||||
datasets.DELETE("/:dataset_id/skills/:skill_kwd", r.datasetArtifactHandler.DeleteSkill)
|
||||
|
||||
datasets.DELETE("/:dataset_id/:index_type", r.datasetsHandler.DeleteIndex)
|
||||
//datasets.DELETE("/:dataset_id/graph", r.datasetsHandler.DeleteKnowledgeGraph)
|
||||
datasets.POST("", r.datasetsHandler.CreateDataset)
|
||||
@@ -391,6 +428,8 @@ func (r *Router) Setup(engine *gin.Engine) {
|
||||
datasets.DELETE("/:dataset_id/chunks", r.chunkHandler.StopParsing)
|
||||
datasets.DELETE("/:dataset_id/documents/:document_id/chunks", r.chunkHandler.RemoveChunks)
|
||||
datasets.PUT("/:dataset_id/documents/:document_id/metadata/config", r.datasetsHandler.UpdateDocumentMetadataConfig)
|
||||
datasets.GET("/:dataset_id/documents/:document_id/structure/graph", r.datasetArtifactHandler.GetDocumentGraph)
|
||||
datasets.DELETE("/:dataset_id/documents/:document_id/structure/graph", r.datasetArtifactHandler.DeleteDocumentGraph)
|
||||
datasets.POST("/:dataset_id/metadata/update", r.documentHandler.MetadataBatchUpdate)
|
||||
datasets.PATCH("/:dataset_id/documents/metadatas", r.documentHandler.UpdateDocumentMetadatas)
|
||||
}
|
||||
@@ -517,24 +556,24 @@ func (r *Router) Setup(engine *gin.Engine) {
|
||||
{
|
||||
provider.GET("", r.providerHandler.ListProviders)
|
||||
provider.PUT("", r.providerHandler.AddProvider)
|
||||
provider.GET("/:provider_name", r.providerHandler.ShowProvider)
|
||||
provider.DELETE("/:provider_name", r.providerHandler.DeleteProvider)
|
||||
provider.GET("/:provider_name/models", r.providerHandler.ListModels)
|
||||
provider.GET("/:provider_name/models/:model_name", r.providerHandler.ShowModel)
|
||||
provider.POST("/:provider_name/instances", r.providerHandler.CreateProviderInstance)
|
||||
provider.GET("/:provider_name/instances", r.providerHandler.ListProviderInstances)
|
||||
provider.GET("/:provider_name/instances/:instance_name", r.providerHandler.ShowProviderInstance)
|
||||
provider.GET("/:provider_name/instances/:instance_name/balance", r.providerHandler.ShowInstanceBalance)
|
||||
provider.GET("/:provider_name/instances/:instance_name/connection", r.providerHandler.CheckInstanceConnection)
|
||||
provider.POST("/:provider_name/connection", r.providerHandler.CheckConnection)
|
||||
provider.GET("/:provider_name/instances/:instance_name/tasks", r.providerHandler.ListTasks)
|
||||
provider.GET("/:provider_name/instances/:instance_name/tasks/:task_id", r.providerHandler.ShowTask)
|
||||
provider.PUT("/:provider_name/instances/:instance_name", r.providerHandler.AlterProviderInstance)
|
||||
provider.DELETE("/:provider_name/instances", r.providerHandler.DropProviderInstance)
|
||||
provider.GET("/:provider_name/instances/:instance_name/models", r.providerHandler.ListInstanceModels)
|
||||
provider.PATCH("/:provider_name/instances/:instance_name/models/*model_name", r.providerHandler.AlterModel)
|
||||
provider.POST("/:provider_name/instances/:instance_name/models", r.providerHandler.AddModel)
|
||||
provider.DELETE("/:provider_name/instances/:instance_name/models", r.providerHandler.DropInstanceModels)
|
||||
provider.GET("/:provider_id_or_name", r.providerHandler.ShowProvider)
|
||||
provider.DELETE("/:provider_id_or_name", r.providerHandler.DeleteProvider)
|
||||
provider.GET("/:provider_id_or_name/models", r.providerHandler.ListModels)
|
||||
provider.GET("/:provider_id_or_name/models/:model_name", r.providerHandler.ShowModel)
|
||||
provider.POST("/:provider_id_or_name/instances", r.providerHandler.CreateProviderInstance)
|
||||
provider.GET("/:provider_id_or_name/instances", r.providerHandler.ListProviderInstances)
|
||||
provider.GET("/:provider_id_or_name/instances/:instance_id_or_name", r.providerHandler.ShowProviderInstance)
|
||||
provider.GET("/:provider_id_or_name/instances/:instance_id_or_name/balance", r.providerHandler.ShowInstanceBalance)
|
||||
provider.GET("/:provider_id_or_name/instances/:instance_id_or_name/connection", r.providerHandler.CheckInstanceConnection)
|
||||
provider.POST("/:provider_id_or_name/connection", r.providerHandler.CheckConnection)
|
||||
provider.GET("/:provider_id_or_name/instances/:instance_id_or_name/tasks", r.providerHandler.ListTasks)
|
||||
provider.GET("/:provider_id_or_name/instances/:instance_id_or_name/tasks/:task_id", r.providerHandler.ShowTask)
|
||||
provider.PUT("/:provider_id_or_name/instances/:instance_id_or_name", r.providerHandler.AlterProviderInstance)
|
||||
provider.DELETE("/:provider_id_or_name/instances", r.providerHandler.DropProviderInstance)
|
||||
provider.GET("/:provider_id_or_name/instances/:instance_id_or_name/models", r.providerHandler.ListInstanceModels)
|
||||
provider.PATCH("/:provider_id_or_name/instances/:instance_id_or_name/models/*model_name", r.providerHandler.AlterModel)
|
||||
provider.POST("/:provider_id_or_name/instances/:instance_id_or_name/models", r.providerHandler.AddModel)
|
||||
provider.DELETE("/:provider_id_or_name/instances/:instance_id_or_name/models", r.providerHandler.DropInstanceModels)
|
||||
v1.POST("/chat/to_model", r.providerHandler.ChatToModel)
|
||||
v1.POST("/embeddings", r.providerHandler.EmbedText)
|
||||
v1.POST("/rerank", r.providerHandler.RerankDocument)
|
||||
@@ -581,6 +620,17 @@ func (r *Router) Setup(engine *gin.Engine) {
|
||||
// runtime.DefaultRegistry.
|
||||
v1.GET("/components", r.componentsHandler.Get)
|
||||
|
||||
// Compilation template routes
|
||||
v1.GET("/compilation-templates/builtins", r.compilationTemplateHandler.ListBuiltins)
|
||||
v1.GET("/compilation-templates/wiki-presets", r.compilationTemplateHandler.ListWikiPresets)
|
||||
|
||||
// Compilation template group routes
|
||||
v1.GET("/compilation-template-groups", r.compilationTemplateGroupHandler.List)
|
||||
v1.POST("/compilation-template-groups", r.compilationTemplateGroupHandler.Save)
|
||||
v1.GET("/compilation-template-groups/:group_id", r.compilationTemplateGroupHandler.Get)
|
||||
v1.PUT("/compilation-template-groups/:group_id", r.compilationTemplateGroupHandler.Update)
|
||||
v1.DELETE("/compilation-template-groups/:group_id", r.compilationTemplateGroupHandler.Delete)
|
||||
|
||||
connectors := v1.Group("/connectors")
|
||||
{
|
||||
connectors.GET("", r.connectorHandler.ListConnectors)
|
||||
|
||||
494
internal/service/compilation_template_group_service.go
Normal file
494
internal/service/compilation_template_group_service.go
Normal file
@@ -0,0 +1,494 @@
|
||||
//
|
||||
// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
//
|
||||
|
||||
package service
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"ragflow/internal/dao"
|
||||
"ragflow/internal/entity"
|
||||
"ragflow/internal/utility"
|
||||
|
||||
"gorm.io/gorm"
|
||||
)
|
||||
|
||||
// timeStr formats a Unix-millisecond timestamp for the read-side JSON payloads.
|
||||
func timeStr(ms *int64) string {
|
||||
if ms == nil {
|
||||
return ""
|
||||
}
|
||||
return time.UnixMilli(*ms).UTC().Format("2006-01-02 15:04:05")
|
||||
}
|
||||
|
||||
const (
|
||||
groupScopeFile = "file"
|
||||
groupScopeDataset = "dataset"
|
||||
// maxGroupNameLen is the DB column size for compilation_template_group.name.
|
||||
maxGroupNameLen = 128
|
||||
)
|
||||
|
||||
// GroupTemplate is a child-template entry in a group create/update payload.
|
||||
type GroupTemplate struct {
|
||||
ID string `json:"id,omitempty"`
|
||||
Name string `json:"name,omitempty"`
|
||||
Description string `json:"description,omitempty"`
|
||||
Kind string `json:"kind,omitempty"`
|
||||
Config entity.JSONMap `json:"config,omitempty"`
|
||||
}
|
||||
|
||||
// GroupRequest is the create/update payload for a compilation template group.
|
||||
type GroupRequest struct {
|
||||
Name string `json:"name,omitempty"`
|
||||
Description string `json:"description,omitempty"`
|
||||
Templates []*GroupTemplate `json:"templates,omitempty"`
|
||||
}
|
||||
|
||||
// CompilationTemplateGroupService implements the knowledge-compilation template
|
||||
// group REST operations, mirroring the Python CompilationTemplateGroupService.
|
||||
type CompilationTemplateGroupService struct {
|
||||
groupDAO *dao.CompilationTemplateGroupDAO
|
||||
templateDAO *dao.CompilationTemplateDAO
|
||||
tenantDAO *dao.TenantDAO
|
||||
}
|
||||
|
||||
// NewCompilationTemplateGroupService creates a CompilationTemplateGroupService.
|
||||
func NewCompilationTemplateGroupService() *CompilationTemplateGroupService {
|
||||
return &CompilationTemplateGroupService{
|
||||
groupDAO: dao.NewCompilationTemplateGroupDAO(),
|
||||
templateDAO: dao.NewCompilationTemplateDAO(),
|
||||
tenantDAO: dao.NewTenantDAO(),
|
||||
}
|
||||
}
|
||||
|
||||
// GroupListItem is the read-side representation of a group with its nested
|
||||
// templates, mirroring Python _group_to_dict.
|
||||
type GroupListItem struct {
|
||||
ID string `json:"id"`
|
||||
Name string `json:"name"`
|
||||
Description string `json:"description"`
|
||||
Scope string `json:"scope"`
|
||||
CreateTime string `json:"create_time,omitempty"`
|
||||
UpdateTime string `json:"update_time,omitempty"`
|
||||
Templates []*TemplateListItem `json:"templates"`
|
||||
}
|
||||
|
||||
// ListSaved returns the tenant's groups with nested templates. Mirrors Python
|
||||
// list_saved(). total is derived from the full (unpaginated) result set.
|
||||
func (s *CompilationTemplateGroupService) ListSaved(ctx context.Context, tenantID, keywords, scope, orderby string, desc bool) ([]*GroupListItem, error) {
|
||||
groups, err := s.groupDAO.ListSaved(ctx, tenantID, keywords, scope, orderby, desc)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return s.buildGroupItems(ctx, tenantID, groups)
|
||||
}
|
||||
|
||||
// GetSaved returns a single group with its nested templates, or nil. Mirrors
|
||||
// Python get_saved().
|
||||
func (s *CompilationTemplateGroupService) GetSaved(ctx context.Context, tenantID, groupID string) (*GroupListItem, error) {
|
||||
group, err := s.groupDAO.GetSaved(ctx, tenantID, groupID)
|
||||
if err != nil || group == nil {
|
||||
return nil, err
|
||||
}
|
||||
return s.buildGroupItem(ctx, tenantID, group)
|
||||
}
|
||||
|
||||
// CreateGroup creates a group plus its child templates. Mirrors Python
|
||||
// create_group(). It returns a GroupValidationError for payload/scope problems.
|
||||
// The group and all child writes are committed atomically so a mid-way failure
|
||||
// cannot leave a partially-populated group behind.
|
||||
func (s *CompilationTemplateGroupService) CreateGroup(ctx context.Context, tenantID string, req *GroupRequest) (*GroupListItem, error) {
|
||||
if err := validateGroupPayload(req, true); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
scope, err := s.deriveScope(req.Templates)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
groupID := utility.GenerateUUID()
|
||||
if err := dao.DB.WithContext(ctx).Transaction(func(tx *gorm.DB) error {
|
||||
group := &entity.CompilationTemplateGroup{
|
||||
ID: groupID,
|
||||
TenantID: tenantID,
|
||||
Name: strings.TrimSpace(req.Name),
|
||||
Description: strptr(req.Description),
|
||||
Scope: scope,
|
||||
Status: strptr(string(entity.StatusValid)),
|
||||
}
|
||||
if cerr := s.groupDAO.Create(ctx, tx, group); cerr != nil {
|
||||
return cerr
|
||||
}
|
||||
return s.insertChildren(ctx, tx, tenantID, groupID, req.Templates, nil)
|
||||
}); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return s.GetSaved(ctx, tenantID, groupID)
|
||||
}
|
||||
|
||||
// UpdateGroup applies a partial update to a group and reconciles its child
|
||||
// templates. Mirrors Python update_group(). The field update and child
|
||||
// reconciliation are committed atomically.
|
||||
func (s *CompilationTemplateGroupService) UpdateGroup(ctx context.Context, tenantID, groupID string, req *GroupRequest) (*GroupListItem, error) {
|
||||
existing, err := s.groupDAO.GetSaved(ctx, tenantID, groupID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if existing == nil {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
if err := validateGroupPayload(req, false); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if err := dao.DB.WithContext(ctx).Transaction(func(tx *gorm.DB) error {
|
||||
updates := map[string]interface{}{}
|
||||
if strings.TrimSpace(req.Name) != "" {
|
||||
updates["name"] = strings.TrimSpace(req.Name)
|
||||
}
|
||||
if req.Description != "" {
|
||||
updates["description"] = req.Description
|
||||
}
|
||||
if req.Templates != nil {
|
||||
scope, serr := s.deriveScope(req.Templates)
|
||||
if serr != nil {
|
||||
return serr
|
||||
}
|
||||
updates["scope"] = scope
|
||||
}
|
||||
if len(updates) > 0 {
|
||||
if uerr := s.groupDAO.UpdateFields(ctx, tx, groupID, updates); uerr != nil {
|
||||
return uerr
|
||||
}
|
||||
}
|
||||
if req.Templates != nil {
|
||||
return s.reconcileChildren(ctx, tx, tenantID, groupID, req.Templates)
|
||||
}
|
||||
return nil
|
||||
}); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return s.GetSaved(ctx, tenantID, groupID)
|
||||
}
|
||||
|
||||
// DeleteGroup soft-deletes a group and its valid child templates. Mirrors
|
||||
// Python delete_group(). Returns (false, nil) when the group is missing.
|
||||
func (s *CompilationTemplateGroupService) DeleteGroup(ctx context.Context, tenantID, groupID string) (bool, error) {
|
||||
existing, err := s.groupDAO.GetSaved(ctx, tenantID, groupID)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
if existing == nil {
|
||||
return false, nil
|
||||
}
|
||||
if err := s.templateDAO.UpdateStatusByGroup(ctx, dao.DB, groupID, string(entity.StatusInvalid)); err != nil {
|
||||
return false, err
|
||||
}
|
||||
if err := s.groupDAO.Delete(ctx, dao.DB, tenantID, groupID); err != nil {
|
||||
return false, err
|
||||
}
|
||||
return true, nil
|
||||
}
|
||||
|
||||
// GroupValidationError is returned for group payload/scope problems so handlers
|
||||
// can map it to a 400 without an HTTP 500.
|
||||
type GroupValidationError struct{ msg string }
|
||||
|
||||
func (e *GroupValidationError) Error() string { return e.msg }
|
||||
|
||||
func groupValidationErrorf(format string, a ...interface{}) error {
|
||||
return &GroupValidationError{msg: fmt.Sprintf(format, a...)}
|
||||
}
|
||||
|
||||
// NameExists reports whether a valid group with the given name exists for the
|
||||
// tenant (optionally excluding excludeID), mirroring Python name_exists().
|
||||
func (s *CompilationTemplateGroupService) NameExists(ctx context.Context, tenantID, name string, excludeIDs ...string) (bool, error) {
|
||||
excludeID := ""
|
||||
if len(excludeIDs) > 0 {
|
||||
excludeID = excludeIDs[0]
|
||||
}
|
||||
return s.groupDAO.NameExists(ctx, tenantID, name, excludeID)
|
||||
}
|
||||
|
||||
// ValidateGroupRequest validates the fields present in a group create/update
|
||||
// payload before any DB write, mirroring the Python blueprint
|
||||
// _validate_group_payload. Required-field enforcement for create happens inside
|
||||
// CreateGroup (require_all=True).
|
||||
func ValidateGroupRequest(req *GroupRequest) error {
|
||||
return validateGroupPayload(req, false)
|
||||
}
|
||||
|
||||
// buildGroupItems loads child templates for many groups at once.
|
||||
func (s *CompilationTemplateGroupService) buildGroupItems(ctx context.Context, tenantID string, groups []*entity.CompilationTemplateGroup) ([]*GroupListItem, error) {
|
||||
items := make([]*GroupListItem, 0, len(groups))
|
||||
for _, g := range groups {
|
||||
item, err := s.buildGroupItem(ctx, tenantID, g)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
items = append(items, item)
|
||||
}
|
||||
return items, nil
|
||||
}
|
||||
|
||||
func (s *CompilationTemplateGroupService) buildGroupItem(ctx context.Context, tenantID string, group *entity.CompilationTemplateGroup) (*GroupListItem, error) {
|
||||
children, err := s.templateDAO.ListByGroup(ctx, dao.DB, group.ID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
items := make([]*TemplateListItem, 0, len(children))
|
||||
for _, c := range children {
|
||||
items = append(items, &TemplateListItem{
|
||||
ID: c.ID,
|
||||
Name: c.Name,
|
||||
Description: derefString(c.Description),
|
||||
Kind: c.Kind,
|
||||
Config: fillConfigDefaultLLM(ctx, s.tenantDAO, c.Config, c.TenantID),
|
||||
CreateTime: timeStr(c.CreateTime),
|
||||
UpdateTime: timeStr(c.UpdateTime),
|
||||
})
|
||||
}
|
||||
return &GroupListItem{
|
||||
ID: group.ID,
|
||||
Name: group.Name,
|
||||
Description: derefString(group.Description),
|
||||
Scope: group.Scope,
|
||||
CreateTime: timeStr(group.CreateTime),
|
||||
UpdateTime: timeStr(group.UpdateTime),
|
||||
Templates: items,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// deriveScope mirrors Python _derive_scope: one wiki child => dataset scope,
|
||||
// otherwise file scope (with a wiki-combination guard and re-chunk tree guard).
|
||||
func (s *CompilationTemplateGroupService) deriveScope(templates []*GroupTemplate) (string, error) {
|
||||
if len(templates) == 0 {
|
||||
return "", groupValidationErrorf("a template group must contain at least one template.")
|
||||
}
|
||||
artifactCount := 0
|
||||
for _, t := range templates {
|
||||
if strings.TrimSpace(t.Kind) == "wiki" {
|
||||
artifactCount++
|
||||
}
|
||||
}
|
||||
if artifactCount > 0 {
|
||||
if artifactCount != 1 || len(templates) != 1 {
|
||||
return "", groupValidationErrorf("a wiki template cannot be combined with other templates in the same group.")
|
||||
}
|
||||
return groupScopeDataset, nil
|
||||
}
|
||||
if err := s.enforceSingleRechunkTree(templates); err != nil {
|
||||
return "", err
|
||||
}
|
||||
return groupScopeFile, nil
|
||||
}
|
||||
|
||||
// enforceSingleRechunkTree mirrors Python _enforce_single_rechunk_tree.
|
||||
func (s *CompilationTemplateGroupService) enforceSingleRechunkTree(templates []*GroupTemplate) error {
|
||||
rechunkTrees := 0
|
||||
for _, t := range templates {
|
||||
if strings.TrimSpace(t.Kind) != "tree" {
|
||||
continue
|
||||
}
|
||||
raptor, _ := t.Config["raptor"].(map[string]interface{})
|
||||
if b, ok := raptor["rechunk"].(bool); ok && b {
|
||||
rechunkTrees++
|
||||
}
|
||||
}
|
||||
if rechunkTrees > 1 {
|
||||
return groupValidationErrorf("only one tree template in a group may enable re-chunking.")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// validateGroupPayload mirrors Python _validate_group_payload in the group
|
||||
// blueprint.
|
||||
func validateGroupPayload(req *GroupRequest, requireAll bool) error {
|
||||
if requireAll {
|
||||
if strings.TrimSpace(req.Name) == "" {
|
||||
return groupValidationErrorf("missing required field: name.")
|
||||
}
|
||||
if len(req.Templates) == 0 {
|
||||
return groupValidationErrorf("missing required field: templates.")
|
||||
}
|
||||
}
|
||||
if strings.TrimSpace(req.Name) != "" {
|
||||
if len([]byte(req.Name)) > maxGroupNameLen {
|
||||
return groupValidationErrorf("template group name is too long.")
|
||||
}
|
||||
}
|
||||
if len(req.Description) > 1024 {
|
||||
return groupValidationErrorf("invalid template group description.")
|
||||
}
|
||||
if req.Templates != nil {
|
||||
if len(req.Templates) == 0 {
|
||||
return groupValidationErrorf("a template group must contain at least one template.")
|
||||
}
|
||||
seen := map[string]struct{}{}
|
||||
for _, child := range req.Templates {
|
||||
if child == nil {
|
||||
return groupValidationErrorf("invalid template entry in group.")
|
||||
}
|
||||
payload := map[string]interface{}{
|
||||
"name": child.Name, "description": child.Description,
|
||||
"kind": child.Kind, "config": child.Config,
|
||||
}
|
||||
if err := ValidateTemplatePayload(payload, true); err != nil {
|
||||
return err
|
||||
}
|
||||
name := strings.TrimSpace(child.Name)
|
||||
if _, dup := seen[name]; dup {
|
||||
return groupValidationErrorf("template name '%s' is duplicated in this group.", name)
|
||||
}
|
||||
seen[name] = struct{}{}
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// insertChildren inserts new child templates. usedForReconcile controls whether
|
||||
// the duplicate-name guard applies (create always; reconcile passes seen set).
|
||||
func (s *CompilationTemplateGroupService) insertChildren(ctx context.Context, db *gorm.DB, tenantID, groupID string, templates []*GroupTemplate, seen map[string]struct{}) error {
|
||||
for _, child := range templates {
|
||||
name := strings.TrimSpace(child.Name)
|
||||
if exists, err := s.templateDAO.NameExistsInGroup(ctx, tenantID, groupID, name, ""); err != nil {
|
||||
return err
|
||||
} else if exists {
|
||||
return groupValidationErrorf("template name '%s' already exists in this group.", name)
|
||||
}
|
||||
desc := child.Description
|
||||
config := fillConfigDefaultLLM(ctx, s.tenantDAO, child.Config, &tenantID)
|
||||
if config == nil {
|
||||
config = entity.JSONMap{}
|
||||
}
|
||||
valid := string(entity.StatusValid)
|
||||
tmpl := &entity.CompilationTemplate{
|
||||
ID: utility.GenerateUUID(),
|
||||
TenantID: &tenantID,
|
||||
GroupID: &groupID,
|
||||
Name: name,
|
||||
Kind: strings.TrimSpace(child.Kind),
|
||||
Config: config,
|
||||
Status: &valid,
|
||||
}
|
||||
if desc != "" {
|
||||
tmpl.Description = &desc
|
||||
}
|
||||
if err := s.templateDAO.Save(ctx, db, tmpl); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// reconcileChildren mirrors the Python update_group child reconciliation:
|
||||
// existing children (matched by id, or by submitted order for legacy clients)
|
||||
// are updated in place, new ones inserted, and removed ones soft-deleted.
|
||||
func (s *CompilationTemplateGroupService) reconcileChildren(ctx context.Context, db *gorm.DB, tenantID, groupID string, templates []*GroupTemplate) error {
|
||||
current, err := s.templateDAO.ListByGroup(ctx, db, groupID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
currentByID := map[string]*entity.CompilationTemplate{}
|
||||
for _, c := range current {
|
||||
currentByID[c.ID] = c
|
||||
}
|
||||
seenNames := map[string]struct{}{}
|
||||
retained := map[string]struct{}{}
|
||||
|
||||
for index, child := range templates {
|
||||
name := strings.TrimSpace(child.Name)
|
||||
if _, dup := seenNames[name]; dup {
|
||||
return groupValidationErrorf("template name '%s' is duplicated in this group.", name)
|
||||
}
|
||||
seenNames[name] = struct{}{}
|
||||
|
||||
var target *entity.CompilationTemplate
|
||||
if child.ID != "" {
|
||||
target = currentByID[child.ID]
|
||||
if target == nil {
|
||||
return groupValidationErrorf("template %s does not belong to this group.", child.ID)
|
||||
}
|
||||
} else if index < len(current) {
|
||||
// Positional fallback for legacy clients that omit IDs. Skip rows
|
||||
// already retained by an explicit ID above so a mixed payload cannot
|
||||
// bind two submitted entries to one existing row.
|
||||
if _, taken := retained[current[index].ID]; !taken {
|
||||
target = current[index]
|
||||
}
|
||||
}
|
||||
|
||||
config := fillConfigDefaultLLM(ctx, s.tenantDAO, child.Config, &tenantID)
|
||||
if config == nil {
|
||||
config = entity.JSONMap{}
|
||||
}
|
||||
desc := child.Description
|
||||
if target != nil {
|
||||
updates := map[string]interface{}{
|
||||
"name": name, "kind": strings.TrimSpace(child.Kind), "config": config,
|
||||
}
|
||||
if desc != "" {
|
||||
updates["description"] = desc
|
||||
}
|
||||
if err := s.templateDAO.UpdateFields(ctx, db, target.ID, updates); err != nil {
|
||||
return err
|
||||
}
|
||||
retained[target.ID] = struct{}{}
|
||||
continue
|
||||
}
|
||||
newID := utility.GenerateUUID()
|
||||
valid := string(entity.StatusValid)
|
||||
tmpl := &entity.CompilationTemplate{
|
||||
ID: newID,
|
||||
TenantID: &tenantID,
|
||||
GroupID: &groupID,
|
||||
Name: name,
|
||||
Kind: strings.TrimSpace(child.Kind),
|
||||
Config: config,
|
||||
Status: &valid,
|
||||
}
|
||||
if desc != "" {
|
||||
tmpl.Description = &desc
|
||||
}
|
||||
if err := s.templateDAO.Save(ctx, db, tmpl); err != nil {
|
||||
return err
|
||||
}
|
||||
retained[newID] = struct{}{}
|
||||
}
|
||||
|
||||
for _, c := range current {
|
||||
if _, keep := retained[c.ID]; !keep {
|
||||
if err := s.templateDAO.UpdateStatusByID(ctx, db, c.ID, string(entity.StatusInvalid)); err != nil {
|
||||
return err
|
||||
}
|
||||
// Mirror Python _purge_stale_invalid_children: permanently drop
|
||||
// any stale invalid non-builtin template of the same name, scoped to
|
||||
// this group so a same-named template in another group is untouched.
|
||||
if c.Name != "" && !c.IsBuiltin {
|
||||
if err := s.templateDAO.HardDeleteOrphansByName(ctx, db, tenantID, groupID, c.Name); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func strptr(s string) *string { return &s }
|
||||
379
internal/service/compilation_template_service.go
Normal file
379
internal/service/compilation_template_service.go
Normal file
@@ -0,0 +1,379 @@
|
||||
//
|
||||
// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
//
|
||||
|
||||
package service
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"strings"
|
||||
"unicode"
|
||||
|
||||
"ragflow/internal/dao"
|
||||
"ragflow/internal/entity"
|
||||
"ragflow/internal/utility"
|
||||
|
||||
"gopkg.in/yaml.v3"
|
||||
"gorm.io/gorm"
|
||||
)
|
||||
|
||||
// fillConfigDefaultLLM mirrors Python CompilationTemplateService.
|
||||
// fill_config_default_llm: when the config has no explicit llm_id it lazily
|
||||
// fills the tenant's default chat model id (read-side only; the DB row is left
|
||||
// untouched).
|
||||
func fillConfigDefaultLLM(ctx context.Context, tenantDAO *dao.TenantDAO, config entity.JSONMap, tenantID *string) entity.JSONMap {
|
||||
if len(config) == 0 || tenantID == nil || *tenantID == "" {
|
||||
return config
|
||||
}
|
||||
if _, ok := config["llm_id"]; ok {
|
||||
return config
|
||||
}
|
||||
tenant, err := tenantDAO.GetByID(ctx, dao.DB, *tenantID)
|
||||
if err != nil || tenant == nil || tenant.TenantLLMID == nil {
|
||||
return config
|
||||
}
|
||||
out := make(entity.JSONMap, len(config)+1)
|
||||
for k, v := range config {
|
||||
out[k] = v
|
||||
}
|
||||
out["llm_id"] = *tenant.TenantLLMID
|
||||
return out
|
||||
}
|
||||
|
||||
// capitalizeTitle capitalizes the first letter of a lowercase section name for
|
||||
// error messages (replaces deprecated strings.Title).
|
||||
func capitalizeTitle(s string) string {
|
||||
if s == "" {
|
||||
return s
|
||||
}
|
||||
r := []rune(s)
|
||||
r[0] = unicode.ToUpper(r[0])
|
||||
return string(r)
|
||||
}
|
||||
|
||||
// TemplateListItem is the read-side representation of a compilation template,
|
||||
// mirroring Python CompilationTemplateService._to_saved_dict.
|
||||
type TemplateListItem struct {
|
||||
ID string `json:"id"`
|
||||
Name string `json:"name"`
|
||||
Description string `json:"description"`
|
||||
Kind string `json:"kind"`
|
||||
Config entity.JSONMap `json:"config"`
|
||||
CreateTime string `json:"create_time,omitempty"`
|
||||
UpdateTime string `json:"update_time,omitempty"`
|
||||
}
|
||||
|
||||
// BuiltinTemplate is the palette representation of a built-in template,
|
||||
// mirroring Python CompilationTemplateService._to_builtin_dict.
|
||||
type BuiltinTemplate struct {
|
||||
ID string `json:"id"`
|
||||
Kind string `json:"kind"`
|
||||
DisplayName string `json:"display_name"`
|
||||
Description string `json:"description"`
|
||||
Config entity.JSONMap `json:"config"`
|
||||
}
|
||||
|
||||
// WikiPreset is a wiki page-structure preset loaded from YAML, mirroring
|
||||
// Python load_wiki_presets_from_files.
|
||||
type WikiPreset struct {
|
||||
ID string `json:"id"`
|
||||
Topic string `json:"topic"`
|
||||
Instruction string `json:"instruction"`
|
||||
PageExample string `json:"page_example"`
|
||||
}
|
||||
|
||||
// CompilationTemplateService implements the read-side compilation template
|
||||
// operations (builtins palette + wiki presets) used by the REST APIs.
|
||||
type CompilationTemplateService struct {
|
||||
templateDAO *dao.CompilationTemplateDAO
|
||||
tenantDAO *dao.TenantDAO
|
||||
}
|
||||
|
||||
// NewCompilationTemplateService creates a CompilationTemplateService.
|
||||
func NewCompilationTemplateService() *CompilationTemplateService {
|
||||
return &CompilationTemplateService{
|
||||
templateDAO: dao.NewCompilationTemplateDAO(),
|
||||
tenantDAO: dao.NewTenantDAO(),
|
||||
}
|
||||
}
|
||||
|
||||
// ListBuiltins returns the built-in template palette with the tenant's default
|
||||
// LLM id lazily filled in, mirroring the Python list_builtins blueprint flow
|
||||
// (list from DB; if empty, seed from files and retry; then fill default LLM).
|
||||
func (s *CompilationTemplateService) ListBuiltins(ctx context.Context, tenantID string) ([]*BuiltinTemplate, error) {
|
||||
templates, err := s.templateDAO.ListBuiltins(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if len(templates) == 0 {
|
||||
if err := s.seedBuiltins(ctx); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
templates, err = s.templateDAO.ListBuiltins(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
out := make([]*BuiltinTemplate, 0, len(templates))
|
||||
for _, t := range templates {
|
||||
config := fillConfigDefaultLLM(ctx, s.tenantDAO, t.Config, t.TenantID)
|
||||
out = append(out, &BuiltinTemplate{
|
||||
ID: t.ID,
|
||||
Kind: t.Kind,
|
||||
DisplayName: t.Name,
|
||||
Description: derefString(t.Description),
|
||||
Config: config,
|
||||
})
|
||||
}
|
||||
return s.sortBuiltins(out), nil
|
||||
}
|
||||
|
||||
// LoadWikiPresets loads the wiki page-structure presets from the
|
||||
// init_data/compilation_templates/wiki/*.yaml files, filesystem-fresh per call.
|
||||
func (s *CompilationTemplateService) LoadWikiPresets() ([]*WikiPreset, error) {
|
||||
wikiDir := filepath.Join(utility.GetProjectBaseDirectory(),
|
||||
"api", "db", "init_data", "compilation_templates", "wiki")
|
||||
entries, err := os.ReadDir(wikiDir)
|
||||
if err != nil {
|
||||
if os.IsNotExist(err) {
|
||||
return []*WikiPreset{}, nil
|
||||
}
|
||||
return nil, err
|
||||
}
|
||||
var presets []*WikiPreset
|
||||
for _, entry := range entries {
|
||||
if entry.IsDir() || !strings.HasSuffix(entry.Name(), ".yaml") && !strings.HasSuffix(entry.Name(), ".yml") {
|
||||
continue
|
||||
}
|
||||
path := filepath.Join(wikiDir, entry.Name())
|
||||
data, rerr := os.ReadFile(path)
|
||||
if rerr != nil {
|
||||
continue
|
||||
}
|
||||
var doc map[string]interface{}
|
||||
if yerr := yaml.Unmarshal(data, &doc); yerr != nil || doc == nil {
|
||||
continue
|
||||
}
|
||||
presets = append(presets, &WikiPreset{
|
||||
ID: strings.TrimSuffix(entry.Name(), filepath.Ext(entry.Name())),
|
||||
Topic: strings.TrimSpace(yamlStr(doc["topic"])),
|
||||
Instruction: yamlStr(doc["instruction"]),
|
||||
PageExample: yamlStr(doc["page_example"]),
|
||||
})
|
||||
}
|
||||
return presets, nil
|
||||
}
|
||||
|
||||
// ValidateTemplatePayload validates a single template payload, mirroring the
|
||||
// Python compilation_template_validation module. It returns an error describing
|
||||
// the first problem found.
|
||||
func ValidateTemplatePayload(req map[string]interface{}, requireAll bool) error {
|
||||
if requireAll {
|
||||
for _, key := range []string{"name", "kind", "config"} {
|
||||
if _, ok := req[key]; !ok {
|
||||
return fmt.Errorf("missing required field: %s.", key)
|
||||
}
|
||||
}
|
||||
}
|
||||
if name, ok := req["name"]; ok {
|
||||
nameStr, ok2 := name.(string)
|
||||
if !ok2 || strings.TrimSpace(nameStr) == "" || len([]byte(nameStr)) > 128 {
|
||||
return errors.New("invalid template name.")
|
||||
}
|
||||
}
|
||||
if desc, ok := req["description"]; ok {
|
||||
if descStr, ok2 := desc.(string); !ok2 || len(descStr) > 1024 {
|
||||
return errors.New("invalid template description.")
|
||||
}
|
||||
}
|
||||
if kind, ok := req["kind"]; ok {
|
||||
if kindStr, ok2 := kind.(string); !ok2 || kindStr == "" {
|
||||
return errors.New("invalid template kind.")
|
||||
}
|
||||
}
|
||||
config, hasConfig := req["config"]
|
||||
if hasConfig {
|
||||
configMap, ok := config.(map[string]interface{})
|
||||
if !ok {
|
||||
return errors.New("invalid template config.")
|
||||
}
|
||||
if len(fmt.Sprint(configMap["global_rules"])) > 4096 {
|
||||
return errors.New("global compilation rules is too long.")
|
||||
}
|
||||
for _, section := range []string{"entity", "relation"} {
|
||||
sec, _ := configMap[section].(map[string]interface{})
|
||||
fields, _ := sec["fields"].([]interface{})
|
||||
seen := map[string]struct{}{}
|
||||
for _, f := range fields {
|
||||
fm, _ := f.(map[string]interface{})
|
||||
fieldType := strings.TrimSpace(yamlStr(fm["type"]))
|
||||
if fieldType == "" {
|
||||
return fmt.Errorf("%s type is required.", capitalizeTitle(section))
|
||||
}
|
||||
if _, dup := seen[fieldType]; dup {
|
||||
return fmt.Errorf("%s type can not be duplicated.", capitalizeTitle(section))
|
||||
}
|
||||
seen[fieldType] = struct{}{}
|
||||
if strings.TrimSpace(yamlStr(fm["description"])) == "" {
|
||||
return fmt.Errorf("%s field description is required.", capitalizeTitle(section))
|
||||
}
|
||||
if len(yamlStr(fm["description"])) > 1024 {
|
||||
return fmt.Errorf("%s field description is too long.", capitalizeTitle(section))
|
||||
}
|
||||
if len(yamlStr(fm["rule"])) > 1024 {
|
||||
return fmt.Errorf("%s field rule is too long.", capitalizeTitle(section))
|
||||
}
|
||||
}
|
||||
}
|
||||
if configMap["kind"] == "wiki" || req["kind"] == "wiki" {
|
||||
for _, group := range []string{"claim", "concept"} {
|
||||
sec, _ := configMap[group].(map[string]interface{})
|
||||
fields, _ := sec["fields"].([]interface{})
|
||||
for _, f := range fields {
|
||||
fm, _ := f.(map[string]interface{})
|
||||
switch group {
|
||||
case "claim":
|
||||
if strings.TrimSpace(yamlStr(fm["statement"])) == "" {
|
||||
return errors.New("claim statement is required.")
|
||||
}
|
||||
if strings.TrimSpace(yamlStr(fm["subject"])) == "" {
|
||||
return errors.New("claim subject is required.")
|
||||
}
|
||||
if len(yamlStr(fm["statement"])) > 1024 {
|
||||
return errors.New("claim statement is too long.")
|
||||
}
|
||||
if len(yamlStr(fm["subject"])) > 1024 {
|
||||
return errors.New("claim subject is too long.")
|
||||
}
|
||||
case "concept":
|
||||
if strings.TrimSpace(yamlStr(fm["term"])) == "" {
|
||||
return errors.New("concept term is required.")
|
||||
}
|
||||
if strings.TrimSpace(yamlStr(fm["definition_excerpt"])) == "" {
|
||||
return errors.New("concept definition excerpt is required.")
|
||||
}
|
||||
if len(yamlStr(fm["term"])) > 1024 {
|
||||
return errors.New("concept term is too long.")
|
||||
}
|
||||
if len(yamlStr(fm["definition_excerpt"])) > 1024 {
|
||||
return errors.New("concept definition excerpt is too long.")
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// sortBuiltins mirrors Python _sort_builtins: empty-kind entries first, then by
|
||||
// display name.
|
||||
func (s *CompilationTemplateService) sortBuiltins(templates []*BuiltinTemplate) []*BuiltinTemplate {
|
||||
sort.SliceStable(templates, func(i, j int) bool {
|
||||
emptyI := templates[i].Kind == "empty" || templates[i].ID == "empty"
|
||||
emptyJ := templates[j].Kind == "empty" || templates[j].ID == "empty"
|
||||
if emptyI != emptyJ {
|
||||
return emptyI
|
||||
}
|
||||
return strings.ToLower(templates[i].DisplayName) < strings.ToLower(templates[j].DisplayName)
|
||||
})
|
||||
return templates
|
||||
}
|
||||
|
||||
// seedBuiltins loads the built-in templates from the init_data YAML files and
|
||||
// upserts them (as a safety net when the DB has no built-ins yet), mirroring
|
||||
// Python seed_builtins_from_files.
|
||||
func (s *CompilationTemplateService) seedBuiltins(ctx context.Context) error {
|
||||
dir := filepath.Join(utility.GetProjectBaseDirectory(),
|
||||
"api", "db", "init_data", "compilation_templates")
|
||||
entries, err := os.ReadDir(dir)
|
||||
if err != nil {
|
||||
return nil // directory absent is acceptable; nothing to seed
|
||||
}
|
||||
for _, entry := range entries {
|
||||
if entry.IsDir() || !strings.HasSuffix(entry.Name(), ".yaml") && !strings.HasSuffix(entry.Name(), ".yml") {
|
||||
continue
|
||||
}
|
||||
data, rerr := os.ReadFile(filepath.Join(dir, entry.Name()))
|
||||
if rerr != nil {
|
||||
continue
|
||||
}
|
||||
var doc struct {
|
||||
Kind string `yaml:"kind"`
|
||||
DisplayName string `yaml:"display_name"`
|
||||
Description string `yaml:"description"`
|
||||
Config map[string]interface{} `yaml:"config"`
|
||||
}
|
||||
if yerr := yaml.Unmarshal(data, &doc); yerr != nil {
|
||||
continue
|
||||
}
|
||||
if doc.Kind == "" || doc.DisplayName == "" || doc.Config == nil {
|
||||
continue
|
||||
}
|
||||
id := strings.TrimSuffix(entry.Name(), filepath.Ext(entry.Name()))
|
||||
desc := doc.Description
|
||||
valid := string(entity.StatusValid)
|
||||
tmpl := &entity.CompilationTemplate{
|
||||
ID: id,
|
||||
Name: doc.DisplayName,
|
||||
Kind: doc.Kind,
|
||||
Config: entity.JSONMap(doc.Config),
|
||||
IsBuiltin: true,
|
||||
Status: &valid,
|
||||
}
|
||||
if doc.Description != "" {
|
||||
tmpl.Description = &desc
|
||||
}
|
||||
_ = s.upsertBuiltin(ctx, tmpl)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *CompilationTemplateService) upsertBuiltin(ctx context.Context, t *entity.CompilationTemplate) error {
|
||||
existing, err := s.templateDAO.GetByID(ctx, t.ID)
|
||||
if errors.Is(err, gorm.ErrRecordNotFound) {
|
||||
return s.templateDAO.Save(ctx, dao.DB, t)
|
||||
}
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
m := map[string]interface{}{
|
||||
"name": t.Name, "kind": t.Kind, "config": t.Config,
|
||||
"description": t.Description, "is_builtin": true,
|
||||
"status": t.Status,
|
||||
}
|
||||
return s.templateDAO.UpdateFields(ctx, dao.DB, existing.ID, m)
|
||||
}
|
||||
|
||||
func derefString(p *string) string {
|
||||
if p == nil {
|
||||
return ""
|
||||
}
|
||||
return *p
|
||||
}
|
||||
|
||||
// yamlStr safely stringifies a YAML value, returning "" when the key is absent
|
||||
// (instead of fmt.Sprint(nil) which yields "<nil>").
|
||||
func yamlStr(v interface{}) string {
|
||||
if v == nil {
|
||||
return ""
|
||||
}
|
||||
return fmt.Sprint(v)
|
||||
}
|
||||
846
internal/service/dataset_artifact_service.go
Normal file
846
internal/service/dataset_artifact_service.go
Normal file
@@ -0,0 +1,846 @@
|
||||
//
|
||||
// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
//
|
||||
|
||||
package service
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"sort"
|
||||
|
||||
"ragflow/internal/engine"
|
||||
"ragflow/internal/engine/types"
|
||||
)
|
||||
|
||||
// Compile keyword constants used by the knowledge-compilation artifacts stored
|
||||
// in the document engine.
|
||||
const (
|
||||
CompileKwdWikiPage = "wiki_page"
|
||||
CompileKwdWikiEntity = "wiki_entity"
|
||||
CompileKwdWikiRelation = "wiki_relation"
|
||||
CompileKwdWikiAlter = "wiki_alter"
|
||||
CompileKwdSkill = "skill"
|
||||
CompileKwdSkillAll = "skill_all"
|
||||
CompileKwdDatasetNav = "dataset_nav"
|
||||
CompileKwdRaptorGraph = "raptor_graph"
|
||||
|
||||
// Structure / graph compilation keywords.
|
||||
CompileKwdStructure = "structure"
|
||||
CompileKwdStructureIndex = "structureIndex"
|
||||
CompileKwdStructureEntity = "structureEntity"
|
||||
CompileKwdStructureRelation = "structureRelation"
|
||||
CompileKwdStructureCommunity = "structureCommunity"
|
||||
|
||||
// Field name for the structure index type discriminator.
|
||||
FieldStructureIndexType = "structure_index_type"
|
||||
FieldStructureKind = "structure_kind"
|
||||
FieldPageID = "page_id"
|
||||
FieldGraphType = "graph_type"
|
||||
)
|
||||
|
||||
// DatasetArtifactService reads knowledge-compilation artifacts (wiki pages,
|
||||
// graphs, structures, navigation, skills) from the document engine.
|
||||
type DatasetArtifactService struct{}
|
||||
|
||||
// NewDatasetArtifactService creates a DatasetArtifactService.
|
||||
func NewDatasetArtifactService() *DatasetArtifactService {
|
||||
return &DatasetArtifactService{}
|
||||
}
|
||||
|
||||
// wikiIndexName returns the tenant document index name.
|
||||
func wikiIndexName(tenantID string) string {
|
||||
return fmt.Sprintf("ragflow_%s", tenantID)
|
||||
}
|
||||
|
||||
// searchCompiled runs a filtered search over the tenant's document index for
|
||||
// the given dataset, returning the matching chunks.
|
||||
func (s *DatasetArtifactService) searchCompiled(ctx context.Context, tenantID, datasetID string, filter map[string]interface{}, selectFields []string, offset, limit int, orderBy *types.OrderByExpr) ([]map[string]interface{}, int64, error) {
|
||||
docEngine := engine.Get()
|
||||
if docEngine == nil {
|
||||
return nil, 0, fmt.Errorf("document engine is not initialized")
|
||||
}
|
||||
// Copy caller filters first, then pin the dataset scope so kb_id always wins
|
||||
// even if a caller passes its own kb_id.
|
||||
merged := make(map[string]interface{}, len(filter)+1)
|
||||
for k, v := range filter {
|
||||
merged[k] = v
|
||||
}
|
||||
merged["kb_id"] = []string{datasetID}
|
||||
res, err := docEngine.Search(ctx, &types.SearchRequest{
|
||||
IndexNames: []string{wikiIndexName(tenantID)},
|
||||
KbIDs: []string{datasetID},
|
||||
Offset: offset,
|
||||
Limit: limit,
|
||||
SelectFields: selectFields,
|
||||
Filter: merged,
|
||||
OrderBy: orderBy,
|
||||
})
|
||||
if err != nil {
|
||||
return nil, 0, err
|
||||
}
|
||||
if res == nil {
|
||||
return nil, 0, nil
|
||||
}
|
||||
return res.Chunks, res.Total, nil
|
||||
}
|
||||
|
||||
// intValue coerces an engine field value into an int, tolerating the numeric
|
||||
// types a document engine may decode a number as (float64, int, int64,
|
||||
// json.Number) as well as a string form and a list-valued form (first element
|
||||
// wins). It returns 0 for anything else.
|
||||
func intValue(v interface{}) int {
|
||||
if v == nil {
|
||||
return 0
|
||||
}
|
||||
switch n := v.(type) {
|
||||
case int:
|
||||
return n
|
||||
case int64:
|
||||
return int(n)
|
||||
case int32:
|
||||
return int(n)
|
||||
case float64:
|
||||
return int(n)
|
||||
case float32:
|
||||
return int(n)
|
||||
case json.Number:
|
||||
if iv, err := n.Int64(); err == nil {
|
||||
return int(iv)
|
||||
}
|
||||
if fv, err := n.Float64(); err == nil {
|
||||
return int(fv)
|
||||
}
|
||||
case string:
|
||||
var iv int
|
||||
if _, err := fmt.Sscanf(n, "%d", &iv); err == nil {
|
||||
return iv
|
||||
}
|
||||
case []interface{}:
|
||||
if len(n) > 0 {
|
||||
return intValue(n[0])
|
||||
}
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
// HasWiki reports whether the dataset has any compiled wiki artifact.
|
||||
func (s *DatasetArtifactService) HasWiki(ctx context.Context, tenantID, datasetID string) (bool, error) {
|
||||
_, total, err := s.searchCompiled(ctx, tenantID, datasetID,
|
||||
map[string]interface{}{"compile_kwd": []string{CompileKwdWikiPage}}, nil, 0, 1, nil)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
return total > 0, nil
|
||||
}
|
||||
|
||||
// HasSkill reports whether the dataset has any compiled skill artifact.
|
||||
func (s *DatasetArtifactService) HasSkill(ctx context.Context, tenantID, datasetID string) (bool, error) {
|
||||
_, total, err := s.searchCompiled(ctx, tenantID, datasetID,
|
||||
map[string]interface{}{"compile_kwd": []string{CompileKwdSkillAll}}, nil, 0, 1, nil)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
return total > 0, nil
|
||||
}
|
||||
|
||||
// WikiPageItem is a single wiki page summary returned by ListWikiPages.
|
||||
type WikiPageItem struct {
|
||||
Slug string `json:"slug"`
|
||||
Title string `json:"title"`
|
||||
PageType string `json:"page_type"`
|
||||
Topic string `json:"topic"`
|
||||
Summary string `json:"summary"`
|
||||
}
|
||||
|
||||
// ListWikiPages lists wiki pages for a dataset with optional page_type/topic
|
||||
// filters and pagination.
|
||||
func (s *DatasetArtifactService) ListWikiPages(ctx context.Context, tenantID, datasetID, pageType, topic string, page, pageSize int) ([]WikiPageItem, int64, error) {
|
||||
filter := map[string]interface{}{"compile_kwd": []string{CompileKwdWikiPage}}
|
||||
if pageType != "" {
|
||||
filter["page_type_kwd"] = []string{pageType}
|
||||
}
|
||||
if topic != "" {
|
||||
filter["topic_kwd"] = []string{topic}
|
||||
}
|
||||
offset := (page - 1) * pageSize
|
||||
chunks, total, err := s.searchCompiled(ctx, tenantID, datasetID, filter,
|
||||
[]string{"slug_kwd", "title_kwd", "page_type_kwd", "topic_kwd", "outlinks_int", "summary_with_weight"},
|
||||
offset, pageSize, (&types.OrderByExpr{}).Desc("outlinks_int").Asc("title_kwd"))
|
||||
if err != nil {
|
||||
return nil, 0, err
|
||||
}
|
||||
items := make([]WikiPageItem, 0, len(chunks))
|
||||
for _, c := range chunks {
|
||||
items = append(items, WikiPageItem{
|
||||
Slug: firstStringValue(c["slug_kwd"]),
|
||||
Title: firstStringValue(c["title_kwd"]),
|
||||
PageType: firstStringValue(c["page_type_kwd"]),
|
||||
Topic: firstStringValue(c["topic_kwd"]),
|
||||
Summary: firstStringValue(c["summary_with_weight"]),
|
||||
})
|
||||
}
|
||||
return items, total, nil
|
||||
}
|
||||
|
||||
// WikiPageDetail is the full wiki page payload.
|
||||
type WikiPageDetail struct {
|
||||
Slug string `json:"slug"`
|
||||
Title string `json:"title"`
|
||||
PageType string `json:"page_type"`
|
||||
Topic string `json:"topic"`
|
||||
ContentMd string `json:"content_md"`
|
||||
Summary string `json:"summary"`
|
||||
EntityNames []string `json:"entity_names"`
|
||||
Outlinks []string `json:"outlinks"`
|
||||
RelatedKbPages []string `json:"related_kb_pages"`
|
||||
SourceChunkIDs []string `json:"source_chunk_ids"`
|
||||
SourceDocIDs []string `json:"source_doc_ids"`
|
||||
}
|
||||
|
||||
// GetWikiPage returns a single wiki page by page_type and slug.
|
||||
func (s *DatasetArtifactService) GetWikiPage(ctx context.Context, tenantID, datasetID, pageType, slug string) (*WikiPageDetail, error) {
|
||||
slugKwd := pageType + "/" + slug
|
||||
filter := map[string]interface{}{
|
||||
"compile_kwd": []string{CompileKwdWikiPage},
|
||||
"page_type_kwd": []string{pageType},
|
||||
"slug_kwd": []string{slugKwd},
|
||||
}
|
||||
chunks, _, err := s.searchCompiled(ctx, tenantID, datasetID, filter,
|
||||
[]string{"slug_kwd", "title_kwd", "page_type_kwd", "topic_kwd", "content_with_weight",
|
||||
"summary_with_weight", "entity_names_kwd", "outlinks_kwd", "related_kb_pages_kwd",
|
||||
"source_chunk_ids", "source_doc_ids"},
|
||||
0, 1, nil)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if len(chunks) == 0 {
|
||||
return nil, nil
|
||||
}
|
||||
c := chunks[0]
|
||||
detail := &WikiPageDetail{
|
||||
Slug: firstStringValue(c["slug_kwd"]),
|
||||
Title: firstStringValue(c["title_kwd"]),
|
||||
PageType: firstStringValue(c["page_type_kwd"]),
|
||||
Topic: firstStringValue(c["topic_kwd"]),
|
||||
ContentMd: firstStringValue(c["content_with_weight"]),
|
||||
Summary: firstStringValue(c["summary_with_weight"]),
|
||||
EntityNames: toStringSlice(c["entity_names_kwd"]),
|
||||
Outlinks: toStringSlice(c["outlinks_kwd"]),
|
||||
RelatedKbPages: toStringSlice(c["related_kb_pages_kwd"]),
|
||||
SourceChunkIDs: toStringSlice(c["source_chunk_ids"]),
|
||||
SourceDocIDs: toStringSlice(c["source_doc_ids"]),
|
||||
}
|
||||
return detail, nil
|
||||
}
|
||||
|
||||
// UpdateWikiPage performs a partial field update of a wiki page's content,
|
||||
// title and outlinks through the document engine, then returns the refreshed
|
||||
// page.
|
||||
func (s *DatasetArtifactService) UpdateWikiPage(ctx context.Context, tenantID, datasetID, pageType, slug, contentMd, title string, outlinks []string) (*WikiPageDetail, error) {
|
||||
docEngine := engine.Get()
|
||||
if docEngine == nil {
|
||||
return nil, fmt.Errorf("document engine is not initialized")
|
||||
}
|
||||
slugKwd := pageType + "/" + slug
|
||||
filter := map[string]interface{}{
|
||||
"compile_kwd": []string{CompileKwdWikiPage},
|
||||
"page_type_kwd": []string{pageType},
|
||||
"slug_kwd": []string{slugKwd},
|
||||
}
|
||||
chunks, _, err := s.searchCompiled(ctx, tenantID, datasetID, filter, []string{"id"}, 0, 1, nil)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if len(chunks) == 0 {
|
||||
return nil, nil
|
||||
}
|
||||
id, ok := chunks[0]["id"].(string)
|
||||
if !ok || id == "" {
|
||||
return nil, fmt.Errorf("wiki page chunk has no id")
|
||||
}
|
||||
update := map[string]interface{}{}
|
||||
if contentMd != "" {
|
||||
update["content_with_weight"] = contentMd
|
||||
}
|
||||
if title != "" {
|
||||
update["title_kwd"] = title
|
||||
}
|
||||
if len(outlinks) > 0 {
|
||||
update["outlinks_kwd"] = outlinks
|
||||
}
|
||||
if len(update) > 0 {
|
||||
cond := map[string]interface{}{"id": id, "kb_id": datasetID}
|
||||
if err := docEngine.UpdateChunks(ctx, cond, update, wikiIndexName(tenantID), datasetID); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
return s.GetWikiPage(ctx, tenantID, datasetID, pageType, slug)
|
||||
}
|
||||
|
||||
// WikiTopicItem is a single topic entry returned by ListWikiTopics.
|
||||
type WikiTopicItem struct {
|
||||
Topic string `json:"topic"`
|
||||
Title string `json:"title"`
|
||||
Slug string `json:"slug"`
|
||||
PageCount int `json:"page_count"`
|
||||
}
|
||||
|
||||
// ListWikiTopics aggregates wiki topics for a dataset.
|
||||
func (s *DatasetArtifactService) ListWikiTopics(ctx context.Context, tenantID, datasetID string) ([]WikiTopicItem, int64, error) {
|
||||
filter := map[string]interface{}{
|
||||
"compile_kwd": []string{CompileKwdWikiPage},
|
||||
"page_type_kwd": []string{"concept", "entity"},
|
||||
}
|
||||
chunks, _, err := s.searchCompiled(ctx, tenantID, datasetID, filter,
|
||||
[]string{"topic_kwd", "title_kwd", "slug_kwd"}, 0, 1000, nil)
|
||||
if err != nil {
|
||||
return nil, 0, err
|
||||
}
|
||||
counts := map[string]int{}
|
||||
metas := map[string]WikiTopicItem{}
|
||||
for _, c := range chunks {
|
||||
t := firstStringValue(c["topic_kwd"])
|
||||
if t == "" {
|
||||
continue
|
||||
}
|
||||
counts[t]++
|
||||
if _, ok := metas[t]; !ok {
|
||||
metas[t] = WikiTopicItem{
|
||||
Topic: t,
|
||||
Title: firstStringValue(c["title_kwd"]),
|
||||
Slug: firstStringValue(c["slug_kwd"]),
|
||||
}
|
||||
}
|
||||
}
|
||||
items := make([]WikiTopicItem, 0, len(metas))
|
||||
for t, it := range metas {
|
||||
it.PageCount = counts[t]
|
||||
items = append(items, it)
|
||||
}
|
||||
sort.Slice(items, func(i, j int) bool { return items[i].Topic < items[j].Topic })
|
||||
return items, int64(len(items)), nil
|
||||
}
|
||||
|
||||
// WikiGraph is the entity/relation graph for a dataset's wiki artifacts.
|
||||
type WikiGraph struct {
|
||||
Entities []WikiGraphEntity `json:"entities"`
|
||||
Relations []WikiGraphRelation `json:"relations"`
|
||||
}
|
||||
|
||||
// WikiGraphEntity is a single wiki graph entity.
|
||||
type WikiGraphEntity struct {
|
||||
Slug string `json:"slug"`
|
||||
Name string `json:"name"`
|
||||
Aliases []string `json:"aliases"`
|
||||
Description string `json:"description"`
|
||||
Type string `json:"type"`
|
||||
Weight int `json:"weight"`
|
||||
SourceChunkIDs []string `json:"source_chunk_ids"`
|
||||
}
|
||||
|
||||
// WikiGraphRelation is a single wiki graph relation.
|
||||
type WikiGraphRelation struct {
|
||||
From string `json:"from"`
|
||||
To string `json:"to"`
|
||||
}
|
||||
|
||||
// GetWikiGraph returns the wiki entity/relation graph for a dataset.
|
||||
func (s *DatasetArtifactService) GetWikiGraph(ctx context.Context, tenantID, datasetID string) (*WikiGraph, error) {
|
||||
entityChunks, _, err := s.searchCompiled(ctx, tenantID, datasetID,
|
||||
map[string]interface{}{"compile_kwd": []string{CompileKwdWikiEntity}},
|
||||
[]string{"slug_kwd", "title_kwd", "aliases_kwd", "description_with_weight", "entity_type_kwd", "weight_int", "source_chunk_ids"},
|
||||
0, 1000, (&types.OrderByExpr{}).Desc("weight_int"))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
fromKwds := make([]string, 0, len(entityChunks))
|
||||
for _, c := range entityChunks {
|
||||
fromKwds = append(fromKwds, firstStringValue(c["slug_kwd"]))
|
||||
}
|
||||
graph := &WikiGraph{Entities: []WikiGraphEntity{}, Relations: []WikiGraphRelation{}}
|
||||
for _, c := range entityChunks {
|
||||
w := intValue(c["weight_int"])
|
||||
graph.Entities = append(graph.Entities, WikiGraphEntity{
|
||||
Slug: firstStringValue(c["slug_kwd"]),
|
||||
Name: firstStringValue(c["title_kwd"]),
|
||||
Aliases: toStringSlice(c["aliases_kwd"]),
|
||||
Description: firstStringValue(c["description_with_weight"]),
|
||||
Type: firstStringValue(c["entity_type_kwd"]),
|
||||
Weight: w,
|
||||
SourceChunkIDs: toStringSlice(c["source_chunk_ids"]),
|
||||
})
|
||||
}
|
||||
if len(fromKwds) > 0 {
|
||||
relChunks, _, err := s.searchCompiled(ctx, tenantID, datasetID,
|
||||
map[string]interface{}{"compile_kwd": []string{CompileKwdWikiRelation}, "from_kwd": fromKwds},
|
||||
[]string{"from_kwd", "to_kwd"}, 0, 10000, nil)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
for _, c := range relChunks {
|
||||
graph.Relations = append(graph.Relations, WikiGraphRelation{
|
||||
From: firstStringValue(c["from_kwd"]),
|
||||
To: firstStringValue(c["to_kwd"]),
|
||||
})
|
||||
}
|
||||
}
|
||||
return graph, nil
|
||||
}
|
||||
|
||||
// WikiAlteration is the alteration summary for a dataset's wiki artifacts.
|
||||
type WikiAlteration struct {
|
||||
Removed int `json:"removed"`
|
||||
NewlyUploaded int `json:"newly_uploaded"`
|
||||
RemovedDocIDs []string `json:"removed_doc_ids"`
|
||||
NewlyUploadedDocIDs []string `json:"newly_uploaded_doc_ids"`
|
||||
InvolvedDocIDs []string `json:"involved_doc_ids"`
|
||||
EligibleDocIDs []string `json:"eligible_doc_ids"`
|
||||
}
|
||||
|
||||
// GetWikiAlteration returns the wiki alteration summary for a dataset.
|
||||
func (s *DatasetArtifactService) GetWikiAlteration(ctx context.Context, tenantID, datasetID string) (*WikiAlteration, error) {
|
||||
chunks, _, err := s.searchCompiled(ctx, tenantID, datasetID,
|
||||
map[string]interface{}{"compile_kwd": []string{CompileKwdWikiPage}},
|
||||
[]string{"source_doc_ids"}, 0, 10000, nil)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
involved := map[string]struct{}{}
|
||||
for _, c := range chunks {
|
||||
for _, d := range toStringSlice(c["source_doc_ids"]) {
|
||||
involved[d] = struct{}{}
|
||||
}
|
||||
}
|
||||
ids := make([]string, 0, len(involved))
|
||||
for d := range involved {
|
||||
ids = append(ids, d)
|
||||
}
|
||||
sort.Strings(ids)
|
||||
return &WikiAlteration{
|
||||
Removed: 0,
|
||||
NewlyUploaded: 0,
|
||||
RemovedDocIDs: []string{},
|
||||
NewlyUploadedDocIDs: []string{},
|
||||
InvolvedDocIDs: ids,
|
||||
EligibleDocIDs: ids,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// ClearWiki deletes all wiki artifacts for a dataset.
|
||||
func (s *DatasetArtifactService) ClearWiki(ctx context.Context, tenantID, datasetID string) (map[string]int, error) {
|
||||
docEngine := engine.Get()
|
||||
if docEngine == nil {
|
||||
return nil, fmt.Errorf("document engine is not initialized")
|
||||
}
|
||||
kwds := []string{CompileKwdWikiPage, CompileKwdWikiEntity, CompileKwdWikiRelation, CompileKwdWikiAlter}
|
||||
deleted := map[string]int{}
|
||||
for _, kwd := range kwds {
|
||||
chunks, _, err := s.searchCompiled(ctx, tenantID, datasetID,
|
||||
map[string]interface{}{"compile_kwd": []string{kwd}}, []string{"id"}, 0, 10000, nil)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if len(chunks) == 0 {
|
||||
deleted[kwd] = 0
|
||||
continue
|
||||
}
|
||||
ids := make([]string, 0, len(chunks))
|
||||
for _, c := range chunks {
|
||||
if id, ok := c["id"].(string); ok {
|
||||
ids = append(ids, id)
|
||||
}
|
||||
}
|
||||
cond := map[string]interface{}{"id": ids, "kb_id": datasetID}
|
||||
if _, err := docEngine.DeleteChunks(ctx, cond, wikiIndexName(tenantID), datasetID); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
deleted[kwd] = len(ids)
|
||||
}
|
||||
return deleted, nil
|
||||
}
|
||||
|
||||
// StructureItem is a single compiled structure entry for a dataset.
|
||||
type StructureItem struct {
|
||||
PageID string `json:"page_id"`
|
||||
StructureKind string `json:"structure_kind"`
|
||||
StructureIndexType string `json:"structure_index_type"`
|
||||
Data string `json:"data"`
|
||||
}
|
||||
|
||||
// ListStructures returns the compiled structures of a dataset, filtered by
|
||||
// optional structure_kind and structure_index_type.
|
||||
func (s *DatasetArtifactService) ListStructures(ctx context.Context, tenantID, datasetID, structureKind, structureIndexType string) ([]StructureItem, int64, error) {
|
||||
filter := map[string]interface{}{"compile_kwd": []string{CompileKwdStructure}}
|
||||
if structureKind != "" {
|
||||
filter[FieldStructureKind] = []string{structureKind}
|
||||
}
|
||||
if structureIndexType != "" {
|
||||
filter[FieldStructureIndexType] = []string{structureIndexType}
|
||||
}
|
||||
chunks, total, err := s.searchCompiled(ctx, tenantID, datasetID, filter,
|
||||
[]string{FieldPageID, FieldStructureKind, FieldStructureIndexType, "content_with_weight"}, 0, 10000, nil)
|
||||
if err != nil {
|
||||
return nil, 0, err
|
||||
}
|
||||
items := make([]StructureItem, 0, len(chunks))
|
||||
for _, c := range chunks {
|
||||
items = append(items, StructureItem{
|
||||
PageID: firstStringValue(c[FieldPageID]),
|
||||
StructureKind: firstStringValue(c[FieldStructureKind]),
|
||||
StructureIndexType: firstStringValue(c[FieldStructureIndexType]),
|
||||
Data: firstStringValue(c["content_with_weight"]),
|
||||
})
|
||||
}
|
||||
return items, total, nil
|
||||
}
|
||||
|
||||
// DeleteStructures deletes the compiled structures of a dataset, optionally
|
||||
// scoped by structure_kind and structure_index_type.
|
||||
func (s *DatasetArtifactService) DeleteStructures(ctx context.Context, tenantID, datasetID, structureKind, structureIndexType string) (int, error) {
|
||||
docEngine := engine.Get()
|
||||
if docEngine == nil {
|
||||
return 0, fmt.Errorf("document engine is not initialized")
|
||||
}
|
||||
filter := map[string]interface{}{"compile_kwd": []string{CompileKwdStructure}}
|
||||
if structureKind != "" {
|
||||
filter[FieldStructureKind] = []string{structureKind}
|
||||
}
|
||||
if structureIndexType != "" {
|
||||
filter[FieldStructureIndexType] = []string{structureIndexType}
|
||||
}
|
||||
chunks, _, err := s.searchCompiled(ctx, tenantID, datasetID, filter, []string{"id"}, 0, 10000, nil)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
if len(chunks) == 0 {
|
||||
return 0, nil
|
||||
}
|
||||
ids := make([]string, 0, len(chunks))
|
||||
for _, c := range chunks {
|
||||
if id, ok := c["id"].(string); ok {
|
||||
ids = append(ids, id)
|
||||
}
|
||||
}
|
||||
cond := map[string]interface{}{"id": ids, "kb_id": datasetID}
|
||||
if _, err := docEngine.DeleteChunks(ctx, cond, wikiIndexName(tenantID), datasetID); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return len(ids), nil
|
||||
}
|
||||
|
||||
// DocGraphItem is a single node/edge entry in a document's structure graph.
|
||||
type DocGraphItem struct {
|
||||
ID string `json:"id"`
|
||||
Content string `json:"content"`
|
||||
SourceID string `json:"source_id"`
|
||||
}
|
||||
|
||||
// GetDocumentGraph returns the structure graph of a single document.
|
||||
func (s *DatasetArtifactService) GetDocumentGraph(ctx context.Context, tenantID, datasetID, documentID, graphType string) ([]DocGraphItem, int64, error) {
|
||||
filter := map[string]interface{}{
|
||||
"doc_id": []string{documentID},
|
||||
"compiled_graph_kwd": []string{"graph"},
|
||||
}
|
||||
if graphType != "" {
|
||||
filter[FieldGraphType] = []string{graphType}
|
||||
}
|
||||
chunks, total, err := s.searchCompiled(ctx, tenantID, datasetID, filter,
|
||||
[]string{"id", "content_with_weight", "source_id"}, 0, 10000, nil)
|
||||
if err != nil {
|
||||
return nil, 0, err
|
||||
}
|
||||
items := make([]DocGraphItem, 0, len(chunks))
|
||||
for _, c := range chunks {
|
||||
items = append(items, DocGraphItem{
|
||||
ID: firstStringValue(c["id"]),
|
||||
Content: firstStringValue(c["content_with_weight"]),
|
||||
SourceID: firstStringValue(c["source_id"]),
|
||||
})
|
||||
}
|
||||
return items, total, nil
|
||||
}
|
||||
|
||||
// DeleteDocumentGraph deletes the structure graph of a single document.
|
||||
func (s *DatasetArtifactService) DeleteDocumentGraph(ctx context.Context, tenantID, datasetID, documentID string) (int, error) {
|
||||
docEngine := engine.Get()
|
||||
if docEngine == nil {
|
||||
return 0, fmt.Errorf("document engine is not initialized")
|
||||
}
|
||||
filter := map[string]interface{}{
|
||||
"doc_id": []string{documentID},
|
||||
"compiled_graph_kwd": []string{"graph"},
|
||||
}
|
||||
chunks, _, err := s.searchCompiled(ctx, tenantID, datasetID, filter, []string{"id"}, 0, 10000, nil)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
if len(chunks) == 0 {
|
||||
return 0, nil
|
||||
}
|
||||
ids := make([]string, 0, len(chunks))
|
||||
for _, c := range chunks {
|
||||
if id, ok := c["id"].(string); ok {
|
||||
ids = append(ids, id)
|
||||
}
|
||||
}
|
||||
cond := map[string]interface{}{"id": ids, "kb_id": datasetID}
|
||||
if _, err := docEngine.DeleteChunks(ctx, cond, wikiIndexName(tenantID), datasetID); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return len(ids), nil
|
||||
}
|
||||
|
||||
// NavigationItem is a single navigation cluster.
|
||||
type NavigationItem struct {
|
||||
Name string `json:"name"`
|
||||
Title string `json:"title"`
|
||||
Count int `json:"count"`
|
||||
}
|
||||
|
||||
// ListNavClusters returns the navigation clusters of a dataset.
|
||||
func (s *DatasetArtifactService) ListNavClusters(ctx context.Context, tenantID, datasetID string) ([]NavigationItem, int64, error) {
|
||||
chunks, total, err := s.searchCompiled(ctx, tenantID, datasetID,
|
||||
map[string]interface{}{"compile_kwd": []string{CompileKwdDatasetNav}},
|
||||
[]string{"nav_cluster_kwd", "title_kwd", "count_int"}, 0, 10000, nil)
|
||||
if err != nil {
|
||||
return nil, 0, err
|
||||
}
|
||||
items := make([]NavigationItem, 0, len(chunks))
|
||||
for _, c := range chunks {
|
||||
count := intValue(c["count_int"])
|
||||
items = append(items, NavigationItem{
|
||||
Name: firstStringValue(c["nav_cluster_kwd"]),
|
||||
Title: firstStringValue(c["title_kwd"]),
|
||||
Count: count,
|
||||
})
|
||||
}
|
||||
return items, total, nil
|
||||
}
|
||||
|
||||
// DeleteNav deletes all navigation clusters of a dataset.
|
||||
func (s *DatasetArtifactService) DeleteNav(ctx context.Context, tenantID, datasetID string) (int, error) {
|
||||
docEngine := engine.Get()
|
||||
if docEngine == nil {
|
||||
return 0, fmt.Errorf("document engine is not initialized")
|
||||
}
|
||||
chunks, _, err := s.searchCompiled(ctx, tenantID, datasetID,
|
||||
map[string]interface{}{"compile_kwd": []string{CompileKwdDatasetNav}}, []string{"id"}, 0, 10000, nil)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
if len(chunks) == 0 {
|
||||
return 0, nil
|
||||
}
|
||||
ids := make([]string, 0, len(chunks))
|
||||
for _, c := range chunks {
|
||||
if id, ok := c["id"].(string); ok {
|
||||
ids = append(ids, id)
|
||||
}
|
||||
}
|
||||
cond := map[string]interface{}{"id": ids, "kb_id": datasetID}
|
||||
if _, err := docEngine.DeleteChunks(ctx, cond, wikiIndexName(tenantID), datasetID); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return len(ids), nil
|
||||
}
|
||||
|
||||
// DeleteNavNode deletes a single navigation cluster by name.
|
||||
func (s *DatasetArtifactService) DeleteNavNode(ctx context.Context, tenantID, datasetID, name string) (int, error) {
|
||||
docEngine := engine.Get()
|
||||
if docEngine == nil {
|
||||
return 0, fmt.Errorf("document engine is not initialized")
|
||||
}
|
||||
chunks, _, err := s.searchCompiled(ctx, tenantID, datasetID,
|
||||
map[string]interface{}{"compile_kwd": []string{CompileKwdDatasetNav}, "nav_cluster_kwd": []string{name}},
|
||||
[]string{"id"}, 0, 10000, nil)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
if len(chunks) == 0 {
|
||||
return 0, nil
|
||||
}
|
||||
ids := make([]string, 0, len(chunks))
|
||||
for _, c := range chunks {
|
||||
if id, ok := c["id"].(string); ok {
|
||||
ids = append(ids, id)
|
||||
}
|
||||
}
|
||||
cond := map[string]interface{}{"id": ids, "kb_id": datasetID}
|
||||
if _, err := docEngine.DeleteChunks(ctx, cond, wikiIndexName(tenantID), datasetID); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return len(ids), nil
|
||||
}
|
||||
|
||||
// NavChildItem is a single child entry under a navigation cluster.
|
||||
type NavChildItem struct {
|
||||
Name string `json:"name"`
|
||||
Title string `json:"title"`
|
||||
Count int `json:"count"`
|
||||
}
|
||||
|
||||
// ListNavChildren returns the children of a navigation cluster.
|
||||
func (s *DatasetArtifactService) ListNavChildren(ctx context.Context, tenantID, datasetID, name string) ([]NavChildItem, int64, error) {
|
||||
chunks, total, err := s.searchCompiled(ctx, tenantID, datasetID,
|
||||
map[string]interface{}{"compile_kwd": []string{CompileKwdDatasetNav}, "nav_cluster_kwd": []string{name}},
|
||||
[]string{"nav_child_kwd", "title_kwd", "count_int"}, 0, 10000, nil)
|
||||
if err != nil {
|
||||
return nil, 0, err
|
||||
}
|
||||
items := make([]NavChildItem, 0, len(chunks))
|
||||
for _, c := range chunks {
|
||||
count := intValue(c["count_int"])
|
||||
items = append(items, NavChildItem{
|
||||
Name: firstStringValue(c["nav_child_kwd"]),
|
||||
Title: firstStringValue(c["title_kwd"]),
|
||||
Count: count,
|
||||
})
|
||||
}
|
||||
return items, total, nil
|
||||
}
|
||||
|
||||
// SkillTreeItem is a single skill-tree page summary.
|
||||
type SkillTreeItem struct {
|
||||
Kwd string `json:"kwd"`
|
||||
Title string `json:"title"`
|
||||
PageType string `json:"page_type"`
|
||||
Outlinks int `json:"outlinks"`
|
||||
Inlinks int `json:"inlinks"`
|
||||
}
|
||||
|
||||
// GetSkillTree returns the skill tree of a dataset.
|
||||
func (s *DatasetArtifactService) GetSkillTree(ctx context.Context, tenantID, datasetID, kwd string) ([]SkillTreeItem, int64, error) {
|
||||
filter := map[string]interface{}{"compile_kwd": []string{CompileKwdSkillAll}}
|
||||
if kwd != "" {
|
||||
filter["kwd"] = []string{kwd}
|
||||
}
|
||||
chunks, total, err := s.searchCompiled(ctx, tenantID, datasetID, filter,
|
||||
[]string{"kwd", "title_kwd", "page_type_kwd", "outlinks_int", "inlinks_int"}, 0, 10000, nil)
|
||||
if err != nil {
|
||||
return nil, 0, err
|
||||
}
|
||||
items := make([]SkillTreeItem, 0, len(chunks))
|
||||
for _, c := range chunks {
|
||||
outlinks, inlinks := intValue(c["outlinks_int"]), intValue(c["inlinks_int"])
|
||||
items = append(items, SkillTreeItem{
|
||||
Kwd: firstStringValue(c["kwd"]),
|
||||
Title: firstStringValue(c["title_kwd"]),
|
||||
PageType: firstStringValue(c["page_type_kwd"]),
|
||||
Outlinks: outlinks,
|
||||
Inlinks: inlinks,
|
||||
})
|
||||
}
|
||||
return items, total, nil
|
||||
}
|
||||
|
||||
// SkillPageDetail is the full skill page payload.
|
||||
type SkillPageDetail struct {
|
||||
Kwd string `json:"kwd"`
|
||||
Title string `json:"title"`
|
||||
ContentMd string `json:"content_md"`
|
||||
Outlinks []string `json:"outlinks"`
|
||||
Inlinks []string `json:"inlinks"`
|
||||
}
|
||||
|
||||
// GetSkillPage returns a single skill page by keyword.
|
||||
func (s *DatasetArtifactService) GetSkillPage(ctx context.Context, tenantID, datasetID, kwd string) (*SkillPageDetail, error) {
|
||||
chunks, _, err := s.searchCompiled(ctx, tenantID, datasetID,
|
||||
map[string]interface{}{"compile_kwd": []string{CompileKwdSkillAll}, "kwd": []string{kwd}},
|
||||
[]string{"kwd", "title_kwd", "content_with_weight", "outlinks_kwd", "inlinks_kwd"}, 0, 1, nil)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if len(chunks) == 0 {
|
||||
return nil, nil
|
||||
}
|
||||
c := chunks[0]
|
||||
return &SkillPageDetail{
|
||||
Kwd: firstStringValue(c["kwd"]),
|
||||
Title: firstStringValue(c["title_kwd"]),
|
||||
ContentMd: firstStringValue(c["content_with_weight"]),
|
||||
Outlinks: toStringSlice(c["outlinks_kwd"]),
|
||||
Inlinks: toStringSlice(c["inlinks_kwd"]),
|
||||
}, nil
|
||||
}
|
||||
|
||||
// DeleteSkills deletes skills of a dataset, optionally scoped by keyword.
|
||||
func (s *DatasetArtifactService) DeleteSkills(ctx context.Context, tenantID, datasetID, kwd string) (int, error) {
|
||||
docEngine := engine.Get()
|
||||
if docEngine == nil {
|
||||
return 0, fmt.Errorf("document engine is not initialized")
|
||||
}
|
||||
filter := map[string]interface{}{"compile_kwd": []string{CompileKwdSkillAll}}
|
||||
if kwd != "" {
|
||||
filter["kwd"] = []string{kwd}
|
||||
}
|
||||
chunks, _, err := s.searchCompiled(ctx, tenantID, datasetID, filter, []string{"id"}, 0, 10000, nil)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
if len(chunks) == 0 {
|
||||
return 0, nil
|
||||
}
|
||||
ids := make([]string, 0, len(chunks))
|
||||
for _, c := range chunks {
|
||||
if id, ok := c["id"].(string); ok {
|
||||
ids = append(ids, id)
|
||||
}
|
||||
}
|
||||
cond := map[string]interface{}{"id": ids, "kb_id": datasetID}
|
||||
if _, err := docEngine.DeleteChunks(ctx, cond, wikiIndexName(tenantID), datasetID); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return len(ids), nil
|
||||
}
|
||||
|
||||
// firstStringValue returns the first string in a possibly-list ES field value.
|
||||
func firstStringValue(v interface{}) string {
|
||||
switch t := v.(type) {
|
||||
case string:
|
||||
return t
|
||||
case []interface{}:
|
||||
if len(t) > 0 {
|
||||
if s, ok := t[0].(string); ok {
|
||||
return s
|
||||
}
|
||||
}
|
||||
case []string:
|
||||
if len(t) > 0 {
|
||||
return t[0]
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// toStringSlice normalizes an ES field value (string or list) to []string.
|
||||
func toStringSlice(v interface{}) []string {
|
||||
switch t := v.(type) {
|
||||
case string:
|
||||
if t == "" {
|
||||
return []string{}
|
||||
}
|
||||
return []string{t}
|
||||
case []interface{}:
|
||||
out := make([]string, 0, len(t))
|
||||
for _, e := range t {
|
||||
if s, ok := e.(string); ok {
|
||||
out = append(out, s)
|
||||
}
|
||||
}
|
||||
return out
|
||||
case []string:
|
||||
return t
|
||||
}
|
||||
return []string{}
|
||||
}
|
||||
@@ -28,6 +28,8 @@ import (
|
||||
"ragflow/internal/storage"
|
||||
"ragflow/internal/utility"
|
||||
"sort"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"go.uber.org/zap"
|
||||
@@ -233,6 +235,209 @@ func (s *FileCommitService) CreateCommit(ctx context.Context, folderID, authorID
|
||||
return commit, nil
|
||||
}
|
||||
|
||||
// PageEditCommitInput carries the data needed to record a single wiki/skill
|
||||
// page edit as an audit commit.
|
||||
type PageEditCommitInput struct {
|
||||
DatasetID string // knowledgebase scope, stored on the commit for isolation
|
||||
DocID string // ES doc id of the page content
|
||||
Slug string
|
||||
PageType string
|
||||
Title string
|
||||
AuthorID string
|
||||
OldContent string
|
||||
NewContent string
|
||||
}
|
||||
|
||||
// wikiFileID derives the stable file key used to scope page-edit commits to a
|
||||
// specific knowledgebase and page, so identical slugs in different
|
||||
// knowledgebases never share a commit parent or history.
|
||||
func wikiFileID(datasetID, pageType, slug string) string {
|
||||
return datasetID + "/" + pageType + "/" + slug
|
||||
}
|
||||
|
||||
// RecordPageEdit records a wiki/skill page edit as an audit commit with a
|
||||
// git-style parent chain (each edit points at the previous commit for the same
|
||||
// page). The new content_after is referenced in ES by doc_id; a unified diff of
|
||||
// old vs new content is stored on the commit item. The commit is scoped to the
|
||||
// dataset via FolderID and a derived page file key so page histories never cross
|
||||
// knowledgebase boundaries.
|
||||
//
|
||||
// This path is independent of the workspace File tree (it does not require a
|
||||
// File record or a tree_state snapshot).
|
||||
func (s *FileCommitService) RecordPageEdit(ctx context.Context, in PageEditCommitInput) (*entity.FileCommit, error) {
|
||||
// Parent chain: previous commit that touched the same page file key.
|
||||
fileID := wikiFileID(in.DatasetID, in.PageType, in.Slug)
|
||||
|
||||
commitID := utility.GenerateUUID()
|
||||
|
||||
diffText := unifiedDiff(in.OldContent, in.NewContent)
|
||||
contentAfterStorage := "es"
|
||||
contentAfterLocation := in.DocID
|
||||
slugKwd := in.Slug
|
||||
pageTypeKwd := in.PageType
|
||||
|
||||
item := &entity.FileCommitItem{
|
||||
ID: utility.GenerateUUID(),
|
||||
CommitID: commitID,
|
||||
FileID: fileID,
|
||||
Operation: "modify",
|
||||
Diff: &diffText,
|
||||
ContentAfterStorage: &contentAfterStorage,
|
||||
ContentAfterLocation: &contentAfterLocation,
|
||||
SlugKwd: &slugKwd,
|
||||
PageTypeKwd: &pageTypeKwd,
|
||||
}
|
||||
|
||||
var commit *entity.FileCommit
|
||||
// Serialize parent selection with insertion in process so two concurrent
|
||||
// edits on the same page cannot both read the same parent and fork the
|
||||
// chain. This is backend-agnostic (works identically on MySQL and SQLite)
|
||||
// and cheaper than row-level DB locks.
|
||||
mu := pageCommitLock(fileID)
|
||||
mu.Lock()
|
||||
defer mu.Unlock()
|
||||
|
||||
// Read the parent inside the lock, on the shared connection, so it always
|
||||
// reflects the previously committed edit for this page.
|
||||
parentID, perr := s.commitItemDAO.GetLatestCommitIDByFileID(ctx, dao.DB, fileID)
|
||||
if perr != nil {
|
||||
return nil, fmt.Errorf("failed to resolve page commit parent: %w", perr)
|
||||
}
|
||||
|
||||
commit = &entity.FileCommit{
|
||||
ID: commitID,
|
||||
FolderID: in.DatasetID,
|
||||
Message: in.Title,
|
||||
AuthorID: in.AuthorID,
|
||||
Title: &in.Title,
|
||||
FileCount: 1,
|
||||
}
|
||||
if parentID != "" {
|
||||
commit.ParentID = &parentID
|
||||
}
|
||||
|
||||
if err := dao.DB.WithContext(ctx).Transaction(func(tx *gorm.DB) error {
|
||||
if cerr := tx.Create(commit).Error; cerr != nil {
|
||||
return fmt.Errorf("failed to create page commit: %w", cerr)
|
||||
}
|
||||
if ierr := tx.Create(item).Error; ierr != nil {
|
||||
return fmt.Errorf("failed to create page commit item: %w", ierr)
|
||||
}
|
||||
return nil
|
||||
}); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return commit, nil
|
||||
}
|
||||
|
||||
// pageCommitLocks is a per-page-file-key mutex registry that serializes
|
||||
// RecordPageEdit calls for the same page. Entries are retained for the process
|
||||
// lifetime (bounded by the number of distinct pages edited).
|
||||
var pageCommitLocks sync.Map // fileID -> *sync.Mutex
|
||||
|
||||
func pageCommitLock(fileID string) *sync.Mutex {
|
||||
v, _ := pageCommitLocks.LoadOrStore(fileID, &sync.Mutex{})
|
||||
return v.(*sync.Mutex)
|
||||
}
|
||||
|
||||
// ListPageCommits lists audit commits for a specific wiki/skill page.
|
||||
func (s *FileCommitService) ListPageCommits(ctx context.Context, datasetID, pageType, slug string, page, pageSize int) ([]*entity.FileCommit, int64, error) {
|
||||
items, err := s.commitItemDAO.ListByFileID(ctx, dao.DB, wikiFileID(datasetID, pageType, slug))
|
||||
if err != nil {
|
||||
return nil, 0, err
|
||||
}
|
||||
commitIDs := make([]string, 0, len(items))
|
||||
for _, it := range items {
|
||||
commitIDs = append(commitIDs, it.CommitID)
|
||||
}
|
||||
if len(commitIDs) == 0 {
|
||||
return []*entity.FileCommit{}, 0, nil
|
||||
}
|
||||
|
||||
commits, total, err := s.commitDAO.ListByIDs(ctx, dao.DB, commitIDs, page, pageSize)
|
||||
if err != nil {
|
||||
return nil, 0, err
|
||||
}
|
||||
return commits, total, nil
|
||||
}
|
||||
|
||||
// unifiedDiff produces a simple line-based unified diff between two texts.
|
||||
func unifiedDiff(oldText, newText string) string {
|
||||
oldLines := strings.Split(oldText, "\n")
|
||||
newLines := strings.Split(newText, "\n")
|
||||
|
||||
const maxCtx = 3
|
||||
type hunkLine struct {
|
||||
prefix string
|
||||
text string
|
||||
}
|
||||
var hunks []hunkLine
|
||||
|
||||
// Longest common subsequence over lines, then render the diff.
|
||||
cur := make([][]int, len(oldLines)+1)
|
||||
for i := range cur {
|
||||
cur[i] = make([]int, len(newLines)+1)
|
||||
}
|
||||
for i := len(oldLines) - 1; i >= 0; i-- {
|
||||
for j := len(newLines) - 1; j >= 0; j-- {
|
||||
if oldLines[i] == newLines[j] {
|
||||
cur[i][j] = cur[i+1][j+1] + 1
|
||||
} else if cur[i+1][j] >= cur[i][j+1] {
|
||||
cur[i][j] = cur[i+1][j]
|
||||
} else {
|
||||
cur[i][j] = cur[i][j+1]
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
i, j := 0, 0
|
||||
for i < len(oldLines) && j < len(newLines) {
|
||||
if oldLines[i] == newLines[j] {
|
||||
i++
|
||||
j++
|
||||
} else if cur[i+1][j] >= cur[i][j+1] {
|
||||
hunks = append(hunks, hunkLine{"-", oldLines[i]})
|
||||
i++
|
||||
} else {
|
||||
hunks = append(hunks, hunkLine{"+", newLines[j]})
|
||||
j++
|
||||
}
|
||||
}
|
||||
for ; i < len(oldLines); i++ {
|
||||
hunks = append(hunks, hunkLine{"-", oldLines[i]})
|
||||
}
|
||||
for ; j < len(newLines); j++ {
|
||||
hunks = append(hunks, hunkLine{"+", newLines[j]})
|
||||
}
|
||||
|
||||
if len(hunks) == 0 {
|
||||
return ""
|
||||
}
|
||||
if len(hunks) > maxCtx*2 {
|
||||
var b strings.Builder
|
||||
for k := 0; k < maxCtx; k++ {
|
||||
b.WriteString(hunks[k].prefix)
|
||||
b.WriteString(hunks[k].text)
|
||||
b.WriteString("\n")
|
||||
}
|
||||
b.WriteString("... (" + fmt.Sprintf("%d", len(hunks)-maxCtx*2) + " lines omitted) ...\n")
|
||||
for k := len(hunks) - maxCtx; k < len(hunks); k++ {
|
||||
b.WriteString(hunks[k].prefix)
|
||||
b.WriteString(hunks[k].text)
|
||||
b.WriteString("\n")
|
||||
}
|
||||
return b.String()
|
||||
}
|
||||
var b strings.Builder
|
||||
for _, h := range hunks {
|
||||
b.WriteString(h.prefix)
|
||||
b.WriteString(h.text)
|
||||
b.WriteString("\n")
|
||||
}
|
||||
return b.String()
|
||||
}
|
||||
|
||||
// ListCommits lists commits for a workspace folder with pagination
|
||||
func (s *FileCommitService) ListCommits(ctx context.Context, folderID string, page, pageSize int, orderBy string, desc bool) ([]*entity.FileCommit, int64, error) {
|
||||
return s.commitDAO.ListByFolderID(ctx, dao.DB, folderID, page, pageSize, orderBy, desc)
|
||||
|
||||
281
internal/service/file/file_commit_page_test.go
Normal file
281
internal/service/file/file_commit_page_test.go
Normal file
@@ -0,0 +1,281 @@
|
||||
package file
|
||||
|
||||
import (
|
||||
"context"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
|
||||
"ragflow/internal/dao"
|
||||
"ragflow/internal/entity"
|
||||
|
||||
"github.com/glebarez/sqlite"
|
||||
"gorm.io/gorm"
|
||||
)
|
||||
|
||||
func newPageCommitTestDB(t *testing.T) *gorm.DB {
|
||||
t.Helper()
|
||||
db, err := gorm.Open(sqlite.Open(":memory:"), &gorm.Config{TranslateError: true})
|
||||
if err != nil {
|
||||
t.Fatalf("open sqlite: %v", err)
|
||||
}
|
||||
// Keep a single connection so the :memory: database is shared across all
|
||||
// goroutines (sqlite :memory: is otherwise per-connection); this also
|
||||
// serializes the concurrent page-commit test through one connection.
|
||||
if sqlDB, serr := db.DB(); serr == nil {
|
||||
sqlDB.SetMaxOpenConns(1)
|
||||
sqlDB.SetMaxIdleConns(1)
|
||||
}
|
||||
if err := db.AutoMigrate(&entity.FileCommit{}, &entity.FileCommitItem{}); err != nil {
|
||||
t.Fatalf("auto migrate: %v", err)
|
||||
}
|
||||
old := dao.DB
|
||||
dao.DB = db
|
||||
t.Cleanup(func() { dao.DB = old })
|
||||
return db
|
||||
}
|
||||
|
||||
func TestRecordPageEdit_CreatesCommitAndItem(t *testing.T) {
|
||||
newPageCommitTestDB(t)
|
||||
svc := NewFileCommitService()
|
||||
ctx := context.Background()
|
||||
|
||||
in := PageEditCommitInput{
|
||||
DatasetID: "kb1",
|
||||
DocID: "wiki/page-a",
|
||||
Slug: "page-a",
|
||||
PageType: "wiki",
|
||||
Title: "First edit",
|
||||
AuthorID: "u1",
|
||||
OldContent: "hello world",
|
||||
NewContent: "hello world, edited",
|
||||
}
|
||||
commit, err := svc.RecordPageEdit(ctx, in)
|
||||
if err != nil {
|
||||
t.Fatalf("RecordPageEdit: %v", err)
|
||||
}
|
||||
if commit.ID == "" {
|
||||
t.Fatal("expected a generated commit id")
|
||||
}
|
||||
if commit.AuthorID != "u1" || commit.Message != "First edit" || commit.FileCount != 1 {
|
||||
t.Fatalf("unexpected commit: %+v", commit)
|
||||
}
|
||||
if commit.FolderID != "kb1" {
|
||||
t.Fatalf("expected folder_id kb1 for dataset scope, got %s", commit.FolderID)
|
||||
}
|
||||
if commit.ParentID != nil {
|
||||
t.Fatalf("first edit should have no parent, got %v", *commit.ParentID)
|
||||
}
|
||||
|
||||
var items []entity.FileCommitItem
|
||||
if err := dao.DB.Where("commit_id = ?", commit.ID).Find(&items).Error; err != nil {
|
||||
t.Fatalf("load items: %v", err)
|
||||
}
|
||||
if len(items) != 1 {
|
||||
t.Fatalf("expected 1 item, got %d", len(items))
|
||||
}
|
||||
it := items[0]
|
||||
if it.Operation != "modify" {
|
||||
t.Fatalf("expected operation modify, got %s", it.Operation)
|
||||
}
|
||||
if it.FileID != "kb1/wiki/page-a" {
|
||||
t.Fatalf("expected file_id kb1/wiki/page-a (dataset-scoped), got %s", it.FileID)
|
||||
}
|
||||
if it.SlugKwd == nil || *it.SlugKwd != "page-a" {
|
||||
t.Fatalf("expected slug_kwd page-a, got %v", it.SlugKwd)
|
||||
}
|
||||
if it.PageTypeKwd == nil || *it.PageTypeKwd != "wiki" {
|
||||
t.Fatalf("expected page_type_kwd wiki, got %v", it.PageTypeKwd)
|
||||
}
|
||||
if it.Diff == nil || !strings.Contains(*it.Diff, "hello world, edited") {
|
||||
t.Fatalf("expected diff containing new content, got %v", it.Diff)
|
||||
}
|
||||
if it.ContentAfterStorage == nil || *it.ContentAfterStorage != "es" {
|
||||
t.Fatalf("expected content_after_storage es, got %v", it.ContentAfterStorage)
|
||||
}
|
||||
if it.ContentAfterLocation == nil || *it.ContentAfterLocation != "wiki/page-a" {
|
||||
t.Fatalf("expected content_after_location wiki/page-a, got %v", it.ContentAfterLocation)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRecordPageEdit_SecondEditLinksParent(t *testing.T) {
|
||||
newPageCommitTestDB(t)
|
||||
svc := NewFileCommitService()
|
||||
ctx := context.Background()
|
||||
|
||||
base := PageEditCommitInput{
|
||||
DatasetID: "kb1",
|
||||
DocID: "wiki/page-a",
|
||||
Slug: "page-a",
|
||||
PageType: "wiki",
|
||||
Title: "first",
|
||||
AuthorID: "u1",
|
||||
}
|
||||
if _, err := svc.RecordPageEdit(ctx, base); err != nil {
|
||||
t.Fatalf("first RecordPageEdit: %v", err)
|
||||
}
|
||||
|
||||
second := base
|
||||
second.Title = "second"
|
||||
second.OldContent = "hello"
|
||||
second.NewContent = "hello world"
|
||||
commit2, err := svc.RecordPageEdit(ctx, second)
|
||||
if err != nil {
|
||||
t.Fatalf("second RecordPageEdit: %v", err)
|
||||
}
|
||||
if commit2.ParentID == nil {
|
||||
t.Fatal("second edit should link to the first commit as parent")
|
||||
}
|
||||
var first entity.FileCommit
|
||||
if err := dao.DB.Where("title = ?", "first").First(&first).Error; err != nil {
|
||||
t.Fatalf("load first commit: %v", err)
|
||||
}
|
||||
if *commit2.ParentID != first.ID {
|
||||
t.Fatalf("parent id mismatch: got %s want %s", *commit2.ParentID, first.ID)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRecordPageEdit_IsolatesDatasets(t *testing.T) {
|
||||
newPageCommitTestDB(t)
|
||||
svc := NewFileCommitService()
|
||||
ctx := context.Background()
|
||||
|
||||
mk := func(datasetID string) PageEditCommitInput {
|
||||
return PageEditCommitInput{
|
||||
DatasetID: datasetID,
|
||||
DocID: datasetID + "/wiki/page-a",
|
||||
Slug: "page-a",
|
||||
PageType: "wiki",
|
||||
Title: datasetID + "-edit",
|
||||
AuthorID: "u1",
|
||||
OldContent: "old",
|
||||
NewContent: "new",
|
||||
}
|
||||
}
|
||||
if _, err := svc.RecordPageEdit(ctx, mk("kb1")); err != nil {
|
||||
t.Fatalf("kb1 RecordPageEdit: %v", err)
|
||||
}
|
||||
if _, err := svc.RecordPageEdit(ctx, mk("kb2")); err != nil {
|
||||
t.Fatalf("kb2 RecordPageEdit: %v", err)
|
||||
}
|
||||
|
||||
// A second edit in kb1 must parent to kb1's own first commit, not kb2's.
|
||||
kb1Again := mk("kb1")
|
||||
kb1Again.Title = "kb1-edit-2"
|
||||
kb1Again.OldContent = "old"
|
||||
kb1Again.NewContent = "newer"
|
||||
kb1Commit2, err := svc.RecordPageEdit(ctx, kb1Again)
|
||||
if err != nil {
|
||||
t.Fatalf("kb1 second RecordPageEdit: %v", err)
|
||||
}
|
||||
if kb1Commit2.ParentID == nil {
|
||||
t.Fatal("kb1 second edit should have a parent")
|
||||
}
|
||||
|
||||
var kb1First, kb2First entity.FileCommit
|
||||
if err := dao.DB.Where("title = ?", "kb1-edit").First(&kb1First).Error; err != nil {
|
||||
t.Fatalf("load kb1 first commit: %v", err)
|
||||
}
|
||||
if err := dao.DB.Where("title = ?", "kb2-edit").First(&kb2First).Error; err != nil {
|
||||
t.Fatalf("load kb2 first commit: %v", err)
|
||||
}
|
||||
if *kb1Commit2.ParentID != kb1First.ID {
|
||||
t.Fatalf("kb1 parent mismatch: got %s want %s", *kb1Commit2.ParentID, kb1First.ID)
|
||||
}
|
||||
if *kb1Commit2.ParentID == kb2First.ID {
|
||||
t.Fatal("kb1 parent must not cross into kb2 history")
|
||||
}
|
||||
}
|
||||
|
||||
func TestRecordPageEdit_ConcurrentEditsFormLinearChain(t *testing.T) {
|
||||
newPageCommitTestDB(t)
|
||||
svc := NewFileCommitService()
|
||||
ctx := context.Background()
|
||||
|
||||
const edits = 8
|
||||
var wg sync.WaitGroup
|
||||
errs := make([]error, edits)
|
||||
for i := 0; i < edits; i++ {
|
||||
wg.Add(1)
|
||||
go func(idx int) {
|
||||
defer wg.Done()
|
||||
in := PageEditCommitInput{
|
||||
DatasetID: "kb1",
|
||||
DocID: "wiki/page-a",
|
||||
Slug: "page-a",
|
||||
PageType: "wiki",
|
||||
Title: "edit-" + strconv.Itoa(idx),
|
||||
AuthorID: "u1",
|
||||
OldContent: "old",
|
||||
NewContent: "new-" + strconv.Itoa(idx),
|
||||
}
|
||||
_, errs[idx] = svc.RecordPageEdit(ctx, in)
|
||||
}(i)
|
||||
}
|
||||
wg.Wait()
|
||||
for i, e := range errs {
|
||||
if e != nil {
|
||||
t.Fatalf("edit %d failed: %v", i, e)
|
||||
}
|
||||
}
|
||||
|
||||
var commits []entity.FileCommit
|
||||
if err := dao.DB.Where("folder_id = ?", "kb1").Order("create_time ASC").Find(&commits).Error; err != nil {
|
||||
t.Fatalf("load commits: %v", err)
|
||||
}
|
||||
if len(commits) != edits {
|
||||
t.Fatalf("expected %d commits, got %d", edits, len(commits))
|
||||
}
|
||||
// Every commit except the first must have a parent, and the parents must
|
||||
// form a linear chain (no two commits share the same parent).
|
||||
seenParents := map[string]bool{}
|
||||
for i, c := range commits {
|
||||
if i == 0 {
|
||||
if c.ParentID != nil {
|
||||
t.Fatalf("first commit should have no parent")
|
||||
}
|
||||
continue
|
||||
}
|
||||
if c.ParentID == nil {
|
||||
t.Fatalf("commit %s should have a parent", c.ID)
|
||||
}
|
||||
if seenParents[*c.ParentID] {
|
||||
t.Fatalf("two commits share parent %s -> forked chain", *c.ParentID)
|
||||
}
|
||||
seenParents[*c.ParentID] = true
|
||||
}
|
||||
}
|
||||
|
||||
func TestUnifiedDiff_EmptyWhenNoChange(t *testing.T) {
|
||||
if d := unifiedDiff("same", "same"); d != "" {
|
||||
t.Fatalf("expected empty diff for identical text, got %q", d)
|
||||
}
|
||||
}
|
||||
|
||||
func TestUnifiedDiff_DetectsAddition(t *testing.T) {
|
||||
d := unifiedDiff("a\nb\nc\n", "a\nb\nc\nd\n")
|
||||
if !strings.Contains(d, "+d") {
|
||||
t.Fatalf("expected diff to contain added line, got %q", d)
|
||||
}
|
||||
}
|
||||
|
||||
func TestUnifiedDiff_DetectsRemoval(t *testing.T) {
|
||||
d := unifiedDiff("a\nb\nc\n", "a\nc\n")
|
||||
if !strings.Contains(d, "-b") {
|
||||
t.Fatalf("expected diff to contain removed line, got %q", d)
|
||||
}
|
||||
}
|
||||
|
||||
func TestUnifiedDiff_TruncatesLongDiff(t *testing.T) {
|
||||
oldLines := make([]string, 50)
|
||||
newLines := make([]string, 50)
|
||||
for i := range oldLines {
|
||||
oldLines[i] = "old-line"
|
||||
newLines[i] = "new-line"
|
||||
}
|
||||
d := unifiedDiff(strings.Join(oldLines, "\n"), strings.Join(newLines, "\n"))
|
||||
if !strings.Contains(d, "... ") || !strings.Contains(d, "lines omitted") {
|
||||
t.Fatalf("expected long diff to be truncated, got %q", d)
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user