From 21af67f6f9a261cd927464cf880f1ebf73d6ca53 Mon Sep 17 00:00:00 2001 From: chanx <1243304602@qq.com> Date: Fri, 3 Apr 2026 15:08:43 +0800 Subject: [PATCH] feat(File Management): Refactor File List API and Add Knowledge Base Document Initialization (#13914) ### What problem does this PR solve? feat(File Management): Refactor File List API and Add Knowledge Base Document Initialization - Migrate the file list API endpoint from `/v1/file/list` to `/api/v1/files` to align with the Python implementation. - Add logic for initializing knowledge base documents; automatically create the `.knowledgebase` folder and associated documents when retrieving the root directory. - Enhance parameter validation and error handling, including the introduction of a new `CodeParamError` error code. - Optimize the file list response structure to match the implementation on the Python side. - Update the Vite configuration to support proxying the new `/api/v1/files` endpoint. ### Type of change - [x] New Feature (non-breaking change which adds functionality) --- internal/common/error_code.go | 2 + internal/dao/file.go | 136 ++++++++++++++++++++++++++++++++ internal/handler/file.go | 142 ++++++++++++---------------------- internal/router/router.go | 2 +- internal/service/file.go | 36 ++++++--- web/vite.config.ts | 2 +- 6 files changed, 217 insertions(+), 103 deletions(-) diff --git a/internal/common/error_code.go b/internal/common/error_code.go index dce537c436..912d7bb6d7 100644 --- a/internal/common/error_code.go +++ b/internal/common/error_code.go @@ -31,6 +31,7 @@ const ( CodeResourceExhausted ErrorCode = 107 CodePermissionError ErrorCode = 108 CodeAuthenticationError ErrorCode = 109 + CodeParamError ErrorCode = 110 CodeLicenseValid ErrorCode = 320 CodeLicenseInactiveError ErrorCode = 321 CodeLicenseExpiredError ErrorCode = 322 @@ -59,6 +60,7 @@ var errorMessages = map[ErrorCode]string{ CodeResourceExhausted: "Resource exhausted", CodePermissionError: "Permission denied", CodeAuthenticationError: "Authentication failed", + CodeParamError: "Invalid parameters", CodeLicenseValid: "License valid", CodeLicenseInactiveError: "License inactive", CodeLicenseExpiredError: "License expired", diff --git a/internal/dao/file.go b/internal/dao/file.go index f8806e21d4..de4fffb76a 100644 --- a/internal/dao/file.go +++ b/internal/dao/file.go @@ -302,3 +302,139 @@ func generateUUID() string { id := uuid.New().String() return strings.ReplaceAll(id, "-", "") } + +// KnowledgebaseFolderName is the folder name for knowledgebase +const KnowledgebaseFolderName = ".knowledgebase" + +// InitKnowledgebaseDocs initializes knowledgebase documents for tenant +// This matches Python's FileService.init_knowledgebase_docs method +func (dao *FileDAO) InitKnowledgebaseDocs(rootID, tenantID string, file2DocumentDAO *File2DocumentDAO) error { + var count int64 + err := DB.Model(&entity.File{}). + Where("name = ? AND parent_id = ?", KnowledgebaseFolderName, rootID). + Count(&count).Error + if err != nil { + return err + } + + if count > 0 { + return nil + } + + kbFolder, err := dao.newAFileFromKB(tenantID, KnowledgebaseFolderName, rootID) + if err != nil { + return err + } + + var knowledgebases []entity.Knowledgebase + err = DB.Select("id", "name"). + Where("tenant_id = ?", tenantID). + Find(&knowledgebases).Error + if err != nil { + return err + } + + for _, kb := range knowledgebases { + kbFolderForKB, err := dao.newAFileFromKB(tenantID, kb.Name, kbFolder.ID) + if err != nil { + continue + } + + var documents []entity.Document + err = DB.Where("kb_id = ?", kb.ID).Find(&documents).Error + if err != nil { + continue + } + + for _, doc := range documents { + dao.addFileFromKB(&doc, kbFolderForKB.ID, tenantID, file2DocumentDAO) + } + } + + return nil +} + +// newAFileFromKB creates a new file from knowledgebase +func (dao *FileDAO) newAFileFromKB(tenantID, name, parentID string) (*entity.File, error) { + var existingFiles []*entity.File + err := DB.Where("tenant_id = ? AND parent_id = ? AND name = ?", tenantID, parentID, name).Find(&existingFiles).Error + if err != nil { + return nil, err + } + + if len(existingFiles) > 0 { + return existingFiles[0], nil + } + + fileID := generateUUID() + file := &entity.File{ + ID: fileID, + ParentID: parentID, + TenantID: tenantID, + CreatedBy: tenantID, + Name: name, + Type: "folder", + Size: 0, + SourceType: "knowledgebase", + } + + if err := DB.Create(file).Error; err != nil { + return nil, err + } + return file, nil +} + +// addFileFromKB adds a file record from knowledgebase document +func (dao *FileDAO) addFileFromKB(doc *entity.Document, kbFolderID, tenantID string, file2DocumentDAO *File2DocumentDAO) error { + var f2dCount int64 + err := DB.Model(&entity.File2Document{}). + Where("document_id = ?", doc.ID). + Count(&f2dCount).Error + if err != nil { + return err + } + + if f2dCount > 0 { + return nil + } + + docName := "" + if doc.Name != nil { + docName = *doc.Name + } + + docLocation := "" + if doc.Location != nil { + docLocation = *doc.Location + } + + fileID := generateUUID() + file := &entity.File{ + ID: fileID, + ParentID: kbFolderID, + TenantID: tenantID, + CreatedBy: tenantID, + Name: docName, + Type: doc.Type, + Size: doc.Size, + Location: &docLocation, + SourceType: "knowledgebase", + } + + if err := DB.Create(file).Error; err != nil { + return err + } + + f2dID := generateUUID() + f2d := &entity.File2Document{ + ID: f2dID, + FileID: &fileID, + DocumentID: &doc.ID, + } + + if err := DB.Create(f2d).Error; err != nil { + return err + } + + return nil +} diff --git a/internal/handler/file.go b/internal/handler/file.go index 93828277fa..cc31a1db39 100644 --- a/internal/handler/file.go +++ b/internal/handler/file.go @@ -41,20 +41,20 @@ func NewFileHandler(fileService *service.FileService, userService *service.UserS } } -// ListFiles list files +// ListFiles list files (new endpoint at /api/v1/files matching Python /files) // @Summary List Files -// @Description Get list of files for the current user with filtering, pagination and sorting +// @Description Get list of files under a folder with filtering, pagination and sorting (matches Python /files endpoint) // @Tags file // @Accept json // @Produce json -// @Param parent_id query string false "parent folder ID" -// @Param keywords query string false "search keywords" -// @Param page query int false "page number (default: 1)" -// @Param page_size query int false "items per page (default: 15)" +// @Param parent_id query string false "parent folder ID (empty means root folder)" +// @Param keywords query string false "search keywords (case-insensitive)" +// @Param page query int false "page number (default: 1, min: 1)" +// @Param page_size query int false "items per page (default: 15, min: 1, max: 100)" // @Param orderby query string false "order by field (default: create_time)" // @Param desc query bool false "descending order (default: true)" // @Success 200 {object} service.ListFilesResponse -// @Router /v1/file/list [get] +// @Router /api/v1/files [get] func (h *FileHandler) ListFiles(c *gin.Context) { user, errorCode, errorMessage := GetUser(c) if errorCode != common.CodeSuccess { @@ -63,49 +63,52 @@ func (h *FileHandler) ListFiles(c *gin.Context) { } userID := user.ID - // Parse query parameters parentID := c.Query("parent_id") keywords := c.Query("keywords") - // Parse page (default: 1) page := 1 if pageStr := c.Query("page"); pageStr != "" { - if p, err := strconv.Atoi(pageStr); err == nil && p > 0 { + if p, err := strconv.Atoi(pageStr); err == nil && p >= 1 { page = p + } else if err != nil { + jsonError(c, common.CodeParamError, "Invalid page parameter: must be a positive integer") + return } } - // Parse page_size (default: 15) pageSize := 15 if pageSizeStr := c.Query("page_size"); pageSizeStr != "" { - if ps, err := strconv.Atoi(pageSizeStr); err == nil && ps > 0 { + if ps, err := strconv.Atoi(pageSizeStr); err == nil { + if ps < 1 { + jsonError(c, common.CodeParamError, "Invalid page_size parameter: must be at least 1") + return + } + if ps > 100 { + ps = 100 + } pageSize = ps + } else { + jsonError(c, common.CodeParamError, "Invalid page_size parameter: must be a positive integer") + return } } - // Parse orderby (default: create_time) orderby := c.DefaultQuery("orderby", "create_time") - - // Parse desc (default: true) desc := true if descStr := c.Query("desc"); descStr != "" { desc = descStr != "false" } - // List files result, err := h.fileService.ListFiles(userID, parentID, page, pageSize, orderby, desc, keywords) if err != nil { - c.JSON(http.StatusInternalServerError, gin.H{ - "code": 500, - "message": err.Error(), - }) + jsonError(c, common.CodeServerError, err.Error()) return } c.JSON(http.StatusOK, gin.H{ - "code": 0, + "code": common.CodeSuccess, "data": result, - "message": "success", + "message": common.CodeSuccess.Message(), }) } @@ -128,17 +131,14 @@ func (h *FileHandler) GetRootFolder(c *gin.Context) { // Get root folder rootFolder, err := h.fileService.GetRootFolder(userID) if err != nil { - c.JSON(http.StatusInternalServerError, gin.H{ - "code": 500, - "message": err.Error(), - }) + jsonError(c, common.CodeServerError, err.Error()) return } c.JSON(http.StatusOK, gin.H{ - "code": 0, + "code": common.CodeSuccess, "data": gin.H{"root_folder": rootFolder}, - "message": "success", + "message": common.CodeSuccess.Message(), }) } @@ -161,27 +161,21 @@ func (h *FileHandler) GetParentFolder(c *gin.Context) { // Get file_id from query fileID := c.Query("file_id") if fileID == "" { - c.JSON(http.StatusBadRequest, gin.H{ - "code": 400, - "message": "file_id is required", - }) + jsonError(c, common.CodeBadRequest, "file_id is required") return } // Get parent folder parentFolder, err := h.fileService.GetParentFolder(fileID) if err != nil { - c.JSON(http.StatusInternalServerError, gin.H{ - "code": 500, - "message": err.Error(), - }) + jsonError(c, common.CodeServerError, err.Error()) return } c.JSON(http.StatusOK, gin.H{ - "code": 0, + "code": common.CodeSuccess, "data": gin.H{"parent_folder": parentFolder}, - "message": "success", + "message": common.CodeSuccess.Message(), }) } @@ -204,27 +198,21 @@ func (h *FileHandler) GetAllParentFolders(c *gin.Context) { // Get file_id from query fileID := c.Query("file_id") if fileID == "" { - c.JSON(http.StatusBadRequest, gin.H{ - "code": 400, - "message": "file_id is required", - }) + jsonError(c, common.CodeBadRequest, "file_id is required") return } // Get all parent folders parentFolders, err := h.fileService.GetAllParentFolders(fileID) if err != nil { - c.JSON(http.StatusInternalServerError, gin.H{ - "code": 500, - "message": err.Error(), - }) + jsonError(c, common.CodeServerError, err.Error()) return } c.JSON(http.StatusOK, gin.H{ - "code": 0, + "code": common.CodeSuccess, "data": gin.H{"parent_folders": parentFolders}, - "message": "success", + "message": common.CodeSuccess.Message(), }) } @@ -248,10 +236,7 @@ type CreateFolderRequest struct { func (h *FileHandler) UploadFile(c *gin.Context) { user, errorCode, errorMessage := GetUser(c) if errorCode != common.CodeSuccess { - c.JSON(http.StatusBadRequest, gin.H{ - "code": errorCode, - "message": errorMessage, - }) + jsonError(c, errorCode, errorMessage) return } @@ -261,29 +246,20 @@ func (h *FileHandler) UploadFile(c *gin.Context) { if strings.Contains(contentType, "multipart/form-data") { if err := c.Request.ParseMultipartForm(32 << 20); err != nil { - c.JSON(http.StatusBadRequest, gin.H{ - "code": 400, - "message": "Failed to parse multipart form: " + err.Error(), - }) + jsonError(c, common.CodeBadRequest, "Failed to parse multipart form: "+err.Error()) return } form := c.Request.MultipartForm if form == nil { - c.JSON(http.StatusBadRequest, gin.H{ - "code": 400, - "message": "No file part!", - }) + jsonError(c, common.CodeBadRequest, "No file part!") return } parentID := c.PostForm("parent_id") if parentID == "" { rootFolder, err := h.fileService.GetRootFolder(userID) if err != nil { - c.JSON(http.StatusInternalServerError, gin.H{ - "code": 500, - "message": err.Error(), - }) + jsonError(c, common.CodeServerError, err.Error()) return } parentID = rootFolder["id"].(string) @@ -291,36 +267,27 @@ func (h *FileHandler) UploadFile(c *gin.Context) { files := form.File["file"] if len(files) == 0 { - c.JSON(http.StatusBadRequest, gin.H{ - "code": 400, - "message": "No file selected!", - }) + jsonError(c, common.CodeBadRequest, "No file selected!") return } for _, fileHeader := range files { if fileHeader.Filename == "" { - c.JSON(http.StatusBadRequest, gin.H{ - "code": 400, - "message": "No file selected!", - }) + jsonError(c, common.CodeBadRequest, "No file selected!") return } } result, err := h.fileService.UploadFile(userID, parentID, files) if err != nil { - c.JSON(http.StatusBadRequest, gin.H{ - "code": 400, - "message": err.Error(), - }) + jsonError(c, common.CodeBadRequest, err.Error()) return } c.JSON(http.StatusOK, gin.H{ - "code": 0, + "code": common.CodeSuccess, "data": result, - "message": "success", + "message": common.CodeSuccess.Message(), }) return } @@ -339,10 +306,7 @@ func (h *FileHandler) UploadFile(c *gin.Context) { if parentID == "" { rootFolder, err := h.fileService.GetRootFolder(userID) if err != nil { - c.JSON(http.StatusInternalServerError, gin.H{ - "code": 500, - "message": err.Error(), - }) + jsonError(c, common.CodeServerError, err.Error()) return } parentID = rootFolder["id"].(string) @@ -350,24 +314,18 @@ func (h *FileHandler) UploadFile(c *gin.Context) { result, err := h.fileService.CreateFolder(userID, req.Name, parentID, req.Type) if err != nil { - c.JSON(http.StatusBadRequest, gin.H{ - "code": 400, - "message": err.Error(), - }) + jsonError(c, common.CodeBadRequest, err.Error()) return } c.JSON(http.StatusOK, gin.H{ - "code": 0, + "code": common.CodeSuccess, "data": result, - "message": "success", + "message": common.CodeSuccess.Message(), }) return } - c.JSON(http.StatusBadRequest, gin.H{ - "code": 400, - "message": "Unsupported content type", - }) + jsonError(c, common.CodeBadRequest, "Unsupported content type") return } diff --git a/internal/router/router.go b/internal/router/router.go index 3668effe06..66003f5388 100644 --- a/internal/router/router.go +++ b/internal/router/router.go @@ -193,6 +193,7 @@ func (r *Router) Setup(engine *gin.Engine) { file := v1.Group("/files") { file.POST("", r.fileHandler.UploadFile) + file.GET("", r.fileHandler.ListFiles) } // provider pool route group @@ -307,7 +308,6 @@ func (r *Router) Setup(engine *gin.Engine) { // File routes file := authorized.Group("/v1/file") { - file.GET("/list", r.fileHandler.ListFiles) file.GET("/root_folder", r.fileHandler.GetRootFolder) file.GET("/parent_folder", r.fileHandler.GetParentFolder) file.GET("/all_parent_folder", r.fileHandler.GetAllParentFolders) diff --git a/internal/service/file.go b/internal/service/file.go index 1c8b89adee..f47af98137 100644 --- a/internal/service/file.go +++ b/internal/service/file.go @@ -68,20 +68,26 @@ func (s *FileService) GetRootFolder(tenantID string) (map[string]interface{}, er return s.toFileResponse(file), nil } -// ListFiles lists files by parent folder ID +// ListFiles lists files by parent folder ID (matching Python /files endpoint) +// This method includes init_knowledgebase_docs initialization when parent_id is empty func (s *FileService) ListFiles(tenantID, pfID string, page, pageSize int, orderby string, desc bool, keywords string) (*ListFilesResponse, error) { - // If pfID is empty, get root folder + // If pfID is empty, get root folder and initialize knowledgebase docs if pfID == "" { rootFolder, err := s.fileDAO.GetRootFolder(tenantID) if err != nil { - return nil, err + return nil, fmt.Errorf("failed to get root folder: %w", err) } pfID = rootFolder.ID + + // Initialize knowledgebase docs (matching Python init_knowledgebase_docs logic) + if err := s.initKnowledgebaseDocs(pfID, tenantID); err != nil { + return nil, fmt.Errorf("failed to initialize knowledgebase docs: %w", err) + } } // Check if parent folder exists if _, err := s.fileDAO.GetByID(pfID); err != nil { - return nil, err + return nil, fmt.Errorf("Folder not found!") } // Get files by parent folder ID @@ -93,16 +99,16 @@ func (s *FileService) ListFiles(tenantID, pfID string, page, pageSize int, order // Get parent folder parentFolder, err := s.fileDAO.GetParentFolder(pfID) if err != nil { - return nil, err + return nil, fmt.Errorf("File not found!") } // Process files to add additional info - fileResponses := make([]map[string]interface{}, len(files)) - for i, file := range files { + fileResponses := make([]map[string]interface{}, 0, len(files)) + for _, file := range files { fileInfo := s.toFileInfo(file) // If folder, calculate size and check for child folders - if file.Type == "folder" { + if file.Type == FileTypeFolder { folderSize, err := s.fileDAO.GetFolderSize(file.ID) if err == nil { fileInfo.Size = folderSize @@ -121,7 +127,7 @@ func (s *FileService) ListFiles(tenantID, pfID string, page, pageSize int, order fileInfo.KbsInfo = kbsInfo } - fileResponses[i] = s.fileInfoToResponse(fileInfo) + fileResponses = append(fileResponses, s.fileInfoToResponse(fileInfo)) } return &ListFilesResponse{ @@ -131,6 +137,18 @@ func (s *FileService) ListFiles(tenantID, pfID string, page, pageSize int, order }, nil } +// initKnowledgebaseDocs initializes knowledgebase documents for tenant +// This matches Python's FileService.init_knowledgebase_docs method +func (s *FileService) initKnowledgebaseDocs(rootID, tenantID string) error { + return s.fileDAO.InitKnowledgebaseDocs(rootID, tenantID, s.file2DocumentDAO) +} + +// KnowledgebaseFolderName is the folder name for knowledgebase +const KnowledgebaseFolderName = ".knowledgebase" + +// FileSourceKnowledgebase represents knowledgebase as file source +const FileSourceKnowledgebase = "knowledgebase" + // toFileResponse converts file model to response format func (s *FileService) toFileResponse(file *entity.File) map[string]interface{} { result := map[string]interface{}{ diff --git a/web/vite.config.ts b/web/vite.config.ts index d3cb1865d7..966fa5fd21 100644 --- a/web/vite.config.ts +++ b/web/vite.config.ts @@ -49,7 +49,7 @@ export default defineConfig(({ mode }) => { }, }, hybrid: { - '^(/api/v1/memories)|^(/v1/user/info)|^(/v1/user/tenant_info)|^(/v1/tenant/list)|^(/v1/system/config)|^(/v1/user/login)|^(/v1/user/logout)': + '^(/api/v1/memories)|^(/v1/user/info)|^(/v1/user/tenant_info)|^(/v1/tenant/list)|^(/v1/system/config)|^(/v1/user/login)|^(/v1/user/logout)|^(/api/v1/files)': { target: 'http://127.0.0.1:9384/', changeOrigin: true,