diff --git a/cmd/admin_server.go b/cmd/admin_server.go index 99c438def6..3775d038b7 100644 --- a/cmd/admin_server.go +++ b/cmd/admin_server.go @@ -25,6 +25,7 @@ import ( "os" "os/signal" "ragflow/internal/cache" + "ragflow/internal/common" "ragflow/internal/engine" "syscall" "time" @@ -34,7 +35,6 @@ import ( "ragflow/internal/admin" "ragflow/internal/dao" - "ragflow/internal/logger" "ragflow/internal/server" "ragflow/internal/utility" ) @@ -45,13 +45,13 @@ func main() { flag.Parse() // Initialize logger - if err := logger.Init("info"); err != nil { + if err := common.Init("info"); err != nil { panic("failed to initialize logger: " + err.Error()) } // Initialize configuration if err := server.Init(configPath); err != nil { - logger.Error("Failed to initialize configuration", err) + common.Error("Failed to initialize configuration", err) os.Exit(1) } @@ -59,15 +59,15 @@ func main() { // Reinitialize logger with configured level if different if cfg.Log.Level != "" && cfg.Log.Level != "info" { - if err := logger.Init(cfg.Log.Level); err != nil { - logger.Error("Failed to reinitialize logger with configured level", err) + if err := common.Init(cfg.Log.Level); err != nil { + common.Error("Failed to reinitialize logger with configured level", err) } } // Set logger for server package - server.SetLogger(logger.Logger) + server.SetLogger(common.Logger) - logger.Info("Server mode", zap.String("mode", cfg.Server.Mode)) + common.Info("Server mode", zap.String("mode", cfg.Server.Mode)) // Set Gin mode if cfg.Server.Mode == "release" { @@ -78,26 +78,26 @@ func main() { // Initialize database if err := dao.InitDB(); err != nil { - logger.Error("Failed to initialize database", err) + common.Error("Failed to initialize database", err) os.Exit(1) } // Initialize doc engine if err := engine.Init(&cfg.DocEngine); err != nil { - logger.Fatal("Failed to initialize doc engine", zap.Error(err)) + common.Fatal("Failed to initialize doc engine", zap.Error(err)) } defer engine.Close() // Initialize Redis cache if err := cache.Init(&cfg.Redis); err != nil { - logger.Fatal("Failed to initialize Redis", zap.Error(err)) + common.Fatal("Failed to initialize Redis", zap.Error(err)) } defer cache.Close() // Initialize server variables (runtime variables that can change during operation) // This must be done after Cache is initialized if err := server.InitVariables(cache.Get()); err != nil { - logger.Warn("Failed to initialize server variables from Redis, using defaults", zap.String("error", err.Error())) + common.Warn("Failed to initialize server variables from Redis, using defaults", zap.String("error", err.Error())) } adminService := admin.NewService() @@ -105,7 +105,7 @@ func main() { // Initialize default admin user if err := adminService.InitDefaultAdmin(); err != nil { - logger.Error("Failed to initialize default admin user", err) + common.Error("Failed to initialize default admin user", err) } // Initialize router @@ -121,7 +121,7 @@ func main() { ginEngine.Use(gin.Recovery()) // Log request URL for every request ginEngine.Use(func(c *gin.Context) { - logger.Info("HTTP Request", zap.String("url", c.Request.URL.String()), zap.String("method", c.Request.Method)) + common.Info("HTTP Request", zap.String("url", c.Request.URL.String()), zap.String("method", c.Request.Method)) c.Next() }) @@ -136,13 +136,13 @@ func main() { } // Print RAGFlow version - logger.Info("RAGFlow version", zap.String("version", utility.GetRAGFlowVersion())) + common.Info("RAGFlow version", zap.String("version", utility.GetRAGFlowVersion())) // Print all configuration settings server.PrintAll() // Print RAGFlow Admin logo - logger.Info("" + + common.Info("" + "\n ____ ___ ______________ ___ __ _ \n" + " / __ \\/ | / ____/ ____/ /___ _ __ / | ____/ /___ ___ (_)___ \n" + " / /_/ / /| |/ / __/ /_ / / __ \\ | /| / / / /| |/ __ / __ `__ \\/ / __ \\ \n" + @@ -151,10 +151,10 @@ func main() { // Start server in a goroutine go func() { - logger.Info(fmt.Sprintf("Admin Go Version: %s", utility.GetRAGFlowVersion())) - logger.Info(fmt.Sprintf("Starting RAGFlow admin server on port: %d", cfg.Admin.Port)) + common.Info(fmt.Sprintf("Admin Go Version: %s", utility.GetRAGFlowVersion())) + common.Info(fmt.Sprintf("Starting RAGFlow admin server on port: %d", cfg.Admin.Port)) if err := srv.ListenAndServe(); err != nil && !errors.Is(err, http.ErrServerClosed) { - logger.Fatal("Failed to start server", zap.Error(err)) + common.Fatal("Failed to start server", zap.Error(err)) } }() @@ -163,8 +163,8 @@ func main() { signal.Notify(quit, syscall.SIGINT, syscall.SIGTERM, syscall.SIGQUIT, syscall.SIGUSR2) sig := <-quit - logger.Info("Received signal", zap.String("signal", sig.String())) - logger.Info("Shutting down server...") + common.Info("Received signal", zap.String("signal", sig.String())) + common.Info("Shutting down server...") // Create context with timeout for graceful shutdown ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) @@ -172,8 +172,8 @@ func main() { // Shutdown server if err := srv.Shutdown(ctx); err != nil { - logger.Fatal("Server forced to shutdown", zap.Error(err)) + common.Fatal("Server forced to shutdown", zap.Error(err)) } - logger.Info("Server exited") + common.Info("Server exited") } diff --git a/cmd/cli/ragflow_cli.go b/cmd/ragflow_cli.go similarity index 94% rename from cmd/cli/ragflow_cli.go rename to cmd/ragflow_cli.go index 0b27397ffa..cc2043687c 100644 --- a/cmd/cli/ragflow_cli.go +++ b/cmd/ragflow_cli.go @@ -4,10 +4,10 @@ import ( "fmt" "os" "os/signal" + "ragflow/internal/common" "syscall" "ragflow/internal/cli" - "ragflow/internal/logger" ) func main() { @@ -23,7 +23,7 @@ func main() { if args.Verbose { logLevel = "info" } - if err := logger.Init(logLevel); err != nil { + if err = common.Init(logLevel); err != nil { fmt.Printf("Warning: Failed to initialize logger: %v\n", err) } diff --git a/cmd/server_main.go b/cmd/server_main.go index 0da92fb032..e4a634e72a 100644 --- a/cmd/server_main.go +++ b/cmd/server_main.go @@ -24,7 +24,6 @@ import ( "ragflow/internal/dao" "ragflow/internal/engine" "ragflow/internal/handler" - "ragflow/internal/logger" "ragflow/internal/router" "ragflow/internal/service" "ragflow/internal/service/nlp" @@ -56,80 +55,80 @@ func main() { // Initialize logger with default level // logger.Init("info"); // set debug log level - if err := logger.Init("info"); err != nil { + if err := common.Init("info"); err != nil { panic(fmt.Sprintf("Failed to initialize logger: %v", err)) } // Initialize configuration if err := server.Init(""); err != nil { - logger.Fatal("Failed to initialize config", zap.Error(err)) + common.Fatal("Failed to initialize config", zap.Error(err)) } // Override port with command line argument if provided config := server.GetConfig() if portFlag > 0 { config.Server.Port = portFlag - logger.Info("Port overridden by command line argument", zap.Int("port", portFlag)) + common.Info("Port overridden by command line argument", zap.Int("port", portFlag)) } if config.Server.Port == 0 { - logger.Fatal("Server port is not configured. Please specify via --port flag or config file.") + common.Fatal("Server port is not configured. Please specify via --port flag or config file.") } // Load model providers configuration if err := server.LoadModelProviders(""); err != nil { - logger.Fatal("Failed to load model providers", zap.Error(err)) + common.Fatal("Failed to load model providers", zap.Error(err)) } - logger.Info("Model providers loaded", zap.Int("count", len(server.GetModelProviders()))) + common.Info("Model providers loaded", zap.Int("count", len(server.GetModelProviders()))) // Reinitialize logger with configured level if different if config.Log.Level != "" && config.Log.Level != "info" { - if err := logger.Init(config.Log.Level); err != nil { - logger.Error("Failed to reinitialize logger with configured level", err) + if err := common.Init(config.Log.Level); err != nil { + common.Error("Failed to reinitialize logger with configured level", err) } } - server.SetLogger(logger.Logger) + server.SetLogger(common.Logger) if config.Log.Level == "" { - config.Log.Level = logger.GetLevel() + config.Log.Level = common.GetLevel() } - logger.Info("Server mode", zap.String("mode", config.Server.Mode)) + common.Info("Server mode", zap.String("mode", config.Server.Mode)) // Print all configuration settings server.PrintAll() // Initialize database if err := dao.InitDB(); err != nil { - logger.Fatal("Failed to initialize database", zap.Error(err)) + common.Fatal("Failed to initialize database", zap.Error(err)) } // Initialize LLM factory data models from configuration file if err := dao.InitLLMFactory(); err != nil { - logger.Error("Failed to initialize LLM factory", err) + common.Error("Failed to initialize LLM factory", err) } else { - logger.Info("LLM factory initialized successfully") + common.Info("LLM factory initialized successfully") } // Initialize doc engine if err := engine.Init(&config.DocEngine); err != nil { - logger.Fatal("Failed to initialize doc engine", zap.Error(err)) + common.Fatal("Failed to initialize doc engine", zap.Error(err)) } defer engine.Close() // Initialize Redis cache if err := cache.Init(&config.Redis); err != nil { - logger.Fatal("Failed to initialize Redis", zap.Error(err)) + common.Fatal("Failed to initialize Redis", zap.Error(err)) } defer cache.Close() if err := storage.InitStorageFactory(); err != nil { - logger.Fatal("Failed to initialize storage factory", zap.Error(err)) + common.Fatal("Failed to initialize storage factory", zap.Error(err)) } // Initialize server variables (runtime variables that can change during operation) // This must be done after Cache is initialized if err := server.InitVariables(cache.Get()); err != nil { - logger.Warn("Failed to initialize server variables from Redis, using defaults", zap.String("error", err.Error())) + common.Warn("Failed to initialize server variables from Redis, using defaults", zap.String("error", err.Error())) } // Initialize admin status (default: unavailable=1) @@ -140,19 +139,19 @@ func main() { DictPath: "/usr/share/infinity/resource", } if err := tokenizer.Init(tokenizerCfg); err != nil { - logger.Fatal("Failed to initialize tokenizer", zap.Error(err)) + common.Fatal("Failed to initialize tokenizer", zap.Error(err)) } defer tokenizer.Close() // Initialize global QueryBuilder using tokenizer's DictPath // This ensures the Synonym uses the same wordnet directory as tokenizer if err := nlp.InitQueryBuilderFromTokenizer(tokenizerCfg.DictPath); err != nil { - logger.Fatal("Failed to initialize query builder", zap.Error(err)) + common.Fatal("Failed to initialize query builder", zap.Error(err)) } startServer(config) - logger.Info("Server exited") + common.Info("Server exited") } func startServer(config *server.Config) { @@ -231,36 +230,36 @@ func startServer(config *server.Config) { // Start server in a goroutine go func() { - logger.Info( + common.Info( "\n ____ ___ ______ ______ __\n" + " / __ \\ / | / ____// ____// /____ _ __\n" + " / /_/ // /| | / / __ / /_ / // __ \\| | /| / /\n" + " / _, _// ___ |/ /_/ // __/ / // /_/ /| |/ |/ /\n" + " /_/ |_|/_/ |_|\\____//_/ /_/ \\____/ |__/|__/\n", ) - logger.Info(fmt.Sprintf("RAGFlow Go Version: %s", utility.GetRAGFlowVersion())) - logger.Info(fmt.Sprintf("Server starting on port: %d", config.Server.Port)) + common.Info(fmt.Sprintf("RAGFlow Go Version: %s", utility.GetRAGFlowVersion())) + common.Info(fmt.Sprintf("Server starting on port: %d", config.Server.Port)) if err := srv.ListenAndServe(); err != nil && !errors.Is(err, http.ErrServerClosed) { - logger.Fatal("Failed to start server", zap.Error(err)) + common.Fatal("Failed to start server", zap.Error(err)) } }() // Get local IP address for heartbeat reporting localIP, err := utility.GetLocalIP() if err != nil { - logger.Fatal("fail to get local ip address") + common.Fatal("fail to get local ip address") } // Initialize and start heartbeat reporter to admin server heartbeatService := service.NewHeartbeatSender( - logger.Logger, + common.Logger, common.ServerTypeAPI, fmt.Sprintf("ragflow-server-%d", config.Server.Port), localIP, config.Server.Port, ) if err = heartbeatService.InitHTTPClient(); err != nil { - logger.Warn("Failed to initialize heartbeat service", zap.Error(err)) + common.Warn("Failed to initialize heartbeat service", zap.Error(err)) } else { // Start heartbeat reporter with 30 seconds interval heartbeatReporter := utility.NewScheduledTask("Heartbeat reporter", 3*time.Second, func() { @@ -280,8 +279,8 @@ func startServer(config *server.Config) { signal.Notify(quit, syscall.SIGINT, syscall.SIGTERM, syscall.SIGQUIT, syscall.SIGUSR2) sig := <-quit - logger.Info(fmt.Sprintf("Receives %s signal to shutdown server", strings.ToUpper(sig.String()))) - logger.Info("Shutting down server...") + common.Info(fmt.Sprintf("Receives %s signal to shutdown server", strings.ToUpper(sig.String()))) + common.Info("Shutting down server...") // Create context with timeout for graceful shutdown ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) @@ -289,6 +288,6 @@ func startServer(config *server.Config) { // Shutdown server if err = srv.Shutdown(ctx); err != nil { - logger.Fatal("Server forced to shutdown", zap.Error(err)) + common.Fatal("Server forced to shutdown", zap.Error(err)) } } diff --git a/internal/admin/handler.go b/internal/admin/handler.go index 61f77d509b..e083c825b4 100644 --- a/internal/admin/handler.go +++ b/internal/admin/handler.go @@ -22,7 +22,6 @@ import ( "net/http" "ragflow/internal/common" "ragflow/internal/dao" - "ragflow/internal/logger" "ragflow/internal/server" "ragflow/internal/service" "ragflow/internal/utility" @@ -1220,7 +1219,7 @@ func (h *Handler) HandleNoRoute(c *gin.Context) { // GetLogLevel returns the current log level func (h *Handler) GetLogLevel(c *gin.Context) { - level := logger.GetLevel() + level := common.GetLevel() success(c, gin.H{"level": level}, "") } @@ -1237,7 +1236,7 @@ func (h *Handler) SetLogLevel(c *gin.Context) { return } - if err := logger.SetLevel(req.Level); err != nil { + if err := common.SetLevel(req.Level); err != nil { errorResponse(c, err.Error(), 400) return } diff --git a/internal/admin/service.go b/internal/admin/service.go index 306a561d31..acd411f259 100644 --- a/internal/admin/service.go +++ b/internal/admin/service.go @@ -30,8 +30,6 @@ import ( "ragflow/internal/dao" "ragflow/internal/engine/elasticsearch" "ragflow/internal/entity" - "ragflow/internal/logger" - "ragflow/internal/server" "ragflow/internal/utility" "regexp" @@ -215,7 +213,7 @@ func (s *Service) CreateUser(username, password, role string) (map[string]interf // Rollback helper function rollbackTx := func() { if rbErr := tx.Rollback(); rbErr.Error != nil { - logger.Error("failed to rollback transaction", rbErr.Error) + common.Error("failed to rollback transaction", rbErr.Error) } } @@ -294,11 +292,11 @@ func (s *Service) CreateUser(username, password, role string) (map[string]interf // 4. Create tenant LLM configurations tenantLLMs, err := s.getInitTenantLLM(userID) if err != nil { - logger.Warn("failed to get init tenant LLM configs", zap.Error(err)) + common.Warn("failed to get init tenant LLM configs", zap.Error(err)) // Continue without LLM configs - not a critical error } else if len(tenantLLMs) > 0 { if err := tx.Create(&tenantLLMs).Error; err != nil { - logger.Warn("failed to create tenant LLM configs", zap.Error(err)) + common.Warn("failed to create tenant LLM configs", zap.Error(err)) // Continue without LLM configs - not a critical error } } @@ -332,7 +330,7 @@ func (s *Service) CreateUser(username, password, role string) (map[string]interf return nil, fmt.Errorf("failed to commit transaction: %w", err) } - logger.Info("Create user success with tenant and related data", zap.String("username", username)) + common.Info("Create user success with tenant and related data", zap.String("username", username)) return map[string]interface{}{ "id": user.ID, @@ -381,7 +379,7 @@ func (s *Service) getInitTenantLLM(userID string) ([]*entity.TenantLLM, error) { for _, factoryConfig := range uniqueFactories { llms, err := s.llmDAO.GetByFactory(factoryConfig.Factory) if err != nil { - logger.Warn("failed to get LLMs for factory", zap.String("factory", factoryConfig.Factory), zap.Error(err)) + common.Warn("failed to get LLMs for factory", zap.String("factory", factoryConfig.Factory), zap.Error(err)) continue } @@ -544,7 +542,7 @@ func (s *Service) DeleteUser(username string) (*DeleteUserResult, error) { // Get user-tenant relations tenants, err := s.userTenantDAO.GetByUserIDAll(user.ID) if err != nil { - logger.Warn("failed to get user-tenant relations", zap.Error(err)) + common.Warn("failed to get user-tenant relations", zap.Error(err)) } // Find owned tenant (role = "owner") @@ -565,7 +563,7 @@ func (s *Service) DeleteUser(username string) (*DeleteUserResult, error) { // Rollback helper function rollbackTx := func() { if rbErr := tx.Rollback(); rbErr.Error != nil { - logger.Error("failed to rollback transaction", rbErr.Error) + common.Error("failed to rollback transaction", rbErr.Error) } } @@ -575,14 +573,14 @@ func (s *Service) DeleteUser(username string) (*DeleteUserResult, error) { // 1. Get knowledge base IDs kbIDs, err := s.kbDAO.GetKBIDsByTenantIDSimple(ownedTenantID) if err != nil { - logger.Warn("failed to get knowledge base IDs", zap.Error(err)) + common.Warn("failed to get knowledge base IDs", zap.Error(err)) } if len(kbIDs) > 0 { // 2. Get document IDs docIDs, err := s.documentDAO.GetAllDocIDsByKBIDs(kbIDs) if err != nil { - logger.Warn("failed to get document IDs", zap.Error(err)) + common.Warn("failed to get document IDs", zap.Error(err)) } // 3. Delete tasks by document IDs @@ -592,58 +590,58 @@ func (s *Service) DeleteUser(username string) (*DeleteUserResult, error) { docIDList[i] = d["id"] } if delErr := tx.Unscoped().Where("doc_id IN ?", docIDList).Delete(&entity.Task{}); delErr.Error != nil { - logger.Warn("failed to delete tasks", zap.Error(delErr.Error)) + common.Warn("failed to delete tasks", zap.Error(delErr.Error)) } } // 4. Delete documents if delErr := tx.Unscoped().Where("kb_id IN ?", kbIDs).Delete(&entity.Document{}); delErr.Error != nil { - logger.Warn("failed to delete documents", zap.Error(delErr.Error)) + common.Warn("failed to delete documents", zap.Error(delErr.Error)) } // 5. Delete knowledge bases if delErr := tx.Unscoped().Where("id IN ?", kbIDs).Delete(&entity.Knowledgebase{}); delErr.Error != nil { - logger.Warn("failed to delete knowledge bases", zap.Error(delErr.Error)) + common.Warn("failed to delete knowledge bases", zap.Error(delErr.Error)) } } // 6. Delete files if delErr := tx.Unscoped().Where("tenant_id = ?", ownedTenantID).Delete(&entity.File{}); delErr.Error != nil { - logger.Warn("failed to delete files", zap.Error(delErr.Error)) + common.Warn("failed to delete files", zap.Error(delErr.Error)) } // 7. Delete user canvas (agents) if delErr := tx.Unscoped().Where("user_id = ?", ownedTenantID).Delete(&entity.UserCanvas{}); delErr.Error != nil { - logger.Warn("failed to delete user canvas", zap.Error(delErr.Error)) + common.Warn("failed to delete user canvas", zap.Error(delErr.Error)) } // 8. Get dialog IDs var dialogIDs []string if pluckErr := tx.Model(&entity.Chat{}).Where("tenant_id = ?", ownedTenantID).Pluck("id", &dialogIDs); pluckErr.Error != nil { - logger.Warn("failed to get dialog IDs", zap.Error(pluckErr.Error)) + common.Warn("failed to get dialog IDs", zap.Error(pluckErr.Error)) } // 9. Delete chat sessions if len(dialogIDs) > 0 { if delErr := tx.Unscoped().Where("dialog_id IN ?", dialogIDs).Delete(&entity.ChatSession{}); delErr.Error != nil { - logger.Warn("failed to delete chat sessions", zap.Error(delErr.Error)) + common.Warn("failed to delete chat sessions", zap.Error(delErr.Error)) } } // 10. Delete chats/dialogs if delErr := tx.Unscoped().Where("tenant_id = ?", ownedTenantID).Delete(&entity.Chat{}); delErr.Error != nil { - logger.Warn("failed to delete chats", zap.Error(delErr.Error)) + common.Warn("failed to delete chats", zap.Error(delErr.Error)) } // 11. Delete API tokens if delErr := tx.Unscoped().Where("tenant_id = ?", ownedTenantID).Delete(&entity.APIToken{}); delErr.Error != nil { - logger.Warn("failed to delete API tokens", zap.Error(delErr.Error)) + common.Warn("failed to delete API tokens", zap.Error(delErr.Error)) } // 12. Delete API4Conversations if len(dialogIDs) > 0 { if delErr := tx.Unscoped().Where("dialog_id IN ?", dialogIDs).Delete(&entity.API4Conversation{}); delErr.Error != nil { - logger.Warn("failed to delete API4Conversations", zap.Error(delErr.Error)) + common.Warn("failed to delete API4Conversations", zap.Error(delErr.Error)) } } @@ -661,7 +659,7 @@ func (s *Service) DeleteUser(username string) (*DeleteUserResult, error) { // 13. Delete tenant LLM configurations if delErr := tx.Unscoped().Where("tenant_id = ?", ownedTenantID).Delete(&entity.TenantLLM{}); delErr.Error != nil { - logger.Warn("failed to delete tenant LLM", zap.Error(delErr.Error)) + common.Warn("failed to delete tenant LLM", zap.Error(delErr.Error)) } var tenantCount int64 @@ -669,7 +667,7 @@ func (s *Service) DeleteUser(username string) (*DeleteUserResult, error) { result.TenantCount = int(tenantCount) // 14. Delete tenant if delErr := tx.Unscoped().Where("id = ?", ownedTenantID).Delete(&entity.Tenant{}); delErr.Error != nil { - logger.Warn("failed to delete tenant", zap.Error(delErr.Error)) + common.Warn("failed to delete tenant", zap.Error(delErr.Error)) } result.DeletedDetails = append(result.DeletedDetails, fmt.Sprintf("- Deleted %d tenant.", result.TenantCount)) } @@ -680,7 +678,7 @@ func (s *Service) DeleteUser(username string) (*DeleteUserResult, error) { // 15. Delete user-tenant relations if delErr := tx.Unscoped().Where("user_id = ?", user.ID).Delete(&entity.UserTenant{}); delErr.Error != nil { - logger.Warn("failed to delete user-tenant relations", zap.Error(delErr.Error)) + common.Warn("failed to delete user-tenant relations", zap.Error(delErr.Error)) } result.DeletedDetails = append(result.DeletedDetails, fmt.Sprintf("- Deleted %d user-tenant records.", result.UserTenantCount)) @@ -699,7 +697,7 @@ func (s *Service) DeleteUser(username string) (*DeleteUserResult, error) { result.DeletedDetails = append(result.DeletedDetails, "Delete done!") - logger.Info("Delete user success with all related data", zap.String("username", username)) + common.Info("Delete user success with all related data", zap.String("username", username)) return result, nil } diff --git a/internal/cache/redis.go b/internal/cache/redis.go index 36270e8b64..e56b172510 100644 --- a/internal/cache/redis.go +++ b/internal/cache/redis.go @@ -22,6 +22,7 @@ import ( "fmt" "math" "math/rand" + "ragflow/internal/common" "strconv" "sync" "time" @@ -30,7 +31,6 @@ import ( "github.com/redis/go-redis/v9" "go.uber.org/zap" - "ragflow/internal/logger" "ragflow/internal/server" ) @@ -109,7 +109,7 @@ func Init(cfg *server.RedisConfig) error { var initErr error once.Do(func() { if cfg.Host == "" { - logger.Info("Redis host not configured, skipping Redis initialization") + common.Info("Redis host not configured, skipping Redis initialization") return } @@ -135,7 +135,7 @@ func Init(cfg *server.RedisConfig) error { luaTokenBucket: redis.NewScript(luaTokenBucketScript), } - logger.Info("Redis client initialized", + common.Info("Redis client initialized", zap.String("host", cfg.Host), zap.Int("port", cfg.Port), zap.Int("db", cfg.DB), @@ -193,7 +193,7 @@ func (r *RedisClient) Info() map[string]interface{} { ctx := context.Background() infoStr, err := r.client.Info(ctx).Result() if err != nil { - logger.Warn("Failed to get Redis info", zap.Error(err)) + common.Warn("Failed to get Redis info", zap.Error(err)) return nil } @@ -281,7 +281,7 @@ func (r *RedisClient) Exist(key string) (bool, error) { ctx := context.Background() exists, err := r.client.Exists(ctx, key).Result() if err != nil { - logger.Warn("Redis Exist error", zap.String("key", key), zap.Error(err)) + common.Warn("Redis Exist error", zap.String("key", key), zap.Error(err)) return false, err } return exists > 0, nil @@ -298,7 +298,7 @@ func (r *RedisClient) Get(key string) (string, error) { return "", nil } if err != nil { - logger.Warn("Redis Get error", zap.String("key", key), zap.Error(err)) + common.Warn("Redis Get error", zap.String("key", key), zap.Error(err)) return "", err } return val, nil @@ -312,11 +312,11 @@ func (r *RedisClient) SetObj(key string, obj interface{}, exp time.Duration) boo ctx := context.Background() data, err := json.Marshal(obj) if err != nil { - logger.Warn("Redis SetObj marshal error", zap.String("key", key), zap.Error(err)) + common.Warn("Redis SetObj marshal error", zap.String("key", key), zap.Error(err)) return false } if err := r.client.Set(ctx, key, data, exp).Err(); err != nil { - logger.Warn("Redis SetObj error", zap.String("key", key), zap.Error(err)) + common.Warn("Redis SetObj error", zap.String("key", key), zap.Error(err)) return false } return true @@ -333,11 +333,11 @@ func (r *RedisClient) GetObj(key string, dest interface{}) bool { return false } if err != nil { - logger.Warn("Redis GetObj error", zap.String("key", key), zap.Error(err)) + common.Warn("Redis GetObj error", zap.String("key", key), zap.Error(err)) return false } if err := json.Unmarshal([]byte(data), dest); err != nil { - logger.Warn("Redis GetObj unmarshal error", zap.String("key", key), zap.Error(err)) + common.Warn("Redis GetObj unmarshal error", zap.String("key", key), zap.Error(err)) return false } return true @@ -350,7 +350,7 @@ func (r *RedisClient) Set(key string, value string, exp time.Duration) bool { } ctx := context.Background() if err := r.client.Set(ctx, key, value, exp).Err(); err != nil { - logger.Warn("Redis Set error", zap.String("key", key), zap.Error(err)) + common.Warn("Redis Set error", zap.String("key", key), zap.Error(err)) return false } return true @@ -364,7 +364,7 @@ func (r *RedisClient) SetNX(key string, value string, exp time.Duration) bool { ctx := context.Background() ok, err := r.client.SetNX(ctx, key, value, exp).Result() if err != nil { - logger.Warn("Redis SetNX error", zap.String("key", key), zap.Error(err)) + common.Warn("Redis SetNX error", zap.String("key", key), zap.Error(err)) return false } return ok @@ -380,7 +380,7 @@ func (r *RedisClient) GetOrCreateKey(key string, value string) (string, error) { // First, try to get the existing key existingKey, err := r.client.Get(ctx, key).Result() if err == nil { - logger.Warn("Redis Get error", zap.String("key", key), zap.Error(err)) + common.Warn("Redis Get error", zap.String("key", key), zap.Error(err)) // Successfully retrieved existing key return existingKey, nil } @@ -414,7 +414,7 @@ func (r *RedisClient) SAdd(key string, member string) bool { } ctx := context.Background() if err := r.client.SAdd(ctx, key, member).Err(); err != nil { - logger.Warn("Redis SAdd error", zap.String("key", key), zap.Error(err)) + common.Warn("Redis SAdd error", zap.String("key", key), zap.Error(err)) return false } return true @@ -427,7 +427,7 @@ func (r *RedisClient) SRem(key string, member string) bool { } ctx := context.Background() if err := r.client.SRem(ctx, key, member).Err(); err != nil { - logger.Warn("Redis SRem error", zap.String("key", key), zap.Error(err)) + common.Warn("Redis SRem error", zap.String("key", key), zap.Error(err)) return false } return true @@ -441,7 +441,7 @@ func (r *RedisClient) SMembers(key string) ([]string, error) { ctx := context.Background() members, err := r.client.SMembers(ctx, key).Result() if err != nil { - logger.Warn("Redis SMembers error", zap.String("key", key), zap.Error(err)) + common.Warn("Redis SMembers error", zap.String("key", key), zap.Error(err)) return nil, err } return members, nil @@ -455,7 +455,7 @@ func (r *RedisClient) SIsMember(key string, member string) bool { ctx := context.Background() ok, err := r.client.SIsMember(ctx, key, member).Result() if err != nil { - logger.Warn("Redis SIsMember error", zap.String("key", key), zap.Error(err)) + common.Warn("Redis SIsMember error", zap.String("key", key), zap.Error(err)) return false } return ok @@ -468,7 +468,7 @@ func (r *RedisClient) ZAdd(key string, member string, score float64) bool { } ctx := context.Background() if err := r.client.ZAdd(ctx, key, redis.Z{Score: score, Member: member}).Err(); err != nil { - logger.Warn("Redis ZAdd error", zap.String("key", key), zap.Error(err)) + common.Warn("Redis ZAdd error", zap.String("key", key), zap.Error(err)) return false } return true @@ -482,7 +482,7 @@ func (r *RedisClient) ZCount(key string, min, max float64) int64 { ctx := context.Background() count, err := r.client.ZCount(ctx, key, fmt.Sprintf("%f", min), fmt.Sprintf("%f", max)).Result() if err != nil { - logger.Warn("Redis ZCount error", zap.String("key", key), zap.Error(err)) + common.Warn("Redis ZCount error", zap.String("key", key), zap.Error(err)) return 0 } return count @@ -496,7 +496,7 @@ func (r *RedisClient) ZPopMin(key string, count int) ([]redis.Z, error) { ctx := context.Background() members, err := r.client.ZPopMin(ctx, key, int64(count)).Result() if err != nil { - logger.Warn("Redis ZPopMin error", zap.String("key", key), zap.Error(err)) + common.Warn("Redis ZPopMin error", zap.String("key", key), zap.Error(err)) return nil, err } return members, nil @@ -513,7 +513,7 @@ func (r *RedisClient) ZRangeByScore(key string, min, max float64) ([]string, err Max: fmt.Sprintf("%f", max), }).Result() if err != nil { - logger.Warn("Redis ZRangeByScore error", zap.String("key", key), zap.Error(err)) + common.Warn("Redis ZRangeByScore error", zap.String("key", key), zap.Error(err)) return nil, err } return members, nil @@ -527,7 +527,7 @@ func (r *RedisClient) ZRemRangeByScore(key string, min, max float64) int64 { ctx := context.Background() count, err := r.client.ZRemRangeByScore(ctx, key, fmt.Sprintf("%f", min), fmt.Sprintf("%f", max)).Result() if err != nil { - logger.Warn("Redis ZRemRangeByScore error", zap.String("key", key), zap.Error(err)) + common.Warn("Redis ZRemRangeByScore error", zap.String("key", key), zap.Error(err)) return 0 } return count @@ -541,7 +541,7 @@ func (r *RedisClient) IncrBy(key string, increment int64) (int64, error) { ctx := context.Background() val, err := r.client.IncrBy(ctx, key, increment).Result() if err != nil { - logger.Warn("Redis IncrBy error", zap.String("key", key), zap.Error(err)) + common.Warn("Redis IncrBy error", zap.String("key", key), zap.Error(err)) return 0, err } return val, nil @@ -555,7 +555,7 @@ func (r *RedisClient) DecrBy(key string, decrement int64) (int64, error) { ctx := context.Background() val, err := r.client.DecrBy(ctx, key, decrement).Result() if err != nil { - logger.Warn("Redis DecrBy error", zap.String("key", key), zap.Error(err)) + common.Warn("Redis DecrBy error", zap.String("key", key), zap.Error(err)) return 0, err } return val, nil @@ -582,7 +582,7 @@ func (r *RedisClient) GenerateAutoIncrementID(keyPrefix string, namespace string // Check if key exists exists, err := r.client.Exists(ctx, redisKey).Result() if err != nil { - logger.Warn("Redis GenerateAutoIncrementID error", zap.Error(err)) + common.Warn("Redis GenerateAutoIncrementID error", zap.Error(err)) return -1 } @@ -604,7 +604,7 @@ func (r *RedisClient) GenerateAutoIncrementID(keyPrefix string, namespace string // Increment nextID, err := r.client.IncrBy(ctx, redisKey, increment).Result() if err != nil { - logger.Warn("Redis GenerateAutoIncrementID increment error", zap.Error(err)) + common.Warn("Redis GenerateAutoIncrementID increment error", zap.Error(err)) return -1 } @@ -621,7 +621,7 @@ func (r *RedisClient) Transaction(key string, value string, exp time.Duration) b pipe.SetNX(ctx, key, value, exp) _, err := pipe.Exec(ctx) if err != nil { - logger.Warn("Redis Transaction error", zap.String("key", key), zap.Error(err)) + common.Warn("Redis Transaction error", zap.String("key", key), zap.Error(err)) return false } return true @@ -637,7 +637,7 @@ func (r *RedisClient) QueueProduct(queue string, message interface{}) bool { for i := 0; i < 3; i++ { data, err := json.Marshal(message) if err != nil { - logger.Warn("Redis QueueProduct marshal error", zap.Error(err)) + common.Warn("Redis QueueProduct marshal error", zap.Error(err)) return false } @@ -648,7 +648,7 @@ func (r *RedisClient) QueueProduct(queue string, message interface{}) bool { if err == nil { return true } - logger.Warn("Redis QueueProduct error", zap.String("queue", queue), zap.Error(err)) + common.Warn("Redis QueueProduct error", zap.String("queue", queue), zap.Error(err)) time.Sleep(100 * time.Millisecond) } return false @@ -665,7 +665,7 @@ func (r *RedisClient) QueueConsumer(queueName, groupName, consumerName string, m // Create consumer group if not exists groups, err := r.client.XInfoGroups(ctx, queueName).Result() if err != nil && err.Error() != "no such key" { - logger.Warn("Redis QueueConsumer XInfoGroups error", zap.Error(err)) + common.Warn("Redis QueueConsumer XInfoGroups error", zap.Error(err)) } groupExists := false @@ -679,7 +679,7 @@ func (r *RedisClient) QueueConsumer(queueName, groupName, consumerName string, m if !groupExists { err = r.client.XGroupCreateMkStream(ctx, queueName, groupName, "0").Err() if err != nil && err.Error() != "BUSYGROUP Consumer Group name already exists" { - logger.Warn("Redis QueueConsumer XGroupCreate error", zap.Error(err)) + common.Warn("Redis QueueConsumer XGroupCreate error", zap.Error(err)) } } @@ -699,7 +699,7 @@ func (r *RedisClient) QueueConsumer(queueName, groupName, consumerName string, m return nil, nil } if err != nil { - logger.Warn("Redis QueueConsumer XReadGroup error", zap.Error(err)) + common.Warn("Redis QueueConsumer XReadGroup error", zap.Error(err)) time.Sleep(100 * time.Millisecond) continue } @@ -733,7 +733,7 @@ func (m *RedisMsg) Ack() bool { ctx := context.Background() err := m.consumer.XAck(ctx, m.queueName, m.groupName, m.msgID).Err() if err != nil { - logger.Warn("RedisMsg Ack error", zap.Error(err)) + common.Warn("RedisMsg Ack error", zap.Error(err)) return false } return true @@ -764,7 +764,7 @@ func (r *RedisClient) GetPendingMsg(queue, groupName string) ([]redis.XPendingEx }).Result() if err != nil { if err.Error() != "No such key" { - logger.Warn("Redis GetPendingMsg error", zap.Error(err)) + common.Warn("Redis GetPendingMsg error", zap.Error(err)) } return nil, err } @@ -781,7 +781,7 @@ func (r *RedisClient) RequeueMsg(queue, groupName, msgID string) { for i := 0; i < 3; i++ { msgs, err := r.client.XRange(ctx, queue, msgID, msgID).Result() if err != nil { - logger.Warn("Redis RequeueMsg XRange error", zap.Error(err)) + common.Warn("Redis RequeueMsg XRange error", zap.Error(err)) time.Sleep(100 * time.Millisecond) continue } @@ -808,7 +808,7 @@ func (r *RedisClient) QueueInfo(queue, groupName string) (map[string]interface{} for i := 0; i < 3; i++ { groups, err := r.client.XInfoGroups(ctx, queue).Result() if err != nil { - logger.Warn("Redis QueueInfo error", zap.Error(err)) + common.Warn("Redis QueueInfo error", zap.Error(err)) time.Sleep(100 * time.Millisecond) continue } @@ -836,7 +836,7 @@ func (r *RedisClient) DeleteIfEqual(key, expectedValue string) bool { ctx := context.Background() result, err := r.luaDeleteIfEqual.Run(ctx, r.client, []string{key}, expectedValue).Result() if err != nil { - logger.Warn("Redis DeleteIfEqual error", zap.Error(err)) + common.Warn("Redis DeleteIfEqual error", zap.Error(err)) return false } return result.(int64) == 1 @@ -849,7 +849,7 @@ func (r *RedisClient) Delete(key string) bool { } ctx := context.Background() if err := r.client.Del(ctx, key).Err(); err != nil { - logger.Warn("Redis Delete error", zap.String("key", key), zap.Error(err)) + common.Warn("Redis Delete error", zap.String("key", key), zap.Error(err)) return false } return true @@ -862,7 +862,7 @@ func (r *RedisClient) Expire(key string, exp time.Duration) bool { } ctx := context.Background() if err := r.client.Expire(ctx, key, exp).Err(); err != nil { - logger.Warn("Redis Expire error", zap.String("key", key), zap.Error(err)) + common.Warn("Redis Expire error", zap.String("key", key), zap.Error(err)) return false } return true @@ -876,7 +876,7 @@ func (r *RedisClient) TTL(key string) time.Duration { ctx := context.Background() ttl, err := r.client.TTL(ctx, key).Result() if err != nil { - logger.Warn("Redis TTL error", zap.String("key", key), zap.Error(err)) + common.Warn("Redis TTL error", zap.String("key", key), zap.Error(err)) return -2 } return ttl @@ -974,7 +974,7 @@ func (tb *TokenBucket) Allow(cost float64) (bool, float64) { result, err := tb.client.luaTokenBucket.Run(ctx, tb.client.client, []string{tb.key}, tb.capacity, tb.rate, now, cost).Result() if err != nil { - logger.Warn("TokenBucket Allow error", zap.Error(err)) + common.Warn("TokenBucket Allow error", zap.Error(err)) return true, 0 } diff --git a/internal/cli/filesystem/skill.go b/internal/cli/filesystem/skill.go index bb66b2af71..1664eed60b 100644 --- a/internal/cli/filesystem/skill.go +++ b/internal/cli/filesystem/skill.go @@ -25,6 +25,7 @@ import ( "net/url" "os" "path/filepath" + "ragflow/internal/common" "regexp" "strings" "time" @@ -32,8 +33,6 @@ import ( "github.com/google/uuid" "go.uber.org/zap" "gopkg.in/yaml.v3" - - "ragflow/internal/logger" ) // SkillProvider handles skill operations using /skills API @@ -56,7 +55,7 @@ import ( const ( MaxSkillTotalSize = 50 * 1024 * 1024 // 50MB MaxSkillFileSize = 5 * 1024 * 1024 // 5MB per file - DefaultSpaceID = "default" + DefaultSpaceID = "default" ) // Text file extensions allowed in skills @@ -166,7 +165,7 @@ func (p *SkillProvider) List(ctx stdctx.Context, subPath string, opts *ListOptio } parts := SplitPath(subPath) - + switch len(parts) { case 1: // skills/{space_id} - list skills in space @@ -214,10 +213,10 @@ func (p *SkillProvider) Search(ctx stdctx.Context, subPath string, opts *SearchO page = (opts.Offset / pageSize) + 1 } payload := map[string]interface{}{ - "query": opts.Query, - "space_id": spaceID, - "page": page, - "page_size": pageSize, + "query": opts.Query, + "space_id": spaceID, + "page": page, + "page_size": pageSize, } // Call skill search API @@ -538,7 +537,7 @@ func (p *SkillProvider) listSkillsInSpace(ctx stdctx.Context, spaceName string, "sort_order": opts.SortOrder, } - logger.Debug("Listing skills via search API", zap.String("space", spaceName), zap.String("spaceUUID", spaceUUID), zap.Int("limit", limit)) + common.Debug("Listing skills via search API", zap.String("space", spaceName), zap.String("spaceUUID", spaceUUID), zap.Int("limit", limit)) resp, err := p.httpClient.Request("POST", "/skills/search", true, "auto", nil, payload) if err == nil { @@ -560,7 +559,7 @@ func (p *SkillProvider) listSkillsInSpace(ctx stdctx.Context, spaceName string, } if err := json.Unmarshal(resp.Body, &result); err == nil && result.Code == 0 { - logger.Debug("Search API response", zap.Int("skills_count", len(result.Data.Skills)), zap.Int64("total", result.Data.Total)) + common.Debug("Search API response", zap.Int("skills_count", len(result.Data.Skills)), zap.Int64("total", result.Data.Total)) // If search returned results, use them if len(result.Data.Skills) > 0 { nodes := make([]*Node, 0, len(result.Data.Skills)) @@ -582,7 +581,7 @@ func (p *SkillProvider) listSkillsInSpace(ctx stdctx.Context, spaceName string, }, }) } - logger.Info("Listed skills via SEARCH", zap.String("space", spaceName), zap.Int("count", len(nodes)), zap.Int64("total", result.Data.Total)) + common.Info("Listed skills via SEARCH", zap.String("space", spaceName), zap.Int("count", len(nodes)), zap.Int64("total", result.Data.Total)) return &Result{ Nodes: nodes, Total: int(result.Data.Total), @@ -591,16 +590,16 @@ func (p *SkillProvider) listSkillsInSpace(ctx stdctx.Context, spaceName string, }, nil } // Search returned empty result, fall through to file system - logger.Debug("Search returned empty result, falling back to file system") + common.Debug("Search returned empty result, falling back to file system") } else { - logger.Debug("Search API error", zap.Error(err), zap.Int("code", result.Code), zap.String("msg", result.Msg)) + common.Debug("Search API error", zap.Error(err), zap.Int("code", result.Code), zap.String("msg", result.Msg)) } } else { - logger.Debug("Search request failed", zap.Error(err)) + common.Debug("Search request failed", zap.Error(err)) } // Fall back to file system listing (for skills not yet indexed) - logger.Info("Listing skills via FILE SYSTEM (search unavailable)", zap.String("space", spaceName)) + common.Info("Listing skills via FILE SYSTEM (search unavailable)", zap.String("space", spaceName)) return p.listSkillsInSpaceFromFileSystem(ctx, spaceName, opts) } @@ -611,14 +610,14 @@ func (p *SkillProvider) listSkillsInSpaceFromFileSystem(ctx stdctx.Context, spac if err != nil { return nil, fmt.Errorf("failed to get skills folder: %w", err) } - logger.Debug("Got skills folder ID", zap.String("skillsFolderID", skillsFolderID)) + common.Debug("Got skills folder ID", zap.String("skillsFolderID", skillsFolderID)) // Find the space folder spaceFolderID, err := p.findFolderID(ctx, skillsFolderID, spaceName) if err != nil { return nil, fmt.Errorf("failed to find space folder: %w", err) } - logger.Debug("Got space folder ID", zap.String("spaceName", spaceName), zap.String("spaceFolderID", spaceFolderID)) + common.Debug("Got space folder ID", zap.String("spaceName", spaceName), zap.String("spaceFolderID", spaceFolderID)) // List all subfolders in the space folder (each subfolder is a skill) skillsResp, err := p.httpClient.Request("GET", fmt.Sprintf("/files?parent_id=%s", spaceFolderID), true, "auto", nil, nil) @@ -646,7 +645,7 @@ func (p *SkillProvider) listSkillsInSpaceFromFileSystem(ctx stdctx.Context, spac if skillsResult.Code != 0 { return nil, fmt.Errorf("failed to list skills: %s", skillsResult.Msg) } - logger.Debug("File system list response", zap.Int("files_count", len(skillsResult.Data.Files))) + common.Debug("File system list response", zap.Int("files_count", len(skillsResult.Data.Files))) // Convert folders to nodes nodes := make([]*Node, 0) @@ -675,7 +674,7 @@ func (p *SkillProvider) listSkillsInSpaceFromFileSystem(ctx stdctx.Context, spac nodes = nodes[:limit] } - logger.Info("Listed skills via FILE SYSTEM", zap.String("space", spaceName), zap.Int("count", len(nodes)), zap.Int("total", total)) + common.Info("Listed skills via FILE SYSTEM", zap.String("space", spaceName), zap.Int("count", len(nodes)), zap.Int("total", total)) return &Result{ Nodes: nodes, @@ -1039,10 +1038,10 @@ func (p *SkillProvider) listSkillContent(ctx stdctx.Context, spaceID, skillName, } nodes = append(nodes, &Node{ - Name: file.Name, - Type: nodeType, - Path: currentPath + "/" + file.Name, - Size: file.Size, + Name: file.Name, + Type: nodeType, + Path: currentPath + "/" + file.Name, + Size: file.Size, UpdatedAt: time.UnixMilli(file.UpdateTime), Metadata: map[string]interface{}{ "id": file.ID, @@ -1142,9 +1141,9 @@ func (p *SkillProvider) IndexSkill(ctx stdctx.Context, spaceID string, skillInfo // Build index request payload := map[string]interface{}{ - "skills": []interface{}{skillInfo}, + "skills": []interface{}{skillInfo}, "space_id": spaceUUID, - "embd_id": embdID, + "embd_id": embdID, } // Call index API diff --git a/internal/logger/logger.go b/internal/common/logger.go similarity index 99% rename from internal/logger/logger.go rename to internal/common/logger.go index f01f613ecf..8650ba06c6 100644 --- a/internal/logger/logger.go +++ b/internal/common/logger.go @@ -14,7 +14,7 @@ // limitations under the License. // -package logger +package common import ( "fmt" diff --git a/internal/dao/database.go b/internal/dao/database.go index cb33929d43..fe525959ae 100644 --- a/internal/dao/database.go +++ b/internal/dao/database.go @@ -22,12 +22,11 @@ import ( "log" "os" "path/filepath" + "ragflow/internal/common" "ragflow/internal/entity" "strings" "time" - "ragflow/internal/logger" - "ragflow/internal/server" "ragflow/internal/utility" @@ -166,13 +165,13 @@ func InitDB() error { return fmt.Errorf("failed to run manual migrations: %w", err) } - logger.Info("Database connected and migrated successfully") + common.Info("Database connected and migrated successfully") modelProviderManager, err = entity.NewProviderManager("conf/models") if err != nil { log.Fatal("Failed to load model providers:", err) } - logger.Info("Model providers loaded successfully") + common.Info("Model providers loaded successfully") return nil } @@ -197,17 +196,17 @@ func autoMigrateSafely(db *gorm.DB, model interface{}) error { // Check if error is MySQL duplicate index error (Error 1061) errStr := err.Error() if strings.Contains(errStr, "Error 1061") && strings.Contains(errStr, "Duplicate key name") { - logger.Info("Index already exists, skipping", zap.String("error", errStr)) + common.Info("Index already exists, skipping", zap.String("error", errStr)) return nil } if strings.Contains(errStr, "Error 1060") && strings.Contains(errStr, "Duplicate column name") { - logger.Info("Column already exists, skipping", zap.String("error", errStr)) + common.Info("Column already exists, skipping", zap.String("error", errStr)) return nil } if strings.Contains(errStr, "Error 1050") && strings.Contains(errStr, "Table") { - logger.Info("Table already exists, skipping", zap.String("error", errStr)) + common.Info("Table already exists, skipping", zap.String("error", errStr)) return nil } diff --git a/internal/dao/migration.go b/internal/dao/migration.go index d7f227a5a9..ca5bd3d06b 100644 --- a/internal/dao/migration.go +++ b/internal/dao/migration.go @@ -18,8 +18,8 @@ package dao import ( "fmt" + "ragflow/internal/common" "ragflow/internal/entity" - "ragflow/internal/logger" "strings" "go.uber.org/zap" @@ -62,7 +62,7 @@ func RunMigrations(db *gorm.DB) error { return fmt.Errorf("failed to migrate skill space tables: %w", err) } - logger.Info("All manual migrations completed successfully") + common.Info("All manual migrations completed successfully") return nil } @@ -102,7 +102,7 @@ func migrateTenantLLMPrimaryKey(db *gorm.DB) error { } } - logger.Info("Migrating tenant_llm to use ID primary key...") + common.Info("Migrating tenant_llm to use ID primary key...") // Start transaction return db.Transaction(func(tx *gorm.DB) error { @@ -112,7 +112,7 @@ func migrateTenantLLMPrimaryKey(db *gorm.DB) error { WHERE TABLE_NAME = 'tenant_llm' AND COLUMN_NAME = 'temp_id'`).Scan(&tempIdExists) if tempIdExists > 0 { if err := tx.Exec("ALTER TABLE tenant_llm DROP COLUMN temp_id").Error; err != nil { - logger.Warn("Failed to drop temp_id column", zap.Error(err)) + common.Warn("Failed to drop temp_id column", zap.Error(err)) } } @@ -144,11 +144,11 @@ func migrateTenantLLMPrimaryKey(db *gorm.DB) error { ALTER TABLE tenant_llm ADD UNIQUE INDEX idx_tenant_llm_unique (tenant_id, llm_factory, llm_name) `).Error; err != nil { - logger.Warn("Failed to add unique index idx_tenant_llm_unique", zap.Error(err)) + common.Warn("Failed to add unique index idx_tenant_llm_unique", zap.Error(err)) } } - logger.Info("tenant_llm primary key migration completed") + common.Info("tenant_llm primary key migration completed") return nil }) } @@ -179,17 +179,17 @@ func migrateAddUniqueEmail(db *gorm.DB) error { } if duplicateCount > 0 { - logger.Warn("Found duplicate emails in user table, cannot add unique index", zap.Int64("count", duplicateCount)) + common.Warn("Found duplicate emails in user table, cannot add unique index", zap.Int64("count", duplicateCount)) return nil } - logger.Info("Adding unique index on user.email...") + common.Info("Adding unique index on user.email...") if err = db.Exec(`ALTER TABLE user ADD UNIQUE INDEX idx_user_email_unique (email)`).Error; err != nil { // Check if error is MySQL duplicate index error (Error 1061) errStr := err.Error() if strings.Contains(errStr, "Error 1061") && strings.Contains(errStr, "Duplicate key name") { - logger.Info("Index already exists, skipping", zap.String("error", errStr)) + common.Info("Index already exists, skipping", zap.String("error", errStr)) return nil } return fmt.Errorf("failed to add unique index on email: %w", err) @@ -211,21 +211,21 @@ func modifyColumnTypes(db *gorm.DB) error { // dialog.top_k: ensure it's INTEGER with default 1024 if db.Migrator().HasTable("dialog") && columnExists("dialog", "top_k") { if err := db.Exec(`ALTER TABLE dialog MODIFY COLUMN top_k BIGINT NOT NULL DEFAULT 1024`).Error; err != nil { - logger.Warn("Failed to modify dialog.top_k", zap.Error(err)) + common.Warn("Failed to modify dialog.top_k", zap.Error(err)) } } // tenant_llm.api_key: ensure it's TEXT type if db.Migrator().HasTable("tenant_llm") && columnExists("tenant_llm", "api_key") { if err := db.Exec(`ALTER TABLE tenant_llm MODIFY COLUMN api_key LONGTEXT`).Error; err != nil { - logger.Warn("Failed to modify tenant_llm.api_key", zap.Error(err)) + common.Warn("Failed to modify tenant_llm.api_key", zap.Error(err)) } } // api_token.dialog_id: ensure it's varchar(32) if db.Migrator().HasTable("api_token") && columnExists("api_token", "dialog_id") { if err := db.Exec(`ALTER TABLE api_token MODIFY COLUMN dialog_id VARCHAR(32)`).Error; err != nil { - logger.Warn("Failed to modify api_token.dialog_id", zap.Error(err)) + common.Warn("Failed to modify api_token.dialog_id", zap.Error(err)) } } @@ -234,12 +234,12 @@ func modifyColumnTypes(db *gorm.DB) error { if db.Migrator().HasTable("canvas_template") { if columnExists("canvas_template", "title") { if err := db.Exec(`ALTER TABLE canvas_template MODIFY COLUMN title LONGTEXT NULL`).Error; err != nil { - logger.Warn("Failed to modify canvas_template.title", zap.Error(err)) + common.Warn("Failed to modify canvas_template.title", zap.Error(err)) } } if columnExists("canvas_template", "description") { if err := db.Exec(`ALTER TABLE canvas_template MODIFY COLUMN description LONGTEXT NULL`).Error; err != nil { - logger.Warn("Failed to modify canvas_template.description", zap.Error(err)) + common.Warn("Failed to modify canvas_template.description", zap.Error(err)) } } } @@ -247,21 +247,21 @@ func modifyColumnTypes(db *gorm.DB) error { // system_settings.value: ensure it's LONGTEXT if db.Migrator().HasTable("system_settings") && columnExists("system_settings", "value") { if err := db.Exec(`ALTER TABLE system_settings MODIFY COLUMN value LONGTEXT NOT NULL`).Error; err != nil { - logger.Warn("Failed to modify system_settings.value", zap.Error(err)) + common.Warn("Failed to modify system_settings.value", zap.Error(err)) } } // knowledgebase.raptor_task_finish_at: ensure it's DateTime if db.Migrator().HasTable("knowledgebase") && columnExists("knowledgebase", "raptor_task_finish_at") { if err := db.Exec(`ALTER TABLE knowledgebase MODIFY COLUMN raptor_task_finish_at DATETIME`).Error; err != nil { - logger.Warn("Failed to modify knowledgebase.raptor_task_finish_at", zap.Error(err)) + common.Warn("Failed to modify knowledgebase.raptor_task_finish_at", zap.Error(err)) } } // knowledgebase.mindmap_task_finish_at: ensure it's DateTime if db.Migrator().HasTable("knowledgebase") && columnExists("knowledgebase", "mindmap_task_finish_at") { if err := db.Exec(`ALTER TABLE knowledgebase MODIFY COLUMN mindmap_task_finish_at DATETIME`).Error; err != nil { - logger.Warn("Failed to modify knowledgebase.mindmap_task_finish_at", zap.Error(err)) + common.Warn("Failed to modify knowledgebase.mindmap_task_finish_at", zap.Error(err)) } } @@ -290,14 +290,14 @@ func renameColumnIfExists(db *gorm.DB, tableName, oldName, newName string) error // Check if new column already exists if columnExists(newName) { // Both exist, drop the old one - logger.Warn("Both old and new columns exist, dropping old one", + common.Warn("Both old and new columns exist, dropping old one", zap.String("table", tableName), zap.String("oldColumn", oldName), zap.String("newColumn", newName)) return db.Migrator().DropColumn(tableName, oldName) } - logger.Info("Renaming column", + common.Info("Renaming column", zap.String("table", tableName), zap.String("oldColumn", oldName), zap.String("newColumn", newName)) @@ -318,7 +318,7 @@ func addColumnIfNotExists(db *gorm.DB, tableName, columnName, columnDef string) return nil } - logger.Info("Adding column", + common.Info("Adding column", zap.String("table", tableName), zap.String("column", columnName)) sql := fmt.Sprintf("ALTER TABLE %s ADD COLUMN %s %s", tableName, columnName, columnDef) @@ -329,7 +329,7 @@ func addColumnIfNotExists(db *gorm.DB, tableName, columnName, columnDef string) func migrateSkillSearchTables(db *gorm.DB) error { // Create skill_search_configs table only if !db.Migrator().HasTable("skill_search_configs") { - logger.Info("Creating skill_search_configs table...") + common.Info("Creating skill_search_configs table...") sql := ` CREATE TABLE IF NOT EXISTS skill_search_configs ( id VARCHAR(32) PRIMARY KEY, @@ -352,12 +352,12 @@ func migrateSkillSearchTables(db *gorm.DB) error { ) ` if err := db.Exec(sql).Error; err != nil { - logger.Warn("Failed to create skill_search_configs table with MySQL dialect, trying generic", zap.Error(err)) + common.Warn("Failed to create skill_search_configs table with MySQL dialect, trying generic", zap.Error(err)) if err := db.AutoMigrate(&entity.SkillSearchConfig{}); err != nil { return err } // AutoMigrate doesn't create unique indexes, so create them explicitly - logger.Info("Creating unique indexes for skill_search_configs...") + common.Info("Creating unique indexes for skill_search_configs...") if err := db.Exec(`ALTER TABLE skill_search_configs ADD UNIQUE INDEX idx_tenant_space_embd (tenant_id, space_id, embd_id)`).Error; err != nil { return fmt.Errorf("failed to create unique index idx_tenant_space_embd: %w", err) } @@ -373,7 +373,7 @@ func migrateSkillSearchTables(db *gorm.DB) error { db.Raw(`SELECT COUNT(*) FROM INFORMATION_SCHEMA.STATISTICS WHERE TABLE_NAME = 'skill_search_configs' AND INDEX_NAME = 'idx_tenant_embd'`).Scan(&legacyIndexExists) if legacyIndexExists > 0 { - logger.Info("Dropping legacy unique index idx_tenant_embd from skill_search_configs...") + common.Info("Dropping legacy unique index idx_tenant_embd from skill_search_configs...") if err := db.Exec(`ALTER TABLE skill_search_configs DROP INDEX idx_tenant_embd`).Error; err != nil { return fmt.Errorf("failed to drop legacy unique index idx_tenant_embd: %w", err) } @@ -384,7 +384,7 @@ func migrateSkillSearchTables(db *gorm.DB) error { db.Raw(`SELECT COUNT(*) FROM INFORMATION_SCHEMA.STATISTICS WHERE TABLE_NAME = 'skill_search_configs' AND INDEX_NAME = 'idx_tenant_space_embd'`).Scan(&indexExists) if indexExists == 0 { - logger.Info("Adding unique index idx_tenant_space_embd to skill_search_configs...") + common.Info("Adding unique index idx_tenant_space_embd to skill_search_configs...") if err := db.Exec(`ALTER TABLE skill_search_configs ADD UNIQUE INDEX idx_tenant_space_embd (tenant_id, space_id, embd_id)`).Error; err != nil { return fmt.Errorf("failed to add unique index idx_tenant_space_embd: %w", err) @@ -398,7 +398,7 @@ func migrateSkillSearchTables(db *gorm.DB) error { // migrateSkillSpaceTables creates skill space related tables func migrateSkillSpaceTables(db *gorm.DB) error { if !db.Migrator().HasTable("skill_spaces") { - logger.Info("Creating skill_spaces table...") + common.Info("Creating skill_spaces table...") sql := ` CREATE TABLE IF NOT EXISTS skill_spaces ( id VARCHAR(32) PRIMARY KEY, @@ -417,13 +417,13 @@ func migrateSkillSpaceTables(db *gorm.DB) error { ) ` if err := db.Exec(sql).Error; err != nil { - logger.Warn("Failed to create skill_spaces table with MySQL dialect, trying generic", zap.Error(err)) + common.Warn("Failed to create skill_spaces table with MySQL dialect, trying generic", zap.Error(err)) // Try with AutoMigrate as fallback if err := db.AutoMigrate(&entity.SkillSpace{}); err != nil { return err } // AutoMigrate doesn't create unique indexes, so create them explicitly - logger.Info("Creating unique indexes for skill_spaces...") + common.Info("Creating unique indexes for skill_spaces...") if err := db.Exec(`ALTER TABLE skill_spaces ADD UNIQUE INDEX idx_tenant_name_status (tenant_id, name, status)`).Error; err != nil { return fmt.Errorf("failed to create unique index idx_tenant_name_status: %w", err) } @@ -450,27 +450,27 @@ func migrateSkillSpaceIndex(db *gorm.DB) error { SELECT COUNT(*) FROM INFORMATION_SCHEMA.STATISTICS WHERE TABLE_NAME = 'skill_spaces' AND INDEX_NAME = 'idx_tenant_name' `).Scan(&oldIndexExists) - + if oldIndexExists > 0 { - logger.Info("Dropping old idx_tenant_name index from skill_spaces...") + common.Info("Dropping old idx_tenant_name index from skill_spaces...") if err := db.Exec(`DROP INDEX idx_tenant_name ON skill_spaces`).Error; err != nil { return fmt.Errorf("failed to drop old index idx_tenant_name: %w", err) } } - + // Check if new index exists var newIndexExists int64 db.Raw(` SELECT COUNT(*) FROM INFORMATION_SCHEMA.STATISTICS WHERE TABLE_NAME = 'skill_spaces' AND INDEX_NAME = 'idx_tenant_name_status' `).Scan(&newIndexExists) - + if newIndexExists == 0 { - logger.Info("Creating new idx_tenant_name_status index on skill_spaces...") + common.Info("Creating new idx_tenant_name_status index on skill_spaces...") if err := db.Exec(`CREATE UNIQUE INDEX idx_tenant_name_status ON skill_spaces(tenant_id, name, status)`).Error; err != nil { return fmt.Errorf("failed to create unique index idx_tenant_name_status: %w", err) } } - + return nil } diff --git a/internal/engine/elasticsearch/search.go b/internal/engine/elasticsearch/search.go index 507618be64..b3c68fbc11 100644 --- a/internal/engine/elasticsearch/search.go +++ b/internal/engine/elasticsearch/search.go @@ -22,13 +22,13 @@ import ( "encoding/json" "fmt" "io" + "ragflow/internal/common" "strings" "github.com/elastic/go-elasticsearch/v8/esapi" "go.uber.org/zap" "ragflow/internal/engine/types" - "ragflow/internal/logger" ) // SearchResponse Elasticsearch search response @@ -203,8 +203,8 @@ func (e *elasticsearchEngine) searchUnified(ctx context.Context, req *types.Sear } // Log search details - logger.Debug("Elasticsearch searching indices", zap.Strings("indices", req.IndexNames)) - logger.Debug("Elasticsearch DSL", zap.Any("dsl", queryBody)) + common.Debug("Elasticsearch searching indices", zap.Strings("indices", req.IndexNames)) + common.Debug("Elasticsearch DSL", zap.Any("dsl", queryBody)) // Build search request reqES := esapi.SearchRequest{ @@ -222,9 +222,9 @@ func (e *elasticsearchEngine) searchUnified(ctx context.Context, req *types.Sear if res.IsError() { bodyBytes, err := io.ReadAll(res.Body) if err != nil { - logger.Error("Elasticsearch failed to read error response body", err) + common.Error("Elasticsearch failed to read error response body", err) } else { - logger.Warn("Elasticsearch error response", zap.String("body", string(bodyBytes))) + common.Warn("Elasticsearch error response", zap.String("body", string(bodyBytes))) } return nil, fmt.Errorf("Elasticsearch returned error: %s", res.Status()) } @@ -560,24 +560,24 @@ func AddMustNot(query map[string]interface{}, clauses ...map[string]interface{}) // GetFields is not implemented for Elasticsearch func (e *elasticsearchEngine) GetFields(chunks []map[string]interface{}, fields []string) map[string]map[string]interface{} { - logger.Warn("GetFields not implemented for Elasticsearch") + common.Warn("GetFields not implemented for Elasticsearch") return nil } // GetAggregation is not implemented for Elasticsearch func (e *elasticsearchEngine) GetAggregation(chunks []map[string]interface{}, fieldName string) []map[string]interface{} { - logger.Warn("GetAggregation not implemented for Elasticsearch") + common.Warn("GetAggregation not implemented for Elasticsearch") return nil } // GetHighlight is not implemented for Elasticsearch func (e *elasticsearchEngine) GetHighlight(chunks []map[string]interface{}, keywords []string, fieldName string) map[string]string { - logger.Warn("GetHighlight not implemented for Elasticsearch") + common.Warn("GetHighlight not implemented for Elasticsearch") return nil } // GetDocIDs is not implemented for Elasticsearch func (e *elasticsearchEngine) GetDocIDs(chunks []map[string]interface{}) []string { - logger.Warn("GetDocIDs not implemented for Elasticsearch") + common.Warn("GetDocIDs not implemented for Elasticsearch") return nil } diff --git a/internal/engine/global.go b/internal/engine/global.go index fb213e65f6..baf178e61f 100644 --- a/internal/engine/global.go +++ b/internal/engine/global.go @@ -18,6 +18,7 @@ package engine import ( "fmt" + "ragflow/internal/common" "ragflow/internal/server" "sync" @@ -25,7 +26,6 @@ import ( "ragflow/internal/engine/elasticsearch" "ragflow/internal/engine/infinity" - "ragflow/internal/logger" ) var ( @@ -53,7 +53,7 @@ func Init(cfg *server.DocEngineConfig) error { initErr = fmt.Errorf("failed to create doc engine: %w", err) return } - logger.Info("Doc engine initialized", zap.String("type", string(cfg.Type))) + common.Info("Doc engine initialized", zap.String("type", string(cfg.Type))) }) return initErr } diff --git a/internal/engine/infinity/client.go b/internal/engine/infinity/client.go index 9b30602fb4..aca5d68074 100644 --- a/internal/engine/infinity/client.go +++ b/internal/engine/infinity/client.go @@ -19,14 +19,15 @@ package infinity import ( "context" "fmt" + "ragflow/internal/common" "reflect" "strconv" "strings" "time" - infinity "github.com/infiniflow/infinity-go-sdk" "ragflow/internal/server" - "ragflow/internal/logger" + + infinity "github.com/infiniflow/infinity-go-sdk" ) // infinityClient Infinity SDK client wrapper @@ -52,7 +53,7 @@ func NewInfinityClient(cfg *server.InfinityConfig) (*infinityClient, error) { } // Retry connecting for up to 120 seconds (24 attempts * 5 seconds) - logger.Info("Connecting to Infinity") + common.Info("Connecting to Infinity") var conn *infinity.InfinityConnection var err error for i := 0; i < 24; i++ { @@ -78,7 +79,7 @@ func NewInfinityClient(cfg *server.InfinityConfig) (*infinityClient, error) { // WaitForHealthy blocks until Infinity is healthy or timeout func (c *infinityClient) WaitForHealthy(ctx context.Context, timeout time.Duration) error { - logger.Info("Waiting for Infinity to be healthy") + common.Info("Waiting for Infinity to be healthy") deadline := time.Now().Add(timeout) for time.Now().Before(deadline) { select { @@ -110,7 +111,7 @@ func (c *infinityClient) WaitForHealthy(ctx context.Context, timeout time.Durati if errorCode.Int() == 0 { status := serverStatus.String() if status == "started" || status == "alive" { - logger.Info("Infinity is healthy") + common.Info("Infinity is healthy") return nil } } @@ -121,7 +122,7 @@ func (c *infinityClient) WaitForHealthy(ctx context.Context, timeout time.Durati // Engine Infinity engine implementation using Go SDK type infinityEngine struct { - config *server.InfinityConfig + config *server.InfinityConfig client *infinityClient mappingFileName string docMetaMappingFileName string @@ -155,9 +156,9 @@ func NewEngine(cfg interface{}) (*infinityEngine, error) { } engine := &infinityEngine{ - config: infConfig, - client: client, - mappingFileName: mappingFileName, + config: infConfig, + client: client, + mappingFileName: mappingFileName, docMetaMappingFileName: docMetaMappingFileName, } diff --git a/internal/engine/infinity/common.go b/internal/engine/infinity/common.go index 199bd0b3c0..47bd09a0b7 100644 --- a/internal/engine/infinity/common.go +++ b/internal/engine/infinity/common.go @@ -21,10 +21,9 @@ import ( "context" "encoding/json" "fmt" + "ragflow/internal/common" "strings" - "ragflow/internal/logger" - infinity "github.com/infiniflow/infinity-go-sdk" ) @@ -46,7 +45,7 @@ func (e *infinityEngine) Delete(ctx context.Context, condition map[string]interf table, err := db.GetTable(tableName) if err != nil { - logger.Warn(fmt.Sprintf("Table %s does not exist, skipping delete", tableName)) + common.Warn(fmt.Sprintf("Table %s does not exist, skipping delete", tableName)) return 0, nil } @@ -199,7 +198,7 @@ func existsCondition(field string, tableColumns map[string]struct { }) string { col, colOk := tableColumns[field] if !colOk { - logger.Warn(fmt.Sprintf("Column '%s' not found in table columns", field)) + common.Warn(fmt.Sprintf("Column '%s' not found in table columns", field)) return fmt.Sprintf("%s!=null", field) } if strings.Contains(strings.ToLower(col.Type), "char") { diff --git a/internal/engine/infinity/dataset.go b/internal/engine/infinity/dataset.go index b8814783ad..8fb80ab572 100644 --- a/internal/engine/infinity/dataset.go +++ b/internal/engine/infinity/dataset.go @@ -22,14 +22,15 @@ import ( "fmt" "os" "path/filepath" + "ragflow/internal/common" "regexp" "strconv" "strings" - infinity "github.com/infiniflow/infinity-go-sdk" - "ragflow/internal/logger" "ragflow/internal/utility" + infinity "github.com/infiniflow/infinity-go-sdk" + "go.uber.org/zap" ) @@ -48,12 +49,12 @@ func (e *infinityEngine) CreateDataset(ctx context.Context, indexName, datasetID // Skill index: table name is just indexName (e.g., "skill_abc123_def456") tableName = indexName mappingFile = "skill_infinity_mapping.json" - logger.Info("Creating skill index table", zap.String("tableName", tableName), zap.String("mappingFile", mappingFile)) + common.Info("Creating skill index table", zap.String("tableName", tableName), zap.String("mappingFile", mappingFile)) } else { // Regular document index: table name is {indexName}_{datasetID} tableName = fmt.Sprintf("%s_%s", indexName, datasetID) mappingFile = e.mappingFileName - logger.Info("Creating regular index table", zap.String("tableName", tableName), zap.String("mappingFile", mappingFile)) + common.Info("Creating regular index table", zap.String("tableName", tableName), zap.String("mappingFile", mappingFile)) } // Use configured schema @@ -87,7 +88,7 @@ func (e *infinityEngine) CreateDataset(ctx context.Context, indexName, datasetID var table *infinity.Table if exists { // Table exists, open it and check if vector column needs to be added - logger.Info("Table already exists, checking for vector column", zap.String("tableName", tableName)) + common.Info("Table already exists, checking for vector column", zap.String("tableName", tableName)) table, err = db.GetTable(tableName) if err != nil { return fmt.Errorf("Failed to open existing table %s: %w", tableName, err) @@ -96,12 +97,12 @@ func (e *infinityEngine) CreateDataset(ctx context.Context, indexName, datasetID // Check if vector column exists (for embedding model changes) colExists, err := e.columnExists(table, vectorColName) if err != nil { - logger.Warn("Failed to check column existence", zap.String("column", vectorColName), zap.Error(err)) + common.Warn("Failed to check column existence", zap.String("column", vectorColName), zap.Error(err)) } // Add new vector column if it doesn't exist (handles embedding model change) if !colExists { - logger.Info("Adding new vector column for embedding model change", zap.String("column", vectorColName), zap.Int("size", vecSize)) + common.Info("Adding new vector column for embedding model change", zap.String("column", vectorColName), zap.Int("size", vecSize)) addColSchema := infinity.TableSchema{ &infinity.ColumnDefinition{ Name: vectorColName, @@ -109,14 +110,14 @@ func (e *infinityEngine) CreateDataset(ctx context.Context, indexName, datasetID }, } if _, err := table.AddColumns(addColSchema); err != nil { - logger.Error("Failed to add vector column "+vectorColName, err) + common.Error("Failed to add vector column "+vectorColName, err) return fmt.Errorf("Failed to add vector column %s: %w", vectorColName, err) } - logger.Info("Successfully added vector column", zap.String("column", vectorColName)) + common.Info("Successfully added vector column", zap.String("column", vectorColName)) } } else { // Table doesn't exist, create it with vector column in the initial schema - logger.Info(fmt.Sprintf("Creating table with vector column: %s with dimension %d", vectorColName, vecSize)) + common.Info(fmt.Sprintf("Creating table with vector column: %s with dimension %d", vectorColName, vecSize)) // Build column definitions (preserving JSON order) var columns infinity.TableSchema @@ -151,7 +152,7 @@ func (e *infinityEngine) CreateDataset(ctx context.Context, indexName, datasetID if err != nil { return fmt.Errorf("Failed to create table: %w", err) } - logger.Debug("Infinity created table", zap.String("tableName", tableName)) + common.Debug("Infinity created table", zap.String("tableName", tableName)) } // Create HNSW index on vector column with unique name based on vector size @@ -171,7 +172,7 @@ func (e *infinityEngine) CreateDataset(ctx context.Context, indexName, datasetID if err != nil { return fmt.Errorf("Failed to create HNSW index %s: %w", vectorIndexName, err) } - logger.Info("Created vector index", zap.String("indexName", vectorIndexName), zap.String("column", vectorColName)) + common.Info("Created vector index", zap.String("indexName", vectorIndexName), zap.String("column", vectorColName)) // Create full-text indexes for varchar fields with analyzers for _, fieldName := range schema.Keys { @@ -255,7 +256,7 @@ func (e *infinityEngine) CreateDataset(ctx context.Context, indexName, datasetID // Delete existing rows with matching IDs before insert func (e *infinityEngine) InsertDataset(ctx context.Context, chunks []map[string]interface{}, tableNamePrefix string, knowledgebaseID string) ([]string, error) { tableName := fmt.Sprintf("%s_%s", tableNamePrefix, knowledgebaseID) - logger.Info("InfinityConnection.InsertDataset called", zap.String("tableName", tableName), zap.Int("chunkCount", len(chunks))) + common.Info("InfinityConnection.InsertDataset called", zap.String("tableName", tableName), zap.Int("chunkCount", len(chunks))) db, err := e.client.conn.GetDatabase(e.client.dbName) if err != nil { @@ -346,12 +347,12 @@ func (e *infinityEngine) InsertDataset(ctx context.Context, chunks []map[string] idList[i] = fmt.Sprintf("'%v'", chunk["id"]) } filter := fmt.Sprintf("id IN (%s)", strings.Join(idList, ", ")) - logger.Debug(fmt.Sprintf("Deleting existing rows with filter: %s", filter)) + common.Debug(fmt.Sprintf("Deleting existing rows with filter: %s", filter)) delResp, delErr := table.Delete(filter) if delErr != nil { - logger.Warn(fmt.Sprintf("Failed to delete existing rows: %v", delErr)) + common.Warn(fmt.Sprintf("Failed to delete existing rows: %v", delErr)) } else { - logger.Info(fmt.Sprintf("Deleted %d existing rows", delResp.DeletedRows)) + common.Info(fmt.Sprintf("Deleted %d existing rows", delResp.DeletedRows)) } } @@ -361,7 +362,7 @@ func (e *infinityEngine) InsertDataset(ctx context.Context, chunks []map[string] return nil, fmt.Errorf("Failed to insert chunks to dataset: %w", err) } - logger.Info("InfinityConnection.InsertDataset result", zap.String("tableName", tableName), zap.Int("count", len(insertChunks))) + common.Info("InfinityConnection.InsertDataset result", zap.String("tableName", tableName), zap.Int("count", len(insertChunks))) return []string{}, nil } @@ -369,7 +370,7 @@ func (e *infinityEngine) InsertDataset(ctx context.Context, chunks []map[string] // Table name format: {tableNamePrefix}_{knowledgebaseID} func (e *infinityEngine) UpdateDataset(ctx context.Context, condition map[string]interface{}, newValue map[string]interface{}, tableNamePrefix string, knowledgebaseID string) error { tableName := fmt.Sprintf("%s_%s", tableNamePrefix, knowledgebaseID) - logger.Info("InfinityConnection.UpdateDataset called", zap.String("tableName", tableName), zap.Any("condition", condition)) + common.Info("InfinityConnection.UpdateDataset called", zap.String("tableName", tableName), zap.Any("condition", condition)) db, err := e.client.conn.GetDatabase(e.client.dbName) if err != nil { @@ -448,7 +449,7 @@ func (e *infinityEngine) UpdateDataset(ctx context.Context, condition map[string // Query rows to be updated queryResult, err := table.Output(colToRemove).Filter(filter).ToResult() if err != nil { - logger.Warn(fmt.Sprintf("Failed to query rows for remove operation: %v", err)) + common.Warn(fmt.Sprintf("Failed to query rows for remove operation: %v", err)) } else { qr, ok := queryResult.(*infinity.QueryResult) if ok && len(qr.Data) > 0 { @@ -491,10 +492,10 @@ func (e *infinityEngine) UpdateDataset(ctx context.Context, condition map[string for colName, valueToIDs := range removeOpt { for newVal, ids := range valueToIDs { idFilter := filter + " AND id IN (" + strings.Join(ids, ", ") + ")" - logger.Info(fmt.Sprintf("INFINITY remove update: table=%s, idFilter=%s, column=%s, newValue=%v", tableName, idFilter, colName, newVal)) + common.Info(fmt.Sprintf("INFINITY remove update: table=%s, idFilter=%s, column=%s, newValue=%v", tableName, idFilter, colName, newVal)) _, err := table.Update(idFilter, map[string]interface{}{colName: newVal}) if err != nil { - logger.Warn(fmt.Sprintf("Failed to remove value from column %s: %v", colName, err)) + common.Warn(fmt.Sprintf("Failed to remove value from column %s: %v", colName, err)) } } } @@ -503,13 +504,13 @@ func (e *infinityEngine) UpdateDataset(ctx context.Context, condition map[string } // Execute the main update - logger.Info(fmt.Sprintf("INFINITY update: table=%s, filter=%s, newValue=%v", tableName, filter, newValue)) + common.Info(fmt.Sprintf("INFINITY update: table=%s, filter=%s, newValue=%v", tableName, filter, newValue)) _, err = table.Update(filter, newValue) if err != nil { return fmt.Errorf("Failed to update chunks: %w", err) } - logger.Info("InfinityConnection.UpdateDataset completes", zap.String("tableName", tableName)) + common.Info("InfinityConnection.UpdateDataset completes", zap.String("tableName", tableName)) return nil } diff --git a/internal/engine/infinity/document.go b/internal/engine/infinity/document.go index 0bc2b2dbc3..1cd1ee8ce2 100644 --- a/internal/engine/infinity/document.go +++ b/internal/engine/infinity/document.go @@ -19,10 +19,10 @@ package infinity import ( "context" "fmt" + "ragflow/internal/common" "strings" "go.uber.org/zap" - "ragflow/internal/logger" ) // IndexDocument indexes a single document @@ -77,9 +77,9 @@ func (e *infinityEngine) InsertSkill(ctx context.Context, tableName, docID strin filter := fmt.Sprintf("skill_id = '%s'", docIDEscaped) delResp, delErr := table.Delete(filter) if delErr != nil { - logger.Warn(fmt.Sprintf("Failed to delete existing skill document: %v", delErr)) + common.Warn(fmt.Sprintf("Failed to delete existing skill document: %v", delErr)) } else if delResp.DeletedRows > 0 { - logger.Debug(fmt.Sprintf("Deleted %d existing skill document(s)", delResp.DeletedRows)) + common.Debug(fmt.Sprintf("Deleted %d existing skill document(s)", delResp.DeletedRows)) } // Insert the document @@ -124,7 +124,7 @@ func (e *infinityEngine) BulkInsertSkill(ctx context.Context, tableName string, for _, doc := range docs { docMap, ok := doc.(map[string]interface{}) if !ok { - logger.Warn("Invalid doc type in bulk insert, expected map[string]interface{}") + common.Warn("Invalid doc type in bulk insert, expected map[string]interface{}") continue } @@ -150,7 +150,7 @@ func (e *infinityEngine) BulkInsertSkill(ctx context.Context, tableName string, } if len(insertDocs) == 0 { - logger.Warn("No valid documents to bulk insert", zap.String("tableName", tableName)) + common.Warn("No valid documents to bulk insert", zap.String("tableName", tableName)) return 0, nil } @@ -161,12 +161,12 @@ func (e *infinityEngine) BulkInsertSkill(ctx context.Context, tableName string, filter := fmt.Sprintf("skill_id = '%s'", docIDEscaped) delResp, delErr := table.Delete(filter) if delErr != nil { - logger.Warn("Failed to delete existing skill document before bulk insert", + common.Warn("Failed to delete existing skill document before bulk insert", zap.String("tableName", tableName), zap.String("skill_id", skillID), zap.Error(delErr)) } else if delResp.DeletedRows > 0 { - logger.Debug("Deleted existing skill document before bulk insert", + common.Debug("Deleted existing skill document before bulk insert", zap.String("tableName", tableName), zap.String("skill_id", skillID), zap.Int64("deletedRows", delResp.DeletedRows)) @@ -179,7 +179,7 @@ func (e *infinityEngine) BulkInsertSkill(ctx context.Context, tableName string, return 0, fmt.Errorf("failed to bulk insert skill documents: %w", err) } - logger.Debug("Bulk upserted skill documents", + common.Debug("Bulk upserted skill documents", zap.String("tableName", tableName), zap.Int("count", len(insertDocs)), zap.Int("skillIDs", len(skillIDs))) @@ -229,7 +229,7 @@ func (e *infinityEngine) DeleteDocument(ctx context.Context, tableName, docID st return fmt.Errorf("failed to delete document: %w", err) } - logger.Debug("Deleted document from Infinity", + common.Debug("Deleted document from Infinity", zap.String("tableName", tableName), zap.String("docID", docID), zap.String("idField", idField), diff --git a/internal/engine/infinity/get.go b/internal/engine/infinity/get.go index fe42f92837..8adbb4aded 100644 --- a/internal/engine/infinity/get.go +++ b/internal/engine/infinity/get.go @@ -19,9 +19,9 @@ package infinity import ( "context" "fmt" + "ragflow/internal/common" "strings" - "ragflow/internal/logger" "ragflow/internal/utility" infinity "github.com/infiniflow/infinity-go-sdk" @@ -115,7 +115,7 @@ func (e *infinityEngine) GetChunk(ctx context.Context, tableName, chunkID string return nil, nil } - logger.Debug("infinity get chunk", zap.String("chunkID", chunkID), zap.Any("tables", tableNames)) + common.Debug("infinity get chunk", zap.String("chunkID", chunkID), zap.Any("tables", tableNames)) // Apply field mappings (same as in GetFields) // docnm -> docnm_kwd, title_tks, title_sm_tks diff --git a/internal/engine/infinity/metadata.go b/internal/engine/infinity/metadata.go index afb6679993..31ef64bccb 100644 --- a/internal/engine/infinity/metadata.go +++ b/internal/engine/infinity/metadata.go @@ -22,12 +22,13 @@ import ( "fmt" "os" "path/filepath" + "ragflow/internal/common" "strings" - infinity "github.com/infiniflow/infinity-go-sdk" - "ragflow/internal/logger" "ragflow/internal/utility" + infinity "github.com/infiniflow/infinity-go-sdk" + "go.uber.org/zap" ) @@ -65,9 +66,9 @@ func (e *infinityEngine) CreateMetadata(ctx context.Context, indexName string) e var columns infinity.TableSchema for fieldName, fieldInfo := range schema { col := infinity.ColumnDefinition{ - Name: fieldName, + Name: fieldName, DataType: fieldInfo.Type, - Default: fieldInfo.Default, + Default: fieldInfo.Default, // Comment: fieldInfo.Comment, } columns = append(columns, &col) @@ -78,7 +79,7 @@ func (e *infinityEngine) CreateMetadata(ctx context.Context, indexName string) e if err != nil { return fmt.Errorf("Failed to create doc meta table: %w", err) } - logger.Debug("Infinity created doc meta table", zap.String("tableName", indexName)) + common.Debug("Infinity created doc meta table", zap.String("tableName", indexName)) // Get table for creating indexes table, err := db.GetTable(indexName) @@ -117,7 +118,7 @@ func (e *infinityEngine) CreateMetadata(ctx context.Context, indexName string) e // Replace existing metadata with same id and kb_id func (e *infinityEngine) InsertMetadata(ctx context.Context, metadata []map[string]interface{}, tenantID string) ([]string, error) { tableName := fmt.Sprintf("ragflow_doc_meta_%s", tenantID) - logger.Info("InfinityConnection.InsertMetadata called", zap.String("tableName", tableName), zap.Int("metaCount", len(metadata))) + common.Info("InfinityConnection.InsertMetadata called", zap.String("tableName", tableName), zap.Int("metaCount", len(metadata))) db, err := e.client.conn.GetDatabase(e.client.dbName) if err != nil { @@ -167,12 +168,12 @@ func (e *infinityEngine) InsertMetadata(ctx context.Context, metadata []map[stri idList[i] = fmt.Sprintf("(id = %s AND kb_id = %s)", docID, kbID) } filter := strings.Join(idList, " OR ") - logger.Debug(fmt.Sprintf("Deleting existing metadata with filter: %s", filter)) + common.Debug(fmt.Sprintf("Deleting existing metadata with filter: %s", filter)) delResp, delErr := table.Delete(filter) if delErr != nil { - logger.Warn(fmt.Sprintf("Failed to delete existing metadata: %v", delErr)) + common.Warn(fmt.Sprintf("Failed to delete existing metadata: %v", delErr)) } else if delResp.DeletedRows > 0 { - logger.Info(fmt.Sprintf("Deleted %d existing metadata entries", delResp.DeletedRows)) + common.Info(fmt.Sprintf("Deleted %d existing metadata entries", delResp.DeletedRows)) } } @@ -182,7 +183,7 @@ func (e *infinityEngine) InsertMetadata(ctx context.Context, metadata []map[stri return nil, fmt.Errorf("Failed to insert metadata: %w", err) } - logger.Info("InfinityConnection.InsertMetadata result", zap.String("tableName", tableName), zap.Int("metaCount", len(metadata))) + common.Info("InfinityConnection.InsertMetadata result", zap.String("tableName", tableName), zap.Int("metaCount", len(metadata))) return []string{}, nil } @@ -192,7 +193,7 @@ func (e *infinityEngine) InsertMetadata(ctx context.Context, metadata []map[stri // Table name format: ragflow_doc_meta_{tenant_id} func (e *infinityEngine) UpdateMetadata(ctx context.Context, docID string, kbID string, metaFields map[string]interface{}, tenantID string) error { tableName := fmt.Sprintf("ragflow_doc_meta_%s", tenantID) - logger.Info("InfinityConnection.UpdateMetadata called", zap.String("tableName", tableName), zap.String("docID", docID), zap.String("kbID", kbID)) + common.Info("InfinityConnection.UpdateMetadata called", zap.String("tableName", tableName), zap.String("docID", docID), zap.String("kbID", kbID)) db, err := e.client.conn.GetDatabase(e.client.dbName) if err != nil { @@ -216,7 +217,7 @@ func (e *infinityEngine) UpdateMetadata(ctx context.Context, docID string, kbID result, err := queryTable.ToResult() rowExists := false if err != nil { - logger.Warn(fmt.Sprintf("Failed to query existing metadata: %v", err)) + common.Warn(fmt.Sprintf("Failed to query existing metadata: %v", err)) // If query fails, treat as not exists and insert } else { // Get results - ToResult returns *infinity.QueryResult @@ -234,7 +235,7 @@ func (e *infinityEngine) UpdateMetadata(ctx context.Context, docID string, kbID switch v := existingMetaFieldsVal.(type) { case string: if err := json.Unmarshal([]byte(v), &existingMetaFields); err != nil { - logger.Warn(fmt.Sprintf("Failed to parse existing meta_fields: %v", err)) + common.Warn(fmt.Sprintf("Failed to parse existing meta_fields: %v", err)) existingMetaFields = make(map[string]interface{}) } case map[string]interface{}: @@ -261,7 +262,7 @@ func (e *infinityEngine) UpdateMetadata(ctx context.Context, docID string, kbID if rowExists { // Row exists: update it with merged metadata - logger.Info(fmt.Sprintf("UpdateMetadata: updating existing row, table=%s, filter=%s, newValue=%v", tableName, filter, updatedFields)) + common.Info(fmt.Sprintf("UpdateMetadata: updating existing row, table=%s, filter=%s, newValue=%v", tableName, filter, updatedFields)) _, err = table.Update(filter, updatedFields) if err != nil { return fmt.Errorf("failed to update metadata: %w", err) @@ -273,14 +274,13 @@ func (e *infinityEngine) UpdateMetadata(ctx context.Context, docID string, kbID "kb_id": kbID, "meta_fields": utility.ConvertMapToJSONString(metaFields), } - logger.Info(fmt.Sprintf("UpdateMetadata: inserting new row, table=%s, newValue=%v", tableName, insertFields)) + common.Info(fmt.Sprintf("UpdateMetadata: inserting new row, table=%s, newValue=%v", tableName, insertFields)) _, err = table.Insert(insertFields) if err != nil { return fmt.Errorf("failed to insert metadata: %w", err) } } - logger.Info("InfinityConnection.UpdateMetadata completes", zap.String("tableName", tableName), zap.String("docID", docID)) + common.Info("InfinityConnection.UpdateMetadata completes", zap.String("tableName", tableName), zap.String("docID", docID)) return nil } - diff --git a/internal/engine/infinity/search.go b/internal/engine/infinity/search.go index 52d780c777..3656854b31 100644 --- a/internal/engine/infinity/search.go +++ b/internal/engine/infinity/search.go @@ -30,8 +30,6 @@ import ( "strings" "unicode" - "ragflow/internal/logger" - infinity "github.com/infiniflow/infinity-go-sdk" "go.uber.org/zap" ) @@ -40,8 +38,8 @@ import ( // It supports three matching types: MatchTextExpr (full-text), MatchDenseExpr (vector), and FusionExpr (combined). // If no match expressions are provided, Search relies solely on filter (e.g., doc_id, available_int) to find results. func (e *infinityEngine) Search(ctx context.Context, req *types.SearchRequest) (*types.SearchResult, error) { - logger.Debug("Search in Infinity started", zap.Any("indexNames", req.IndexNames)) - if logger.IsDebugEnabled() { + common.Debug("Search in Infinity started", zap.Any("indexNames", req.IndexNames)) + if common.IsDebugEnabled() { // Format match expressions for logging var matchExprsStr string for i, expr := range req.MatchExprs { @@ -56,7 +54,7 @@ func (e *infinityEngine) Search(ctx context.Context, req *types.SearchRequest) ( matchExprsStr += fmt.Sprintf(" [%d] unknown type\n", i) } } - logger.Debug(fmt.Sprintf("Search request:\n"+ + common.Debug(fmt.Sprintf("Search request:\n"+ " indexNames=%v\n"+ " KbIDs=%v\n"+ " offset=%d, limit=%d\n"+ @@ -298,12 +296,12 @@ func (e *infinityEngine) Search(ctx context.Context, req *types.SearchRequest) ( if matchText != nil && len(matchText.Fields) > 0 { textFields = matchText.Fields } else if isSkillIndex { - textFields = []string{ - "name^10", - "tags^5", - "description^3", - "content^1", - } + textFields = []string{ + "name^10", + "tags^5", + "description^3", + "content^1", + } } else { textFields = []string{ "title_tks^10", @@ -352,7 +350,7 @@ func (e *infinityEngine) Search(ctx context.Context, req *types.SearchRequest) ( table = table.MatchText(fields, questionText, textTopN, extraOptions) - logger.Debug(fmt.Sprintf( + common.Debug(fmt.Sprintf( "MatchTextExpr:\n"+ " fields=%s\n"+ " matching_text=%s\n"+ @@ -386,14 +384,14 @@ func (e *infinityEngine) Search(ctx context.Context, req *types.SearchRequest) ( vectorTopN = int(matchDense.TopN) } - denseFilterStr := filterStr - if denseFilterStr == "" { - if isSkillIndex { - denseFilterStr = "status='1'" - } else { - denseFilterStr = "available_int=1" + denseFilterStr := filterStr + if denseFilterStr == "" { + if isSkillIndex { + denseFilterStr = "status='1'" + } else { + denseFilterStr = "available_int=1" + } } - } if hasTextMatch && fusionExpr == nil { fieldsStr := strings.Join(convertedFields, ",") @@ -405,7 +403,7 @@ func (e *infinityEngine) Search(ctx context.Context, req *types.SearchRequest) ( "filter": denseFilterStr, } - logger.Debug("MatchDense for hybrid search", + common.Debug("MatchDense for hybrid search", zap.String("fieldName", fieldName), zap.String("distanceType", distanceType), zap.Int("topN", vectorTopN), @@ -430,7 +428,7 @@ func (e *infinityEngine) Search(ctx context.Context, req *types.SearchRequest) ( } } - logger.Debug("Applying Fusion for hybrid search", + common.Debug("Applying Fusion for hybrid search", zap.String("method", fusionMethod), zap.Int("topN", fusionTopK), zap.Any("params", fusionParams)) @@ -453,7 +451,7 @@ func (e *infinityEngine) Search(ctx context.Context, req *types.SearchRequest) ( // Add filter when there's no text/vector match (like metadata queries) if !hasTextMatch && !hasVectorMatch && filterStr != "" { - logger.Debug(fmt.Sprintf("Adding filter for no-match query: %s", filterStr)) + common.Debug(fmt.Sprintf("Adding filter for no-match query: %s", filterStr)) table = table.Filter(filterStr) } @@ -469,7 +467,7 @@ func (e *infinityEngine) Search(ctx context.Context, req *types.SearchRequest) ( // Execute query df, err := table.ToDataFrame() if err != nil { - logger.Warn("Infinity query failed", + common.Warn("Infinity query failed", zap.String("tableName", tableName), zap.Bool("hasTextMatch", hasTextMatch), zap.Bool("hasVectorMatch", hasVectorMatch), @@ -547,7 +545,7 @@ func (e *infinityEngine) Search(ctx context.Context, req *types.SearchRequest) ( allResults = allResults[:pageSize] } - logger.Debug("Search in Infinity completed", zap.Int("returnedRows", len(allResults)), zap.Int64("totalHits", totalHits)) + common.Debug("Search in Infinity completed", zap.Int("returnedRows", len(allResults)), zap.Int64("totalHits", totalHits)) return &types.SearchResult{ Chunks: allResults, @@ -647,10 +645,10 @@ func convertMatchingField(fieldWeightStr string) string { "authors_sm_tks": "authors@ft_authors_rag_fine", "tag_kwd": "tag_kwd@ft_tag_kwd_whitespace__", // Skill index fields - "name": "name@ft_name_rag_coarse", - "tags": "tags@ft_tags_rag_coarse", - "description": "description@ft_description_rag_coarse", - "content": "content@ft_content_rag_coarse", + "name": "name@ft_name_rag_coarse", + "tags": "tags@ft_tags_rag_coarse", + "description": "description@ft_description_rag_coarse", + "content": "content@ft_content_rag_coarse", } if newField, ok := fieldMapping[field]; ok { @@ -1099,4 +1097,4 @@ func (e *infinityEngine) GetHighlight(chunks []map[string]interface{}, keywords } return result -} \ No newline at end of file +} diff --git a/internal/entity/models/aliyun.go b/internal/entity/models/aliyun.go index 0d0316347e..2578be56d5 100644 --- a/internal/entity/models/aliyun.go +++ b/internal/entity/models/aliyun.go @@ -23,7 +23,7 @@ import ( "fmt" "io" "net/http" - "ragflow/internal/logger" + "ragflow/internal/common" "strings" "time" ) @@ -285,7 +285,7 @@ func (z *AliyunModel) ChatStreamlyWithSender(modelName string, messages []Messag scanner := bufio.NewScanner(resp.Body) for scanner.Scan() { line := scanner.Text() - logger.Info(line) + common.Info(line) // SSE data line starts with "data:" if !strings.HasPrefix(line, "data:") { diff --git a/internal/entity/models/deepseek.go b/internal/entity/models/deepseek.go index 03835dcfd8..8218d0116c 100644 --- a/internal/entity/models/deepseek.go +++ b/internal/entity/models/deepseek.go @@ -23,7 +23,7 @@ import ( "fmt" "io" "net/http" - "ragflow/internal/logger" + "ragflow/internal/common" "strings" "time" ) @@ -349,7 +349,7 @@ func (z *DeepSeekModel) ChatStreamlyWithSender(modelName string, messages []Mess scanner := bufio.NewScanner(resp.Body) for scanner.Scan() { line := scanner.Text() - logger.Info(line) + common.Info(line) // SSE data line starts with "data:" if !strings.HasPrefix(line, "data:") { diff --git a/internal/entity/models/gitee.go b/internal/entity/models/gitee.go index 6372e17b80..9e5d64090e 100644 --- a/internal/entity/models/gitee.go +++ b/internal/entity/models/gitee.go @@ -23,7 +23,7 @@ import ( "fmt" "io" "net/http" - "ragflow/internal/logger" + "ragflow/internal/common" "strings" "time" ) @@ -84,7 +84,7 @@ func (z *GiteeModel) ChatWithMessages(modelName string, messages []Message, apiC "content": msg.Content, } } - logger.Info(fmt.Sprintf("GiteeAPI messages: %+v", apiMessages)) + common.Info(fmt.Sprintf("GiteeAPI messages: %+v", apiMessages)) // Build request body reqBody := map[string]interface{}{ @@ -133,7 +133,7 @@ func (z *GiteeModel) ChatWithMessages(modelName string, messages []Message, apiC return nil, fmt.Errorf("failed to marshal request: %w", err) } - logger.Info(fmt.Sprintf("GiteeAPI request body: %s", string(jsonData))) + common.Info(fmt.Sprintf("GiteeAPI request body: %s", string(jsonData))) req, err := http.NewRequest("POST", url, bytes.NewBuffer(jsonData)) if err != nil { @@ -309,7 +309,7 @@ func (z *GiteeModel) ChatStreamlyWithSender(modelName string, messages []Message scanner := bufio.NewScanner(resp.Body) for scanner.Scan() { line := scanner.Text() - logger.Info(line) + common.Info(line) // SSE data line starts with "data:" if !strings.HasPrefix(line, "data:") { diff --git a/internal/entity/models/google.go b/internal/entity/models/google.go index 315fe4d178..d442b66399 100644 --- a/internal/entity/models/google.go +++ b/internal/entity/models/google.go @@ -19,7 +19,7 @@ package models import ( "context" "fmt" - "ragflow/internal/logger" + "ragflow/internal/common" "google.golang.org/genai" ) @@ -195,14 +195,14 @@ func (z *GoogleModel) ChatStreamlyWithSender(modelName string, messages []Messag } if responseContent != "" { - logger.Info(fmt.Sprintf("Thinking: %s", responseContent)) + common.Info(fmt.Sprintf("Thinking: %s", responseContent)) if err = sender(nil, &responseContent); err != nil { return err } } if content != "" { - logger.Info(fmt.Sprintf("Answer: %s", content)) + common.Info(fmt.Sprintf("Answer: %s", content)) if err = sender(&content, nil); err != nil { return err } diff --git a/internal/entity/models/minimax.go b/internal/entity/models/minimax.go index 2ee550962a..916a407375 100644 --- a/internal/entity/models/minimax.go +++ b/internal/entity/models/minimax.go @@ -23,7 +23,7 @@ import ( "fmt" "io" "net/http" - "ragflow/internal/logger" + "ragflow/internal/common" "strings" "time" ) @@ -397,7 +397,7 @@ func (z *MinimaxModel) ChatStreamlyWithSender(modelName string, messages []Messa scanner := bufio.NewScanner(resp.Body) for scanner.Scan() { line := scanner.Text() - logger.Info(line) + common.Info(line) // SSE data line start with data: if !strings.HasPrefix(line, "data:") { diff --git a/internal/entity/models/moonshot.go b/internal/entity/models/moonshot.go index cf0849c7c9..15db3327ab 100644 --- a/internal/entity/models/moonshot.go +++ b/internal/entity/models/moonshot.go @@ -23,7 +23,7 @@ import ( "fmt" "io" "net/http" - "ragflow/internal/logger" + "ragflow/internal/common" "strings" "time" ) @@ -292,7 +292,7 @@ func (k *MoonshotModel) ChatStreamlyWithSender(modelName string, messages []Mess scanner := bufio.NewScanner(resp.Body) for scanner.Scan() { line := scanner.Text() - logger.Info(line) + common.Info(line) // SSE data line starts with "data:" if !strings.HasPrefix(line, "data:") { diff --git a/internal/entity/models/siliconflow.go b/internal/entity/models/siliconflow.go index 450539b709..c1a1db07ef 100644 --- a/internal/entity/models/siliconflow.go +++ b/internal/entity/models/siliconflow.go @@ -23,7 +23,7 @@ import ( "fmt" "io" "net/http" - "ragflow/internal/logger" + "ragflow/internal/common" "strings" "time" ) @@ -310,7 +310,7 @@ func (z *SiliconflowModel) ChatStreamlyWithSender(modelName string, messages []M scanner := bufio.NewScanner(resp.Body) for scanner.Scan() { line := scanner.Text() - logger.Info(line) + common.Info(line) // SSE data line starts with "data:" if !strings.HasPrefix(line, "data:") { diff --git a/internal/entity/models/vllm.go b/internal/entity/models/vllm.go index 9d61c5f766..904252e127 100644 --- a/internal/entity/models/vllm.go +++ b/internal/entity/models/vllm.go @@ -23,7 +23,7 @@ import ( "fmt" "io" "net/http" - "ragflow/internal/logger" + "ragflow/internal/common" "strings" "time" ) @@ -429,7 +429,7 @@ func (z *VllmModel) ChatStreamlyWithSender(modelName string, messages []Message, scanner := bufio.NewScanner(resp.Body) for scanner.Scan() { line := scanner.Text() - logger.Info(line) + common.Info(line) // SSE data line starts with "data:" if !strings.HasPrefix(line, "data:") { diff --git a/internal/entity/models/volcengine.go b/internal/entity/models/volcengine.go index a31bf8c871..d500a5ad36 100644 --- a/internal/entity/models/volcengine.go +++ b/internal/entity/models/volcengine.go @@ -23,7 +23,7 @@ import ( "fmt" "io" "net/http" - "ragflow/internal/logger" + "ragflow/internal/common" "strings" "time" ) @@ -491,7 +491,7 @@ func (z *VolcEngine) ChatStreamlyWithSender(modelName string, messages []Message scanner := bufio.NewScanner(resp.Body) for scanner.Scan() { line := scanner.Text() - logger.Info(line) + common.Info(line) // SSE data line start with data: if !strings.HasPrefix(line, "data:") { diff --git a/internal/entity/models/zhipu-ai.go b/internal/entity/models/zhipu-ai.go index b4be9977b8..8785b26fb1 100644 --- a/internal/entity/models/zhipu-ai.go +++ b/internal/entity/models/zhipu-ai.go @@ -23,7 +23,7 @@ import ( "fmt" "io" "net/http" - "ragflow/internal/logger" + "ragflow/internal/common" "strings" "time" ) @@ -297,7 +297,7 @@ func (z *ZhipuAIModel) ChatStreamlyWithSender(modelName string, messages []Messa scanner := bufio.NewScanner(resp.Body) for scanner.Scan() { line := scanner.Text() - logger.Info(line) + common.Info(line) // SSE data line starts with "data:" if !strings.HasPrefix(line, "data:") { diff --git a/internal/handler/auth.go b/internal/handler/auth.go index a983e9b404..813bc205fc 100644 --- a/internal/handler/auth.go +++ b/internal/handler/auth.go @@ -20,7 +20,6 @@ import ( "fmt" "net/http" "ragflow/internal/common" - "ragflow/internal/logger" "ragflow/internal/server/local" "ragflow/internal/service" @@ -78,7 +77,7 @@ func (h *AuthHandler) AuthMiddleware() gin.HandlerFunc { if !local.IsAdminAvailable() { license := local.GetAdminStatus() errMsg := fmt.Sprintf("server license %s", license.Reason) - logger.Warn(errMsg) + common.Warn(errMsg) c.JSON(http.StatusServiceUnavailable, gin.H{ "code": common.CodeUnauthorized, "message": errMsg, diff --git a/internal/handler/error.go b/internal/handler/error.go index 9ca6b6c5fd..774da8b323 100644 --- a/internal/handler/error.go +++ b/internal/handler/error.go @@ -18,17 +18,16 @@ package handler import ( "net/http" + "ragflow/internal/common" "github.com/gin-gonic/gin" "go.uber.org/zap" - - "ragflow/internal/logger" ) // HandleNoRoute handles requests to undefined routes func HandleNoRoute(c *gin.Context) { // Log the request details on server side - logger.Logger.Warn("The requested URL was not found", + common.Logger.Warn("The requested URL was not found", zap.String("method", c.Request.Method), zap.String("path", c.Request.URL.Path), zap.String("query", c.Request.URL.RawQuery), diff --git a/internal/handler/skill_search.go b/internal/handler/skill_search.go index 2e6b2a9583..99d139587d 100644 --- a/internal/handler/skill_search.go +++ b/internal/handler/skill_search.go @@ -21,7 +21,6 @@ import ( "net/http" "ragflow/internal/common" "ragflow/internal/engine" - "ragflow/internal/logger" "ragflow/internal/service" "github.com/gin-gonic/gin" @@ -146,9 +145,9 @@ func (h *SkillSearchHandler) Search(c *gin.Context) { // IndexSkillsRequest represents the request to index skills type IndexSkillsRequest struct { - Skills []service.SkillInfo `json:"skills" binding:"required"` - SpaceID string `json:"space_id"` - EmbdID string `json:"embd_id"` // Optional, will use config's embd_id if empty + Skills []service.SkillInfo `json:"skills" binding:"required"` + SpaceID string `json:"space_id"` + EmbdID string `json:"embd_id"` // Optional, will use config's embd_id if empty } // IndexSkills handles the index skills request @@ -191,7 +190,7 @@ func (h *SkillSearchHandler) IndexSkills(c *gin.Context) { } // Ensure index exists before indexing (for both ES and Infinity) - logger.Info("Ensuring skill index exists before indexing", + common.Info("Ensuring skill index exists before indexing", zap.String("tenantID", user.ID), zap.String("spaceID", req.SpaceID), zap.String("engineType", h.docEngine.GetType()), @@ -205,12 +204,12 @@ func (h *SkillSearchHandler) IndexSkills(c *gin.Context) { } if err := h.indexerService.BatchIndexSkills(c.Request.Context(), user.ID, req.SpaceID, req.Skills, h.docEngine, embdID); err != nil { - logger.Error(fmt.Sprintf("Failed to batch index skills: tenantID=%s, spaceID=%s, error=%v", user.ID, req.SpaceID, err), err) + common.Error(fmt.Sprintf("Failed to batch index skills: tenantID=%s, spaceID=%s, error=%v", user.ID, req.SpaceID, err), err) jsonError(c, common.CodeOperatingError, err.Error()) return } - logger.Info("Successfully indexed skills", + common.Info("Successfully indexed skills", zap.String("tenantID", user.ID), zap.String("spaceID", req.SpaceID), zap.Int("indexedCount", len(req.Skills))) diff --git a/internal/handler/system.go b/internal/handler/system.go index cb645b9c03..3a87f60dcf 100644 --- a/internal/handler/system.go +++ b/internal/handler/system.go @@ -18,7 +18,7 @@ package handler import ( "net/http" - "ragflow/internal/logger" + "ragflow/internal/common" "ragflow/internal/server" "ragflow/internal/service" @@ -133,7 +133,7 @@ func (h *SystemHandler) GetVersion(c *gin.Context) { // GetLogLevel returns the current log level func (h *SystemHandler) GetLogLevel(c *gin.Context) { - level := logger.GetLevel() + level := common.GetLevel() c.JSON(http.StatusOK, gin.H{ "code": 0, "message": "success", @@ -157,7 +157,7 @@ func (h *SystemHandler) SetLogLevel(c *gin.Context) { return } - if err := logger.SetLevel(req.Level); err != nil { + if err := common.SetLevel(req.Level); err != nil { c.JSON(http.StatusBadRequest, gin.H{ "code": 400, "message": err.Error(), diff --git a/internal/logger/README.md b/internal/logger/README.md deleted file mode 100644 index adc941baf1..0000000000 --- a/internal/logger/README.md +++ /dev/null @@ -1,70 +0,0 @@ -# Logger Package - -This package provides structured logging using Uber's Zap library. - -## Installation - -Install zap dependency: - -```bash -go get go.uber.org/zap -``` - -## Usage - -The logger is initialized in `cmd/server_main.go` and is available throughout the application. - -### Basic Usage - -```go -import ( - "ragflow/internal/logger" - "go.uber.org/zap" -) - -// Log with structured fields -logger.Info("User login", zap.String("user_id", userID), zap.String("ip", clientIP)) - -// Log error -logger.Error("Failed to connect database", err) - -// Log fatal (exits application) -logger.Fatal("Failed to start server", err) - -// Debug level -logger.Debug("Processing request", zap.String("request_id", reqID)) - -// Warning level -logger.Warn("Slow query", zap.Duration("duration", duration)) -``` - -### Access Logger Directly - -If you need the underlying Zap logger: - -```go -logger.Logger.Info("Message", zap.String("key", "value")) -``` - -Or use the SugaredLogger for more flexible API: - -```go -logger.Sugar.Infow("Message", "key", "value") -``` - -## Fallback to Standard Logger - -If zap is not installed or fails to initialize, the logger will fallback to the standard library `log` package, ensuring the application continues to work. - -## Log Levels - -The logger supports the following levels: -- `debug` - Detailed information for debugging -- `info` - General informational messages -- `warn` - Warning messages -- `error` - Error messages -- `fatal` - Fatal errors that stop the application - -The log level is configured via the server mode in the configuration: -- `debug` mode uses `debug` level -- `release` mode uses `info` level diff --git a/internal/server/local/admin_status.go b/internal/server/local/admin_status.go index 31252b1418..4bbb377e0c 100644 --- a/internal/server/local/admin_status.go +++ b/internal/server/local/admin_status.go @@ -18,7 +18,7 @@ package local import ( "fmt" - "ragflow/internal/logger" + "ragflow/internal/common" "sync" ) @@ -70,7 +70,7 @@ func SetAdminStatus(status int, reason string) { adminStatus.Reason = reason if adminStatus.Status != 0 { - logger.Warn(fmt.Sprintf("Admin server is unavailable, reason: %s", adminStatus.Reason)) + common.Warn(fmt.Sprintf("Admin server is unavailable, reason: %s", adminStatus.Reason)) } } diff --git a/internal/server/variable.go b/internal/server/variable.go index 23f1b4c94b..14a6399e13 100644 --- a/internal/server/variable.go +++ b/internal/server/variable.go @@ -19,13 +19,12 @@ package server import ( "context" "fmt" + "ragflow/internal/common" "ragflow/internal/utility" "sync" "time" "go.uber.org/zap" - - "ragflow/internal/logger" ) // Variables holds all runtime variables that can be changed during system operation @@ -74,10 +73,10 @@ func InitVariables(store VariableStore) error { initErr = fmt.Errorf("failed to initialize secret key: %w", err) } else { globalVariables.SecretKey = secretKey - logger.Info("Secret key initialized from store") + common.Info("Secret key initialized from store") } - logger.Info("Server variables initialized successfully") + common.Info("Server variables initialized successfully") }) return initErr } @@ -105,7 +104,7 @@ func SetSecretKey(key string) { defer variablesMu.Unlock() if globalVariables != nil { globalVariables.SecretKey = key - logger.Info("Secret key updated at runtime") + common.Info("Secret key updated at runtime") } } @@ -116,41 +115,41 @@ func SetSecretKey(key string) { func GetOrCreateKey(store VariableStore, key string, newValue string) (string, error) { if store == nil { err := fmt.Errorf("store is nil") - logger.Warn("VariableStore is nil, cannot get or create key", zap.String("key", key)) + common.Warn("VariableStore is nil, cannot get or create key", zap.String("key", key)) return "store is nil", err } // Try to get existing value value, err := store.Get(key) if err != nil { - logger.Warn("Failed to get key from store", zap.String("key", key), zap.Error(err)) + common.Warn("Failed to get key from store", zap.String("key", key), zap.Error(err)) return "", err } // Key exists, return the value if value != "" { - logger.Debug("Key found in store", zap.String("key", key)) + common.Debug("Key found in store", zap.String("key", key)) return value, nil } // Key doesn't exist, generate new value - logger.Info("Generating new value for key", zap.String("key", key)) + common.Info("Generating new value for key", zap.String("key", key)) // Try to set with NX (only if not exists) - ensures atomicity if store.SetNX(key, newValue, SecretKeyTTL) { - logger.Info("New value stored successfully", zap.String("key", key)) + common.Info("New value stored successfully", zap.String("key", key)) return newValue, nil } // Another process might have set it, try to get again value, err = store.Get(key) if err != nil { - logger.Warn("Failed to get key after SetNX", zap.String("key", key), zap.Error(err)) + common.Warn("Failed to get key after SetNX", zap.String("key", key), zap.Error(err)) return newValue, nil // Return our generated value as fallback } if value != "" { - logger.Info("Using value set by another process", zap.String("key", key)) + common.Info("Using value set by another process", zap.String("key", key)) return value, nil } @@ -175,12 +174,12 @@ func RefreshVariables(store VariableStore) error { // Refresh SecretKey secretKey, err := store.Get(SecretKeyRedisKey) if err != nil { - logger.Warn("Failed to refresh secret key from store", zap.Error(err)) + common.Warn("Failed to refresh secret key from store", zap.Error(err)) return err } if secretKey != "" { globalVariables.SecretKey = secretKey - logger.Info("Secret key refreshed from store") + common.Info("Secret key refreshed from store") } return nil @@ -214,21 +213,21 @@ func (w *VariableWatcher) Start(interval time.Duration) { select { case <-ticker.C: if err := RefreshVariables(w.store); err != nil { - logger.Debug("Failed to refresh variables", zap.Error(err)) + common.Debug("Failed to refresh variables", zap.Error(err)) } case <-w.stopChan: return } } }() - logger.Info("Variable watcher started", zap.Duration("interval", interval)) + common.Info("Variable watcher started", zap.Duration("interval", interval)) } // Stop stops the variable watcher func (w *VariableWatcher) Stop() { close(w.stopChan) w.wg.Wait() - logger.Info("Variable watcher stopped") + common.Info("Variable watcher stopped") } // SaveToStorage saves current variables to persistent storage @@ -249,7 +248,7 @@ func SaveToStorage(store VariableStore) error { return fmt.Errorf("failed to save secret key to store") } - logger.Info("Variables saved to storage") + common.Info("Variables saved to storage") return nil } diff --git a/internal/service/chat_session.go b/internal/service/chat_session.go index 661565e3f8..dc28e9ed6c 100644 --- a/internal/service/chat_session.go +++ b/internal/service/chat_session.go @@ -20,6 +20,7 @@ import ( "encoding/json" "errors" "fmt" + "ragflow/internal/common" "strings" "time" @@ -29,7 +30,6 @@ import ( "ragflow/internal/dao" "ragflow/internal/entity" modelModule "ragflow/internal/entity/models" - "ragflow/internal/logger" ) // ChatSessionService chat session (conversation) service @@ -524,7 +524,7 @@ func (s *ChatSessionService) asyncChatStream(dialog *entity.Chat, session *entit // asyncChatSolo performs simple chat without RAG (non-streaming) func (s *ChatSessionService) asyncChatSolo(dialog *entity.Chat, session *entity.ChatSession, messages []map[string]interface{}, config map[string]interface{}, messageID string, reference []interface{}, stream bool) (map[string]interface{}, error) { - logger.Info("asyncChatSolo started", + common.Info("asyncChatSolo started", zap.String("tenant_id", dialog.TenantID), zap.String("llm_id", dialog.LLMID), zap.String("dialog_id", dialog.ID), @@ -538,7 +538,7 @@ func (s *ChatSessionService) asyncChatSolo(dialog *entity.Chat, session *entity. chatModel, err := s.modelProviderSvc.GetChatModel(dialog.TenantID, dialog.LLMID) if err != nil { - logger.Error("asyncChatSolo failed to get chat model", err) + common.Error("asyncChatSolo failed to get chat model", err) return nil, err } @@ -564,11 +564,11 @@ func (s *ChatSessionService) asyncChatSolo(dialog *entity.Chat, session *entity. // Perform chat response, err := chatModel.ModelDriver.ChatWithMessages(*chatModel.ModelName, msgs, chatModel.APIConfig, chatConfig) if err != nil { - logger.Error("asyncChatSolo chat failed", err) + common.Error("asyncChatSolo chat failed", err) return nil, err } - logger.Info("asyncChatSolo completed", + common.Info("asyncChatSolo completed", zap.String("tenant_id", dialog.TenantID), zap.String("llm_id", dialog.LLMID), zap.Int("response_length", len(*response.Answer))) @@ -585,7 +585,7 @@ func (s *ChatSessionService) asyncChatSolo(dialog *entity.Chat, session *entity. // asyncChatSoloStream performs simple streaming chat without RAG func (s *ChatSessionService) asyncChatSoloStream(dialog *entity.Chat, session *entity.ChatSession, messages []map[string]interface{}, config map[string]interface{}, messageID string, reference []interface{}, resultChan chan<- map[string]interface{}) { - logger.Info("asyncChatSoloStream started", + common.Info("asyncChatSoloStream started", zap.String("tenant_id", dialog.TenantID), zap.String("llm_id", dialog.LLMID), zap.String("dialog_id", dialog.ID), @@ -599,7 +599,7 @@ func (s *ChatSessionService) asyncChatSoloStream(dialog *entity.Chat, session *e chatModel, err := s.modelProviderSvc.GetChatModel(dialog.TenantID, dialog.LLMID) if err != nil { - logger.Error("asyncChatSoloStream failed to get chat model", err) + common.Error("asyncChatSoloStream failed to get chat model", err) resultChan <- s.structureAnswer(session, "**ERROR**: "+err.Error(), messageID, session.ID, reference) return } @@ -647,7 +647,7 @@ func (s *ChatSessionService) asyncChatSoloStream(dialog *entity.Chat, session *e return } - logger.Info("asyncChatSoloStream completed", + common.Info("asyncChatSoloStream completed", zap.String("tenant_id", dialog.TenantID), zap.String("llm_id", dialog.LLMID), zap.Int("response_length", len(fullAnswer))) diff --git a/internal/service/chunk.go b/internal/service/chunk.go index 0da359d9d6..b041ef2374 100644 --- a/internal/service/chunk.go +++ b/internal/service/chunk.go @@ -19,6 +19,7 @@ package service import ( "context" "fmt" + "ragflow/internal/common" "ragflow/internal/entity" "ragflow/internal/entity/models" "ragflow/internal/server" @@ -30,8 +31,6 @@ import ( "ragflow/internal/dao" "ragflow/internal/engine" "ragflow/internal/engine/types" - "ragflow/internal/logger" - "ragflow/internal/service/nlp" "ragflow/internal/tokenizer" "ragflow/internal/utility" @@ -106,9 +105,9 @@ type RetrievalTestResponse struct { // 7. knowledge graph retrieval (not implemented) // 8. Apply retrieval by children to group child chunks under parent chunks func (s *ChunkService) RetrievalTest(req *RetrievalTestRequest, userID string) (*RetrievalTestResponse, error) { - logger.Info("RetrievalTest started", zap.String("userID", userID), zap.Any("kbID", req.KbID), zap.String("question", req.Question)) + common.Info("RetrievalTest started", zap.String("userID", userID), zap.Any("kbID", req.KbID), zap.String("question", req.Question)) - logger.Debug(fmt.Sprintf("RetrievalTest request:\n"+ + common.Debug(fmt.Sprintf("RetrievalTest request:\n"+ " kbID=%v\n"+ " question=%s\n"+ " page=%v, size=%v\n"+ @@ -156,7 +155,7 @@ func (s *ChunkService) RetrievalTest(req *RetrievalTestRequest, userID string) ( if len(tenants) == 0 { return nil, fmt.Errorf("user has no accessible tenants") } - logger.Debug("Retrieved user tenants from database", zap.String("userID", userID), zap.Int("tenantCount", len(tenants))) + common.Debug("Retrieved user tenants from database", zap.String("userID", userID), zap.Int("tenantCount", len(tenants))) var tenantIDs []string var kbRecords []*entity.Knowledgebase @@ -165,7 +164,7 @@ func (s *ChunkService) RetrievalTest(req *RetrievalTestRequest, userID string) ( for _, tenant := range tenants { kb, err := s.kbDAO.GetByIDAndTenantID(kbID, tenant.TenantID) if err == nil && kb != nil { - logger.Debug("Found knowledge base in database", + common.Debug("Found knowledge base in database", zap.String("kbID", kbID), zap.String("tenantID", tenant.TenantID), zap.String("kbName", kb.Name), @@ -200,14 +199,14 @@ func (s *ChunkService) RetrievalTest(req *RetrievalTestRequest, userID string) ( // If search_id is set, get meta_data_filter and chat_id from search_config searchDetail, err := s.searchService.GetDetail(*req.SearchID) if err != nil { - logger.Warn("Failed to get search detail for search_id, proceeding without it", zap.String("searchID", *req.SearchID), zap.Error(err)) + common.Warn("Failed to get search detail for search_id, proceeding without it", zap.String("searchID", *req.SearchID), zap.Error(err)) } else if searchConfig, ok := searchDetail["search_config"].(entity.JSONMap); ok && searchConfig != nil { if searchMetaFilter, ok := searchConfig["meta_data_filter"].(map[string]interface{}); ok { filter = searchMetaFilter } chatID, _ = searchConfig["chat_id"].(string) } else { - logger.Warn("No search_config found in search detail", zap.String("searchID", *req.SearchID)) + common.Warn("No search_config found in search detail", zap.String("searchID", *req.SearchID)) } } @@ -220,9 +219,9 @@ func (s *ChunkService) RetrievalTest(req *RetrievalTestRequest, userID string) ( // Use chat_id from search_config creds, err = modelProviderSvc.GetModelByName(chatID, tenantIDs[0]) if err != nil { - logger.Warn("Failed to get chat model from search_config chat_id, using tenant default", zap.String("chatID", chatID), zap.Error(err)) + common.Warn("Failed to get chat model from search_config chat_id, using tenant default", zap.String("chatID", chatID), zap.Error(err)) } else { - logger.Info("Fetched chat model (from search_config) for metadata filter", + common.Info("Fetched chat model (from search_config) for metadata filter", zap.String("chatID", chatID), zap.String("tenantID", tenantIDs[0]), zap.String("providerName", creds.ProviderName), @@ -233,9 +232,9 @@ func (s *ChunkService) RetrievalTest(req *RetrievalTestRequest, userID string) ( if creds == nil { creds, err = modelProviderSvc.GetDefaultModel(entity.ModelTypeChat, tenantIDs[0]) if err != nil { - logger.Warn("Failed to get tenant default chat model for meta_data_filter", zap.Error(err)) + common.Warn("Failed to get tenant default chat model for meta_data_filter", zap.Error(err)) } else { - logger.Info("Fetched chat model (tenant default) for metadata filter", + common.Info("Fetched chat model (tenant default) for metadata filter", zap.String("tenantID", tenantIDs[0]), zap.String("providerName", creds.ProviderName), zap.String("modelName", creds.ModelName)) @@ -252,12 +251,12 @@ func (s *ChunkService) RetrievalTest(req *RetrievalTestRequest, userID string) ( metadataSvc := NewMetadataService() flattedMeta, err := metadataSvc.GetFlattedMetaByKBs(kbIDs) if err != nil { - logger.Warn("Failed to get flatted metadata", zap.Error(err)) + common.Warn("Failed to get flatted metadata", zap.Error(err)) } else { - logger.Info("metadata filter conditions", zap.Any("filter", filter)) + common.Info("metadata filter conditions", zap.Any("filter", filter)) filteredDocIDs, _ := ApplyMetaDataFilter(ctx, filter, flattedMeta, req.Question, creds, req.DocIDs) docIDs = filteredDocIDs - logger.Info("ApplyMetaDataFilter result", zap.Strings("docIDs", docIDs)) + common.Info("ApplyMetaDataFilter result", zap.Strings("docIDs", docIDs)) } } @@ -269,9 +268,9 @@ func (s *ChunkService) RetrievalTest(req *RetrievalTestRequest, userID string) ( modelProviderSvc := NewModelProviderService() creds, err = modelProviderSvc.GetDefaultModel(entity.ModelTypeChat, tenantIDs[0]) if err != nil { - logger.Warn("Failed to get default chat model for LLM transformations", zap.Error(err)) + common.Warn("Failed to get default chat model for LLM transformations", zap.Error(err)) } else { - logger.Info("Fetched chat model (tenant default) for cross_languages/keyword_extraction", + common.Info("Fetched chat model (tenant default) for cross_languages/keyword_extraction", zap.String("tenantID", tenantIDs[0]), zap.String("providerName", creds.ProviderName), zap.String("modelName", creds.ModelName)) @@ -282,7 +281,7 @@ func (s *ChunkService) RetrievalTest(req *RetrievalTestRequest, userID string) ( if creds != nil && len(req.CrossLanguages) > 0 { translated, err := CrossLanguages(ctx, creds, req.Question, req.CrossLanguages) if err != nil { - logger.Warn("Failed to translate question", zap.Error(err)) + common.Warn("Failed to translate question", zap.Error(err)) } else { modifiedQuestion = translated } @@ -292,14 +291,14 @@ func (s *ChunkService) RetrievalTest(req *RetrievalTestRequest, userID string) ( if creds != nil && req.Keyword != nil && *req.Keyword { extractedKeywords, err := KeywordExtraction(ctx, creds, modifiedQuestion, 3) if err != nil { - logger.Warn("Failed to extract keywords from question", zap.Error(err)) + common.Warn("Failed to extract keywords from question", zap.Error(err)) } else if extractedKeywords != "" { modifiedQuestion = modifiedQuestion + " " + extractedKeywords } } if modifiedQuestion != req.Question { - logger.Info("Modified question after transformations", + common.Info("Modified question after transformations", zap.String("originalQuestion", req.Question), zap.String("modifiedQuestion", modifiedQuestion), zap.Strings("crossLanguages", req.CrossLanguages), @@ -309,7 +308,7 @@ func (s *ChunkService) RetrievalTest(req *RetrievalTestRequest, userID string) ( // Get tag-based rank features via LabelQuestion metadataSvc := NewMetadataService() labels := metadataSvc.LabelQuestion(modifiedQuestion, kbRecords) - logger.Debug("LabelQuestion result", zap.Any("labels", labels)) + common.Debug("LabelQuestion result", zap.Any("labels", labels)) // Determine embedding model var embdID string @@ -346,7 +345,7 @@ func (s *ChunkService) RetrievalTest(req *RetrievalTestRequest, userID string) ( if err != nil { return nil, fmt.Errorf("failed to get embedding model: %w", err) } - logger.Info("Fetched embedding model for retrieval", + common.Info("Fetched embedding model for retrieval", zap.String("tenantID", tenantIDs[0]), zap.String("embdID", embdID)) @@ -376,7 +375,7 @@ func (s *ChunkService) RetrievalTest(req *RetrievalTestRequest, userID string) ( } if rerankModel != nil { - logger.Info("Fetched rerank model", + common.Info("Fetched rerank model", zap.String("tenantID", tenantIDs[0]), zap.String("rerankCompositeName", rerankCompositeName)) } @@ -407,7 +406,7 @@ func (s *ChunkService) RetrievalTest(req *RetrievalTestRequest, userID string) ( // Handle knowledge graph retrieval // TODO: KG retrieval requires GraphRAG infrastructure which is not yet implemented in Go if req.UseKG != nil && *req.UseKG { - logger.Warn("use_kg is not yet implemented in Go - skipping KG retrieval") + common.Warn("use_kg is not yet implemented in Go - skipping KG retrieval") } // Apply retrieval_by_children - aggregate child chunks into parent chunks @@ -418,7 +417,7 @@ func (s *ChunkService) RetrievalTest(req *RetrievalTestRequest, userID string) ( delete(filteredChunks[i], "vector") } - logger.Info("RetrievalTest completed", zap.String("userID", userID), zap.Any("kbID", req.KbID), zap.String("question", req.Question), zap.Int64("chunkCount", int64(len(filteredChunks)))) + common.Info("RetrievalTest completed", zap.String("userID", userID), zap.Any("kbID", req.KbID), zap.String("question", req.Question), zap.Int64("chunkCount", int64(len(filteredChunks)))) return &RetrievalTestResponse{ Chunks: filteredChunks, diff --git a/internal/service/file.go b/internal/service/file.go index 90433f17bb..24d27f3acb 100644 --- a/internal/service/file.go +++ b/internal/service/file.go @@ -23,10 +23,10 @@ import ( "mime/multipart" "os" "path/filepath" + "ragflow/internal/common" "ragflow/internal/dao" "ragflow/internal/engine" "ragflow/internal/entity" - "ragflow/internal/logger" "ragflow/internal/storage" "ragflow/internal/utility" "strings" @@ -579,7 +579,7 @@ func (s *FileService) deleteSingleFile(ctx context.Context, file *entity.File) e storageImpl := storage.GetStorageFactory().GetStorage() if storageImpl != nil { if err := storageImpl.Remove(file.ParentID, *file.Location); err != nil { - logger.Logger.Error(fmt.Sprintf("Fail to remove object: %s/%s, error: %v", file.ParentID, *file.Location, err)) + common.Logger.Error(fmt.Sprintf("Fail to remove object: %s/%s, error: %v", file.ParentID, *file.Location, err)) } } } @@ -608,14 +608,14 @@ func (s *FileService) deleteSingleFile(ctx context.Context, file *entity.File) e if tenantID != "" { // Delete from document engine if err := s.deleteDocumentFromEngine(ctx, doc, tenantID); err != nil { - logger.Logger.Error(fmt.Sprintf("Fail to delete document from engine: %s, error: %v", doc.ID, err)) + common.Logger.Error(fmt.Sprintf("Fail to delete document from engine: %s, error: %v", doc.ID, err)) } } } // Delete document record if err := documentDAO.Delete(docID); err != nil { - logger.Logger.Error(fmt.Sprintf("Fail to delete document: %s, error: %v", docID, err)) + common.Logger.Error(fmt.Sprintf("Fail to delete document: %s, error: %v", docID, err)) } } diff --git a/internal/service/generator.go b/internal/service/generator.go index 901a486790..889f6ed87c 100644 --- a/internal/service/generator.go +++ b/internal/service/generator.go @@ -19,6 +19,7 @@ package service import ( "context" "fmt" + "ragflow/internal/common" "regexp" "strings" @@ -26,7 +27,6 @@ import ( "ragflow/internal/entity" modelModule "ragflow/internal/entity/models" - "ragflow/internal/logger" ) // KeywordExtraction extracts keywords from content using LLM. @@ -73,7 +73,7 @@ func KeywordExtraction(ctx context.Context, creds *entity.ModelCredentials, cont } response := *responsePtr - logger.Info("KeywordExtraction result", zap.String("response", response)) + common.Info("KeywordExtraction result", zap.String("response", response)) // Clean up response - remove thinking tags if present response = strings.TrimSpace(response) diff --git a/internal/service/metadata_filter.go b/internal/service/metadata_filter.go index 5e445cf347..fc76158cde 100644 --- a/internal/service/metadata_filter.go +++ b/internal/service/metadata_filter.go @@ -21,6 +21,7 @@ import ( "encoding/json" "fmt" "os" + "ragflow/internal/common" "regexp" "strings" "time" @@ -29,7 +30,6 @@ import ( "ragflow/internal/entity" modelModule "ragflow/internal/entity/models" - "ragflow/internal/logger" ) // MetaFilterCondition represents a single filter condition @@ -115,7 +115,7 @@ func renderMetaFilterTemplate(currentDate, metadataKeys, question, constraints s func genMetaFilterPrompt(metaDataJSON, question, constraintsJSON, currentDate string) string { prompt, err := renderMetaFilterTemplate(currentDate, metaDataJSON, question, constraintsJSON) if err != nil { - logger.Warn("Failed to render meta filter template, using fallback", zap.Error(err)) + common.Warn("Failed to render meta filter template, using fallback", zap.Error(err)) // Fallback to empty prompt return "" } @@ -168,7 +168,7 @@ func GenMetaFilter(ctx context.Context, creds *entity.ModelCredentials, metaData modelProviderSvc := NewModelProviderService() response, code, err := modelProviderSvc.ChatWithMessagesToModelByApiKey(creds.ProviderName, creds.ModelName, creds.APIKey, messages) if err != nil { - logger.Warn("ChatWithMessagesToModelByApiKey failed for GenMetaFilter", + common.Warn("ChatWithMessagesToModelByApiKey failed for GenMetaFilter", zap.String("provider", creds.ProviderName), zap.String("model", creds.ModelName), zap.Int("code", int(code)), @@ -190,11 +190,11 @@ func GenMetaFilter(ctx context.Context, creds *entity.ModelCredentials, metaData // Parse JSON var result MetaFilterResult if err := json.Unmarshal([]byte(responseStr), &result); err != nil { - logger.Warn("Failed to parse meta filter response, returning empty conditions", zap.Error(err)) + common.Warn("Failed to parse meta filter response, returning empty conditions", zap.Error(err)) return &MetaFilterResult{Conditions: []MetaFilterCondition{}, Logic: "and"}, nil } - logger.Info("GenMetaFilter result", zap.Any("conditions", result.Conditions), zap.String("logic", result.Logic)) + common.Info("GenMetaFilter result", zap.Any("conditions", result.Conditions), zap.String("logic", result.Logic)) return &result, nil } @@ -464,7 +464,7 @@ func ApplyMetaDataFilter( case "auto": filters, err := GenMetaFilter(ctx, creds, metaData, question, nil) if err != nil { - logger.Warn("Failed to generate meta filter", zap.Error(err)) + common.Warn("Failed to generate meta filter", zap.Error(err)) return docIDs, false } filteredIDs := ApplyMetaFilter(metaData, filters.Conditions, filters.Logic) @@ -505,7 +505,7 @@ func ApplyMetaDataFilter( if len(filteredMeta) > 0 { filters, err := GenMetaFilter(ctx, creds, filteredMeta, question, constraints) if err != nil { - logger.Warn("Failed to generate meta filter", zap.Error(err)) + common.Warn("Failed to generate meta filter", zap.Error(err)) return docIDs, false } filteredIDs := ApplyMetaFilter(metaData, filters.Conditions, filters.Logic) diff --git a/internal/service/nlp/reranker.go b/internal/service/nlp/reranker.go index 25abf5ff58..f127c10009 100644 --- a/internal/service/nlp/reranker.go +++ b/internal/service/nlp/reranker.go @@ -24,7 +24,6 @@ import ( "ragflow/internal/common" "ragflow/internal/entity/models" - "ragflow/internal/logger" "go.uber.org/zap" ) @@ -102,14 +101,14 @@ func RerankByModel( chunkCount := len(chunks) - logger.Info("RerankByModel started", zap.String("query", query), zap.Int("chunkCount", chunkCount), zap.Float64("tkWeight", tkWeight), zap.Float64("vtWeight", vtWeight)) + common.Info("RerankByModel started", zap.String("query", query), zap.Int("chunkCount", chunkCount), zap.Float64("tkWeight", tkWeight), zap.Float64("vtWeight", vtWeight)) // Extract keywords from query keywords := []string{} if qb != nil { _, keywords = qb.Question(query, "qa", 0.6) } - logger.Info("RerankByModel keywords extracted", zap.Any("keywords", keywords)) + common.Info("RerankByModel keywords extracted", zap.Any("keywords", keywords)) // Build token lists and document texts for each chunk insTw := make([][]string, 0, chunkCount) @@ -138,12 +137,12 @@ func RerankByModel( // Get similarity scores from reranker model modelSim, err := rerankModel.ModelDriver.Rerank(rerankModel.ModelName, query, docs, rerankModel.APIConfig) if err != nil { - logger.Error("RerankByModel: rerankModel.Rerank failed; falling back to token-only similarity", err) + common.Error("RerankByModel: rerankModel.Rerank failed; falling back to token-only similarity", err) // If model fails, fall back to token similarity only modelSim = make([]float64, len(tsim)) } if len(modelSim) != chunkCount { - logger.Warn("reranker returned mismatched score length; padding/truncating", + common.Warn("reranker returned mismatched score length; padding/truncating", zap.Int("got", len(modelSim)), zap.Int("want", chunkCount)) fixed := make([]float64, chunkCount) copy(fixed, modelSim) @@ -160,7 +159,7 @@ func RerankByModel( // Always apply pageranks, even when rankFeature is nil/empty sim = applyRankFeatureScores(chunks, sim, rankFeature) - logger.Info("RerankByModel completed") + common.Info("RerankByModel completed") return sim, tsim, modelSim } @@ -181,13 +180,13 @@ func RerankStandard( return []float64{}, []float64{}, []float64{} } - logger.Info("RerankStandard started", zap.Int("chunkCount", chunkCount), zap.Float64("tkWeight", tkWeight), zap.Float64("vtWeight", vtWeight)) + common.Info("RerankStandard started", zap.Int("chunkCount", chunkCount), zap.Float64("tkWeight", tkWeight), zap.Float64("vtWeight", vtWeight)) // Compute keywords fresh from query if qb != nil && len(keywords) == 0 { _, keywords = qb.Question(query, "qa", 0.6) } - logger.Info("RerankStandard keywords", zap.Any("keywords", keywords)) + common.Info("RerankStandard keywords", zap.Any("keywords", keywords)) // Get vector information vectorSize := len(questionVector) @@ -236,7 +235,7 @@ func RerankStandard( // Always apply pageranks, even when rankFeature is nil/empty sim = applyRankFeatureScores(chunks, sim, rankFeature) - logger.Info("RerankStandard completed") + common.Info("RerankStandard completed") return sim, tsim, vsim } @@ -245,7 +244,7 @@ func RerankStandard( // so we check multiple possible field names. If no score is found, we default to 1.0 // to ensure the chunk passes through any similarity threshold filters. func RerankInfinityFallback(chunks []map[string]interface{}) (sim []float64, tsim []float64, vsim []float64) { - logger.Info("RerankInfinityFallback started", zap.Int("chunkCount", len(chunks))) + common.Info("RerankInfinityFallback started", zap.Int("chunkCount", len(chunks))) sim = make([]float64, len(chunks)) for i, chunk := range chunks { @@ -262,7 +261,7 @@ func RerankInfinityFallback(chunks []map[string]interface{}) (sim []float64, tsi sim[i] = 1.0 } } - logger.Info("RerankInfinityFallback completed") + common.Info("RerankInfinityFallback completed") return sim, sim, sim } diff --git a/internal/service/nlp/retrieval.go b/internal/service/nlp/retrieval.go index 36e38cf2d4..2754571120 100644 --- a/internal/service/nlp/retrieval.go +++ b/internal/service/nlp/retrieval.go @@ -20,11 +20,11 @@ import ( "context" "fmt" "math" + "ragflow/internal/common" "ragflow/internal/dao" "ragflow/internal/engine" "ragflow/internal/engine/types" "ragflow/internal/entity/models" - "ragflow/internal/logger" "sort" "strings" @@ -128,7 +128,7 @@ func (s *RetrievalService) Retrieval(ctx context.Context, req *RetrievalRequest) page := req.Page globalOffset := (page - 1) * pageSize searchPage := globalOffset/rerankLimit + 1 - logger.Debug("Retrieval rerank params", zap.Int("page", req.Page), zap.Int("pageSize", pageSize), + common.Debug("Retrieval rerank params", zap.Int("page", req.Page), zap.Int("pageSize", pageSize), zap.Int("searchPage", searchPage), zap.Int("rerankLimit", rerankLimit), zap.Int("globalOffset", globalOffset)) // Execute search via Search() @@ -225,7 +225,7 @@ func (s *RetrievalService) Retrieval(ctx context.Context, req *RetrievalRequest) } pageIdx = validIdx[begin:end] } - logger.Debug("Pagination result info", zap.Int("totalValid", len(validIdx)), zap.Int("begin", begin), + common.Debug("Pagination result info", zap.Int("totalValid", len(validIdx)), zap.Int("begin", begin), zap.Int("end", end), zap.Int("chunkCount", len(pageIdx))) // Build chunks for pageIdx, transforms raw search results into the API response format @@ -651,7 +651,7 @@ func (r *RetrievalSearchRequest) GetFilters() map[string]interface{} { // RetrievalByChildren aggregates child chunks into parent chunks func RetrievalByChildren(chunks []map[string]interface{}, tenantIDs []string, docEngine engine.DocEngine, ctx context.Context) []map[string]interface{} { - logger.Info("RetrievalByChildren started", zap.Int("chunks", len(chunks)), zap.Strings("tenantIDs", tenantIDs)) + common.Info("RetrievalByChildren started", zap.Int("chunks", len(chunks)), zap.Strings("tenantIDs", tenantIDs)) indexNames := buildIndexNames(tenantIDs) if len(chunks) == 0 || len(indexNames) == 0 { @@ -677,7 +677,7 @@ func RetrievalByChildren(chunks []map[string]interface{}, tenantIDs []string, do } if len(momChunks) == 0 { - logger.Info("RetrievalByChildren finished", zap.Int("momChunks", len(momChunks)), zap.Int("resultChunks", len(chunks))) + common.Info("RetrievalByChildren finished", zap.Int("momChunks", len(momChunks)), zap.Int("resultChunks", len(chunks))) return chunks } @@ -696,7 +696,7 @@ func RetrievalByChildren(chunks []map[string]interface{}, tenantIDs []string, do parent, err := docEngine.GetChunk(ctx, indexNames[0], momID, kbIDs) if err != nil { - logger.Warn("Failed to get parent chunk", zap.String("momID", momID), zap.Error(err)) + common.Warn("Failed to get parent chunk", zap.String("momID", momID), zap.Error(err)) continue } parentMap, ok := parent.(map[string]interface{}) @@ -784,7 +784,7 @@ func RetrievalByChildren(chunks []map[string]interface{}, tenantIDs []string, do } } - logger.Info("RetrievalByChildren finished", zap.Int("momChunks", len(momChunks)), zap.Int("resultChunks", len(remainingChunks))) + common.Info("RetrievalByChildren finished", zap.Int("momChunks", len(momChunks)), zap.Int("resultChunks", len(remainingChunks))) return remainingChunks } @@ -871,7 +871,7 @@ func (s *RetrievalService) PruneDeletedChunks(result *RetrievalSearchResult) (*R } if removed > 0 { - logger.Warn("Pruned stale chunks whose documents no longer exist", zap.Int("removed", removed)) + common.Warn("Pruned stale chunks whose documents no longer exist", zap.Int("removed", removed)) } return &RetrievalSearchResult{ diff --git a/internal/service/nlp/synonym.go b/internal/service/nlp/synonym.go index f5f0871cd9..335e252590 100644 --- a/internal/service/nlp/synonym.go +++ b/internal/service/nlp/synonym.go @@ -18,12 +18,11 @@ import ( "encoding/json" "os" "path/filepath" + "ragflow/internal/common" "regexp" "strings" "time" - "ragflow/internal/logger" - "go.uber.org/zap" ) @@ -96,18 +95,18 @@ func NewSynonym(redis RedisClient, resPath string, wordnetDir string) *Synonym { } } } else { - logger.Warn("Failed to parse synonym.json", zap.Error(err)) + common.Warn("Failed to parse synonym.json", zap.Error(err)) } } else { - logger.Warn("Missing synonym.json", zap.Error(err)) + common.Warn("Missing synonym.json", zap.Error(err)) } if redis == nil { - logger.Warn("Realtime synonym is disabled, since no redis connection.") + common.Warn("Realtime synonym is disabled, since no redis connection.") } if len(s.dictionary) == 0 { - logger.Warn("Fail to load synonym") + common.Warn("Fail to load synonym") } s.load() diff --git a/internal/service/nlp/term_weight.go b/internal/service/nlp/term_weight.go index 215d608bac..e72463674d 100644 --- a/internal/service/nlp/term_weight.go +++ b/internal/service/nlp/term_weight.go @@ -19,11 +19,11 @@ import ( "math" "os" "path/filepath" + "ragflow/internal/common" "regexp" "strconv" "strings" - "ragflow/internal/logger" "ragflow/internal/tokenizer" "go.uber.org/zap" @@ -59,10 +59,10 @@ func NewTermWeightDealer(resPath string) *TermWeightDealer { nerPath := filepath.Join(resPath, "ner.json") if data, err := os.ReadFile(nerPath); err == nil { if err := json.Unmarshal(data, &d.ne); err != nil { - logger.Warn("Failed to load ner.json", zap.Error(err)) + common.Warn("Failed to load ner.json", zap.Error(err)) } } else { - logger.Warn("Failed to load ner.json", zap.Error(err)) + common.Warn("Failed to load ner.json", zap.Error(err)) } // Load term frequency dictionary @@ -93,7 +93,7 @@ func loadDict(fnm string) map[string]int { res := make(map[string]int) data, err := os.ReadFile(fnm) if err != nil { - logger.Warn("Failed to load dictionary", zap.String("file", fnm), zap.Error(err)) + common.Warn("Failed to load dictionary", zap.String("file", fnm), zap.Error(err)) return res } diff --git a/internal/service/skill_indexer.go b/internal/service/skill_indexer.go index 414d1ccb9c..ec36a7948e 100644 --- a/internal/service/skill_indexer.go +++ b/internal/service/skill_indexer.go @@ -21,10 +21,10 @@ import ( "encoding/json" "fmt" "path/filepath" + "ragflow/internal/common" "ragflow/internal/dao" "ragflow/internal/engine" "ragflow/internal/entity" - "ragflow/internal/logger" "ragflow/internal/storage" "ragflow/internal/tokenizer" "strings" @@ -100,7 +100,7 @@ func (s *SkillIndexerService) IndexSkill(ctx context.Context, tenantID, spaceID // Generate embedding (optional - continue on failure) vector, err := s.generateEmbedding(ctx, vectorText, embdID, tenantID) if err != nil { - logger.Warn(fmt.Sprintf("Failed to generate embedding for skill %s: %v. Continuing with text-only index.", skill.ID, err)) + common.Warn(fmt.Sprintf("Failed to generate embedding for skill %s: %v. Continuing with text-only index.", skill.ID, err)) } // Build document with RAG tokenization for ES @@ -172,7 +172,7 @@ func (s *SkillIndexerService) IndexSkill(ctx context.Context, tenantID, spaceID if docEngine.GetType() == "infinity" { exists, _ := docEngine.TableExists(ctx, indexName) if !exists { - logger.Info(fmt.Sprintf("Creating Infinity table with dimension %d", dimension)) + common.Info(fmt.Sprintf("Creating Infinity table with dimension %d", dimension)) if err := s.createIndexWithDimension(ctx, tenantID, spaceID, docEngine, embdID, dimension); err != nil { return fmt.Errorf("failed to create index with dimension %d: %w", dimension, err) } @@ -181,20 +181,20 @@ func (s *SkillIndexerService) IndexSkill(ctx context.Context, tenantID, spaceID // Delete old versions (both new format and old format with version suffix) // This ensures only the latest version is indexed - logger.Debug(fmt.Sprintf("Deleting old versions of skill if exists: indexName=%s, skillName=%s", indexName, skill.Name)) + common.Debug(fmt.Sprintf("Deleting old versions of skill if exists: indexName=%s, skillName=%s", indexName, skill.Name)) if err := s.DeleteSkillByName(ctx, tenantID, spaceID, skill.Name, docEngine); err != nil { - logger.Debug(fmt.Sprintf("No existing document to delete for skill %s (this is normal for new skills)", skill.Name)) + common.Debug(fmt.Sprintf("No existing document to delete for skill %s (this is normal for new skills)", skill.Name)) } // ES document ID cannot contain '/' - replace with '_' docID := strings.ReplaceAll(skill.ID, "/", "_") - logger.Info(fmt.Sprintf("Calling IndexDocument: indexName=%s, docID=%s, engineType=%s", indexName, docID, docEngine.GetType())) + common.Info(fmt.Sprintf("Calling IndexDocument: indexName=%s, docID=%s, engineType=%s", indexName, docID, docEngine.GetType())) if err := docEngine.IndexDocument(ctx, indexName, docID, doc); err != nil { - logger.Error(fmt.Sprintf("IndexDocument failed: indexName=%s, docID=%s", indexName, docID), err) + common.Error(fmt.Sprintf("IndexDocument failed: indexName=%s, docID=%s", indexName, docID), err) return fmt.Errorf("failed to index document: %w", err) } - logger.Info(fmt.Sprintf("IndexDocument succeeded: indexName=%s, docID=%s", indexName, docID)) + common.Info(fmt.Sprintf("IndexDocument succeeded: indexName=%s, docID=%s", indexName, docID)) return nil } @@ -232,39 +232,39 @@ func (s *SkillIndexerService) BatchIndexSkills(ctx context.Context, tenantID, sp if err != nil { return fmt.Errorf("failed to get embedding dimension: %w", err) } - logger.Info(fmt.Sprintf("Using embedding dimension: %d", dimension)) + common.Info(fmt.Sprintf("Using embedding dimension: %d", dimension)) vectorField := fmt.Sprintf("q_%d_vec", dimension) // Generate embeddings in batch - logger.Info(fmt.Sprintf("Generating embeddings for %d skills with embdID=%s", len(skills), embdID)) + common.Info(fmt.Sprintf("Generating embeddings for %d skills with embdID=%s", len(skills), embdID)) vectors, err := s.generateEmbeddings(ctx, vectorTexts, embdID, tenantID) if err != nil { - logger.Warn(fmt.Sprintf("Failed to generate embeddings: %v. Continuing with text-only index.", err)) + common.Warn(fmt.Sprintf("Failed to generate embeddings: %v. Continuing with text-only index.", err)) vectors = nil // Continue without vectors } else { - logger.Info(fmt.Sprintf("Generated %d vectors", len(vectors))) + common.Info(fmt.Sprintf("Generated %d vectors", len(vectors))) } // Ensure index exists with correct dimension indexName := getSkillIndexName(tenantID, spaceID) if docEngine.GetType() == "infinity" { // For Infinity: must ensure table exists with correct dimension BEFORE inserting - logger.Info(fmt.Sprintf("Checking if index exists: %s", indexName)) + common.Info(fmt.Sprintf("Checking if index exists: %s", indexName)) exists, err := docEngine.TableExists(ctx, indexName) if err != nil { - logger.Warn(fmt.Sprintf("Error checking index existence: %v", err)) + common.Warn(fmt.Sprintf("Error checking index existence: %v", err)) } - logger.Info(fmt.Sprintf("Index exists: %v", exists)) + common.Info(fmt.Sprintf("Index exists: %v", exists)) if !exists { // Only create if table doesn't exist - logger.Info(fmt.Sprintf("Creating index with actual dimension %d", dimension)) + common.Info(fmt.Sprintf("Creating index with actual dimension %d", dimension)) if err := s.createIndexWithDimension(ctx, tenantID, spaceID, docEngine, embdID, dimension); err != nil { return fmt.Errorf("failed to create index with dimension %d: %w", dimension, err) } - logger.Info("Index created successfully") + common.Info("Index created successfully") } else { - logger.Info("Index already exists, skipping creation") + common.Info("Index already exists, skipping creation") } } else { // For ES: just ensure index exists @@ -283,7 +283,7 @@ func (s *SkillIndexerService) BatchIndexSkills(ctx context.Context, tenantID, sp // Delete old versions (both new format and old format with version suffix) // This ensures only the latest version is indexed if err := s.DeleteSkillByName(ctx, tenantID, spaceID, skill.Name, docEngine); err != nil { - logger.Debug(fmt.Sprintf("No existing document to delete for skill %s (this is normal for new skills)", skill.Name)) + common.Debug(fmt.Sprintf("No existing document to delete for skill %s (this is normal for new skills)", skill.Name)) } // ES document ID cannot contain '/' - replace with '_' @@ -313,7 +313,7 @@ func (s *SkillIndexerService) BatchIndexSkills(ctx context.Context, tenantID, sp if vectors != nil && i < len(vectors) { doc[vectorField] = vectors[i] } else { - logger.Info(fmt.Sprintf("No vector for skill %s, creating text-only index", skill.ID)) + common.Info(fmt.Sprintf("No vector for skill %s, creating text-only index", skill.ID)) // For Infinity: use zero vector as placeholder (table schema requires vector column) if docEngine.GetType() == "infinity" { zeroVector := make([]float64, dimension) @@ -340,9 +340,9 @@ func (s *SkillIndexerService) BatchIndexSkills(ctx context.Context, tenantID, sp } } - logger.Info("Batch: Calling IndexDocument", zap.String("indexName", indexName), zap.String("docID", docID), zap.Int("index", i)) + common.Info("Batch: Calling IndexDocument", zap.String("indexName", indexName), zap.String("docID", docID), zap.Int("index", i)) if err := docEngine.IndexDocument(ctx, indexName, docID, doc); err != nil { - logger.Error(fmt.Sprintf("Failed to index skill %s", skill.ID), err) + common.Error(fmt.Sprintf("Failed to index skill %s", skill.ID), err) indexErrors = append(indexErrors, fmt.Sprintf("%s: %v", skill.ID, err)) continue } @@ -365,10 +365,10 @@ func (s *SkillIndexerService) DeleteSkillIndex(ctx context.Context, tenantID, sp if err := docEngine.DeleteDocument(ctx, indexName, docID); err != nil { // Check if it's a "not found" error - this is OK, document might not have been indexed if strings.Contains(err.Error(), "not found") { - logger.Debug(fmt.Sprintf("Document %s not found in index %s, treating as already deleted", skillID, indexName)) + common.Debug(fmt.Sprintf("Document %s not found in index %s, treating as already deleted", skillID, indexName)) return nil } - logger.Error(fmt.Sprintf("Failed to delete document %s from index %s", skillID, indexName), err) + common.Error(fmt.Sprintf("Failed to delete document %s from index %s", skillID, indexName), err) return err } return nil @@ -382,7 +382,7 @@ func (s *SkillIndexerService) DeleteSkillByName(ctx context.Context, tenantID, s docID := strings.ReplaceAll(skillName, "/", "_") if err := docEngine.DeleteDocument(ctx, indexName, docID); err != nil { - logger.Debug(fmt.Sprintf("Document %s not found in index %s", skillName, indexName)) + common.Debug(fmt.Sprintf("Document %s not found in index %s", skillName, indexName)) } return nil @@ -394,7 +394,7 @@ func (s *SkillIndexerService) UpdateSkillVersion(ctx context.Context, tenantID, // Delete old version first (upsert behavior) if err := s.DeleteSkillByName(ctx, tenantID, spaceID, skill.Name, docEngine); err != nil { // Log but don't fail - the document might not exist - logger.Debug(fmt.Sprintf("No existing index to delete for skill %s", skill.Name)) + common.Debug(fmt.Sprintf("No existing index to delete for skill %s", skill.Name)) } // Index new version @@ -405,10 +405,10 @@ func (s *SkillIndexerService) UpdateSkillVersion(ctx context.Context, tenantID, // Increments semantic version, deletes old table, and reindexes all skills from file system // For Infinity: if embedding model changed (different dimension), recreates the table // Behavior: -// 1. Delete the existing table -// 2. Traverse all skill folders under the space -// 3. For each skill, get the latest version -// 4. Reindex all skills +// 1. Delete the existing table +// 2. Traverse all skill folders under the space +// 3. For each skill, get the latest version +// 4. Reindex all skills func (s *SkillIndexerService) ReindexAll(ctx context.Context, tenantID, spaceID string, docEngine engine.DocEngine, embdID string) (map[string]interface{}, error) { spaceID = normalizeSpaceID(spaceID) // Get current config and increment semantic version @@ -430,20 +430,20 @@ func (s *SkillIndexerService) ReindexAll(ctx context.Context, tenantID, spaceID if err != nil { return nil, fmt.Errorf("failed to get new embedding dimension: %w", err) } - logger.Info(fmt.Sprintf("ReindexAll: new embedding dimension is %d", newDimension)) + common.Info(fmt.Sprintf("ReindexAll: new embedding dimension is %d", newDimension)) // Delete existing index and recreate with new dimension (for both ES and Infinity) indexName := getSkillIndexName(tenantID, spaceID) exists, _ := docEngine.TableExists(ctx, indexName) if exists { - logger.Info(fmt.Sprintf("ReindexAll: deleting existing index %s", indexName)) + common.Info(fmt.Sprintf("ReindexAll: deleting existing index %s", indexName)) if err := docEngine.DropTable(ctx, indexName); err != nil { - logger.Warn(fmt.Sprintf("ReindexAll: failed to delete existing index: %v", err)) + common.Warn(fmt.Sprintf("ReindexAll: failed to delete existing index: %v", err)) } } // Create new index with correct dimension - logger.Info(fmt.Sprintf("ReindexAll: creating new index %s with dimension %d", indexName, newDimension)) + common.Info(fmt.Sprintf("ReindexAll: creating new index %s with dimension %d", indexName, newDimension)) if err := s.createIndexWithDimension(ctx, tenantID, spaceID, docEngine, embdID, newDimension); err != nil { return nil, fmt.Errorf("failed to create index with dimension %d: %w", newDimension, err) } @@ -463,7 +463,7 @@ func (s *SkillIndexerService) ReindexAll(ctx context.Context, tenantID, spaceID if err != nil { return nil, fmt.Errorf("failed to find space folder: %w", err) } - logger.Info(fmt.Sprintf("ReindexAll: found space folder ID %s for space %s (stored FolderID was %s)", spaceFolderID, space.Name, space.FolderID)) + common.Info(fmt.Sprintf("ReindexAll: found space folder ID %s for space %s (stored FolderID was %s)", spaceFolderID, space.Name, space.FolderID)) // Traverse all skill folders under the space skills, err := s.getSkillsFromFileSystem(ctx, tenantID, spaceFolderID, spaceID) @@ -471,20 +471,20 @@ func (s *SkillIndexerService) ReindexAll(ctx context.Context, tenantID, spaceID return nil, fmt.Errorf("failed to get skills from file system: %w", err) } - logger.Info(fmt.Sprintf("ReindexAll: found %d skills to index", len(skills))) + common.Info(fmt.Sprintf("ReindexAll: found %d skills to index", len(skills))) // Index all skills with new version using batch indexing for better performance if len(skills) > 0 { - logger.Info(fmt.Sprintf("ReindexAll: batch indexing %d skills", len(skills))) + common.Info(fmt.Sprintf("ReindexAll: batch indexing %d skills", len(skills))) if err := s.BatchIndexSkills(ctx, tenantID, spaceID, skills, docEngine, embdID); err != nil { - logger.Error("ReindexAll: batch indexing failed", err) + common.Error("ReindexAll: batch indexing failed", err) return nil, fmt.Errorf("failed to batch index skills: %w", err) } } // Clean up old version documents if err := s.cleanupOldVersions(ctx, tenantID, spaceID, newVersion, docEngine); err != nil { - logger.Error("Failed to cleanup old versions", err) + common.Error("Failed to cleanup old versions", err) } result := map[string]interface{}{ @@ -507,7 +507,7 @@ func (s *SkillIndexerService) getSkillsFromFileSystem(ctx context.Context, tenan return nil, fmt.Errorf("failed to list skill folders: %w", err) } - logger.Info(fmt.Sprintf("getSkillsFromFileSystem: found %d skill folders in space %s", len(skillFolders), spaceID)) + common.Info(fmt.Sprintf("getSkillsFromFileSystem: found %d skill folders in space %s", len(skillFolders), spaceID)) for _, skillFolder := range skillFolders { if skillFolder.Type != "folder" { @@ -517,31 +517,31 @@ func (s *SkillIndexerService) getSkillsFromFileSystem(ctx context.Context, tenan // Get all versions of this skill versions, err := s.fileDAO.ListByParentID(skillFolder.ID) if err != nil { - logger.Warn(fmt.Sprintf("failed to list versions for skill %s: %v", skillFolder.Name, err)) + common.Warn(fmt.Sprintf("failed to list versions for skill %s: %v", skillFolder.Name, err)) continue } if len(versions) == 0 { - logger.Info(fmt.Sprintf("no versions found for skill %s", skillFolder.Name)) + common.Info(fmt.Sprintf("no versions found for skill %s", skillFolder.Name)) continue } // Find the latest version (highest semantic version) latestVersion := s.findLatestVersion(versions) if latestVersion == nil { - logger.Warn(fmt.Sprintf("no valid version found for skill %s", skillFolder.Name)) + common.Warn(fmt.Sprintf("no valid version found for skill %s", skillFolder.Name)) continue } // Get skill content from the latest version folder skillInfo, err := s.getSkillContentFromFolder(ctx, tenantID, skillFolder, latestVersion, spaceID) if err != nil { - logger.Warn(fmt.Sprintf("failed to get skill content for %s: %v", skillFolder.Name, err)) + common.Warn(fmt.Sprintf("failed to get skill content for %s: %v", skillFolder.Name, err)) continue } skills = append(skills, *skillInfo) - logger.Info(fmt.Sprintf("added skill %s version %s for indexing", skillFolder.Name, latestVersion.Name)) + common.Info(fmt.Sprintf("added skill %s version %s for indexing", skillFolder.Name, latestVersion.Name)) } return skills, nil @@ -609,7 +609,7 @@ func (s *SkillIndexerService) getSkillContentFromFolder(ctx context.Context, ten // Get file content (this might need to be implemented based on your storage system) fileContent, err := s.getFileContent(ctx, tenantID, file) if err != nil { - logger.Warn(fmt.Sprintf("failed to get content for file %s: %v", file.Name, err)) + common.Warn(fmt.Sprintf("failed to get content for file %s: %v", file.Name, err)) continue } @@ -842,20 +842,20 @@ func (s *SkillIndexerService) InitializeIndex(ctx context.Context, tenantID, spa // Check if index exists indexName := getSkillIndexName(tenantID, spaceID) - logger.Info("Checking skill index existence", zap.String("indexName", indexName), zap.String("tenantID", tenantID), zap.String("spaceID", spaceID)) + common.Info("Checking skill index existence", zap.String("indexName", indexName), zap.String("tenantID", tenantID), zap.String("spaceID", spaceID)) exists, err := docEngine.TableExists(ctx, indexName) if err != nil { - logger.Error("Failed to check index existence", err) + common.Error("Failed to check index existence", err) return fmt.Errorf("failed to check index existence: %w", err) } if !exists { - logger.Info("Skill index does not exist, creating...", zap.String("indexName", indexName)) + common.Info("Skill index does not exist, creating...", zap.String("indexName", indexName)) return s.createIndex(ctx, tenantID, spaceID, docEngine, embdID) } - logger.Info("Skill search index already exists", zap.String("indexName", indexName)) + common.Info("Skill search index already exists", zap.String("indexName", indexName)) return nil } @@ -873,7 +873,7 @@ func (s *SkillIndexerService) createIndex(ctx context.Context, tenantID, spaceID func (s *SkillIndexerService) createIndexWithDimension(ctx context.Context, tenantID, spaceID string, docEngine engine.DocEngine, embdID string, dimension int) error { indexName := getSkillIndexName(tenantID, spaceID) - logger.Info(fmt.Sprintf("Creating skill index with dimension %d", dimension), + common.Info(fmt.Sprintf("Creating skill index with dimension %d", dimension), zap.String("indexName", indexName), zap.String("spaceID", spaceID), zap.Int("dimension", dimension), @@ -883,13 +883,13 @@ func (s *SkillIndexerService) createIndexWithDimension(ctx context.Context, tena if docEngine.GetType() == "infinity" { exists, err := docEngine.TableExists(ctx, indexName) if err != nil { - logger.Warn(fmt.Sprintf("Error checking if index exists: %v", err)) + common.Warn(fmt.Sprintf("Error checking if index exists: %v", err)) } if exists { - logger.Info(fmt.Sprintf("Index exists, deleting for recreation with dimension %d", dimension), + common.Info(fmt.Sprintf("Index exists, deleting for recreation with dimension %d", dimension), zap.String("indexName", indexName)) if err := docEngine.DropTable(ctx, indexName); err != nil { - logger.Warn(fmt.Sprintf("Failed to delete existing index: %v", err)) + common.Warn(fmt.Sprintf("Failed to delete existing index: %v", err)) } } } @@ -898,10 +898,10 @@ func (s *SkillIndexerService) createIndexWithDimension(ctx context.Context, tena // The mapping file is loaded from conf/skill_es_mapping.json or conf/skill_infinity_mapping.json err := docEngine.CreateDataset(ctx, indexName, "skill", dimension, "") if err != nil { - logger.Error("Failed to create skill index", err) + common.Error("Failed to create skill index", err) return err } - logger.Info("Successfully created skill index", zap.String("indexName", indexName)) + common.Info("Successfully created skill index", zap.String("indexName", indexName)) return nil } @@ -946,7 +946,7 @@ func (s *SkillIndexerService) generateEmbedding(ctx context.Context, text, embdI // generateEmbeddings generates embeddings for multiple texts in batch // This is more efficient than calling generateEmbedding individually func (s *SkillIndexerService) generateEmbeddings(ctx context.Context, texts []string, embdID, tenantID string) ([][]float64, error) { - logger.Info(fmt.Sprintf("generateEmbeddings called: texts=%d, embdID=%s, tenantID=%s", len(texts), embdID, tenantID)) + common.Info(fmt.Sprintf("generateEmbeddings called: texts=%d, embdID=%s, tenantID=%s", len(texts), embdID, tenantID)) if s.modelProvider == nil { return nil, fmt.Errorf("model provider not set") @@ -956,10 +956,10 @@ func (s *SkillIndexerService) generateEmbeddings(ctx context.Context, texts []st return nil, fmt.Errorf("embedding model ID not configured") } - logger.Info(fmt.Sprintf("Getting embedding model for %s", embdID)) + common.Info(fmt.Sprintf("Getting embedding model for %s", embdID)) embeddingModel, err := s.modelProvider.GetEmbeddingModel(tenantID, embdID) if err != nil { - logger.Error(fmt.Sprintf("Failed to get embedding model: %v", err), err) + common.Error(fmt.Sprintf("Failed to get embedding model: %v", err), err) return nil, fmt.Errorf("failed to get embedding model: %w", err) } @@ -973,17 +973,17 @@ func (s *SkillIndexerService) generateEmbeddings(ctx context.Context, texts []st truncatedTexts[i] = truncate(text, maxLen-10) } - logger.Info(fmt.Sprintf("Encoding %d texts", len(truncatedTexts))) + common.Info(fmt.Sprintf("Encoding %d texts", len(truncatedTexts))) // Use batch encode API (consistent with Python's encode(texts: list)) vectors, err := embeddingModel.ModelDriver.Encode(embeddingModel.ModelName, truncatedTexts, embeddingModel.APIConfig, nil) if err != nil { - logger.Error(fmt.Sprintf("Failed to encode texts: %v", err), err) + common.Error(fmt.Sprintf("Failed to encode texts: %v", err), err) return nil, fmt.Errorf("failed to encode texts: %w", err) } - logger.Info(fmt.Sprintf("Encoded successfully, got %d vectors", len(vectors))) + common.Info(fmt.Sprintf("Encoded successfully, got %d vectors", len(vectors))) if len(vectors) > 0 { - logger.Info(fmt.Sprintf("Vector dimension: %d", len(vectors[0]))) + common.Info(fmt.Sprintf("Vector dimension: %d", len(vectors[0]))) } return vectors, nil @@ -1031,6 +1031,6 @@ func (s *SkillIndexerService) getEmbeddingDimension(ctx context.Context, tenantI } dimension := len(vectors[0]) - logger.Info(fmt.Sprintf("Got embedding dimension from API: %d", dimension)) + common.Info(fmt.Sprintf("Got embedding dimension from API: %d", dimension)) return dimension, nil } diff --git a/internal/service/skill_search.go b/internal/service/skill_search.go index b96f076ca0..c48d0f1314 100644 --- a/internal/service/skill_search.go +++ b/internal/service/skill_search.go @@ -27,7 +27,6 @@ import ( "ragflow/internal/engine" "ragflow/internal/engine/types" "ragflow/internal/entity" - "ragflow/internal/logger" "ragflow/internal/utility" "strings" @@ -144,7 +143,7 @@ func (s *SkillSearchService) UpdateConfig(req *UpdateConfigRequest) (map[string] // Config exists, clean up any other active records for this tenant+space // to ensure only one active config per tenant+space if err := s.configDAO.DeleteAllByTenantSpaceExceptID(req.TenantID, req.SpaceID, config.ID); err != nil { - logger.Warn("Failed to clean up duplicate configs", zap.Error(err)) + common.Warn("Failed to clean up duplicate configs", zap.Error(err)) } } @@ -224,18 +223,18 @@ func (s *SkillSearchService) Search(ctx context.Context, req *SearchRequest, doc // Check if index exists before searching indexName := getSkillIndexName(req.TenantID, req.SpaceID) - logger.Debug("Searching skills", zap.String("indexName", indexName), zap.String("query", req.Query)) + common.Debug("Searching skills", zap.String("indexName", indexName), zap.String("query", req.Query)) indexExists, err := docEngine.TableExists(ctx, indexName) if err != nil { - logger.Error("Failed to check index existence", err) + common.Error("Failed to check index existence", err) return nil, common.CodeOperatingError, fmt.Errorf("failed to check index existence: %w", err) } - logger.Debug("Index existence check", zap.String("indexName", indexName), zap.Bool("exists", indexExists)) + common.Debug("Index existence check", zap.String("indexName", indexName), zap.Bool("exists", indexExists)) if !indexExists { // Return empty result if index doesn't exist (no skills indexed yet) // This allows listing skills via file system API as fallback - logger.Warn("Skill index does not exist, returning empty result", zap.String("indexName", indexName), zap.String("tenantID", req.TenantID), zap.String("spaceID", req.SpaceID)) + common.Warn("Skill index does not exist, returning empty result", zap.String("indexName", indexName), zap.String("tenantID", req.TenantID), zap.String("spaceID", req.SpaceID)) return &SearchResponse{ Skills: []entity.SkillSearchResult{}, Total: 0, @@ -286,7 +285,7 @@ func (s *SkillSearchService) Search(ctx context.Context, req *SearchRequest, doc searchType = "vector" results, err = s.vectorSearch(ctx, docEngine, indexName, req.Query, config, req.TenantID) if err != nil { - logger.Warn("Vector search failed, falling back to keyword search", zap.Error(err)) + common.Warn("Vector search failed, falling back to keyword search", zap.Error(err)) searchType = "keyword" results, err = s.keywordSearch(ctx, docEngine, indexName, req.Query, config, config.SimilarityThreshold, req.SortBy, req.SortOrder) } @@ -301,7 +300,7 @@ func (s *SkillSearchService) Search(ctx context.Context, req *SearchRequest, doc } if err != nil { - logger.Error("Skill search failed", err) + common.Error("Skill search failed", err) return nil, common.CodeOperatingError, fmt.Errorf("search failed: %w", err) } @@ -374,12 +373,12 @@ func (s *SkillSearchService) vectorSearch(ctx context.Context, docEngine engine. // Get embedding for query vector, err := s.getEmbedding(ctx, query, config.EmbdID, tenantID) if err != nil { - logger.Warn("Vector search: failed to get embedding, will fallback to keyword search", + common.Warn("Vector search: failed to get embedding, will fallback to keyword search", zap.String("embdID", config.EmbdID), zap.Error(err)) return nil, fmt.Errorf("failed to get embedding: %w", err) } - logger.Debug("Vector search: successfully got embedding", + common.Debug("Vector search: successfully got embedding", zap.String("embdID", config.EmbdID), zap.Int("dimension", len(vector))) @@ -418,20 +417,20 @@ func (s *SkillSearchService) vectorSearch(ctx context.Context, docEngine engine. searchResult, err := docEngine.Search(ctx, searchReq) if err != nil { - logger.Warn("Vector search: search execution failed", + common.Warn("Vector search: search execution failed", zap.String("indexName", indexName), zap.Error(err)) return nil, err } results := s.convertChunksToResults(searchResult.Chunks, config.SimilarityThreshold) - logger.Debug("Vector search: completed", + common.Debug("Vector search: completed", zap.Int("totalChunks", len(searchResult.Chunks)), zap.Int("filteredResults", len(results))) // If no results, return error to trigger fallback if len(results) == 0 { - logger.Info("Vector search: no results found, will fallback to keyword search", + common.Info("Vector search: no results found, will fallback to keyword search", zap.String("indexName", indexName), zap.String("query", query)) return nil, fmt.Errorf("vector search returned no results") @@ -451,19 +450,19 @@ func (s *SkillSearchService) hybridSearch(ctx context.Context, docEngine engine. "description^3", "content^1", }, - TopN: int(config.TopK), + TopN: int(config.TopK), } // Get embedding for query vector, err := s.getEmbedding(ctx, query, config.EmbdID, tenantID) if err != nil { - logger.Warn("Hybrid search: failed to get embedding, falling back to keyword search", + common.Warn("Hybrid search: failed to get embedding, falling back to keyword search", zap.String("embdID", config.EmbdID), zap.Error(err)) // Fallback to keyword search with analyzed query return s.executeKeywordSearch(ctx, docEngine, indexName, query, matchExpr, config) } - logger.Debug("Hybrid search: successfully got embedding", + common.Debug("Hybrid search: successfully got embedding", zap.String("embdID", config.EmbdID), zap.Int("dimension", len(vector))) @@ -500,20 +499,20 @@ func (s *SkillSearchService) hybridSearch(ctx context.Context, docEngine engine. searchResult, err := docEngine.Search(ctx, searchReq) if err != nil { - logger.Warn("Hybrid search: search execution failed, falling back to keyword search", + common.Warn("Hybrid search: search execution failed, falling back to keyword search", zap.String("indexName", indexName), zap.Error(err)) return s.executeKeywordSearch(ctx, docEngine, indexName, query, matchExpr, config) } results := s.convertChunksToResults(searchResult.Chunks, config.SimilarityThreshold) - logger.Debug("Hybrid search completed", + common.Debug("Hybrid search completed", zap.Int("totalChunks", len(searchResult.Chunks)), zap.Int("filteredResults", len(results))) // If no results, fallback to keyword search if len(results) == 0 { - logger.Info("Hybrid search: no results found, falling back to keyword search", + common.Info("Hybrid search: no results found, falling back to keyword search", zap.String("indexName", indexName), zap.String("query", query)) return s.executeKeywordSearch(ctx, docEngine, indexName, query, matchExpr, config) @@ -524,7 +523,7 @@ func (s *SkillSearchService) hybridSearch(ctx context.Context, docEngine engine. // executeKeywordSearch executes a keyword search (used for fallback) func (s *SkillSearchService) executeKeywordSearch(ctx context.Context, docEngine engine.DocEngine, indexName, query string, matchExpr *types.MatchTextExpr, config *entity.SkillSearchConfig) ([]entity.SkillSearchResult, error) { - logger.Debug("Executing fallback keyword search", + common.Debug("Executing fallback keyword search", zap.String("indexName", indexName), zap.String("query", query)) @@ -537,12 +536,12 @@ func (s *SkillSearchService) executeKeywordSearch(ctx context.Context, docEngine searchResult, err := docEngine.Search(ctx, searchReq) if err != nil { - logger.Error("Keyword search fallback failed", err) + common.Error("Keyword search fallback failed", err) return nil, err } results := s.convertChunksToResults(searchResult.Chunks, config.SimilarityThreshold) - logger.Debug("Keyword search fallback completed", + common.Debug("Keyword search fallback completed", zap.Int("totalChunks", len(searchResult.Chunks)), zap.Int("results", len(results))) @@ -624,21 +623,21 @@ func (s *SkillSearchService) convertChunksToResults(chunks []map[string]interfac createTime = ctVal } - // Extract version - version := getString(chunk, "version") + // Extract version + version := getString(chunk, "version") - result := entity.SkillSearchResult{ - SkillID: skillID, - FolderID: folderID, - Name: name, - Description: description, - Tags: tags, - Score: score, - BM25Score: bm25Score, - VectorScore: vectorScore, - CreateTime: createTime, - Version: version, - } + result := entity.SkillSearchResult{ + SkillID: skillID, + FolderID: folderID, + Name: name, + Description: description, + Tags: tags, + Score: score, + BM25Score: bm25Score, + VectorScore: vectorScore, + CreateTime: createTime, + Version: version, + } // Keep only the highest scored result for each skill if existing, ok := skillMap[skillKey]; !ok || score > existing.Score { @@ -725,8 +724,6 @@ func normalizeSpaceID(spaceID string) string { return spaceID } - - func getString(m map[string]interface{}, key string) string { if v, ok := m[key].(string); ok { return v @@ -962,14 +959,14 @@ func (s *SkillSearchService) buildOrderBy(sortBy, sortOrder string, isEmptyQuery // Map frontend field names to backend field names fieldMapping := map[string]string{ - "name": "name", - "update_time": "update_time", - "create_time": "create_time", - "updateTime": "update_time", - "createTime": "create_time", - "relevance": "", // Empty means sort by score/relevance - "updated_at": "update_time", - "created_at": "create_time", + "name": "name", + "update_time": "update_time", + "create_time": "create_time", + "updateTime": "update_time", + "createTime": "create_time", + "relevance": "", // Empty means sort by score/relevance + "updated_at": "update_time", + "created_at": "create_time", } backendField, ok := fieldMapping[sortBy] diff --git a/internal/service/skill_space.go b/internal/service/skill_space.go index 29d1b780f9..e40907fec4 100644 --- a/internal/service/skill_space.go +++ b/internal/service/skill_space.go @@ -28,7 +28,6 @@ import ( "ragflow/internal/dao" "ragflow/internal/engine" "ragflow/internal/entity" - "ragflow/internal/logger" "strings" "sync" "time" @@ -39,14 +38,14 @@ import ( // SkillSpaceService handles business logic for skills space operations type SkillSpaceService struct { - spaceDAO *dao.SkillSpaceDAO - fileDAO *dao.FileDAO - configDAO *dao.SkillSearchConfigDAO - tenantDAO *dao.TenantDAO - skillsFolderCache map[string]string // tenant-keyed cache for skills folder ID - skillsFolderMu sync.RWMutex // protects skillsFolderCache + spaceDAO *dao.SkillSpaceDAO + fileDAO *dao.FileDAO + configDAO *dao.SkillSearchConfigDAO + tenantDAO *dao.TenantDAO + skillsFolderCache map[string]string // tenant-keyed cache for skills folder ID + skillsFolderMu sync.RWMutex // protects skillsFolderCache skillsFolderCreateMu sync.Map // tenant-scoped locks for folder creation - spaceCreateMu sync.Map // tenant-scoped locks for space creation (prevents TOCTOU races) + spaceCreateMu sync.Map // tenant-scoped locks for space creation (prevents TOCTOU races) } // NewSkillSpaceService creates a new SkillSpaceService instance @@ -125,7 +124,7 @@ func (s *SkillSpaceService) getSkillsFolderID(tenantID string) (string, error) { } // Skills folder not found, create it - logger.Info("Creating skills folder", zap.String("tenant_id", tenantID)) + common.Info("Creating skills folder", zap.String("tenant_id", tenantID)) folderID := generateSpaceID() now := time.Now() createTime := now.UnixMilli() @@ -193,13 +192,13 @@ func (s *SkillSpaceService) CreateSpace(req *CreateSpaceRequest) (map[string]int // This handles the case where a previous creation failed partially // Only delete non-active spaces (status != '1') to prevent TOCTOU race if err := s.spaceDAO.DeletePermanentByName(req.TenantID, req.Name); err != nil { - logger.Warn("Failed to delete permanent space by name", zap.Error(err)) + common.Warn("Failed to delete permanent space by name", zap.Error(err)) } // Get skills folder ID skillsFolderID, err := s.getSkillsFolderID(req.TenantID) if err != nil { - logger.Error("Failed to get skills folder ID", err) + common.Error("Failed to get skills folder ID", err) return nil, common.CodeOperatingError, err } @@ -208,9 +207,9 @@ func (s *SkillSpaceService) CreateSpace(req *CreateSpaceRequest) (map[string]int existingFolders := s.fileDAO.Query(req.Name, skillsFolderID) for _, f := range existingFolders { if f.Type == "folder" && f.Name == req.Name { - logger.Info("Deleting existing space folder with same name", zap.String("folderID", f.ID), zap.String("name", req.Name)) + common.Info("Deleting existing space folder with same name", zap.String("folderID", f.ID), zap.String("name", req.Name)) if err := s.deleteFolderRecursive(f.ID); err != nil { - logger.Warn("Failed to delete existing folder", zap.String("folderID", f.ID), zap.Error(err)) + common.Warn("Failed to delete existing folder", zap.String("folderID", f.ID), zap.Error(err)) } break } @@ -235,7 +234,7 @@ func (s *SkillSpaceService) CreateSpace(req *CreateSpaceRequest) (map[string]int } if err := s.fileDAO.Create(folder); err != nil { - logger.Error("Failed to create space folder", err) + common.Error("Failed to create space folder", err) return nil, common.CodeOperatingError, fmt.Errorf("failed to create space folder: %w", err) } @@ -256,7 +255,7 @@ func (s *SkillSpaceService) CreateSpace(req *CreateSpaceRequest) (map[string]int if err := s.spaceDAO.Create(space); err != nil { // Rollback: delete the created folder - logger.Error("Failed to create space in database", err) + common.Error("Failed to create space in database", err) s.fileDAO.DeleteByIDs([]string{folderID}) return nil, common.CodeOperatingError, fmt.Errorf("failed to create space: %w", err) } @@ -267,14 +266,14 @@ func (s *SkillSpaceService) CreateSpace(req *CreateSpaceRequest) (map[string]int tenant, err := s.tenantDAO.GetByID(req.TenantID) if err == nil && tenant != nil && tenant.EmbdID != "" { defaultEmbdID = tenant.EmbdID - logger.Info("Using tenant default embedding model", zap.String("tenantID", req.TenantID), zap.String("embdID", defaultEmbdID)) + common.Info("Using tenant default embedding model", zap.String("tenantID", req.TenantID), zap.String("embdID", defaultEmbdID)) } else { - logger.Warn("Tenant has no default embedding model, skill search will not work until configured", zap.String("tenantID", req.TenantID)) + common.Warn("Tenant has no default embedding model, skill search will not work until configured", zap.String("tenantID", req.TenantID)) } } if defaultEmbdID != "" { if _, err := s.configDAO.GetOrCreate(req.TenantID, spaceID, defaultEmbdID); err != nil { - logger.Warn("Failed to create skill search config for new space", + common.Warn("Failed to create skill search config for new space", zap.String("tenantID", req.TenantID), zap.String("spaceID", spaceID), zap.String("embdID", defaultEmbdID), @@ -338,7 +337,7 @@ func (s *SkillSpaceService) UpdateSpace(spaceID string, tenantID string, req *Up // Build updates updates := make(map[string]interface{}) - + if req.Name != "" && req.Name != space.Name { // Check if name already exists existingSpace, _ := s.spaceDAO.GetByTenantAndName(tenantID, req.Name) @@ -356,10 +355,10 @@ func (s *SkillSpaceService) UpdateSpace(spaceID string, tenantID string, req *Up // Update folder name as well - if this fails, rollback space name if err := s.fileDAO.UpdateByID(space.FolderID, map[string]interface{}{"name": req.Name}); err != nil { - logger.Error("Failed to update folder name, rolling back space name", err) + common.Error("Failed to update folder name, rolling back space name", err) // Rollback space name if rollbackErr := s.spaceDAO.UpdateByID(spaceID, map[string]interface{}{"name": originalName}); rollbackErr != nil { - logger.Error("Failed to rollback space name after folder rename failure", rollbackErr) + common.Error("Failed to rollback space name after folder rename failure", rollbackErr) } return nil, common.CodeOperatingError, fmt.Errorf("failed to update folder name: %w", err) } @@ -367,7 +366,7 @@ func (s *SkillSpaceService) UpdateSpace(spaceID string, tenantID string, req *Up // Clear updates map since we've already applied name change delete(updates, "name") } - + if req.Description != space.Description { updates["description"] = req.Description } @@ -443,7 +442,7 @@ func (s *SkillSpaceService) deleteFolderViaPythonAPI(folderID, tenantID, authHea // Set tenant ID header for Python backend req.Header.Set("X-tenant-id", tenantID) - logger.Info("Calling Python API to delete folder", zap.String("folderID", folderID), zap.String("tenantID", tenantID)) + common.Info("Calling Python API to delete folder", zap.String("folderID", folderID), zap.String("tenantID", tenantID)) client := &http.Client{Timeout: 60 * time.Second} resp, err := client.Do(req) @@ -453,7 +452,7 @@ func (s *SkillSpaceService) deleteFolderViaPythonAPI(folderID, tenantID, authHea defer resp.Body.Close() body, _ := io.ReadAll(resp.Body) - logger.Info("Python API delete folder response", zap.String("folderID", folderID), zap.Int("status", resp.StatusCode), zap.String("body", string(body))) + common.Info("Python API delete folder response", zap.String("folderID", folderID), zap.Int("status", resp.StatusCode), zap.String("body", string(body))) if resp.StatusCode != http.StatusOK { return fmt.Errorf("Python API returned status %d: %s", resp.StatusCode, string(body)) @@ -473,7 +472,7 @@ func (s *SkillSpaceService) deleteFolderViaPythonAPI(folderID, tenantID, authHea return fmt.Errorf("Python API returned error: %s", message) } - logger.Info("Successfully deleted folder via Python API", zap.String("folderID", folderID)) + common.Info("Successfully deleted folder via Python API", zap.String("folderID", folderID)) return nil } @@ -493,13 +492,13 @@ func (s *SkillSpaceService) DeleteSpace(spaceID, tenantID string, docEngine engi // If already deleting, return success (idempotent) if space.Status == entity.SpaceStatusDeleting { - logger.Info("Space is already being deleted", zap.String("spaceID", spaceID)) + common.Info("Space is already being deleted", zap.String("spaceID", spaceID)) return common.CodeSuccess, nil } // If already deleted, return success (idempotent) if space.Status == entity.SpaceStatusDeleted { - logger.Info("Space is already deleted", zap.String("spaceID", spaceID)) + common.Info("Space is already deleted", zap.String("spaceID", spaceID)) return common.CodeSuccess, nil } @@ -513,7 +512,7 @@ func (s *SkillSpaceService) DeleteSpace(spaceID, tenantID string, docEngine engi return common.CodeOperatingError, fmt.Errorf("space is being modified by another request") } - logger.Info("Space marked as deleting, starting async cleanup", zap.String("spaceID", spaceID), zap.String("tenantID", tenantID)) + common.Info("Space marked as deleting, starting async cleanup", zap.String("spaceID", spaceID), zap.String("tenantID", tenantID)) // Launch async deletion in background goroutine go s.asyncDeleteSpace(spaceID, space.FolderID, tenantID, docEngine, authHeader) @@ -526,7 +525,7 @@ func (s *SkillSpaceService) DeleteSpace(spaceID, tenantID string, docEngine engi func (s *SkillSpaceService) asyncDeleteSpace(spaceID, folderID, tenantID string, docEngine engine.DocEngine, authHeader string) { defer func() { if r := recover(); r != nil { - logger.Warn("Panic in asyncDeleteSpace, marking space as deleted", zap.Any("recover", r), zap.String("spaceID", spaceID)) + common.Warn("Panic in asyncDeleteSpace, marking space as deleted", zap.Any("recover", r), zap.String("spaceID", spaceID)) _, _ = s.spaceDAO.CASStatus(spaceID, entity.SpaceStatusDeleting, entity.SpaceStatusDeleted) } }() @@ -534,29 +533,29 @@ func (s *SkillSpaceService) asyncDeleteSpace(spaceID, folderID, tenantID string, // Step 1: Delete the search index if docEngine != nil { indexName := getSkillIndexName(tenantID, spaceID) - logger.Info("Async deleting space index", zap.String("index", indexName), zap.String("spaceID", spaceID)) + common.Info("Async deleting space index", zap.String("index", indexName), zap.String("spaceID", spaceID)) deleteCtx, cancel := context.WithTimeout(context.Background(), 60*time.Second) if err := docEngine.DropTable(deleteCtx, indexName); err != nil { - logger.Warn("Failed to delete space index during async delete", zap.String("index", indexName), zap.Error(err)) + common.Warn("Failed to delete space index during async delete", zap.String("index", indexName), zap.Error(err)) // Continue with other cleanup steps } else { - logger.Info("Successfully deleted space index", zap.String("index", indexName)) + common.Info("Successfully deleted space index", zap.String("index", indexName)) } cancel() } // Step 2: Delete folder and storage via Python API - logger.Info("Async deleting space folder via Python API", zap.String("folderID", folderID), zap.String("spaceID", spaceID)) + common.Info("Async deleting space folder via Python API", zap.String("folderID", folderID), zap.String("spaceID", spaceID)) if err := s.deleteFolderViaPythonAPI(folderID, tenantID, authHeader); err != nil { - logger.Error(fmt.Sprintf("Failed to delete space folder via Python API during async delete, spaceID=%s", spaceID), err) + common.Error(fmt.Sprintf("Failed to delete space folder via Python API during async delete, spaceID=%s", spaceID), err) // Retry once with a delay time.Sleep(5 * time.Second) if retryErr := s.deleteFolderViaPythonAPI(folderID, tenantID, authHeader); retryErr != nil { - logger.Error(fmt.Sprintf("Retry failed to delete space folder, marking space as deleted anyway, spaceID=%s", spaceID), retryErr) + common.Error(fmt.Sprintf("Retry failed to delete space folder, marking space as deleted anyway, spaceID=%s", spaceID), retryErr) // Mark as deleted even if folder deletion fails - orphaned folders can be cleaned up later } } else { - logger.Info("Successfully deleted space folder via Python API", zap.String("folderID", folderID)) + common.Info("Successfully deleted space folder via Python API", zap.String("folderID", folderID)) } // Step 3: Soft delete the space record (status "2" → "0") @@ -569,15 +568,15 @@ func (s *SkillSpaceService) asyncDeleteSpace(spaceID, folderID, tenantID string, swapped, err := s.spaceDAO.CASStatus(spaceID, entity.SpaceStatusDeleting, entity.SpaceStatusDeleted) if err != nil { - logger.Error(fmt.Sprintf("Failed to update space status to deleted, spaceID=%s", spaceID), err) + common.Error(fmt.Sprintf("Failed to update space status to deleted, spaceID=%s", spaceID), err) return } if !swapped { - logger.Warn("Space status was not 'deleting' when trying to mark as deleted", zap.String("spaceID", spaceID)) + common.Warn("Space status was not 'deleting' when trying to mark as deleted", zap.String("spaceID", spaceID)) return } - logger.Info("Successfully completed async space deletion", zap.String("spaceID", spaceID)) + common.Info("Successfully completed async space deletion", zap.String("spaceID", spaceID)) } // deleteFolderRecursive recursively deletes a folder and all its contents @@ -585,41 +584,41 @@ func (s *SkillSpaceService) deleteFolderRecursive(folderID string) error { // Get all children children, err := s.fileDAO.ListByParentID(folderID) if err != nil { - logger.Error(fmt.Sprintf("Failed to list children for folder %s", folderID), err) + common.Error(fmt.Sprintf("Failed to list children for folder %s", folderID), err) return err } - logger.Info("Deleting folder contents", zap.String("folder_id", folderID), zap.Int("child_count", len(children))) + common.Info("Deleting folder contents", zap.String("folder_id", folderID), zap.Int("child_count", len(children))) // Collect file IDs (non-folder) and recurse into subfolders var fileIDs []string for _, child := range children { if child.Type == "folder" { - logger.Debug("Recursively deleting child folder", zap.String("folder_id", child.ID), zap.String("folder_name", child.Name)) + common.Debug("Recursively deleting child folder", zap.String("folder_id", child.ID), zap.String("folder_name", child.Name)) if err := s.deleteFolderRecursive(child.ID); err != nil { - logger.Warn("Failed to delete child folder", zap.String("folder_id", child.ID), zap.Error(err)) + common.Warn("Failed to delete child folder", zap.String("folder_id", child.ID), zap.Error(err)) } } else { // Collect non-folder files for batch deletion - logger.Debug("Collecting file for deletion", zap.String("file_id", child.ID), zap.String("file_name", child.Name)) + common.Debug("Collecting file for deletion", zap.String("file_id", child.ID), zap.String("file_name", child.Name)) fileIDs = append(fileIDs, child.ID) } } // Delete all non-folder files in batch if len(fileIDs) > 0 { - logger.Info("Deleting files in folder", zap.String("folder_id", folderID), zap.Int("file_count", len(fileIDs))) + common.Info("Deleting files in folder", zap.String("folder_id", folderID), zap.Int("file_count", len(fileIDs))) if _, err := s.fileDAO.DeleteByIDs(fileIDs); err != nil { - logger.Warn("Failed to delete files in folder", zap.String("folder_id", folderID), zap.Strings("file_ids", fileIDs), zap.Error(err)) + common.Warn("Failed to delete files in folder", zap.String("folder_id", folderID), zap.Strings("file_ids", fileIDs), zap.Error(err)) // Continue to delete folder even if file deletion fails } } // Delete the folder itself - logger.Info("Deleting folder", zap.String("folder_id", folderID)) + common.Info("Deleting folder", zap.String("folder_id", folderID)) _, err = s.fileDAO.DeleteByIDs([]string{folderID}) if err != nil { - logger.Error(fmt.Sprintf("Failed to delete folder %s", folderID), err) + common.Error(fmt.Sprintf("Failed to delete folder %s", folderID), err) } return err } diff --git a/internal/service/tag.go b/internal/service/tag.go index edb6a88e24..7c86d45fa8 100644 --- a/internal/service/tag.go +++ b/internal/service/tag.go @@ -20,6 +20,7 @@ import ( "context" "encoding/json" "fmt" + "ragflow/internal/common" "sort" "strings" "time" @@ -30,7 +31,6 @@ import ( "ragflow/internal/dao" "ragflow/internal/engine/types" "ragflow/internal/entity" - "ragflow/internal/logger" "ragflow/internal/service/nlp" "github.com/cespare/xxhash/v2" @@ -63,7 +63,7 @@ func GetTagsFromCache(kbIDs []string) (map[string]float64, error) { redisClient := cache.Get() if redisClient == nil { - logger.Warn("Redis client not available, skipping cache lookup") + common.Warn("Redis client not available, skipping cache lookup") return nil, nil } @@ -76,7 +76,7 @@ func GetTagsFromCache(kbIDs []string) (map[string]float64, error) { var tags map[string]float64 if err := json.Unmarshal([]byte(data), &tags); err != nil { - logger.Warn("Failed to unmarshal cached tags", zap.Error(err)) + common.Warn("Failed to unmarshal cached tags", zap.Error(err)) return nil, nil } @@ -91,7 +91,7 @@ func SetTagsToCache(kbIDs []string, tags map[string]float64) error { redisClient := cache.Get() if redisClient == nil { - logger.Warn("Redis client not available, skipping cache store") + common.Warn("Redis client not available, skipping cache store") return nil } @@ -104,7 +104,7 @@ func SetTagsToCache(kbIDs []string, tags map[string]float64) error { // Cache for 10 minutes (600 seconds) ok := redisClient.Set(key, string(data), 10*time.Minute) if !ok { - logger.Warn("Failed to set tags cache") + common.Warn("Failed to set tags cache") return fmt.Errorf("failed to set tags cache") } @@ -176,12 +176,12 @@ func (s *MetadataService) TagQuery(question string, tenantIDs []string, kbIDs [] queryBuilder := nlp.GetQueryBuilder() matchTextExpr, warns := queryBuilder.Question(question, "qa", 0.0) // min_match=0.0 if len(warns) > 0 { - logger.Warn("TagQuery: failed to build match text", zap.Any("warnings", warns)) + common.Warn("TagQuery: failed to build match text", zap.Any("warnings", warns)) return make(map[string]float64), nil } matchText := matchTextExpr.MatchingText - logger.Debug("TagQuery match_text", zap.String("match_text", matchText)) + common.Debug("TagQuery match_text", zap.String("match_text", matchText)) // Search with match text to get relevant docs searchReq := &types.SearchRequest{ @@ -284,23 +284,23 @@ func (s *MetadataService) LabelQuestion(question string, kbs []*Knowledgebase) m return nil } - logger.Debug("tag_kb_ids found in parser_config", zap.Strings("tag_kb_ids", tagKBIDs)) + common.Debug("tag_kb_ids found in parser_config", zap.Strings("tag_kb_ids", tagKBIDs)) // Get all tags from cache or compute and cache allTags, err := GetTagsFromCache(tagKBIDs) if err != nil { - logger.Warn("Failed to get tags from cache", zap.Error(err)) + common.Warn("Failed to get tags from cache", zap.Error(err)) } if allTags == nil { // Cache miss - compute all_tags_in_portion allTags, err = s.GetAllTagsInPortion(lastKB.TenantID, tagKBIDs) if err != nil { - logger.Warn("Failed to get all tags in portion", zap.Error(err)) + common.Warn("Failed to get all tags in portion", zap.Error(err)) return nil } // Store in cache for future lookups if err := SetTagsToCache(tagKBIDs, allTags); err != nil { - logger.Warn("Failed to set tags cache", zap.Error(err)) + common.Warn("Failed to set tags cache", zap.Error(err)) } } diff --git a/internal/storage/minio.go b/internal/storage/minio.go index 11a9b6f9cf..abdb28504a 100644 --- a/internal/storage/minio.go +++ b/internal/storage/minio.go @@ -22,7 +22,7 @@ import ( "crypto/tls" "fmt" "net/http" - "ragflow/internal/logger" + "ragflow/internal/common" "ragflow/internal/server" "time" @@ -83,7 +83,7 @@ func (m *MinioStorage) connect() error { func (m *MinioStorage) reconnect() { if err := m.connect(); err != nil { - logger.Fatal(fmt.Sprintf("Failed to reconnect to MinIO, %s", err.Error())) + common.Fatal(fmt.Sprintf("Failed to reconnect to MinIO, %s", err.Error())) } } @@ -115,7 +115,7 @@ func (m *MinioStorage) Health() bool { } if err != nil { - logger.Warn("Failed to check MinIO health", zap.Error(err)) + common.Warn("Failed to check MinIO health", zap.Error(err)) return false } @@ -136,14 +136,14 @@ func (m *MinioStorage) Put(bucket, fnm string, binary []byte, tenantID ...string if m.bucket == "" { exists, err = m.client.BucketExists(ctx, bucket) if err != nil { - logger.Warn("Failed to check bucket existence", zap.String("bucket", bucket), zap.Error(err)) + common.Warn("Failed to check bucket existence", zap.String("bucket", bucket), zap.Error(err)) m.reconnect() time.Sleep(time.Second) continue } if !exists { if err = m.client.MakeBucket(ctx, bucket, minio.MakeBucketOptions{}); err != nil { - logger.Warn("Failed to create bucket", zap.String("bucket", bucket), zap.Error(err)) + common.Warn("Failed to create bucket", zap.String("bucket", bucket), zap.Error(err)) m.reconnect() time.Sleep(time.Second) continue @@ -154,7 +154,7 @@ func (m *MinioStorage) Put(bucket, fnm string, binary []byte, tenantID ...string reader := bytes.NewReader(binary) _, err = m.client.PutObject(ctx, bucket, fnm, reader, int64(len(binary)), minio.PutObjectOptions{}) if err != nil { - logger.Warn("Failed to put object", zap.String("bucket", bucket), zap.String("key", fnm), zap.Error(err)) + common.Warn("Failed to put object", zap.String("bucket", bucket), zap.String("key", fnm), zap.Error(err)) m.reconnect() time.Sleep(time.Second) continue @@ -175,7 +175,7 @@ func (m *MinioStorage) Get(bucket, fnm string, tenantID ...string) ([]byte, erro for i := 0; i < 2; i++ { obj, err := m.client.GetObject(ctx, bucket, fnm, minio.GetObjectOptions{}) if err != nil { - logger.Warn("Failed to get object", zap.String("bucket", bucket), zap.String("key", fnm), zap.Error(err)) + common.Warn("Failed to get object", zap.String("bucket", bucket), zap.String("key", fnm), zap.Error(err)) m.reconnect() time.Sleep(time.Second) continue @@ -184,7 +184,7 @@ func (m *MinioStorage) Get(bucket, fnm string, tenantID ...string) ([]byte, erro buf := new(bytes.Buffer) if _, err := buf.ReadFrom(obj); err != nil { - logger.Warn("Failed to read object data", zap.String("bucket", bucket), zap.String("key", fnm), zap.Error(err)) + common.Warn("Failed to read object data", zap.String("bucket", bucket), zap.String("key", fnm), zap.Error(err)) m.reconnect() time.Sleep(time.Second) continue @@ -203,7 +203,7 @@ func (m *MinioStorage) Remove(bucket, fnm string, tenantID ...string) error { ctx := context.Background() if err := m.client.RemoveObject(ctx, bucket, fnm, minio.RemoveObjectOptions{}); err != nil { - logger.Warn("Failed to remove object", zap.String("bucket", bucket), zap.String("key", fnm), zap.Error(err)) + common.Warn("Failed to remove object", zap.String("bucket", bucket), zap.String("key", fnm), zap.Error(err)) return err } @@ -227,7 +227,7 @@ func (m *MinioStorage) ObjExist(bucket, fnm string, tenantID ...string) bool { if errResponse.Code == "NoSuchKey" || errResponse.Code == "NoSuchBucket" { return false } - logger.Warn("Failed to stat object", zap.String("bucket", bucket), zap.String("key", fnm), zap.Error(err)) + common.Warn("Failed to stat object", zap.String("bucket", bucket), zap.String("key", fnm), zap.Error(err)) return false } @@ -243,7 +243,7 @@ func (m *MinioStorage) GetPresignedURL(bucket, fnm string, expires time.Duration for i := 0; i < 10; i++ { url, err := m.client.PresignedGetObject(ctx, bucket, fnm, expires, nil) if err != nil { - logger.Warn("Failed to get presigned URL", zap.String("bucket", bucket), zap.String("key", fnm), zap.Error(err)) + common.Warn("Failed to get presigned URL", zap.String("bucket", bucket), zap.String("key", fnm), zap.Error(err)) m.reconnect() time.Sleep(time.Second) continue @@ -266,7 +266,7 @@ func (m *MinioStorage) BucketExists(bucket string) bool { exists, err := m.client.BucketExists(ctx, actualBucket) if err != nil { - logger.Warn("Failed to check bucket existence", zap.String("bucket", actualBucket), zap.Error(err)) + common.Warn("Failed to check bucket existence", zap.String("bucket", actualBucket), zap.Error(err)) return false } @@ -303,7 +303,7 @@ func (m *MinioStorage) RemoveBucket(bucket string) error { Recursive: true, }) { if obj.Err != nil { - logger.Warn("Failed to list objects", zap.Error(obj.Err)) + common.Warn("Failed to list objects", zap.Error(obj.Err)) return } objectsCh <- obj @@ -311,13 +311,13 @@ func (m *MinioStorage) RemoveBucket(bucket string) error { }() for err := range m.client.RemoveObjects(ctx, actualBucket, objectsCh, minio.RemoveObjectsOptions{}) { - logger.Warn(fmt.Sprintf("Failed to remove object, key: %s", err.ObjectName), zap.Error(err.Err)) + common.Warn(fmt.Sprintf("Failed to remove object, key: %s", err.ObjectName), zap.Error(err.Err)) } // Only remove the actual bucket if not in single-bucket mode if m.bucket == "" { if err := m.client.RemoveBucket(ctx, actualBucket); err != nil { - logger.Warn("Failed to remove bucket", zap.String("bucket", actualBucket), zap.Error(err)) + common.Warn("Failed to remove bucket", zap.String("bucket", actualBucket), zap.Error(err)) return err } } @@ -336,12 +336,12 @@ func (m *MinioStorage) Copy(srcBucket, srcPath, destBucket, destPath string) boo if m.bucket == "" { exists, err := m.client.BucketExists(ctx, destBucket) if err != nil { - logger.Warn("Failed to check bucket existence", zap.String("bucket", destBucket), zap.Error(err)) + common.Warn("Failed to check bucket existence", zap.String("bucket", destBucket), zap.Error(err)) return false } if !exists { if err = m.client.MakeBucket(ctx, destBucket, minio.MakeBucketOptions{}); err != nil { - logger.Warn("Failed to create bucket", zap.String("bucket", destBucket), zap.Error(err)) + common.Warn("Failed to create bucket", zap.String("bucket", destBucket), zap.Error(err)) return false } } @@ -350,7 +350,7 @@ func (m *MinioStorage) Copy(srcBucket, srcPath, destBucket, destPath string) boo // Check if source object exists _, err := m.client.StatObject(ctx, srcBucket, srcPath, minio.StatObjectOptions{}) if err != nil { - logger.Warn("Failed to stat source object", zap.String("bucket", srcBucket), zap.String("key", srcPath), zap.Error(err)) + common.Warn("Failed to stat source object", zap.String("bucket", srcBucket), zap.String("key", srcPath), zap.Error(err)) return false } @@ -366,7 +366,7 @@ func (m *MinioStorage) Copy(srcBucket, srcPath, destBucket, destPath string) boo _, err = m.client.CopyObject(ctx, destOpts, srcOpts) if err != nil { - logger.Warn("Failed to copy object", zap.String("src", fmt.Sprintf("%s/%s", srcBucket, srcPath)), zap.String("dest", fmt.Sprintf("%s/%s", destBucket, destPath)), zap.Error(err)) + common.Warn("Failed to copy object", zap.String("src", fmt.Sprintf("%s/%s", srcBucket, srcPath)), zap.String("dest", fmt.Sprintf("%s/%s", destBucket, destPath)), zap.Error(err)) return false } @@ -377,7 +377,7 @@ func (m *MinioStorage) Copy(srcBucket, srcPath, destBucket, destPath string) boo func (m *MinioStorage) Move(srcBucket, srcPath, destBucket, destPath string) bool { if m.Copy(srcBucket, srcPath, destBucket, destPath) { if err := m.Remove(srcBucket, srcPath); err != nil { - logger.Warn("Failed to remove source object after copy", zap.String("bucket", srcBucket), zap.String("key", srcPath), zap.Error(err)) + common.Warn("Failed to remove source object after copy", zap.String("bucket", srcBucket), zap.String("key", srcPath), zap.Error(err)) return false } return true diff --git a/internal/storage/storage_factory.go b/internal/storage/storage_factory.go index 3ee45606df..147d003524 100644 --- a/internal/storage/storage_factory.go +++ b/internal/storage/storage_factory.go @@ -18,7 +18,7 @@ package storage import ( "fmt" - "ragflow/internal/logger" + "ragflow/internal/common" "ragflow/internal/server" "sync" ) @@ -55,7 +55,7 @@ func InitStorageFactory() error { return err } - logger.Info(fmt.Sprintf("Storage initialized: %s", factory.config.Type)) + common.Info(fmt.Sprintf("Storage initialized: %s", factory.config.Type)) return nil } diff --git a/internal/tokenizer/tokenizer.go b/internal/tokenizer/tokenizer.go index 8355f7b2e0..724a9fdeb4 100644 --- a/internal/tokenizer/tokenizer.go +++ b/internal/tokenizer/tokenizer.go @@ -19,6 +19,7 @@ package tokenizer import ( "context" "fmt" + "ragflow/internal/common" "ragflow/internal/engine" "runtime" "sync" @@ -28,7 +29,6 @@ import ( "go.uber.org/zap" rag "ragflow/internal/binding" - "ragflow/internal/logger" ) // PoolConfig configures the elastic analyzer pool @@ -98,7 +98,7 @@ func Init(cfg *PoolConfig) error { cfg.AcquireTimeout = 10 * time.Second } - logger.Info("Initializing analyzer pool", + common.Info("Initializing analyzer pool", zap.String("dict_path", cfg.DictPath), zap.Int("min_size", cfg.MinSize), zap.Int("max_size", cfg.MaxSize), @@ -115,13 +115,13 @@ func Init(cfg *PoolConfig) error { baseAnalyzer, err := rag.NewAnalyzer(cfg.DictPath) if err != nil { poolInitError = fmt.Errorf("failed to create base analyzer: %w", err) - logger.Error("Failed to create base analyzer", poolInitError) + common.Error("Failed to create base analyzer", poolInitError) return } if err = baseAnalyzer.Load(); err != nil { poolInitError = fmt.Errorf("failed to load base analyzer: %w", err) - logger.Error("Failed to load base analyzer", poolInitError) + common.Error("Failed to load base analyzer", poolInitError) baseAnalyzer.Close() return } @@ -133,7 +133,7 @@ func Init(cfg *PoolConfig) error { instance, err := globalPool.createInstance() if err != nil { poolInitError = fmt.Errorf("failed to create instance %d: %w", i, err) - logger.Error("Failed to create pool instance", poolInitError) + common.Error("Failed to create pool instance", poolInitError) globalPool.Close() return } @@ -142,7 +142,7 @@ func Init(cfg *PoolConfig) error { } globalPool.initialized = true - logger.Info("Analyzer pool initialized successfully", + common.Info("Analyzer pool initialized successfully", zap.Int("pre_warmed", cfg.MinSize), zap.Int32("current_size", atomic.LoadInt32(&globalPool.currentSize))) @@ -198,7 +198,7 @@ func (p *analyzerPool) acquire() (*poolInstance, error) { atomic.AddInt32(&p.currentSize, -1) return nil, fmt.Errorf("failed to dynamically create instance: %w", err) } - logger.Info("Pool expanded dynamically", + common.Info("Pool expanded dynamically", zap.Int32("previous_size", current), zap.Int32("new_size", current+1), zap.Int("max_size", p.config.MaxSize)) @@ -237,7 +237,7 @@ func (p *analyzerPool) release(instance *poolInstance) { // Successfully returned to pool default: // Pool is full (shouldn't happen normally), close this instance - logger.Warn("Pool full when releasing instance, destroying it", + common.Warn("Pool full when releasing instance, destroying it", zap.Int32("current_size", atomic.LoadInt32(&p.currentSize))) instance.analyzer.Close() atomic.AddInt32(&p.currentSize, -1) @@ -308,7 +308,7 @@ func (p *analyzerPool) shrink() { } newSize := atomic.AddInt32(&p.currentSize, -int32(len(toRemove))) - logger.Info("Pool shrunk", + common.Info("Pool shrunk", zap.Int("removed_instances", len(toRemove)), zap.Int32("previous_size", currentSize), zap.Int32("new_size", newSize), @@ -348,7 +348,7 @@ func (p *analyzerPool) Close() { p.baseAnalyzer = nil } - logger.Info(fmt.Sprintf("Analyzer pool closed, final_size: %d", atomic.LoadInt32(&p.currentSize))) + common.Info(fmt.Sprintf("Analyzer pool closed, final_size: %d", atomic.LoadInt32(&p.currentSize))) } // GetPoolStats returns current pool statistics @@ -440,7 +440,7 @@ func Analyze(text string) ([]rag.Token, error) { func SetFineGrained(fineGrained bool) { // In pool mode, we don't set global state on instances // Each request gets a fresh instance with default settings - logger.Debug("SetFineGrained is no-op in pool mode", zap.Bool("fine_grained", fineGrained)) + common.Debug("SetFineGrained is no-op in pool mode", zap.Bool("fine_grained", fineGrained)) } // FineGrainedTokenize performs fine-grained tokenization on space-separated tokens @@ -460,7 +460,7 @@ func FineGrainedTokenize(tokens string) (string, error) { // SetEnablePosition sets whether to enable position tracking // Note: This is a no-op in pool mode as each request uses its own instance func SetEnablePosition(enablePosition bool) { - logger.Debug("SetEnablePosition is no-op in pool mode", zap.Bool("enable_position", enablePosition)) + common.Debug("SetEnablePosition is no-op in pool mode", zap.Bool("enable_position", enablePosition)) } // IsInitialized checks whether the tokenizer pool has been initialized diff --git a/internal/tokenizer/tokenizer_concurrent_test.go b/internal/tokenizer/tokenizer_concurrent_test.go index 319a693324..ea4b793cee 100644 --- a/internal/tokenizer/tokenizer_concurrent_test.go +++ b/internal/tokenizer/tokenizer_concurrent_test.go @@ -18,6 +18,7 @@ package tokenizer import ( "fmt" + "ragflow/internal/common" "runtime" "sync" "sync/atomic" @@ -25,13 +26,11 @@ import ( "time" "go.uber.org/zap" - - "ragflow/internal/logger" ) func init() { // Initialize logger for tests - if err := logger.Init("info"); err != nil { + if err := common.Init("info"); err != nil { fmt.Printf("Failed to initialize logger: %v\n", err) } } @@ -482,7 +481,7 @@ func ExampleGetPoolStats() { // logPoolStats logs pool statistics using the zap logger func logPoolStats(msg string) { stats := GetPoolStats() - logger.Info(msg, + common.Info(msg, zap.Bool("initialized", stats["initialized"].(bool)), zap.Int32("current_size", stats["current_size"].(int32)), zap.Int("min_size", stats["min_size"].(int)), diff --git a/internal/utility/scheduled_task.go b/internal/utility/scheduled_task.go index 88c9886d17..c085f27baf 100644 --- a/internal/utility/scheduled_task.go +++ b/internal/utility/scheduled_task.go @@ -19,7 +19,7 @@ package utility import ( "encoding/json" "fmt" - "ragflow/internal/logger" + "ragflow/internal/common" "sync/atomic" "time" @@ -51,7 +51,7 @@ func StatusMessageSending() { // Serialize to JSON jsonData, err := json.Marshal(statusMessage) if err != nil { - logger.Error("Failed to marshal status message", err) + common.Error("Failed to marshal status message", err) return } @@ -66,13 +66,13 @@ func StatusMessageSending() { // Send POST request resp, err := client.PostJSON("/v1/admin/status", jsonData) if err != nil { - logger.Error("Error sending status message", err) + common.Error("Error sending status message", err) return } defer resp.Body.Close() if resp.StatusCode < 200 || resp.StatusCode >= 300 { - logger.Error("Failed to send status message", fmt.Errorf("status: %d", resp.StatusCode)) + common.Error("Failed to send status message", fmt.Errorf("status: %d", resp.StatusCode)) } } @@ -107,14 +107,14 @@ func (t *ScheduledTask) Start() { ticker := time.NewTicker(t.Interval) defer ticker.Stop() - logger.Info("Task started", zap.String("name", t.Name)) + common.Info("Task started", zap.String("name", t.Name)) for { select { case <-ticker.C: t.runSafely() case <-t.stop: - logger.Info("Task stopped", zap.String("name", t.Name)) + common.Info("Task stopped", zap.String("name", t.Name)) return } } @@ -125,7 +125,7 @@ func (t *ScheduledTask) Start() { func (t *ScheduledTask) runSafely() { // Attempt to set the flag if !atomic.CompareAndSwapInt32(&t.executing, 0, 1) { - logger.Warn("Task skipped - previous execution still running", zap.String("name", t.Name)) + common.Warn("Task skipped - previous execution still running", zap.String("name", t.Name)) return } @@ -134,7 +134,7 @@ func (t *ScheduledTask) runSafely() { defer func() { if r := recover(); r != nil { - logger.Fatal("Task panicked", zap.String("name", t.Name), zap.Any("recover", r)) + common.Fatal("Task panicked", zap.String("name", t.Name), zap.Any("recover", r)) } }() diff --git a/pyproject.toml b/pyproject.toml index 8dc40295cc..c4d5c7c426 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -12,7 +12,7 @@ dependencies = [ "anthropic==0.34.1", "arxiv==2.1.3", "atlassian-python-api==4.0.7", - "azure-identity==1.25.3", + "azure-identity>=1.25.3", "azure-storage-file-datalake==12.16.0", "beartype>=0.20.0,<1.0.0", "bio==1.7.1", @@ -75,7 +75,7 @@ dependencies = [ "opensearch-py==2.7.1", "ormsgpack==1.5.0", "pdfplumber==0.10.4", - "pluginlib==0.10.0", + "pluginlib>=0.10.0", "psycopg2-binary>=2.9.11,<3.0.0", "pyclipper>=1.4.0,<2.0.0", # "pywencai>=0.13.1,<1.0.0", # Temporarily disabled: conflicts with agentrun-sdk (pydash>=8), needed for agent/tools/wencai.py diff --git a/web/src/pages/user-setting/data-source/constant/index.tsx b/web/src/pages/user-setting/data-source/constant/index.tsx index 803d8ca9b8..327bbc826c 100644 --- a/web/src/pages/user-setting/data-source/constant/index.tsx +++ b/web/src/pages/user-setting/data-source/constant/index.tsx @@ -106,7 +106,7 @@ export const DataSourceFeatureVisibilityMap: Partial< syncDeletedFiles: true, }, [DataSourceKey.WEBDAV]: { - syncDeletedFiles: true, + syncDeletedFiles: true, }, [DataSourceKey.ZENDESK]: { syncDeletedFiles: true,