From 36ae39bc60b1893f3b38b553e9d692f817faac0a Mon Sep 17 00:00:00 2001 From: Jin Hai Date: Fri, 31 Jul 2026 17:18:45 +0800 Subject: [PATCH] Go: refactor config (#17544) Signed-off-by: Jin Hai --- cmd/ragflow_server.go | 102 +- conf/service_conf.yaml | 3 +- docker/service_conf.yaml.template | 3 +- internal/admin/service.go | 55 +- internal/common/logger.go | 7 - internal/dao/database.go | 18 +- internal/engine/elasticsearch/chunk.go | 54 +- .../elasticsearch/chunk_helpers_test.go | 16 +- internal/engine/elasticsearch/client.go | 43 +- internal/engine/elasticsearch/common.go | 4 +- internal/engine/elasticsearch/document.go | 8 +- internal/engine/elasticsearch/metadata.go | 20 +- internal/engine/elasticsearch/sql.go | 4 +- internal/engine/elasticsearch/sql_test.go | 22 +- internal/engine/global.go | 35 +- internal/engine/infinity/chunk.go | 32 +- internal/engine/infinity/client.go | 43 +- internal/engine/infinity/common.go | 8 +- internal/engine/infinity/document.go | 12 +- internal/engine/infinity/metadata.go | 22 +- internal/engine/infinity/sql.go | 2 +- internal/engine/redis/redis.go | 24 +- internal/handler/system.go | 2 +- .../task/pipeline_real_integration_test.go | 3 +- .../chunk/{chenk_type.go => chunk_type.go} | 0 internal/server/config.go | 1066 ++++++++--------- internal/server/config/admin_config.go | 58 + .../server/config/analytic_engine_config.go | 84 ++ internal/server/config/api_server_config.go | 95 ++ internal/server/config/base.go | 46 + internal/server/config/billing_ee.go | 26 + internal/server/config/cache_engine_config.go | 105 ++ internal/server/config/database_config.go | 113 ++ internal/server/config/doc_engine_config.go | 124 ++ internal/server/config/environments.go | 167 +++ internal/server/config/general_config.go | 122 ++ internal/server/config/ingestor_config.go | 42 + internal/server/config/log_config.go | 103 ++ internal/server/config/model_config_ee.go | 70 ++ internal/server/config/oauth_ee.go | 26 + .../server/config/open_telemetry_config.go | 72 ++ internal/server/config/queue_engine_config.go | 74 ++ internal/server/config/smtp_config.go | 86 ++ .../server/config/storage_engine_config.go | 297 +++++ internal/server/config/syncer_config.go | 52 + internal/server/variable.go | 4 +- internal/service/admin_client.go | 9 +- internal/service/memory.go | 2 +- internal/service/nlp/retrieval.go | 4 +- internal/service/oauth_login.go | 37 +- internal/service/system.go | 11 +- internal/service/user.go | 85 +- internal/storage/gcs.go | 6 +- internal/storage/minio.go | 6 +- internal/storage/oss.go | 14 +- internal/storage/s3.go | 16 +- internal/storage/storage_factory.go | 140 +-- internal/tokenizer/tokenizer.go | 18 +- .../tokenizer/tokenizer_concurrent_test.go | 2 +- internal/tokenizer/tokenizer_test.go | 24 +- 60 files changed, 2708 insertions(+), 1040 deletions(-) rename internal/parser/chunk/{chenk_type.go => chunk_type.go} (100%) create mode 100644 internal/server/config/admin_config.go create mode 100644 internal/server/config/analytic_engine_config.go create mode 100644 internal/server/config/api_server_config.go create mode 100644 internal/server/config/base.go create mode 100644 internal/server/config/billing_ee.go create mode 100644 internal/server/config/cache_engine_config.go create mode 100644 internal/server/config/database_config.go create mode 100644 internal/server/config/doc_engine_config.go create mode 100644 internal/server/config/environments.go create mode 100644 internal/server/config/general_config.go create mode 100644 internal/server/config/ingestor_config.go create mode 100644 internal/server/config/log_config.go create mode 100644 internal/server/config/model_config_ee.go create mode 100644 internal/server/config/oauth_ee.go create mode 100644 internal/server/config/open_telemetry_config.go create mode 100644 internal/server/config/queue_engine_config.go create mode 100644 internal/server/config/smtp_config.go create mode 100644 internal/server/config/storage_engine_config.go create mode 100644 internal/server/config/syncer_config.go diff --git a/cmd/ragflow_server.go b/cmd/ragflow_server.go index 897b0d6ce3..1594dd5748 100644 --- a/cmd/ragflow_server.go +++ b/cmd/ragflow_server.go @@ -271,24 +271,26 @@ func main() { os.Exit(1) } - config := server.GetConfig() + globalConfig := server.GetConfig() // override default port if provided switch *arguments.mode { case "api": - port := config.APIServer.Port + apiServerConfig := globalConfig.GetAPIServerConfig() + port := apiServerConfig.HTTPPort if arguments.port != nil { port = *arguments.port - config.APIServer.Port = port + apiServerConfig.HTTPPort = port } if arguments.name == nil { serverName = fmt.Sprintf("api_server_%d", port) } case "admin": - port := config.Admin.Port + adminServerConfig := globalConfig.GetAdminServerConfig() + port := adminServerConfig.HTTPPort if arguments.port != nil { port = *arguments.port - config.Admin.Port = port + adminServerConfig.HTTPPort = port } if arguments.name == nil { serverName = fmt.Sprintf("admin_server_%d", port) @@ -313,8 +315,10 @@ func main() { server.SetServerName(serverName) logFile = fmt.Sprintf("%s.log", serverName) + logConfig := globalConfig.GetLogConfig() + // Reinitialize logger with configured level if different - logLevel = config.Log.Level + logLevel = logConfig.Level if logLevel == "" { logLevel = "info" } @@ -323,17 +327,17 @@ func main() { logLevel = "debug" } - config.Log.Level = logLevel + globalConfig.SetLogLevel(logLevel) fileOut := common.FileOutput{ Path: logFile, - MaxSize: config.Log.MaxSize, - MaxBackups: config.Log.MaxBackups, - MaxAge: config.Log.MaxAge, - Compress: common.ResolveCompress(config.Log.Compress), + MaxSize: logConfig.MaxSize, + MaxBackups: logConfig.MaxBackups, + MaxAge: logConfig.MaxAge, + Compress: logConfig.Compress, } - if config.Log.Path != "" { - fileOut.Path = config.Log.Path + if logConfig.Path != "" { + fileOut.Path = logConfig.Path } common.SyncLog() @@ -344,7 +348,7 @@ func main() { server.SetLogger(common.Logger) // Print all configuration settings - common.Info(fmt.Sprintf("Starting %s server: %s, mode: %s", *arguments.mode, serverName, config.General.Mode)) + common.Info(fmt.Sprintf("Starting %s server: %s, mode: %s", *arguments.mode, serverName, globalConfig.GetMode())) server.PrintAll() // Initialize database @@ -353,13 +357,13 @@ func main() { } // Initialize doc engine - if err = engine.Init(&config.DocEngine); err != nil { + if err = engine.Init(); err != nil { common.Fatal("Failed to initialize doc engine", zap.Error(err)) } defer engine.Close() // Initialize Redis cache - if err = redis.Init(&config.Redis); err != nil { + if err = redis.Init(); err != nil { common.Fatal("Failed to initialize Redis", zap.Error(err)) } defer redis.Close() @@ -369,7 +373,7 @@ func main() { } defer storage.CloseStorage() - if err = engine.InitMessageQueueEngine(config.Ingestor.MQType); err != nil { + if err = engine.InitMessageQueueEngine(); err != nil { common.Error("Failed to initialize message queue engine", err) } @@ -391,22 +395,22 @@ func main() { switch *arguments.mode { case "api": - if err = runAPI(ctx, arguments, config); err != nil { + if err = runAPI(ctx, arguments); err != nil { fmt.Printf("Failed to start API server: %v\n", err) os.Exit(1) } case "admin": - if err = runAdmin(ctx, arguments, config); err != nil { + if err = runAdmin(ctx, arguments); err != nil { fmt.Printf("Failed to start ADMIN server: %v\n", err) os.Exit(1) } case "ingestor": - if err = runIngestor(ctx, cancel, arguments, config); err != nil { + if err = runIngestor(ctx, cancel, arguments); err != nil { fmt.Printf("Failed to start INGESTION worker: %v\n", err) os.Exit(1) } case "syncer": - if err = runSyncer(ctx, cancel, arguments, config); err != nil { + if err = runSyncer(ctx, cancel, arguments); err != nil { fmt.Printf("Failed to start SYNCER: %v\n", err) os.Exit(1) } @@ -416,10 +420,13 @@ func main() { } } -func runAdmin(ctx context.Context, args *serverArgs, config *server.Config) error { +func runAdmin(ctx context.Context, args *serverArgs) error { + + globalConfig := server.GetConfig() + serverMode := globalConfig.GetMode() // Set Gin mode - if config.General.Mode == "debug" { + if serverMode == "debug" { gin.SetMode(gin.DebugMode) } else { gin.SetMode(gin.ReleaseMode) @@ -454,7 +461,8 @@ func runAdmin(ctx context.Context, args *serverArgs, config *server.Config) erro // Setup routes r.Setup(ginEngine) - addr := fmt.Sprintf(":%d", config.Admin.Port) + adminConfig := globalConfig.GetAdminServerConfig() + addr := fmt.Sprintf(":%d", adminConfig.HTTPPort) srv := &http.Server{ Addr: addr, Handler: ginEngine, @@ -473,7 +481,7 @@ func runAdmin(ctx context.Context, args *serverArgs, config *server.Config) erro // Start HTTP server in a goroutine go func() { - common.Info(fmt.Sprintf("Starting RAGFlow admin HTTP server on port: %d", config.Admin.Port)) + common.Info(fmt.Sprintf("Starting RAGFlow admin HTTP server on port: %d", adminConfig.HTTPPort)) if err := srv.ListenAndServe(); err != nil && !errors.Is(err, http.ErrServerClosed) { common.Fatal("Failed to start server", zap.Error(err)) } @@ -501,7 +509,7 @@ func runAdmin(ctx context.Context, args *serverArgs, config *server.Config) erro // startHeartbeat initializes and starts the heartbeat reporter to the admin server. // It is shared by API, ingestion, and syncer server modes. // The caller must defer the returned *utility.ScheduledTask's Stop() method. -func startHeartbeat(serverType common.ServerType, serverID string, port int, config *server.Config) *utility.ScheduledTask { +func startHeartbeat(serverType common.ServerType, serverID string, port int, heartBeatInterval time.Duration) *utility.ScheduledTask { localIP, err := utility.GetLocalIP() if err != nil { common.Fatal("fail to get local ip address") @@ -519,7 +527,7 @@ func startHeartbeat(serverType common.ServerType, serverID string, port int, con return nil } - heartbeatReporter := utility.NewScheduledTask("Heartbeat reporter", config.General.HeartbeatInterval*time.Second, func() { + heartbeatReporter := utility.NewScheduledTask("Heartbeat reporter", heartBeatInterval*time.Second, func() { if err = service.AdminServiceClient.SendHeartbeat(); err == nil { local.SetAdminStatus(0, "") } else { @@ -530,7 +538,7 @@ func startHeartbeat(serverType common.ServerType, serverID string, port int, con return heartbeatReporter } -func runIngestor(ctx context.Context, cancel context.CancelFunc, args *serverArgs, config *server.Config) error { +func runIngestor(ctx context.Context, cancel context.CancelFunc, args *serverArgs) error { // Initialize tokenizer (rag_analyzer) // tokenizer.Init handles DictPath fallback: env var → /usr/share/infinity/resource if err := tokenizer.Init(&tokenizer.PoolConfig{}); err != nil { @@ -546,11 +554,11 @@ func runIngestor(ctx context.Context, cancel context.CancelFunc, args *serverArg // provisioning error is logged by the Ingestor and the pipeline still // writes available_int=0 compiled chunks; they just won't be merged until // the consumer is available. - cfg := server.GetConfig() + globalConfig := server.GetConfig() ingestor := ingestion.NewIngestor(*args.name, 2, []string{"pdf", "docx", "txt"}) ingestor.SetKnowledgeCompileModelConfig( - cfg.UserDefaultLLM.DefaultModels.ChatModel.Name, - cfg.UserDefaultLLM.DefaultModels.EmbeddingModel.Name, + globalConfig.GetDefaultChatModel().Name, + globalConfig.GetDefaultEmbeddingModel().Name, ) // Start returns immediately (it launches the owned consume/compile @@ -582,7 +590,7 @@ func runIngestor(ctx context.Context, cancel context.CancelFunc, args *serverArg common.ServerTypeIngestion, fmt.Sprintf("ingestor-%s", ingestor.ID()), -1, - config, + globalConfig.GetHeartbeatInterval(), ); hb != nil { defer hb.Stop() } @@ -608,8 +616,10 @@ func runIngestor(ctx context.Context, cancel context.CancelFunc, args *serverArg return nil } -func runSyncer(ctx context.Context, cancel context.CancelFunc, args *serverArgs, config *server.Config) error { - fileSyncer := syncer.NewSyncer(config.FileSyncer.MaxConcurrentSyncs, time.Duration(config.FileSyncer.SyncInterval)*time.Second) +func runSyncer(ctx context.Context, cancel context.CancelFunc, args *serverArgs) error { + globalConfig := server.GetConfig() + syncerConfig := globalConfig.GetSyncerConfig() + fileSyncer := syncer.NewSyncer(syncerConfig.MaxConcurrentSyncs, time.Duration(syncerConfig.SyncInterval)*time.Second) go func() { err := fileSyncer.Start() @@ -634,7 +644,7 @@ func runSyncer(ctx context.Context, cancel context.CancelFunc, args *serverArgs, common.ServerTypeFileSyncer, fmt.Sprintf("syncer-%s", fileSyncer.ID()), -1, - config, + globalConfig.GetHeartbeatInterval(), ); hb != nil { defer hb.Stop() } @@ -655,7 +665,7 @@ func runSyncer(ctx context.Context, cancel context.CancelFunc, args *serverArgs, return nil } -func runAPI(ctx context.Context, args *serverArgs, config *server.Config) error { +func runAPI(ctx context.Context, args *serverArgs) error { // Initialize admin status (default: unavailable=1) local.InitAdminStatus(1, "admin server not connected") @@ -674,17 +684,19 @@ func runAPI(ctx context.Context, args *serverArgs, config *server.Config) error common.Fatal("Failed to initialize query builder", zap.Error(err)) } - startServer(ctx, config) + startServer(ctx) common.Info("Server exited") return nil } -func startServer(ctx context.Context, config *server.Config) { +func startServer(ctx context.Context) { + globalConfig := server.GetConfig() + serverMode := globalConfig.GetMode() // Set Gin mode - if config.General.Mode == "debug" { + if serverMode == "debug" { gin.SetMode(gin.DebugMode) } else { gin.SetMode(gin.ReleaseMode) @@ -872,8 +884,10 @@ func startServer(ctx context.Context, config *server.Config) { channels.Start(ctx) + apiServerConfig := globalConfig.GetAPIServerConfig() + // Create HTTP server with timeouts to prevent slow clients from blocking shutdown - addr := fmt.Sprintf(":%d", config.APIServer.Port) + addr := fmt.Sprintf(":%d", apiServerConfig.HTTPPort) srv := &http.Server{ Addr: addr, Handler: ginEngine, @@ -893,7 +907,7 @@ func startServer(ctx context.Context, config *server.Config) { " /_/ |_|/_/ |_|\\____//_/ /_/ \\____/ |__/|__/\n", ) common.Info(fmt.Sprintf("RAGFlow Go Version: %s", common.GetRAGFlowVersion())) - common.Info(fmt.Sprintf("Server starting on port: %d", config.APIServer.Port)) + common.Info(fmt.Sprintf("Server starting on port: %d", apiServerConfig.HTTPPort)) if err := srv.ListenAndServe(); err != nil && !errors.Is(err, http.ErrServerClosed) { common.Fatal("Failed to start server", zap.Error(err)) } @@ -902,9 +916,9 @@ func startServer(ctx context.Context, config *server.Config) { // Start heartbeat reporter to admin server if hb := startHeartbeat( common.ServerTypeAPI, - fmt.Sprintf("ragflow-server-%d", config.APIServer.Port), - config.APIServer.Port, - config, + fmt.Sprintf("ragflow-server-%d", apiServerConfig.HTTPPort), + apiServerConfig.HTTPPort, + globalConfig.GetHeartbeatInterval(), ); hb != nil { defer hb.Stop() } diff --git a/conf/service_conf.yaml b/conf/service_conf.yaml index 81a6d95d6c..e275e31aca 100644 --- a/conf/service_conf.yaml +++ b/conf/service_conf.yaml @@ -68,8 +68,9 @@ otel: enable: false ingestor: mq_type: 'nats' + max_concurrent_workers: 1 file_syncer: - max_concurrent_syncs: 4 + max_concurrent_syncs: 1 sync_interval: 3 user_default_llm: default_models: diff --git a/docker/service_conf.yaml.template b/docker/service_conf.yaml.template index dd9f39c8ef..06ca10dcc4 100644 --- a/docker/service_conf.yaml.template +++ b/docker/service_conf.yaml.template @@ -80,8 +80,9 @@ otel: enable: false ingestor: mq_type: 'nats' + max_concurrent_workers: 1 file_syncer: - max_concurrent_syncs: 4 + max_concurrent_syncs: 1 sync_interval: 3 user_default_llm: default_models: diff --git a/internal/admin/service.go b/internal/admin/service.go index 03d5953a9f..389b828c49 100644 --- a/internal/admin/service.go +++ b/internal/admin/service.go @@ -34,6 +34,7 @@ import ( "ragflow/internal/entity" modelModule "ragflow/internal/entity/models" "ragflow/internal/server" + "ragflow/internal/server/config" servicepkg "ragflow/internal/service" "ragflow/internal/utility" "regexp" @@ -241,14 +242,20 @@ func (s *Service) CreateUser(ctx context.Context, username, password, role strin asrModel := "" vlmModel := "" rerankModel := "" + var ttsModel *string = nil + var ocrModel *string = nil parserIDs := "naive:General,qa:Q&A,resume:Resume,manual:Manual,table:Table,paper:Paper,book:Book,laws:Laws,presentation:Presentation,picture:Picture,one:One,audio:Audio,email:Email" if cfg != nil { - chatModel = cfg.UserDefaultLLM.DefaultModels.ChatModel.Name - embeddingModel = cfg.UserDefaultLLM.DefaultModels.EmbeddingModel.Name - asrModel = cfg.UserDefaultLLM.DefaultModels.ASRModel.Name - vlmModel = cfg.UserDefaultLLM.DefaultModels.Image2TextModel.Name - rerankModel = cfg.UserDefaultLLM.DefaultModels.RerankModel.Name + chatModel = cfg.GetDefaultChatModel().Name + embeddingModel = cfg.GetDefaultEmbeddingModel().Name + asrModel = cfg.GetDefaultASRModel().Name + vlmModel = cfg.GetDefaultVisionModel().Name + rerankModel = cfg.GetDefaultRerankModel().Name + ttsModelStr := cfg.GetDefaultTTSModel().Name + ttsModel = &ttsModelStr + ocrModelStr := cfg.GetDefaultOCRModel().Name + ocrModel = &ocrModelStr } tenantStatus := "1" @@ -260,6 +267,8 @@ func (s *Service) CreateUser(ctx context.Context, username, password, role strin ASRID: asrModel, Img2TxtID: vlmModel, RerankID: rerankModel, + TTSID: ttsModel, + OCRID: ocrModel, ParserIDs: parserIDs, Credit: 512, Status: &tenantStatus, @@ -342,17 +351,19 @@ func (s *Service) getInitTenantLLM(ctx context.Context, userID string) ([]*entit var tenantLLMs []*entity.TenantLLM // Get model configs from configuration - modelConfigs := []server.ModelConfig{ - cfg.UserDefaultLLM.DefaultModels.ChatModel, - cfg.UserDefaultLLM.DefaultModels.EmbeddingModel, - cfg.UserDefaultLLM.DefaultModels.RerankModel, - cfg.UserDefaultLLM.DefaultModels.ASRModel, - cfg.UserDefaultLLM.DefaultModels.Image2TextModel, + modelConfigs := []config.ModelConfig{ + cfg.GetDefaultChatModel(), + cfg.GetDefaultEmbeddingModel(), + cfg.GetDefaultRerankModel(), + cfg.GetDefaultASRModel(), + cfg.GetDefaultVisionModel(), + cfg.GetDefaultTTSModel(), + cfg.GetDefaultOCRModel(), } // Track seen factories to avoid duplicates seenFactories := make(map[string]bool) - var uniqueFactories []server.ModelConfig + var uniqueFactories []config.ModelConfig for _, mc := range modelConfigs { if mc.Factory == "" { @@ -380,8 +391,8 @@ func (s *Service) getInitTenantLLM(ctx context.Context, userID string) ([]*entit apiKey = factoryConfig.APIKey apiBase = factoryConfig.BaseURL case entity.ModelTypeEmbedding.String(): - apiKey = cfg.UserDefaultLLM.DefaultModels.EmbeddingModel.APIKey - apiBase = cfg.UserDefaultLLM.DefaultModels.EmbeddingModel.BaseURL + apiKey = cfg.GetDefaultEmbeddingModel().APIKey + apiBase = cfg.GetDefaultEmbeddingModel().BaseURL if apiKey == "" { apiKey = factoryConfig.APIKey } @@ -389,8 +400,8 @@ func (s *Service) getInitTenantLLM(ctx context.Context, userID string) ([]*entit apiBase = factoryConfig.BaseURL } case entity.ModelTypeRerank.String(): - apiKey = cfg.UserDefaultLLM.DefaultModels.RerankModel.APIKey - apiBase = cfg.UserDefaultLLM.DefaultModels.RerankModel.BaseURL + apiKey = cfg.GetDefaultRerankModel().APIKey + apiBase = cfg.GetDefaultRerankModel().BaseURL if apiKey == "" { apiKey = factoryConfig.APIKey } @@ -398,8 +409,8 @@ func (s *Service) getInitTenantLLM(ctx context.Context, userID string) ([]*entit apiBase = factoryConfig.BaseURL } case entity.ModelTypeSpeech2Text.String(): - apiKey = cfg.UserDefaultLLM.DefaultModels.ASRModel.APIKey - apiBase = cfg.UserDefaultLLM.DefaultModels.ASRModel.BaseURL + apiKey = cfg.GetDefaultASRModel().APIKey + apiBase = cfg.GetDefaultASRModel().BaseURL if apiKey == "" { apiKey = factoryConfig.APIKey } @@ -407,8 +418,8 @@ func (s *Service) getInitTenantLLM(ctx context.Context, userID string) ([]*entit apiBase = factoryConfig.BaseURL } case entity.ModelTypeImage2Text.String(): - apiKey = cfg.UserDefaultLLM.DefaultModels.Image2TextModel.APIKey - apiBase = cfg.UserDefaultLLM.DefaultModels.Image2TextModel.BaseURL + apiKey = cfg.GetDefaultVisionModel().APIKey + apiBase = cfg.GetDefaultVisionModel().BaseURL if apiKey == "" { apiKey = factoryConfig.APIKey } @@ -1135,7 +1146,7 @@ func (s *Service) getESClusterStats(name string) (map[string]interface{}, error) // Get ES config from server config cfg := server.GetConfig() - if cfg == nil || cfg.DocEngine.ES == nil { + if cfg == nil || !cfg.IsElasticConfigured() { return map[string]interface{}{ "service_name": name, "status": "timeout", @@ -1144,7 +1155,7 @@ func (s *Service) getESClusterStats(name string) (map[string]interface{}, error) } // Create ES engine and get cluster stats - esEngine, err := elasticsearch.NewEngine(cfg.DocEngine.ES) + esEngine, err := elasticsearch.NewEngine(cfg.GetElasticsearchConfig()) if err != nil { return map[string]interface{}{ "service_name": name, diff --git a/internal/common/logger.go b/internal/common/logger.go index 61bfbf01b5..cfef59f40d 100644 --- a/internal/common/logger.go +++ b/internal/common/logger.go @@ -234,13 +234,6 @@ func SetLogLevel(level string) error { return nil } -func ResolveCompress(c *bool) bool { - if c == nil { - return true - } - return *c -} - // GinLogger returns a gin middleware that emits one log line per request // through Logger. Level is chosen by status: // diff --git a/internal/dao/database.go b/internal/dao/database.go index d11de3ce93..6dfc8b1c3e 100644 --- a/internal/dao/database.go +++ b/internal/dao/database.go @@ -67,21 +67,21 @@ type LLMFactoriesFile struct { // InitDB initialize database connection func InitDB(ctx context.Context, migrateDB bool) error { - cfg := server.GetConfig() - dbCfg := cfg.Database + globalConfig := server.GetConfig() + databaseConfig := globalConfig.GetMySQLConfig() dsn := fmt.Sprintf("%s:%s@tcp(%s:%d)/%s?charset=%s&parseTime=True&loc=Local", - dbCfg.Username, - dbCfg.Password, - dbCfg.Host, - dbCfg.Port, - dbCfg.Database, - dbCfg.Charset, + databaseConfig.User, + databaseConfig.Password, + databaseConfig.Host, + databaseConfig.Port, + databaseConfig.DatabaseName, + databaseConfig.Charset, ) // Set log level var gormLogLevel gormLogger.LogLevel - if cfg.General.Mode == "debug" { + if globalConfig.GetMode() == "debug" { gormLogLevel = gormLogger.Info } else { gormLogLevel = gormLogger.Silent diff --git a/internal/engine/elasticsearch/chunk.go b/internal/engine/elasticsearch/chunk.go index 8a330bfecc..119da87611 100644 --- a/internal/engine/elasticsearch/chunk.go +++ b/internal/engine/elasticsearch/chunk.go @@ -57,7 +57,7 @@ var ( ) // CreateChunkStore creates an index -func (e *elasticsearchEngine) CreateChunkStore(ctx context.Context, baseName, datasetID string, vectorSize int, parserID string) error { +func (e *Engine) CreateChunkStore(ctx context.Context, baseName, datasetID string, vectorSize int, parserID string) error { if baseName == "" { return fmt.Errorf("index name cannot be empty") } @@ -159,7 +159,7 @@ func (e *elasticsearchEngine) CreateChunkStore(ctx context.Context, baseName, da // InsertChunks inserts chunks into a chunk index // If a chunk with the same id + doc_id + kb_id already exists, it will be updated with the new value -func (e *elasticsearchEngine) InsertChunks(ctx context.Context, chunks []map[string]interface{}, baseName string, datasetID string) ([]string, error) { +func (e *Engine) InsertChunks(ctx context.Context, chunks []map[string]interface{}, baseName string, datasetID string) ([]string, error) { common.Info("ElasticsearchConnection.InsertChunks called", zap.String("index_name", baseName), zap.Int("chunkCount", len(chunks))) if len(chunks) == 0 { @@ -279,7 +279,7 @@ func (e *elasticsearchEngine) InsertChunks(ctx context.Context, chunks []map[str } // UpdateChunks updates chunks by condition -func (e *elasticsearchEngine) UpdateChunks(ctx context.Context, condition map[string]interface{}, newValue map[string]interface{}, baseName string, datasetID string) error { +func (e *Engine) UpdateChunks(ctx context.Context, condition map[string]interface{}, newValue map[string]interface{}, baseName string, datasetID string) error { fullIndexName := baseName common.Info("ElasticsearchConnection.UpdateChunks called", zap.String("index_name", fullIndexName), zap.Any("condition", condition), zap.Any("new_value", newValue)) @@ -319,7 +319,7 @@ func (e *elasticsearchEngine) UpdateChunks(ctx context.Context, condition map[st // AdjustChunkPagerank atomically adjusts pagerank_fea and clamps it to // [minWeight, maxWeight]. -func (e *elasticsearchEngine) AdjustChunkPagerank(ctx context.Context, indexName, chunkID, kbID string, delta, minWeight, maxWeight float64) error { +func (e *Engine) AdjustChunkPagerank(ctx context.Context, indexName, chunkID, kbID string, delta, minWeight, maxWeight float64) error { if indexName == "" { return fmt.Errorf("index name cannot be empty") } @@ -404,7 +404,7 @@ func (e *elasticsearchEngine) AdjustChunkPagerank(ctx context.Context, indexName return nil } -func (e *elasticsearchEngine) updateSingleMemoryMessage(ctx context.Context, indexName, messageDocID string, newValue map[string]interface{}) error { +func (e *Engine) updateSingleMemoryMessage(ctx context.Context, indexName, messageDocID string, newValue map[string]interface{}) error { doc := mapMemoryMessageESUpdateFields(newValue) delete(doc, "id") if len(doc) == 0 { @@ -464,7 +464,7 @@ func mapMemoryMessageESConditionFields(condition map[string]interface{}) map[str } // updateSingleChunk handles single document update -func (e *elasticsearchEngine) updateSingleChunk(ctx context.Context, indexName, chunkID string, newValue map[string]interface{}) error { +func (e *Engine) updateSingleChunk(ctx context.Context, indexName, chunkID string, newValue map[string]interface{}) error { common.Debug("ElasticsearchConnection.updateSingleChunk called", zap.String("indexName", indexName), zap.String("chunkID", chunkID)) // First find the document by id field to get the actual _id @@ -633,7 +633,7 @@ func (e *elasticsearchEngine) updateSingleChunk(ctx context.Context, indexName, } // updateChunksByQuery handles multi-document update -func (e *elasticsearchEngine) updateChunksByQuery(ctx context.Context, indexName string, condition map[string]interface{}, newValue map[string]interface{}) error { +func (e *Engine) updateChunksByQuery(ctx context.Context, indexName string, condition map[string]interface{}, newValue map[string]interface{}) error { common.Debug("ElasticsearchConnection.updateChunksByQuery called", zap.String("indexName", indexName)) // Build bool query from condition @@ -783,7 +783,7 @@ func copyFields(m map[string]interface{}) map[string]interface{} { } // DeleteChunks deletes chunks from a dataset index by condition -func (e *elasticsearchEngine) DeleteChunks(ctx context.Context, condition map[string]interface{}, indexName string, datasetID string) (int64, error) { +func (e *Engine) DeleteChunks(ctx context.Context, condition map[string]interface{}, indexName string, datasetID string) (int64, error) { // For ES, index name is just indexName (e.g., "ragflow_{tenantID}"), not indexName_datasetID fullIndexName := indexName common.Info("Deleting chunks from Elasticsearch index", zap.String("index_name", fullIndexName), zap.Any("condition", condition)) @@ -964,7 +964,7 @@ type SearchResponse struct { } // Search executes search with unified types.SearchRequest -func (e *elasticsearchEngine) Search(ctx context.Context, req *types.SearchRequest) (*types.SearchResult, error) { +func (e *Engine) Search(ctx context.Context, req *types.SearchRequest) (*types.SearchResult, error) { types.LogSearchRequest("Elasticsearch", req) // Validate inputs and set defaults @@ -1314,7 +1314,7 @@ type searchAfterFetcher func( // gets an accurate total; subsequent requests skip it for efficiency. // Returns the (possibly empty) collected hits and the total hit count // from the first response. -func (e *elasticsearchEngine) searchAfterCursor( +func (e *Engine) searchAfterCursor( ctx context.Context, req *types.SearchRequest, baseQuery map[string]interface{}, @@ -1331,7 +1331,7 @@ func (e *elasticsearchEngine) searchAfterCursor( // buildSearchAfterFetcher returns a fetcher that delegates each // iteration to executeSearchRequest, which talks to the real ES client. -func (e *elasticsearchEngine) buildSearchAfterFetcher(req *types.SearchRequest) searchAfterFetcher { +func (e *Engine) buildSearchAfterFetcher(req *types.SearchRequest) searchAfterFetcher { return func( ctx context.Context, baseQuery map[string]interface{}, @@ -1463,7 +1463,7 @@ func searchAfterPaginate( // batch size and search_after cursor. If trackTotalHits is true the // request asks ES to compute an exact total (cheap to omit on // pagination iterations after the first). -func (e *elasticsearchEngine) executeSearchRequest( +func (e *Engine) executeSearchRequest( ctx context.Context, req *types.SearchRequest, baseQuery map[string]interface{}, @@ -1923,7 +1923,7 @@ func buildRankFeatureQuery(rankFeature map[string]float64) []map[string]interfac } // GetChunk gets a chunk by ID using ES search API -func (e *elasticsearchEngine) GetChunk(ctx context.Context, baseName, chunkID string, datasetIDs []string) (interface{}, error) { +func (e *Engine) GetChunk(ctx context.Context, baseName, chunkID string, datasetIDs []string) (interface{}, error) { if strings.HasPrefix(baseName, "memory_") { return e.getMemoryMessage(ctx, baseName, chunkID) } @@ -1996,7 +1996,7 @@ func (e *elasticsearchEngine) GetChunk(ctx context.Context, baseName, chunkID st return nil, nil } -func (e *elasticsearchEngine) getMemoryMessage(ctx context.Context, indexName, docID string) (interface{}, error) { +func (e *Engine) getMemoryMessage(ctx context.Context, indexName, docID string) (interface{}, error) { req := esapi.GetRequest{ Index: indexName, DocumentID: docID, @@ -2037,7 +2037,7 @@ func (e *elasticsearchEngine) getMemoryMessage(ctx context.Context, indexName, d // The original requested field names ARE the database column names: // - "content_with_weight" is stored and returned as "content_with_weight" // - No field name mapping is needed in GetFields -func (e *elasticsearchEngine) GetFields(chunks []map[string]interface{}, fields []string) map[string]map[string]interface{} { +func (e *Engine) GetFields(chunks []map[string]interface{}, fields []string) map[string]map[string]interface{} { common.Info("GetFields called", zap.Int("chunkCount", len(chunks)), zap.Strings("fields", fields)) result := make(map[string]map[string]interface{}) @@ -2116,7 +2116,7 @@ func (e *elasticsearchEngine) GetFields(chunks []map[string]interface{}, fields // GetAggregation aggregates chunk values by field name // Input: [{"docnm_kwd": "docA"}, {"docnm_kwd": "docA"}, {"docnm_kwd": "docB"}] // Returns: [{"key": "docA", "count": 2}, {"key": "docB", "count": 1}] -func (e *elasticsearchEngine) GetAggregation(chunks []map[string]interface{}, fieldName string) []map[string]interface{} { +func (e *Engine) GetAggregation(chunks []map[string]interface{}, fieldName string) []map[string]interface{} { if len(chunks) == 0 || fieldName == "" { return []map[string]interface{}{} } @@ -2176,7 +2176,7 @@ func (e *elasticsearchEngine) GetAggregation(chunks []map[string]interface{}, fi // GetChunkIDs extracts chunk IDs from ES search response chunks. // Uses _id field (composite: {doc_id}_{kb_id}_{chunk_id}). -func (e *elasticsearchEngine) GetChunkIDs(chunks []map[string]interface{}) []string { +func (e *Engine) GetChunkIDs(chunks []map[string]interface{}) []string { ids := make([]string, 0, len(chunks)) for _, chunk := range chunks { if id, ok := elasticsearchChunkID(chunk); ok { @@ -2187,7 +2187,7 @@ func (e *elasticsearchEngine) GetChunkIDs(chunks []map[string]interface{}) []str } // GetHighlight returns highlighted text for matching keywords -func (e *elasticsearchEngine) GetHighlight(chunks []map[string]interface{}, keywords []string, fieldName string) map[string]string { +func (e *Engine) GetHighlight(chunks []map[string]interface{}, keywords []string, fieldName string) map[string]string { result := make(map[string]string) if len(chunks) == 0 || len(keywords) == 0 { return result @@ -2323,18 +2323,18 @@ func normalizeElasticsearchHighlightKeywords(keywords []string) []string { } // DropChunkStore deletes a chunk index -func (e *elasticsearchEngine) DropChunkStore(ctx context.Context, baseName, datasetID string) error { +func (e *Engine) DropChunkStore(ctx context.Context, baseName, datasetID string) error { return e.dropIndex(ctx, baseName) } // ChunkStoreExists checks if a chunk index exists -func (e *elasticsearchEngine) ChunkStoreExists(ctx context.Context, baseName, datasetID string) (bool, error) { +func (e *Engine) ChunkStoreExists(ctx context.Context, baseName, datasetID string) (bool, error) { return e.indexExists(ctx, baseName) } // KNNScores performs a second-pass KNN search to get clean cosine similarities for ES. // This keeps chunk vectors in the index and asks ES to compute the cosine similarity. -func (e *elasticsearchEngine) KNNScores(ctx context.Context, chunks []map[string]interface{}, queryVector []float64, topK int) (map[string]interface{}, error) { +func (e *Engine) KNNScores(ctx context.Context, chunks []map[string]interface{}, queryVector []float64, topK int) (map[string]interface{}, error) { if len(chunks) == 0 || len(queryVector) == 0 { return nil, nil } @@ -2424,7 +2424,7 @@ func (e *elasticsearchEngine) KNNScores(ctx context.Context, chunks []map[string } // GetScores extracts similarity scores from KNN search result -func (e *elasticsearchEngine) GetScores(knnResult map[string]interface{}) map[string]float64 { +func (e *Engine) GetScores(knnResult map[string]interface{}) map[string]float64 { scores := make(map[string]float64) hits, ok := knnResult["hits"].(map[string]interface{}) if !ok { @@ -2543,7 +2543,7 @@ func parseMemoryMessageVectorSize(field string) (int, bool) { return vectorSize, true } -func (e *elasticsearchEngine) memoryMessageVectorMappingExists(ctx context.Context, indexName, fieldName string) (bool, error) { +func (e *Engine) memoryMessageVectorMappingExists(ctx context.Context, indexName, fieldName string) (bool, error) { req := esapi.IndicesGetMappingRequest{ Index: []string{indexName}, } @@ -2586,7 +2586,7 @@ func (e *elasticsearchEngine) memoryMessageVectorMappingExists(ctx context.Conte return ok, nil } -func (e *elasticsearchEngine) ensureMemoryMessageVectorMapping(ctx context.Context, indexName string, vectorSize int) error { +func (e *Engine) ensureMemoryMessageVectorMapping(ctx context.Context, indexName string, vectorSize int) error { if vectorSize <= 0 { return fmt.Errorf("memory vector size must be positive, got %d", vectorSize) } @@ -2632,7 +2632,7 @@ func (e *elasticsearchEngine) ensureMemoryMessageVectorMapping(ctx context.Conte return nil } -func (e *elasticsearchEngine) ensureMemoryMessageVectorMappingsForDocs(ctx context.Context, indexName string, chunks []map[string]interface{}) error { +func (e *Engine) ensureMemoryMessageVectorMappingsForDocs(ctx context.Context, indexName string, chunks []map[string]interface{}) error { seen := map[int]struct{}{} for _, chunk := range chunks { for field := range chunk { @@ -2652,7 +2652,7 @@ func (e *elasticsearchEngine) ensureMemoryMessageVectorMappingsForDocs(ctx conte return nil } -func (e *elasticsearchEngine) ensureMemoryMessageSearchVectorMappings(ctx context.Context, indexNames []string, vectorFieldName string, fallbackVectorSize int) error { +func (e *Engine) ensureMemoryMessageSearchVectorMappings(ctx context.Context, indexNames []string, vectorFieldName string, fallbackVectorSize int) error { vectorSize, ok := parseMemoryMessageVectorSize(vectorFieldName) if !ok { vectorSize = fallbackVectorSize @@ -2672,7 +2672,7 @@ func (e *elasticsearchEngine) ensureMemoryMessageSearchVectorMappings(ctx contex if !exists { continue } - if err := e.ensureMemoryMessageVectorMapping(ctx, indexName, vectorSize); err != nil { + if err = e.ensureMemoryMessageVectorMapping(ctx, indexName, vectorSize); err != nil { return err } } diff --git a/internal/engine/elasticsearch/chunk_helpers_test.go b/internal/engine/elasticsearch/chunk_helpers_test.go index e77d5f151d..60afe15a1a 100644 --- a/internal/engine/elasticsearch/chunk_helpers_test.go +++ b/internal/engine/elasticsearch/chunk_helpers_test.go @@ -62,8 +62,8 @@ func TestUpdateSingleMemoryMessageWaitsForRefresh(t *testing.T) { t.Fatalf("new elasticsearch client: %v", err) } - engine := &elasticsearchEngine{client: client} - if err := engine.updateSingleMemoryMessage(context.Background(), "memory_tenant", "memory-1_42", map[string]interface{}{"forget_at": "2026-07-27 10:00:00"}); err != nil { + engine := &Engine{client: client} + if err = engine.updateSingleMemoryMessage(context.Background(), "memory_tenant", "memory-1_42", map[string]interface{}{"forget_at": "2026-07-27 10:00:00"}); err != nil { t.Fatalf("updateSingleMemoryMessage: %v", err) } if gotRefresh != "wait_for" { @@ -72,7 +72,7 @@ func TestUpdateSingleMemoryMessageWaitsForRefresh(t *testing.T) { } func TestElasticsearchGetFieldsFiltersAndUsesIDFallback(t *testing.T) { - engine := &elasticsearchEngine{} + engine := &Engine{} chunks := []map[string]interface{}{ { "_id": "fallback-chunk", @@ -100,7 +100,7 @@ func TestElasticsearchGetFieldsFiltersAndUsesIDFallback(t *testing.T) { } func TestElasticsearchGetFieldsEmptyAndSkippedIDs(t *testing.T) { - engine := &elasticsearchEngine{} + engine := &Engine{} if got := engine.GetFields(nil, nil); got == nil || len(got) != 0 { t.Fatalf("GetFields(nil)=%#v, want empty non-nil map", got) @@ -131,7 +131,7 @@ func TestElasticsearchGetFieldsEmptyAndSkippedIDs(t *testing.T) { } func TestElasticsearchGetAggregationSplitsCountsAndSorts(t *testing.T) { - engine := &elasticsearchEngine{} + engine := &Engine{} chunks := []map[string]interface{}{ {"tag_kwd": "red###blue###"}, {"tag_kwd": []interface{}{"blue", " green ", ""}}, @@ -161,7 +161,7 @@ func TestElasticsearchGetAggregationSplitsCountsAndSorts(t *testing.T) { } func TestElasticsearchGetChunkIDsPreservesOrderWithFallback(t *testing.T) { - engine := &elasticsearchEngine{} + engine := &Engine{} chunks := []map[string]interface{}{ {"id": "source-id", "_id": "hit-id"}, {"_id": "fallback-id"}, @@ -179,7 +179,7 @@ func TestElasticsearchGetChunkIDsPreservesOrderWithFallback(t *testing.T) { } func TestElasticsearchGetHighlightFallbackAndBoundaries(t *testing.T) { - engine := &elasticsearchEngine{} + engine := &Engine{} chunks := []map[string]interface{}{ { "_id": "fallback-id", @@ -228,7 +228,7 @@ func TestElasticsearchGetHighlightFallbackAndBoundaries(t *testing.T) { } func TestElasticsearchGetHighlightPreservesExistingAndNonEnglish(t *testing.T) { - engine := &elasticsearchEngine{} + engine := &Engine{} gotExisting := engine.GetHighlight([]map[string]interface{}{ {"id": "existing", "content_with_weight": "already marked text"}, diff --git a/internal/engine/elasticsearch/client.go b/internal/engine/elasticsearch/client.go index 01be8803bb..e63cb36b6f 100644 --- a/internal/engine/elasticsearch/client.go +++ b/internal/engine/elasticsearch/client.go @@ -25,7 +25,7 @@ import ( "io" "net/http" "os" - "ragflow/internal/server" + "ragflow/internal/server/config" "ragflow/internal/utility" "time" @@ -33,25 +33,14 @@ import ( "github.com/elastic/go-elasticsearch/v8/esapi" ) -// elasticsearchEngine is the Elasticsearch engine implementation -type elasticsearchEngine struct { +// Engine is the Elasticsearch engine implementation +type Engine struct { client *elasticsearch.Client - config *server.ElasticsearchConfig + config *config.ElasticsearchConfig } // NewEngine creates an Elasticsearch engine -func NewEngine(cfg interface{}) (*elasticsearchEngine, error) { - if cfg == nil { - return nil, fmt.Errorf("elasticsearch config is nil, please check your configuration file for 'doc_engine.es' settings") - } - esConfig, ok := cfg.(*server.ElasticsearchConfig) - if !ok { - return nil, fmt.Errorf("invalid Elasticsearch config type, expected *config.ElasticsearchConfig") - } - if esConfig == nil { - return nil, fmt.Errorf("elasticsearch config is nil, please check your configuration file for 'doc_engine.es' settings") - } - +func NewEngine(esConfig config.ElasticsearchConfig) (*Engine, error) { // Create ES client client, err := elasticsearch.NewClient(elasticsearch.Config{ Addresses: []string{esConfig.Hosts}, @@ -82,9 +71,9 @@ func NewEngine(cfg interface{}) (*elasticsearchEngine, error) { return nil, fmt.Errorf("Elasticsearch returned error: %s", res.Status()) } - engine := &elasticsearchEngine{ + engine := &Engine{ client: client, - config: esConfig, + config: &esConfig, } // Create two index templates for different index types @@ -101,17 +90,17 @@ func NewEngine(cfg interface{}) (*elasticsearchEngine, error) { } // GetType returns the engine type -func (e *elasticsearchEngine) GetType() string { +func (e *Engine) GetType() string { return "elasticsearch" } // SupportsPageRank returns true because Elasticsearch supports pagerank. -func (e *elasticsearchEngine) SupportsPageRank() bool { +func (e *Engine) SupportsPageRank() bool { return true } // Ping health check -func (e *elasticsearchEngine) Ping(ctx context.Context) error { +func (e *Engine) Ping(ctx context.Context) error { req := esapi.InfoRequest{} res, err := req.Do(ctx, e.client) if err != nil { @@ -125,14 +114,14 @@ func (e *elasticsearchEngine) Ping(ctx context.Context) error { } // Close closes the connection -func (e *elasticsearchEngine) Close() error { +func (e *Engine) Close() error { // Go-elasticsearch client doesn't have a Close method, connection is managed by the transport return nil } // CreateIndexTemplate creates an index template with the specified mapping // The template will be automatically applied to any new index matching the pattern -func (e *elasticsearchEngine) CreateIndexTemplate(ctx context.Context, templateName, indexPattern, mappingFileName string, priority ...int) error { +func (e *Engine) CreateIndexTemplate(ctx context.Context, templateName, indexPattern, mappingFileName string, priority ...int) error { if templateName == "" || indexPattern == "" { return fmt.Errorf("template name and index pattern cannot be empty") } @@ -212,7 +201,7 @@ func (e *elasticsearchEngine) CreateIndexTemplate(ctx context.Context, templateN // GetClusterStats gets Elasticsearch cluster statistics // Reference: curl -XGET "http://{es_host}/_cluster/stats" -H "kbn-xsrf: reporting" -func (e *elasticsearchEngine) GetClusterStats() (map[string]interface{}, error) { +func (e *Engine) GetClusterStats() (map[string]interface{}, error) { req := esapi.ClusterStatsRequest{} res, err := req.Do(context.Background(), e.client) if err != nil { @@ -323,7 +312,7 @@ func (e *elasticsearchEngine) GetClusterStats() (map[string]interface{}, error) return result, nil } -// convertBytes converts bytes to human readable format +// convertBytes converts bytes to human-readable format func convertBytes(bytes int64) string { const ( KB = 1024 @@ -389,7 +378,7 @@ func extractErrorReason(bodyBytes []byte) string { // GetIndexStats gets statistics for specified indices using the _cat/indices API // Returns index, health, status, docs.count, store.size, dataset.size for each index -func (e *elasticsearchEngine) GetIndexStats(indices []string) ([]map[string]interface{}, error) { +func (e *Engine) GetIndexStats(indices []string) ([]map[string]interface{}, error) { if len(indices) == 0 { return []map[string]interface{}{}, nil } @@ -415,7 +404,7 @@ func (e *elasticsearchEngine) GetIndexStats(indices []string) ([]map[string]inte } var results []map[string]interface{} - if err := json.NewDecoder(res.Body).Decode(&results); err != nil { + if err = json.NewDecoder(res.Body).Decode(&results); err != nil { return nil, fmt.Errorf("failed to decode index stats: %w", err) } diff --git a/internal/engine/elasticsearch/common.go b/internal/engine/elasticsearch/common.go index 63e5f44702..080e3f0732 100644 --- a/internal/engine/elasticsearch/common.go +++ b/internal/engine/elasticsearch/common.go @@ -25,7 +25,7 @@ import ( ) // dropIndex deletes an index -func (e *elasticsearchEngine) dropIndex(ctx context.Context, indexName string) error { +func (e *Engine) dropIndex(ctx context.Context, indexName string) error { if indexName == "" { return fmt.Errorf("index name cannot be empty") } @@ -65,7 +65,7 @@ func (e *elasticsearchEngine) dropIndex(ctx context.Context, indexName string) e } // indexExists checks if index exists -func (e *elasticsearchEngine) indexExists(ctx context.Context, indexName string) (bool, error) { +func (e *Engine) indexExists(ctx context.Context, indexName string) (bool, error) { if indexName == "" { return false, fmt.Errorf("index name cannot be empty") } diff --git a/internal/engine/elasticsearch/document.go b/internal/engine/elasticsearch/document.go index a79be0dd4c..b021881286 100644 --- a/internal/engine/elasticsearch/document.go +++ b/internal/engine/elasticsearch/document.go @@ -27,7 +27,7 @@ import ( ) // IndexDocument indexes a single document -func (e *elasticsearchEngine) IndexDocument(ctx context.Context, indexName, docID string, doc interface{}) error { +func (e *Engine) IndexDocument(ctx context.Context, indexName, docID string, doc interface{}) error { if indexName == "" { return fmt.Errorf("index name cannot be empty") } @@ -71,7 +71,7 @@ func (e *elasticsearchEngine) IndexDocument(ctx context.Context, indexName, docI } // BulkIndex indexes documents in bulk -func (e *elasticsearchEngine) BulkIndex(ctx context.Context, indexName string, docs []interface{}) (interface{}, error) { +func (e *Engine) BulkIndex(ctx context.Context, indexName string, docs []interface{}) (interface{}, error) { if indexName == "" { return nil, fmt.Errorf("index name cannot be empty") } @@ -174,7 +174,7 @@ type BulkResponse struct { } // GetDocument gets a document -func (e *elasticsearchEngine) GetDocument(ctx context.Context, indexName, docID string) (interface{}, error) { +func (e *Engine) GetDocument(ctx context.Context, indexName, docID string) (interface{}, error) { if indexName == "" { return nil, fmt.Errorf("index name cannot be empty") } @@ -221,7 +221,7 @@ func (e *elasticsearchEngine) GetDocument(ctx context.Context, indexName, docID } // DeleteDocument deletes a document -func (e *elasticsearchEngine) DeleteDocument(ctx context.Context, indexName, docID string) error { +func (e *Engine) DeleteDocument(ctx context.Context, indexName, docID string) error { if indexName == "" { return fmt.Errorf("index name cannot be empty") } diff --git a/internal/engine/elasticsearch/metadata.go b/internal/engine/elasticsearch/metadata.go index f997e9f5e2..b805ecadc4 100644 --- a/internal/engine/elasticsearch/metadata.go +++ b/internal/engine/elasticsearch/metadata.go @@ -37,7 +37,7 @@ import ( ) // CreateMetadataStore creates the document metadata index -func (e *elasticsearchEngine) CreateMetadataStore(ctx context.Context, tenantID string) error { +func (e *Engine) CreateMetadataStore(ctx context.Context, tenantID string) error { indexName := buildMetadataIndexName(tenantID) // Check if index already exists @@ -88,7 +88,7 @@ func (e *elasticsearchEngine) CreateMetadataStore(ctx context.Context, tenantID // InsertMetadata inserts documents into tenant's metadata index // If a document with the same id and kb_id already exists, it will be updated with the new value -func (e *elasticsearchEngine) InsertMetadata(ctx context.Context, metadata []map[string]interface{}, tenantID string) ([]string, error) { +func (e *Engine) InsertMetadata(ctx context.Context, metadata []map[string]interface{}, tenantID string) ([]string, error) { indexName := buildMetadataIndexName(tenantID) common.Info("ElasticsearchConnection.InsertMetadata called", zap.String("index_name", indexName), zap.String("tenant_id", tenantID), zap.Int("doc_count", len(metadata))) @@ -177,7 +177,7 @@ func (e *elasticsearchEngine) InsertMetadata(ctx context.Context, metadata []map // // The metadata index must already exist; the service layer is responsible // for creating it before writing. -func (e *elasticsearchEngine) UpdateMetadata(ctx context.Context, docID string, datasetID string, metaFields map[string]interface{}, tenantID string) error { +func (e *Engine) UpdateMetadata(ctx context.Context, docID string, datasetID string, metaFields map[string]interface{}, tenantID string) error { indexName := buildMetadataIndexName(tenantID) common.Info("ElasticsearchConnection.UpdateMetadata called", zap.String("index_name", indexName), zap.String("docID", docID), zap.String("datasetID", datasetID)) @@ -257,7 +257,7 @@ func (e *elasticsearchEngine) UpdateMetadata(ctx context.Context, docID string, // DeleteMetadata deletes metadata from tenant's metadata index by condition // The condition is a map used to build an ES query (e.g., map["kb_id"]="xxx") // Returns the number of deleted documents -func (e *elasticsearchEngine) DeleteMetadata(ctx context.Context, condition map[string]interface{}, tenantID string) (int64, error) { +func (e *Engine) DeleteMetadata(ctx context.Context, condition map[string]interface{}, tenantID string) (int64, error) { indexName := buildMetadataIndexName(tenantID) common.Info("ElasticsearchConnection.DeleteMetadata called", zap.String("index_name", indexName), zap.Any("condition", condition)) @@ -325,7 +325,7 @@ func (e *elasticsearchEngine) DeleteMetadata(ctx context.Context, condition map[ // DeleteMetadataKeys deletes specific metadata keys from a document's meta_fields. // If deleting those keys leaves no metadata entries, the metadata document is removed. -func (e *elasticsearchEngine) DeleteMetadataKeys(ctx context.Context, docID string, datasetID string, keys []string, tenantID string) error { +func (e *Engine) DeleteMetadataKeys(ctx context.Context, docID string, datasetID string, keys []string, tenantID string) error { indexName := buildMetadataIndexName(tenantID) common.Info("ElasticsearchConnection.DeleteMetadataKeys called", zap.String("index_name", indexName), zap.String("docID", docID), zap.Any("keys", keys)) @@ -520,19 +520,19 @@ func (e *elasticsearchEngine) DeleteMetadataKeys(ctx context.Context, docID stri } // DropMetadataStore drops a metadata index from Elasticsearch -func (e *elasticsearchEngine) DropMetadataStore(ctx context.Context, tenantID string) error { +func (e *Engine) DropMetadataStore(ctx context.Context, tenantID string) error { indexName := buildMetadataIndexName(tenantID) return e.dropIndex(ctx, indexName) } // MetadataStoreExists checks if a metadata index exists in Elasticsearch -func (e *elasticsearchEngine) MetadataStoreExists(ctx context.Context, tenantID string) (bool, error) { +func (e *Engine) MetadataStoreExists(ctx context.Context, tenantID string) (bool, error) { indexName := buildMetadataIndexName(tenantID) return e.indexExists(ctx, indexName) } // SearchMetadata executes search specifically for metadata indices (ragflow_doc_meta_*) -func (e *elasticsearchEngine) SearchMetadata(ctx context.Context, req *types.SearchMetadataRequest) (*types.SearchMetadataResult, error) { +func (e *Engine) SearchMetadata(ctx context.Context, req *types.SearchMetadataRequest) (*types.SearchMetadataResult, error) { tenantID := req.TenantID common.Debug("SearchMetadata in Elasticsearch started", zap.String("tenantID", tenantID)) @@ -631,7 +631,7 @@ func (e *elasticsearchEngine) SearchMetadata(ctx context.Context, req *types.Sea } // buildMetadataQueryFromCondition builds an ES query for metadata index -func (e *elasticsearchEngine) buildMetadataQueryFromCondition(condition map[string]interface{}) map[string]interface{} { +func (e *Engine) buildMetadataQueryFromCondition(condition map[string]interface{}) map[string]interface{} { if len(condition) == 0 { return nil } @@ -712,7 +712,7 @@ const metaPushdownMaxSize = 10000 // nil -> push-down was not viable / errored / result overflowed the // push-down cap (caller should fall back to in-memory) // []string{} -> push-down succeeded but found 0 matching docs (empty result is definitive) -func (e *elasticsearchEngine) FilterDocIdsByMetaPushdown(ctx context.Context, sqlDB *gorm.DB, kbIDs []string, conditions []map[string]interface{}, logic string) []string { +func (e *Engine) FilterDocIdsByMetaPushdown(ctx context.Context, sqlDB *gorm.DB, kbIDs []string, conditions []map[string]interface{}, logic string) []string { if len(conditions) == 0 || len(kbIDs) == 0 { return nil } diff --git a/internal/engine/elasticsearch/sql.go b/internal/engine/elasticsearch/sql.go index 65b74e11bb..1d869c8d3a 100644 --- a/internal/engine/elasticsearch/sql.go +++ b/internal/engine/elasticsearch/sql.go @@ -86,7 +86,7 @@ func Preprocess(sql string) string { // RunSQL posts SQL to `/_sql`, translates the response into chunk-shaped maps. // Returns (nil, nil) on empty rows; (nil, error) when retries exhausted. -func (e *elasticsearchEngine) RunSQL(ctx context.Context, tableName string, sqlText string, kbIDs []string, format string) ([]map[string]interface{}, error) { +func (e *Engine) RunSQL(ctx context.Context, tableName string, sqlText string, kbIDs []string, format string) ([]map[string]interface{}, error) { if e == nil || e.client == nil { return nil, fmt.Errorf("Elasticsearch RunSQL: client not initialized") } @@ -127,7 +127,7 @@ func (e *elasticsearchEngine) RunSQL(ctx context.Context, tableName string, sqlT return nil, fmt.Errorf("Elasticsearch RunSQL: timeout after %d attempts: %w", esSQLRetryAttempts, lastErr) } -func (e *elasticsearchEngine) runSQLOnce(ctx context.Context, sqlText string, format string) ([]map[string]interface{}, error) { +func (e *Engine) runSQLOnce(ctx context.Context, sqlText string, format string) ([]map[string]interface{}, error) { ctx, cancel := context.WithTimeout(ctx, esSQLRequestTimeout) defer cancel() diff --git a/internal/engine/elasticsearch/sql_test.go b/internal/engine/elasticsearch/sql_test.go index 6d486b8ad6..6a4221a0f3 100644 --- a/internal/engine/elasticsearch/sql_test.go +++ b/internal/engine/elasticsearch/sql_test.go @@ -49,26 +49,26 @@ type capturedRequest struct { // incoming request and replies with the given body / status. func newCapturingServer(t *testing.T, replyStatus int, replyBody string) (*httptest.Server, *capturedRequest) { t.Helper() - cap := &capturedRequest{} + capRequest := &capturedRequest{} srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { body, _ := io.ReadAll(r.Body) - cap.mu.Lock() - cap.method = r.Method - cap.path = r.URL.Path - cap.body = string(body) - cap.mu.Unlock() + capRequest.mu.Lock() + capRequest.method = r.Method + capRequest.path = r.URL.Path + capRequest.body = string(body) + capRequest.mu.Unlock() w.Header().Set("X-Elastic-Product", "Elasticsearch") w.WriteHeader(replyStatus) _, _ = w.Write([]byte(replyBody)) })) t.Cleanup(srv.Close) - return srv, cap + return srv, capRequest } -// newTestEngine constructs an elasticsearchEngine pointing at the given +// newTestEngine constructs an Engine pointing at the given // test server. Bypasses NewEngine (which calls ES Info to verify // connectivity) — the test server is a stub, not a real ES cluster. -func newTestEngine(t *testing.T, srvURL string) *elasticsearchEngine { +func newTestEngine(t *testing.T, srvURL string) *Engine { t.Helper() client, err := elasticsearch.NewClient(elasticsearch.Config{ Addresses: []string{srvURL}, @@ -76,7 +76,7 @@ func newTestEngine(t *testing.T, srvURL string) *elasticsearchEngine { if err != nil { t.Fatalf("elasticsearch.NewClient: %v", err) } - return &elasticsearchEngine{client: client} + return &Engine{client: client} } const sampleESResponse = `{ @@ -374,7 +374,7 @@ func TestRunSQL_PostsToSQLPath(t *testing.T) { // as-is. This lets the rewrite tests assert on the SHAPE of the MATCH() // substitution without depending on a real tokenizer pool. func TestMain(m *testing.M) { - tokenizer.RegisterEngineType(func() string { return "infinity" }) + tokenizer.SetEngineType("infinity") m.Run() } diff --git a/internal/engine/global.go b/internal/engine/global.go index 32a6f0f6b1..eff017bcf3 100644 --- a/internal/engine/global.go +++ b/internal/engine/global.go @@ -33,41 +33,40 @@ import ( var ( globalEngine DocEngine - engineType EngineType + engineType string messageQueueEngine MessageQueue once sync.Once ) // Init initializes document engine -func Init(cfg *server.DocEngineConfig) error { +func Init() error { + var initErr error once.Do(func() { - tokenizer.RegisterEngineType(func() string { - return string(GetEngineType()) - }) - - engineType = EngineType(cfg.Type) + globalConfig := server.GetConfig() + engineType = globalConfig.DocEngineType() + tokenizer.SetEngineType(engineType) var err error switch engineType { - case EngineElasticsearch: - globalEngine, err = elasticsearch.NewEngine(cfg.ES) - case EngineInfinity: - globalEngine, err = infinity.NewEngine(cfg.Infinity) + case "elasticsearch": + globalEngine, err = elasticsearch.NewEngine(globalConfig.GetElasticsearchConfig()) + case "infinity": + globalEngine, err = infinity.NewEngine(globalConfig.GetInfinityConfig()) default: - err = fmt.Errorf("unsupported doc engine type: %s", cfg.Type) + err = fmt.Errorf("unsupported doc engine type: %s", engineType) } if err != nil { initErr = fmt.Errorf("failed to create doc engine: %w", err) return } - common.Info("Doc engine initialized", zap.String("type", string(cfg.Type))) + common.Info("Doc engine initialized", zap.String("type", engineType)) }) return initErr } // GetEngineType returns the document engine type -func GetEngineType() EngineType { +func GetEngineType() string { return engineType } @@ -95,11 +94,13 @@ func SetMessageQueueEngine(mq MessageQueue) { messageQueueEngine = mq } -func InitMessageQueueEngine(messageQueueType string) error { - config := server.GetConfig() +func InitMessageQueueEngine() error { + globalConfig := server.GetConfig() + messageQueueType := globalConfig.QueueEngineType() switch messageQueueType { case "nats": - messageQueueEngine = nats.NewNatsEngine(config.Nats.Host, config.Nats.Port) + natsConfig := globalConfig.GetNATSConfig() + messageQueueEngine = nats.NewNatsEngine(natsConfig.Host, natsConfig.Port) err := messageQueueEngine.Init() if err != nil { return err diff --git a/internal/engine/infinity/chunk.go b/internal/engine/infinity/chunk.go index d7ba963334..bc977c52b1 100644 --- a/internal/engine/infinity/chunk.go +++ b/internal/engine/infinity/chunk.go @@ -50,7 +50,7 @@ var pagerankAdjustLocks [pagerankAdjustLockCount]sync.Mutex // baseName is the table name prefix (e.g., "ragflow_") // The full table name is built as "{baseName}_{datasetID}" // For skill index (datasetID="skill"), tableName is just baseName and uses skill_infinity_mapping.json -func (e *infinityEngine) CreateChunkStore(ctx context.Context, baseName, datasetID string, vectorSize int, parserID string) error { +func (e *Engine) CreateChunkStore(ctx context.Context, baseName, datasetID string, vectorSize int, parserID string) error { db, release, err := e.client.checkoutDatabase(ctx, "chunk.go") if err != nil { return fmt.Errorf("failed to get database: %w", err) @@ -60,7 +60,7 @@ func (e *infinityEngine) CreateChunkStore(ctx context.Context, baseName, dataset return e.createChunkStoreWithDB(db, baseName, datasetID, vectorSize, parserID) } -func (e *infinityEngine) createChunkStoreWithDB(db *infinity.Database, baseName, datasetID string, vectorSize int, parserID string) error { +func (e *Engine) createChunkStoreWithDB(db *infinity.Database, baseName, datasetID string, vectorSize int, parserID string) error { vecSize := vectorSize // Determine table name and mapping file based on index type @@ -268,7 +268,7 @@ func (e *infinityEngine) createChunkStoreWithDB(db *infinity.Database, baseName, // Table name format: {baseName}_{datasetID} // Auto-create the table if it doesn't exist // Delete existing rows with matching IDs before insert -func (e *infinityEngine) InsertChunks(ctx context.Context, chunks []map[string]interface{}, baseName string, datasetID string) ([]string, error) { +func (e *Engine) InsertChunks(ctx context.Context, chunks []map[string]interface{}, baseName string, datasetID string) ([]string, error) { tableName := buildChunkTableName(baseName, datasetID) common.Info("InfinityConnection.InsertChunks called", zap.String("tableName", tableName), zap.Int("chunkCount", len(chunks))) @@ -388,7 +388,7 @@ func (e *infinityEngine) InsertChunks(ctx context.Context, chunks []map[string]i // UpdateChunks updates chunks in a dataset table // Table name format: {baseName}_{datasetID} -func (e *infinityEngine) UpdateChunks(ctx context.Context, condition map[string]interface{}, newValue map[string]interface{}, baseName string, datasetID string) error { +func (e *Engine) UpdateChunks(ctx context.Context, condition map[string]interface{}, newValue map[string]interface{}, baseName string, datasetID string) error { tableName := buildChunkTableName(baseName, datasetID) common.Info("InfinityConnection.UpdateChunks called", zap.String("tableName", tableName), zap.Any("condition", condition)) @@ -542,7 +542,7 @@ func (e *infinityEngine) UpdateChunks(ctx context.Context, condition map[string] } // AdjustChunkPagerank adjusts pagerank_fea and clamps it to [minWeight, maxWeight]. -func (e *infinityEngine) AdjustChunkPagerank(ctx context.Context, baseName, chunkID, datasetID string, delta, minWeight, maxWeight float64) error { +func (e *Engine) AdjustChunkPagerank(ctx context.Context, baseName, chunkID, datasetID string, delta, minWeight, maxWeight float64) error { if baseName == "" { return fmt.Errorf("index name cannot be empty") } @@ -636,7 +636,7 @@ func (e *infinityEngine) AdjustChunkPagerank(ctx context.Context, baseName, chun // DeleteChunks deletes chunks from a dataset table // Table name format: {baseName}_{datasetID} // condition specifies which chunks to delete -func (e *infinityEngine) DeleteChunks(ctx context.Context, condition map[string]interface{}, baseName string, datasetID string) (int64, error) { +func (e *Engine) DeleteChunks(ctx context.Context, condition map[string]interface{}, baseName string, datasetID string) (int64, error) { tableName := buildChunkTableName(baseName, datasetID) db, release, err := e.client.checkoutDatabase(ctx, "chunk.go") @@ -696,7 +696,7 @@ func (e *infinityEngine) DeleteChunks(ctx context.Context, condition map[string] // Search searches the Infinity engine for matching chunks. // It supports three matching types: MatchTextExpr (full-text), MatchDenseExpr (vector), and FusionExpr (combined). // If no match expressions are provided, Search relies solely on filter (e.g., doc_id, available_int) to find results. -func (e *infinityEngine) Search(ctx context.Context, req *types.SearchRequest) (*types.SearchResult, error) { +func (e *Engine) Search(ctx context.Context, req *types.SearchRequest) (*types.SearchResult, error) { types.LogSearchRequest("Infinity", req) if len(req.IndexNames) == 0 { @@ -1212,7 +1212,7 @@ func (e *infinityEngine) Search(ctx context.Context, req *types.SearchRequest) ( } // GetChunk gets a chunk by ID -func (e *infinityEngine) GetChunk(ctx context.Context, tableName, chunkID string, datasetIDs []string) (interface{}, error) { +func (e *Engine) GetChunk(ctx context.Context, tableName, chunkID string, datasetIDs []string) (interface{}, error) { if e.client == nil || e.client.pool == nil { return nil, fmt.Errorf("Infinity client not initialized") } @@ -1478,7 +1478,7 @@ func memoryMessageStatusBool(value interface{}) bool { } // GetFields extracts the requested fields from Infinity search results -func (e *infinityEngine) GetFields(chunks []map[string]interface{}, fields []string) map[string]map[string]interface{} { +func (e *Engine) GetFields(chunks []map[string]interface{}, fields []string) map[string]map[string]interface{} { result := make(map[string]map[string]interface{}) // Python: if not fields, return {} @@ -1779,7 +1779,7 @@ func (e *infinityEngine) GetFields(chunks []map[string]interface{}, fields []str // // For tag_kwd field, splits values by "###" separator. // For other fields, uses comma separation. -func (e *infinityEngine) GetAggregation(chunks []map[string]interface{}, fieldName string) []map[string]interface{} { +func (e *Engine) GetAggregation(chunks []map[string]interface{}, fieldName string) []map[string]interface{} { if len(chunks) == 0 { return []map[string]interface{}{} } @@ -1875,7 +1875,7 @@ func (e *infinityEngine) GetAggregation(chunks []map[string]interface{}, fieldNa } // GetChunkIDs extracts chunk IDs from Infinity search results. -func (e *infinityEngine) GetChunkIDs(chunks []map[string]interface{}) []string { +func (e *Engine) GetChunkIDs(chunks []map[string]interface{}) []string { ids := make([]string, 0, len(chunks)) for _, chunk := range chunks { if id, ok := chunk["id"].(string); ok { @@ -1887,7 +1887,7 @@ func (e *infinityEngine) GetChunkIDs(chunks []map[string]interface{}) []string { // GetHighlight generates highlighted text snippets for search results. // Matches keywords in text and wraps them with tags. -func (e *infinityEngine) GetHighlight(chunks []map[string]interface{}, keywords []string, fieldName string) map[string]string { +func (e *Engine) GetHighlight(chunks []map[string]interface{}, keywords []string, fieldName string) map[string]string { result := make(map[string]string) if len(chunks) == 0 || len(keywords) == 0 { return result @@ -1901,7 +1901,7 @@ func (e *infinityEngine) GetHighlight(chunks []map[string]interface{}, keywords // KNNScores for Infinity - since Infinity normalizes scores during fusion, // we just need to return a result structure that GetScores can parse. // This matches Python's approach where Infinity doesn't use the two-pass KNN. -func (e *infinityEngine) KNNScores(ctx context.Context, chunks []map[string]interface{}, queryVector []float64, topK int) (map[string]interface{}, error) { +func (e *Engine) KNNScores(ctx context.Context, chunks []map[string]interface{}, queryVector []float64, topK int) (map[string]interface{}, error) { if len(chunks) == 0 { return nil, nil } @@ -1927,7 +1927,7 @@ func (e *infinityEngine) KNNScores(ctx context.Context, chunks []map[string]inte // GetScores extracts similarity scores from KNN search result. // For Infinity, it parses the result from KNNScores and extracts _score values. -func (e *infinityEngine) GetScores(knnResult map[string]interface{}) map[string]float64 { +func (e *Engine) GetScores(knnResult map[string]interface{}) map[string]float64 { scores := make(map[string]float64) hits, ok := knnResult["hits"].(map[string]interface{}) if !ok { @@ -2513,11 +2513,11 @@ func transformChunkFields(chunk map[string]interface{}, embeddingCols [][2]inter } // DropChunkStore drops a chunk table from Infinity -func (e *infinityEngine) DropChunkStore(ctx context.Context, baseName, datasetID string) error { +func (e *Engine) DropChunkStore(ctx context.Context, baseName, datasetID string) error { return e.dropTable(ctx, buildChunkTableName(baseName, datasetID)) } // ChunkStoreExists checks if a chunk table exists in Infinity -func (e *infinityEngine) ChunkStoreExists(ctx context.Context, baseName, datasetID string) (bool, error) { +func (e *Engine) ChunkStoreExists(ctx context.Context, baseName, datasetID string) (bool, error) { return e.tableExists(ctx, buildChunkTableName(baseName, datasetID)) } diff --git a/internal/engine/infinity/client.go b/internal/engine/infinity/client.go index 43a8196467..d6c3edf215 100644 --- a/internal/engine/infinity/client.go +++ b/internal/engine/infinity/client.go @@ -22,6 +22,7 @@ import ( "fmt" "os" "ragflow/internal/common" + "ragflow/internal/server/config" "reflect" "strconv" "strings" @@ -115,7 +116,7 @@ func ensureDeadline(ctx context.Context, timeout time.Duration) (context.Context return context.WithTimeout(ctx, timeout) } -func NewInfinityClient(cfg *server.InfinityConfig) (*infinityClient, error) { +func NewInfinityClient(cfg config.InfinityConfig) (*infinityClient, error) { // Parse URI like "localhost:23817" to get IP and port host := "127.0.0.1" port := 23817 @@ -331,54 +332,44 @@ func (c *infinityClient) checkoutDatabase(ctx context.Context, caller string) (* } // Engine Infinity engine implementation using Go SDK -type infinityEngine struct { - config *server.InfinityConfig +type Engine struct { + config config.InfinityConfig client *infinityClient mappingFileName string docMetaMappingFileName string } // NewEngine creates an Infinity engine -func NewEngine(cfg interface{}) (*infinityEngine, error) { - if cfg == nil { - return nil, fmt.Errorf("infinity config is nil, please check your configuration file for 'doc_engine.infinity' settings") - } - infConfig, ok := cfg.(*server.InfinityConfig) - if !ok { - return nil, fmt.Errorf("invalid infinity config type, expected *config.InfinityConfig") - } - if infConfig == nil { - return nil, fmt.Errorf("infinity config is nil, please check your configuration file for 'doc_engine.infinity' settings") - } +func NewEngine(infinityConfig config.InfinityConfig) (*Engine, error) { - client, err := NewInfinityClient(infConfig) + client, err := NewInfinityClient(infinityConfig) if err != nil { return nil, err } - mappingFileName := infConfig.MappingFileName + mappingFileName := infinityConfig.MappingFileName if mappingFileName == "" { mappingFileName = "infinity_mapping.json" } - docMetaMappingFileName := infConfig.DocMetaMappingFileName + docMetaMappingFileName := infinityConfig.DocMetaMappingFileName if docMetaMappingFileName == "" { docMetaMappingFileName = "doc_meta_infinity_mapping.json" } - engine := &infinityEngine{ - config: infConfig, + engine := &Engine{ + config: infinityConfig, client: client, mappingFileName: mappingFileName, docMetaMappingFileName: docMetaMappingFileName, } // Wait for Infinity to be healthy - if err := client.WaitForHealthy(context.Background(), 120*time.Second); err != nil { + if err = client.WaitForHealthy(context.Background(), 120*time.Second); err != nil { return nil, fmt.Errorf("Infinity not healthy: %w", err) } // MigrateDB creates the database if it doesn't exist - if err := engine.MigrateDB(context.Background()); err != nil { + if err = engine.MigrateDB(context.Background()); err != nil { return nil, fmt.Errorf("failed to migrate database: %w", err) } @@ -386,17 +377,17 @@ func NewEngine(cfg interface{}) (*infinityEngine, error) { } // GetType returns the engine type -func (e *infinityEngine) GetType() string { +func (e *Engine) GetType() string { return "infinity" } // SupportsPageRank returns false because Infinity does not support pagerank. -func (e *infinityEngine) SupportsPageRank() bool { +func (e *Engine) SupportsPageRank() bool { return false } // Ping checks if Infinity is accessible -func (e *infinityEngine) Ping(ctx context.Context) error { +func (e *Engine) Ping(ctx context.Context) error { if e.client == nil || e.client.pool == nil { return fmt.Errorf("Infinity client not initialized") } @@ -412,7 +403,7 @@ func (e *infinityEngine) Ping(ctx context.Context) error { } // Close closes the Infinity connection -func (e *infinityEngine) Close() error { +func (e *Engine) Close() error { if e.client != nil && e.client.pool != nil { return e.client.pool.Close() } @@ -420,7 +411,7 @@ func (e *infinityEngine) Close() error { } // MigrateDB creates the database if it doesn't exist -func (e *infinityEngine) MigrateDB(ctx context.Context) error { +func (e *Engine) MigrateDB(ctx context.Context) error { conn, release, err := e.client.checkoutConn(ctx, "MigrateDB") if err != nil { return fmt.Errorf("failed to get connection: %w", err) diff --git a/internal/engine/infinity/common.go b/internal/engine/infinity/common.go index 97754434c1..56238b78ea 100644 --- a/internal/engine/infinity/common.go +++ b/internal/engine/infinity/common.go @@ -30,7 +30,7 @@ import ( ) // dropTable drops a table from Infinity -func (e *infinityEngine) dropTable(ctx context.Context, tableName string) error { +func (e *Engine) dropTable(ctx context.Context, tableName string) error { if tableName == "" { return fmt.Errorf("table name cannot be empty") } @@ -60,7 +60,7 @@ func (e *infinityEngine) dropTable(ctx context.Context, tableName string) error } // tableExists checks if a table exists in Infinity -func (e *infinityEngine) tableExists(ctx context.Context, tableName string) (bool, error) { +func (e *Engine) tableExists(ctx context.Context, tableName string) (bool, error) { if tableName == "" { return false, fmt.Errorf("table name cannot be empty") } @@ -74,7 +74,7 @@ func (e *infinityEngine) tableExists(ctx context.Context, tableName string) (boo return e.tableExistsWithDB(db, tableName) } -func (e *infinityEngine) tableExistsWithDB(db *infinity.Database, tableName string) (bool, error) { +func (e *Engine) tableExistsWithDB(db *infinity.Database, tableName string) (bool, error) { if db == nil { return false, fmt.Errorf("database is nil") } @@ -291,7 +291,7 @@ func buildFilterFromCondition(condition map[string]interface{}, tableColumns map } // columnExists checks if a column exists in the table -func (e *infinityEngine) columnExists(table *infinity.Table, columnName string) (bool, error) { +func (e *Engine) columnExists(table *infinity.Table, columnName string) (bool, error) { colsResp, err := table.ShowColumns() if err != nil { return false, err diff --git a/internal/engine/infinity/document.go b/internal/engine/infinity/document.go index 10369377d8..62a3c8b3c7 100644 --- a/internal/engine/infinity/document.go +++ b/internal/engine/infinity/document.go @@ -28,7 +28,7 @@ import ( // IndexDocument indexes a single document // For skill index (tableName starts with "skill_"), uses InsertSkill // For regular document index, returns not implemented error -func (e *infinityEngine) IndexDocument(ctx context.Context, tableName, docID string, doc interface{}) error { +func (e *Engine) IndexDocument(ctx context.Context, tableName, docID string, doc interface{}) error { // Check if this is a skill index if strings.HasPrefix(tableName, "skill_") { return e.InsertSkill(ctx, tableName, docID, doc) @@ -38,7 +38,7 @@ func (e *infinityEngine) IndexDocument(ctx context.Context, tableName, docID str // InsertSkill inserts a skill document into skill index // Auto-creates the table if it doesn't exist -func (e *infinityEngine) InsertSkill(ctx context.Context, tableName, docID string, doc interface{}) error { +func (e *Engine) InsertSkill(ctx context.Context, tableName, docID string, doc interface{}) error { db, release, err := e.client.checkoutDatabase(ctx, "document.go") if err != nil { return fmt.Errorf("failed to get database: %w", err) @@ -94,7 +94,7 @@ func (e *infinityEngine) InsertSkill(ctx context.Context, tableName, docID strin // BulkIndex indexes documents in bulk // For skill index (tableName starts with "skill_"), uses BulkInsertSkill // For regular document index, returns not implemented error -func (e *infinityEngine) BulkIndex(ctx context.Context, tableName string, docs []interface{}) (interface{}, error) { +func (e *Engine) BulkIndex(ctx context.Context, tableName string, docs []interface{}) (interface{}, error) { // Check if this is a skill index if strings.HasPrefix(tableName, "skill_") { inserted, err := e.BulkInsertSkill(ctx, tableName, docs) @@ -107,7 +107,7 @@ func (e *infinityEngine) BulkIndex(ctx context.Context, tableName string, docs [ // For each document, deletes existing rows with the same skill_id before inserting, // matching the behavior of InsertSkill. Creates shallow copies of input maps to // avoid mutating caller data. -func (e *infinityEngine) BulkInsertSkill(ctx context.Context, tableName string, docs []interface{}) (int, error) { +func (e *Engine) BulkInsertSkill(ctx context.Context, tableName string, docs []interface{}) (int, error) { db, release, err := e.client.checkoutDatabase(ctx, "document.go") if err != nil { return 0, fmt.Errorf("failed to get database: %w", err) @@ -194,12 +194,12 @@ type BulkResponse struct { } // GetDocument gets a document -func (e *infinityEngine) GetDocument(ctx context.Context, tableName, docID string) (interface{}, error) { +func (e *Engine) GetDocument(ctx context.Context, tableName, docID string) (interface{}, error) { return nil, fmt.Errorf("infinity get document not implemented: waiting for official Go SDK") } // DeleteDocument deletes a document by ID -func (e *infinityEngine) DeleteDocument(ctx context.Context, tableName, docID string) error { +func (e *Engine) DeleteDocument(ctx context.Context, tableName, docID string) error { if tableName == "" { return fmt.Errorf("table name cannot be empty") } diff --git a/internal/engine/infinity/metadata.go b/internal/engine/infinity/metadata.go index ad1f17b908..833325e394 100644 --- a/internal/engine/infinity/metadata.go +++ b/internal/engine/infinity/metadata.go @@ -36,7 +36,7 @@ import ( // CreateMetadataStore creates a metadata table in Infinity // tenantID is the tenant identifier used to build the table name -func (e *infinityEngine) CreateMetadataStore(ctx context.Context, tenantID string) error { +func (e *Engine) CreateMetadataStore(ctx context.Context, tenantID string) error { db, release, err := e.client.checkoutDatabase(ctx, "metadata.go") if err != nil { return fmt.Errorf("failed to get database: %w", err) @@ -46,7 +46,7 @@ func (e *infinityEngine) CreateMetadataStore(ctx context.Context, tenantID strin return e.createMetadataStoreWithDB(db, tenantID) } -func (e *infinityEngine) createMetadataStoreWithDB(db *infinity.Database, tenantID string) error { +func (e *Engine) createMetadataStoreWithDB(db *infinity.Database, tenantID string) error { tableName := buildMetadataTableName(tenantID) // Check if table already exists @@ -138,7 +138,7 @@ func (e *infinityEngine) createMetadataStoreWithDB(db *infinity.Database, tenant // The metadata table must already exist; the service layer is responsible // for creating it before writing. // Replace existing metadata with same id and kb_id -func (e *infinityEngine) InsertMetadata(ctx context.Context, metadata []map[string]interface{}, tenantID string) ([]string, error) { +func (e *Engine) InsertMetadata(ctx context.Context, metadata []map[string]interface{}, tenantID string) ([]string, error) { tableName := buildMetadataTableName(tenantID) common.Info("InfinityConnection.InsertMetadata called", zap.String("tableName", tableName), zap.Int("metaCount", len(metadata))) @@ -222,7 +222,7 @@ func (e *infinityEngine) InsertMetadata(ctx context.Context, metadata []map[stri // produced by the LLM extraction pipeline. See the CLI parser in // internal/cli/user_parser.go (parseDevSetMeta) for the user-facing // surface that drives this engine method. -func (e *infinityEngine) UpdateMetadata(ctx context.Context, docID string, datasetID string, metaFields map[string]interface{}, tenantID string) error { +func (e *Engine) UpdateMetadata(ctx context.Context, docID string, datasetID string, metaFields map[string]interface{}, tenantID string) error { tableName := buildMetadataTableName(tenantID) common.Info("InfinityConnection.UpdateMetadata called", zap.String("tableName", tableName), zap.String("docID", docID), zap.String("datasetID", datasetID)) @@ -329,7 +329,7 @@ func (e *infinityEngine) UpdateMetadata(ctx context.Context, docID string, datas // DeleteMetadata deletes metadata from tenant's metadata table by condition // Returns the number of deleted documents. -func (e *infinityEngine) DeleteMetadata(ctx context.Context, condition map[string]interface{}, tenantID string) (int64, error) { +func (e *Engine) DeleteMetadata(ctx context.Context, condition map[string]interface{}, tenantID string) (int64, error) { tableName := buildMetadataTableName(tenantID) db, release, err := e.client.checkoutDatabase(ctx, "metadata.go") @@ -347,7 +347,7 @@ func (e *infinityEngine) DeleteMetadata(ctx context.Context, condition map[strin return e.deleteMetadataWithTable(table, condition) } -func (e *infinityEngine) deleteMetadataWithTable(table *infinity.Table, condition map[string]interface{}) (int64, error) { +func (e *Engine) deleteMetadataWithTable(table *infinity.Table, condition map[string]interface{}) (int64, error) { if table == nil { return 0, fmt.Errorf("metadata table is nil") } @@ -396,7 +396,7 @@ func (e *infinityEngine) deleteMetadataWithTable(table *infinity.Table, conditio // DeleteMetadataKeys deletes specific metadata keys from a document's meta_fields. // If deleting those keys leaves no metadata entries, the metadata row is removed. -func (e *infinityEngine) DeleteMetadataKeys(ctx context.Context, docID string, datasetID string, keys []string, tenantID string) error { +func (e *Engine) DeleteMetadataKeys(ctx context.Context, docID string, datasetID string, keys []string, tenantID string) error { tableName := buildMetadataTableName(tenantID) common.Info("InfinityConnection.DeleteMetadataKeys called", zap.String("tableName", tableName), zap.String("docID", docID), zap.Any("keys", keys)) @@ -539,20 +539,20 @@ func (e *infinityEngine) DeleteMetadataKeys(ctx context.Context, docID string, d } // DropMetadataStore drops a metadata table from Infinity -func (e *infinityEngine) DropMetadataStore(ctx context.Context, tenantID string) error { +func (e *Engine) DropMetadataStore(ctx context.Context, tenantID string) error { tableName := buildMetadataTableName(tenantID) return e.dropTable(ctx, tableName) } // MetadataStoreExists checks if a metadata table exists in Infinity -func (e *infinityEngine) MetadataStoreExists(ctx context.Context, tenantID string) (bool, error) { +func (e *Engine) MetadataStoreExists(ctx context.Context, tenantID string) (bool, error) { tableName := buildMetadataTableName(tenantID) return e.tableExists(ctx, tableName) } // SearchMetadata executes search specifically for metadata tables // This is separate from Search() which handles only chunk tables -func (e *infinityEngine) SearchMetadata(ctx context.Context, req *types.SearchMetadataRequest) (*types.SearchMetadataResult, error) { +func (e *Engine) SearchMetadata(ctx context.Context, req *types.SearchMetadataRequest) (*types.SearchMetadataResult, error) { tenantID := req.TenantID common.Debug("SearchMetadata in Infinity started", zap.String("tenantID", tenantID)) @@ -788,7 +788,7 @@ const metaPushdownMaxSize = 10000 // nil -> push-down was not viable / errored / result overflowed the // push-down cap (caller should fall back to in-memory) // []string{} -> push-down succeeded but found 0 matching docs (empty result is definitive) -func (e *infinityEngine) FilterDocIdsByMetaPushdown(ctx context.Context, sqlDB *gorm.DB, kbIDs []string, conditions []map[string]interface{}, logic string) []string { +func (e *Engine) FilterDocIdsByMetaPushdown(ctx context.Context, sqlDB *gorm.DB, kbIDs []string, conditions []map[string]interface{}, logic string) []string { if len(conditions) == 0 || len(kbIDs) == 0 { return nil } diff --git a/internal/engine/infinity/sql.go b/internal/engine/infinity/sql.go index e7b15b812c..3c2b8745d0 100644 --- a/internal/engine/infinity/sql.go +++ b/internal/engine/infinity/sql.go @@ -286,7 +286,7 @@ func resolvePsqlHostPort(hostURI string, postgresPort int) (host, port string) { // RunSQL implements the SQL retrieval path: preprocess, rewrite aliases, // run psql subprocess, parse output. -func (e *infinityEngine) RunSQL(ctx context.Context, tableName string, sqlText string, kbIDs []string, _ string) ([]map[string]interface{}, error) { +func (e *Engine) RunSQL(ctx context.Context, tableName string, sqlText string, kbIDs []string, _ string) ([]map[string]interface{}, error) { if e == nil || e.client == nil { return nil, fmt.Errorf("infinity RunSQL: client not initialized") } diff --git a/internal/engine/redis/redis.go b/internal/engine/redis/redis.go index ca44a2992d..00f7023ff8 100644 --- a/internal/engine/redis/redis.go +++ b/internal/engine/redis/redis.go @@ -23,6 +23,7 @@ import ( "math" "math/rand" "ragflow/internal/common" + "ragflow/internal/server/config" "strconv" "sync" "time" @@ -45,7 +46,7 @@ type Client struct { luaDeleteIfEqual *redis.Script luaTokenBucket *redis.Script luaAutoIncrement *redis.Script - config *server.RedisConfig + config config.RedisConfig } // Message represents a message from Redis Stream @@ -105,18 +106,21 @@ const ( ) // Init initializes Redis client -func Init(cfg *server.RedisConfig) error { +func Init() error { var initErr error once.Do(func() { - if cfg.Host == "" { + globalConfig := server.GetConfig() + redisConfig := globalConfig.GetRedisConfig() + + if redisConfig.Host == "" { common.Info("Redis host not configured, skipping Redis initialization") return } client := redis.NewClient(&redis.Options{ - Addr: fmt.Sprintf("%s:%d", cfg.Host, cfg.Port), - Password: cfg.Password, - DB: cfg.DB, + Addr: fmt.Sprintf("%s:%d", redisConfig.Host, redisConfig.Port), + Password: redisConfig.Password, + DB: redisConfig.DB, }) // Test connection @@ -130,15 +134,15 @@ func Init(cfg *server.RedisConfig) error { globalClient = &Client{ client: client, - config: cfg, + config: redisConfig, luaDeleteIfEqual: redis.NewScript(luaDeleteIfEqualScript), luaTokenBucket: redis.NewScript(luaTokenBucketScript), } common.Info("Redis client initialized", - zap.String("host", cfg.Host), - zap.Int("port", cfg.Port), - zap.Int("db", cfg.DB), + zap.String("host", redisConfig.Host), + zap.Int("port", redisConfig.Port), + zap.Int("db", redisConfig.DB), ) }) return initErr diff --git a/internal/handler/system.go b/internal/handler/system.go index 0b2db57a4d..d5d68a5879 100644 --- a/internal/handler/system.go +++ b/internal/handler/system.go @@ -178,7 +178,7 @@ func (h *SystemHandler) SetLogLevel(c *gin.Context) { } if config := server.GetConfig(); config != nil { - config.Log.Level = common.GetLogLevel() + config.SetLogLevel(req.Level) } common.SuccessWithData(c, gin.H{"level": req.Level}, "SUCCESS") diff --git a/internal/ingestion/task/pipeline_real_integration_test.go b/internal/ingestion/task/pipeline_real_integration_test.go index d047271873..5c46d9c8b5 100644 --- a/internal/ingestion/task/pipeline_real_integration_test.go +++ b/internal/ingestion/task/pipeline_real_integration_test.go @@ -8,6 +8,7 @@ import ( "io" "os" "path/filepath" + "ragflow/internal/server/config" "sort" "strings" "testing" @@ -389,7 +390,7 @@ func taskRepoRoot(t *testing.T) string { return filepath.Clean(filepath.Join(wd, "..", "..", "..")) } -func mustLoadTaskTestConfig(t *testing.T) *server.Config { +func mustLoadTaskTestConfig(t *testing.T) *config.Config { t.Helper() if err := common.Init("info", common.FileOutput{}, ""); err != nil { t.Fatalf("init common logger: %v", err) diff --git a/internal/parser/chunk/chenk_type.go b/internal/parser/chunk/chunk_type.go similarity index 100% rename from internal/parser/chunk/chenk_type.go rename to internal/parser/chunk/chunk_type.go diff --git a/internal/server/config.go b/internal/server/config.go index 3a1e643fd0..4b3cb5e13e 100644 --- a/internal/server/config.go +++ b/internal/server/config.go @@ -20,8 +20,8 @@ import ( "errors" "fmt" "net" - "net/mail" "net/url" + "ragflow/internal/server/config" "strconv" "strings" "time" @@ -311,7 +311,7 @@ type NatsConfig struct { } var ( - globalConfig *Config + globalConfig *config.Config globalViper *viper.Viper zapLogger *zap.Logger allConfigs []map[string]interface{} @@ -330,285 +330,165 @@ func Init(configPath string) error { return err } - id := 0 - for k, v := range globalViper.AllSettings() { - configDict, ok := v.(map[string]interface{}) - if !ok { - continue - } - - switch k { - case "ragflow": - configDict["id"] = id - configDict["name"] = fmt.Sprintf("ragflow_%d", id) - configDict["service_type"] = "ragflow_server" - configDict["extra"] = map[string]interface{}{} - configDict["port"] = configDict["http_port"] - delete(configDict, "http_port") - case "es": - // Skip if retrieval_type doesn't match doc_engine - if globalConfig.DocEngine.Type != "elasticsearch" { - continue - } - hosts := getString(configDict, "hosts") - host, port := parseHostPort(hosts) - username := getString(configDict, "username") - password := getString(configDict, "password") - configDict["id"] = id - configDict["name"] = "elasticsearch" - configDict["host"] = host - configDict["port"] = port - configDict["service_type"] = "retrieval" - configDict["extra"] = map[string]interface{}{ - "retrieval_type": "elasticsearch", - "username": username, - "password": password, - } - delete(configDict, "hosts") - delete(configDict, "username") - delete(configDict, "password") - case "infinity": - // Skip if retrieval_type doesn't match doc_engine - if globalConfig.DocEngine.Type != "infinity" { - continue - } - uri := getString(configDict, "uri") - host, port := parseHostPort(uri) - dbName := getString(configDict, "db_name") - if dbName == "" { - dbName = "default_db" - } - configDict["id"] = id - configDict["name"] = "infinity" - configDict["host"] = host - configDict["port"] = port - configDict["service_type"] = "retrieval" - configDict["extra"] = map[string]interface{}{ - "retrieval_type": "infinity", - "db_name": dbName, - } - case "minio": - hostPort := getString(configDict, "host") - host, port := parseHostPort(hostPort) - user := getString(configDict, "user") - password := getString(configDict, "password") - configDict["id"] = id - configDict["name"] = "minio" - configDict["host"] = host - configDict["port"] = port - configDict["service_type"] = "file_store" - configDict["extra"] = map[string]interface{}{ - "store_type": "minio", - "user": user, - "password": password, - } - delete(configDict, "bucket") - delete(configDict, "user") - delete(configDict, "password") - case "redis": - hostPort := getString(configDict, "host") - host, port := parseHostPort(hostPort) - password := getString(configDict, "password") - db := getInt(configDict, "db") - configDict["id"] = id - configDict["name"] = "redis" - configDict["host"] = host - configDict["port"] = port - configDict["service_type"] = "cache" - configDict["extra"] = map[string]interface{}{ - "mq_type": "redis", - "database": db, - "password": password, - } - delete(configDict, "password") - delete(configDict, "db") - case "mysql": - host := getString(configDict, "host") - port := getInt(configDict, "port") - user := getString(configDict, "user") - password := getString(configDict, "password") - configDict["id"] = id - configDict["name"] = "mysql" - configDict["host"] = host - configDict["port"] = port - configDict["service_type"] = "meta_data" - configDict["extra"] = map[string]interface{}{ - "meta_type": "mysql", - "username": user, - "password": password, - } - delete(configDict, "stale_timeout") - delete(configDict, "max_connections") - delete(configDict, "max_allowed_packet") - delete(configDict, "user") - delete(configDict, "password") - case "ingestor": - mqType := getString(configDict, "mq_type") - configDict["id"] = id - configDict["name"] = "ingestor" - configDict["service_type"] = "ingestor" - configDict["extra"] = map[string]interface{}{ - "message_queue_type": mqType, - } - delete(configDict, "message_queue_type") - case "nats": - configDict["id"] = id - configDict["name"] = "nats" - configDict["service_type"] = "message_queue" - case "otel": - configDict["id"] = id - configDict["name"] = "jaeger" - configDict["service_type"] = "tracing" - case "clickhouse": - configDict["id"] = id - configDict["name"] = "clickhouse" - configDict["service_type"] = "olap" - case "admin": - // Skip admin section - continue - default: - // Skip unknown sections - continue - } - - // Set default values for empty host/port - if configDict["host"] == "" { - configDict["host"] = "-" - } - if configDict["port"] == 0 { - configDict["port"] = "-" - } - - delete(configDict, "prefix_path") - delete(configDict, "username") - allConfigs = append(allConfigs, configDict) - id++ - } + //id := 0 + //for k, v := range globalViper.AllSettings() { + // configDict, ok := v.(map[string]interface{}) + // if !ok { + // continue + // } + // + // switch k { + // case "ragflow": + // configDict["id"] = id + // configDict["name"] = fmt.Sprintf("ragflow_%d", id) + // configDict["service_type"] = "ragflow_server" + // configDict["extra"] = map[string]interface{}{} + // configDict["port"] = configDict["http_port"] + // delete(configDict, "http_port") + // case "es": + // // Skip if retrieval_type doesn't match doc_engine + // if globalConfig.DocEngine.Type != "elasticsearch" { + // continue + // } + // hosts := getString(configDict, "hosts") + // host, port := parseHostPort(hosts) + // username := getString(configDict, "username") + // password := getString(configDict, "password") + // configDict["id"] = id + // configDict["name"] = "elasticsearch" + // configDict["host"] = host + // configDict["port"] = port + // configDict["service_type"] = "retrieval" + // configDict["extra"] = map[string]interface{}{ + // "retrieval_type": "elasticsearch", + // "username": username, + // "password": password, + // } + // delete(configDict, "hosts") + // delete(configDict, "username") + // delete(configDict, "password") + // case "infinity": + // // Skip if retrieval_type doesn't match doc_engine + // if globalConfig.DocEngine.Type != "infinity" { + // continue + // } + // uri := getString(configDict, "uri") + // host, port := parseHostPort(uri) + // dbName := getString(configDict, "db_name") + // if dbName == "" { + // dbName = "default_db" + // } + // configDict["id"] = id + // configDict["name"] = "infinity" + // configDict["host"] = host + // configDict["port"] = port + // configDict["service_type"] = "retrieval" + // configDict["extra"] = map[string]interface{}{ + // "retrieval_type": "infinity", + // "db_name": dbName, + // } + // case "minio": + // hostPort := getString(configDict, "host") + // host, port := parseHostPort(hostPort) + // user := getString(configDict, "user") + // password := getString(configDict, "password") + // configDict["id"] = id + // configDict["name"] = "minio" + // configDict["host"] = host + // configDict["port"] = port + // configDict["service_type"] = "file_store" + // configDict["extra"] = map[string]interface{}{ + // "store_type": "minio", + // "user": user, + // "password": password, + // } + // delete(configDict, "bucket") + // delete(configDict, "user") + // delete(configDict, "password") + // case "redis": + // hostPort := getString(configDict, "host") + // host, port := parseHostPort(hostPort) + // password := getString(configDict, "password") + // db := getInt(configDict, "db") + // configDict["id"] = id + // configDict["name"] = "redis" + // configDict["host"] = host + // configDict["port"] = port + // configDict["service_type"] = "cache" + // configDict["extra"] = map[string]interface{}{ + // "mq_type": "redis", + // "database": db, + // "password": password, + // } + // delete(configDict, "password") + // delete(configDict, "db") + // case "mysql": + // host := getString(configDict, "host") + // port := getInt(configDict, "port") + // user := getString(configDict, "user") + // password := getString(configDict, "password") + // configDict["id"] = id + // configDict["name"] = "mysql" + // configDict["host"] = host + // configDict["port"] = port + // configDict["service_type"] = "meta_data" + // configDict["extra"] = map[string]interface{}{ + // "meta_type": "mysql", + // "username": user, + // "password": password, + // } + // delete(configDict, "stale_timeout") + // delete(configDict, "max_connections") + // delete(configDict, "max_allowed_packet") + // delete(configDict, "user") + // delete(configDict, "password") + // case "ingestor": + // mqType := getString(configDict, "mq_type") + // configDict["id"] = id + // configDict["name"] = "ingestor" + // configDict["service_type"] = "ingestor" + // configDict["extra"] = map[string]interface{}{ + // "message_queue_type": mqType, + // } + // delete(configDict, "message_queue_type") + // case "nats": + // configDict["id"] = id + // configDict["name"] = "nats" + // configDict["service_type"] = "message_queue" + // case "otel": + // configDict["id"] = id + // configDict["name"] = "jaeger" + // configDict["service_type"] = "tracing" + // case "clickhouse": + // configDict["id"] = id + // configDict["name"] = "clickhouse" + // configDict["service_type"] = "olap" + // case "admin": + // // Skip admin section + // continue + // default: + // // Skip unknown sections + // continue + // } + // + // // Set default values for empty host/port + // if configDict["host"] == "" { + // configDict["host"] = "-" + // } + // if configDict["port"] == 0 { + // configDict["port"] = "-" + // } + // + // delete(configDict, "prefix_path") + // delete(configDict, "username") + // allConfigs = append(allConfigs, configDict) + // id++ + //} return nil } func FromEnvironments() error { - // Secret key - if envVal := common.GetEnv(common.EnvRAGFlowSecretKey); envVal != "" { - globalConfig.General.SecretKey = &envVal - } - - // Load REGISTER_ENABLED from environment variable (default: true) - if envVal := common.GetEnv(common.EnvRegisterEnabled); envVal != "" { - str := strings.ToLower(envVal) - if str == "true" || str == "1" || str == "yes" { - globalConfig.Authentication.RegisterEnabled = true - } else { - globalConfig.Authentication.RegisterEnabled = false - } - } - - // Load DISABLE_PASSWORD_LOGIN from environment variable (default: false) - if envVal := common.GetEnv(common.EnvDisablePasswordLogin); envVal != "" { - str := strings.ToLower(envVal) - if str == "true" || str == "1" || str == "yes" { - globalConfig.Authentication.DisablePasswordLogin = true - } else { - globalConfig.Authentication.DisablePasswordLogin = false - } - } - - // Doc engine - docEngine := common.GetEnvSmall(common.EnvDocEngine) - switch docEngine { - case "infinity": - globalConfig.DocEngine.Type = EngineInfinity - case "": - // Default - if globalConfig.DocEngine.Type == "" { - globalConfig.DocEngine.Type = EngineElasticsearch - } - case "elasticsearch": - globalConfig.DocEngine.Type = EngineElasticsearch - case "opensearch": - case "oceanbase": - return fmt.Errorf("not implemented: %s", docEngine) - default: - return fmt.Errorf("invalid doc engine: %s", docEngine) - } - - // Default super user email - globalConfig.DefaultSuperUser.Email = "admin@ragflow.io" - superUserEmail := common.GetEnv(common.EnvDefaultSuperuserEmail) - if superUserEmail != "" { - _, err := mail.ParseAddress(superUserEmail) - if err != nil { - return fmt.Errorf("invalid super user email: %s", superUserEmail) - } - globalConfig.DefaultSuperUser.Email = superUserEmail - } - - globalConfig.DefaultSuperUser.Password = "admin" - superUserPassword := common.GetEnv(common.EnvDefaultSuperuserPassword) - if superUserPassword != "" { - globalConfig.DefaultSuperUser.Password = superUserPassword - } - - globalConfig.DefaultSuperUser.Nickname = "admin" - superUserNickname := common.GetEnv(common.EnvDefaultSuperuserNickname) - if superUserNickname != "" { - globalConfig.DefaultSuperUser.Nickname = superUserNickname - } - - // Meta database - databaseType := common.GetEnvSmall(common.EnvDBType) - switch databaseType { - case "mysql": - globalConfig.Database.Driver = "mysql" - case "": - // Default - if globalConfig.Database.Driver == "" { - globalConfig.Database.Driver = "mysql" - } - default: - return fmt.Errorf("invalid database type: %s", databaseType) - } - - // Storage - storageType := common.GetEnvSmall(common.EnvStorageImpl) - switch storageType { - case "minio": - globalConfig.StorageEngine.Type = StorageMinio - case "s3": - globalConfig.StorageEngine.Type = StorageS3 - case "oss": - globalConfig.StorageEngine.Type = StorageOSS - case "gcs": - globalConfig.StorageEngine.Type = StorageGCS - case "": - // Default - if globalConfig.StorageEngine.Type == "" { - globalConfig.StorageEngine.Type = StorageMinio - } - default: - return fmt.Errorf("invalid storage type: %s", storageType) - } - - // Minio - minioHost := strings.ToLower(common.GetEnv(common.EnvMinioHost)) - if minioHost != "" { - globalConfig.StorageEngine.Minio.Host = minioEndpoint(minioHost, globalConfig.StorageEngine.Minio.Host) - } - - minioRegion := strings.ToLower(common.GetEnv(common.EnvMinioRegion)) - if minioRegion != "" { - if globalConfig.StorageEngine.Minio == nil { - return fmt.Errorf("minio config not found") - } - globalConfig.StorageEngine.Minio.Region = minioRegion - } - - // Language - if globalConfig.Language == "" { - globalConfig.Language = GetLanguage() - } - return nil } @@ -657,299 +537,384 @@ func FromConfigFile(configPath string) error { // Save viper instance globalViper = v - // Unmarshal configuration to globalConfig - // Note: This will only unmarshal fields that match the Config struct - if err := v.Unmarshal(&globalConfig); err != nil { - return fmt.Errorf("unmarshal config error: %w", err) + globalConfig = &config.Config{} + err := globalConfig.ParseGeneralConfig(v) + if err != nil { + return fmt.Errorf("parse general config error: %w", err) } - // Set default values for admin configuration if not configured - if globalConfig.Admin.Host == "" { - globalConfig.Admin.Host = "127.0.0.1" - } - if globalConfig.Admin.Port == 0 { - globalConfig.Admin.Port = 9383 - } else { - globalConfig.Admin.Port += 2 + err = globalConfig.ParseDatabaseConfig(v) + if err != nil { + return fmt.Errorf("parse database config error: %w", err) } - // authentication section - if globalConfig != nil { - // Try to map from mysql section - globalConfig.Authentication.DisablePasswordLogin = false - globalConfig.Authentication.RegisterEnabled = true - if v.IsSet("authentication") { - authenticationConfig := v.Sub("authentication") - if authenticationConfig != nil { - if authenticationConfig.IsSet("disable_password_login") { - globalConfig.Authentication.DisablePasswordLogin = authenticationConfig.GetBool("disable_password_login") - } - if authenticationConfig.IsSet("enable_register") { - globalConfig.Authentication.RegisterEnabled = authenticationConfig.GetBool("enable_register") - } - } - } + err = globalConfig.ParseDocEngineConfig(v) + if err != nil { + return fmt.Errorf("parse doc engine config error: %w", err) } - // If we loaded service_conf.yaml, map mysql fields to DatabaseConfig - if globalConfig != nil && globalConfig.Database.Host == "" { - // Try to map from mysql section - if v.IsSet("mysql") { - mysqlConfig := v.Sub("mysql") - if mysqlConfig != nil { - globalConfig.Database.Driver = "mysql" - globalConfig.Database.Host = mysqlConfig.GetString("host") - globalConfig.Database.Port = mysqlConfig.GetInt("port") - globalConfig.Database.Database = mysqlConfig.GetString("name") - globalConfig.Database.Username = mysqlConfig.GetString("user") - globalConfig.Database.Password = mysqlConfig.GetString("password") - globalConfig.Database.Charset = "utf8mb4" - } - } + err = globalConfig.ParseStorageEngineConfig(v) + if err != nil { + return fmt.Errorf("parse storage engine config error: %w", err) } - // Map ragflow section to ServerConfig - if globalConfig != nil && globalConfig.APIServer.Port == 0 { - // Try to map from ragflow section - if v.IsSet("ragflow") { - ragflowConfig := v.Sub("ragflow") - if ragflowConfig != nil { - globalConfig.APIServer.Port = ragflowConfig.GetInt("http_port") + 4 // 9384, by default - //globalConfig.Server.Port = ragflowConfig.GetInt("http_port") // Correct - // If mode is not set, default to debug - if globalConfig.General.Mode == "" { - globalConfig.General.Mode = "release" - } - secretKey := ragflowConfig.GetString("secret_key") - if secretKey != "" { - globalConfig.General.SecretKey = &secretKey - } - } - } + err = globalConfig.ParseCacheEngineConfig(v) + if err != nil { + return fmt.Errorf("parse cache engine config error: %w", err) } - if globalConfig.APIServer.Port == 0 { - globalConfig.APIServer.Port = 9384 - } else { - globalConfig.APIServer.Port += 4 + err = globalConfig.ParseQueueEngineConfig(v) + if err != nil { + return fmt.Errorf("parse queue engine config error: %w", err) } - // Map redis section to RedisConfig - if globalConfig != nil && globalConfig.Redis.Host != "" { - if v.IsSet("redis") { - redisConfig := v.Sub("redis") - if redisConfig != nil { - hostStr := redisConfig.GetString("host") - // Handle host:port format (e.g., "localhost:6379") - if hostStr == "" { - return fmt.Errorf("empty host of Redis configuration") - } - - if idx := strings.LastIndex(hostStr, ":"); idx != -1 { - globalConfig.Redis.Host = hostStr[:idx] - if portStr := hostStr[idx+1:]; portStr != "" { - if port, err := strconv.Atoi(portStr); err == nil { - globalConfig.Redis.Port = port - } - } - } else { - return fmt.Errorf("error address format of Redis: %s", hostStr) - } - - globalConfig.Redis.Password = redisConfig.GetString("password") - globalConfig.Redis.DB = redisConfig.GetInt("db") - } - } + err = globalConfig.ParseAnalyticEngineConfig(v) + if err != nil { + return fmt.Errorf("parse analytic engine config error: %w", err) } - // Map doc_engine section to DocEngineConfig - if globalConfig != nil { - // First, ensure engine type is set - if globalConfig.DocEngine.Type == "" { - if v.IsSet("doc_engine") { - docEngineConfig := v.Sub("doc_engine") - if docEngineConfig != nil { - globalConfig.DocEngine.Type = EngineType(docEngineConfig.GetString("type")) - } - } - } - - // Map es section from top-level (service_conf.yaml format) - if v.IsSet("es") { - esConfig := v.Sub("es") - if esConfig != nil { - // Set default engine type if not set - if globalConfig.DocEngine.Type == "" { - globalConfig.DocEngine.Type = EngineElasticsearch - } - // Always populate ES config if es section exists - if globalConfig.DocEngine.ES == nil { - globalConfig.DocEngine.ES = &ElasticsearchConfig{ - Hosts: esConfig.GetString("hosts"), - Username: esConfig.GetString("username"), - Password: esConfig.GetString("password"), - } - } - } - } - - // Map infinity section from top-level (service_conf.yaml format) - if v.IsSet("infinity") { - infConfig := v.Sub("infinity") - if infConfig != nil { - // Set default engine type if not set - if globalConfig.DocEngine.Type == "" { - globalConfig.DocEngine.Type = EngineInfinity - } - // Always populate Infinity config if infinity section exists - if globalConfig.DocEngine.Infinity == nil { - globalConfig.DocEngine.Infinity = &InfinityConfig{ - URI: infConfig.GetString("uri"), - PostgresPort: infConfig.GetInt("postgres_port"), - DBName: infConfig.GetString("db_name"), - MappingFileName: infConfig.GetString("mapping_file_name"), - DocMetaMappingFileName: infConfig.GetString("doc_meta_mapping_file_name"), - } - } - } - } + err = globalConfig.ParseOpenTelemetryConfig(v) + if err != nil { + return fmt.Errorf("parse open telemetry config error: %w", err) } - if globalConfig != nil && globalConfig.StorageEngine.Type == "" { - // Also check legacy es section for backward compatibility - if v.IsSet("minio") { - minioConfig := v.Sub("minio") - if minioConfig != nil { - if globalConfig.StorageEngine.Minio == nil { - globalConfig.StorageEngine.Minio = &MinioConfig{ - Host: minioConfig.GetString("host"), - User: minioConfig.GetString("user"), - Password: minioConfig.GetString("password"), - Secure: minioConfig.GetBool("secure"), - PrefixPath: minioConfig.GetString("prefix_path"), - Verify: minioConfig.GetBool("verify"), - Region: minioConfig.GetString("region"), - Bucket: minioConfig.GetString("bucket"), - } - } - } - } - - if v.IsSet("gcs") { - gcsConfig := v.Sub("gcs") - if gcsConfig != nil { - if globalConfig.StorageEngine.GCS == nil { - globalConfig.StorageEngine.GCS = &GCSConfig{ - Bucket: gcsConfig.GetString("bucket"), - PrefixPath: gcsConfig.GetString("prefix_path"), - EndpointURL: gcsConfig.GetString("endpoint_url"), - } - } - } - } - - if v.IsSet("minio_0") { - minioConfig := v.Sub("minio_0") - if minioConfig != nil { - if globalConfig.StorageEngine.Minio == nil { - globalConfig.StorageEngine.Minio = &MinioConfig{ - Host: minioConfig.GetString("host"), - User: minioConfig.GetString("user"), - Password: minioConfig.GetString("password"), - Secure: minioConfig.GetBool("secure"), - PrefixPath: minioConfig.GetString("prefix_path"), - Verify: minioConfig.GetBool("verify"), - Bucket: minioConfig.GetString("bucket"), - } - } - } - } - - if v.IsSet("s3") { - s3Config := v.Sub("s3") - if s3Config != nil { - if globalConfig.StorageEngine.S3 == nil { - globalConfig.StorageEngine.S3 = &S3Config{ - AccessKey: s3Config.GetString("access_key"), - SecretKey: s3Config.GetString("secret_key"), - Region: s3Config.GetString("region"), - } - } - } - } - - if v.IsSet("oss") { - ossConfig := v.Sub("oss") - if ossConfig != nil { - if globalConfig.StorageEngine.OSS == nil { - globalConfig.StorageEngine.OSS = &OSSConfig{ - AccessKey: ossConfig.GetString("access_key"), - SecretKey: ossConfig.GetString("secret_key"), - EndpointURL: ossConfig.GetString("endpoint_url"), - Region: ossConfig.GetString("region"), - Bucket: ossConfig.GetString("bucket"), - SignatureVersion: ossConfig.GetString("signature_version"), - AddressingStyle: ossConfig.GetString("addressing_style"), - } - } - } - } + err = globalConfig.ParseAdminConfig(v) + if err != nil { + return fmt.Errorf("parse admin config error: %w", err) } - // Map user_default_llm section to UserDefaultLLMConfig - if v.IsSet("user_default_llm") { - userDefaultLLMConfig := v.Sub("user_default_llm") - if userDefaultLLMConfig != nil { - if defaultModels := userDefaultLLMConfig.Sub("default_models"); defaultModels != nil { - globalConfig.UserDefaultLLM.DefaultModels.ChatModel = ModelConfig{ - Name: defaultModels.GetString("chat_model.name"), - APIKey: defaultModels.GetString("chat_model.api_key"), - BaseURL: defaultModels.GetString("chat_model.base_url"), - Factory: defaultModels.GetString("chat_model.factory"), - } - globalConfig.UserDefaultLLM.DefaultModels.EmbeddingModel = ModelConfig{ - Name: defaultModels.GetString("embedding_model.name"), - APIKey: defaultModels.GetString("embedding_model.api_key"), - BaseURL: defaultModels.GetString("embedding_model.base_url"), - Factory: defaultModels.GetString("embedding_model.factory"), - } - globalConfig.UserDefaultLLM.DefaultModels.RerankModel = ModelConfig{ - Name: defaultModels.GetString("rerank_model.name"), - APIKey: defaultModels.GetString("rerank_model.api_key"), - BaseURL: defaultModels.GetString("rerank_model.base_url"), - Factory: defaultModels.GetString("rerank_model.factory"), - } - globalConfig.UserDefaultLLM.DefaultModels.ASRModel = ModelConfig{ - Name: defaultModels.GetString("asr_model.name"), - APIKey: defaultModels.GetString("asr_model.api_key"), - BaseURL: defaultModels.GetString("asr_model.base_url"), - Factory: defaultModels.GetString("asr_model.factory"), - } - globalConfig.UserDefaultLLM.DefaultModels.Image2TextModel = ModelConfig{ - Name: defaultModels.GetString("image2text_model.name"), - APIKey: defaultModels.GetString("image2text_model.api_key"), - BaseURL: defaultModels.GetString("image2text_model.base_url"), - Factory: defaultModels.GetString("image2text_model.factory"), - } - } - } + err = globalConfig.ParseAPIServerConfig(v) + if err != nil { + return fmt.Errorf("parse API server config error: %w", err) } + err = globalConfig.ParseIngestorConfig(v) + if err != nil { + return fmt.Errorf("parse ingestor config error: %w", err) + } + + err = globalConfig.ParseSyncerConfig(v) + if err != nil { + return fmt.Errorf("parse syncer config error: %w", err) + } + + err = globalConfig.ParseLogConfig(v) + if err != nil { + return fmt.Errorf("parse log config error: %w", err) + } + + err = globalConfig.ParseSMTPConfig(v) + if err != nil { + return fmt.Errorf("parse SMTP config error: %w", err) + } + + err = globalConfig.GetEnvironments() + if err != nil { + return fmt.Errorf("get environments error: %w", err) + } + + err = globalConfig.ParseBillingConfig(v) + if err != nil { + return fmt.Errorf("parse billing config error: %w", err) + } + + err = globalConfig.ParseDefaultModelsConfig(v) + if err != nil { + return fmt.Errorf("parse default models config error: %w", err) + } + + err = globalConfig.ParseOAuthConfig(v) + if err != nil { + return fmt.Errorf("parse OAuth config error: %w", err) + } + + //// Set default values for admin configuration if not configured + //if globalConfig.Admin.Host == "" { + // globalConfig.Admin.Host = "127.0.0.1" + //} + //if globalConfig.Admin.Port == 0 { + // globalConfig.Admin.Port = 9383 + //} else { + // globalConfig.Admin.Port += 2 + //} + + //// authentication section + //if globalConfig != nil { + // // Try to map from mysql section + // globalConfig.Authentication.DisablePasswordLogin = false + // globalConfig.Authentication.RegisterEnabled = true + // if v.IsSet("authentication") { + // authenticationConfig := v.Sub("authentication") + // if authenticationConfig != nil { + // if authenticationConfig.IsSet("disable_password_login") { + // globalConfig.Authentication.DisablePasswordLogin = authenticationConfig.GetBool("disable_password_login") + // } + // if authenticationConfig.IsSet("enable_register") { + // globalConfig.Authentication.RegisterEnabled = authenticationConfig.GetBool("enable_register") + // } + // } + // } + //} + // + //// If we loaded service_conf.yaml, map mysql fields to DatabaseConfig + //if globalConfig != nil && globalConfig.Database.Host == "" { + // // Try to map from mysql section + // if v.IsSet("mysql") { + // mysqlConfig := v.Sub("mysql") + // if mysqlConfig != nil { + // globalConfig.Database.Driver = "mysql" + // globalConfig.Database.Host = mysqlConfig.GetString("host") + // globalConfig.Database.Port = mysqlConfig.GetInt("port") + // globalConfig.Database.Database = mysqlConfig.GetString("name") + // globalConfig.Database.Username = mysqlConfig.GetString("user") + // globalConfig.Database.Password = mysqlConfig.GetString("password") + // globalConfig.Database.Charset = "utf8mb4" + // } + // } + //} + // + //// Map ragflow section to ServerConfig + //if globalConfig != nil && globalConfig.APIServer.Port == 0 { + // // Try to map from ragflow section + // if v.IsSet("ragflow") { + // ragflowConfig := v.Sub("ragflow") + // if ragflowConfig != nil { + // globalConfig.APIServer.Port = ragflowConfig.GetInt("http_port") + 4 // 9384, by default + // //globalConfig.Server.Port = ragflowConfig.GetInt("http_port") // Correct + // // If mode is not set, default to debug + // if globalConfig.General.Mode == "" { + // globalConfig.General.Mode = "release" + // } + // secretKey := ragflowConfig.GetString("secret_key") + // if secretKey != "" { + // globalConfig.General.SecretKey = &secretKey + // } + // } + // } + //} + // + //if globalConfig.APIServer.Port == 0 { + // globalConfig.APIServer.Port = 9384 + //} else { + // globalConfig.APIServer.Port += 4 + //} + // + //// Map redis section to RedisConfig + //if globalConfig != nil && globalConfig.Redis.Host != "" { + // if v.IsSet("redis") { + // redisConfig := v.Sub("redis") + // if redisConfig != nil { + // hostStr := redisConfig.GetString("host") + // // Handle host:port format (e.g., "localhost:6379") + // if hostStr == "" { + // return fmt.Errorf("empty host of Redis configuration") + // } + // + // if idx := strings.LastIndex(hostStr, ":"); idx != -1 { + // globalConfig.Redis.Host = hostStr[:idx] + // if portStr := hostStr[idx+1:]; portStr != "" { + // if port, err := strconv.Atoi(portStr); err == nil { + // globalConfig.Redis.Port = port + // } + // } + // } else { + // return fmt.Errorf("error address format of Redis: %s", hostStr) + // } + // + // globalConfig.Redis.Password = redisConfig.GetString("password") + // globalConfig.Redis.DB = redisConfig.GetInt("db") + // } + // } + //} + // + //// Map doc_engine section to DocEngineConfig + //if globalConfig != nil { + // // First, ensure engine type is set + // if globalConfig.DocEngine.Type == "" { + // if v.IsSet("doc_engine") { + // docEngineConfig := v.Sub("doc_engine") + // if docEngineConfig != nil { + // globalConfig.DocEngine.Type = EngineType(docEngineConfig.GetString("type")) + // } + // } + // } + // + // // Map es section from top-level (service_conf.yaml format) + // if v.IsSet("es") { + // esConfig := v.Sub("es") + // if esConfig != nil { + // // Set default engine type if not set + // if globalConfig.DocEngine.Type == "" { + // globalConfig.DocEngine.Type = EngineElasticsearch + // } + // // Always populate ES config if es section exists + // if globalConfig.DocEngine.ES == nil { + // globalConfig.DocEngine.ES = &ElasticsearchConfig{ + // Hosts: esConfig.GetString("hosts"), + // Username: esConfig.GetString("username"), + // Password: esConfig.GetString("password"), + // } + // } + // } + // } + // + // // Map infinity section from top-level (service_conf.yaml format) + // if v.IsSet("infinity") { + // infConfig := v.Sub("infinity") + // if infConfig != nil { + // // Set default engine type if not set + // if globalConfig.DocEngine.Type == "" { + // globalConfig.DocEngine.Type = EngineInfinity + // } + // // Always populate Infinity config if infinity section exists + // if globalConfig.DocEngine.Infinity == nil { + // globalConfig.DocEngine.Infinity = &InfinityConfig{ + // URI: infConfig.GetString("uri"), + // PostgresPort: infConfig.GetInt("postgres_port"), + // DBName: infConfig.GetString("db_name"), + // MappingFileName: infConfig.GetString("mapping_file_name"), + // DocMetaMappingFileName: infConfig.GetString("doc_meta_mapping_file_name"), + // } + // } + // } + // } + //} + // + //if globalConfig != nil && globalConfig.StorageEngine.Type == "" { + // // Also check legacy es section for backward compatibility + // if v.IsSet("minio") { + // minioConfig := v.Sub("minio") + // if minioConfig != nil { + // if globalConfig.StorageEngine.Minio == nil { + // globalConfig.StorageEngine.Minio = &MinioConfig{ + // Host: minioConfig.GetString("host"), + // User: minioConfig.GetString("user"), + // Password: minioConfig.GetString("password"), + // Secure: minioConfig.GetBool("secure"), + // PrefixPath: minioConfig.GetString("prefix_path"), + // Verify: minioConfig.GetBool("verify"), + // Region: minioConfig.GetString("region"), + // Bucket: minioConfig.GetString("bucket"), + // } + // } + // } + // } + // + // if v.IsSet("gcs") { + // gcsConfig := v.Sub("gcs") + // if gcsConfig != nil { + // if globalConfig.StorageEngine.GCS == nil { + // globalConfig.StorageEngine.GCS = &GCSConfig{ + // Bucket: gcsConfig.GetString("bucket"), + // PrefixPath: gcsConfig.GetString("prefix_path"), + // EndpointURL: gcsConfig.GetString("endpoint_url"), + // } + // } + // } + // } + // + // if v.IsSet("minio_0") { + // minioConfig := v.Sub("minio_0") + // if minioConfig != nil { + // if globalConfig.StorageEngine.Minio == nil { + // globalConfig.StorageEngine.Minio = &MinioConfig{ + // Host: minioConfig.GetString("host"), + // User: minioConfig.GetString("user"), + // Password: minioConfig.GetString("password"), + // Secure: minioConfig.GetBool("secure"), + // PrefixPath: minioConfig.GetString("prefix_path"), + // Verify: minioConfig.GetBool("verify"), + // Bucket: minioConfig.GetString("bucket"), + // } + // } + // } + // } + // + // if v.IsSet("s3") { + // s3Config := v.Sub("s3") + // if s3Config != nil { + // if globalConfig.StorageEngine.S3 == nil { + // globalConfig.StorageEngine.S3 = &S3Config{ + // AccessKey: s3Config.GetString("access_key"), + // SecretKey: s3Config.GetString("secret_key"), + // Region: s3Config.GetString("region"), + // } + // } + // } + // } + // + // if v.IsSet("oss") { + // ossConfig := v.Sub("oss") + // if ossConfig != nil { + // if globalConfig.StorageEngine.OSS == nil { + // globalConfig.StorageEngine.OSS = &OSSConfig{ + // AccessKey: ossConfig.GetString("access_key"), + // SecretKey: ossConfig.GetString("secret_key"), + // EndpointURL: ossConfig.GetString("endpoint_url"), + // Region: ossConfig.GetString("region"), + // Bucket: ossConfig.GetString("bucket"), + // SignatureVersion: ossConfig.GetString("signature_version"), + // AddressingStyle: ossConfig.GetString("addressing_style"), + // } + // } + // } + // } + //} + // + //// Map user_default_llm section to UserDefaultLLMConfig + //if v.IsSet("user_default_llm") { + // userDefaultLLMConfig := v.Sub("user_default_llm") + // if userDefaultLLMConfig != nil { + // if defaultModels := userDefaultLLMConfig.Sub("default_models"); defaultModels != nil { + // globalConfig.UserDefaultLLM.DefaultModels.ChatModel = ModelConfig{ + // Name: defaultModels.GetString("chat_model.name"), + // APIKey: defaultModels.GetString("chat_model.api_key"), + // BaseURL: defaultModels.GetString("chat_model.base_url"), + // Factory: defaultModels.GetString("chat_model.factory"), + // } + // globalConfig.UserDefaultLLM.DefaultModels.EmbeddingModel = ModelConfig{ + // Name: defaultModels.GetString("embedding_model.name"), + // APIKey: defaultModels.GetString("embedding_model.api_key"), + // BaseURL: defaultModels.GetString("embedding_model.base_url"), + // Factory: defaultModels.GetString("embedding_model.factory"), + // } + // globalConfig.UserDefaultLLM.DefaultModels.RerankModel = ModelConfig{ + // Name: defaultModels.GetString("rerank_model.name"), + // APIKey: defaultModels.GetString("rerank_model.api_key"), + // BaseURL: defaultModels.GetString("rerank_model.base_url"), + // Factory: defaultModels.GetString("rerank_model.factory"), + // } + // globalConfig.UserDefaultLLM.DefaultModels.ASRModel = ModelConfig{ + // Name: defaultModels.GetString("asr_model.name"), + // APIKey: defaultModels.GetString("asr_model.api_key"), + // BaseURL: defaultModels.GetString("asr_model.base_url"), + // Factory: defaultModels.GetString("asr_model.factory"), + // } + // globalConfig.UserDefaultLLM.DefaultModels.Image2TextModel = ModelConfig{ + // Name: defaultModels.GetString("image2text_model.name"), + // APIKey: defaultModels.GetString("image2text_model.api_key"), + // BaseURL: defaultModels.GetString("image2text_model.base_url"), + // Factory: defaultModels.GetString("image2text_model.factory"), + // } + // } + // } + //} + return nil } // GetConfig gets the global configuration -func GetConfig() *Config { +func GetConfig() *config.Config { return globalConfig } -// GetAdminConfig gets the admin server configuration -func GetAdminConfig() *AdminConfig { - if globalConfig == nil { - return nil - } - return &globalConfig.Admin -} +// GetAdminServerConfig gets the admin server configuration +//func GetAdminServerConfig() *config.AdminConfig { +// if globalConfig == nil { +// return nil +// } +// return &globalConfig.Admin +//} // SetLogger sets the logger instance func SetLogger(l *zap.Logger) { @@ -1017,20 +982,3 @@ func getInt(m map[string]interface{}, key string) int { } return 0 } - -func GetLanguage() string { - lang := common.GetEnv(common.EnvLang) - if lang == "" { - lang = common.GetEnv(common.EnvLanguage) - } - - lang = strings.ToLower(lang) - - if strings.Contains(lang, "zh_") || - strings.Contains(lang, "zh-") || - strings.HasPrefix(lang, "zh") { - return "Chinese" - } - - return "English" -} diff --git a/internal/server/config/admin_config.go b/internal/server/config/admin_config.go new file mode 100644 index 0000000000..849726a01c --- /dev/null +++ b/internal/server/config/admin_config.go @@ -0,0 +1,58 @@ +// +// Copyright 2026 The InfiniFlow Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +package config + +import ( + "github.com/spf13/viper" +) + +type AdminConfig struct { + Host string `mapstructure:"host"` + HTTPPort int `mapstructure:"http_port"` +} + +func (c *Config) ParseAdminConfig(v *viper.Viper) error { + // Default Admin config + c.admin.Host = "localhost" + c.admin.HTTPPort = 9383 + + if !v.IsSet("admin") { + return nil + } + sub := v.Sub("admin") + if sub == nil { + return nil + } + + if sub.IsSet("host") { + c.admin.Host = sub.GetString("host") + } + + if sub.IsSet("http_port") { + c.admin.HTTPPort = sub.GetInt("http_port") + } + + if c.admin.HTTPPort == 9381 { + c.admin.HTTPPort = 9383 + } + + return nil +} + +func (c *Config) GetAdminServerConfig() AdminConfig { + return c.admin +} diff --git a/internal/server/config/analytic_engine_config.go b/internal/server/config/analytic_engine_config.go new file mode 100644 index 0000000000..35bce352b3 --- /dev/null +++ b/internal/server/config/analytic_engine_config.go @@ -0,0 +1,84 @@ +// +// Copyright 2026 The InfiniFlow Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +package config + +import ( + "fmt" + + "github.com/spf13/viper" +) + +type AnalyticEngineConfig struct { + Clickhouse ClickhouseConfig `mapstructure:"clickhouse"` +} + +type ClickhouseConfig struct { + Host string `mapstructure:"host"` + Port int `mapstructure:"port"` + User string `mapstructure:"user"` + Password string `mapstructure:"password"` + Database string `mapstructure:"database"` +} + +func (c *Config) ParseAnalyticEngineConfig(v *viper.Viper) error { + analyticEngineType := c.general.AnalyticEngine + var err error + switch analyticEngineType { + case "clickhouse": + err = c.parseClickhouseConfig(v) + default: + return fmt.Errorf("analytic engine type %s is not supported", analyticEngineType) + } + + return err +} + +func (c *Config) parseClickhouseConfig(v *viper.Viper) error { + // Default Clickhouse config + c.analyticEngine.Clickhouse.Host = "localhost" + c.analyticEngine.Clickhouse.Port = 9900 + c.analyticEngine.Clickhouse.User = "ragflow" + c.analyticEngine.Clickhouse.Password = "infini_rag_flow" + c.analyticEngine.Clickhouse.Database = "ragflow" + + if !v.IsSet("clickhouse") { + return nil + } + sub := v.Sub("clickhouse") + if sub == nil { + return nil + } + + if sub.IsSet("host") { + c.analyticEngine.Clickhouse.Host = sub.GetString("host") + } + + if sub.IsSet("port") { + c.analyticEngine.Clickhouse.Port = sub.GetInt("port") + } + + if sub.IsSet("user") { + c.analyticEngine.Clickhouse.User = sub.GetString("user") + } + if sub.IsSet("password") { + c.analyticEngine.Clickhouse.Password = sub.GetString("password") + } + if sub.IsSet("database") { + c.analyticEngine.Clickhouse.Database = sub.GetString("database") + } + return nil +} diff --git a/internal/server/config/api_server_config.go b/internal/server/config/api_server_config.go new file mode 100644 index 0000000000..cd4bfbd064 --- /dev/null +++ b/internal/server/config/api_server_config.go @@ -0,0 +1,95 @@ +// +// Copyright 2026 The InfiniFlow Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +package config + +import "github.com/spf13/viper" + +type AuthenticationConfig struct { + DisablePasswordLogin bool `mapstructure:"disable_password_login"` + RegisterEnabled bool `mapstructure:"register_enabled"` +} + +type APIServerConfig struct { + Host string `mapstructure:"host"` + HTTPPort int `mapstructure:"http_port"` + + Authentication AuthenticationConfig `mapstructure:"authentication"` +} + +func (c *Config) ParseAPIServerConfig(v *viper.Viper) error { + // Default Admin config + c.apiServer.Host = "localhost" + c.apiServer.HTTPPort = 9384 + + if !v.IsSet("ragflow") { + return nil + } + sub := v.Sub("ragflow") + if sub == nil { + return nil + } + + if sub.IsSet("host") { + c.apiServer.Host = sub.GetString("host") + } + + if sub.IsSet("http_port") { + c.apiServer.HTTPPort = sub.GetInt("http_port") + } + + if c.apiServer.HTTPPort == 9380 { + c.apiServer.HTTPPort = 9384 + } + + c.parseAuthenticationConfig(v) + + return nil +} + +func (c *Config) parseAuthenticationConfig(v *viper.Viper) { + apiServerConfig := &c.apiServer + apiServerConfig.Authentication.DisablePasswordLogin = false + apiServerConfig.Authentication.RegisterEnabled = true + + if !v.IsSet("authentication") { + return + } + sub := v.Sub("authentication") + if sub == nil { + return + } + + if sub.IsSet("disable_password_login") { + apiServerConfig.Authentication.DisablePasswordLogin = sub.GetBool("disable_password_login") + } + + if sub.IsSet("enable_register") { + apiServerConfig.Authentication.RegisterEnabled = sub.GetBool("enable_register") + } +} + +func (c *Config) DisablePasswordLogin() bool { + return c.apiServer.Authentication.DisablePasswordLogin +} + +func (c *Config) RegisterEnabled() bool { + return c.apiServer.Authentication.RegisterEnabled +} + +func (c *Config) GetAPIServerConfig() APIServerConfig { + return c.apiServer +} diff --git a/internal/server/config/base.go b/internal/server/config/base.go new file mode 100644 index 0000000000..7a769b1636 --- /dev/null +++ b/internal/server/config/base.go @@ -0,0 +1,46 @@ +// +// Copyright 2026 The InfiniFlow Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +package config + +import "ragflow/internal/common" + +type Config struct { + general GeneralConfig + database DatabaseConfig + docEngine DocEngineConfig + storageEngine StorageConfig + cacheEngine CacheEngineConfig + queueEngine QueueEngineConfig + analyticEngine AnalyticEngineConfig + oTel OpenTelemetryConfig + + admin AdminConfig + apiServer APIServerConfig + ingestor IngestorConfig + syncer SyncerConfig + + log LogConfig + smtp common.SMTPConfig + + // From environments + environments Environments + + // For EE + defaultModels DefaultModelsConfig + billing BillingConfig + oAuth OAuthConfig +} diff --git a/internal/server/config/billing_ee.go b/internal/server/config/billing_ee.go new file mode 100644 index 0000000000..f5ecb03be1 --- /dev/null +++ b/internal/server/config/billing_ee.go @@ -0,0 +1,26 @@ +// +// Copyright 2026 The InfiniFlow Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +package config + +import "github.com/spf13/viper" + +type BillingConfig struct { +} + +func (c *Config) ParseBillingConfig(v *viper.Viper) error { + return nil +} diff --git a/internal/server/config/cache_engine_config.go b/internal/server/config/cache_engine_config.go new file mode 100644 index 0000000000..35274a680d --- /dev/null +++ b/internal/server/config/cache_engine_config.go @@ -0,0 +1,105 @@ +// +// Copyright 2026 The InfiniFlow Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +package config + +import ( + "fmt" + "strconv" + "strings" + + "github.com/spf13/viper" +) + +type CacheEngineConfig struct { + Redis RedisConfig `mapstructure:"redis"` +} + +// RedisConfig Redis configuration +type RedisConfig struct { + Host string `mapstructure:"host"` + Port int `mapstructure:"port"` + Username string `mapstructure:"username"` + Password string `mapstructure:"password"` + DB int `mapstructure:"db"` +} + +func (c *Config) ParseCacheEngineConfig(v *viper.Viper) error { + cacheEngineType := c.general.CacheEngine + var err error + switch cacheEngineType { + case "redis": + err = c.parseRedisConfig(v) + default: + return fmt.Errorf("cache engine type %s is not supported", cacheEngineType) + } + + return err +} + +func (c *Config) parseRedisConfig(v *viper.Viper) error { + // Default Redis config + c.cacheEngine.Redis.Host = "localhost" + c.cacheEngine.Redis.Port = 6379 + c.cacheEngine.Redis.DB = 1 + c.cacheEngine.Redis.Username = "" + c.cacheEngine.Redis.Password = "infini_rag_flow" + + if !v.IsSet("redis") { + return nil + } + sub := v.Sub("redis") + if sub == nil { + return nil + } + + if sub.IsSet("host") { + hostStr := sub.GetString("host") + // Handle host:port format (e.g., "localhost:6379") + if hostStr == "" { + return fmt.Errorf("empty host of Redis configuration") + } + + if idx := strings.LastIndex(hostStr, ":"); idx != -1 { + c.cacheEngine.Redis.Host = hostStr[:idx] + if portStr := hostStr[idx+1:]; portStr != "" { + if port, err := strconv.Atoi(portStr); err == nil { + c.cacheEngine.Redis.Port = port + } + } + } else { + return fmt.Errorf("error address format of Redis: %s", hostStr) + } + } + + if sub.IsSet("db") { + c.cacheEngine.Redis.DB = sub.GetInt("db") + } + + if sub.IsSet("username") { + c.cacheEngine.Redis.Username = sub.GetString("username") + } + + if sub.IsSet("password") { + c.cacheEngine.Redis.Password = sub.GetString("password") + } + + return nil +} + +func (c *Config) GetRedisConfig() RedisConfig { + return c.cacheEngine.Redis +} diff --git a/internal/server/config/database_config.go b/internal/server/config/database_config.go new file mode 100644 index 0000000000..a012ac8dda --- /dev/null +++ b/internal/server/config/database_config.go @@ -0,0 +1,113 @@ +// +// Copyright 2026 The InfiniFlow Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +package config + +import ( + "fmt" + + "github.com/spf13/viper" +) + +// DatabaseConfig database configuration +type DatabaseConfig struct { + MySQL MySQLConfig `mapstructure:"mysql"` +} + +type MySQLConfig struct { + DatabaseName string `mapstructure:"name"` // database name + User string `mapstructure:"user"` + Password string `mapstructure:"password"` + Host string `mapstructure:"host"` + Port int `mapstructure:"port"` + MaxConnections int `mapstructure:"max_connections"` + StaleTimeout int `mapstructure:"stale_timeout"` + MaxAllowedPacket int `mapstructure:"max_allowed_packet"` + Charset string `mapstructure:"charset"` +} + +func (c *Config) ParseDatabaseConfig(v *viper.Viper) error { + databaseType := c.general.Database + switch databaseType { + case "mysql": + c.parseMySQLConfig(v) + default: + return fmt.Errorf("database type %s is not supported", databaseType) + } + return nil +} + +func (c *Config) parseMySQLConfig(v *viper.Viper) { + + // Default MySQL config + c.database.MySQL.DatabaseName = "rag_flow" + c.database.MySQL.User = "root" + c.database.MySQL.Password = "infini_rag_flow" + c.database.MySQL.Host = "localhost" + c.database.MySQL.Port = 3306 + c.database.MySQL.MaxConnections = 900 + c.database.MySQL.StaleTimeout = 300 + c.database.MySQL.MaxAllowedPacket = 1073741824 + c.database.MySQL.Charset = "utf8mb4" + + if !v.IsSet("mysql") { + return + } + sub := v.Sub("mysql") + if sub == nil { + return + } + + if sub.IsSet("name") { + c.database.MySQL.DatabaseName = sub.GetString("name") + } + + if sub.IsSet("user") { + c.database.MySQL.User = sub.GetString("user") + } + + if sub.IsSet("password") { + c.database.MySQL.Password = sub.GetString("password") + } + + if sub.IsSet("host") { + c.database.MySQL.Host = sub.GetString("host") + } + + if sub.IsSet("port") { + c.database.MySQL.Port = sub.GetInt("port") + } + + if sub.IsSet("max_connections") { + c.database.MySQL.MaxConnections = sub.GetInt("max_connections") + } + + if sub.IsSet("stale_timeout") { + c.database.MySQL.StaleTimeout = sub.GetInt("stale_timeout") + } + + if sub.IsSet("max_allowed_packet") { + c.database.MySQL.MaxAllowedPacket = sub.GetInt("max_allowed_packet") + } + + if sub.IsSet("charset") { + c.database.MySQL.Charset = sub.GetString("charset") + } +} + +func (c *Config) GetMySQLConfig() MySQLConfig { + return c.database.MySQL +} diff --git a/internal/server/config/doc_engine_config.go b/internal/server/config/doc_engine_config.go new file mode 100644 index 0000000000..361e876834 --- /dev/null +++ b/internal/server/config/doc_engine_config.go @@ -0,0 +1,124 @@ +// +// Copyright 2026 The InfiniFlow Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +package config + +import ( + "github.com/spf13/viper" +) + +type DocEngineConfig struct { + ES ElasticsearchConfig `mapstructure:"es"` + Infinity InfinityConfig `mapstructure:"infinity"` +} + +// ElasticsearchConfig Elasticsearch configuration +type ElasticsearchConfig struct { + Hosts string `mapstructure:"hosts"` + Username string `mapstructure:"username"` + Password string `mapstructure:"password"` +} + +// InfinityConfig Infinity configuration +type InfinityConfig struct { + URI string `mapstructure:"uri"` + PostgresPort int `mapstructure:"postgres_port"` + DBName string `mapstructure:"db_name"` + MappingFileName string `mapstructure:"mapping_file_name"` + DocMetaMappingFileName string `mapstructure:"doc_meta_mapping_file_name"` +} + +func (c *Config) ParseDocEngineConfig(v *viper.Viper) error { + c.parseInfinityConfig(v) + c.parseElasticsearchConfig(v) + return nil +} + +func (c *Config) parseInfinityConfig(v *viper.Viper) { + // Default Infinity config + c.docEngine.Infinity.URI = "localhost:23817" + c.docEngine.Infinity.PostgresPort = 5432 + c.docEngine.Infinity.DBName = "default_db" + c.docEngine.Infinity.MappingFileName = "infinity_mapping.json" + c.docEngine.Infinity.DocMetaMappingFileName = "doc_meta_infinity_mapping.json" + + if !v.IsSet("infinity") { + return + } + sub := v.Sub("infinity") + if sub == nil { + return + } + + if sub.IsSet("uri") { + c.docEngine.Infinity.URI = sub.GetString("uri") + } + + if sub.IsSet("postgres_port") { + c.docEngine.Infinity.PostgresPort = sub.GetInt("postgres_port") + } + + if sub.IsSet("db_name") { + c.docEngine.Infinity.DBName = sub.GetString("db_name") + } + + if sub.IsSet("mapping_file_name") { + c.docEngine.Infinity.MappingFileName = sub.GetString("mapping_file_name") + } + + if sub.IsSet("doc_meta_mapping_file_name") { + c.docEngine.Infinity.DocMetaMappingFileName = sub.GetString("doc_meta_mapping_file_name") + } +} + +func (c *Config) parseElasticsearchConfig(v *viper.Viper) { + // Default Elasticsearch config + c.docEngine.ES.Hosts = "http://localhost:1200" + c.docEngine.ES.Username = "elastic" + c.docEngine.ES.Password = "infini_rag_flow" + + if !v.IsSet("es") { + return + } + sub := v.Sub("es") + if sub == nil { + return + } + + if sub.IsSet("hosts") { + c.docEngine.ES.Hosts = sub.GetString("hosts") + } + + if sub.IsSet("username") { + c.docEngine.ES.Username = sub.GetString("username") + } + + if sub.IsSet("password") { + c.docEngine.ES.Password = sub.GetString("password") + } +} + +func (c *Config) GetElasticsearchConfig() ElasticsearchConfig { + return c.docEngine.ES +} + +func (c *Config) IsElasticConfigured() bool { + return c.docEngine.ES.Hosts != "" +} + +func (c *Config) GetInfinityConfig() InfinityConfig { + return c.docEngine.Infinity +} diff --git a/internal/server/config/environments.go b/internal/server/config/environments.go new file mode 100644 index 0000000000..a5becdc8da --- /dev/null +++ b/internal/server/config/environments.go @@ -0,0 +1,167 @@ +// +// Copyright 2026 The InfiniFlow Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +package config + +import ( + "fmt" + "net/mail" + "ragflow/internal/common" + "strings" +) + +type Environments struct { + Language string `mapstructure:"language"` + + SecretKey string `mapstructure:"secret_key"` + + RegisterEnabled bool `mapstructure:"register_enabled"` + DisablePasswordLogin bool `mapstructure:"disable_password_login"` + + DocumentEngineType string `mapstructure:"document_engine_type"` + DatabaseType string `mapstructure:"database_type"` + + StorageType string `mapstructure:"storage_type"` + MinioHost string `mapstructure:"minio_host"` + MinioRegion string `mapstructure:"minio_region"` + + CreateDefaultSuperUser bool `mapstructure:"create_default_super_user"` + DefaultSuperUser DefaultSuperUser `mapstructure:"default_super_user"` +} + +type DefaultSuperUser struct { + Email string `mapstructure:"email"` + Password string `mapstructure:"password"` + Nickname string `mapstructure:"nickname"` +} + +func (c *Config) GetEnvironments() error { + // Secret key + if envVal := common.GetEnv(common.EnvRAGFlowSecretKey); envVal != "" { + c.environments.SecretKey = envVal + } + + // Load REGISTER_ENABLED from environment variable (default: true) + if envVal := common.GetEnv(common.EnvRegisterEnabled); envVal != "" { + str := strings.ToLower(envVal) + if str == "true" || str == "1" || str == "yes" { + c.environments.RegisterEnabled = true + } else { + c.environments.RegisterEnabled = false + } + } + + // Load DISABLE_PASSWORD_LOGIN from environment variable (default: false) + if envVal := common.GetEnv(common.EnvDisablePasswordLogin); envVal != "" { + str := strings.ToLower(envVal) + if str == "true" || str == "1" || str == "yes" { + c.environments.DisablePasswordLogin = true + } else { + c.environments.DisablePasswordLogin = false + } + } + + // Doc engine + docEngine := common.GetEnvSmall(common.EnvDocEngine) + if docEngine != "" { + switch docEngine { + case "infinity", "elasticsearch": + c.environments.DocumentEngineType = docEngine + case "opensearch", "oceanbase": + return fmt.Errorf("not implemented: %s", docEngine) + default: + return fmt.Errorf("invalid doc engine: %s", docEngine) + } + } + + // Default super user email + superUserEmail := common.GetEnv(common.EnvDefaultSuperuserEmail) + if superUserEmail != "" { + _, err := mail.ParseAddress(superUserEmail) + if err != nil { + return fmt.Errorf("invalid super user email: %s", superUserEmail) + } + c.environments.DefaultSuperUser.Email = superUserEmail + } + + superUserPassword := common.GetEnv(common.EnvDefaultSuperuserPassword) + if superUserPassword != "" { + c.environments.DefaultSuperUser.Password = superUserPassword + } + + superUserNickname := common.GetEnv(common.EnvDefaultSuperuserNickname) + if superUserNickname != "" { + c.environments.DefaultSuperUser.Nickname = superUserNickname + } + + // Meta database + databaseType := common.GetEnvSmall(common.EnvDBType) + if databaseType != "" { + switch databaseType { + case "mysql": + c.environments.DatabaseType = "mysql" + default: + return fmt.Errorf("invalid database type: %s", databaseType) + } + } + + // Storage + storageType := common.GetEnvSmall(common.EnvStorageImpl) + if storageType != "" { + switch storageType { + case "minio", "s3", "oss", "gcs": + c.environments.StorageType = storageType + default: + return fmt.Errorf("invalid storage type: %s", storageType) + } + } + + // Minio + minioHost := strings.ToLower(common.GetEnv(common.EnvMinioHost)) + if minioHost != "" { + c.environments.MinioHost = minioHost + } + + minioRegion := strings.ToLower(common.GetEnv(common.EnvMinioRegion)) + if minioRegion != "" { + c.environments.MinioRegion = minioRegion + } + + c.environments.Language = GetLanguage() + + return nil +} + +func GetLanguage() string { + lang := common.GetEnv(common.EnvLang) + if lang == "" { + lang = common.GetEnv(common.EnvLanguage) + } + + lang = strings.ToLower(lang) + + if strings.Contains(lang, "zh_") || + strings.Contains(lang, "zh-") || + strings.HasPrefix(lang, "zh") { + return "Chinese" + } + + return "English" +} + +func (c *Config) GetSecretKey() string { + return c.environments.SecretKey +} diff --git a/internal/server/config/general_config.go b/internal/server/config/general_config.go new file mode 100644 index 0000000000..4482b3aa9a --- /dev/null +++ b/internal/server/config/general_config.go @@ -0,0 +1,122 @@ +// +// Copyright 2026 The InfiniFlow Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +package config + +import ( + "time" + + "github.com/spf13/viper" +) + +// GeneralConfig general configuration +type GeneralConfig struct { + HeartbeatInterval time.Duration `mapstructure:"heartbeat_interval"` + Mode string `mapstructure:"mode"` // debug, release + Database string `mapstructure:"database"` + DocEngine string `mapstructure:"doc_engine"` // Infinity, Elasticsearch + StorageEngine string `mapstructure:"storage_engine"` // Minio, S3 + CacheEngine string `mapstructure:"cache_engine"` // Redis + QueueEngine string `mapstructure:"queue_engine"` // NATS + AnalyticEngine string `mapstructure:"analytic_engine"` // Clickhouse + Language string `mapstructure:"language"` +} + +func (c *Config) ParseGeneralConfig(v *viper.Viper) error { + + // Default General config + c.general.HeartbeatInterval = 3 + c.general.Mode = "release" + c.general.Database = "mysql" + c.general.DocEngine = "elasticsearch" + c.general.StorageEngine = "minio" + c.general.CacheEngine = "redis" + c.general.QueueEngine = "nats" + c.general.AnalyticEngine = "clickhouse" + c.general.Language = "english" + + if !v.IsSet("general") { + return nil + } + sub := v.Sub("general") + if sub == nil { + return nil + } + + if sub.IsSet("heartbeat_interval") { + c.general.HeartbeatInterval = sub.GetDuration("heartbeat_interval") + } + if sub.IsSet("mode") { + c.general.Mode = sub.GetString("mode") + } + if sub.IsSet("database") { + c.general.Database = sub.GetString("database") + } + if sub.IsSet("doc_engine") { + c.general.DocEngine = sub.GetString("doc_engine") + } + if sub.IsSet("storage_engine") { + c.general.StorageEngine = sub.GetString("storage_engine") + } + if sub.IsSet("cache_engine") { + c.general.CacheEngine = sub.GetString("cache_engine") + } + if sub.IsSet("queue_engine") { + c.general.QueueEngine = sub.GetString("queue_engine") + } + if sub.IsSet("analytic_engine") { + c.general.AnalyticEngine = sub.GetString("analytic_engine") + } + if sub.IsSet("language") { + c.general.Language = sub.GetString("language") + } + return nil +} + +func (c *Config) GetHeartbeatInterval() time.Duration { + return c.general.HeartbeatInterval +} + +func (c *Config) GetMode() string { + return c.general.Mode +} + +func (c *Config) DatabaseType() string { + return c.general.Database +} + +func (c *Config) DocEngineType() string { + if c.environments.DocumentEngineType != "" { + return c.environments.DocumentEngineType + } + return c.general.DocEngine +} + +func (c *Config) StorageEngineType() string { + return c.general.StorageEngine +} + +func (c *Config) CacheEngineType() string { + return c.general.CacheEngine +} + +func (c *Config) QueueEngineType() string { + return c.general.QueueEngine +} + +func (c *Config) AnalyticEngineType() string { + return c.general.AnalyticEngine +} diff --git a/internal/server/config/ingestor_config.go b/internal/server/config/ingestor_config.go new file mode 100644 index 0000000000..7ea6edb0e5 --- /dev/null +++ b/internal/server/config/ingestor_config.go @@ -0,0 +1,42 @@ +// +// Copyright 2026 The InfiniFlow Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +package config + +import "github.com/spf13/viper" + +type IngestorConfig struct { + MaxConcurrentWorkers int `mapstructure:"max_concurrent_workers"` +} + +func (c *Config) ParseIngestorConfig(v *viper.Viper) error { + // Default Ingestor config + c.ingestor.MaxConcurrentWorkers = 1 + + if !v.IsSet("ingestor") { + return nil + } + sub := v.Sub("ingestor") + if sub == nil { + return nil + } + + if sub.IsSet("max_concurrent_workers") { + c.ingestor.MaxConcurrentWorkers = sub.GetInt("max_concurrent_workers") + } + + return nil +} diff --git a/internal/server/config/log_config.go b/internal/server/config/log_config.go new file mode 100644 index 0000000000..b6d4406a75 --- /dev/null +++ b/internal/server/config/log_config.go @@ -0,0 +1,103 @@ +// +// Copyright 2026 The InfiniFlow Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +package config + +import "github.com/spf13/viper" + +// LogConfig logging configuration. +// +// Path, MaxSize, MaxBackups, MaxAge, and Compress configure the rotated +// log file. The cmd/* entry points hardcode per-service defaults +// (e.g. "server_main.log" for the API server, "admin_server.log" for +// the admin server, "ingestion_server.log" for the ingestion worker), +// so a typical deployment gets a rotated file without any YAML +// configuration. When Path is empty (the default) the binary's +// hardcoded default filename is used — it does NOT disable file +// output. Set log.path in service_conf.yaml to override the +// per-service default filename. +// +// Compress is a pointer so callers can distinguish "not set" (nil, +// defaults to true) from "explicitly false" (*bool=false). All other +// numeric fields use plain int because their zero values are sensible +// defaults (100 MB / 10 files / 30 days) and there is no operator-meaningful +// reason to distinguish "not set" from "0". +type LogConfig struct { + Level string `mapstructure:"level"` // debug, info, warn, error; default "info" + Format string `mapstructure:"format"` // json, text (reserved for future use); default "json" + Path string `mapstructure:"path"` // per-binary file override; empty = use cmd/* hardcoded default + MaxSize int `mapstructure:"max_size"` // MB before rotation; default 100 + MaxBackups int `mapstructure:"max_backups"` // retained rotated files; default 10 + MaxAge int `mapstructure:"max_age"` // days; default 30 + Compress bool `mapstructure:"compress"` // gzip rotated files; default false +} + +func (c *Config) ParseLogConfig(v *viper.Viper) error { + // Default Log config + c.log.Level = "info" + c.log.Format = "json" + c.log.Path = "logs" + c.log.MaxSize = 100 * 1024 * 1024 // 1024MB + c.log.MaxBackups = 10 + c.log.MaxAge = 30 + c.log.Compress = false + + if !v.IsSet("log") { + return nil + } + sub := v.Sub("log") + if sub == nil { + return nil + } + + if sub.IsSet("level") { + c.log.Level = sub.GetString("level") + } + + if sub.IsSet("format") { + c.log.Format = sub.GetString("format") + } + + if sub.IsSet("path") { + c.log.Path = sub.GetString("path") + } + + if sub.IsSet("max_size") { + c.log.MaxSize = sub.GetInt("max_size") + } + + if sub.IsSet("max_backups") { + c.log.MaxBackups = sub.GetInt("max_backups") + } + + if sub.IsSet("max_age") { + c.log.MaxAge = sub.GetInt("max_age") + } + + if sub.IsSet("compress") { + c.log.Compress = sub.GetBool("compress") + } + + return nil +} + +func (c *Config) GetLogConfig() LogConfig { + return c.log +} + +func (c *Config) SetLogLevel(level string) { + c.log.Level = level +} diff --git a/internal/server/config/model_config_ee.go b/internal/server/config/model_config_ee.go new file mode 100644 index 0000000000..504991a3f0 --- /dev/null +++ b/internal/server/config/model_config_ee.go @@ -0,0 +1,70 @@ +// +// Copyright 2026 The InfiniFlow Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +package config + +import "github.com/spf13/viper" + +// DefaultModelsConfig default models configuration +type DefaultModelsConfig struct { + ChatModel ModelConfig `mapstructure:"chat_model"` + EmbeddingModel ModelConfig `mapstructure:"embedding_model"` + RerankModel ModelConfig `mapstructure:"rerank_model"` + ASRModel ModelConfig `mapstructure:"asr_model"` + VisionModel ModelConfig `mapstructure:"vision_model"` + OCRModel ModelConfig `mapstructure:"ocr_model"` + TTSModel ModelConfig `mapstructure:"tts_model"` +} + +// ModelConfig model configuration +type ModelConfig struct { + Name string `mapstructure:"name"` + APIKey string `mapstructure:"api_key"` + BaseURL string `mapstructure:"base_url"` + Factory string `mapstructure:"factory"` +} + +func (c *Config) ParseDefaultModelsConfig(v *viper.Viper) error { + return nil +} + +func (c *Config) GetDefaultChatModel() ModelConfig { + return c.defaultModels.ChatModel +} + +func (c *Config) GetDefaultEmbeddingModel() ModelConfig { + return c.defaultModels.EmbeddingModel +} + +func (c *Config) GetDefaultRerankModel() ModelConfig { + return c.defaultModels.RerankModel +} + +func (c *Config) GetDefaultASRModel() ModelConfig { + return c.defaultModels.ASRModel +} + +func (c *Config) GetDefaultVisionModel() ModelConfig { + return c.defaultModels.VisionModel +} + +func (c *Config) GetDefaultOCRModel() ModelConfig { + return c.defaultModels.OCRModel +} + +func (c *Config) GetDefaultTTSModel() ModelConfig { + return c.defaultModels.TTSModel +} diff --git a/internal/server/config/oauth_ee.go b/internal/server/config/oauth_ee.go new file mode 100644 index 0000000000..87a7020668 --- /dev/null +++ b/internal/server/config/oauth_ee.go @@ -0,0 +1,26 @@ +// +// Copyright 2026 The InfiniFlow Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +package config + +import "github.com/spf13/viper" + +type OAuthConfig struct { +} + +func (c *Config) ParseOAuthConfig(v *viper.Viper) error { + return nil +} diff --git a/internal/server/config/open_telemetry_config.go b/internal/server/config/open_telemetry_config.go new file mode 100644 index 0000000000..14335566ad --- /dev/null +++ b/internal/server/config/open_telemetry_config.go @@ -0,0 +1,72 @@ +// +// Copyright 2026 The InfiniFlow Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +package config + +import "github.com/spf13/viper" + +type OpenTelemetryConfig struct { + Host string `mapstructure:"host"` + Port int `mapstructure:"port"` + Secure bool `mapstructure:"secure"` + SampleRatio float64 `mapstructure:"sample_ratio"` + Stdout bool `mapstructure:"stdout"` + Enable bool `mapstructure:"enable"` +} + +func (c *Config) ParseOpenTelemetryConfig(v *viper.Viper) error { + // Default OpenTelemetry config + c.oTel.Host = "localhost" + c.oTel.Port = 4318 + c.oTel.Secure = false + c.oTel.SampleRatio = 1.0 + c.oTel.Stdout = false + c.oTel.Enable = false + + if !v.IsSet("otel") { + return nil + } + sub := v.Sub("otel") + if sub == nil { + return nil + } + + if sub.IsSet("host") { + c.oTel.Host = sub.GetString("host") + } + + if sub.IsSet("port") { + c.oTel.Port = sub.GetInt("port") + } + + if sub.IsSet("secure") { + c.oTel.Secure = sub.GetBool("secure") + } + + if sub.IsSet("sample_ratio") { + c.oTel.SampleRatio = sub.GetFloat64("sample_ratio") + } + + if sub.IsSet("stdout") { + c.oTel.Stdout = sub.GetBool("stdout") + } + + if sub.IsSet("enable") { + c.oTel.Enable = sub.GetBool("enable") + } + + return nil +} diff --git a/internal/server/config/queue_engine_config.go b/internal/server/config/queue_engine_config.go new file mode 100644 index 0000000000..a4e364c909 --- /dev/null +++ b/internal/server/config/queue_engine_config.go @@ -0,0 +1,74 @@ +// +// Copyright 2026 The InfiniFlow Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +package config + +import ( + "fmt" + + "github.com/spf13/viper" +) + +type QueueEngineConfig struct { + NATS NATSConfig `mapstructure:"nats"` +} + +// NATSConfig NATS queue configuration +type NATSConfig struct { + Host string `mapstructure:"host"` + Port int `mapstructure:"port"` +} + +func (c *Config) ParseQueueEngineConfig(v *viper.Viper) error { + queueEngineType := c.general.QueueEngine + var err error + switch queueEngineType { + case "nats": + err = c.parseNATSConfig(v) + default: + return fmt.Errorf("queue engine type %s is not supported", queueEngineType) + } + + return err +} + +func (c *Config) parseNATSConfig(v *viper.Viper) error { + // Default NATS config + c.queueEngine.NATS.Host = "localhost" + c.queueEngine.NATS.Port = 4222 + + if !v.IsSet("nats") { + return nil + } + sub := v.Sub("nats") + if sub == nil { + return nil + } + + if sub.IsSet("host") { + c.queueEngine.NATS.Host = sub.GetString("host") + } + + if sub.IsSet("port") { + c.queueEngine.NATS.Port = sub.GetInt("port") + } + + return nil +} + +func (c *Config) GetNATSConfig() NATSConfig { + return c.queueEngine.NATS +} diff --git a/internal/server/config/smtp_config.go b/internal/server/config/smtp_config.go new file mode 100644 index 0000000000..bb3dfa1ca8 --- /dev/null +++ b/internal/server/config/smtp_config.go @@ -0,0 +1,86 @@ +// +// Copyright 2026 The InfiniFlow Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +package config + +import ( + "ragflow/internal/common" + + "github.com/spf13/viper" +) + +func (c *Config) ParseSMTPConfig(v *viper.Viper) error { + // Default SMTP config + c.smtp.MailServer = "" + c.smtp.MailPort = 465 + c.smtp.MailUseSSL = true + c.smtp.MailUseTLS = false + c.smtp.MailUsername = "" + c.smtp.MailPassword = "" + c.smtp.MailFromName = "RAGFlow" + c.smtp.MailFromAddress = "" + c.smtp.MailFrontendURL = "https://your-frontend.example.com" + + if !v.IsSet("smtp") { + return nil + } + sub := v.Sub("smtp") + if sub == nil { + return nil + } + + if sub.IsSet("mail_server") { + c.smtp.MailServer = sub.GetString("mail_server") + } + + if sub.IsSet("mail_port") { + c.smtp.MailPort = sub.GetInt("mail_port") + } + + if sub.IsSet("mail_use_ssl") { + c.smtp.MailUseSSL = sub.GetBool("mail_use_ssl") + } + + if sub.IsSet("mail_use_tls") { + c.smtp.MailUseTLS = sub.GetBool("mail_use_tls") + } + + if sub.IsSet("mail_username") { + c.smtp.MailUsername = sub.GetString("mail_username") + } + + if sub.IsSet("mail_password") { + c.smtp.MailPassword = sub.GetString("mail_password") + } + + if sub.IsSet("mail_from_name") { + c.smtp.MailFromName = sub.GetString("mail_from_name") + } + + if sub.IsSet("mail_from_address") { + c.smtp.MailFromAddress = sub.GetString("mail_from_address") + } + + if sub.IsSet("mail_frontend_url") { + c.smtp.MailFrontendURL = sub.GetString("mail_frontend_url") + } + + return nil +} + +func (c *Config) GetSMTPConfig() common.SMTPConfig { + return c.smtp +} diff --git a/internal/server/config/storage_engine_config.go b/internal/server/config/storage_engine_config.go new file mode 100644 index 0000000000..293db4a2bb --- /dev/null +++ b/internal/server/config/storage_engine_config.go @@ -0,0 +1,297 @@ +// +// Copyright 2026 The InfiniFlow Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +package config + +import ( + "fmt" + + "github.com/spf13/viper" +) + +// StorageConfig holds all storage-related configurations +type StorageConfig struct { + Minio MinioConfig `mapstructure:"minio"` + S3 S3Config `mapstructure:"s3"` + OSS OSSConfig `mapstructure:"oss"` + GCS GCSConfig `mapstructure:"gcs"` +} + +func (c *Config) ParseStorageEngineConfig(v *viper.Viper) error { + switch c.general.StorageEngine { + case "minio": + c.parseMinioConfig(v) + case "s3": + c.parseS3Config(v) + case "oss": + c.parseOSSConfig(v) + case "gcs": + c.parseGCSConfig(v) + default: + return fmt.Errorf("invalid storage type: %s", c.general.StorageEngine) + } + + return nil +} + +// MinioConfig holds MinIO storage configuration +type MinioConfig struct { + Host string `mapstructure:"host"` // MinIO server host (e.g., "localhost:9000") + User string `mapstructure:"user"` // Access key + Password string `mapstructure:"password"` // Secret key + Bucket string `mapstructure:"bucket"` // Default bucket (optional) + PrefixPath string `mapstructure:"prefix_path"` // Path prefix (optional) + Secure bool `mapstructure:"secure"` // Use HTTPS + Verify bool `mapstructure:"verify"` // Verify SSL certificates + Region string `mapstructure:"region"` // optional +} + +func (c *Config) parseMinioConfig(v *viper.Viper) { + // Default MinIO config + c.storageEngine.Minio.Host = "localhost:23817" + c.storageEngine.Minio.User = "rag_flow" + c.storageEngine.Minio.Password = "infini_rag_flow" + c.storageEngine.Minio.Bucket = "" + c.storageEngine.Minio.PrefixPath = "" + c.storageEngine.Minio.Secure = false + c.storageEngine.Minio.Verify = false + c.storageEngine.Minio.Region = "" + + if !v.IsSet("minio") { + return + } + sub := v.Sub("minio") + if sub == nil { + return + } + + if sub.IsSet("host") { + c.storageEngine.Minio.Host = sub.GetString("host") + } + + if sub.IsSet("user") { + c.storageEngine.Minio.User = sub.GetString("user") + } + + if sub.IsSet("password") { + c.storageEngine.Minio.Password = sub.GetString("password") + } + + if sub.IsSet("bucket") { + c.storageEngine.Minio.Bucket = sub.GetString("bucket") + } + + if sub.IsSet("prefix_path") { + c.storageEngine.Minio.PrefixPath = sub.GetString("prefix_path") + } + + if sub.IsSet("secure") { + c.storageEngine.Minio.Secure = sub.GetBool("secure") + } + + if sub.IsSet("verify") { + c.storageEngine.Minio.Verify = sub.GetBool("verify") + } + + if sub.IsSet("region") { + c.storageEngine.Minio.Region = sub.GetString("region") + } +} + +// S3Config holds AWS S3 storage configuration +type S3Config struct { + AccessKey string `mapstructure:"access_key"` // AWS Access Key ID + SecretKey string `mapstructure:"secret_key"` // AWS Secret Access Key + Region string `mapstructure:"region_name"` // AWS Region + SessionToken string `mapstructure:"session_token"` // AWS Session Token (optional) + EndpointURL string `mapstructure:"endpoint_url"` // Custom endpoint (optional) + SignatureVersion string `mapstructure:"signature_version"` // Signature version + AddressingStyle string `mapstructure:"addressing_style"` // Addressing style + Bucket string `mapstructure:"bucket"` // Default bucket (optional) + PrefixPath string `mapstructure:"prefix_path"` // Path prefix (optional) +} + +func (c *Config) parseS3Config(v *viper.Viper) { + // Default S3 config + c.storageEngine.S3.AccessKey = "" + c.storageEngine.S3.SecretKey = "" + c.storageEngine.S3.Region = "" + c.storageEngine.S3.SessionToken = "" + c.storageEngine.S3.EndpointURL = "" + c.storageEngine.S3.SignatureVersion = "v4" + c.storageEngine.S3.AddressingStyle = "path" + c.storageEngine.S3.Bucket = "" + c.storageEngine.S3.PrefixPath = "" + + if !v.IsSet("s3") { + return + } + sub := v.Sub("s3") + if sub == nil { + return + } + + if sub.IsSet("access_key") { + c.storageEngine.S3.AccessKey = sub.GetString("access_key") + } + + if sub.IsSet("secret_key") { + c.storageEngine.S3.SecretKey = sub.GetString("secret_key") + } + + if sub.IsSet("region") { + c.storageEngine.S3.Region = sub.GetString("region") + } + + if sub.IsSet("session_token") { + c.storageEngine.S3.SessionToken = sub.GetString("session_token") + } + + if sub.IsSet("endpoint_url") { + c.storageEngine.S3.EndpointURL = sub.GetString("endpoint_url") + } + + if sub.IsSet("signature_version") { + c.storageEngine.S3.SignatureVersion = sub.GetString("signature_version") + } + + if sub.IsSet("addressing_style") { + c.storageEngine.S3.AddressingStyle = sub.GetString("addressing_style") + } + + if sub.IsSet("bucket") { + c.storageEngine.S3.Bucket = sub.GetString("bucket") + } + + if sub.IsSet("prefix_path") { + c.storageEngine.S3.PrefixPath = sub.GetString("prefix_path") + } +} + +// OSSConfig holds Aliyun OSS storage configuration +// OSS is compatible with S3 API +type OSSConfig struct { + AccessKey string `mapstructure:"access_key"` // OSS Access Key ID + SecretKey string `mapstructure:"secret_key"` // OSS Secret Access Key + EndpointURL string `mapstructure:"endpoint_url"` // OSS Endpoint (e.g., "https://oss-cn-hangzhou.aliyuncs.com") + Region string `mapstructure:"region"` // Region (e.g., "cn-hangzhou") + Bucket string `mapstructure:"bucket"` // Default bucket (optional) + PrefixPath string `mapstructure:"prefix_path"` // Path prefix (optional) + SignatureVersion string `mapstructure:"signature_version"` // Signature version + AddressingStyle string `mapstructure:"addressing_style"` // Addressing style +} + +func (c *Config) parseOSSConfig(v *viper.Viper) { + // Default OSS config + c.storageEngine.OSS.AccessKey = "" + c.storageEngine.OSS.SecretKey = "" + c.storageEngine.OSS.EndpointURL = "" + c.storageEngine.OSS.Region = "" + c.storageEngine.OSS.Bucket = "" + c.storageEngine.OSS.PrefixPath = "" + c.storageEngine.OSS.SignatureVersion = "v4" + c.storageEngine.OSS.AddressingStyle = "path" + + if !v.IsSet("oss") { + return + } + sub := v.Sub("oss") + if sub == nil { + return + } + + if sub.IsSet("access_key") { + c.storageEngine.OSS.AccessKey = sub.GetString("access_key") + } + + if sub.IsSet("secret_key") { + c.storageEngine.OSS.SecretKey = sub.GetString("secret_key") + } + + if sub.IsSet("endpoint_url") { + c.storageEngine.OSS.EndpointURL = sub.GetString("endpoint_url") + } + + if sub.IsSet("region") { + c.storageEngine.OSS.Region = sub.GetString("region") + } + + if sub.IsSet("bucket") { + c.storageEngine.OSS.Bucket = sub.GetString("bucket") + } + + if sub.IsSet("prefix_path") { + c.storageEngine.OSS.PrefixPath = sub.GetString("prefix_path") + } + + if sub.IsSet("signature_version") { + c.storageEngine.OSS.SignatureVersion = sub.GetString("signature_version") + } + + if sub.IsSet("addressing_style") { + c.storageEngine.OSS.AddressingStyle = sub.GetString("addressing_style") + } + +} + +type GCSConfig struct { + Bucket string `mapstructure:"bucket"` // Default bucket (optional) + PrefixPath string `mapstructure:"prefix_path"` // Path prefix (optional) + EndpointURL string `mapstructure:"endpoint_url"` // Custom endpoint (optional) +} + +func (c *Config) parseGCSConfig(v *viper.Viper) { + // Default GCS config + c.storageEngine.GCS.Bucket = "" + c.storageEngine.GCS.PrefixPath = "" + c.storageEngine.GCS.EndpointURL = "" + + if !v.IsSet("gcs") { + return + } + sub := v.Sub("gcs") + if sub == nil { + return + } + + if sub.IsSet("bucket") { + c.storageEngine.GCS.Bucket = sub.GetString("bucket") + } + + if sub.IsSet("prefix_path") { + c.storageEngine.GCS.PrefixPath = sub.GetString("prefix_path") + } + + if sub.IsSet("endpoint_url") { + c.storageEngine.GCS.EndpointURL = sub.GetString("endpoint_url") + } +} + +func (c *Config) GetMinioConfig() MinioConfig { + return c.storageEngine.Minio +} + +func (c *Config) GetS3Config() S3Config { + return c.storageEngine.S3 +} + +func (c *Config) GetOSSConfig() OSSConfig { + return c.storageEngine.OSS +} + +func (c *Config) GetGCSConfig() GCSConfig { + return c.storageEngine.GCS +} diff --git a/internal/server/config/syncer_config.go b/internal/server/config/syncer_config.go new file mode 100644 index 0000000000..164b0ed73b --- /dev/null +++ b/internal/server/config/syncer_config.go @@ -0,0 +1,52 @@ +// +// Copyright 2026 The InfiniFlow Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +package config + +import "github.com/spf13/viper" + +type SyncerConfig struct { + MaxConcurrentSyncs int `mapstructure:"max_concurrent_syncs"` + SyncInterval int `mapstructure:"sync_interval"` +} + +func (c *Config) ParseSyncerConfig(v *viper.Viper) error { + // Default Syncer config + c.syncer.MaxConcurrentSyncs = 1 + c.syncer.SyncInterval = 3 + + if !v.IsSet("file_syncer") { + return nil + } + sub := v.Sub("file_syncer") + if sub == nil { + return nil + } + + if sub.IsSet("max_concurrent_syncs") { + c.syncer.MaxConcurrentSyncs = sub.GetInt("max_concurrent_syncs") + } + + if sub.IsSet("sync_interval") { + c.syncer.SyncInterval = sub.GetInt("sync_interval") + } + + return nil +} + +func (c *Config) GetSyncerConfig() *SyncerConfig { + return &c.syncer +} diff --git a/internal/server/variable.go b/internal/server/variable.go index 5d0a42f169..953f488eaa 100644 --- a/internal/server/variable.go +++ b/internal/server/variable.go @@ -91,8 +91,8 @@ func InitVariables(store VariableStore) error { // GetSecretKey returns the current secret key func GetSecretKey(store VariableStore) (string, error) { - if globalConfig.General.SecretKey != nil { - return *globalConfig.General.SecretKey, nil + if globalConfig.GetSecretKey() != "" { + return globalConfig.GetSecretKey(), nil } generatedKey, err := utility.GenerateSecretKey() diff --git a/internal/service/admin_client.go b/internal/service/admin_client.go index bb5426d5ef..c13208f403 100644 --- a/internal/service/admin_client.go +++ b/internal/service/admin_client.go @@ -59,20 +59,21 @@ func NewAdminClient(logger *zap.Logger, serverType common.ServerType, serverName // InitHTTPClient initializes the HTTP client with admin server configuration func (h *AdminClient) InitHTTPClient() error { - adminConfig := server.GetAdminConfig() - if adminConfig == nil { + config := server.GetConfig() + adminConfig := config.GetAdminServerConfig() + if adminConfig.Host == "" || adminConfig.HTTPPort == 0 { return fmt.Errorf("admin configuration not found") } h.client = utility.NewHTTPClientBuilder(). WithHost(adminConfig.Host). - WithPort(adminConfig.Port). + WithPort(adminConfig.HTTPPort). WithTimeout(10 * time.Second). Build() h.logger.Info("Heartbeat HTTP client initialized", zap.String("admin_host", adminConfig.Host), - zap.Int("admin_port", adminConfig.Port), + zap.Int("admin_port", adminConfig.HTTPPort), ) return nil diff --git a/internal/service/memory.go b/internal/service/memory.go index f28be1b4a7..230e888ae6 100644 --- a/internal/service/memory.go +++ b/internal/service/memory.go @@ -1286,7 +1286,7 @@ func memorySearchIndexNames(memories []*entity.Memory) []string { continue } indexName := memoryIndexName(memory.TenantID) - if engine.GetEngineType() == engine.EngineInfinity { + if engine.GetEngineType() == "infinity" { indexName = fmt.Sprintf("%s_%s", indexName, memory.ID) } if _, ok := seen[indexName]; ok { diff --git a/internal/service/nlp/retrieval.go b/internal/service/nlp/retrieval.go index addc1dabc3..68e0cc61ce 100644 --- a/internal/service/nlp/retrieval.go +++ b/internal/service/nlp/retrieval.go @@ -166,7 +166,7 @@ func (s *RetrievalService) Retrieval(ctx context.Context, req *RetrievalRequest) tkWeight := tkWeightOrig vtWeight := vtWeightOrig qb := GetQueryBuilder() - useInfinity := engine.GetEngineType() != engine.EngineElasticsearch + useInfinity := engine.GetEngineType() == "infinity" useOceanBase := false // TODO: add OceanBase detection when supported // For ES path: call GetScores() for second-pass KNN to get clean cosine similarity @@ -665,7 +665,7 @@ func (s *RetrievalService) Search(ctx context.Context, req *RetrievalSearchReque // Build source with vector column for ES searchSrc := make([]string, len(searchRequest.SelectFields)) copy(searchSrc, searchRequest.SelectFields) - if engine.GetEngineType() == engine.EngineElasticsearch { + if engine.GetEngineType() == "elasticsearch" { searchSrc = append(searchSrc, matchDense.VectorColumnName) } diff --git a/internal/service/oauth_login.go b/internal/service/oauth_login.go index 54ec4e72fa..6bb77b03f5 100644 --- a/internal/service/oauth_login.go +++ b/internal/service/oauth_login.go @@ -23,7 +23,6 @@ import ( "errors" "fmt" "ragflow/internal/engine/redis" - "strings" "time" "ragflow/internal/common" @@ -230,11 +229,11 @@ func (s *UserService) registerOAuthUser(ctx context.Context, channel string, inf tenant := &entity.Tenant{ ID: userID, Name: &tenantName, - LLMID: cfg.UserDefaultLLM.DefaultModels.ChatModel.Name, - EmbdID: cfg.UserDefaultLLM.DefaultModels.EmbeddingModel.Name, - ASRID: cfg.UserDefaultLLM.DefaultModels.ASRModel.Name, - Img2TxtID: cfg.UserDefaultLLM.DefaultModels.Image2TextModel.Name, - RerankID: cfg.UserDefaultLLM.DefaultModels.RerankModel.Name, + LLMID: cfg.GetDefaultChatModel().Name, + EmbdID: cfg.GetDefaultEmbeddingModel().Name, + ASRID: cfg.GetDefaultASRModel().Name, + Img2TxtID: cfg.GetDefaultVisionModel().Name, + RerankID: cfg.GetDefaultRerankModel().Name, ParserIDs: "naive:General,qa:Q&A,resume:Resume,manual:Manual,table:Table,paper:Research Paper,book:Book,laws:Laws,presentation:Presentation,picture:Picture,one:One,audio:Audio,email:Email,tag:Tag", Status: &status, } @@ -288,19 +287,19 @@ func (s *UserService) registerOAuthUser(ctx context.Context, channel string, inf // is case-insensitive against the keys server.GetConfig() materialises from // the yaml config file. func lookupOAuthConfig(channel string) (server.OAuthConfig, bool) { - cfg := server.GetConfig() - if cfg == nil { - return server.OAuthConfig{}, false - } - if found, ok := cfg.OAuth[channel]; ok { - return found, true - } - want := strings.ToLower(channel) - for k, v := range cfg.OAuth { - if strings.ToLower(k) == want { - return v, true - } - } + //cfg := server.GetConfig() + //if cfg == nil { + // return server.OAuthConfig{}, false + //} + //if found, ok := cfg.OAuth[channel]; ok { + // return found, true + //} + //want := strings.ToLower(channel) + //for k, v := range cfg.OAuth { + // if strings.ToLower(k) == want { + // return v, true + // } + //} return server.OAuthConfig{}, false } diff --git a/internal/service/system.go b/internal/service/system.go index 33076a358f..6877e92433 100644 --- a/internal/service/system.go +++ b/internal/service/system.go @@ -23,7 +23,6 @@ import ( "ragflow/internal/common" "ragflow/internal/engine/redis" "ragflow/internal/entity" - "strings" "time" "ragflow/internal/dao" @@ -54,12 +53,12 @@ type ConfigResponse struct { func (s *SystemService) GetConfig() (*ConfigResponse, error) { cfg := server.GetConfig() registerEnabled := 1 - if !cfg.Authentication.RegisterEnabled { + if !cfg.RegisterEnabled() { registerEnabled = 0 } return &ConfigResponse{ RegisterEnabled: registerEnabled, - DisablePasswordLogin: cfg.Authentication.DisablePasswordLogin, + DisablePasswordLogin: cfg.DisablePasswordLogin(), }, nil } @@ -121,7 +120,7 @@ func (s *SystemService) getDocEngineStatus() ComponentStatus { cfg := server.GetConfig() docEngineType := "" if cfg != nil { - docEngineType = strings.ToLower(string(cfg.DocEngine.Type)) + docEngineType = cfg.DocEngineType() } startedAt := time.Now() @@ -157,7 +156,7 @@ func (s *SystemService) getStorageStatus() ComponentStatus { cfg := server.GetConfig() storageType := "" if cfg != nil { - storageType = strings.ToLower(string(cfg.StorageEngine.Type)) + storageType = cfg.StorageEngineType() } startedAt := time.Now() @@ -194,7 +193,7 @@ func (s *SystemService) getDatabaseStatus() ComponentStatus { cfg := server.GetConfig() databaseType := "" if cfg != nil { - databaseType = cfg.Database.Driver + databaseType = cfg.DatabaseType() } startedAt := time.Now() diff --git a/internal/service/user.go b/internal/service/user.go index 90ca4591b8..a7dc8003ec 100644 --- a/internal/service/user.go +++ b/internal/service/user.go @@ -29,6 +29,7 @@ import ( "ragflow/internal/engine/redis" "ragflow/internal/entity" "ragflow/internal/server" + "ragflow/internal/server/config" "regexp" "strconv" "strings" @@ -105,7 +106,7 @@ type UserResponse struct { // Register user registration func (s *UserService) Register(ctx context.Context, req *RegisterRequest) (*entity.User, common.ErrorCode, error) { cfg := server.GetConfig() - if !cfg.Authentication.RegisterEnabled { + if !cfg.RegisterEnabled() { return nil, common.CodeOperatingError, fmt.Errorf("user registration is disabled") } @@ -163,31 +164,31 @@ func (s *UserService) Register(ctx context.Context, req *RegisterRequest) (*enti tenantName := req.Nickname + "'s Kingdom" - llmID := cfg.UserDefaultLLM.DefaultModels.ChatModel.Name + llmID := cfg.GetDefaultChatModel().Name if llmID == "" { llmID = "" } - embdID := cfg.UserDefaultLLM.DefaultModels.EmbeddingModel.Name + embdID := cfg.GetDefaultEmbeddingModel().Name if embdID == "" { embdID = "" } - asrID := cfg.UserDefaultLLM.DefaultModels.ASRModel.Name + asrID := cfg.GetDefaultASRModel().Name if asrID == "" { asrID = "" } - img2txtID := cfg.UserDefaultLLM.DefaultModels.Image2TextModel.Name + img2txtID := cfg.GetDefaultVisionModel().Name if img2txtID == "" { img2txtID = "" } - rerankID := cfg.UserDefaultLLM.DefaultModels.RerankModel.Name + rerankID := cfg.GetDefaultRerankModel().Name if rerankID == "" { rerankID = "" } - ttsID := cfg.UserDefaultLLM.DefaultModels.TTSModel.Name + ttsID := cfg.GetDefaultTTSModel().Name if ttsID == "" { ttsID = "" } - ocrID := cfg.UserDefaultLLM.DefaultModels.OCRModel.Name + ocrID := cfg.GetDefaultOCRModel().Name if ocrID == "" { ocrID = "" } @@ -269,22 +270,26 @@ func (s *UserService) getInitTenantLLM(ctx context.Context, userID string) ([]*e return nil, fmt.Errorf("config not initialized") } - modelConfigs := map[string]server.ModelConfig{ - entity.ModelTypeChat.String(): cfg.UserDefaultLLM.DefaultModels.ChatModel, - entity.ModelTypeEmbedding.String(): cfg.UserDefaultLLM.DefaultModels.EmbeddingModel, - entity.ModelTypeSpeech2Text.String(): cfg.UserDefaultLLM.DefaultModels.ASRModel, - entity.ModelTypeImage2Text.String(): cfg.UserDefaultLLM.DefaultModels.Image2TextModel, - entity.ModelTypeRerank.String(): cfg.UserDefaultLLM.DefaultModels.RerankModel, + modelConfigs := map[string]config.ModelConfig{ + entity.ModelTypeChat.String(): cfg.GetDefaultChatModel(), + entity.ModelTypeEmbedding.String(): cfg.GetDefaultEmbeddingModel(), + entity.ModelTypeSpeech2Text.String(): cfg.GetDefaultASRModel(), + entity.ModelTypeImage2Text.String(): cfg.GetDefaultVisionModel(), + entity.ModelTypeRerank.String(): cfg.GetDefaultRerankModel(), + entity.ModelTypeTTS.String(): cfg.GetDefaultTTSModel(), + entity.ModelTypeOCR.String(): cfg.GetDefaultOCRModel(), } seenFactories := make(map[string]bool) - factoryConfigs := make([]server.ModelConfig, 0, len(modelConfigs)) - for _, modelConfig := range []server.ModelConfig{ - cfg.UserDefaultLLM.DefaultModels.ChatModel, - cfg.UserDefaultLLM.DefaultModels.EmbeddingModel, - cfg.UserDefaultLLM.DefaultModels.ASRModel, - cfg.UserDefaultLLM.DefaultModels.Image2TextModel, - cfg.UserDefaultLLM.DefaultModels.RerankModel, + factoryConfigs := make([]config.ModelConfig, 0, len(modelConfigs)) + for _, modelConfig := range []config.ModelConfig{ + cfg.GetDefaultChatModel(), + cfg.GetDefaultEmbeddingModel(), + cfg.GetDefaultASRModel(), + cfg.GetDefaultVisionModel(), + cfg.GetDefaultRerankModel(), + cfg.GetDefaultTTSModel(), + cfg.GetDefaultOCRModel(), } { if modelConfig.Factory == "" || seenFactories[modelConfig.Factory] { continue @@ -840,26 +845,26 @@ type LoginChannel struct { // GetLoginChannels gets all supported authentication channels func (s *UserService) GetLoginChannels() ([]*LoginChannel, common.ErrorCode, error) { - cfg := server.GetConfig() + //cfg := server.GetConfig() channels := make([]*LoginChannel, 0) - for channel, oauthCfg := range cfg.OAuth { - displayName := oauthCfg.DisplayName - if displayName == "" { - displayName = strings.Title(channel) - } - - icon := oauthCfg.Icon - if icon == "" { - icon = "sso" - } - - channels = append(channels, &LoginChannel{ - Channel: channel, - DisplayName: displayName, - Icon: icon, - }) - } + //for channel, oauthCfg := range cfg.OAuth { + // displayName := oauthCfg.DisplayName + // if displayName == "" { + // displayName = strings.Title(channel) + // } + // + // icon := oauthCfg.Icon + // if icon == "" { + // icon = "sso" + // } + // + // channels = append(channels, &LoginChannel{ + // Channel: channel, + // DisplayName: displayName, + // Icon: icon, + // }) + //} return channels, common.CodeSuccess, nil } @@ -1193,7 +1198,7 @@ func (s *UserService) ForgotSendOTP(ctx context.Context, email, captchaID, captc ttlMin := int(utility.OTPTTL.Minutes()) cfg := server.GetConfig() - if err := utility.SendResetCodeEmail(cfg.SMTP, email, otp, ttlMin); err != nil { + if err = utility.SendResetCodeEmail(cfg.GetSMTPConfig(), email, otp, ttlMin); err != nil { // Roll back: restore prior code/attempts/last-sent or remove the // keys we just wrote so the next attempt isn't blocked by the // resend cooldown a failed send just installed. diff --git a/internal/storage/gcs.go b/internal/storage/gcs.go index 2d23ef86c8..9d97b67a9f 100644 --- a/internal/storage/gcs.go +++ b/internal/storage/gcs.go @@ -22,7 +22,7 @@ import ( "fmt" "io" "ragflow/internal/common" - "ragflow/internal/server" + "ragflow/internal/server/config" "time" "cloud.google.com/go/storage" @@ -33,11 +33,11 @@ import ( // GCSStorage implements Storage interface for GCS type GCSStorage struct { client *storage.Client - config *server.GCSConfig + config config.GCSConfig } // NewGCSStorage creates a new GCS storage instance -func NewGCSStorage(config *server.GCSConfig) (*GCSStorage, error) { +func NewGCSStorage(config config.GCSConfig) (*GCSStorage, error) { gcsStorage := &GCSStorage{ config: config, } diff --git a/internal/storage/minio.go b/internal/storage/minio.go index d29cfe83a5..41fccb9653 100644 --- a/internal/storage/minio.go +++ b/internal/storage/minio.go @@ -23,7 +23,7 @@ import ( "fmt" "net/http" "ragflow/internal/common" - "ragflow/internal/server" + "ragflow/internal/server/config" "time" "github.com/minio/minio-go/v7" @@ -36,11 +36,11 @@ type MinioStorage struct { client *minio.Client bucket string // default bucket prefixPath string // default prefix path - config *server.MinioConfig + config config.MinioConfig } // NewMinioStorage creates a new MinIO storage instance -func NewMinioStorage(config *server.MinioConfig) (*MinioStorage, error) { +func NewMinioStorage(config config.MinioConfig) (*MinioStorage, error) { storage := &MinioStorage{ bucket: config.Bucket, prefixPath: config.PrefixPath, diff --git a/internal/storage/oss.go b/internal/storage/oss.go index 31f7b23ca3..30936259b9 100644 --- a/internal/storage/oss.go +++ b/internal/storage/oss.go @@ -21,11 +21,11 @@ import ( "context" "errors" "fmt" - "ragflow/internal/server" + "ragflow/internal/server/config" "time" "github.com/aws/aws-sdk-go-v2/aws" - "github.com/aws/aws-sdk-go-v2/config" + s3Config "github.com/aws/aws-sdk-go-v2/config" "github.com/aws/aws-sdk-go-v2/credentials" "github.com/aws/aws-sdk-go-v2/service/s3" "github.com/aws/smithy-go" @@ -38,11 +38,11 @@ type OSSStorage struct { client *s3.Client bucket string prefixPath string - config *server.OSSConfig + config config.OSSConfig } // NewOSSStorage creates a new OSS storage instance -func NewOSSStorage(config *server.OSSConfig) (*OSSStorage, error) { +func NewOSSStorage(config config.OSSConfig) (*OSSStorage, error) { storage := &OSSStorage{ bucket: config.Bucket, prefixPath: config.PrefixPath, @@ -67,9 +67,9 @@ func (o *OSSStorage) connect() error { ) // Load configuration - cfg, err := config.LoadDefaultConfig(ctx, - config.WithRegion(o.config.Region), - config.WithCredentialsProvider(creds), + cfg, err := s3Config.LoadDefaultConfig(ctx, + s3Config.WithRegion(o.config.Region), + s3Config.WithCredentialsProvider(creds), ) if err != nil { return fmt.Errorf("failed to load OSS config: %w", err) diff --git a/internal/storage/s3.go b/internal/storage/s3.go index 1b9da54746..78a11d9cde 100644 --- a/internal/storage/s3.go +++ b/internal/storage/s3.go @@ -21,11 +21,11 @@ import ( "context" "errors" "fmt" - "ragflow/internal/server" + "ragflow/internal/server/config" "time" "github.com/aws/aws-sdk-go-v2/aws" - "github.com/aws/aws-sdk-go-v2/config" + s3Config "github.com/aws/aws-sdk-go-v2/config" "github.com/aws/aws-sdk-go-v2/credentials" "github.com/aws/aws-sdk-go-v2/service/s3" "github.com/aws/smithy-go" @@ -37,11 +37,11 @@ type S3Storage struct { client *s3.Client bucket string prefixPath string - config *server.S3Config + config config.S3Config } // NewS3Storage creates a new S3 storage instance -func NewS3Storage(config *server.S3Config) (*S3Storage, error) { +func NewS3Storage(config config.S3Config) (*S3Storage, error) { storage := &S3Storage{ config: config, } @@ -56,11 +56,11 @@ func NewS3Storage(config *server.S3Config) (*S3Storage, error) { func (s *S3Storage) connect() error { ctx := context.Background() - var opts []func(*config.LoadOptions) error + var opts []func(*s3Config.LoadOptions) error // Configure region if s.config.Region != "" { - opts = append(opts, config.WithRegion(s.config.Region)) + opts = append(opts, s3Config.WithRegion(s.config.Region)) } // Configure credentials if provided @@ -70,11 +70,11 @@ func (s *S3Storage) connect() error { s.config.SecretKey, s.config.SessionToken, ) - opts = append(opts, config.WithCredentialsProvider(creds)) + opts = append(opts, s3Config.WithCredentialsProvider(creds)) } // Load configuration - cfg, err := config.LoadDefaultConfig(ctx, opts...) + cfg, err := s3Config.LoadDefaultConfig(ctx, opts...) if err != nil { return fmt.Errorf("failed to load AWS config: %w", err) } diff --git a/internal/storage/storage_factory.go b/internal/storage/storage_factory.go index c6673fb7de..e65897f79e 100644 --- a/internal/storage/storage_factory.go +++ b/internal/storage/storage_factory.go @@ -30,10 +30,8 @@ var ( // StorageFactory creates storage instances based on configuration type StorageFactory struct { - storageType StorageType - storage Storage - config *server.StorageConfig - mu sync.RWMutex + storage Storage + mu sync.RWMutex } // GetStorageFactory returns the singleton storage factory instance @@ -49,13 +47,12 @@ func InitStorageFactory() error { factory := GetStorageFactory() globalConfig := server.GetConfig() - factory.config = &globalConfig.StorageEngine // Initialize storage based on type if err := factory.initStorage(); err != nil { return err } - common.Info(fmt.Sprintf("Storage initialized: %s", factory.config.Type)) + common.Info(fmt.Sprintf("Storage initialized: %s", globalConfig.StorageEngineType())) return nil } @@ -69,78 +66,73 @@ func CloseStorage() error { } func (f *StorageFactory) initStorage() error { - switch f.config.Type { + globalConfig := server.GetConfig() + switch globalConfig.StorageEngineType() { case "minio": - return f.initMinio(f.config.Minio) + return f.initMinio() case "s3": - return f.initS3(f.config.S3) + return f.initS3() case "oss": - return f.initOSS(f.config.OSS) + return f.initOSS() case "gcs": - return f.initGCS(f.config.GCS) + return f.initGCS() default: - return fmt.Errorf("unsupported storage type: %s", f.config.Type) + return fmt.Errorf("unsupported storage type: %s", globalConfig.StorageEngineType()) } } -func (f *StorageFactory) initMinio(minioConfig *server.MinioConfig) error { - storage, err := NewMinioStorage(minioConfig) +func (f *StorageFactory) initMinio() error { + globalConfig := server.GetConfig() + storage, err := NewMinioStorage(globalConfig.GetMinioConfig()) if err != nil { return fmt.Errorf("failed to create MinIO storage: %w", err) } f.mu.Lock() defer f.mu.Unlock() - f.storageType = StorageMinio f.storage = storage - f.config.Minio = minioConfig return nil } -func (f *StorageFactory) initS3(s3Config *server.S3Config) error { - storage, err := NewS3Storage(s3Config) +func (f *StorageFactory) initS3() error { + globalConfig := server.GetConfig() + storage, err := NewS3Storage(globalConfig.GetS3Config()) if err != nil { return fmt.Errorf("failed to create S3 storage: %w", err) } f.mu.Lock() defer f.mu.Unlock() - f.storageType = StorageAWSS3 f.storage = storage - f.config.S3 = s3Config return nil } -func (f *StorageFactory) initOSS(ossConfig *server.OSSConfig) error { - - storage, err := NewOSSStorage(ossConfig) +func (f *StorageFactory) initOSS() error { + globalConfig := server.GetConfig() + storage, err := NewOSSStorage(globalConfig.GetOSSConfig()) if err != nil { return fmt.Errorf("failed to create OSS storage: %w", err) } f.mu.Lock() defer f.mu.Unlock() - f.storageType = StorageOSS f.storage = storage - f.config.OSS = ossConfig return nil } -func (f *StorageFactory) initGCS(gcsConfig *server.GCSConfig) error { - - storage, err := NewGCSStorage(gcsConfig) +func (f *StorageFactory) initGCS() error { + globalConfig := server.GetConfig() + storage, err := NewGCSStorage(globalConfig.GetGCSConfig()) if err != nil { return fmt.Errorf("failed to create GCS storage: %w", err) } f.mu.Lock() defer f.mu.Unlock() - f.storageType = StorageGCS f.storage = storage - f.config.GCS = gcsConfig return nil } @@ -152,48 +144,34 @@ func (f *StorageFactory) GetStorage() Storage { return f.storage } -// GetStorageType returns the current storage type -func (f *StorageFactory) GetStorageType() StorageType { - f.mu.RLock() - defer f.mu.RUnlock() - return f.storageType -} - // Create creates a new storage instance based on the storage type // This is the factory method equivalent to Python's StorageFactory.create() -func (f *StorageFactory) Create(storageType StorageType) (Storage, error) { - var storage Storage - var err error - - switch storageType { - case StorageMinio: - if f.config.Minio != nil { - storage, err = NewMinioStorage(f.config.Minio) - } else { - return nil, fmt.Errorf("MinIO config not available") - } - case StorageAWSS3: - if f.config.S3 != nil { - storage, err = NewS3Storage(f.config.S3) - } else { - return nil, fmt.Errorf("S3 config not available") - } - case StorageOSS: - if f.config.OSS != nil { - storage, err = NewOSSStorage(f.config.OSS) - } else { - return nil, fmt.Errorf("OSS config not available") - } - default: - return nil, fmt.Errorf("unsupported storage type: %v", storageType) - } - - if err != nil { - return nil, err - } - - return storage, nil -} +//func (f *StorageFactory) Create(storageType StorageType) (Storage, error) { +// var storage Storage +// var err error +// +// switch storageType { +// case StorageMinio: +// storage, err = NewMinioStorage(f.config.Minio) +// if err != nil { +// return nil, fmt.Errorf("MinIO config not available: %w, %v", err, f.config.Minio) +// } +// case StorageAWSS3: +// storage, err = NewS3Storage(f.config.S3) +// if err != nil { +// return nil, fmt.Errorf("S3 config not available: %w, %v", err, f.config.S3) +// } +// case StorageOSS: +// storage, err = NewOSSStorage(f.config.OSS) +// if err != nil { +// return nil, fmt.Errorf("OSS config not available: %w, %v", err, f.config.OSS) +// } +// default: +// return nil, fmt.Errorf("unsupported storage type: %v", storageType) +// } +// +// return storage, nil +//} // SetStorage sets the storage instance (useful for testing) func (f *StorageFactory) SetStorage(storage Storage) { @@ -201,25 +179,3 @@ func (f *StorageFactory) SetStorage(storage Storage) { defer f.mu.Unlock() f.storage = storage } - -// StorageTypeMapping returns the storage type mapping (equivalent to Python's storage_mapping) -var StorageTypeMapping = map[StorageType]func(*server.StorageConfig) (Storage, error){ - StorageMinio: func(config *server.StorageConfig) (Storage, error) { - if config.Minio == nil { - return nil, fmt.Errorf("MinIO config not available") - } - return NewMinioStorage(config.Minio) - }, - StorageAWSS3: func(config *server.StorageConfig) (Storage, error) { - if config.S3 == nil { - return nil, fmt.Errorf("S3 config not available") - } - return NewS3Storage(config.S3) - }, - StorageOSS: func(config *server.StorageConfig) (Storage, error) { - if config.OSS == nil { - return nil, fmt.Errorf("OSS config not available") - } - return NewOSSStorage(config.OSS) - }, -} diff --git a/internal/tokenizer/tokenizer.go b/internal/tokenizer/tokenizer.go index 87ccc17312..fcf9acdd66 100644 --- a/internal/tokenizer/tokenizer.go +++ b/internal/tokenizer/tokenizer.go @@ -32,18 +32,10 @@ import ( rag "ragflow/internal/binding" ) -// engineTypeProvider is injected at startup by engine.RegisterEngineType -// to break the tokenizer → engine import cycle. -var engineTypeProvider = func() string { return "" } +var engineType string -// RegisterEngineType wires the engine package's GetEngineType into the -// tokenizer, breaking the circular import (engine/elasticsearch → tokenizer → engine). -func RegisterEngineType(get func() string) { - if get == nil { - engineTypeProvider = func() string { return "" } - return - } - engineTypeProvider = get +func SetEngineType(engine string) { + engineType = engine } // PoolConfig configures the elastic analyzer pool @@ -464,7 +456,7 @@ func Tokenize(text string) (string, error) { // Tokenize tokenizes the text using the tokenizer's request-scoped language. func (t Tokenizer) Tokenize(text string) (string, error) { - if engineTypeProvider() == "infinity" { + if engineType == "infinity" { return text, nil } return withAnalyzerResult(t.lang, func(a *rag.Analyzer) (string, error) { @@ -518,7 +510,7 @@ func FineGrainedTokenize(tokens string) (string, error) { // FineGrainedTokenize performs fine-grained tokenization using the tokenizer's // request-scoped language. func (t Tokenizer) FineGrainedTokenize(tokens string) (string, error) { - if engineTypeProvider() == "infinity" { + if engineType == "infinity" { return tokens, nil } return withAnalyzerResult(t.lang, func(a *rag.Analyzer) (string, error) { diff --git a/internal/tokenizer/tokenizer_concurrent_test.go b/internal/tokenizer/tokenizer_concurrent_test.go index 4fb6ae50be..f154e0e016 100644 --- a/internal/tokenizer/tokenizer_concurrent_test.go +++ b/internal/tokenizer/tokenizer_concurrent_test.go @@ -175,7 +175,7 @@ func TestConcurrentTokenize(t *testing.T) { func TestConcurrentTokenizeLanguageIsolation(t *testing.T) { restore := saveEngineType() defer restore() - RegisterEngineType(func() string { return "" }) + SetEngineType("") cfg := &PoolConfig{ DictPath: "", diff --git a/internal/tokenizer/tokenizer_test.go b/internal/tokenizer/tokenizer_test.go index f81f94a4d8..7670ccb59d 100644 --- a/internal/tokenizer/tokenizer_test.go +++ b/internal/tokenizer/tokenizer_test.go @@ -32,8 +32,8 @@ type languageDifferentiator struct { // to restore it. Use this when a test modifies the engine type to avoid // leaking global state between tests. func saveEngineType() func() { - original := engineTypeProvider - return func() { engineTypeProvider = original } + original := engineType + return func() { engineType = original } } // --------------------------------------------------------------------------- @@ -116,8 +116,8 @@ func TestRegisterEngineType_Basic(t *testing.T) { restore := saveEngineType() defer restore() - RegisterEngineType(func() string { return "infinity" }) - if got := engineTypeProvider(); got != "infinity" { + SetEngineType("infinity") + if got := engineType; got != "infinity" { t.Errorf("expected 'infinity', got %q", got) } } @@ -126,9 +126,9 @@ func TestRegisterEngineType_Overwrite(t *testing.T) { restore := saveEngineType() defer restore() - RegisterEngineType(func() string { return "first" }) - RegisterEngineType(func() string { return "second" }) - if got := engineTypeProvider(); got != "second" { + SetEngineType("first") + SetEngineType("second") + if got := engineType; got != "second" { t.Errorf("expected 'second', got %q", got) } } @@ -140,7 +140,7 @@ func TestRegisterEngineType_Overwrite(t *testing.T) { func TestTokenize_InfinityEngine(t *testing.T) { restore := saveEngineType() defer restore() - RegisterEngineType(func() string { return "infinity" }) + SetEngineType("infinity") inputs := []string{"hello world", "你好 世界", "", "a single word"} for _, input := range inputs { @@ -158,7 +158,7 @@ func TestTokenize_PoolNotInitialized(t *testing.T) { restore := saveEngineType() defer restore() // Ensure engine type is not "infinity" so we hit the pool path - RegisterEngineType(func() string { return "" }) + SetEngineType("") _, err := Tokenize("hello world") if err == nil { @@ -173,7 +173,7 @@ func TestTokenize_PoolNotInitialized(t *testing.T) { func TestFineGrainedTokenize_InfinityEngine(t *testing.T) { restore := saveEngineType() defer restore() - RegisterEngineType(func() string { return "infinity" }) + SetEngineType("infinity") inputs := []string{"hello world", "测试 分词", ""} for _, input := range inputs { @@ -190,7 +190,7 @@ func TestFineGrainedTokenize_InfinityEngine(t *testing.T) { func TestFineGrainedTokenize_PoolNotInitialized(t *testing.T) { restore := saveEngineType() defer restore() - RegisterEngineType(func() string { return "" }) + SetEngineType("") _, err := FineGrainedTokenize("hello world") if err == nil { @@ -233,7 +233,7 @@ func TestGetTermTag_PoolNotInitialized(t *testing.T) { func TestTokenize_DefaultLanguageResetsAnalyzerState(t *testing.T) { restore := saveEngineType() defer restore() - RegisterEngineType(func() string { return "" }) + SetEngineType("") if err := Init(&PoolConfig{ DictPath: "",