From add7b9486f8eb584a5b9d7b7bdf69863383036c3 Mon Sep 17 00:00:00 2001 From: Jin Hai Date: Fri, 10 Jul 2026 11:58:32 +0800 Subject: [PATCH] Go: merge duplicate codes (#16783) ### Summary 1. merge heartbeat function. 2. introduce all environments --------- Signed-off-by: Jin Hai --- cmd/ragflow_server.go | 4 +- internal/admin/service.go | 17 +- .../agent/audio/model_provider_synthesizer.go | 4 +- internal/agent/canvas/node_body.go | 4 +- internal/agent/canvas/timeout.go | 4 +- internal/agent/component/stagehand_runtime.go | 6 +- .../stagehand_runtime_integration_test.go | 19 +- internal/agent/runtime/selector.go | 4 +- internal/agent/sandbox/aliyun.go | 16 +- internal/agent/sandbox/e2b.go | 14 +- internal/agent/sandbox/e2b_test.go | 6 +- internal/agent/sandbox/local.go | 35 ++-- internal/agent/sandbox/local_test.go | 19 +- internal/agent/sandbox/manager.go | 4 +- internal/agent/sandbox/self_managed.go | 13 +- internal/agent/sandbox/ssh.go | 31 +-- internal/agent/tool/exesql_trino_stub.go | 4 +- internal/agent/tool/keenable.go | 4 +- internal/agent/tool/ssrf.go | 4 +- internal/agent/tool/tavily.go | 4 +- internal/cli/cli.go | 4 +- .../filesystem/skill_hub/source/interface.go | 6 +- internal/cli/filesystem/skill_install.go | 5 +- internal/common/environments.go | 188 ++++++++++++++++++ internal/deepdoc/client.go | 6 +- .../deepdoc/parser/pdf/batch_smoke_test.go | 15 +- internal/deepdoc/parser/pdf/compare_test.go | 9 +- internal/deepdoc/parser/pdf/helpers_test.go | 3 +- internal/deepdoc/parser/pdf/ocr_merge_test.go | 2 +- .../pdf/parser_pipeline_integration_test.go | 2 +- .../parser/pdf/pipeline_parity_test.go | 2 +- .../pdf/table_rotate_integration_test.go | 4 +- internal/deepdoc/parser/pdf/text_dump_test.go | 2 +- internal/deepdoc/parser/pdf/tool/compare.go | 3 +- .../deepdoc/parser/pdf/tool/compare_test.go | 8 +- internal/deepdoc/parser/pdf/tool/config.go | 17 +- internal/engine/elasticsearch/kg_test.go | 10 +- internal/entity/models/builtin.go | 6 +- internal/entity/models/gitee_test.go | 7 +- internal/handler/memory.go | 3 +- internal/harness/core/interrupt.go | 4 +- .../ingestion/compilation/extractor/ner.go | 12 +- .../ingestion/component/tokenizer_test.go | 3 +- .../pipeline/real_storage_integration_test.go | 6 +- .../pipeline/template_integration_test.go | 6 +- internal/ingestion/pipeline/test_helpers.go | 4 +- internal/ingestion/task/dataflow_e2e_test.go | 12 +- ...dataflow_real_pipeline_integration_test.go | 8 +- internal/parser/parser/pdf_parser_common.go | 4 +- internal/parser/parser/pdf_parser_docling.go | 6 +- internal/parser/parser/pdf_parser_mineru.go | 8 +- .../parser/pdf_parser_opendataloader.go | 12 +- .../parser/parser/pdf_parser_paddleocr.go | 10 +- internal/parser/parser/pdf_parser_somark.go | 38 ++-- internal/parser/parser/pdf_parser_tcadp.go | 6 +- internal/server/config.go | 52 ++--- internal/service/chat_session.go | 5 +- internal/service/connector.go | 13 +- internal/service/document.go | 3 +- internal/service/file.go | 23 ++- internal/service/memory.go | 3 +- internal/service/model_service.go | 7 +- internal/service/system.go | 11 +- internal/service/user.go | 3 +- internal/storage/minio_test.go | 17 -- internal/tokenizer/tokenizer.go | 3 +- internal/utility/path.go | 11 +- 67 files changed, 479 insertions(+), 329 deletions(-) create mode 100644 internal/common/environments.go diff --git a/cmd/ragflow_server.go b/cmd/ragflow_server.go index ca32c177d1..dda3434af2 100644 --- a/cmd/ragflow_server.go +++ b/cmd/ragflow_server.go @@ -474,7 +474,7 @@ func runAdmin(args *serverArgs) error { func runIngestor(args *serverArgs) error { // Initialize tokenizer (rag_analyzer) - dictPath := os.Getenv("RAGFLOW_DICT_PATH") + dictPath := common.GetEnv(common.EnvRAGFlowDictPath) if dictPath == "" { dictPath = "/usr/share/infinity/resource" } @@ -639,7 +639,7 @@ func runAPI(args *serverArgs) error { local.InitAdminStatus(1, "admin server not connected") // Initialize tokenizer (rag_analyzer) - dictPath := os.Getenv("RAGFLOW_DICT_PATH") + dictPath := common.GetEnv(common.EnvRAGFlowDictPath) if dictPath == "" { dictPath = "/usr/share/infinity/resource" } diff --git a/internal/admin/service.go b/internal/admin/service.go index c3a76e370b..4b47b23e80 100644 --- a/internal/admin/service.go +++ b/internal/admin/service.go @@ -24,7 +24,6 @@ import ( "errors" "fmt" "net/http" - "os" "ragflow/internal/common" "ragflow/internal/dao" "ragflow/internal/engine" @@ -1191,7 +1190,7 @@ func (s *Service) getRedisInfo(name string) (map[string]interface{}, error) { // getESClusterStats gets Elasticsearch cluster stats func (s *Service) getESClusterStats(name string) (map[string]interface{}, error) { // Check if Elasticsearch is the doc engine - docEngine := os.Getenv("DOC_ENGINE") + docEngine := common.GetEnv(common.EnvDocEngine) if docEngine == "" { docEngine = "elasticsearch" } @@ -1564,7 +1563,7 @@ func (s *Service) ListEnvironments() ([]map[string]interface{}, error) { result := make([]map[string]interface{}, 0) // DOC_ENGINE - docEngine := os.Getenv("DOC_ENGINE") + docEngine := common.GetEnv(common.EnvDocEngine) if docEngine == "" { docEngine = "elasticsearch" } @@ -1574,17 +1573,17 @@ func (s *Service) ListEnvironments() ([]map[string]interface{}, error) { }) // DEFAULT_SUPERUSER_EMAIL - defaultSuperuserEmail := os.Getenv("DEFAULT_SUPERUSER_EMAIL") + defaultSuperuserEmail := common.GetEnv(common.EnvDefaultSuperuserEmail) if defaultSuperuserEmail == "" { defaultSuperuserEmail = "admin@ragflow.io" } result = append(result, map[string]interface{}{ - "env": "DEFAULT_SUPERUSER_EMAIL", + "env": common.EnvDefaultSuperuserEmail, "value": defaultSuperuserEmail, }) // DB_TYPE - dbType := os.Getenv("DB_TYPE") + dbType := common.GetEnv(common.EnvDBType) if dbType == "" { dbType = "mysql" } @@ -1594,7 +1593,7 @@ func (s *Service) ListEnvironments() ([]map[string]interface{}, error) { }) // DEVICE - device := os.Getenv("DEVICE") + device := common.GetEnv(common.EnvDevice) if device == "" { device = "cpu" } @@ -1604,12 +1603,12 @@ func (s *Service) ListEnvironments() ([]map[string]interface{}, error) { }) // STORAGE_IMPL - storageImpl := os.Getenv("STORAGE_IMPL") + storageImpl := common.GetEnv(common.EnvStorageImpl) if storageImpl == "" { storageImpl = "MINIO" } result = append(result, map[string]interface{}{ - "env": "STORAGE_IMPL", + "env": common.EnvStorageImpl, "value": storageImpl, }) diff --git a/internal/agent/audio/model_provider_synthesizer.go b/internal/agent/audio/model_provider_synthesizer.go index 2bf3acfb40..0809736814 100644 --- a/internal/agent/audio/model_provider_synthesizer.go +++ b/internal/agent/audio/model_provider_synthesizer.go @@ -40,7 +40,7 @@ import ( "crypto/sha256" "encoding/hex" "fmt" - "os" + "ragflow/internal/common" "strconv" "time" @@ -167,7 +167,7 @@ func buildTTSCacheKey(tenantID string, req SynthesizeRequest) string { // ttsCacheTTL reads RAGFLOW_TTS_CACHE_TTL_SECONDS; default 7 days. // Matches the Python `_ttl_seconds` default of 7 * 24 * 60 * 60. func ttsCacheTTL() time.Duration { - raw := os.Getenv("RAGFLOW_TTS_CACHE_TTL_SECONDS") + raw := common.GetEnv(common.EnvRAGFlowTTSCacheTTLSeconds) if raw == "" { return 7 * 24 * time.Hour } diff --git a/internal/agent/canvas/node_body.go b/internal/agent/canvas/node_body.go index 5ac3181c89..91f40d517f 100644 --- a/internal/agent/canvas/node_body.go +++ b/internal/agent/canvas/node_body.go @@ -33,7 +33,7 @@ import ( "context" "errors" "fmt" - "os" + "ragflow/internal/common" "strconv" "strings" "time" @@ -146,7 +146,7 @@ func legacyNoOpBody(cpnID string) nodeBodyFn { // the default — invalid input must never widen the timeout silently. func componentTimeout() time.Duration { const def = 600 * time.Second - if v := os.Getenv("COMPONENT_EXEC_TIMEOUT"); v != "" { + if v := common.GetEnv(common.EnvComponentExecTimeout); v != "" { if secs, err := strconv.Atoi(v); err == nil && secs > 0 { return time.Duration(secs) * time.Second } diff --git a/internal/agent/canvas/timeout.go b/internal/agent/canvas/timeout.go index e875a52c4f..f283b2a7f2 100644 --- a/internal/agent/canvas/timeout.go +++ b/internal/agent/canvas/timeout.go @@ -40,7 +40,7 @@ package canvas import ( "context" - "os" + "ragflow/internal/common" "strconv" "strings" "time" @@ -145,7 +145,7 @@ func resolveTimeoutFromContext(ctx context.Context, componentClass string) time. // unset / empty / non-numeric / non-positive. Invalid input must // never widen the timeout silently. func parseSecondsEnv(name string) (time.Duration, bool) { - v := os.Getenv(name) + v := common.GetEnv(name) if v == "" { return 0, false } diff --git a/internal/agent/component/stagehand_runtime.go b/internal/agent/component/stagehand_runtime.go index 3233988769..cf9466c2f7 100644 --- a/internal/agent/component/stagehand_runtime.go +++ b/internal/agent/component/stagehand_runtime.go @@ -65,7 +65,7 @@ import ( "errors" "fmt" "math" - "os" + "ragflow/internal/common" "strconv" "sync" "sync/atomic" @@ -176,7 +176,7 @@ func getDefaultStagehandInvoker() StagehandInvoker { // envDuration reads a Go-duration env var, falling back to def on // missing / parse failure. func envDuration(key string, def time.Duration) time.Duration { - v := os.Getenv(key) + v := common.GetEnv(key) if v == "" { return def } @@ -189,7 +189,7 @@ func envDuration(key string, def time.Duration) time.Duration { // envInt reads a non-negative int env var, falling back to def. func envInt(key string, def int) int { - v := os.Getenv(key) + v := common.GetEnv(key) if v == "" { return def } diff --git a/internal/agent/component/stagehand_runtime_integration_test.go b/internal/agent/component/stagehand_runtime_integration_test.go index 4c74f68f31..01c532d381 100644 --- a/internal/agent/component/stagehand_runtime_integration_test.go +++ b/internal/agent/component/stagehand_runtime_integration_test.go @@ -48,6 +48,7 @@ import ( "net/http/httptest" "os" "path/filepath" + "ragflow/internal/common" "testing" "time" @@ -81,9 +82,9 @@ import ( // against https://www.bbc.com/news/world — returns a non-empty // summary string in ~10s. func TestStagehandRuntime_Extract(t *testing.T) { - apiKey := os.Getenv("OPENAI_API_KEY") - baseURL := os.Getenv("OPENAI_BASE_URL") - model := os.Getenv("OPENAI_MODEL") + apiKey := common.GetEnv(common.EnvOpenAIApiKey) + baseURL := common.GetEnv(common.EnvOpenAIBaseURL) + model := common.GetEnv(common.EnvOpenAIModel) if apiKey == "" || baseURL == "" || model == "" { t.Skipf("missing required env (OPENAI_API_KEY/OPENAI_BASE_URL/OPENAI_MODEL); skipping") } @@ -91,7 +92,7 @@ func TestStagehandRuntime_Extract(t *testing.T) { // Default schema: single string. Optional override via env: // STAGEHAND_EXTRACT_SCHEMA_JSON='{"type":"object",...}'. var schema map[string]any - if raw := os.Getenv("STAGEHAND_EXTRACT_SCHEMA_JSON"); raw != "" { + if raw := common.GetEnv(common.EnvStageHandExtractSchemaJSON); raw != "" { if err := json.Unmarshal([]byte(raw), &schema); err != nil { t.Fatalf("STAGEHAND_EXTRACT_SCHEMA_JSON is not valid JSON: %v", err) } @@ -142,7 +143,7 @@ func TestStagehandRuntime_Extract(t *testing.T) { t.Logf("extraction result:\n%s", resultJSON) // Dump for external observers. - dumpPath := os.Getenv("STAGEHAND_EXTRACT_RESULT_FILE") + dumpPath := common.GetEnv(common.EnvStageHandExtractResultFile) if dumpPath == "" { dumpPath = "/tmp/stagehand_extract_result.txt" } @@ -166,7 +167,7 @@ func truncateSchema(s map[string]any) string { // binary is missing. We don't fail on miss because the runtime // falls back to a GitHub download. func cacheDirGuess() string { - if v := os.Getenv("XDG_CACHE_HOME"); v != "" { + if v := common.GetEnv(common.EnvXDGCacheHome); v != "" { return filepath.Join(v, "stagehand", "lib") } home, err := os.UserHomeDir() @@ -184,9 +185,9 @@ func cacheDirGuess() string { // // Skipped unless OPENAI_* env vars are configured. func TestBrowser_E2E_Extract(t *testing.T) { - apiKey := os.Getenv("OPENAI_API_KEY") - baseURL := os.Getenv("OPENAI_BASE_URL") - model := os.Getenv("OPENAI_MODEL") + apiKey := common.GetEnv(common.EnvOpenAIApiKey) + baseURL := common.GetEnv(common.EnvOpenAIBaseURL) + model := common.GetEnv(common.EnvOpenAIModel) if apiKey == "" || baseURL == "" || model == "" { t.Skipf("missing required env (OPENAI_API_KEY/OPENAI_BASE_URL/OPENAI_MODEL); skipping") } diff --git a/internal/agent/runtime/selector.go b/internal/agent/runtime/selector.go index fc62036ee4..c7ac7b5fac 100644 --- a/internal/agent/runtime/selector.go +++ b/internal/agent/runtime/selector.go @@ -30,7 +30,7 @@ package runtime import ( "context" "fmt" - "os" + "ragflow/internal/common" "sync" "github.com/redis/go-redis/v9" @@ -81,7 +81,7 @@ var ( // to RuntimeGo (the new default) so a misconfig still lands on the Go path. func Default() RuntimeMode { defaultOnce.Do(func() { - raw := os.Getenv(defaultEnvKey) + raw := common.GetEnv(defaultEnvKey) switch RuntimeMode(raw) { case RuntimeGo, RuntimePython, RuntimeAuto: defaultMode = RuntimeMode(raw) diff --git a/internal/agent/sandbox/aliyun.go b/internal/agent/sandbox/aliyun.go index eb4ea1c91f..8eb5f2a83e 100644 --- a/internal/agent/sandbox/aliyun.go +++ b/internal/agent/sandbox/aliyun.go @@ -44,7 +44,7 @@ import ( "io" "net/http" "net/url" - "os" + "ragflow/internal/common" "strings" "time" @@ -100,13 +100,13 @@ func newAliyunProviderFromEnv() *AliyunCodeInterpreterProvider { // env vars, mirroring the admin-panel settings JSON shape. func aliyunConfigFromEnv() map[string]any { return map[string]any{ - "ACCESS_KEY_ID": os.Getenv("AGENTRUN_ACCESS_KEY_ID"), - "ACCESS_KEY_SECRET": os.Getenv("AGENTRUN_ACCESS_KEY_SECRET"), - "ACCOUNT_ID": os.Getenv("AGENTRUN_ACCOUNT_ID"), - "REGION": os.Getenv("AGENTRUN_REGION"), - "TEMPLATE_NAME": os.Getenv("AGENTRUN_TEMPLATE_NAME"), - "EXECUTE_HOST": os.Getenv("AGENTRUN_EXECUTE_HOST"), - "TIMEOUT": os.Getenv("AGENTRUN_TIMEOUT"), + "ACCESS_KEY_ID": common.GetEnv(common.EnvAgentRunAccessKeyID), + "ACCESS_KEY_SECRET": common.GetEnv(common.EnvAgentRunAccessKeySecret), + "ACCOUNT_ID": common.GetEnv(common.EnvAgentRunAccountID), + "REGION": common.GetEnv(common.EnvAgentRunRegion), + "TEMPLATE_NAME": common.GetEnv(common.EnvAgentRunTemplateName), + "EXECUTE_HOST": common.GetEnv(common.EnvAgentRunExecuteHost), + "TIMEOUT": common.GetEnv(common.EnvAgentRunTimeout), } } diff --git a/internal/agent/sandbox/e2b.go b/internal/agent/sandbox/e2b.go index c7ae02f9fd..ce249f3f24 100644 --- a/internal/agent/sandbox/e2b.go +++ b/internal/agent/sandbox/e2b.go @@ -43,7 +43,7 @@ import ( "context" "errors" "fmt" - "os" + "ragflow/internal/common" "sync" "time" @@ -93,8 +93,8 @@ func newE2BProviderFromEnv() *E2BProvider { // part of the JSON config map. func e2bConfigFromEnv() map[string]any { return map[string]any{ - "TEMPLATE": os.Getenv("E2B_TEMPLATE"), - "TIMEOUT": os.Getenv("E2B_TIMEOUT"), + "TEMPLATE": common.GetEnv(common.EnvE2BTemplate), + "TIMEOUT": common.GetEnv(common.EnvE2BTimeout), } } @@ -125,8 +125,8 @@ func (p *E2BProvider) ProviderType() ProviderType { return ProviderE2B } // resolves from the same env vars. We return an error if neither // is set so the manager does not register a broken provider. func (p *E2BProvider) Initialize(ctx context.Context) error { - apiKey := os.Getenv("E2B_API_KEY") - accessToken := os.Getenv("E2B_ACCESS_TOKEN") + apiKey := common.GetEnv(common.EnvE2BApiKey) + accessToken := common.GetEnv(common.EnvE2BAccessToken) if apiKey == "" && accessToken == "" { return errors.New( "e2b: E2B_API_KEY or E2B_ACCESS_TOKEN env var is required " + @@ -136,8 +136,8 @@ func (p *E2BProvider) Initialize(ctx context.Context) error { cfg := e2bsdk.Config{ APIKey: apiKey, AccessToken: accessToken, - Domain: os.Getenv("E2B_DOMAIN"), - APIURL: os.Getenv("E2B_API_URL"), + Domain: common.GetEnv(common.EnvE2BDomain), + APIURL: common.GetEnv(common.EnvE2BAPIURL), } c, err := e2bsdk.NewClient(cfg) if err != nil { diff --git a/internal/agent/sandbox/e2b_test.go b/internal/agent/sandbox/e2b_test.go index 230d6fe461..23cf355dd8 100644 --- a/internal/agent/sandbox/e2b_test.go +++ b/internal/agent/sandbox/e2b_test.go @@ -19,7 +19,7 @@ package sandbox import ( "context" "errors" - "os" + "ragflow/internal/common" "strings" "testing" "time" @@ -93,7 +93,7 @@ func TestE2BProvider_Initialize_MissingCreds(t *testing.T) { // key is set (CI without secrets). The skip is loud via a clear // log line so missing-secrets is visible in test output. func TestE2BProvider_Initialize_WithAPIKey(t *testing.T) { - apiKey := os.Getenv("E2B_API_KEY") + apiKey := common.GetEnv(common.EnvE2BApiKey) if apiKey == "" { t.Skip("E2B_API_KEY not set — skipping network-dependent init check") } @@ -265,7 +265,7 @@ func makeFakeCommandResult(stdout string) *e2bsdk.CommandResult { // the test always runs (so missing-secrets shows up in CI logs). // When enabled, it creates a real sandbox, runs Python, kills it. func TestE2BProvider_FullE2E_SkipWithoutKey(t *testing.T) { - apiKey := os.Getenv("E2B_API_KEY") + apiKey := common.GetEnv(common.EnvE2BApiKey) if apiKey == "" { t.Skip("E2B_API_KEY not set — skipping full E2E test (real network call)") } diff --git a/internal/agent/sandbox/local.go b/internal/agent/sandbox/local.go index f5e3b8dee3..559b4149ce 100644 --- a/internal/agent/sandbox/local.go +++ b/internal/agent/sandbox/local.go @@ -49,6 +49,7 @@ import ( "os" "os/exec" "path/filepath" + "ragflow/internal/common" "strings" "sync" "syscall" @@ -116,14 +117,14 @@ func newLocalProviderFromEnv() *LocalProvider { // vars, mirroring the admin-panel settings JSON shape. func localConfigFromEnv() map[string]any { return map[string]any{ - "PYTHON_BIN": os.Getenv("LOCAL_PYTHON_BIN"), - "NODE_BIN": os.Getenv("LOCAL_NODE_BIN"), - "WORK_DIR": os.Getenv("LOCAL_WORK_DIR"), - "TIMEOUT": os.Getenv("LOCAL_TIMEOUT"), - "MAX_MEMORY_MB": os.Getenv("LOCAL_MAX_MEMORY_MB"), - "MAX_OUTPUT_BYTES": os.Getenv("LOCAL_MAX_OUTPUT_BYTES"), - "MAX_ARTIFACTS": os.Getenv("LOCAL_MAX_ARTIFACTS"), - "MAX_ARTIFACT_BYTES": os.Getenv("LOCAL_MAX_ARTIFACT_BYTES"), + "PYTHON_BIN": common.GetEnv(common.EnvLocalPythonBin), + "NODE_BIN": common.GetEnv(common.EnvLocalNodeBin), + "WORK_DIR": common.GetEnv(common.EnvLocalWorkDir), + "TIMEOUT": common.GetEnv(common.EnvLocalTimeout), + "MAX_MEMORY_MB": common.GetEnv(common.EnvLocalMaxMemoryMB), + "MAX_OUTPUT_BYTES": common.GetEnv(common.EnvLocalMaxOutputBytes), + "MAX_ARTIFACTS": common.GetEnv(common.EnvLocalMaxArtifacts), + "MAX_ARTIFACT_BYTES": common.GetEnv(common.EnvLocalMaxArtifactBytes), } } @@ -462,19 +463,19 @@ func buildLocalChildEnv(instanceDir string) []string { } // Append the host PATH so the subprocess can find shared // libs and helper binaries. - if path := os.Getenv("PATH"); path != "" { + if path := common.GetEnv(common.EnvPath); path != "" { env = append(env, "PATH="+path) } // Append thread-related vars from the host so libraries // that honor them (OpenMP, MKL, etc.) behave the same. for _, name := range []string{ - "OMP_NUM_THREADS", - "OPENBLAS_NUM_THREADS", - "MKL_NUM_THREADS", - "VECLIB_MAXIMUM_THREADS", - "NUMEXPR_NUM_THREADS", + common.EnvOMPNumThreads, + common.EnvOpenBLASNumThreads, + common.EnvMKLNumThreads, + common.EnvVECLIBMaximumThreads, + common.EnvNumEXPRNumThreads, } { - if v := os.Getenv(name); v != "" { + if v := common.GetEnv(name); v != "" { env = append(env, name+"="+v) } } @@ -579,7 +580,7 @@ func statusFromExitCode(code int) string { // envOr returns the value of the env var, or fallback if unset. func envOr(key, fallback string) string { - if v := os.Getenv(key); v != "" { + if v := common.GetEnv(key); v != "" { return v } return fallback @@ -588,7 +589,7 @@ func envOr(key, fallback string) string { // envIntOr returns the env var parsed as int, or fallback if // unset / unparseable. func envIntOr(key string, fallback int) int { - if v := os.Getenv(key); v != "" { + if v := common.GetEnv(key); v != "" { var n int if _, err := fmt.Sscanf(v, "%d", &n); err == nil && n > 0 { return n diff --git a/internal/agent/sandbox/local_test.go b/internal/agent/sandbox/local_test.go index 0a794c6cb7..d12f1c95d8 100644 --- a/internal/agent/sandbox/local_test.go +++ b/internal/agent/sandbox/local_test.go @@ -21,6 +21,7 @@ import ( "encoding/base64" "os" "path/filepath" + "ragflow/internal/common" "strings" "testing" "time" @@ -366,25 +367,25 @@ func TestStatusFromExitCode(t *testing.T) { } func TestEnvOr(t *testing.T) { - t.Setenv("RAGFLOW_TEST_ENVOR", "x") - if got := envOr("RAGFLOW_TEST_ENVOR", "fallback"); got != "x" { + t.Setenv(common.EnvRAGFlowTestEnvOr, "x") + if got := envOr(common.EnvRAGFlowTestEnvOr, "fallback"); got != "x" { t.Errorf("got %q, want x", got) } - if got := envOr("RAGFLOW_TEST_ENVOR_UNSET", "fallback"); got != "fallback" { + if got := envOr(common.EnvRAGFlowTestEnvOrUnset, "fallback"); got != "fallback" { t.Errorf("got %q, want fallback", got) } } func TestEnvIntOr(t *testing.T) { - t.Setenv("RAGFLOW_TEST_ENVINTOR", "42") - if got := envIntOr("RAGFLOW_TEST_ENVINTOR", 10); got != 42 { + t.Setenv(common.EnvRAGFlowTestEnvIntOr, "42") + if got := envIntOr(common.EnvRAGFlowTestEnvIntOr, 10); got != 42 { t.Errorf("got %d, want 42", got) } - if got := envIntOr("RAGFLOW_TEST_ENVINTOR_UNSET", 10); got != 10 { + if got := envIntOr(common.EnvRAGFlowTestEnvIntOrUnset, 10); got != 10 { t.Errorf("got %d, want 10", got) } - t.Setenv("RAGFLOW_TEST_ENVINTOR", "garbage") - if got := envIntOr("RAGFLOW_TEST_ENVINTOR", 10); got != 10 { + t.Setenv(common.EnvRAGFlowTestEnvIntOr, "garbage") + if got := envIntOr(common.EnvRAGFlowTestEnvIntOr, 10); got != 10 { t.Errorf("garbage value: got %d, want fallback 10", got) } } @@ -392,7 +393,7 @@ func TestEnvIntOr(t *testing.T) { // findBinary searches PATH for a binary. Used by tests that // skip on missing runtime dependencies. func findBinary(name string) (string, error) { - paths := strings.Split(os.Getenv("PATH"), string(os.PathListSeparator)) + paths := strings.Split(common.GetEnv(common.EnvPath), string(os.PathListSeparator)) for _, dir := range paths { candidate := filepath.Join(dir, name) if info, err := os.Stat(candidate); err == nil && !info.IsDir() && info.Mode()&0o111 != 0 { diff --git a/internal/agent/sandbox/manager.go b/internal/agent/sandbox/manager.go index c3f87058f2..b7d6686a5a 100644 --- a/internal/agent/sandbox/manager.go +++ b/internal/agent/sandbox/manager.go @@ -40,7 +40,7 @@ import ( "encoding/json" "errors" "fmt" - "os" + "ragflow/internal/common" "sync" "ragflow/internal/dao" @@ -275,7 +275,7 @@ var errSettingsMalformed = errors.New("sandbox: admin-panel settings JSON malfor // "self_managed" to match the Python // `_load_provider_from_settings` default. func resolveProviderType() ProviderType { - if v := os.Getenv("SANDBOX_PROVIDER_TYPE"); v != "" { + if v := common.GetEnv(common.EnvSandboxProviderType); v != "" { return ProviderType(v) } return ProviderSelfManaged diff --git a/internal/agent/sandbox/self_managed.go b/internal/agent/sandbox/self_managed.go index d7f7f18a5f..2ae90f4e4e 100644 --- a/internal/agent/sandbox/self_managed.go +++ b/internal/agent/sandbox/self_managed.go @@ -62,7 +62,6 @@ import ( "fmt" "io" "net/http" - "os" "strings" "sync" "time" @@ -119,12 +118,12 @@ func newSelfManagedProviderFromEnv() *SelfManagedProvider { // env vars, mirroring the admin-panel settings JSON shape. func selfManagedConfigFromEnv() map[string]any { return map[string]any{ - "EXECUTOR_MANAGER_URL": os.Getenv("SANDBOX_EXECUTOR_MANAGER_URL"), - "EXECUTOR_MANAGER_TIMEOUT": os.Getenv("SANDBOX_EXECUTOR_MANAGER_TIMEOUT"), - "EXECUTOR_MANAGER_POOL_SIZE": os.Getenv("SANDBOX_EXECUTOR_MANAGER_POOL_SIZE"), - "EXECUTOR_MANAGER_MAX_RETRIES": os.Getenv("SANDBOX_EXECUTOR_MANAGER_MAX_RETRIES"), - "BASE_PYTHON_IMAGE": os.Getenv("SANDBOX_BASE_PYTHON_IMAGE"), - "BASE_NODEJS_IMAGE": os.Getenv("SANDBOX_BASE_NODEJS_IMAGE"), + "EXECUTOR_MANAGER_URL": common.GetEnv(common.EnvSandboxExecutorManagerURL), + "EXECUTOR_MANAGER_TIMEOUT": common.GetEnv(common.EnvSandboxExecutorManagerTimeout), + "EXECUTOR_MANAGER_POOL_SIZE": common.GetEnv(common.EnvSandboxExecutorManagerPoolSize), + "EXECUTOR_MANAGER_MAX_RETRIES": common.GetEnv(common.EnvSandboxExecutorManagerMaxRetries), + "BASE_PYTHON_IMAGE": common.GetEnv(common.EnvSandboxBasePythonImage), + "BASE_NODEJS_IMAGE": common.GetEnv(common.EnvSandboxBaseNodeJSImage), } } diff --git a/internal/agent/sandbox/ssh.go b/internal/agent/sandbox/ssh.go index b1ebf6ce73..21011c54bb 100644 --- a/internal/agent/sandbox/ssh.go +++ b/internal/agent/sandbox/ssh.go @@ -50,6 +50,7 @@ import ( "net" "os" "path" + "ragflow/internal/common" "strconv" "strings" "sync" @@ -117,21 +118,21 @@ func newSSHProviderFromEnv() *SSHProvider { // remote host's key (fail-closed when unset). func sshConfigFromEnv() map[string]any { return map[string]any{ - "HOST": os.Getenv("SSH_HOST"), - "PORT": os.Getenv("SSH_PORT"), - "USERNAME": os.Getenv("SSH_USERNAME"), - "PASSWORD": os.Getenv("SSH_PASSWORD"), - "PRIVATE_KEY": os.Getenv("SSH_PRIVATE_KEY"), - "PRIVATE_KEY_PATH": os.Getenv("SSH_PRIVATE_KEY_PATH"), - "PASSPHRASE": os.Getenv("SSH_PASSPHRASE"), - "PYTHON_BIN": os.Getenv("SSH_PYTHON_BIN"), - "NODE_BIN": os.Getenv("SSH_NODE_BIN"), - "WORK_DIR": os.Getenv("SSH_WORK_DIR"), - "TIMEOUT": os.Getenv("SSH_TIMEOUT"), - "MAX_OUTPUT_BYTES": os.Getenv("SSH_MAX_OUTPUT_BYTES"), - "MAX_ARTIFACTS": os.Getenv("SSH_MAX_ARTIFACTS"), - "MAX_ARTIFACT_BYTES": os.Getenv("SSH_MAX_ARTIFACT_BYTES"), - "KNOWN_HOSTS": os.Getenv("SSH_KNOWN_HOSTS"), + "HOST": common.GetEnv(common.EnvSSHHost), + "PORT": common.GetEnv(common.EnvSSHPort), + "USERNAME": common.GetEnv(common.EnvSSHUsername), + "PASSWORD": common.GetEnv(common.EnvSSHPassword), + "PRIVATE_KEY": common.GetEnv(common.EnvSSHPrivateKey), + "PRIVATE_KEY_PATH": common.GetEnv(common.EnvSSHPrivateKeyPath), + "PASSPHRASE": common.GetEnv(common.EnvSSHPassphrase), + "PYTHON_BIN": common.GetEnv(common.EnvSSHPythonBin), + "NODE_BIN": common.GetEnv(common.EnvSSHNodeBin), + "WORK_DIR": common.GetEnv(common.EnvSSHWorkDir), + "TIMEOUT": common.GetEnv(common.EnvSSHTimeout), + "MAX_OUTPUT_BYTES": common.GetEnv(common.EnvSSHMaxOutputBytes), + "MAX_ARTIFACTS": common.GetEnv(common.EnvSSHMaxArtifacts), + "MAX_ARTIFACT_BYTES": common.GetEnv(common.EnvSSHMaxArtifactBytes), + "KNOWN_HOSTS": common.GetEnv(common.EnvSSHKnownHosts), } } diff --git a/internal/agent/tool/exesql_trino_stub.go b/internal/agent/tool/exesql_trino_stub.go index 58835c6bc9..68a56f6f75 100644 --- a/internal/agent/tool/exesql_trino_stub.go +++ b/internal/agent/tool/exesql_trino_stub.go @@ -43,7 +43,7 @@ package tool import ( "fmt" "net/url" - "os" + "ragflow/internal/common" "strconv" "strings" ) @@ -71,7 +71,7 @@ func splitTrinoCatalogSchema(db string) (catalog, schema string) { // design doc §10.1 + gap analysis §11.4.1 row 5c. func trinoDSN(p exesqlConnParams) string { scheme := "http" - if os.Getenv("TRINO_USE_TLS") != "" { + if common.GetEnv(common.EnvTrinoUseTls) != "" { scheme = "https" } port := p.Port diff --git a/internal/agent/tool/keenable.go b/internal/agent/tool/keenable.go index bbe6fdc5bb..5d946f6b69 100644 --- a/internal/agent/tool/keenable.go +++ b/internal/agent/tool/keenable.go @@ -22,7 +22,7 @@ import ( "fmt" "net/http" neturl "net/url" - "os" + "ragflow/internal/common" "strings" "github.com/cloudwego/eino/components/tool" @@ -137,7 +137,7 @@ func NewKeenableToolWithEnvBaseURL(h *HTTPHelper, envBaseURL func() string) *Kee // Pulled out as a named function (not a var) so tests cannot // accidentally mutate it via package-var assignment. func defaultKeenableEnvBaseURL() string { - if v := strings.TrimSpace(os.Getenv("KEENABLE_API_URL")); v != "" { + if v := strings.TrimSpace(common.GetEnv(common.EnvKeenableAPIURL)); v != "" { return v } return "https://api.keenable.ai" diff --git a/internal/agent/tool/ssrf.go b/internal/agent/tool/ssrf.go index e9c030d2ff..a40e7aee19 100644 --- a/internal/agent/tool/ssrf.go +++ b/internal/agent/tool/ssrf.go @@ -21,7 +21,7 @@ import ( "fmt" "net" "net/url" - "os" + "ragflow/internal/common" "strings" "ragflow/internal/utility" @@ -154,7 +154,7 @@ func isPrivateOrLoopback(ip net.IP) bool { } func allowAnyHost() bool { - switch strings.ToLower(strings.TrimSpace(os.Getenv("ALLOW_ANY_HOST"))) { + switch strings.ToLower(strings.TrimSpace(common.GetEnv(common.EnvAllowAnyHost))) { case "1", "true", "yes", "on": return true default: diff --git a/internal/agent/tool/tavily.go b/internal/agent/tool/tavily.go index b25b79c2bd..d692172643 100644 --- a/internal/agent/tool/tavily.go +++ b/internal/agent/tool/tavily.go @@ -21,7 +21,7 @@ import ( "encoding/json" "fmt" "net/http" - "os" + "ragflow/internal/common" "strings" "github.com/cloudwego/eino/components/tool" @@ -192,7 +192,7 @@ func NewTavilyExtractToolWithEnvKey(h *HTTPHelper, envKey func() string) *Tavily // defaultTavilyEnvKey is the production env-key resolver. Pulled out // as a named function (not a var) so tests cannot accidentally // mutate it via package-var assignment. -func defaultTavilyEnvKey() string { return os.Getenv("TAVILY_API_KEY") } +func defaultTavilyEnvKey() string { return common.GetEnv(common.EnvTavilyApiKey) } // Info returns the tool's metadata for the chat model. func (t *TavilyTool) Info(_ context.Context) (*schema.ToolInfo, error) { diff --git a/internal/cli/cli.go b/internal/cli/cli.go index 0738d3521f..b7aefedd87 100644 --- a/internal/cli/cli.go +++ b/internal/cli/cli.go @@ -21,6 +21,8 @@ import ( "errors" "fmt" "os" + "ragflow/internal/common" + //"os/signal" "path/filepath" "strconv" @@ -482,7 +484,7 @@ Commands: // HistoryFile returns the path to the history file func HistoryFile() string { - return os.Getenv("HOME") + "/" + historyFileName + return common.GetEnv(common.EnvHome) + "/" + historyFileName } const historyFileName = ".ragflow_cli_history" diff --git a/internal/cli/filesystem/skill_hub/source/interface.go b/internal/cli/filesystem/skill_hub/source/interface.go index 8cc8617ecf..9a24f0bc43 100644 --- a/internal/cli/filesystem/skill_hub/source/interface.go +++ b/internal/cli/filesystem/skill_hub/source/interface.go @@ -19,8 +19,8 @@ package source import ( "fmt" "net/url" - "os" "path/filepath" + "ragflow/internal/common" "strings" ) @@ -117,9 +117,9 @@ func (r *SourceResolver) Resolve(ref string) (SkillSource, string, error) { // getHomeDir returns the user's home directory func getHomeDir() (string, error) { - home := os.Getenv("HOME") + home := common.GetEnv(common.EnvHome) if home == "" { - home = os.Getenv("USERPROFILE") + home = common.GetEnv(common.EnvUserProfile) } if home == "" { return "", fmt.Errorf("cannot determine home directory") diff --git a/internal/cli/filesystem/skill_install.go b/internal/cli/filesystem/skill_install.go index 77dc922890..d42535c020 100644 --- a/internal/cli/filesystem/skill_install.go +++ b/internal/cli/filesystem/skill_install.go @@ -24,6 +24,7 @@ import ( "net/url" "os" "path/filepath" + "ragflow/internal/common" "strings" "time" @@ -79,10 +80,10 @@ func (a *sourceHTTPClientAdapter) Get(url string) (*http.Response, error) { // NewInstallSkillCommand creates a new install-skill command handler func NewInstallSkillCommand(client HTTPClientInterface, fileProvider *FileProvider, skillProvider Provider) *SkillInstallCommand { // Log proxy settings - if httpProxy := os.Getenv("http_proxy"); httpProxy != "" { + if httpProxy := common.GetEnv(common.EnvHttpHTTPProxy); httpProxy != "" { fmt.Printf("Using HTTP proxy: %s\n", httpProxy) } - if httpsProxy := os.Getenv("https_proxy"); httpsProxy != "" { + if httpsProxy := common.GetEnv(common.EnvHttpHTTPSProxy); httpsProxy != "" { fmt.Printf("Using HTTPS proxy: %s\n", httpsProxy) } diff --git a/internal/common/environments.go b/internal/common/environments.go new file mode 100644 index 0000000000..98dbd15c03 --- /dev/null +++ b/internal/common/environments.go @@ -0,0 +1,188 @@ +// +// Copyright 2026 The InfiniFlow Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +package common + +import ( + "os" + "strings" +) + +func GetEnv(key string) string { + return os.Getenv(key) +} + +func GetEnvSmall(key string) string { + return strings.ToLower(GetEnv(key)) +} + +// environment variables +const ( + EnvDeepDocURL = "DEEPDOC_URL" + EnvTensorrtDLAServer = "TENSORRT_DLA_SVR" + EnvRAGFlowTTSCacheTTLSeconds = "RAGFLOW_TTS_CACHE_TTL_SECONDS" + EnvComponentExecTimeout = "COMPONENT_EXEC_TIMEOUT" + EnvDocEngine = "DOC_ENGINE" + EnvMaxFileNumPerUser = "MAX_FILE_NUM_PER_USER" + EnvRAGFlowDictPath = "RAGFLOW_DICT_PATH" + EnvDefaultSuperuserEmail = "DEFAULT_SUPERUSER_EMAIL" + EnvDefaultSuperuserNickname = "DEFAULT_SUPERUSER_NICKNAME" + EnvDefaultSuperuserPassword = "DEFAULT_SUPERUSER_PASSWORD" + EnvDBType = "DB_TYPE" + EnvDevice = "DEVICE" + EnvStorageImpl = "STORAGE_IMPL" + EnvStageHandCacheCap = "STAGEHAND_CACHE_CAP" + EnvStageHandCacheTTLSeconds = "STAGEHAND_CACHE_TTL_SECONDS" + EnvStageHandCacheSweepInterval = "STAGEHAND_CACHE_SWEEP_INTERVAL" + EnvStageHandExtractResultFile = "STAGEHAND_EXTRACT_RESULT_FILE" + EnvAgentRunAccessKeyID = "AGENTRUN_ACCESS_KEY_ID" + EnvAgentRunAccessKeySecret = "AGENTRUN_ACCESS_KEY_SECRET" + EnvAgentRunAccountID = "AGENTRUN_ACCOUNT_ID" + EnvAgentRunRegion = "AGENTRUN_REGION" + EnvAgentRunTemplateName = "AGENTRUN_TEMPLATE_NAME" + EnvAgentRunExecuteHost = "AGENTRUN_EXECUTE_HOST" + EnvAgentRunTimeout = "AGENTRUN_TIMEOUT" + EnvE2BTemplate = "E2B_TEMPLATE" + EnvE2BTemplateName = "E2B_TEMPLATE_NAME" + EnvE2BTimeout = "E2B_TIMEOUT" + EnvE2BAPIURL = "E2B_API_URL" + EnvE2BApiKey = "E2B_API_KEY" + EnvE2BAccessToken = "E2B_ACCESS_TOKEN" + EnvE2BDomain = "E2B_DOMAIN" + EnvLocalPythonBin = "LOCAL_PYTHON_BIN" + EnvLocalNodeBin = "LOCAL_NODE_BIN" + EnvLocalWorkDir = "LOCAL_WORK_DIR" + EnvLocalTimeout = "LOCAL_TIMEOUT" + EnvLocalMaxMemoryMB = "LOCAL_MAX_MEMORY_MB" + EnvLocalMaxOutputBytes = "LOCAL_MAX_OUTPUT_BYTES" + EnvLocalMaxArtifacts = "LOCAL_MAX_ARTIFACTS" + EnvLocalMaxArtifactBytes = "LOCAL_MAX_ARTIFACT_BYTES" + EnvPath = "PATH" + EnvOMPNumThreads = "OMP_NUM_THREADS" + EnvOpenBLASNumThreads = "OPENBLAS_NUM_THREADS" + EnvMKLNumThreads = "MKL_NUM_THREADS" + EnvVECLIBMaximumThreads = "VECLIB_MAXIMUM_THREADS" + EnvNumEXPRNumThreads = "NUMEXPR_NUM_THREADS" + EnvXDGCacheHome = "XDG_CACHE_HOME" + EnvOpenAIApiKey = "OPENAI_API_KEY" + EnvOpenAIBaseURL = "OPENAI_BASE_URL" + EnvOpenAIModel = "OPENAI_MODEL" + EnvStageHandExtractSchemaJSON = "STAGEHAND_EXTRACT_SCHEMA_JSON" + EnvSandboxProviderType = "SANDBOX_PROVIDER_TYPE" + EnvSandboxExecutorManagerURL = "SANDBOX_EXECUTOR_MANAGER_URL" + EnvSandboxExecutorManagerTimeout = "SANDBOX_EXECUTOR_MANAGER_TIMEOUT" + EnvSandboxExecutorManagerPoolSize = "SANDBOX_EXECUTOR_MANAGER_POOL_SIZE" + EnvSandboxExecutorManagerMaxRetries = "SANDBOX_EXECUTOR_MANAGER_MAX_RETRIES" + EnvSandboxBasePythonImage = "SANDBOX_BASE_PYTHON_IMAGE" + EnvSandboxBaseNodeJSImage = "SANDBOX_BASE_NODEJS_IMAGE" + EnvSandboxArtifactBucket = "SANDBOX_ARTIFACT_BUCKET" + EnvSSHHost = "SSH_HOST" + EnvSSHPort = "SSH_PORT" + EnvSSHUsername = "SSH_USERNAME" + EnvSSHPassword = "SSH_PASSWORD" + EnvSSHPrivateKey = "SSH_PRIVATE_KEY" + EnvSSHPrivateKeyPath = "SSH_PRIVATE_KEY_PATH" + EnvSSHPassphrase = "SSH_PASSPHRASE" + EnvSSHPythonBin = "SSH_PYTHON_BIN" + EnvSSHNodeBin = "SSH_NODE_BIN" + EnvSSHWorkDir = "SSH_WORK_DIR" + EnvSSHTimeout = "SSH_TIMEOUT" + EnvSSHMaxOutputBytes = "SSH_MAX_OUTPUT_BYTES" + EnvSSHMaxArtifacts = "SSH_MAX_ARTIFACTS" + EnvSSHMaxArtifactBytes = "SSH_MAX_ARTIFACT_BYTES" + EnvSSHKnownHosts = "SSH_KNOWN_HOSTS" + EnvTrinoUseTls = "TRINO_USE_TLS" + EnvSSHEnableAPIURL = "SSH_ENABLE_API_URL" + EnvAllowAnyHost = "ALLOW_ANY_HOST" + EnvTavilyApiKey = "TAVILY_API_KEY" + EnvHome = "HOME" + EnvUserProfile = "USERPROFILE" + EnvHttpHTTPProxy = "http_proxy" + EnvHttpHTTPSProxy = "https_proxy" + EnvBatchSingle = "BATCH_SINGLE" + EnvBatchCount = "BATCH_COUNT" + EnvBatchLogLevel = "BATCH_LOG_LEVEL" + EnvBatchSkipOCR = "BATCH_SKIP_OCR" + EnvBatchCompareOnly = "BATCH_COMPARE_ONLY" + EnvBatchCompareFilter = "BATCH_COMPARE_FILTER" + EnvBatchCompareCSV = "BATCH_COMPARE_CSV" + EnvPYOCRSuffix = "PY_OCR_SUFFIX" + EnvOSSDeepDocURL = "OSSDEEPDOC_URL" + EnvUpdateGolden = "UPDATE_GOLDEN" + EnvBatchParityFilter = "BATCH_PARITY_FILTER" + EnvDumpCount = "DUMP_COUNT" + EnvBatchCSV = "BATCH_CSV" + EnvESTest = "ES_TEST" + EnvESHost = "ES_HOST" + EnvESUsername = "ES_USERNAME" + EnvESPassword = "ES_PASSWORD" + EnvESIndexPrefix = "ES_INDEX_PREFIX" + EnvGiteeListModelsIntegration = "GITEE_LIST_MODELS_INTEGRATION" + EnvGiteeBaseUrl = "GITEE_BASE_URL" + EnvGiteeApiKey = "GITEE_API_KEY" + EnvRAGFlowApiTiming = "RAGFLOW_API_TIMING" + EnvInfinityURI = "INFINITY_URI" + EnvDoclingServerURL = "DOCLING_SERVER_URL" + EnvDoclingApiKey = "DOCLING_API_KEY" + EnvMineruApiServer = "MINERU_APISERVER" + EnvMineruApiKey = "MINERU_API_KEY" + EnvMineruBackend = "MINERU_BACKEND" + EnvOpenDataLoaderApiServer = "OPENDATALOADER_APISERVER" + EnvOpenDataLoaderApiKey = "OPENDATALOADER_API_KEY" + EnvPaddleOCRBaseUrl = "PADDLEOCR_BASE_URL" + EnvPaddleOCRApiURL = "PADDLEOCR_API_URL" + EnvPaddleOCRAccessToken = "PADDLEOCR_ACCESS_TOKEN" + EnvPaddleOCRAlgorithm = "PADDLEOCR_ALGORITHM" + EnvSOMarkBaseUrl = "SOMARK_BASE_URL" + EnvSOMarkApiKey = "SOMARK_API_KEY" + EnvSOMarkImageFormat = "SOMARK_IMAGE_FORMAT" + EnvSOMarkFormulaFormat = "SOMARK_FORMULA_FORMAT" + EnvSOMarkTableFormat = "SOMARK_TABLE_FORMAT" + EnvSOMarkCSFormat = "SOMARK_CS_FORMAT" + EnvSOMarkEnableTextCrossPage = "SOMARK_ENABLE_TEXT_CROSS_PAGE" + EnvSOMarkEnableTableCrossPage = "SOMARK_ENABLE_TABLE_CROSS_PAGE" + EnvSOMarkEnableTitleLevelRecognition = "SOMARK_ENABLE_TITLE_LEVEL_RECOGNITION" + EnvSOMarkEnableInlineImage = "SOMARK_ENABLE_INLINE_IMAGE" + EnvSOMarkEnableTableImage = "SOMARK_ENABLE_TABLE_IMAGE" + EnvSOMarkEnableImageUnderstanding = "SOMARK_ENABLE_IMAGE_UNDERSTANDING" + EnvSOMarkKeepHeaderFooter = "SOMARK_KEEP_HEADER_FOOTER" + EnvTCADPApiServerURL = "TCADP_APISERVER_URL" + EnvTCADPApiKey = "TCADP_API_KEY" + EnvRAGFlowSecretKey = "RAGFLOW_SECRET_KEY" + EnvRegisterEnabled = "REGISTER_ENABLED" + EnvDisablePasswordLogin = "DISABLE_PASSWORD_LOGIN" + EnvMinioHost = "MINIO_HOST" + EnvMinioRegion = "MINIO_REGION" + EnvLang = "LANG" + EnvLanguage = "LANGUAGE" + EnvChunkFeedbackEnabled = "CHUNK_FEEDBACK_ENABLED" + EnvChunkFeedbackWeighting = "CHUNK_FEEDBACK_WEIGHTING" + EnvComposeProfiles = "COMPOSE_PROFILES" + EnvTEIModel = "TEI_MODEL" + EnvTEIBaseURL = "TEI_BASE_URL" + EnvRAGFlowConfDir = "RAGFLOW_CONF_DIR" + EnvRAGProjectBaseURL = "RAG_PROJECT_BASE" + EnvRAGDeployBaseURL = "RAG_DEPLOY_BASE" + EnvRAGFlowTestEnvIntOrUnset = "RAGFLOW_TEST_ENVINTOR_UNSET" + EnvRAGFlowTestEnvIntOr = "RAGFLOW_TEST_ENVINTOR" + EnvRAGFlowTestEnvOr = "RAGFLOW_TEST_ENVOR" + EnvRAGFlowTestEnvOrUnset = "RAGFLOW_TEST_ENVOR_UNSET" + EnvKeenableAPIURL = "KEENABLE_API_URL" + EnvBoxWebOAuthRedirectURI = "BOX_WEB_OAUTH_REDIRECT_URI" + EnvGmailWebOAuthRedirectURI = "GMAIL_WEB_OAUTH_REDIRECT_URI" + EnvGoogleDriveWebOAuthRedirectURI = "GOOGLE_DRIVE_WEB_OAUTH_REDIRECT_URI" + EnvSpacyModelDir = "SPACY_MODEL_DIR" +) diff --git a/internal/deepdoc/client.go b/internal/deepdoc/client.go index 497e149fff..6c25c1ecaf 100644 --- a/internal/deepdoc/client.go +++ b/internal/deepdoc/client.go @@ -29,7 +29,7 @@ import ( "errors" "io" "net/http" - "os" + "ragflow/internal/common" "time" "go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp" @@ -101,9 +101,9 @@ func WithBackoff(d time.Duration) Option { // (legacy alias per deepdoc/vision/layout_recognizer.py:52). When // both are unset, Enabled() reports false. func NewClient(opts ...Option) *Client { - url := os.Getenv("DEEPDOC_URL") + url := common.GetEnv(common.EnvDeepDocURL) if url == "" { - url = os.Getenv("TENSORRT_DLA_SVR") + url = common.GetEnv(common.EnvTensorrtDLAServer) } return NewClientWithURL(url, opts...) } diff --git a/internal/deepdoc/parser/pdf/batch_smoke_test.go b/internal/deepdoc/parser/pdf/batch_smoke_test.go index aee547eecd..6b8678d7bc 100644 --- a/internal/deepdoc/parser/pdf/batch_smoke_test.go +++ b/internal/deepdoc/parser/pdf/batch_smoke_test.go @@ -10,6 +10,7 @@ import ( "math" "os" "path/filepath" + "ragflow/internal/common" "regexp" "sort" "strconv" @@ -44,14 +45,14 @@ func TestBatchResults(t *testing.T) { pdfDir := filepath.Join("testdata", "real_pdfs") all := listRealPDFs(t, pdfDir) - count := countFromEnv("BATCH_COUNT", len(all)) - if single := os.Getenv("BATCH_SINGLE"); single != "" { + count := countFromEnv(common.EnvBatchCount, len(all)) + if single := common.GetEnv(common.EnvBatchSingle); single != "" { all = filterSingle(all, single, t) count = 1 } pdfs := all[:min(count, len(all))] - ddClient, err := inf.NewClient(os.Getenv("DEEPDOC_URL")) + ddClient, err := inf.NewClient(common.GetEnv(common.EnvDeepDocURL)) if err != nil { t.Fatal(err) } @@ -73,7 +74,7 @@ func TestBatchResults(t *testing.T) { func setupLogger() { level := slog.LevelInfo - switch os.Getenv("BATCH_LOG_LEVEL") { + switch common.GetEnv(common.EnvBatchLogLevel) { case "debug": level = slog.LevelDebug case "warn": @@ -83,7 +84,7 @@ func setupLogger() { } func variantFromEnv() string { - if os.Getenv("BATCH_SKIP_OCR") == "1" { + if common.GetEnv(common.EnvBatchSkipOCR) == "1" { return "noocr" } return "ocr" @@ -106,7 +107,7 @@ func mkOutputDirs(variant string) outputDirs { } func countFromEnv(key string, ceiling int) int { - if s := os.Getenv(key); s != "" { + if s := common.GetEnv(key); s != "" { n, err := strconv.Atoi(s) if err == nil && n > 0 && n < ceiling { return n @@ -178,7 +179,7 @@ func processPDFs(t *testing.T, pdfDir string, pdfs []string, deepDoc pdf.DocAnal t.Helper() var results []tool.BatchResult totalChars := 0 - skipOCR := os.Getenv("BATCH_SKIP_OCR") == "1" + skipOCR := common.GetEnv(common.EnvBatchSkipOCR) == "1" for i, name := range pdfs { label := fmt.Sprintf("[%d/%d] %s", i+1, len(pdfs), name) diff --git a/internal/deepdoc/parser/pdf/compare_test.go b/internal/deepdoc/parser/pdf/compare_test.go index 6df639ce48..a4fd628f91 100644 --- a/internal/deepdoc/parser/pdf/compare_test.go +++ b/internal/deepdoc/parser/pdf/compare_test.go @@ -6,6 +6,7 @@ import ( "log/slog" "os" "path/filepath" + "ragflow/internal/common" "testing" "ragflow/internal/deepdoc/parser/pdf/tool" @@ -17,19 +18,19 @@ import ( // compare the noocr variant; PY_OCR_SUFFIX to override the Python variant. func TestBatchCompareWithPython(t *testing.T) { level := slog.LevelInfo - if os.Getenv("BATCH_LOG_LEVEL") == "debug" { + if common.GetEnv(common.EnvBatchLogLevel) == "debug" { level = slog.LevelDebug } - if os.Getenv("BATCH_LOG_LEVEL") == "warn" { + if common.GetEnv(common.EnvBatchLogLevel) == "warn" { level = slog.LevelWarn } slog.SetDefault(slog.New(slog.NewTextHandler(os.Stderr, &slog.HandlerOptions{Level: level}))) goVariant := "ocr" - if os.Getenv("BATCH_SKIP_OCR") == "1" { + if common.GetEnv(common.EnvBatchSkipOCR) == "1" { goVariant = "noocr" } - pyVariant := os.Getenv("PY_OCR_SUFFIX") + pyVariant := common.GetEnv(common.EnvPYOCRSuffix) if pyVariant == "" { pyVariant = goVariant } diff --git a/internal/deepdoc/parser/pdf/helpers_test.go b/internal/deepdoc/parser/pdf/helpers_test.go index 7b8142eb7c..0d649baaa8 100644 --- a/internal/deepdoc/parser/pdf/helpers_test.go +++ b/internal/deepdoc/parser/pdf/helpers_test.go @@ -5,6 +5,7 @@ package pdf import ( "os" "path/filepath" + "ragflow/internal/common" "testing" inf "ragflow/internal/deepdoc/parser/pdf/inference" @@ -14,7 +15,7 @@ import ( // mustConnectInferenceClient returns a InferenceClient for the OSS DeepDoc service. func mustConnectInferenceClient(t *testing.T) *inf.Client { t.Helper() - url := os.Getenv("OSSDEEPDOC_URL") + url := common.GetEnv(common.EnvOSSDeepDocURL) if url == "" { url = "http://localhost:9390" } diff --git a/internal/deepdoc/parser/pdf/ocr_merge_test.go b/internal/deepdoc/parser/pdf/ocr_merge_test.go index 357b105f62..cbea6a50a0 100644 --- a/internal/deepdoc/parser/pdf/ocr_merge_test.go +++ b/internal/deepdoc/parser/pdf/ocr_merge_test.go @@ -18,7 +18,7 @@ import ( // instead of real text. This validates that detect+merge+recognize // produces readable English from the scan. func TestOCR_mergeChars_RealScanned(t *testing.T) { - url := os.Getenv("DEEPDOC_URL") + url := common.GetEnv(common.EnvDeepDocURL) if url == "" { t.Skip("DEEPDOC_URL not set") } diff --git a/internal/deepdoc/parser/pdf/parser_pipeline_integration_test.go b/internal/deepdoc/parser/pdf/parser_pipeline_integration_test.go index 4c5c082536..87decd1f0d 100644 --- a/internal/deepdoc/parser/pdf/parser_pipeline_integration_test.go +++ b/internal/deepdoc/parser/pdf/parser_pipeline_integration_test.go @@ -66,7 +66,7 @@ func writeGolden(t *testing.T, path string, v any) { } func updateGolden() bool { - return os.Getenv("UPDATE_GOLDEN") == "1" + return common.GetEnv(common.EnvUpdateGolden) == "1" } // sectionsToGolden converts []pdf.Section to the snapshot format. diff --git a/internal/deepdoc/parser/pdf/pipeline_parity_test.go b/internal/deepdoc/parser/pdf/pipeline_parity_test.go index 4c000ae4a3..290eabb1cd 100644 --- a/internal/deepdoc/parser/pdf/pipeline_parity_test.go +++ b/internal/deepdoc/parser/pdf/pipeline_parity_test.go @@ -31,7 +31,7 @@ func TestPipelineParity(t *testing.T) { t.Skipf("charspy/ not found: %v", err) } - filter := os.Getenv("BATCH_PARITY_FILTER") + filter := common.GetEnv(common.EnvBatchParityFilter) total, passed := 0, 0 for _, e := range entries { diff --git a/internal/deepdoc/parser/pdf/table_rotate_integration_test.go b/internal/deepdoc/parser/pdf/table_rotate_integration_test.go index 6aa11fc603..b39f67a5c7 100644 --- a/internal/deepdoc/parser/pdf/table_rotate_integration_test.go +++ b/internal/deepdoc/parser/pdf/table_rotate_integration_test.go @@ -29,7 +29,7 @@ func TestTableRotation_Integration(t *testing.T) { t.Skipf("test PDF not found: %s (run tools/generate_rotated_table_pdf.py first)", pdfPath) } - baseURL := os.Getenv("DEEPDOC_URL") + baseURL := common.GetEnv(common.EnvDeepDocURL) if baseURL == "" { baseURL = "http://localhost:9390" } @@ -126,7 +126,7 @@ func TestTableRotation_Integration(t *testing.T) { // TestTableRotation_Stability runs rotation detection on a sample real PDF // and verifies the pipeline doesn't crash. Set BATCH_COUNT to limit. func TestTableRotation_Stability(t *testing.T) { - baseURL := os.Getenv("DEEPDOC_URL") + baseURL := common.GetEnv(common.EnvDeepDocURL) if baseURL == "" { baseURL = "http://localhost:9390" } diff --git a/internal/deepdoc/parser/pdf/text_dump_test.go b/internal/deepdoc/parser/pdf/text_dump_test.go index baa7fabd3b..73a16af147 100644 --- a/internal/deepdoc/parser/pdf/text_dump_test.go +++ b/internal/deepdoc/parser/pdf/text_dump_test.go @@ -24,7 +24,7 @@ func TestDumpTextOutput(t *testing.T) { } count := len(entries) - if n := os.Getenv("DUMP_COUNT"); n != "" { + if n := common.GetEnv(common.EnvDumpCount); n != "" { c := 0 for _, ch := range n { c = c*10 + int(ch-'0') diff --git a/internal/deepdoc/parser/pdf/tool/compare.go b/internal/deepdoc/parser/pdf/tool/compare.go index d81209d5f3..0e169965cb 100644 --- a/internal/deepdoc/parser/pdf/tool/compare.go +++ b/internal/deepdoc/parser/pdf/tool/compare.go @@ -7,6 +7,7 @@ import ( "math" "os" "path/filepath" + "ragflow/internal/common" "sort" "strconv" "strings" @@ -191,7 +192,7 @@ func CompareWithPython(log TLogger, goResults []BatchResult, pyResults []PyResul } // Also write CSV if BATCH_CSV env is set (backward compat). - if csvPath := os.Getenv("BATCH_CSV"); csvPath != "" { + if csvPath := common.GetEnv(common.EnvBatchCSV); csvPath != "" { if err := WriteCSV(csvPath, diffs); err != nil { log.Logf("CSV write error: %v", err) } else { diff --git a/internal/deepdoc/parser/pdf/tool/compare_test.go b/internal/deepdoc/parser/pdf/tool/compare_test.go index e96eec76c7..bd2dee92f8 100644 --- a/internal/deepdoc/parser/pdf/tool/compare_test.go +++ b/internal/deepdoc/parser/pdf/tool/compare_test.go @@ -15,19 +15,19 @@ import ( // compare the noocr variant; PY_OCR_SUFFIX to override the Python variant. func TestBatchCompareWithPython(t *testing.T) { level := slog.LevelInfo - if os.Getenv("BATCH_LOG_LEVEL") == "debug" { + if common.GetEnv(common.EnvBatchLogLevel) == "debug" { level = slog.LevelDebug } - if os.Getenv("BATCH_LOG_LEVEL") == "warn" { + if common.GetEnv(common.EnvBatchLogLevel) == "warn" { level = slog.LevelWarn } slog.SetDefault(slog.New(slog.NewTextHandler(os.Stderr, &slog.HandlerOptions{Level: level}))) goVariant := "ocr" - if os.Getenv("BATCH_SKIP_OCR") == "1" { + if common.GetEnv(common.EnvBatchSkipOCR) == "1" { goVariant = "noocr" } - pyVariant := os.Getenv("PY_OCR_SUFFIX") + pyVariant := common.GetEnv(common.EnvPYOCRSuffix) if pyVariant == "" { pyVariant = goVariant } diff --git a/internal/deepdoc/parser/pdf/tool/config.go b/internal/deepdoc/parser/pdf/tool/config.go index 8c2828619d..665379eba4 100644 --- a/internal/deepdoc/parser/pdf/tool/config.go +++ b/internal/deepdoc/parser/pdf/tool/config.go @@ -4,6 +4,7 @@ import ( "fmt" "os" "path/filepath" + "ragflow/internal/common" "strconv" "time" ) @@ -26,12 +27,12 @@ func LoadConfig() Config { pyVariant := "ocr" td := filepath.Join("testdata") return Config{ - Count: envInt("BATCH_COUNT", 0), - Single: os.Getenv("BATCH_SINGLE"), - SkipOCR: os.Getenv("BATCH_SKIP_OCR") == "1", - CompareOnly: os.Getenv("BATCH_COMPARE_ONLY") == "1", - CompareFilter: os.Getenv("BATCH_COMPARE_FILTER"), - CSVOutput: envStr("BATCH_COMPARE_CSV", filepath.Join(td, "output", fmt.Sprintf("compare_%s.csv", time.Now().Format("20060102_150405")))), + Count: envInt(common.EnvBatchCount, 0), + Single: common.GetEnv(common.EnvBatchSingle), + SkipOCR: common.GetEnv(common.EnvBatchSkipOCR) == "1", + CompareOnly: common.GetEnv(common.EnvBatchCompareOnly) == "1", + CompareFilter: common.GetEnv(common.EnvBatchCompareFilter), + CSVOutput: envStr(common.EnvBatchCompareCSV, filepath.Join(td, "output", fmt.Sprintf("compare_%s.csv", time.Now().Format("20060102_150405")))), GoTextDir: filepath.Join(td, "output", "go", goVariant, "text"), PyTextDir: filepath.Join(td, "output", "py", pyVariant, "text"), TablesDir: filepath.Join(td, "output", "go", goVariant, "tables"), @@ -40,7 +41,7 @@ func LoadConfig() Config { } func envInt(key string, def int) int { - v := os.Getenv(key) + v := common.GetEnv(key) if v == "" { return def } @@ -52,7 +53,7 @@ func envInt(key string, def int) int { } func envStr(key, def string) string { - v := os.Getenv(key) + v := common.GetEnv(key) if v == "" { return def } diff --git a/internal/engine/elasticsearch/kg_test.go b/internal/engine/elasticsearch/kg_test.go index 3fa8464c97..9e4941cc5c 100644 --- a/internal/engine/elasticsearch/kg_test.go +++ b/internal/engine/elasticsearch/kg_test.go @@ -18,7 +18,7 @@ package elasticsearch import ( "context" - "os" + "ragflow/internal/common" "testing" "ragflow/internal/engine/types" @@ -29,7 +29,7 @@ import ( // Requires a running Elasticsearch instance and KG data indexed by Python task executor. // Set ES_TEST=1 to run. func TestKGSearchSelectFields(t *testing.T) { - if os.Getenv("ES_TEST") != "1" { + if common.GetEnv(common.EnvESTest) != "1" { t.Skip("Skipping ES integration test; set ES_TEST=1 to run") } @@ -74,15 +74,15 @@ func TestKGSearchSelectFields(t *testing.T) { // getTestConfig returns a minimal ES config for testing. // Reads from environment or uses defaults pointing to localhost. func getTestConfig() map[string]interface{} { - hosts := os.Getenv("ES_HOSTS") + hosts := common.GetEnv(common.EnvESHost) if hosts == "" { hosts = "http://localhost:1200" } - username := os.Getenv("ES_USERNAME") + username := common.GetEnv(common.EnvESUsername) if username == "" { username = "elastic" } - password := os.Getenv("ES_PASSWORD") + password := common.GetEnv(common.EnvESPassword) if password == "" { password = "infini_rag_flow" } diff --git a/internal/entity/models/builtin.go b/internal/entity/models/builtin.go index 1e7f8788d4..738bb6d525 100644 --- a/internal/entity/models/builtin.go +++ b/internal/entity/models/builtin.go @@ -6,7 +6,7 @@ import ( "fmt" "io" "net/http" - "os" + "ragflow/internal/common" "strings" "time" ) @@ -180,7 +180,7 @@ func (b *BuiltinModel) ShowTask(taskID string, apiConfig *APIConfig) (*TaskRespo func GetBuiltinEmbeddingModel(modelName string) ModelDriver { // Get TEI base URL from environment or config // Default to port 6380 where docker-tei-cpu-1 maps internal port 80 - teiBaseURL := getEnv("TEI_BASE_URL", "http://localhost:6380") + teiBaseURL := getEnv(common.EnvTEIBaseURL, "http://localhost:6380") // Create a builtin model instance with TEI endpoint driver := NewBuiltinModel(teiBaseURL, modelName) @@ -189,7 +189,7 @@ func GetBuiltinEmbeddingModel(modelName string) ModelDriver { // getEnv is a helper to get environment variable with default func getEnv(key, defaultValue string) string { - if value := strings.TrimSpace(strings.Replace(os.Getenv(key), "\\", "/", -1)); value != "" { + if value := strings.TrimSpace(strings.Replace(common.GetEnv(key), "\\", "/", -1)); value != "" { return value } return defaultValue diff --git a/internal/entity/models/gitee_test.go b/internal/entity/models/gitee_test.go index 33839891ac..1c653091e2 100644 --- a/internal/entity/models/gitee_test.go +++ b/internal/entity/models/gitee_test.go @@ -24,6 +24,7 @@ import ( "net/http/httptest" "os" "path/filepath" + "ragflow/internal/common" "sort" "strings" "testing" @@ -181,18 +182,18 @@ func TestGiteeListModelsKeepsOwnedBySuffixAfterAliasMetadataLookup(t *testing.T) } func TestGiteeListModelsIntegration(t *testing.T) { - if os.Getenv("GITEE_LIST_MODELS_INTEGRATION") != "1" { + if common.GetEnv(common.EnvGiteeListModelsIntegration) != "1" { t.Skip("set GITEE_LIST_MODELS_INTEGRATION=1 to call the real Gitee models endpoint") } initProviderManagerWithGiteeForTest(t) - baseURL := os.Getenv("GITEE_BASE_URL") + baseURL := common.GetEnv(common.EnvGiteeBaseUrl) if baseURL == "" { baseURL = "https://api.moark.ai/v1" } apiConfig := &APIConfig{} - if apiKey := os.Getenv("GITEE_API_KEY"); apiKey != "" { + if apiKey := common.GetEnv(common.EnvGiteeApiKey); apiKey != "" { apiConfig.ApiKey = &apiKey } diff --git a/internal/handler/memory.go b/internal/handler/memory.go index ec5a7da3bc..291e322f1b 100644 --- a/internal/handler/memory.go +++ b/internal/handler/memory.go @@ -23,7 +23,6 @@ import ( "encoding/json" "errors" "net/http" - "os" "strconv" "strings" "time" @@ -83,7 +82,7 @@ func NewMemoryHandler(memoryService *service.MemoryService) *MemoryHandler { func (h *MemoryHandler) CreateMemory(c *gin.Context) { // Check if API timing is enabled // If RAGFLOW_API_TIMING environment variable is set, request processing time will be logged - timingEnabled := os.Getenv("RAGFLOW_API_TIMING") + timingEnabled := common.GetEnv(common.EnvRAGFlowApiTiming) var tStart time.Time if timingEnabled != "" { tStart = time.Now() diff --git a/internal/harness/core/interrupt.go b/internal/harness/core/interrupt.go index 99cdb41b5a..e0d9a7e81a 100644 --- a/internal/harness/core/interrupt.go +++ b/internal/harness/core/interrupt.go @@ -10,8 +10,6 @@ import ( "encoding/gob" "errors" "fmt" - "os" - "ragflow/internal/common" "ragflow/internal/harness/core/schema" @@ -224,7 +222,7 @@ const ( var checkpointHMACKey = loadCheckpointHMACKey() func loadCheckpointHMACKey() []byte { - if env := os.Getenv(envHMACKey); env != "" { + if env := common.GetEnv(envHMACKey); env != "" { k, err := base64.StdEncoding.DecodeString(env) if err != nil { panic("checkpoint HMAC key: invalid base64 in " + envHMACKey + ": " + err.Error()) diff --git a/internal/ingestion/compilation/extractor/ner.go b/internal/ingestion/compilation/extractor/ner.go index 0d8ba8fb46..dcb2246524 100644 --- a/internal/ingestion/compilation/extractor/ner.go +++ b/internal/ingestion/compilation/extractor/ner.go @@ -167,7 +167,7 @@ func (e *Extractor) Extract(text string, extractRelations bool) (*ExtractionResu tokensJSON := tokenizeText(text, e.Lang) if tokensJSON != "" { var rawTokens []string - if err := json.Unmarshal([]byte(tokensJSON), &rawTokens); err == nil { + if err = json.Unmarshal([]byte(tokensJSON), &rawTokens); err == nil { for i, t := range rawTokens { tokensMeta = append(tokensMeta, map[string]interface{}{ "text": t, @@ -293,7 +293,7 @@ func (e *Extractor) ExtractEntities(text string) ([]Entity, error) { End int `json:"end"` Confidence float64 `json:"confidence"` } - if err := json.Unmarshal([]byte(resultJSON), &rawEntities); err != nil { + if err = json.Unmarshal([]byte(resultJSON), &rawEntities); err != nil { return nil, fmt.Errorf("failed to parse NER result: %w", err) } @@ -309,7 +309,7 @@ func (e *Extractor) ExtractEntities(text string) ([]Entity, error) { if re.Confidence < e.ConfidenceThreshold { continue } - text := re.Text + text = re.Text if isCJK { text = strings.ReplaceAll(text, " ", "") } @@ -345,7 +345,7 @@ func (e *Extractor) findModelDir() string { if dirExists(base) { return base } - if p := getenv("SPACY_MODEL_DIR"); p != "" { + if p := common.GetEnv(common.EnvSpacyModelDir); p != "" { return p } return base @@ -372,10 +372,6 @@ func tokenizeText(text, lang string) string { return C.GoString(cTokens) } -func getenv(key string) string { - return os.Getenv(key) -} - // DetectLanguage detects text language based on Unicode ranges. // Pure Go, zero dependencies. func DetectLanguage(text string) string { diff --git a/internal/ingestion/component/tokenizer_test.go b/internal/ingestion/component/tokenizer_test.go index 478a64e495..4e550d9ac8 100644 --- a/internal/ingestion/component/tokenizer_test.go +++ b/internal/ingestion/component/tokenizer_test.go @@ -27,6 +27,7 @@ import ( "log" "math" "os" + "ragflow/internal/common" "reflect" "strings" "sync/atomic" @@ -54,7 +55,7 @@ var tokenizerPoolInitErr error // initialized. func TestMain(m *testing.M) { cfg := &tokenizer.PoolConfig{ - DictPath: os.Getenv("RAGFLOW_DICT_PATH"), + DictPath: common.GetEnv(common.EnvRAGFlowDictPath), MinSize: 1, MaxSize: 2, IdleTimeout: 30 * time.Second, diff --git a/internal/ingestion/pipeline/real_storage_integration_test.go b/internal/ingestion/pipeline/real_storage_integration_test.go index 8faa6d31cb..9f5b75f976 100644 --- a/internal/ingestion/pipeline/real_storage_integration_test.go +++ b/internal/ingestion/pipeline/real_storage_integration_test.go @@ -196,18 +196,18 @@ func mustLoadRealIntegrationConfig(t *testing.T) *server.Config { func prepareTokenizerResourceForIntegration(t *testing.T) { t.Helper() - if os.Getenv("RAGFLOW_DICT_PATH") != "" { + if common.GetEnv(common.EnvRAGFlowDictPath) != "" { return } const systemDictPath = "/usr/share/infinity/resource" if _, err := os.Stat(filepath.Join(systemDictPath, "rag", "huqie.txt")); err != nil { t.Skipf("system tokenizer resource not found at %s: %v", systemDictPath, err) } - if err := os.Setenv("RAGFLOW_DICT_PATH", systemDictPath); err != nil { + if err := os.Setenv(common.EnvRAGFlowDictPath, systemDictPath); err != nil { t.Fatalf("set RAGFLOW_DICT_PATH=%s: %v", systemDictPath, err) } t.Cleanup(func() { - _ = os.Unsetenv("RAGFLOW_DICT_PATH") + _ = os.Unsetenv(common.EnvRAGFlowDictPath) }) } diff --git a/internal/ingestion/pipeline/template_integration_test.go b/internal/ingestion/pipeline/template_integration_test.go index 5167b439f2..dfb5dc966f 100644 --- a/internal/ingestion/pipeline/template_integration_test.go +++ b/internal/ingestion/pipeline/template_integration_test.go @@ -677,9 +677,9 @@ func TestPipelineRun_TemplateBook_RealComponents(t *testing.T) { func TestPipelineRun_TemplateResume_RealComponents(t *testing.T) { RequireTokenizerPool(t) - apiKey := os.Getenv("OPENAI_API_KEY") - baseURL := os.Getenv("OPENAI_BASE_URL") - model := os.Getenv("OPENAI_MODEL") + apiKey := common.GetEnv(common.EnvOpenAIApiKey) + baseURL := common.GetEnv(common.EnvOpenAIBaseUrl) + model := common.GetEnv(common.EnvOpenAIModel) if apiKey == "" || baseURL == "" || model == "" { t.Skip("missing required env (OPENAI_API_KEY/OPENAI_BASE_URL/OPENAI_MODEL); skipping real resume extractor integration test") } diff --git a/internal/ingestion/pipeline/test_helpers.go b/internal/ingestion/pipeline/test_helpers.go index 85457c4ac8..56834fa490 100644 --- a/internal/ingestion/pipeline/test_helpers.go +++ b/internal/ingestion/pipeline/test_helpers.go @@ -17,8 +17,8 @@ package pipeline import ( "encoding/json" "math" - "os" "path/filepath" + "ragflow/internal/common" goruntime "runtime" "sort" "strings" @@ -43,7 +43,7 @@ func RequireTokenizerPool(t *testing.T) { return } cfg := &tokenizer.PoolConfig{ - DictPath: os.Getenv("RAGFLOW_DICT_PATH"), + DictPath: common.GetEnv(common.EnvRAGFlowDictPath), MinSize: 1, MaxSize: 2, IdleTimeout: 30 * time.Second, diff --git a/internal/ingestion/task/dataflow_e2e_test.go b/internal/ingestion/task/dataflow_e2e_test.go index 2aa6199439..69e8b8c66e 100644 --- a/internal/ingestion/task/dataflow_e2e_test.go +++ b/internal/ingestion/task/dataflow_e2e_test.go @@ -20,7 +20,7 @@ import ( "context" "fmt" "net" - "os" + "ragflow/internal/common" "testing" "time" @@ -48,18 +48,18 @@ func setupTestDocEngine(t *testing.T, engineType engine.EngineType, tenantID, da switch engineType { case engine.EngineElasticsearch: t.Logf("Setting up Elasticsearch engine...") - esHost := os.Getenv("ES_HOST") + esHost := common.GetEnv(common.EnvESHost) if esHost == "" { esHost = "localhost:1200" } if !startsWithHTTP(esHost) { esHost = "http://" + esHost } - esUser := os.Getenv("ES_USER") + esUser := common.GetEnv(common.EnvESUsername) if esUser == "" { esUser = "elastic" } - esPassword := os.Getenv("ES_PASSWORD") + esPassword := common.GetEnv(common.EnvESPassword) if esPassword == "" { esPassword = "infini_rag_flow" } @@ -77,7 +77,7 @@ func setupTestDocEngine(t *testing.T, engineType engine.EngineType, tenantID, da case engine.EngineInfinity: t.Logf("Setting up Infinity engine...") - infURI := os.Getenv("INFINITY_URI") + infURI := common.GetEnv(common.EnvInfinityURI) if infURI == "" { infURI = "localhost:23817" } @@ -352,7 +352,7 @@ func TestDataflowE2E_TaskHandlerToDataflowService(t *testing.T) { // Verify final task status can be updated to success ingestionTaskDAO := dao.NewIngestionTaskDAO() - if err := ingestionTaskDAO.UpdateStatus(taskID, "success"); err != nil { + if err = ingestionTaskDAO.UpdateStatus(taskID, "success"); err != nil { t.Fatalf("UpdateStatus failed: %v", err) } diff --git a/internal/ingestion/task/dataflow_real_pipeline_integration_test.go b/internal/ingestion/task/dataflow_real_pipeline_integration_test.go index 98d4fe690f..29f8421355 100644 --- a/internal/ingestion/task/dataflow_real_pipeline_integration_test.go +++ b/internal/ingestion/task/dataflow_real_pipeline_integration_test.go @@ -519,18 +519,18 @@ func disableTokenizerEmbeddingForTaskTemplate(t *testing.T, raw []byte) []byte { func prepareTokenizerResourceForTaskIntegration(t *testing.T) { t.Helper() - if os.Getenv("RAGFLOW_DICT_PATH") != "" { + if common.GetEnv(common.EnvRAGFlowDictPath) != "" { return } const systemDictPath = "/usr/share/infinity/resource" if _, err := os.Stat(filepath.Join(systemDictPath, "rag", "huqie.txt")); err != nil { t.Skipf("system tokenizer resource not found at %s: %v", systemDictPath, err) } - if err := os.Setenv("RAGFLOW_DICT_PATH", systemDictPath); err != nil { + if err := os.Setenv(common.EnvRAGFlowDictPath, systemDictPath); err != nil { t.Fatalf("set RAGFLOW_DICT_PATH=%s: %v", systemDictPath, err) } t.Cleanup(func() { - _ = os.Unsetenv("RAGFLOW_DICT_PATH") + _ = os.Unsetenv(common.EnvRAGFlowDictPath) }) } @@ -626,7 +626,7 @@ func requireTokenizerPool(t *testing.T) { return } cfg := &tokenizer.PoolConfig{ - DictPath: os.Getenv("RAGFLOW_DICT_PATH"), + DictPath: common.GetEnv(common.EnvRAGFlowDictPath), MinSize: 1, MaxSize: 2, IdleTimeout: 30 * time.Second, diff --git a/internal/parser/parser/pdf_parser_common.go b/internal/parser/parser/pdf_parser_common.go index 4bebde1083..c2b767adf0 100644 --- a/internal/parser/parser/pdf_parser_common.go +++ b/internal/parser/parser/pdf_parser_common.go @@ -23,7 +23,7 @@ import ( "fmt" "image" "log/slog" - "os" + "ragflow/internal/common" "sort" "strings" "time" @@ -294,7 +294,7 @@ func emptyPDFResult(filename string) ParseResult { } func deepDocAnalyzerFromEnv() deepdoctype.DocAnalyzer { - baseURL := strings.TrimSpace(os.Getenv("DEEPDOC_URL")) + baseURL := strings.TrimSpace(common.GetEnv(common.EnvDeepDocURL)) if baseURL == "" { return &deepdocpdf.MockDocAnalyzer{Healthy: true} } diff --git a/internal/parser/parser/pdf_parser_docling.go b/internal/parser/parser/pdf_parser_docling.go index 7f7006da87..5665a8e071 100644 --- a/internal/parser/parser/pdf_parser_docling.go +++ b/internal/parser/parser/pdf_parser_docling.go @@ -6,7 +6,7 @@ import ( "encoding/json" "fmt" "io" - "os" + "ragflow/internal/common" "strings" models "ragflow/internal/entity/models" @@ -42,14 +42,14 @@ func parsePDFWithDocling(filename string, data []byte, parser *PDFParser) ParseR } serverURL := strings.TrimSpace(parser.DoclingServerURL) if serverURL == "" { - serverURL = strings.TrimSpace(os.Getenv("DOCLING_SERVER_URL")) + serverURL = strings.TrimSpace(common.GetEnv(common.EnvDoclingServerURL)) } if serverURL == "" { return ParseResult{Err: fmt.Errorf("parser: Docling requires docling_server_url or DOCLING_SERVER_URL")} } apiKey := strings.TrimSpace(parser.DoclingAPIKey) if apiKey == "" { - apiKey = strings.TrimSpace(os.Getenv("DOCLING_API_KEY")) + apiKey = strings.TrimSpace(common.GetEnv(common.EnvDoclingApiKey)) } baseURL := strings.TrimRight(serverURL, "/") diff --git a/internal/parser/parser/pdf_parser_mineru.go b/internal/parser/parser/pdf_parser_mineru.go index f5d2f26c9e..095ecb00ac 100644 --- a/internal/parser/parser/pdf_parser_mineru.go +++ b/internal/parser/parser/pdf_parser_mineru.go @@ -2,7 +2,7 @@ package parser import ( "fmt" - "os" + "ragflow/internal/common" "strings" "time" @@ -18,18 +18,18 @@ func parsePDFWithMinerU(filename string, data []byte, parser *PDFParser) ParseRe } apiServer := strings.TrimSpace(parser.MinerUAPIServer) if apiServer == "" { - apiServer = strings.TrimSpace(os.Getenv("MINERU_APISERVER")) + apiServer = strings.TrimSpace(common.GetEnv(common.EnvMineruApiServer)) } if apiServer == "" { return ParseResult{Err: fmt.Errorf("parser: MinerU requires mineru_apiserver or MINERU_APISERVER")} } apiKey := parser.MinerUAPIKey if strings.TrimSpace(apiKey) == "" { - apiKey = strings.TrimSpace(os.Getenv("MINERU_API_KEY")) + apiKey = strings.TrimSpace(common.GetEnv(common.EnvMineruApiKey)) } backend := strings.TrimSpace(parser.MinerUBackend) if backend == "" { - backend = strings.TrimSpace(os.Getenv("MINERU_BACKEND")) + backend = strings.TrimSpace(common.GetEnv(common.EnvMineruBackend)) } if backend == "" { backend = "pipeline" diff --git a/internal/parser/parser/pdf_parser_opendataloader.go b/internal/parser/parser/pdf_parser_opendataloader.go index bed7d9fc8a..9c4474fcff 100644 --- a/internal/parser/parser/pdf_parser_opendataloader.go +++ b/internal/parser/parser/pdf_parser_opendataloader.go @@ -7,7 +7,7 @@ import ( "io" "mime/multipart" "net/http" - "os" + "ragflow/internal/common" "strings" models "ragflow/internal/entity/models" @@ -19,14 +19,14 @@ func parsePDFWithOpenDataLoader(filename string, data []byte, parser *PDFParser) } baseURL := strings.TrimSpace(parser.OpenDataLoaderAPIServer) if baseURL == "" { - baseURL = strings.TrimSpace(os.Getenv("OPENDATALOADER_APISERVER")) + baseURL = strings.TrimSpace(common.GetEnv(common.EnvOpenDataLoaderApiServer)) } if baseURL == "" { return ParseResult{Err: fmt.Errorf("parser: OpenDataLoader requires opendataloader_apiserver or OPENDATALOADER_APISERVER")} } apiKey := strings.TrimSpace(parser.OpenDataLoaderAPIKey) if apiKey == "" { - apiKey = strings.TrimSpace(os.Getenv("OPENDATALOADER_API_KEY")) + apiKey = strings.TrimSpace(common.GetEnv(common.EnvOpenDataLoaderApiKey)) } bodyReader, contentType, err := openDataLoaderMultipart(filename, data, parser) @@ -57,7 +57,7 @@ func parsePDFWithOpenDataLoader(filename string, data []byte, parser *PDFParser) JSONDoc any `json:"json_doc"` MDText string `json:"md_text"` } - if err := json.Unmarshal(raw, &payload); err != nil { + if err = json.Unmarshal(raw, &payload); err != nil { return ParseResult{Err: fmt.Errorf("parser: OpenDataLoader decode: %w", err)} } if payload.JSONDoc != nil { @@ -79,7 +79,7 @@ func openDataLoaderMultipart(filename string, data []byte, parser *PDFParser) (i if err != nil { return nil, "", fmt.Errorf("parser: OpenDataLoader create form file: %w", err) } - if _, err := part.Write(data); err != nil { + if _, err = part.Write(data); err != nil { return nil, "", fmt.Errorf("parser: OpenDataLoader write PDF: %w", err) } if parser.OpenDataLoaderHybrid != "" { @@ -95,7 +95,7 @@ func openDataLoaderMultipart(filename string, data []byte, parser *PDFParser) (i _ = writer.WriteField("sanitize", "false") } } - if err := writer.Close(); err != nil { + if err = writer.Close(); err != nil { return nil, "", fmt.Errorf("parser: OpenDataLoader finalize form: %w", err) } return strings.NewReader(body.String()), writer.FormDataContentType(), nil diff --git a/internal/parser/parser/pdf_parser_paddleocr.go b/internal/parser/parser/pdf_parser_paddleocr.go index 35cc2d6596..6e9d032b6a 100644 --- a/internal/parser/parser/pdf_parser_paddleocr.go +++ b/internal/parser/parser/pdf_parser_paddleocr.go @@ -2,7 +2,7 @@ package parser import ( "fmt" - "os" + "ragflow/internal/common" "strings" models "ragflow/internal/entity/models" @@ -14,21 +14,21 @@ func parsePDFWithPaddleOCR(filename string, data []byte, parser *PDFParser) Pars } baseURL := strings.TrimSpace(parser.PaddleOCRBaseURL) if baseURL == "" { - baseURL = strings.TrimSpace(os.Getenv("PADDLEOCR_BASE_URL")) + baseURL = strings.TrimSpace(common.GetEnv(common.EnvPaddleOCRBaseUrl)) } if baseURL == "" { - baseURL = strings.TrimSpace(os.Getenv("PADDLEOCR_API_URL")) + baseURL = strings.TrimSpace(common.GetEnv(common.EnvPaddleOCRApiURL)) } if baseURL == "" { return ParseResult{Err: fmt.Errorf("parser: PaddleOCR requires paddleocr_base_url or PADDLEOCR_BASE_URL")} } apiKey := parser.PaddleOCRAPIKey if strings.TrimSpace(apiKey) == "" { - apiKey = strings.TrimSpace(os.Getenv("PADDLEOCR_ACCESS_TOKEN")) + apiKey = strings.TrimSpace(common.GetEnv(common.EnvPaddleOCRAccessToken)) } algorithm := strings.TrimSpace(parser.PaddleOCRAlgorithm) if algorithm == "" { - algorithm = strings.TrimSpace(os.Getenv("PADDLEOCR_ALGORITHM")) + algorithm = strings.TrimSpace(common.GetEnv(common.EnvPaddleOCRAlgorithm)) } if algorithm == "" { algorithm = "PaddleOCR-VL" diff --git a/internal/parser/parser/pdf_parser_somark.go b/internal/parser/parser/pdf_parser_somark.go index bce37ea23f..a8ed7db7b7 100644 --- a/internal/parser/parser/pdf_parser_somark.go +++ b/internal/parser/parser/pdf_parser_somark.go @@ -9,7 +9,7 @@ import ( "mime/multipart" "net/http" "net/url" - "os" + "ragflow/internal/common" "strings" models "ragflow/internal/entity/models" @@ -21,14 +21,14 @@ func parsePDFWithSoMark(filename string, data []byte, parser *PDFParser) ParseRe } baseURL := strings.TrimSpace(parser.SoMarkBaseURL) if baseURL == "" { - baseURL = strings.TrimSpace(os.Getenv("SOMARK_BASE_URL")) + baseURL = strings.TrimSpace(common.GetEnv(common.EnvSOMarkBaseUrl)) } if baseURL == "" { return ParseResult{Err: fmt.Errorf("parser: SoMark requires somark_base_url or SOMARK_BASE_URL")} } apiKey := parser.SoMarkAPIKey if strings.TrimSpace(apiKey) == "" { - apiKey = strings.TrimSpace(os.Getenv("SOMARK_API_KEY")) + apiKey = strings.TrimSpace(common.GetEnv(common.EnvSOMarkApiKey)) } taskID, err := soMarkSubmit(strings.TrimRight(baseURL, "/"), filename, data, parser, apiKey) if err != nil { @@ -57,26 +57,26 @@ func soMarkSubmit(baseURL, filename string, data []byte, parser *PDFParser, apiK } _ = writer.WriteField("output_formats", "json") elementFormats, _ := json.Marshal(map[string]any{ - "image": envOrDefault("SOMARK_IMAGE_FORMAT", parser.SoMarkImageFormat, "url"), - "formula": envOrDefault("SOMARK_FORMULA_FORMAT", parser.SoMarkFormulaFormat, "latex"), - "table": envOrDefault("SOMARK_TABLE_FORMAT", parser.SoMarkTableFormat, "html"), - "cs": envOrDefault("SOMARK_CS_FORMAT", parser.SoMarkCSFormat, "image"), + "image": envOrDefault(common.EnvSOMarkImageFormat, parser.SoMarkImageFormat, "url"), + "formula": envOrDefault(common.EnvSOMarkFormulaFormat, parser.SoMarkFormulaFormat, "latex"), + "table": envOrDefault(common.EnvSOMarkTableFormat, parser.SoMarkTableFormat, "html"), + "cs": envOrDefault(common.EnvSOMarkCSFormat, parser.SoMarkCSFormat, "image"), }) featureConfig, _ := json.Marshal(map[string]any{ - "enable_text_cross_page": envOrBool("SOMARK_ENABLE_TEXT_CROSS_PAGE", parser.SoMarkEnableTextCrossPage), - "enable_table_cross_page": envOrBool("SOMARK_ENABLE_TABLE_CROSS_PAGE", parser.SoMarkEnableTableCrossPage), - "enable_title_level_recognition": envOrBool("SOMARK_ENABLE_TITLE_LEVEL_RECOGNITION", parser.SoMarkEnableTitleLevelRecognition), - "enable_inline_image": envOrBool("SOMARK_ENABLE_INLINE_IMAGE", parser.SoMarkEnableInlineImage), - "enable_table_image": envOrBool("SOMARK_ENABLE_TABLE_IMAGE", parser.SoMarkEnableTableImage), - "enable_image_understanding": envOrBool("SOMARK_ENABLE_IMAGE_UNDERSTANDING", parser.SoMarkEnableImageUnderstanding), - "keep_header_footer": envOrBool("SOMARK_KEEP_HEADER_FOOTER", parser.SoMarkKeepHeaderFooter), + "enable_text_cross_page": envOrBool(common.EnvSOMarkEnableTextCrossPage, parser.SoMarkEnableTextCrossPage), + "enable_table_cross_page": envOrBool(common.EnvSOMarkEnableTableCrossPage, parser.SoMarkEnableTableCrossPage), + "enable_title_level_recognition": envOrBool(common.EnvSOMarkEnableTitleLevelRecognition, parser.SoMarkEnableTitleLevelRecognition), + "enable_inline_image": envOrBool(common.EnvSOMarkEnableInlineImage, parser.SoMarkEnableInlineImage), + "enable_table_image": envOrBool(common.EnvSOMarkEnableTableImage, parser.SoMarkEnableTableImage), + "enable_image_understanding": envOrBool(common.EnvSOMarkEnableImageUnderstanding, parser.SoMarkEnableImageUnderstanding), + "keep_header_footer": envOrBool(common.EnvSOMarkKeepHeaderFooter, parser.SoMarkKeepHeaderFooter), }) _ = writer.WriteField("element_formats", string(elementFormats)) _ = writer.WriteField("feature_config", string(featureConfig)) if apiKey != "" { _ = writer.WriteField("api_key", apiKey) } - if err := writer.Close(); err != nil { + if err = writer.Close(); err != nil { return "", fmt.Errorf("parser: SoMark finalize form: %w", err) } req, err := http.NewRequestWithContext(context.Background(), http.MethodPost, baseURL+"/parse/async", &body) @@ -100,7 +100,7 @@ func soMarkSubmit(baseURL, filename string, data []byte, parser *PDFParser, apiK TaskID string `json:"task_id"` } `json:"data"` } - if err := json.Unmarshal(raw, &payload); err != nil { + if err = json.Unmarshal(raw, &payload); err != nil { return "", fmt.Errorf("parser: SoMark decode submit: %w", err) } if payload.Code != 0 { @@ -136,7 +136,7 @@ func soMarkPoll(baseURL, taskID, apiKey string) (map[string]any, error) { Message string `json:"message"` Data map[string]any `json:"data"` } - if err := json.Unmarshal(raw, &payload); err != nil { + if err = json.Unmarshal(raw, &payload); err != nil { return nil, fmt.Errorf("parser: SoMark decode poll: %w", err) } if payload.Code != 0 { @@ -225,14 +225,14 @@ func envOrDefault(envKey, configured, fallback string) string { if configured != "" { return configured } - if raw := strings.TrimSpace(os.Getenv(envKey)); raw != "" { + if raw := strings.TrimSpace(common.GetEnv(envKey)); raw != "" { return raw } return fallback } func envOrBool(envKey string, configured bool) bool { - if raw := strings.TrimSpace(os.Getenv(envKey)); raw != "" { + if raw := strings.TrimSpace(common.GetEnv(envKey)); raw != "" { switch strings.ToLower(raw) { case "1", "true", "yes", "on": return true diff --git a/internal/parser/parser/pdf_parser_tcadp.go b/internal/parser/parser/pdf_parser_tcadp.go index 59842d056e..f32584dc6d 100644 --- a/internal/parser/parser/pdf_parser_tcadp.go +++ b/internal/parser/parser/pdf_parser_tcadp.go @@ -9,7 +9,7 @@ import ( "fmt" "io" "net/http" - "os" + "ragflow/internal/common" "strings" models "ragflow/internal/entity/models" @@ -21,14 +21,14 @@ func parsePDFWithTCADP(filename string, data []byte, parser *PDFParser) ParseRes } baseURL := strings.TrimSpace(parser.TCADPAPIServer) if baseURL == "" { - baseURL = strings.TrimSpace(os.Getenv("TCADP_APISERVER")) + baseURL = strings.TrimSpace(common.GetEnv(common.EnvTCADPApiServerURL)) } if baseURL == "" { return ParseResult{Err: fmt.Errorf("parser: TCADP requires tcadp_apiserver or TCADP_APISERVER")} } apiKey := strings.TrimSpace(parser.TCADPAPIKey) if apiKey == "" { - apiKey = strings.TrimSpace(os.Getenv("TCADP_API_KEY")) + apiKey = strings.TrimSpace(common.GetEnv(common.EnvTCADPApiKey)) } requestBody := map[string]any{ "file_type": "PDF", diff --git a/internal/server/config.go b/internal/server/config.go index 90e9972082..2d41bf4de6 100644 --- a/internal/server/config.go +++ b/internal/server/config.go @@ -18,10 +18,8 @@ package server import ( "fmt" - "net" "net/mail" "net/url" - "os" "strconv" "strings" "time" @@ -453,12 +451,12 @@ func Init(configPath string) error { func FromEnvironments() error { // Secret key - if envVal := os.Getenv("RAGFLOW_SECRET_KEY"); envVal != "" { + if envVal := common.GetEnv(common.EnvRAGFlowSecretKey); envVal != "" { globalConfig.Server.SecretKey = &envVal } // Load REGISTER_ENABLED from environment variable (default: true) - if envVal := os.Getenv("REGISTER_ENABLED"); envVal != "" { + if envVal := common.GetEnv(common.EnvRegisterEnabled); envVal != "" { str := strings.ToLower(envVal) if str == "true" || str == "1" || str == "yes" { globalConfig.Authentication.RegisterEnabled = true @@ -468,7 +466,7 @@ func FromEnvironments() error { } // Load DISABLE_PASSWORD_LOGIN from environment variable (default: false) - if envVal := os.Getenv("DISABLE_PASSWORD_LOGIN"); envVal != "" { + if envVal := common.GetEnv(common.EnvDisablePasswordLogin); envVal != "" { str := strings.ToLower(envVal) if str == "true" || str == "1" || str == "yes" { globalConfig.Authentication.DisablePasswordLogin = true @@ -478,7 +476,7 @@ func FromEnvironments() error { } // Doc engine - docEngine := strings.ToLower(os.Getenv("DOC_ENGINE")) + docEngine := common.GetEnvSmall(common.EnvDocEngine) switch docEngine { case "infinity": globalConfig.DocEngine.Type = EngineInfinity @@ -498,7 +496,7 @@ func FromEnvironments() error { // Default super user email globalConfig.DefaultSuperUser.Email = "admin@ragflow.io" - superUserEmail := os.Getenv("DEFAULT_SUPERUSER_EMAIL") + superUserEmail := common.GetEnv(common.EnvDefaultSuperuserEmail) if superUserEmail != "" { _, err := mail.ParseAddress(superUserEmail) if err != nil { @@ -508,19 +506,19 @@ func FromEnvironments() error { } globalConfig.DefaultSuperUser.Password = "admin" - superUserPassword := os.Getenv("DEFAULT_SUPERUSER_PASSWORD") + superUserPassword := common.GetEnv(common.EnvDefaultSuperuserPassword) if superUserPassword != "" { globalConfig.DefaultSuperUser.Password = superUserPassword } globalConfig.DefaultSuperUser.Nickname = "admin" - superUserNickname := os.Getenv("DEFAULT_SUPERUSER_NICKNAME") + superUserNickname := common.GetEnv(common.EnvDefaultSuperuserNickname) if superUserNickname != "" { globalConfig.DefaultSuperUser.Nickname = superUserNickname } // Meta database - databaseType := strings.ToLower(os.Getenv("DB_TYPE")) + databaseType := common.GetEnvSmall(common.EnvDBType) switch databaseType { case "mysql": globalConfig.Database.Driver = "mysql" @@ -534,7 +532,7 @@ func FromEnvironments() error { } // Storage - storageType := strings.ToLower(os.Getenv("STORAGE_IMPL")) + storageType := common.GetEnvSmall(common.EnvStorageImpl) switch storageType { case "minio": globalConfig.StorageEngine.Type = StorageMinio @@ -554,32 +552,12 @@ func FromEnvironments() error { } // Minio - minioIP := strings.ToLower(os.Getenv("MINIO_IP")) - if minioIP != "" { - if globalConfig.StorageEngine.Minio == nil { - return fmt.Errorf("Minio config not found") - } - _, port, err := net.SplitHostPort(globalConfig.StorageEngine.Minio.Host) - if err != nil { - return fmt.Errorf("Error parsing host address %s: %v\n", globalConfig.StorageEngine.Minio.Host, err) - } - globalConfig.StorageEngine.Minio.Host = fmt.Sprintf("%s:%s", minioIP, port) + minioHost := strings.ToLower(common.GetEnv(common.EnvMinioHost)) + if minioHost != "" { + globalConfig.StorageEngine.Minio.Host = minioHost } - //minioPort := strings.ToLower(os.Getenv("MINIO_PORT")) - //// println(fmt.Sprintf("MINIO ip and port from env: %s:%s", minioIP, minioPort)) - //if minioPort != "" { - // if globalConfig.StorageEngine.Minio == nil { - // return fmt.Errorf("Minio config not found") - // } - // ip, _, err := net.SplitHostPort(globalConfig.StorageEngine.Minio.Host) - // if err != nil { - // return fmt.Errorf("Error parsing host address %s: %v\n", globalConfig.StorageEngine.Minio.Host, err) - // } - // globalConfig.StorageEngine.Minio.Host = fmt.Sprintf("%s:%s", ip, minioPort) - //} - - minioRegion := strings.ToLower(os.Getenv("MINIO_REGION")) + minioRegion := strings.ToLower(common.GetEnv(common.EnvMinioRegion)) if minioRegion != "" { if globalConfig.StorageEngine.Minio == nil { return fmt.Errorf("Minio config not found") @@ -986,9 +964,9 @@ func getInt(m map[string]interface{}, key string) int { } func GetLanguage() string { - lang := os.Getenv("LANG") + lang := common.GetEnv(common.EnvLang) if lang == "" { - lang = os.Getenv("LANGUAGE") + lang = common.GetEnv(common.EnvLanguage) } lang = strings.ToLower(lang) diff --git a/internal/service/chat_session.go b/internal/service/chat_session.go index cf785f6c09..a5b5fc5aa5 100644 --- a/internal/service/chat_session.go +++ b/internal/service/chat_session.go @@ -22,7 +22,6 @@ import ( "errors" "fmt" "math" - "os" "ragflow/internal/common" "ragflow/internal/engine" "ragflow/internal/storage" @@ -894,11 +893,11 @@ type feedbackDelta struct { } func chunkFeedbackEnabled() bool { - return strings.ToLower(os.Getenv("CHUNK_FEEDBACK_ENABLED")) == "true" + return common.GetEnv(common.EnvChunkFeedbackEnabled) == "true" } func chunkFeedbackWeighting() string { - weighting := strings.ToLower(strings.TrimSpace(os.Getenv("CHUNK_FEEDBACK_WEIGHTING"))) + weighting := strings.TrimSpace(common.GetEnvSmall(common.EnvChunkFeedbackWeighting)) if weighting == "uniform" || weighting == "relevance" { return weighting } diff --git a/internal/service/connector.go b/internal/service/connector.go index 3baed9f262..8f84d5c6f4 100644 --- a/internal/service/connector.go +++ b/internal/service/connector.go @@ -28,7 +28,6 @@ import ( "io" "net/http" "net/url" - "os" "ragflow/internal/engine/redis" "ragflow/internal/utility" "strings" @@ -566,13 +565,13 @@ func (s *ConnectorService) PollGoogleWebOAuthResult(userID, source string, req * func defaultGoogleWebOAuthRedirectURI(source string) string { if source == "gmail" { - return getenvDefault("GMAIL_WEB_OAUTH_REDIRECT_URI", "http://localhost:9384/api/v1/connectors/gmail/oauth/web/callback") + return getEnvDefault(common.EnvGmailWebOAuthRedirectURI, "http://localhost:9384/api/v1/connectors/gmail/oauth/web/callback") } - return getenvDefault("GOOGLE_DRIVE_WEB_OAUTH_REDIRECT_URI", "http://localhost:9384/api/v1/connectors/google-drive/oauth/web/callback") + return getEnvDefault(common.EnvGoogleDriveWebOAuthRedirectURI, "http://localhost:9384/api/v1/connectors/google-drive/oauth/web/callback") } -func getenvDefault(key, fallback string) string { - if value := strings.TrimSpace(os.Getenv(key)); value != "" { +func getEnvDefault(key, fallback string) string { + if value := strings.TrimSpace(common.GetEnv(key)); value != "" { return value } return fallback @@ -1163,8 +1162,8 @@ func (s *ConnectorService) PollBoxWebOAuthResult(userID string, req *PollBoxWebO } func defaultBoxWebOAuthRedirectURI() string { - return getenvDefault( - "BOX_WEB_OAUTH_REDIRECT_URI", + return getEnvDefault( + common.EnvBoxWebOAuthRedirectURI, "http://localhost:9384/api/v1/connectors/box/oauth/web/callback", ) } diff --git a/internal/service/document.go b/internal/service/document.go index 12f092d69c..d56f4183c5 100644 --- a/internal/service/document.go +++ b/internal/service/document.go @@ -30,7 +30,6 @@ import ( "math/rand" "mime/multipart" "net/http" - "os" "path/filepath" "reflect" "regexp" @@ -401,7 +400,7 @@ func (s *DocumentService) sandboxArtifactAccessible(filename, userID string) boo } func sandboxArtifactBucket() string { - if bucket := os.Getenv("SANDBOX_ARTIFACT_BUCKET"); bucket != "" { + if bucket := common.GetEnv(common.EnvSandboxArtifactBucket); bucket != "" { return bucket } return "sandbox-artifacts" diff --git a/internal/service/file.go b/internal/service/file.go index 659153f531..a6f8822123 100644 --- a/internal/service/file.go +++ b/internal/service/file.go @@ -26,7 +26,6 @@ import ( "net" "net/http" "net/url" - "os" "path/filepath" "ragflow/internal/common" "ragflow/internal/dao" @@ -352,11 +351,12 @@ func (s *FileService) UploadFile(tenantID, parentID string, files []*multipart.F return nil, fmt.Errorf("Can't find this folder!") } - maxFileNumPerUser := os.Getenv("MAX_FILE_NUM_PER_USER") + maxFileNumPerUser := common.GetEnv(common.EnvMaxFileNumPerUser) if maxFileNumPerUser != "" { var maxNum int64 - if _, err := fmt.Sscanf(maxFileNumPerUser, "%d", &maxNum); err == nil && maxNum > 0 { - docCount, err := s.GetDocCount(tenantID) + if _, err = fmt.Sscanf(maxFileNumPerUser, "%d", &maxNum); err == nil && maxNum > 0 { + var docCount int64 + docCount, err = s.GetDocCount(tenantID) if err != nil { return nil, fmt.Errorf("failed to get document count: %w", err) } @@ -383,7 +383,8 @@ func (s *FileService) UploadFile(tenantID, parentID string, files []*multipart.F fileObjNames := s.parseFilePath(filename) - idList, err := s.fileDAO.GetIDListByID(parentID, fileObjNames, 1, []string{parentID}) + var idList []string + idList, err = s.fileDAO.GetIDListByID(parentID, fileObjNames, 1, []string{parentID}) if err != nil { return nil, fmt.Errorf("failed to get file ID list: %w", err) } @@ -395,7 +396,8 @@ func (s *FileService) UploadFile(tenantID, parentID string, files []*multipart.F if err != nil { return nil, fmt.Errorf("Folder not found!") } - createdFolder, err := s.createFolderRecursive(lastFolder, fileObjNames, len(idList), tenantID) + var createdFolder *entity.File + createdFolder, err = s.createFolderRecursive(lastFolder, fileObjNames, len(idList), tenantID) if err != nil { return nil, fmt.Errorf("failed to create folder: %w", err) } @@ -424,7 +426,7 @@ func (s *FileService) UploadFile(tenantID, parentID string, files []*multipart.F return nil, fmt.Errorf("failed to read file data: %w", err) } - if err := storageImpl.Put(lastFolder.ID, location, data); err != nil { + if err = storageImpl.Put(lastFolder.ID, location, data); err != nil { return nil, fmt.Errorf("failed to store file: %w", err) } @@ -442,7 +444,7 @@ func (s *FileService) UploadFile(tenantID, parentID string, files []*multipart.F SourceType: "", } - if err := s.fileDAO.Insert(fileRecord); err != nil { + if err = s.fileDAO.Insert(fileRecord); err != nil { return nil, fmt.Errorf("failed to insert file record: %w", err) } @@ -1301,11 +1303,12 @@ func (s *FileService) checkUploadInfoHealth(userID, filename string) error { if filename == "" { return fmt.Errorf("No file selected!") } - maxFileNumPerUser := os.Getenv("MAX_FILE_NUM_PER_USER") + maxFileNumPerUser := common.GetEnv(common.EnvMaxFileNumPerUser) if maxFileNumPerUser != "" { var maxNum int64 if _, err := fmt.Sscanf(maxFileNumPerUser, "%d", &maxNum); err == nil && maxNum > 0 { - docCount, err := s.GetDocCount(userID) + var docCount int64 + docCount, err = s.GetDocCount(userID) if err != nil { return fmt.Errorf("failed to get document count: %w", err) } diff --git a/internal/service/memory.go b/internal/service/memory.go index 96011140c5..a7cb4abe56 100644 --- a/internal/service/memory.go +++ b/internal/service/memory.go @@ -20,7 +20,6 @@ import ( "context" "errors" "fmt" - "os" "ragflow/internal/common" "ragflow/internal/entity" models "ragflow/internal/entity/models" @@ -1209,7 +1208,7 @@ func memoryMessageSelectFields() []string { } func memoryIndexName(tenantID string) string { - prefix := strings.TrimSpace(os.Getenv("ES_INDEX_PREFIX")) + prefix := strings.TrimSpace(common.GetEnv(common.EnvESIndexPrefix)) if prefix == "" { return fmt.Sprintf("memory_%s", tenantID) } diff --git a/internal/service/model_service.go b/internal/service/model_service.go index 1a4f6dc5e6..87038a8d5c 100644 --- a/internal/service/model_service.go +++ b/internal/service/model_service.go @@ -20,7 +20,6 @@ import ( "encoding/json" "errors" "fmt" - "os" "ragflow/internal/common" "ragflow/internal/dao" "ragflow/internal/entity" @@ -2443,9 +2442,9 @@ func (m *ModelProviderService) GetModelConfigFromProviderInstance(tenantID strin zap.String("modelType", modelType.String())) // TEI builtin embedding short-circuit - if modelType == entity.ModelTypeEmbedding && strings.Contains(os.Getenv("COMPOSE_PROFILES"), "tei-") { - teiModel := os.Getenv("TEI_MODEL") - teiBaseURL := os.Getenv("TEI_BASE_URL") + if modelType == entity.ModelTypeEmbedding && strings.Contains(common.GetEnv(common.EnvComposeProfiles), "tei-") { + teiModel := common.GetEnv(common.EnvTEIModel) + teiBaseURL := common.GetEnv(common.EnvTEIBaseURL) // First try exact match: handles bare model IDs like "model@q8_0" // where '@' is part of the model name itself. diff --git a/internal/service/system.go b/internal/service/system.go index f6a4b894df..25ef64a092 100644 --- a/internal/service/system.go +++ b/internal/service/system.go @@ -20,7 +20,6 @@ import ( "context" "encoding/json" "fmt" - "os" "ragflow/internal/common" "ragflow/internal/engine/redis" "ragflow/internal/entity" @@ -456,7 +455,7 @@ func (s *SystemService) ListEnvironments() ([]map[string]interface{}, error) { result := make([]map[string]interface{}, 0) // DOC_ENGINE - docEngine := os.Getenv("DOC_ENGINE") + docEngine := common.GetEnv(common.EnvDocEngine) if docEngine == "" { docEngine = "elasticsearch" } @@ -466,7 +465,7 @@ func (s *SystemService) ListEnvironments() ([]map[string]interface{}, error) { }) // DEFAULT_SUPERUSER_EMAIL - defaultSuperuserEmail := os.Getenv("DEFAULT_SUPERUSER_EMAIL") + defaultSuperuserEmail := common.GetEnvSmall(common.EnvDefaultSuperuserEmail) if defaultSuperuserEmail == "" { defaultSuperuserEmail = "admin@ragflow.io" } @@ -476,7 +475,7 @@ func (s *SystemService) ListEnvironments() ([]map[string]interface{}, error) { }) // DB_TYPE - dbType := os.Getenv("DB_TYPE") + dbType := common.GetEnvSmall(common.EnvDBType) if dbType == "" { dbType = "mysql" } @@ -486,7 +485,7 @@ func (s *SystemService) ListEnvironments() ([]map[string]interface{}, error) { }) // DEVICE - device := os.Getenv("DEVICE") + device := common.GetEnvSmall(common.EnvDevice) if device == "" { device = "cpu" } @@ -496,7 +495,7 @@ func (s *SystemService) ListEnvironments() ([]map[string]interface{}, error) { }) // STORAGE_IMPL - storageImpl := os.Getenv("STORAGE_IMPL") + storageImpl := common.GetEnvSmall(common.EnvStorageImpl) if storageImpl == "" { storageImpl = "MINIO" } diff --git a/internal/service/user.go b/internal/service/user.go index 8cb057a5bb..4898ab520d 100644 --- a/internal/service/user.go +++ b/internal/service/user.go @@ -26,7 +26,6 @@ import ( "errors" "fmt" "hash" - "os" "ragflow/internal/common" "ragflow/internal/engine/redis" "ragflow/internal/entity" @@ -605,7 +604,7 @@ func (s *UserService) constantTimeCompare(a, b []byte) bool { } func defaultUserLanguage() string { - if strings.Contains(os.Getenv("LANG"), "zh_CN") { + if strings.Contains(common.GetEnv(common.EnvLang), "zh_CN") { return "Chinese" } return "English" diff --git a/internal/storage/minio_test.go b/internal/storage/minio_test.go index 1ce4248658..6e677f8305 100644 --- a/internal/storage/minio_test.go +++ b/internal/storage/minio_test.go @@ -20,7 +20,6 @@ import ( "bytes" "fmt" "log" - "os" "ragflow/internal/utility" "testing" "time" @@ -44,22 +43,6 @@ func getMinioConfig() (*server.MinioConfig, error) { return config, nil } -// getEnv gets environment variable or returns default value -func getEnv(key, defaultValue string) string { - if value := os.Getenv(key); value != "" { - return value - } - return defaultValue -} - -// getEnvBool gets environment variable as bool or returns default value -func getEnvBool(key string, defaultValue bool) bool { - if value := os.Getenv(key); value != "" { - return value == "true" || value == "1" || value == "yes" - } - return defaultValue -} - // newTestMinioStorage creates a new MinIO storage instance for testing func newTestMinioStorage(t *testing.T) *MinioStorage { rootDir := utility.GetProjectRoot() diff --git a/internal/tokenizer/tokenizer.go b/internal/tokenizer/tokenizer.go index b303210542..60405bd26e 100644 --- a/internal/tokenizer/tokenizer.go +++ b/internal/tokenizer/tokenizer.go @@ -19,7 +19,6 @@ package tokenizer import ( "context" "fmt" - "os" "ragflow/internal/common" "runtime" "sync" @@ -96,7 +95,7 @@ func Init(cfg *PoolConfig) error { // Set default values if cfg.DictPath == "" { - if env := os.Getenv("RAGFLOW_DICT_PATH"); env != "" { + if env := common.GetEnv(common.EnvRAGFlowDictPath); env != "" { cfg.DictPath = env } else { cfg.DictPath = "/usr/share/infinity/resource" diff --git a/internal/utility/path.go b/internal/utility/path.go index 038237cbb9..7fb034b4c5 100644 --- a/internal/utility/path.go +++ b/internal/utility/path.go @@ -20,19 +20,20 @@ import ( "fmt" "os" "path/filepath" + "ragflow/internal/common" "runtime" ) // GetProjectRoot returns the project root directory func GetProjectRoot() string { // Try environment variable first - if confDir := os.Getenv("RAGFLOW_CONF_DIR"); confDir != "" { + if confDir := common.GetEnv(common.EnvRAGFlowConfDir); confDir != "" { return confDir } - if d := os.Getenv("RAG_PROJECT_BASE"); d != "" { + if d := common.GetEnv(common.EnvRAGProjectBaseURL); d != "" { return d } - if d := os.Getenv("RAG_DEPLOY_BASE"); d != "" { + if d := common.GetEnv(common.EnvRAGDeployBaseURL); d != "" { return d } @@ -63,14 +64,14 @@ func GetProjectRoot() string { func FindConfFileInProject(fileName string) (*string, error) { var filePath string - if projDir := os.Getenv("RAG_PROJECT_BASE"); projDir != "" { + if projDir := common.GetEnv(common.EnvRAGProjectBaseURL); projDir != "" { filePath = filepath.Join(projDir, "conf", fileName) if _, err := os.Stat(filePath); err == nil { return &filePath, nil } } - if projDir := os.Getenv("RAG_DEPLOY_BASE"); projDir != "" { + if projDir := common.GetEnv(common.EnvRAGDeployBaseURL); projDir != "" { filePath = filepath.Join(projDir, "conf", fileName) if _, err := os.Stat(filePath); err == nil { return &filePath, nil