diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 934005edec..6d370097f6 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -142,7 +142,7 @@ jobs: RUNNER_WORKSPACE_PREFIX=${RUNNER_WORKSPACE_PREFIX:-${HOME}} RAGFLOW_IMAGE=infiniflow/ragflow:${GITHUB_RUN_ID} echo "RAGFLOW_IMAGE=${RAGFLOW_IMAGE}" >> ${GITHUB_ENV} - sudo docker pull ubuntu:22.04 + sudo docker pull ubuntu:24.04 sudo DOCKER_BUILDKIT=1 docker build --build-arg NEED_MIRROR=1 --build-arg HTTPS_PROXY=${HTTPS_PROXY} --build-arg HTTP_PROXY=${HTTP_PROXY} -f Dockerfile -t ${RAGFLOW_IMAGE} . if [[ ${GITHUB_EVENT_NAME} == "schedule" ]]; then export HTTP_API_TEST_LEVEL=p3 diff --git a/common/settings.py b/common/settings.py index fe3d07b33c..2b67dc34d7 100644 --- a/common/settings.py +++ b/common/settings.py @@ -16,7 +16,6 @@ import os import json import secrets -from datetime import date import logging from common.constants import RAG_FLOW_SERVICE_NAME from common.file_utils import get_project_base_directory @@ -34,6 +33,7 @@ from rag.utils.azure_spn_conn import RAGFlowAzureSpnBlob from rag.utils.gcs_conn import RAGFlowGCS from rag.utils.minio_conn import RAGFlowMinio from rag.utils.opendal_conn import OpenDALStorage +from rag.utils.redis_conn import REDIS_CONN from rag.utils.s3_conn import RAGFlowS3 from rag.utils.oss_conn import RAGFlowOSS @@ -138,21 +138,22 @@ def get_svr_queue_names(): return [get_svr_queue_name(priority) for priority in [1, 0]] def _get_or_create_secret_key(): - secret_key = os.environ.get("RAGFLOW_SECRET_KEY") - if secret_key and len(secret_key) >= 32: - return secret_key - - # Check if there's a configured secret key - configured_key = get_base_config(RAG_FLOW_SERVICE_NAME, {}).get("secret_key") - if configured_key and configured_key != str(date.today()) and len(configured_key) >= 32: - return configured_key + # secret_key = os.environ.get("RAGFLOW_SECRET_KEY") + # if secret_key and len(secret_key) >= 32: + # return secret_key + # + # # Check if there's a configured secret key + # configured_key = get_base_config(RAG_FLOW_SERVICE_NAME, {}).get("secret_key") + # if configured_key and configured_key != str(date.today()) and len(configured_key) >= 32: + # return configured_key # Generate a new secure key and warn about it import logging - new_key = secrets.token_hex(32) + generated_key = secrets.token_hex(32) + secret_key = REDIS_CONN.get_or_create_secret_key("ragflow:system:secret_key", generated_key) logging.warning("SECURITY WARNING: Using auto-generated SECRET_KEY.") - return new_key + return secret_key class StorageFactory: storage_mapping = { diff --git a/rag/utils/redis_conn.py b/rag/utils/redis_conn.py index d134f05331..960e98af81 100644 --- a/rag/utils/redis_conn.py +++ b/rag/utils/redis_conn.py @@ -334,6 +334,42 @@ class RedisDB: self.__open__() return -1 + def get_or_create_secret_key(self, key_name: str, new_value: str) -> str: + """ + Atomically get an existing key or create a new one. + + This method guarantees that across multiple concurrent calls, only one + key will be created and all callers will receive the same key. + + Returns: + The secret key string + + Raises: + redis.RedisError: If Redis operations fail + """ + # First, try to get the existing key + existing_value = self.REDIS.get(key_name) + if existing_value is not None: + logging.debug("Retrieved existing key from Redis") + return existing_value + + # Use SETNX to atomically set the key only if it doesn't exist + # SETNX returns True if the key was set, False if it already existed + if self.REDIS.setnx(key_name, new_value): + logging.info("Successfully created new secret key in Redis") + return new_value + + # SETNX failed, meaning another process created the key concurrently + # Retrieve and return that key + final_key = self.REDIS.get(key_name) + if final_key is None: + # This should rarely happen, but retry if it does + logging.warning("Key disappeared during concurrent access, retrying...") + return self.get_or_create_secret_key(key_name, new_value) + + logging.debug("Retrieved key created by another process") + return final_key + def transaction(self, key, value, exp=3600): try: pipeline = self.REDIS.pipeline(transaction=True)