zlei6/ragflow
1
0
Fork 0
mirror of https://github.com/infiniflow/ragflow.git synced 2026-07-20 06:31:02 +08:00
Code Issues Packages Projects Releases Wiki Activity

Refactor: reformat all code for lefthook using ruff and gofmt (#16585)

Browse Source
This commit is contained in:
Wang Qi
2026-07-03 12:53:39 +08:00
committed by GitHub
parent 19fcb4a981
commit 6a4b9be426
588 changed files with 11123 additions and 15412 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
api/utils/__init__.py
View File

@@ -21,7 +21,6 @@ def from_dict_hook(in_dict: dict):
if in_dict["module"] is None:
return in_dict["data"]
else:
return getattr(importlib.import_module(
in_dict["module"]), in_dict["type"])(**in_dict["data"])
return getattr(importlib.import_module(in_dict["module"]), in_dict["type"])(**in_dict["data"])
else:
return in_dict

21
api/utils/api_utils.py
View File

@@ -51,6 +51,7 @@ from common.misc_utils import thread_pool_exec
requests.models.complexjson.dumps = functools.partial(json.dumps, cls=CustomJSONEncoder)
def _safe_jsonify(payload: dict):
if has_app_context():
return jsonify(payload)
@@ -87,9 +88,11 @@ async def _coerce_request_data() -> dict:
request._cached_payload = payload
return payload
async def get_request_json():
return await _coerce_request_data()
def serialize_for_json(obj):
"""
Recursively serialize objects to make them JSON serializable.
@@ -211,6 +214,7 @@ def not_allowed_parameters(*params):
if inspect.iscoroutinefunction(func):
return await func(*args, **kwargs)
return func(*args, **kwargs)
return wrapper
return decorator
@@ -238,10 +242,12 @@ def add_tenant_id_to_kwargs(func):
@wraps(func)
async def wrapper(**kwargs):
from api.apps import current_user
kwargs["tenant_id"] = current_user.id
if inspect.iscoroutinefunction(func):
return await func(**kwargs)
return func(**kwargs)
return wrapper
@@ -677,24 +683,15 @@ async def is_strong_enough(chat_model, embedding_model):
async def _is_strong_enough():
nonlocal chat_model, embedding_model
if embedding_model:
await asyncio.wait_for(
thread_pool_exec(embedding_model.encode, ["Are you strong enough!?"]),
timeout=10
)
await asyncio.wait_for(thread_pool_exec(embedding_model.encode, ["Are you strong enough!?"]), timeout=10)
if chat_model:
res = await asyncio.wait_for(
chat_model.async_chat("Nothing special.", [{"role": "user", "content": "Are you strong enough!?"}]),
timeout=30
)
res = await asyncio.wait_for(chat_model.async_chat("Nothing special.", [{"role": "user", "content": "Are you strong enough!?"}]), timeout=30)
if "**ERROR**" in res:
raise Exception(res)
# Pressure test for GraphRAG task
tasks = [
asyncio.create_task(_is_strong_enough())
for _ in range(count)
]
tasks = [asyncio.create_task(_is_strong_enough()) for _ in range(count)]
try:
await asyncio.gather(*tasks, return_exceptions=False)
except Exception as e:

48
api/utils/commands.py
View File

@@ -24,54 +24,50 @@ from werkzeug.security import generate_password_hash
from api.db.services import UserService
@click.command('reset-password', help='Reset the account password.')
@click.option('--email', prompt=True, help='The email address of the account whose password you need to reset')
@click.option('--new-password', prompt=True, help='the new password.')
@click.option('--password-confirm', prompt=True, help='the new password confirm.')
@click.command("reset-password", help="Reset the account password.")
@click.option("--email", prompt=True, help="The email address of the account whose password you need to reset")
@click.option("--new-password", prompt=True, help="the new password.")
@click.option("--password-confirm", prompt=True, help="the new password confirm.")
def reset_password(email, new_password, password_confirm):
if str(new_password).strip() != str(password_confirm).strip():
click.echo(click.style('sorry. The two passwords do not match.', fg='red'))
click.echo(click.style("sorry. The two passwords do not match.", fg="red"))
return
user = UserService.query(email=email)
if not user:
click.echo(click.style('sorry. The Email is not registered!.', fg='red'))
click.echo(click.style("sorry. The Email is not registered!.", fg="red"))
return
encode_password = base64.b64encode(new_password.encode('utf-8')).decode('utf-8')
encode_password = base64.b64encode(new_password.encode("utf-8")).decode("utf-8")
password_hash = generate_password_hash(encode_password)
user_dict = {
'password': password_hash
}
UserService.update_user(user[0].id,user_dict)
click.echo(click.style('Congratulations! Password has been reset.', fg='green'))
user_dict = {"password": password_hash}
UserService.update_user(user[0].id, user_dict)
click.echo(click.style("Congratulations! Password has been reset.", fg="green"))
@click.command('reset-email', help='Reset the account email.')
@click.option('--email', prompt=True, help='The old email address of the account whose email you need to reset')
@click.option('--new-email', prompt=True, help='the new email.')
@click.option('--email-confirm', prompt=True, help='the new email confirm.')
@click.command("reset-email", help="Reset the account email.")
@click.option("--email", prompt=True, help="The old email address of the account whose email you need to reset")
@click.option("--new-email", prompt=True, help="the new email.")
@click.option("--email-confirm", prompt=True, help="the new email confirm.")
def reset_email(email, new_email, email_confirm):
if str(new_email).strip() != str(email_confirm).strip():
click.echo(click.style('Sorry, new email and confirm email do not match.', fg='red'))
click.echo(click.style("Sorry, new email and confirm email do not match.", fg="red"))
return
if str(new_email).strip() == str(email).strip():
click.echo(click.style('Sorry, new email and old email are the same.', fg='red'))
click.echo(click.style("Sorry, new email and old email are the same.", fg="red"))
return
user = UserService.query(email=email)
if not user:
click.echo(click.style('sorry. the account: [{}] not exist .'.format(email), fg='red'))
click.echo(click.style("sorry. the account: [{}] not exist .".format(email), fg="red"))
return
if not re.match(r"^[\w\._-]+@([\w_-]+\.)+[\w-]{2,4}$", new_email):
click.echo(click.style('sorry. {} is not a valid email. '.format(new_email), fg='red'))
click.echo(click.style("sorry. {} is not a valid email. ".format(new_email), fg="red"))
return
new_user = UserService.query(email=new_email)
if new_user:
click.echo(click.style('sorry. the account: [{}] is exist .'.format(new_email), fg='red'))
click.echo(click.style("sorry. the account: [{}] is exist .".format(new_email), fg="red"))
return
user_dict = {
'email': new_email
}
UserService.update_user(user[0].id,user_dict)
click.echo(click.style('Congratulations!, email has been reset.', fg='green'))
user_dict = {"email": new_email}
UserService.update_user(user[0].id, user_dict)
click.echo(click.style("Congratulations!, email has been reset.", fg="green"))
def register_commands(app: Quart):

4
api/utils/common.py
View File

@@ -17,13 +17,13 @@ import xxhash
def string_to_bytes(string):
return string if isinstance(
string, bytes) else string.encode(encoding="utf-8")
return string if isinstance(string, bytes) else string.encode(encoding="utf-8")
def bytes_to_string(byte):
return byte.decode(encoding="utf-8")
# 128 bit = 32 character
def hash128(data: str) -> str:
return xxhash.xxh128(data).hexdigest()

15
api/utils/configs.py
View File

@@ -19,21 +19,18 @@ import base64
import pickle
from api.utils.common import bytes_to_string, string_to_bytes
safe_module = {
'numpy',
'rag_flow'
}
safe_module = {"numpy", "rag_flow"}
class RestrictedUnpickler(pickle.Unpickler):
def find_class(self, module, name):
import importlib
if module.split('.')[0] in safe_module:
if module.split(".")[0] in safe_module:
_module = importlib.import_module(module)
return getattr(_module, name)
# Forbid everything else.
raise pickle.UnpicklingError("global '%s.%s' is forbidden" %
(module, name))
raise pickle.UnpicklingError("global '%s.%s' is forbidden" % (module, name))
def restricted_loads(src):
@@ -50,7 +47,5 @@ def serialize_b64(src, to_str=False):
def deserialize_b64(src):
src = base64.b64decode(
string_to_bytes(src) if isinstance(
src, str) else src)
src = base64.b64decode(string_to_bytes(src) if isinstance(src, str) else src)
return restricted_loads(src)

9
api/utils/crypt.py
View File

@@ -30,25 +30,26 @@ def crypt(line):
file_path = os.path.join(get_project_base_directory(), "conf", "public.pem")
rsa_key = RSA.importKey(Path(file_path).read_text(), "Welcome")
cipher = Cipher_pkcs1_v1_5.new(rsa_key)
password_base64 = base64.b64encode(line.encode('utf-8')).decode("utf-8")
password_base64 = base64.b64encode(line.encode("utf-8")).decode("utf-8")
encrypted_password = cipher.encrypt(password_base64.encode())
return base64.b64encode(encrypted_password).decode('utf-8')
return base64.b64encode(encrypted_password).decode("utf-8")
def decrypt(line):
file_path = os.path.join(get_project_base_directory(), "conf", "private.pem")
rsa_key = RSA.importKey(Path(file_path).read_text(), "Welcome")
cipher = Cipher_pkcs1_v1_5.new(rsa_key)
return cipher.decrypt(base64.b64decode(line), "Fail to decrypt password!").decode('utf-8')
return cipher.decrypt(base64.b64decode(line), "Fail to decrypt password!").decode("utf-8")
def decrypt2(crypt_text):
from base64 import b64decode, b16decode
from Crypto.Cipher import PKCS1_v1_5 as Cipher_PKCS1_v1_5
from Crypto.PublicKey import RSA
decode_data = b64decode(crypt_text)
if len(decode_data) == 127:
hex_fixed = '00' + decode_data.hex()
hex_fixed = "00" + decode_data.hex()
decode_data = b16decode(hex_fixed.upper())
file_path = os.path.join(get_project_base_directory(), "conf", "private.pem")

107
api/utils/health_utils.py
View File

@@ -70,14 +70,11 @@ def check_storage() -> tuple[bool, dict]:
def get_es_cluster_stats() -> dict:
doc_engine = os.getenv('DOC_ENGINE', 'elasticsearch')
if doc_engine != 'elasticsearch':
doc_engine = os.getenv("DOC_ENGINE", "elasticsearch")
if doc_engine != "elasticsearch":
raise Exception("Elasticsearch is not in use.")
try:
return {
"status": "alive",
"message": ESConnection().get_cluster_stats()
}
return {"status": "alive", "message": ESConnection().get_cluster_stats()}
except Exception as e:
return {
"status": "timeout",
@@ -86,14 +83,11 @@ def get_es_cluster_stats() -> dict:
def get_infinity_status():
doc_engine = os.getenv('DOC_ENGINE', 'elasticsearch')
if doc_engine != 'infinity':
doc_engine = os.getenv("DOC_ENGINE", "elasticsearch")
if doc_engine != "infinity":
raise Exception("Infinity is not in use.")
try:
return {
"status": "alive",
"message": InfinityConnection().health()
}
return {"status": "alive", "message": InfinityConnection().health()}
except Exception as e:
return {
"status": "timeout",
@@ -104,28 +98,22 @@ def get_infinity_status():
def get_oceanbase_status():
"""
Get OceanBase health status and performance metrics.
Returns:
dict: OceanBase status with health information and performance metrics
"""
doc_engine = os.getenv('DOC_ENGINE', 'elasticsearch')
if doc_engine != 'oceanbase':
doc_engine = os.getenv("DOC_ENGINE", "elasticsearch")
if doc_engine != "oceanbase":
raise Exception("OceanBase is not in use.")
try:
ob_conn = OBConnection()
health_info = ob_conn.health()
performance_metrics = ob_conn.get_performance_metrics()
# Combine health and performance metrics
status = "alive" if health_info.get("status") == "healthy" else "timeout"
return {
"status": status,
"message": {
"health": health_info,
"performance": performance_metrics
}
}
return {"status": status, "message": {"health": health_info, "performance": performance_metrics}}
except Exception as e:
return {
"status": "timeout",
@@ -136,7 +124,7 @@ def get_oceanbase_status():
def check_oceanbase_health() -> dict:
"""
Check OceanBase health status with comprehensive metrics.
This function provides detailed health information including:
- Connection status
- Query latency
@@ -144,28 +132,22 @@ def check_oceanbase_health() -> dict:
- Query throughput (QPS)
- Slow query statistics
- Connection pool statistics
Returns:
dict: Health status with detailed metrics
"""
doc_engine = os.getenv('DOC_ENGINE', 'elasticsearch')
if doc_engine != 'oceanbase':
return {
"status": "not_configured",
"details": {
"connection": "not_configured",
"message": "OceanBase is not configured as the document engine"
}
}
doc_engine = os.getenv("DOC_ENGINE", "elasticsearch")
if doc_engine != "oceanbase":
return {"status": "not_configured", "details": {"connection": "not_configured", "message": "OceanBase is not configured as the document engine"}}
try:
ob_conn = OBConnection()
health_info = ob_conn.health()
performance_metrics = ob_conn.get_performance_metrics()
# Determine overall health status
connection_status = performance_metrics.get("connection", "unknown")
# If connection is disconnected, return unhealthy
if connection_status == "disconnected" or health_info.get("status") != "healthy":
return {
@@ -181,16 +163,15 @@ def check_oceanbase_health() -> dict:
"max_connections": performance_metrics.get("max_connections", 0),
"uri": health_info.get("uri", "unknown"),
"version": health_info.get("version_comment", "unknown"),
"error": health_info.get("error", performance_metrics.get("error"))
}
"error": health_info.get("error", performance_metrics.get("error")),
},
}
# Check if healthy (connected and low latency)
is_healthy = (
connection_status == "connected" and
performance_metrics.get("latency_ms", float('inf')) < 1000 # Latency under 1 second
connection_status == "connected" and performance_metrics.get("latency_ms", float("inf")) < 1000 # Latency under 1 second
)
return {
"status": "healthy" if is_healthy else "degraded",
"details": {
@@ -203,29 +184,20 @@ def check_oceanbase_health() -> dict:
"active_connections": performance_metrics.get("active_connections", 0),
"max_connections": performance_metrics.get("max_connections", 0),
"uri": health_info.get("uri", "unknown"),
"version": health_info.get("version_comment", "unknown")
}
"version": health_info.get("version_comment", "unknown"),
},
}
except Exception as e:
return {
"status": "unhealthy",
"details": {
"connection": "disconnected",
"error": str(e)
}
}
return {"status": "unhealthy", "details": {"connection": "disconnected", "error": str(e)}}
def get_mysql_status():
try:
cursor = DB.execute_sql("SHOW PROCESSLIST;")
res_rows = cursor.fetchall()
headers = ['id', 'user', 'host', 'db', 'command', 'time', 'state', 'info']
headers = ["id", "user", "host", "db", "command", "time", "state", "info"]
cursor.close()
return {
"status": "alive",
"message": [dict(zip(headers, r)) for r in res_rows]
}
return {"status": "alive", "message": [dict(zip(headers, r)) for r in res_rows]}
except Exception as e:
return {
"status": "timeout",
@@ -276,10 +248,7 @@ def check_minio_alive():
def get_redis_info():
try:
return {
"status": "alive",
"message": REDIS_CONN.info()
}
return {"status": "alive", "message": REDIS_CONN.info()}
except Exception as e:
return {
"status": "timeout",
@@ -290,9 +259,9 @@ def get_redis_info():
def check_ragflow_server_alive():
start_time = timer()
try:
url = f'http://{settings.HOST_IP}:{settings.HOST_PORT}/api/v1/system/ping'
if '0.0.0.0' in url:
url = url.replace('0.0.0.0', '127.0.0.1')
url = f"http://{settings.HOST_IP}:{settings.HOST_PORT}/api/v1/system/ping"
if "0.0.0.0" in url:
url = url.replace("0.0.0.0", "127.0.0.1")
response = requests.get(url, timeout=10)
if response.status_code == 200:
return {"status": "alive", "message": f"Confirm elapsed: {(timer() - start_time) * 1000.0:.1f} ms."}
@@ -320,10 +289,7 @@ def check_task_executor_alive():
else:
return {"status": "timeout", "message": "Not found any task executor."}
except Exception as e:
return {
"status": "timeout",
"message": f"error: {str(e)}"
}
return {"status": "timeout", "message": f"error: {str(e)}"}
def run_health_checks() -> tuple[dict, bool]:
@@ -358,7 +324,6 @@ def run_health_checks() -> tuple[dict, bool]:
except Exception:
result["storage"] = "nok"
all_ok = (result.get("db") == "ok") and (result.get("redis") == "ok") and (result.get("doc_engine") == "ok") and (
result.get("storage") == "ok")
all_ok = (result.get("db") == "ok") and (result.get("redis") == "ok") and (result.get("doc_engine") == "ok") and (result.get("storage") == "ok")
result["status"] = "ok" if all_ok else "nok"
return result, all_ok

16
api/utils/json_encode.py
View File

@@ -43,8 +43,7 @@ class BaseType:
data[_k] = _dict(vv)
else:
data = obj
return {"type": obj.__class__.__name__,
"data": data, "module": module}
return {"type": obj.__class__.__name__, "data": data, "module": module}
return _dict(self)
@@ -56,9 +55,9 @@ class CustomJSONEncoder(json.JSONEncoder):
def default(self, obj):
if isinstance(obj, datetime.datetime):
return obj.strftime('%Y-%m-%d %H:%M:%S')
return obj.strftime("%Y-%m-%d %H:%M:%S")
elif isinstance(obj, datetime.date):
return obj.strftime('%Y-%m-%d')
return obj.strftime("%Y-%m-%d")
elif isinstance(obj, datetime.timedelta):
return str(obj)
elif issubclass(type(obj), Enum) or issubclass(type(obj), IntEnum):
@@ -77,11 +76,7 @@ class CustomJSONEncoder(json.JSONEncoder):
def json_dumps(src, byte=False, indent=None, with_type=False):
dest = json.dumps(
src,
indent=indent,
cls=CustomJSONEncoder,
with_type=with_type)
dest = json.dumps(src, indent=indent, cls=CustomJSONEncoder, with_type=with_type)
if byte:
dest = string_to_bytes(dest)
return dest
@@ -90,5 +85,4 @@ def json_dumps(src, byte=False, indent=None, with_type=False):
def json_loads(src, object_hook=None, object_pairs_hook=None):
if isinstance(src, bytes):
src = bytes_to_string(src)
return json.loads(src, object_hook=object_hook,
object_pairs_hook=object_pairs_hook)
return json.loads(src, object_hook=object_hook, object_pairs_hook=object_pairs_hook)

3
api/utils/memory_utils.py
View File

@@ -16,6 +16,7 @@
from typing import List
from common.constants import MemoryType
def format_ret_data_from_memory(memory):
return {
"id": memory.id,
@@ -37,7 +38,7 @@ def format_ret_data_from_memory(memory):
"create_time": memory.create_time,
"create_date": memory.create_date,
"update_time": memory.update_time,
"update_date": memory.update_date
"update_date": memory.update_date,
}

9
api/utils/reference_metadata_utils.py
View File

@@ -51,8 +51,7 @@ def resolve_reference_metadata_preferences(
return include_metadata, None
if not isinstance(fields, list):
logger.warning(
"reference_metadata.fields is not a list; include_metadata=%s fields=%r type=%s resolved=%r. "
"enrich_chunks_with_document_metadata will skip enrichment.",
"reference_metadata.fields is not a list; include_metadata=%s fields=%r type=%s resolved=%r. enrich_chunks_with_document_metadata will skip enrichment.",
include_metadata,
fields,
type(fields).__name__,
@@ -96,12 +95,10 @@ def enrich_chunks_with_document_metadata(
# Resolve service lazily so callers/tests that swap service modules at runtime
# (e.g. via monkeypatch) don't get stuck with a stale class reference.
from api.db.services.doc_metadata_service import DocMetadataService
metadata_getter = getattr(DocMetadataService, "get_metadata_for_documents", None)
if not callable(metadata_getter):
logging.warning(
"DocMetadataService.get_metadata_for_documents is unavailable; "
"skipping metadata enrichment."
)
logging.warning("DocMetadataService.get_metadata_for_documents is unavailable; skipping metadata enrichment.")
return
meta_by_doc: dict[str, dict] = {}