mirror of
https://github.com/infiniflow/ragflow.git
synced 2026-07-26 18:33:30 +08:00
Fix: add deadlock retry (#13552)
### What problem does this PR solve? Add deadlock retry. ### Type of change - [x] Bug Fix (non-breaking change which fixes an issue)
This commit is contained in:
@@ -13,7 +13,11 @@
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
#
|
||||
import logging
|
||||
import time
|
||||
from datetime import datetime
|
||||
from functools import wraps
|
||||
|
||||
from tenacity import retry, stop_after_attempt, wait_exponential, retry_if_exception_type
|
||||
import peewee
|
||||
from peewee import InterfaceError, OperationalError
|
||||
@@ -22,6 +26,38 @@ from api.db.db_models import DB
|
||||
from common.misc_utils import get_uuid
|
||||
from common.time_utils import current_timestamp, datetime_format
|
||||
|
||||
|
||||
def _is_deadlock_error(exc: OperationalError) -> bool:
|
||||
return isinstance(exc, OperationalError) and bool(getattr(exc, "args", ())) and exc.args[0] == 1213
|
||||
|
||||
|
||||
def retry_deadlock_operation(max_retries=3, retry_delay=0.1):
|
||||
"""Retry a full DB operation when MySQL/OceanBase aborts it due to deadlock."""
|
||||
|
||||
def decorator(func):
|
||||
@wraps(func)
|
||||
def wrapper(*args, **kwargs):
|
||||
for attempt in range(max_retries):
|
||||
try:
|
||||
return func(*args, **kwargs)
|
||||
except OperationalError as e:
|
||||
if not _is_deadlock_error(e) or attempt >= max_retries - 1:
|
||||
raise
|
||||
current_delay = retry_delay * (2**attempt)
|
||||
logging.warning(
|
||||
"%s failed due to DB deadlock, retrying (%s/%s): %s",
|
||||
func.__qualname__,
|
||||
attempt + 1,
|
||||
max_retries,
|
||||
e,
|
||||
)
|
||||
time.sleep(current_delay)
|
||||
|
||||
return wrapper
|
||||
|
||||
return decorator
|
||||
|
||||
|
||||
def retry_db_operation(func):
|
||||
@retry(
|
||||
stop=stop_after_attempt(3),
|
||||
@@ -34,6 +70,7 @@ def retry_db_operation(func):
|
||||
return func(*args, **kwargs)
|
||||
return wrapper
|
||||
|
||||
|
||||
class CommonService:
|
||||
"""Base service class that provides common database operations.
|
||||
|
||||
@@ -279,7 +316,7 @@ class CommonService:
|
||||
# Returns:
|
||||
# Number of records deleted
|
||||
return cls.model.delete().where(cls.model.id == pid).execute()
|
||||
|
||||
|
||||
@classmethod
|
||||
@DB.connection_context()
|
||||
def delete_by_ids(cls, pids):
|
||||
|
||||
Reference in New Issue
Block a user