Merge branch 'main' into fix-pr13295-conflicts

# Conflicts:
#	api/apps/canvas_app.py
#	api/apps/restful_apis/mcp_api.py
#	api/db/services/canvas_service.py

api/utils/api_utils.py

@@ -19,7 +19,6 @@ import functools
 import inspect
 import json
 import logging
-import os
 import sys
 import time
 from copy import deepcopy
@@ -28,7 +27,6 @@ from typing import Any
 
 import requests
 from quart import (
-    Response,
     jsonify,
     request,
     has_app_context,
@@ -42,8 +40,7 @@ except ImportError:  # pragma: no cover - optional dependency
 
 from peewee import OperationalError
 
-from common.constants import ActiveEnum
-from api.db.db_models import APIToken
+from common.constants import ActiveEnum, LLMType
 from api.utils.json_encode import CustomJSONEncoder
 from common.mcp_tool_call_conn import MCPToolCallSession, close_multiple_mcp_toolcall_sessions
 from api.db.services.tenant_llm_service import LLMFactoriesService
@@ -148,6 +145,9 @@ def server_error_response(e):
     if repr(e).find("index_not_found_exception") >= 0:
         return get_json_result(code=RetCode.EXCEPTION_ERROR, message="No chunk found, please upload file and parse it.")
 
+    if "not_found" in str(e):
+        return get_error_data_result(message="No chunk found! Check the chunk status please!")
+
     return get_json_result(code=RetCode.EXCEPTION_ERROR, message=repr(e))
 
 
@@ -234,27 +234,22 @@ def active_required(func):
     return wrapper
 
 
+def add_tenant_id_to_kwargs(func):
+    @wraps(func)
+    async def wrapper(**kwargs):
+        from api.apps import current_user
+        kwargs["tenant_id"] = current_user.id
+        if inspect.iscoroutinefunction(func):
+            return await func(**kwargs)
+        return func(**kwargs)
+    return wrapper
+
+
 def get_json_result(code: RetCode = RetCode.SUCCESS, message="success", data=None):
     response = {"code": code, "message": message, "data": data}
     return _safe_jsonify(response)
 
 
-def apikey_required(func):
-    @wraps(func)
-    async def decorated_function(*args, **kwargs):
-        token = request.headers.get("Authorization").split()[1]
-        objs = APIToken.query(token=token)
-        if not objs:
-            return build_error_result(message="API-KEY is invalid!", code=RetCode.FORBIDDEN)
-        kwargs["tenant_id"] = objs[0].tenant_id
-        if inspect.iscoroutinefunction(func):
-            return await func(*args, **kwargs)
-
-        return func(*args, **kwargs)
-
-    return decorated_function
-
-
 def build_error_result(code=RetCode.FORBIDDEN, message="success"):
     response = {"code": code, "message": message}
     response = _safe_jsonify(response)
@@ -269,42 +264,6 @@ def construct_json_result(code: RetCode = RetCode.SUCCESS, message="success", da
     return _safe_jsonify({"code": code, "message": message, "data": data})
 
 
-def token_required(func):
-    def get_tenant_id(**kwargs):
-        if os.environ.get("DISABLE_SDK"):
-            return False, get_json_result(data=False, message="`Authorization` can't be empty")
-        authorization_str = request.headers.get("Authorization")
-        if not authorization_str:
-            return False, get_json_result(data=False, message="`Authorization` can't be empty")
-        authorization_list = authorization_str.split()
-        if len(authorization_list) < 2:
-            return False, get_json_result(data=False, message="Please check your authorization format.")
-        token = authorization_list[1]
-        objs = APIToken.query(token=token)
-        if not objs:
-            return False, get_json_result(data=False, message="Authentication error: API key is invalid!", code=RetCode.AUTHENTICATION_ERROR)
-        kwargs["tenant_id"] = objs[0].tenant_id
-        return True, kwargs
-
-    @wraps(func)
-    def decorated_function(*args, **kwargs):
-        e, kwargs = get_tenant_id(**kwargs)
-        if not e:
-            return kwargs
-        return func(*args, **kwargs)
-
-    @wraps(func)
-    async def adecorated_function(*args, **kwargs):
-        e, kwargs = get_tenant_id(**kwargs)
-        if not e:
-            return kwargs
-        return await func(*args, **kwargs)
-
-    if inspect.iscoroutinefunction(func):
-        return adecorated_function
-    return decorated_function
-
-
 def get_result(code=RetCode.SUCCESS, message="", data=None, total=None):
     """
     Standard API response format:
@@ -396,6 +355,20 @@ def get_parser_config(chunk_method, parser_config):
                     "category",
                 ],
                 "method": "light",
+                "batch_chunk_token_size": 4096,
+                "retry_attempts": 2,
+                "retry_backoff_seconds": 2.0,
+                "retry_backoff_max_seconds": 60.0,
+                "build_subgraph_timeout_per_chunk_seconds": 300,
+                "build_subgraph_min_timeout_seconds": 600,
+                "merge_timeout_seconds": 180,
+                "resolution_timeout_seconds": 1800,
+                "community_timeout_seconds": 1800,
+                "lock_acquire_timeout_seconds": 600,
+            },
+            "parent_child": {
+                "use_parent_child": False,
+                "children_delimiter": "\n",
             },
         },
         "qa": {"raptor": {"use_raptor": False}, "graphrag": {"use_graphrag": False}},
@@ -424,16 +397,23 @@ def get_parser_config(chunk_method, parser_config):
     # If no parser_config provided, return default merged with base defaults
     if not parser_config:
         if default_config is None:
-            return deep_merge(base_defaults, {})
-        return deep_merge(base_defaults, default_config)
+            merged_config = deep_merge(base_defaults, {})
+        else:
+            merged_config = deep_merge(base_defaults, default_config)
+    elif default_config is None:
+        # If parser_config is provided but no defaults for this method
+        merged_config = deep_merge(base_defaults, parser_config)
+    else:
+        # Ensure raptor and graph_rag fields have default values if not provided
+        merged_config = deep_merge(base_defaults, default_config)
+        merged_config = deep_merge(merged_config, parser_config)
 
-    # If parser_config is provided, merge with defaults to ensure required fields exist
-    if default_config is None:
-        return deep_merge(base_defaults, parser_config)
-
-    # Ensure raptor and graph_rag fields have default values if not provided
-    merged_config = deep_merge(base_defaults, default_config)
-    merged_config = deep_merge(merged_config, parser_config)
+    # Flatten parent_child config into children_delimiter for the execution layer
+    pc = merged_config.get("parent_child", {})
+    if pc.get("use_parent_child"):
+        merged_config["children_delimiter"] = pc.get("children_delimiter", "\n")
+    elif pc:
+        merged_config["children_delimiter"] = ""
 
     return merged_config
 
@@ -511,9 +491,8 @@ def check_duplicate_ids(ids, id_type="item"):
     return list(set(ids)), duplicate_messages
 
 
-def verify_embedding_availability(embd_id: str, tenant_id: str) -> tuple[bool, Response | None]:
-    from api.db.services.llm_service import LLMService
-    from api.db.services.tenant_llm_service import TenantLLMService
+def verify_embedding_availability(embd_id: str, tenant_id: str) -> tuple[bool, str | None]:
+    from api.db.joint_services.tenant_model_service import get_model_config_from_provider_instance
 
     """
     Verifies availability of an embedding model for a specific tenant.
@@ -549,21 +528,15 @@ def verify_embedding_availability(embd_id: str, tenant_id: str) -> tuple[bool, R
         (False, {'code': 101, 'message': "Unsupported model: <invalid_model>"})
     """
     try:
-        llm_name, llm_factory = TenantLLMService.split_model_name_and_factory(embd_id)
-        in_llm_service = bool(LLMService.query(llm_name=llm_name, fid=llm_factory, model_type="embedding"))
-
-        tenant_llms = TenantLLMService.get_my_llms(tenant_id=tenant_id)
-        is_tenant_model = any(llm["llm_name"] == llm_name and llm["llm_factory"] == llm_factory and llm["model_type"] == "embedding" for llm in tenant_llms)
-
-        is_builtin_model = llm_factory == "Builtin"
-        if not (is_builtin_model or is_tenant_model or in_llm_service):
-            return False, get_error_argument_result(f"Unsupported model: <{embd_id}>")
-
-        if not (is_builtin_model or is_tenant_model):
-            return False, get_error_argument_result(f"Unauthorized model: <{embd_id}>")
+        get_model_config_from_provider_instance(tenant_id, LLMType.EMBEDDING, embd_id)
+    except LookupError as e:
+        return False, str(e)
     except OperationalError as e:
         logging.exception(e)
-        return False, get_error_data_result(message="Database operation failed")
+        return False, "Database operation failed"
+    except Exception as e:
+        logging.exception(e)
+        return False, "Internal server error"
 
     return True, None
 

api/utils/configs.py

@@ -18,7 +18,6 @@ import io
 import base64
 import pickle
 from api.utils.common import bytes_to_string, string_to_bytes
-from common.config_utils import get_base_config
 
 safe_module = {
     'numpy',
@@ -54,8 +53,4 @@ def deserialize_b64(src):
     src = base64.b64decode(
         string_to_bytes(src) if isinstance(
             src, str) else src)
-    use_deserialize_safe_module = get_base_config(
-        'use_deserialize_safe_module', False)
-    if use_deserialize_safe_module:
-        return restricted_loads(src)
-    return pickle.loads(src)
+    return restricted_loads(src)

api/utils/crypt.py

@@ -17,6 +17,7 @@
 import base64
 import os
 import sys
+from pathlib import Path
 from Cryptodome.PublicKey import RSA
 from Cryptodome.Cipher import PKCS1_v1_5 as Cipher_pkcs1_v1_5
 from common.file_utils import get_project_base_directory
@@ -27,7 +28,7 @@ def crypt(line):
     decrypt(crypt(input_string)) == base64(input_string), which frontend and ragflow_cli use.
     """
     file_path = os.path.join(get_project_base_directory(), "conf", "public.pem")
-    rsa_key = RSA.importKey(open(file_path).read(), "Welcome")
+    rsa_key = RSA.importKey(Path(file_path).read_text(), "Welcome")
     cipher = Cipher_pkcs1_v1_5.new(rsa_key)
     password_base64 = base64.b64encode(line.encode('utf-8')).decode("utf-8")
     encrypted_password = cipher.encrypt(password_base64.encode())
@@ -36,7 +37,7 @@ def crypt(line):
 
 def decrypt(line):
     file_path = os.path.join(get_project_base_directory(), "conf", "private.pem")
-    rsa_key = RSA.importKey(open(file_path).read(), "Welcome")
+    rsa_key = RSA.importKey(Path(file_path).read_text(), "Welcome")
     cipher = Cipher_pkcs1_v1_5.new(rsa_key)
     return cipher.decrypt(base64.b64decode(line), "Fail to decrypt password!").decode('utf-8')
 
@@ -51,7 +52,7 @@ def decrypt2(crypt_text):
         decode_data = b16decode(hex_fixed.upper())
 
     file_path = os.path.join(get_project_base_directory(), "conf", "private.pem")
-    pem = open(file_path).read()
+    pem = Path(file_path).read_text()
     rsa_key = RSA.importKey(pem, "Welcome")
     cipher = Cipher_PKCS1_v1_5.new(rsa_key)
     decrypt_text = cipher.decrypt(decode_data, None)

api/utils/file_utils.py

@@ -35,8 +35,8 @@ from api.db import FileType
 
 # Robustness and resource limits: reject oversized inputs to avoid DoS and OOM.
 MAX_BLOB_SIZE_THUMBNAIL = 50 * 1024 * 1024  # 50 MiB for thumbnail generation
-MAX_BLOB_SIZE_PDF = 100 * 1024 * 1024       # 100 MiB for PDF repair / read
-GHOSTSCRIPT_TIMEOUT_SEC = 120                # Timeout for Ghostscript subprocess
+MAX_BLOB_SIZE_PDF = 100 * 1024 * 1024  # 100 MiB for PDF repair / read
+GHOSTSCRIPT_TIMEOUT_SEC = 120  # Timeout for Ghostscript subprocess
 
 LOCK_KEY_pdfplumber = "global_shared_lock_pdfplumber"
 if LOCK_KEY_pdfplumber not in sys.modules:
@@ -64,13 +64,17 @@ def filename_type(filename):
     if re.match(r".*\.pdf$", filename):
         return FileType.PDF.value
 
-    if re.match(r".*\.(msg|eml|doc|docx|ppt|pptx|yml|xml|htm|json|jsonl|ldjson|csv|txt|ini|xls|xlsx|wps|rtf|hlp|pages|numbers|key|md|mdx|py|js|java|c|cpp|h|php|go|ts|sh|cs|kt|html|sql)$", filename):
+    if re.match(
+        r".*\.(msg|eml|doc|docx|ppt|pptx|yml|xml|htm|json|jsonl|ldjson|csv|txt|ini|xls|xlsx|wps|rtf|hlp|pages|numbers|key|md|mdx|py|js|java|c|cpp|h|php|go|ts|sh|cs|kt|html|sql|epub)$", filename
+    ):
         return FileType.DOC.value
 
     if re.match(r".*\.(wav|flac|ape|alac|wavpack|wv|mp3|aac|ogg|vorbis|opus)$", filename):
         return FileType.AURAL.value
 
-    if re.match(r".*\.(jpg|jpeg|png|tif|gif|pcx|tga|exif|fpx|svg|psd|cdr|pcd|dxf|ufo|eps|ai|raw|WMF|webp|avif|apng|icon|ico|mpg|mpeg|avi|rm|rmvb|mov|wmv|asf|dat|asx|wvx|mpe|mpa|mp4|avi|mkv)$", filename):
+    if re.match(
+        r".*\.(jpg|jpeg|png|tif|gif|pcx|tga|exif|fpx|svg|psd|cdr|pcd|dxf|ufo|eps|ai|raw|WMF|webp|avif|apng|icon|ico|mpg|mpeg|avi|rm|rmvb|mov|wmv|asf|dat|asx|wvx|mpe|mpa|mp4|avi|mkv)$", filename
+    ):
         return FileType.VISUAL.value
 
     return FileType.OTHER.value
@@ -103,23 +107,21 @@ def thumbnail_img(filename, blob):
     if re.match(r".*\.pdf$", filename):
         try:
             with sys.modules[LOCK_KEY_pdfplumber]:
-                pdf = pdfplumber.open(BytesIO(blob))
-                if not pdf.pages:
-                    pdf.close()
-                    return None
-                buffered = BytesIO()
-                resolution = 32
-                img = None
-                for _ in range(10):
-                    pdf.pages[0].to_image(resolution=resolution).annotated.save(buffered, format="png")
-                    img = buffered.getvalue()
-                    if len(img) >= 64000 and resolution >= 2:
-                        resolution = resolution / 2
-                        buffered = BytesIO()
-                    else:
-                        break
-                pdf.close()
-                return img
+                with pdfplumber.open(BytesIO(blob)) as pdf:
+                    if not pdf.pages:
+                        return None
+                    buffered = BytesIO()
+                    resolution = 32
+                    img = None
+                    for _ in range(10):
+                        pdf.pages[0].to_image(resolution=resolution).annotated.save(buffered, format="png")
+                        img = buffered.getvalue()
+                        if len(img) >= 64000 and resolution >= 2:
+                            resolution = resolution / 2
+                            buffered = BytesIO()
+                        else:
+                            break
+                    return img
         except Exception:
             return None
 

api/utils/health_utils.py

@@ -290,10 +290,10 @@ def get_redis_info():
 def check_ragflow_server_alive():
     start_time = timer()
     try:
-        url = f'http://{settings.HOST_IP}:{settings.HOST_PORT}/v1/system/ping'
+        url = f'http://{settings.HOST_IP}:{settings.HOST_PORT}/api/v1/system/ping'
         if '0.0.0.0' in url:
             url = url.replace('0.0.0.0', '127.0.0.1')
-        response = requests.get(url)
+        response = requests.get(url, timeout=10)
         if response.status_code == 200:
             return {"status": "alive", "message": f"Confirm elapsed: {(timer() - start_time) * 1000.0:.1f} ms."}
         else:

api/utils/image_utils.py

@@ -0,0 +1,40 @@
+#
+#  Copyright 2025 The InfiniFlow Authors. All Rights Reserved.
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+#
+
+from io import BytesIO
+
+from PIL import Image
+
+from common import settings
+
+
+def store_chunk_image(bucket, name, image_binary):
+    if settings.STORAGE_IMPL.obj_exist(bucket, name):
+        old_binary = settings.STORAGE_IMPL.get(bucket, name)
+        old_img = Image.open(BytesIO(old_binary))
+        new_img = Image.open(BytesIO(image_binary))
+        old_img = old_img.convert("RGB")
+        new_img = new_img.convert("RGB")
+        width = max(old_img.width, new_img.width)
+        height = old_img.height + new_img.height
+        combined = Image.new("RGB", (width, height), (255, 255, 255))
+        combined.paste(old_img, (0, 0))
+        combined.paste(new_img, (0, old_img.height))
+        buf = BytesIO()
+        combined.save(buf, format="JPEG")
+        settings.STORAGE_IMPL.put(bucket, name, buf.getvalue())
+    else:
+        settings.STORAGE_IMPL.put(bucket, name, image_binary)

api/utils/nickname_validation.py

@@ -0,0 +1,51 @@
+#
+#  Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+#
+import logging
+import re
+
+from api.constants import NICKNAME_MAX_LENGTH
+from common.constants import RetCode
+
+# Match frontend NICKNAME_PATTERN: letters, numbers, space, and . _ ' -
+_NICKNAME_PATTERN = re.compile(r"^[\w ._'-]+$", re.UNICODE)
+
+
+def _reject_nickname(message: str) -> tuple[str, int]:
+    logging.warning("Nickname validation failed: %s", message)
+    return message, RetCode.ARGUMENT_ERROR
+
+
+def validate_nickname(nickname: str | None) -> tuple[str | None, int | None]:
+    """
+    Validate a user nickname/display name.
+
+    Returns:
+        A tuple of (error_message, error_code) if validation fails,
+        or (None, None) if validation passes.
+    """
+    if not isinstance(nickname, (str, type(None))):
+        return _reject_nickname("Nickname must be a string.")
+    if nickname is None:
+        return _reject_nickname("Nickname is required.")
+
+    nickname = nickname.strip()
+    if not nickname:
+        return _reject_nickname("Nickname cannot be empty.")
+    if len(nickname) > NICKNAME_MAX_LENGTH:
+        return _reject_nickname(f"Nickname must be at most {NICKNAME_MAX_LENGTH} characters.")
+    if not _NICKNAME_PATTERN.fullmatch(nickname):
+        return _reject_nickname("Nickname contains invalid characters.")
+    return None, None

api/utils/pagination_utils.py

@@ -0,0 +1,24 @@
+#
+#  Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+#
+
+REST_API_MAX_PAGE_SIZE = 100
+
+
+def validate_rest_api_page_size(page_size: int) -> int:
+    """Validate REST API page_size values against the public maximum."""
+    if page_size > REST_API_MAX_PAGE_SIZE:
+        raise ValueError(f"page_size must be less than or equal to {REST_API_MAX_PAGE_SIZE}")
+    return page_size

api/utils/reference_metadata_utils.py

@@ -0,0 +1,125 @@
+#
+#  Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+#
+
+import logging
+
+logger = logging.getLogger(__name__)
+
+
+def resolve_reference_metadata_preferences(
+    request_payload: dict | None = None,
+    config_payload: dict | None = None,
+) -> tuple[bool, set[str] | None]:
+    """
+    Resolve metadata include/fields from request and optional config.
+    Request values take precedence over config values.
+    Supports legacy request keys: include_metadata / metadata_fields.
+    """
+    request_payload = request_payload or {}
+    config_payload = config_payload or {}
+
+    config_ref = config_payload.get("reference_metadata", {})
+    request_ref = request_payload.get("reference_metadata", {})
+
+    resolved: dict = {}
+    if isinstance(config_ref, dict):
+        resolved.update(config_ref)
+    if isinstance(request_ref, dict):
+        resolved.update(request_ref)
+
+    if "include_metadata" in request_payload:
+        resolved["include"] = bool(request_payload.get("include_metadata"))
+    if "metadata_fields" in request_payload:
+        resolved["fields"] = request_payload.get("metadata_fields")
+
+    include_metadata = bool(resolved.get("include", False))
+    fields = resolved.get("fields")
+    if fields is None:
+        return include_metadata, None
+    if not isinstance(fields, list):
+        logger.warning(
+            "reference_metadata.fields is not a list; include_metadata=%s fields=%r type=%s resolved=%r. "
+            "enrich_chunks_with_document_metadata will skip enrichment.",
+            include_metadata,
+            fields,
+            type(fields).__name__,
+            resolved,
+        )
+        return include_metadata, set()
+    return include_metadata, {f for f in fields if isinstance(f, str)}
+
+
+def enrich_chunks_with_document_metadata(
+    chunks: list[dict],
+    metadata_fields: set[str] | None = None,
+    *,
+    kb_field: str = "kb_id",
+    doc_field: str = "doc_id",
+    output_field: str = "document_metadata",
+) -> None:
+    """
+    Mutates chunk payloads in-place by attaching `document_metadata`.
+    Field names can be customized for different chunk schemas.
+    """
+    if metadata_fields is not None and not metadata_fields:
+        return
+
+    doc_ids_by_kb: dict[str, set[str]] = {}
+    for chunk in chunks:
+        kb_ids = chunk.get(kb_field)
+        doc_id = chunk.get(doc_field)
+        if not kb_ids or not doc_id:
+            continue
+        if isinstance(kb_ids, (list, tuple)):
+            for kid in kb_ids:
+                if kid:
+                    doc_ids_by_kb.setdefault(kid, set()).add(doc_id)
+        else:
+            doc_ids_by_kb.setdefault(kb_ids, set()).add(doc_id)
+
+    if not doc_ids_by_kb:
+        return
+
+    # Resolve service lazily so callers/tests that swap service modules at runtime
+    # (e.g. via monkeypatch) don't get stuck with a stale class reference.
+    from api.db.services.doc_metadata_service import DocMetadataService
+    metadata_getter = getattr(DocMetadataService, "get_metadata_for_documents", None)
+    if not callable(metadata_getter):
+        logging.warning(
+            "DocMetadataService.get_metadata_for_documents is unavailable; "
+            "skipping metadata enrichment."
+        )
+        return
+
+    meta_by_doc: dict[str, dict] = {}
+    for kb_id, doc_ids in doc_ids_by_kb.items():
+        meta_map = metadata_getter(list(doc_ids), kb_id)
+        if meta_map:
+            meta_by_doc.update(meta_map)
+            logging.debug("Fetched metadata for %d docs in kb_id=%s", len(meta_map), kb_id)
+
+    for chunk in chunks:
+        doc_id = chunk.get(doc_field)
+        if not doc_id:
+            continue
+        meta = meta_by_doc.get(doc_id)
+        if not meta:
+            continue
+        if metadata_fields is not None:
+            meta = {k: v for k, v in meta.items() if k in metadata_fields}
+        if meta:
+            chunk[output_field] = meta
+            logging.debug("Enriched chunk for doc_id=%s with %d metadata fields: %s", doc_id, len(meta), list(meta.keys()))

api/utils/validation_utils.py

@@ -1,5 +1,5 @@
 #
-#  Copyright 2025 The InfiniFlow Authors. All Rights Reserved.
+#  Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
 #
 #  Licensed under the Apache License, Version 2.0 (the "License");
 #  you may not use this file except in compliance with the License.
@@ -13,28 +13,29 @@
 #  See the License for the specific language governing permissions and
 #  limitations under the License.
 #
+import logging
+import math
+import pathlib
+import re
 from collections import Counter
 import string
 from typing import Annotated, Any, Literal
 from uuid import UUID
 
 from quart import Request
-from pydantic import (
-    BaseModel,
-    ConfigDict,
-    Field,
-    StringConstraints,
-    ValidationError,
-    field_validator,
-    model_validator,
-)
+from pydantic import BaseModel, ConfigDict, Field, StringConstraints, ValidationError, field_validator, model_validator, ValidationInfo
 from pydantic_core import PydanticCustomError
 from werkzeug.exceptions import BadRequest, UnsupportedMediaType
 
-from api.constants import DATASET_NAME_LIMIT
+from api.constants import DATASET_NAME_LIMIT, FILE_NAME_LEN_LIMIT
+from api.db import FileType
+from api.utils.pagination_utils import validate_rest_api_page_size
+from common.constants import RetCode
 
 
-async def validate_and_parse_json_request(request: Request, validator: type[BaseModel], *, extras: dict[str, Any] | None = None, exclude_unset: bool = False) -> tuple[dict[str, Any] | None, str | None]:
+async def validate_and_parse_json_request(
+    request: Request, validator: type[BaseModel], *, extras: dict[str, Any] | None = None, exclude_unset: bool = False
+) -> tuple[dict[str, Any] | None, str | None]:
     """
     Validates and parses JSON requests through a multi-stage validation pipeline.
 
@@ -160,6 +161,16 @@ def validate_and_parse_request_args(request: Request, validator: type[BaseModel]
         - Preserves type conversion from Pydantic validation
     """
     args = request.args.to_dict(flat=True)
+
+    # Handle ext parameter: parse JSON string to dict if it's a string
+    if "ext" in args and isinstance(args["ext"], str):
+        import json
+
+        try:
+            args["ext"] = json.loads(args["ext"])
+        except json.JSONDecodeError:
+            logging.debug("Failed to decode query arg 'ext' as JSON; passing raw value to validator")
+
     try:
         if extras is not None:
             args.update(extras)
@@ -268,16 +279,20 @@ def normalize_str(v: Any) -> Any:
 
 def validate_uuid1_hex(v: Any) -> str:
     """
-    Validates and converts input to a UUID version 1 hexadecimal string.
+    Validates and converts input to a UUID hexadecimal string.
 
-    This function performs strict validation and normalization:
+    The function name is retained for backward compatibility; only UUID
+    *format* is enforced (any version is accepted), because some IDs in the
+    system originate from external imports and use non-v1 UUIDs.
+
+    This function performs validation and normalization:
     1. Accepts either UUID objects or UUID-formatted strings
-    2. Verifies the UUID is version 1 (time-based)
-    3. Returns the 32-character hexadecimal representation
+    2. Returns the 32-character hexadecimal representation
+    3. Rejects anything that is not a valid UUID
 
     Args:
         v (Any): Input value to validate. Can be:
-                - UUID object (must be version 1)
+                - UUID object (any version)
                 - String in UUID format (e.g. "550e8400-e29b-41d4-a716-446655440000")
 
     Returns:
@@ -285,9 +300,8 @@ def validate_uuid1_hex(v: Any) -> str:
              Example: "550e8400e29b41d4a716446655440000"
 
     Raises:
-        PydanticCustomError: With code "invalid_UUID1_format" when:
+        PydanticCustomError: With code "invalid_uuid_format" when:
             - Input is not a UUID object or valid UUID string
-            - UUID version is not 1
             - String doesn't match UUID format
 
     Examples:
@@ -300,27 +314,33 @@ def validate_uuid1_hex(v: Any) -> str:
         Invalid cases:
             >>> validate_uuid1_hex("not-a-uuid")  # raises PydanticCustomError
             >>> validate_uuid1_hex(12345)  # raises PydanticCustomError
-            >>> validate_uuid1_hex(UUID(int=0))  # v4, raises PydanticCustomError
 
     Notes:
         - Uses Python's built-in UUID parser for format validation
-        - Version check prevents accidental use of other UUID versions
+        - UUID version is no longer enforced (v1, v4, v7, etc. all accepted)
         - Hyphens in input strings are automatically removed in output
     """
     try:
-        uuid_obj = UUID(v) if isinstance(v, str) else v
-        if uuid_obj.version != 1:
-            raise PydanticCustomError("invalid_UUID1_format", "Must be a UUID1 format")
+        if isinstance(v, UUID):
+            uuid_obj = v
+        elif isinstance(v, str):
+            uuid_obj = UUID(v)
+        else:
+            raise TypeError
         return uuid_obj.hex
     except (AttributeError, ValueError, TypeError):
-        raise PydanticCustomError("invalid_UUID1_format", "Invalid UUID1 format")
+        raise PydanticCustomError("invalid_uuid_format", "Invalid UUID format")
 
 
 class Base(BaseModel):
+    """Strict base model that rejects unknown request fields."""
+
     model_config = ConfigDict(extra="forbid", strict=True)
 
 
 class RaptorConfig(Base):
+    """Dataset parser configuration for RAPTOR summary generation."""
+
     use_raptor: Annotated[bool, Field(default=False)]
     prompt: Annotated[
         str,
@@ -333,35 +353,62 @@ class RaptorConfig(Base):
     threshold: Annotated[float, Field(default=0.1, ge=0.0, le=1.0)]
     max_cluster: Annotated[int, Field(default=64, ge=1, le=1024)]
     random_seed: Annotated[int, Field(default=0, ge=0)]
+    scope: Annotated[Literal["file", "dataset"], Field(default="file")]
+    clustering_method: Annotated[Literal["gmm", "ahc"], Field(default="gmm")]
+    tree_builder: Annotated[Literal["raptor", "psi"], Field(default="raptor")]
     auto_disable_for_structured_data: Annotated[bool, Field(default=True)]
+    ext: Annotated[dict, Field(default={})]
 
 
 class GraphragConfig(Base):
+    """Dataset parser configuration for GraphRAG generation."""
+
     use_graphrag: Annotated[bool, Field(default=False)]
     entity_types: Annotated[list[str], Field(default_factory=lambda: ["organization", "person", "geo", "event", "category"])]
-    method: Annotated[Literal["light", "general"], Field(default="light")]
+    method: Annotated[Literal["light", "general", "ner"], Field(default="light")]
     community: Annotated[bool, Field(default=False)]
     resolution: Annotated[bool, Field(default=False)]
+    batch_chunk_token_size: Annotated[int, Field(default=4096, ge=512, le=8196)]
+    retry_attempts: Annotated[int, Field(default=2, ge=1, le=10)]
+    retry_backoff_seconds: Annotated[float, Field(default=2.0, ge=0.0, le=600.0)]
+    retry_backoff_max_seconds: Annotated[float, Field(default=60.0, ge=0.0, le=3600.0)]
+    build_subgraph_timeout_per_chunk_seconds: Annotated[int, Field(default=300, ge=1, le=86400)]
+    build_subgraph_min_timeout_seconds: Annotated[int, Field(default=600, ge=1, le=86400)]
+    merge_timeout_seconds: Annotated[int, Field(default=180, ge=0, le=86400)]
+    resolution_timeout_seconds: Annotated[int, Field(default=1800, ge=0, le=86400)]
+    community_timeout_seconds: Annotated[int, Field(default=1800, ge=0, le=86400)]
+    lock_acquire_timeout_seconds: Annotated[int, Field(default=600, ge=0, le=86400)]
+
+
+class ParentChildConfig(Base):
+    """Dataset parser configuration for parent-child chunking."""
+
+    use_parent_child: Annotated[bool, Field(default=False)]
+    children_delimiter: Annotated[str, Field(default=r"\n", min_length=1)]
 
 
 class AutoMetadataField(Base):
     """Schema for a single auto-metadata field configuration."""
 
-    name: Annotated[str, StringConstraints(strip_whitespace=True, min_length=1, max_length=255), Field(...)]
-    type: Annotated[Literal["string", "list", "time"], Field(...)]
+    key: Annotated[str, StringConstraints(strip_whitespace=True, min_length=1, max_length=255), Field(...)]
+    type: Annotated[Literal["string", "list", "time", "number"], Field(...)]
     description: Annotated[str | None, Field(default=None, max_length=65535)]
-    examples: Annotated[list[str] | None, Field(default=None)]
-    restrict_values: Annotated[bool, Field(default=False)]
+    enum: Annotated[list[str] | None, Field(default=None)]
 
 
 class AutoMetadataConfig(Base):
     """Top-level auto-metadata configuration attached to a dataset."""
 
-    enabled: Annotated[bool, Field(default=True)]
-    fields: Annotated[list[AutoMetadataField], Field(default_factory=list)]
+    metadata: Annotated[list[AutoMetadataField], Field(default_factory=list)]
+    built_in_metadata: Annotated[list[AutoMetadataField], Field(default_factory=list)]
+
+
+TableColumnRole = Literal["indexing", "metadata", "both"]
 
 
 class ParserConfig(Base):
+    """Complete parser configuration accepted by dataset APIs."""
+
     auto_keywords: Annotated[int, Field(default=0, ge=0, le=32)]
     auto_questions: Annotated[int, Field(default=0, ge=0, le=10)]
     chunk_token_num: Annotated[int, Field(default=512, ge=1, le=2048)]
@@ -369,25 +416,119 @@ class ParserConfig(Base):
     graphrag: Annotated[GraphragConfig, Field(default_factory=lambda: GraphragConfig(use_graphrag=False))]
     html4excel: Annotated[bool, Field(default=False)]
     layout_recognize: Annotated[str, Field(default="DeepDOC")]
+    parent_child: Annotated[ParentChildConfig, Field(default_factory=lambda: ParentChildConfig(use_parent_child=False))]
     raptor: Annotated[RaptorConfig, Field(default_factory=lambda: RaptorConfig(use_raptor=False))]
     tag_kb_ids: Annotated[list[str], Field(default_factory=list)]
     topn_tags: Annotated[int, Field(default=1, ge=1, le=10)]
     filename_embd_weight: Annotated[float | None, Field(default=0.1, ge=0.0, le=1.0)]
     task_page_size: Annotated[int | None, Field(default=None, ge=1)]
     pages: Annotated[list[list[int]] | None, Field(default=None)]
+    ext: Annotated[dict, Field(default={})]
+    # Table parser: column name -> "indexing" | "metadata" | "both". Absence => all columns "both".
+    # Table parser: "auto" = all columns both (default), "manual" = use table_column_roles. None → treated as "auto".
+    table_column_mode: Annotated[Literal["auto", "manual"] | None, Field(default=None)]
+    # Table parser: column name -> "indexing" | "metadata" | "both". Used only when table_column_mode == "manual".
+    table_column_roles: Annotated[dict[str, TableColumnRole] | None, Field(default=None)]
+    # Table parser: list of column names (set by backend after first parse; used by frontend for role selector).
+    table_column_names: Annotated[list[str] | None, Field(default=None)]
+
+    @field_validator("table_column_roles", mode="before")
+    @classmethod
+    def legacy_vectorize_table_column_role(cls, v: Any) -> Any:
+        """Normalize legacy role value *vectorize* to *indexing* (chunk text + full-text search)."""
+        if v is None or not isinstance(v, dict):
+            return v
+        out: dict[str, Any] = {}
+        for key, val in v.items():
+            k = key if isinstance(key, str) else str(key)
+            out[k] = "indexing" if val == "vectorize" else val
+        return out
+
+
+class UpdateDocumentReq(Base):
+    """
+    Request model for updating a document.
+
+    This model validates the request parameters for updating a document,
+    including name, chunk method, enabled status, and other metadata.
+    """
+
+    model_config = ConfigDict(extra="ignore")
+    name: Annotated[str | None, Field(default=None, max_length=65535)]
+    chunk_method: Annotated[str | None, Field(default=None, max_length=65535)]
+    pipeline_id: Annotated[str | None, Field(default=None, max_length=65535)]
+    enabled: Annotated[int | None, Field(default=None, ge=0, le=1)]
+    chunk_count: Annotated[int | None, Field(default=None, ge=0)]
+    token_count: Annotated[int | None, Field(default=None, ge=0)]
+    progress: Annotated[float | None, Field(default=None, ge=0.0, le=1.0)]
+    parser_config: Annotated[ParserConfig | None, Field(default=None)]
+    meta_fields: Annotated[dict | None, Field(default={})]
+
+    @field_validator("chunk_method", mode="after")
+    @classmethod
+    def validate_document_chunk_method(cls, chunk_method: str | None):
+        """Validate an optional document parser method."""
+        if chunk_method:
+            # Validate chunk method if present
+            valid_chunk_method = {"naive", "manual", "qa", "table", "paper", "book", "laws", "presentation", "picture", "one", "knowledge_graph", "email", "tag"}
+            if chunk_method not in valid_chunk_method:
+                raise PydanticCustomError("format_invalid", "`chunk_method` {chunk_method} doesn't exist", {"chunk_method": chunk_method})
+
+        return chunk_method
+
+    @field_validator("enabled", mode="after")
+    @classmethod
+    def validate_document_enabled(cls, enabled: str | None):
+        """Validate the optional enabled flag."""
+        if enabled:
+            converted = int(enabled)
+            if converted < 0 or converted > 1:
+                raise PydanticCustomError("format_invalid", "`enabled` value invalid, only accept 0 or 1 but is {enabled}", {"enabled": enabled})
+
+        return enabled
+
+    @field_validator("meta_fields", mode="after")
+    @classmethod
+    def validate_document_meta_fields(cls, meta_fields: dict | None):
+        """Validate user-provided document metadata values."""
+        if meta_fields is None:
+            return None
+
+        if not isinstance(meta_fields, dict):
+            raise PydanticCustomError("format_invalid", "Only dictionary type supported")
+        for k, v in meta_fields.items():
+            if isinstance(v, list):
+                if not all(isinstance(i, (str, int, float)) for i in v):
+                    raise PydanticCustomError("format_invalid", "The type is not supported in list: {v}", {"v": v})
+            elif not isinstance(v, (str, int, float)):
+                raise PydanticCustomError("format_invalid", "The type is not supported: {v}", {"v": v})
+        return meta_fields
 
 
 class CreateDatasetReq(Base):
+    """Request model for creating a dataset."""
+
     name: Annotated[str, StringConstraints(strip_whitespace=True, min_length=1, max_length=DATASET_NAME_LIMIT), Field(...)]
     avatar: Annotated[str | None, Field(default=None, max_length=65535)]
     description: Annotated[str | None, Field(default=None, max_length=65535)]
     embedding_model: Annotated[str | None, Field(default=None, max_length=255, serialization_alias="embd_id")]
     permission: Annotated[Literal["me", "team"], Field(default="me", min_length=1, max_length=16)]
-    chunk_method: Annotated[str | None, Field(default=None, serialization_alias="parser_id")]
     parse_type: Annotated[int | None, Field(default=None, ge=0, le=64)]
     pipeline_id: Annotated[str | None, Field(default=None, min_length=32, max_length=32, serialization_alias="pipeline_id")]
+    chunk_method: Annotated[str | None, Field(default=None, serialization_alias="parser_id")]
     parser_config: Annotated[ParserConfig | None, Field(default=None)]
     auto_metadata_config: Annotated[AutoMetadataConfig | None, Field(default=None)]
+    ext: Annotated[dict, Field(default={})]
+
+    @field_validator("pipeline_id", mode="before")
+    @classmethod
+    def handle_pipeline_id(cls, v: str | None, info: ValidationInfo):
+        """Drop pipeline_id when parse_type selects direct parser mode."""
+        if v is None:
+            return v
+        if info.data.get("parse_type", 0) == 1:
+            v = None
+        return v
 
     @field_validator("avatar", mode="after")
     @classmethod
@@ -422,7 +563,7 @@ class CreateDatasetReq(Base):
             CreateDatasetReq(avatar="data:video/mp4;base64,...")  # Unsupported MIME type
             ```
         """
-        if v is None:
+        if not v: # cover both None and empty string
             return v
 
         if "," in v:
@@ -600,11 +741,11 @@ class CreateDatasetReq(Base):
             # Both provided → allow pipeline mode
             return self
 
-        # parser_id provided (valid): MUST NOT have parse_type or pipeline_id
+        # parser_id provided (valid): parse_type MUST be one of [None, 1], and MUST NOT have pipeline_id
         if isinstance(self.chunk_method, str):
-            if self.parse_type is not None or self.pipeline_id is not None:
-                invalid = []
-                if self.parse_type is not None:
+            invalid = []
+            if self.parse_type not in [None, 1] or self.pipeline_id is not None:
+                if self.parse_type not in [None, 1]:
                     invalid.append("parse_type")
                 if self.pipeline_id is not None:
                     invalid.append("pipeline_id")
@@ -617,37 +758,45 @@ class CreateDatasetReq(Base):
 
     @field_validator("chunk_method", mode="wrap")
     @classmethod
-    def validate_chunk_method(cls, v: Any, handler) -> Any:
+    def validate_chunk_method(cls, v: Any, handler, info: ValidationInfo) -> Any:
         """Wrap validation to unify error messages, including type errors (e.g. list)."""
-        allowed = {"naive", "book", "email", "laws", "manual", "one", "paper", "picture", "presentation", "qa", "table", "tag"}
-        error_msg = "Input should be 'naive', 'book', 'email', 'laws', 'manual', 'one', 'paper', 'picture', 'presentation', 'qa', 'table' or 'tag'"
-        # Omitted field: handler won't be invoked (wrap still gets value); None treated as explicit invalid
-        if v is None:
-            raise PydanticCustomError("literal_error", error_msg)
+        allowed = {"naive", "book", "email", "laws", "manual", "one", "paper", "picture", "presentation", "qa", "table", "tag", "resume"}
+        error_msg = "Input should be 'naive', 'book', 'email', 'laws', 'manual', 'one', 'paper', 'picture', 'presentation', 'qa', 'table', 'tag' or 'resume'"
         try:
             # Run inner validation (type checking)
             result = handler(v)
         except Exception:
             raise PydanticCustomError("literal_error", error_msg)
+            # Omitted field: handler won't be invoked (wrap still gets value); None treated as explicit invalid
+        if not result and not info.data.get("pipeline_id", None):
+            raise PydanticCustomError("literal_error", error_msg)
         # After handler, enforce enumeration
-        if not isinstance(result, str) or result == "" or result not in allowed:
+        if result and result not in allowed:
             raise PydanticCustomError("literal_error", error_msg)
         return result
 
 
 class UpdateDatasetReq(CreateDatasetReq):
+    """Request model for updating a dataset."""
+
     dataset_id: Annotated[str, Field(...)]
     name: Annotated[str, StringConstraints(strip_whitespace=True, min_length=1, max_length=DATASET_NAME_LIMIT), Field(default="")]
     pagerank: Annotated[int, Field(default=0, ge=0, le=100)]
+    language: Annotated[str | None, Field(default=None, max_length=32)]
+    connectors: Annotated[list[dict[str, Any]], Field(default_factory=list)]
 
     @field_validator("dataset_id", mode="before")
     @classmethod
     def validate_dataset_id(cls, v: Any) -> str:
+        """Validate and normalize the dataset id."""
         return validate_uuid1_hex(v)
 
 
 class DeleteReq(Base):
-    ids: Annotated[list[str] | None, Field(...)]
+    """Base request model for batch delete APIs."""
+
+    ids: Annotated[list[str] | None, Field(default=None)]
+    delete_all: Annotated[bool, Field(default=False)]
 
     @field_validator("ids", mode="after")
     @classmethod
@@ -657,7 +806,7 @@ class DeleteReq(Base):
 
         This post-processing validator performs:
         1. None input handling (pass-through)
-        2. UUID version 1 validation for each list item
+        2. UUID format validation for each list item (any version accepted)
         3. Duplicate value detection
         4. Returns normalized UUID hex strings or None
 
@@ -670,18 +819,18 @@ class DeleteReq(Base):
             - None if input was None
             - List of normalized UUID hex strings otherwise:
             * 32-character lowercase
-            * Valid UUID version 1
+            * Valid UUID format (any version)
             * Unique within list
 
         Raises:
             PydanticCustomError: With structured error details when:
-                - "invalid_UUID1_format": Any string fails UUIDv1 validation
+                - "invalid_uuid_format": Any string fails UUID format validation
                 - "duplicate_uuids": If duplicate IDs are detected
 
         Validation Rules:
             - None input returns None
             - Empty list returns empty list
-            - All non-None items must be valid UUIDv1
+            - All non-None items must be valid UUIDs (any version)
             - No duplicates permitted
             - Original order preserved
 
@@ -696,12 +845,12 @@ class DeleteReq(Base):
 
             Invalid cases:
                 >>> validate_ids(["invalid"])
-                # raises PydanticCustomError(invalid_UUID1_format)
+                # raises PydanticCustomError(invalid_uuid_format)
                 >>> validate_ids(["550e...", "550e..."])
                 # raises PydanticCustomError(duplicate_uuids)
 
         Security Notes:
-            - Validates UUID version to prevent version spoofing
+            - Validates UUID format (any version)
             - Duplicate check prevents data injection
             - None handling maintains pipeline integrity
         """
@@ -723,10 +872,85 @@ class DeleteReq(Base):
         return ids_list
 
 
-class DeleteDatasetReq(DeleteReq): ...
+class DeleteDatasetReq(DeleteReq):
+    """Request model for deleting datasets."""
+
+    ...
+
+
+class DeleteDocumentReq(DeleteReq):
+    """Request model for deleting documents."""
+
+    @field_validator("ids", mode="after")
+    @classmethod
+    def validate_ids(cls, v_list: list[str] | None) -> list[str] | None:
+        """
+        Validate document IDs without enforcing UUIDv1.
+
+        Connector-backed documents can use non-UUID identifiers, so we only
+        enforce uniqueness here and leave existence checks to the delete API.
+        """
+        if v_list is None:
+            return None
+
+        duplicates = [item for item, count in Counter(v_list).items() if count > 1]
+        if duplicates:
+            duplicates_str = ", ".join(duplicates)
+            raise PydanticCustomError(
+                "duplicate_uuids",
+                "Duplicate ids: '{duplicate_ids}'",
+                {"duplicate_ids": duplicates_str},
+            )
+
+        return v_list
+
+
+class SearchDatasetReq(BaseModel):
+    """Request model for searching one dataset."""
+
+    model_config = ConfigDict(extra="ignore")
+
+    question: Annotated[str, StringConstraints(strip_whitespace=True, min_length=1), Field(...)]
+    doc_ids: Annotated[list[str], Field(default=[])]
+    page: Annotated[int, Field(default=1, ge=1)]
+    size: Annotated[int, Field(default=30, ge=1)]
+    top_k: Annotated[int, Field(default=1024, ge=1)]
+    similarity_threshold: Annotated[float, Field(default=0.0, ge=0.0, le=1.0)]
+    vector_similarity_weight: Annotated[float, Field(default=0.3, ge=0.0, le=1.0)]
+    use_kg: Annotated[bool, Field(default=False)]
+    cross_languages: Annotated[list[str], Field(default=[])]
+    keyword: Annotated[bool, Field(default=False)]
+    search_id: Annotated[str | None, Field(default=None)]
+    rerank_id: Annotated[str | None, Field(default=None)]
+    tenant_rerank_id: Annotated[int | None, Field(default=None)]
+    meta_data_filter: Annotated[dict | None, Field(default=None)]
+
+
+class SearchDatasetsReq(BaseModel):
+    """Request model for searching multiple datasets."""
+
+    model_config = ConfigDict(extra="ignore")
+
+    dataset_ids: Annotated[list[str], Field(..., min_length=1)]
+    question: Annotated[str, StringConstraints(strip_whitespace=True, min_length=1), Field(...)]
+    doc_ids: Annotated[list[str], Field(default=[])]
+    page: Annotated[int, Field(default=1, ge=1)]
+    size: Annotated[int, Field(default=30, ge=1)]
+    top_k: Annotated[int, Field(default=1024, ge=1)]
+    similarity_threshold: Annotated[float, Field(default=0.0, ge=0.0, le=1.0)]
+    vector_similarity_weight: Annotated[float, Field(default=0.3, ge=0.0, le=1.0)]
+    use_kg: Annotated[bool, Field(default=False)]
+    cross_languages: Annotated[list[str], Field(default=[])]
+    keyword: Annotated[bool, Field(default=False)]
+    search_id: Annotated[str | None, Field(default=None)]
+    rerank_id: Annotated[str | None, Field(default=None)]
+    tenant_rerank_id: Annotated[int | None, Field(default=None)]
+    meta_data_filter: Annotated[dict | None, Field(default=None)]
 
 
 class BaseListReq(BaseModel):
+    """Shared pagination and sorting fields for list APIs."""
+
     model_config = ConfigDict(extra="forbid")
 
     id: Annotated[str | None, Field(default=None)]
@@ -739,7 +963,153 @@ class BaseListReq(BaseModel):
     @field_validator("id", mode="before")
     @classmethod
     def validate_id(cls, v: Any) -> str:
+        """Validate and normalize an optional list filter id."""
         return validate_uuid1_hex(v)
 
+    @field_validator("page_size")
+    @classmethod
+    def validate_page_size(cls, v: int) -> int:
+        return validate_rest_api_page_size(v)
 
-class ListDatasetReq(BaseListReq): ...
+
+class ListDatasetReq(BaseListReq):
+    """Request model for listing datasets."""
+
+    include_parsing_status: Annotated[bool, Field(default=False)]
+    ext: Annotated[dict, Field(default={})]
+
+
+# ---- File Management Request Models ----
+
+
+class CreateFolderReq(Base):
+    """Request model for creating a folder."""
+
+    name: Annotated[str, StringConstraints(strip_whitespace=True, min_length=1, max_length=255), Field(...)]
+    parent_id: Annotated[str | None, Field(default=None)]
+    type: Annotated[str | None, Field(default=None)]
+
+
+class DeleteFileReq(Base):
+    """Request model for deleting files."""
+
+    ids: Annotated[list[str], Field(min_length=1)]
+
+
+class MoveFileReq(Base):
+    """Request model for moving or renaming files."""
+
+    src_file_ids: Annotated[list[str], Field(min_length=1)]
+    dest_file_id: Annotated[str | None, Field(default=None)]
+    new_name: Annotated[str | None, StringConstraints(strip_whitespace=True, min_length=1, max_length=255), Field(default=None)]
+
+    @model_validator(mode="after")
+    def check_operation(self):
+        """Require either a destination folder or a new file name."""
+        if not self.dest_file_id and not self.new_name:
+            raise ValueError("At least one of dest_file_id or new_name must be provided")
+        if self.new_name and len(self.src_file_ids) > 1:
+            raise ValueError("new_name can only be used with a single file")
+        return self
+
+
+class ListFileReq(BaseModel):
+    """Request model for listing files."""
+
+    model_config = ConfigDict(extra="forbid")
+
+    parent_id: Annotated[str | None, Field(default=None)]
+    keywords: Annotated[str, Field(default="")]
+    page: Annotated[int, Field(default=1, ge=1)]
+    page_size: Annotated[int, Field(default=15, ge=1)]
+    orderby: Annotated[str, Field(default="create_time")]
+    desc: Annotated[bool, Field(default=True)]
+
+    @field_validator("page_size")
+    @classmethod
+    def validate_page_size(cls, v: int) -> int:
+        return validate_rest_api_page_size(v)
+
+
+def validate_immutable_fields(update_doc_req: UpdateDocumentReq, doc):
+    """
+    Validate that immutable fields have not been changed.
+
+    Checks that fields like chunk_count, token_count, and progress
+    cannot be modified directly by the user.
+
+    Args:
+        update_doc_req: The validated update document request.
+        doc: The document model from the database.
+
+    Returns:
+        A tuple of (error_message, error_code) if validation fails,
+        or (None, None) if validation passes.
+    """
+    if update_doc_req.chunk_count is not None and update_doc_req.chunk_count != int(getattr(doc, "chunk_num", -1)):
+        return "Can't change `chunk_count`.", RetCode.DATA_ERROR
+
+    if update_doc_req.token_count is not None and update_doc_req.token_count != int(getattr(doc, "token_num", -1)):
+        return "Can't change `token_count`.", RetCode.DATA_ERROR
+
+    if update_doc_req.progress is not None:
+        progress_from_db = float(getattr(doc, "progress", -1.0))
+        # should not use "==" to compare two float values
+        if not math.isclose(update_doc_req.progress, progress_from_db):
+            return "Can't change `progress`.", RetCode.DATA_ERROR
+
+    return None, None
+
+
+def validate_document_name(req_doc_name: str, doc, docs_from_name):
+    """
+    Validate document name update.
+
+    Checks that the new document name is valid:
+    - Must be a string
+    - Must not exceed the file name length limit
+    - File extension cannot be changed
+    - Must not duplicate an existing document name in the same dataset.
+
+    Args:
+        req_doc_name: The new document name to validate.
+        doc: The document model from the database.
+        docs_from_name: Query result for documents with the new name.
+
+    Returns:
+        A tuple of (error_message, error_code) if validation fails,
+        or (None, None) if validation passes.
+    """
+    if not isinstance(req_doc_name, str):
+        return f"AttributeError('{type(req_doc_name).__name__}' object has no attribute 'encode')", RetCode.EXCEPTION_ERROR
+    if len(req_doc_name.encode("utf-8")) > FILE_NAME_LEN_LIMIT:
+        return f"File name must be {FILE_NAME_LEN_LIMIT} bytes or less.", RetCode.ARGUMENT_ERROR
+    if pathlib.Path(req_doc_name.lower()).suffix != pathlib.Path(doc.name.lower()).suffix:
+        return "The extension of file can't be changed", RetCode.ARGUMENT_ERROR
+
+    for d in docs_from_name:
+        if d.name == req_doc_name:
+            return "Duplicated document name in the same dataset.", RetCode.DATA_ERROR
+    return None, None
+
+
+def validate_chunk_method(doc, chunk_method=None):
+    """
+    Validate chunk method update.
+
+    Checks if the chunk method is valid for the given document,
+    particularly for visual documents or specific file types.
+
+    Args:
+        doc: The document model from the database.
+        chunk_method: The chunk method to validate.
+
+    Returns:
+        A tuple of (error_message, error_code) if validation fails,
+        or (None, None) if validation passes.
+    """
+    if chunk_method is not None and len(chunk_method) == 0:  # will not be detected in UpdateDocumentReq
+        return "`chunk_method` (empty string) is not valid", RetCode.DATA_ERROR
+    if doc.type == FileType.VISUAL or re.search(r"\.(ppt|pptx|pages)$", doc.name):
+        return "Not supported yet!", RetCode.DATA_ERROR
+    return None, None

api/utils/web_utils.py

@@ -15,11 +15,8 @@
 #
 
 import base64
-import ipaddress
 import json
 import re
-import socket
-from urllib.parse import urlparse
 import aiosmtplib
 from email.mime.text import MIMEText
 from email.header import Header
@@ -37,10 +34,10 @@ from webdriver_manager.chrome import ChromeDriverManager
 
 
 OTP_LENGTH = 4
-OTP_TTL_SECONDS = 5 * 60 # valid for 5 minutes
-ATTEMPT_LIMIT = 5 # maximum attempts
-ATTEMPT_LOCK_SECONDS = 30 * 60 # lock for 30 minutes
-RESEND_COOLDOWN_SECONDS = 60 # cooldown for 1 minute
+OTP_TTL_SECONDS = 5 * 60  # valid for 5 minutes
+ATTEMPT_LIMIT = 5  # maximum attempts
+ATTEMPT_LOCK_SECONDS = 30 * 60  # lock for 30 minutes
+RESEND_COOLDOWN_SECONDS = 60  # cooldown for 1 minute
 
 
 CONTENT_TYPE_MAP = {
@@ -176,6 +173,9 @@ def __get_pdf_from_html(path: str, timeout: int, install_driver: bool, print_opt
     try:
         WebDriverWait(driver, timeout).until(staleness_of(driver.find_element(by=By.TAG_NAME, value="html")))
     except TimeoutException:
+        pass
+
+    try:
         calculated_print_options = {
             "landscape": False,
             "displayHeaderFooter": False,
@@ -184,33 +184,21 @@ def __get_pdf_from_html(path: str, timeout: int, install_driver: bool, print_opt
         }
         calculated_print_options.update(print_options)
         result = __send_devtools(driver, "Page.printToPDF", calculated_print_options)
-        driver.quit()
         return base64.b64decode(result["data"])
-
-
-def is_private_ip(ip: str) -> bool:
-    try:
-        ip_obj = ipaddress.ip_address(ip)
-        return ip_obj.is_private
-    except ValueError:
-        return False
+    finally:
+        driver.quit()
 
 
 def is_valid_url(url: str) -> bool:
     if not re.match(r"(https?)://[-A-Za-z0-9+&@#/%?=~_|!:,.;]+[-A-Za-z0-9+&@#/%=~_|]", url):
         return False
-    parsed_url = urlparse(url)
-    hostname = parsed_url.hostname
+    from common.ssrf_guard import assert_url_is_safe
 
-    if not hostname:
-        return False
     try:
-        ip = socket.gethostbyname(hostname)
-        if is_private_ip(ip):
-            return False
-    except socket.gaierror:
+        assert_url_is_safe(url)
+        return True
+    except ValueError:
         return False
-    return True
 
 
 def safe_json_parse(data: str | dict) -> dict: