mirror of
https://github.com/infiniflow/ragflow.git
synced 2026-07-14 17:08:31 +08:00
Add file type validation (#13802)
### What problem does this PR solve? This PR fixes WebDAV sync behavior for unsupported file types ([#13795](https://github.com/infiniflow/ragflow/issues/13795)). Previously, the WebDAV connector selected files primarily by modified time (and size threshold) and could still pass unsupported extensions into the download/document-generation path. This caused unnecessary processing and inconsistent behavior compared with connectors that validate file type earlier. This change adds extension validation in two places: 1. **Early filter during recursive listing** to skip unsupported files before they enter the download flow. 2. **Defensive filter before download/document creation** to prevent unsupported files from being processed if any listing edge case slips through. It also wires `allow_images` into the WebDAV sync path so image extension handling follows connector policy. Scope is intentionally limited to WebDAV for a focused bug-fix PR. ### Type of change - [x] Bug Fix (non-breaking change which fixes an issue) ### How was this tested? - Manual verification with mixed file types under the configured WebDAV path: - supported: `.pdf`, `.txt`, `.md` - unsupported: `.exe`, `.bin`, `.dat` - Triggered full sync and polling sync. - Confirmed unsupported files are skipped before download. - Confirmed supported files are still indexed normally. - Confirmed image handling follows `allow_images` setting. Fixes: #13795
This commit is contained in:
@@ -8,6 +8,7 @@ from webdav4.client import Client as WebDAVClient
|
||||
|
||||
from common.data_source.utils import (
|
||||
get_file_ext,
|
||||
is_accepted_file_ext,
|
||||
)
|
||||
from common.data_source.config import DocumentSource, INDEX_BATCH_SIZE, BLOB_STORAGE_SIZE_THRESHOLD
|
||||
from common.data_source.exceptions import (
|
||||
@@ -16,7 +17,7 @@ from common.data_source.exceptions import (
|
||||
CredentialExpiredError,
|
||||
InsufficientPermissionsError
|
||||
)
|
||||
from common.data_source.interfaces import LoadConnector, PollConnector
|
||||
from common.data_source.interfaces import LoadConnector, OnyxExtensionType, PollConnector
|
||||
from common.data_source.models import Document, SecondsSinceUnixEpoch, GenerateDocumentsOutput
|
||||
|
||||
|
||||
@@ -49,6 +50,16 @@ class WebDAVConnector(LoadConnector, PollConnector):
|
||||
self._allow_images: bool | None = None
|
||||
self.size_threshold: int | None = BLOB_STORAGE_SIZE_THRESHOLD
|
||||
|
||||
def _build_extension_type(self) -> OnyxExtensionType:
|
||||
extension_type = OnyxExtensionType.Plain | OnyxExtensionType.Document
|
||||
if bool(self._allow_images):
|
||||
extension_type |= OnyxExtensionType.Multimedia
|
||||
return extension_type
|
||||
|
||||
def _is_supported_file(self, file_name: str) -> bool:
|
||||
file_ext = get_file_ext(file_name)
|
||||
return is_accepted_file_ext(file_ext, self._build_extension_type())
|
||||
|
||||
def set_allow_images(self, allow_images: bool) -> None:
|
||||
"""Set whether to process images"""
|
||||
logging.info(f"Setting allow_images to {allow_images}.")
|
||||
@@ -129,6 +140,11 @@ class WebDAVConnector(LoadConnector, PollConnector):
|
||||
continue
|
||||
else:
|
||||
try:
|
||||
file_name = os.path.basename(item_path)
|
||||
if not self._is_supported_file(file_name):
|
||||
logging.debug(f"Skipping file {item_path} due to unsupported extension.")
|
||||
continue
|
||||
|
||||
modified_time = item.get('modified')
|
||||
if modified_time:
|
||||
if isinstance(modified_time, datetime):
|
||||
@@ -194,6 +210,10 @@ class WebDAVConnector(LoadConnector, PollConnector):
|
||||
batch: list[Document] = []
|
||||
for file_path, file_info in files:
|
||||
file_name = os.path.basename(file_path)
|
||||
|
||||
if not self._is_supported_file(file_name):
|
||||
logging.debug(f"Skipping file {file_path} due to unsupported extension.")
|
||||
continue
|
||||
|
||||
size_bytes = file_info.get('size', 0)
|
||||
if (
|
||||
|
||||
@@ -712,6 +712,7 @@ class WebDAV(SyncBase):
|
||||
base_url=self.conf["base_url"],
|
||||
remote_path=self.conf.get("remote_path", "/")
|
||||
)
|
||||
self.connector.set_allow_images(self.conf.get("allow_images", False))
|
||||
self.connector.load_credentials(self.conf["credentials"])
|
||||
|
||||
logging.info(f"Task info: reindex={task['reindex']}, poll_range_start={task['poll_range_start']}")
|
||||
|
||||
Reference in New Issue
Block a user