Add file type validation (#13802)

### What problem does this PR solve?

This PR fixes WebDAV sync behavior for unsupported file types
([#13795](https://github.com/infiniflow/ragflow/issues/13795)).

Previously, the WebDAV connector selected files primarily by modified
time (and size threshold) and could still pass unsupported extensions
into the download/document-generation path. This caused unnecessary
processing and inconsistent behavior compared with connectors that
validate file type earlier.

This change adds extension validation in two places:

1. **Early filter during recursive listing** to skip unsupported files
before they enter the download flow.
2. **Defensive filter before download/document creation** to prevent
unsupported files from being processed if any listing edge case slips
through.

It also wires `allow_images` into the WebDAV sync path so image
extension handling follows connector policy.

Scope is intentionally limited to WebDAV for a focused bug-fix PR.

### Type of change

- [x] Bug Fix (non-breaking change which fixes an issue)

### How was this tested?

- Manual verification with mixed file types under the configured WebDAV
path:
  - supported: `.pdf`, `.txt`, `.md`
  - unsupported: `.exe`, `.bin`, `.dat`
- Triggered full sync and polling sync.
- Confirmed unsupported files are skipped before download.
- Confirmed supported files are still indexed normally.
- Confirmed image handling follows `allow_images` setting.

Fixes: #13795
This commit is contained in:
NeedmeFordev
2026-04-01 23:12:27 -07:00
committed by GitHub
parent dd529137eb
commit 6b7989b4b4
2 changed files with 22 additions and 1 deletions

View File

@@ -8,6 +8,7 @@ from webdav4.client import Client as WebDAVClient
from common.data_source.utils import (
get_file_ext,
is_accepted_file_ext,
)
from common.data_source.config import DocumentSource, INDEX_BATCH_SIZE, BLOB_STORAGE_SIZE_THRESHOLD
from common.data_source.exceptions import (
@@ -16,7 +17,7 @@ from common.data_source.exceptions import (
CredentialExpiredError,
InsufficientPermissionsError
)
from common.data_source.interfaces import LoadConnector, PollConnector
from common.data_source.interfaces import LoadConnector, OnyxExtensionType, PollConnector
from common.data_source.models import Document, SecondsSinceUnixEpoch, GenerateDocumentsOutput
@@ -49,6 +50,16 @@ class WebDAVConnector(LoadConnector, PollConnector):
self._allow_images: bool | None = None
self.size_threshold: int | None = BLOB_STORAGE_SIZE_THRESHOLD
def _build_extension_type(self) -> OnyxExtensionType:
extension_type = OnyxExtensionType.Plain | OnyxExtensionType.Document
if bool(self._allow_images):
extension_type |= OnyxExtensionType.Multimedia
return extension_type
def _is_supported_file(self, file_name: str) -> bool:
file_ext = get_file_ext(file_name)
return is_accepted_file_ext(file_ext, self._build_extension_type())
def set_allow_images(self, allow_images: bool) -> None:
"""Set whether to process images"""
logging.info(f"Setting allow_images to {allow_images}.")
@@ -129,6 +140,11 @@ class WebDAVConnector(LoadConnector, PollConnector):
continue
else:
try:
file_name = os.path.basename(item_path)
if not self._is_supported_file(file_name):
logging.debug(f"Skipping file {item_path} due to unsupported extension.")
continue
modified_time = item.get('modified')
if modified_time:
if isinstance(modified_time, datetime):
@@ -194,6 +210,10 @@ class WebDAVConnector(LoadConnector, PollConnector):
batch: list[Document] = []
for file_path, file_info in files:
file_name = os.path.basename(file_path)
if not self._is_supported_file(file_name):
logging.debug(f"Skipping file {file_path} due to unsupported extension.")
continue
size_bytes = file_info.get('size', 0)
if (

View File

@@ -712,6 +712,7 @@ class WebDAV(SyncBase):
base_url=self.conf["base_url"],
remote_path=self.conf.get("remote_path", "/")
)
self.connector.set_allow_images(self.conf.get("allow_images", False))
self.connector.load_credentials(self.conf["credentials"])
logging.info(f"Task info: reindex={task['reindex']}, poll_range_start={task['poll_range_start']}")