mirror of
https://github.com/infiniflow/ragflow.git
synced 2026-08-24 17:10:12 +08:00
feat(seafile): add library and directory sync scope support (#13153)
### What problem does this PR solve?
The SeaFile connector currently synchronises the entire account — every
library
visible to the authenticated user. This is impractical for users who
only need
a subset of their data indexed, especially on large SeaFile instances
with many
shared libraries.
This PR introduces granular sync scope support, allowing users to choose
between
syncing their entire account, a single library, or a specific directory
within a
library. It also adds support for SeaFile library-scoped API tokens
(`/api/v2.1/via-repo-token/` endpoints), enabling tighter access control
without
exposing account-level credentials.
### Type of change
- [ ] Bug Fix (non-breaking change which fixes an issue)
- [x] New Feature (non-breaking change which adds functionality)
- [ ] Documentation Update
- [ ] Refactoring
- [ ] Performance Improvement
- [ ] Other (please describe):
### Test
```
from seafile_connector import SeaFileConnector
import logging
import os
logging.basicConfig(level=logging.DEBUG)
URL = os.environ.get("SEAFILE_URL", "https://seafile.example.com")
TOKEN = os.environ.get("SEAFILE_TOKEN", "")
REPO_ID = os.environ.get("SEAFILE_REPO_ID", "")
SYNC_PATH = os.environ.get("SEAFILE_SYNC_PATH", "/Documents")
REPO_TOKEN = os.environ.get("SEAFILE_REPO_TOKEN", "")
def _test_scope(scope, repo_id=None, sync_path=None):
print(f"\n{'='*50}")
print(f"Testing scope: {scope}")
print(f"{'='*50}")
creds = {"seafile_token": TOKEN} if TOKEN else {}
if REPO_TOKEN and scope in ("library", "directory"):
creds["repo_token"] = REPO_TOKEN
connector = SeaFileConnector(
seafile_url=URL,
batch_size=5,
sync_scope=scope,
include_shared = False,
repo_id=repo_id,
sync_path=sync_path,
)
connector.load_credentials(creds)
connector.validate_connector_settings()
count = 0
for batch in connector.load_from_state():
for doc in batch:
count += 1
print(f" [{count}] {doc.semantic_identifier} "
f"({doc.size_bytes} bytes, {doc.extension})")
print(f"\n-> {scope} scope: {count} document(s) found.\n")
# 1. Account scope
if TOKEN:
_test_scope("account")
else:
print("\nSkipping account scope (set SEAFILE_TOKEN)")
# 2. Library scope
if REPO_ID and (TOKEN or REPO_TOKEN):
_test_scope("library", repo_id=REPO_ID)
else:
print("\nSkipping library scope (set SEAFILE_REPO_ID + token)")
# 3. Directory scope
if REPO_ID and SYNC_PATH and (TOKEN or REPO_TOKEN):
_test_scope("directory", repo_id=REPO_ID, sync_path=SYNC_PATH)
else:
print("\nSkipping directory scope (set SEAFILE_REPO_ID + SEAFILE_SYNC_PATH + token)")
```
This commit is contained in:
committed by
GitHub
parent
aec2ef4232
commit
d1afcc9e71
@@ -58,7 +58,7 @@ from common.data_source import (
|
||||
)
|
||||
from common.constants import FileSource, TaskStatus
|
||||
from common.data_source.config import INDEX_BATCH_SIZE
|
||||
from common.data_source.models import ConnectorFailure
|
||||
from common.data_source.models import ConnectorFailure, SeafileSyncScope
|
||||
from common.data_source.webdav_connector import WebDAVConnector
|
||||
from common.data_source.confluence_connector import ConfluenceConnector
|
||||
from common.data_source.gmail_connector import GmailConnector
|
||||
@@ -1180,21 +1180,23 @@ class Bitbucket(SyncBase):
|
||||
|
||||
return wrapper()
|
||||
|
||||
|
||||
class SeaFile(SyncBase):
|
||||
SOURCE_NAME: str = FileSource.SEAFILE
|
||||
|
||||
async def _generate(self, task: dict):
|
||||
conf = self.conf
|
||||
self.connector = SeaFileConnector(
|
||||
seafile_url=self.conf["seafile_url"],
|
||||
batch_size=self.conf.get("batch_size", INDEX_BATCH_SIZE),
|
||||
include_shared=self.conf.get("include_shared", True)
|
||||
seafile_url=conf["seafile_url"],
|
||||
batch_size=conf.get("batch_size", INDEX_BATCH_SIZE),
|
||||
include_shared=conf.get("include_shared", True),
|
||||
sync_scope=conf.get("sync_scope", SeafileSyncScope.ACCOUNT),
|
||||
repo_id=conf.get("repo_id") or None,
|
||||
sync_path=conf.get("sync_path") or None,
|
||||
)
|
||||
self.connector.load_credentials(conf["credentials"])
|
||||
|
||||
self.connector.load_credentials(self.conf["credentials"])
|
||||
|
||||
# Determine the time range for synchronization based on reindex or poll_range_start
|
||||
poll_start = task.get("poll_range_start")
|
||||
|
||||
if task["reindex"] == "1" or poll_start is None:
|
||||
document_generator = self.connector.load_from_state()
|
||||
begin_info = "totally"
|
||||
@@ -1205,12 +1207,16 @@ class SeaFile(SyncBase):
|
||||
)
|
||||
begin_info = f"from {poll_start}"
|
||||
|
||||
scope = conf.get("sync_scope", "account")
|
||||
extra = ""
|
||||
if scope in ("library", "directory"):
|
||||
extra = f" repo_id={conf.get('repo_id')}"
|
||||
if scope == "directory":
|
||||
extra += f" path={conf.get('sync_path')}"
|
||||
|
||||
logging.info(
|
||||
"Connect to SeaFile: {} (include_shared: {}) {}".format(
|
||||
self.conf["seafile_url"],
|
||||
self.conf.get("include_shared", True),
|
||||
begin_info
|
||||
)
|
||||
"Connect to SeaFile: %s (scope=%s%s) %s",
|
||||
conf["seafile_url"], scope, extra, begin_info,
|
||||
)
|
||||
return document_generator
|
||||
|
||||
|
||||
Reference in New Issue
Block a user