mirror of
https://github.com/infiniflow/ragflow.git
synced 2026-08-25 09:32:38 +08:00
feat: support rss datasource (#13721)
### What problem does this PR solve? Supporting public RSS/Atom feed URLs as data sources for RagFlow. link https://github.com/infiniflow/ragflow/issues/12313 ### Type of change - [x] New Feature (non-breaking change which adds functionality)
This commit is contained in:
@@ -43,6 +43,7 @@ from common import settings
|
||||
from common.config_utils import show_configs
|
||||
from common.data_source import (
|
||||
BlobStorageConnector,
|
||||
RSSConnector,
|
||||
NotionConnector,
|
||||
DiscordConnector,
|
||||
GoogleDriveConnector,
|
||||
@@ -243,6 +244,26 @@ class GOOGLE_CLOUD_STORAGE(_BlobLikeBase):
|
||||
DEFAULT_BUCKET_TYPE: str = "google_cloud_storage"
|
||||
|
||||
|
||||
class RSS(SyncBase):
|
||||
SOURCE_NAME: str = FileSource.RSS
|
||||
|
||||
async def _generate(self, task: dict):
|
||||
self.connector = RSSConnector(
|
||||
feed_url=self.conf["feed_url"],
|
||||
batch_size=self.conf.get("batch_size", INDEX_BATCH_SIZE),
|
||||
)
|
||||
self.connector.load_credentials(self.conf.get("credentials", {}))
|
||||
self.connector.validate_connector_settings()
|
||||
|
||||
if task["reindex"] == "1" or not task["poll_range_start"]:
|
||||
return self.connector.load_from_state()
|
||||
|
||||
return self.connector.poll_source(
|
||||
task["poll_range_start"].timestamp(),
|
||||
datetime.now(timezone.utc).timestamp(),
|
||||
)
|
||||
|
||||
|
||||
class Confluence(SyncBase):
|
||||
SOURCE_NAME: str = FileSource.CONFLUENCE
|
||||
|
||||
@@ -1347,6 +1368,7 @@ class PostgreSQL(SyncBase):
|
||||
|
||||
|
||||
func_factory = {
|
||||
FileSource.RSS: RSS,
|
||||
FileSource.S3: S3,
|
||||
FileSource.R2: R2,
|
||||
FileSource.OCI_STORAGE: OCI_STORAGE,
|
||||
|
||||
Reference in New Issue
Block a user