feat: implement Slack data source connector (#15188)

### What problem does this PR solve?

Closes #15187.

RAGFlow shipped a Slack connector
(`common/data_source/slack_connector.py`) but it was never usable:
`Slack._generate()` in the sync worker was a `pass` stub, the
connector's document-generating code was incompatible with the current
data model,
and Slack was commented out of the data-source settings UI. As a result,
teams had no way to index Slack channels/threads into a knowledge base.

This PR completes the connector end to end.

**Backend**

- `common/data_source/slack_connector.py`
- Rewrote `thread_to_doc` to produce a blob-based `Document`
(`extension`/`blob`/`size_bytes`). The previous implementation built the
doc with a `sections=[...]` argument and omitted the now-required
`blob`/`extension`/ `size_bytes` fields, so it raised a validation error
against the current `Document` model. Thread messages are now cleaned
and flattened into a single UTF-8 text blob.
- Added `load_from_state()` / `poll_source(start, end)` generators. The
connector's checkpoint interface is a no-op stub, so both full and
incremental syncs run through a single channel-iterating generator built
on the existing module helpers (`get_channels`, `filter_channels`,
`get_channel_messages`, `_process_message`), with per-channel thread
de-duplication.
- `rag/svr/sync_data_source.py`
- Implemented `Slack._generate()`. Credentials are loaded via
`StaticCredentialsProvider` (the connector requires `slack_bot_token`
and does not support `load_credentials`). Supports full reindex and
incremental polling from `poll_range_start`, plus the optional channel
filter. Modeled on the Confluence/Dropbox wrappers.
- `SlackConnector` was already exported from
`common/data_source/__init__.py`.

**Frontend (`web/`)**

- Enabled the `SLACK` data-source enum and added its form fields (Slack
bot token + optional channel filter), default values, display metadata,
and a Slack icon.
- Added `slackDescription` / `slackBotTokenTip` / `slackChannelsTip`
strings to `en.ts` and `zh.ts`.

**Tests**

- `test/unit_test/data_source/test_slack_connector_unit.py`: unit tests
covering credential loading (`load_credentials` raises,
`set_credentials_provider` initializes clients, missing credentials
raises) and document generation (standalone message + flattened thread,
blob/extension/size_bytes/metadata, and the incremental poll time
window). All 5 pass; `ruff check` is clean.

Required Slack scopes: `channels:read`, `channels:history`,
`users:read`.

### Type of change

- [x] New Feature (non-breaking change which adds functionality)
This commit is contained in:
web-dev0521
2026-05-28 01:46:07 -06:00
committed by GitHub
parent 7e83643536
commit 5de021ebb4
8 changed files with 428 additions and 19 deletions

View File

@@ -64,16 +64,19 @@ def test_documents_upload_and_list(rest_client, create_dataset, tmp_path):
assert any(doc["name"] == fp.name for doc in list_payload["data"]["docs"]), list_payload
def _upload_files(rest_client, dataset_id, file_paths):
def _upload_files(rest_client, dataset_id, file_paths, timeout=None):
with ExitStack() as stack:
files = [("file", (fp.name, stack.enter_context(fp.open("rb")))) for fp in file_paths]
return rest_client.post(f"/datasets/{dataset_id}/documents", files=files)
kwargs = {"files": files}
if timeout is not None:
kwargs["timeout"] = timeout
return rest_client.post(f"/datasets/{dataset_id}/documents", **kwargs)
def _seed_documents(rest_client, create_dataset, tmp_path, count=5):
def _seed_documents(rest_client, create_dataset, tmp_path, count=5, timeout=None):
dataset_id = create_dataset("dataset_list_contract")
file_paths = [create_txt_file(tmp_path / f"ragflow_test_upload_{i}.txt") for i in range(count)]
res = _upload_files(rest_client, dataset_id, file_paths)
res = _upload_files(rest_client, dataset_id, file_paths, timeout=timeout)
assert res.status_code == 200
payload = res.json()
assert payload["code"] == 0, payload
@@ -1166,7 +1169,9 @@ def test_documents_delete_invalid_dataset_partial_duplicate_repeat_and_cross_dat
@pytest.mark.p2
def test_documents_delete_concurrent_and_bulk_contract(rest_client, create_dataset, tmp_path):
dataset_id, uploaded_docs = _seed_documents(rest_client, create_dataset, tmp_path, count=60)
dataset_id, uploaded_docs = _seed_documents(
rest_client, create_dataset, tmp_path, count=60, timeout=120
)
document_ids = [doc["id"] for doc in uploaded_docs]
with ThreadPoolExecutor(max_workers=8) as executor:
@@ -1192,9 +1197,15 @@ def test_documents_delete_concurrent_and_bulk_contract(rest_client, create_datas
assert list_after_payload["code"] == 0, list_after_payload
assert list_after_payload["data"]["total"] == 0, list_after_payload
bulk_dataset_id, bulk_docs = _seed_documents(rest_client, create_dataset, tmp_path, count=120)
bulk_dataset_id, bulk_docs = _seed_documents(
rest_client, create_dataset, tmp_path, count=120, timeout=120
)
bulk_ids = [doc["id"] for doc in bulk_docs]
bulk_delete_res = rest_client.delete(f"/datasets/{bulk_dataset_id}/documents", json={"ids": bulk_ids})
bulk_delete_res = rest_client.delete(
f"/datasets/{bulk_dataset_id}/documents",
json={"ids": bulk_ids},
timeout=120,
)
assert bulk_delete_res.status_code == 200
bulk_delete_payload = bulk_delete_res.json()
assert bulk_delete_payload["code"] == 0, bulk_delete_payload

View File

@@ -0,0 +1,206 @@
#
# Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
import importlib.util
import sys
from pathlib import Path
from types import ModuleType
import pytest
def _load_slack_connector_module():
"""Load slack_connector.py in isolation.
Importing ``common.data_source`` directly would execute its ``__init__``
and pull in every connector's (heavy) dependencies. We stub the package and
exec only the Slack module, mirroring the Dropbox connector unit test.
"""
repo_root = Path(__file__).resolve().parents[3]
package_name = "common.data_source"
saved_modules = {
name: module
for name, module in sys.modules.items()
if name == package_name or name.startswith(f"{package_name}.")
}
package_stub = ModuleType(package_name)
package_stub.__path__ = [str(repo_root / "common" / "data_source")]
sys.modules[package_name] = package_stub
try:
spec = importlib.util.spec_from_file_location(
"_slack_connector_under_test",
repo_root / "common" / "data_source" / "slack_connector.py",
)
module = importlib.util.module_from_spec(spec)
spec.loader.exec_module(module)
return module
finally:
for name in list(sys.modules):
if name == package_name or name.startswith(f"{package_name}."):
if name in saved_modules:
sys.modules[name] = saved_modules[name]
else:
sys.modules.pop(name, None)
slack_connector = _load_slack_connector_module()
SlackConnector = slack_connector.SlackConnector
SlackTextCleaner = slack_connector.SlackTextCleaner
class _FakeResponse(dict):
"""Mimics slack_sdk SlackResponse: dict-like, plus .validate() and .data."""
def validate(self):
return self
@property
def data(self):
return self
class _FakeSlackClient:
def __init__(self):
self.token = "xoxb-test"
self.joined = []
self._users = {
"U1": {
"real_name": "Alice",
"profile": {"display_name": "alice", "first_name": "Alice", "last_name": "A"},
},
"U2": {"real_name": "Bob", "profile": {"display_name": "bob"}},
"U3": {"real_name": "Carol", "profile": {"display_name": "carol"}},
}
def conversations_list(self, cursor=None, limit=None, **kwargs):
return _FakeResponse(
{
"channels": [
{"id": "C1", "name": "general", "is_member": True, "is_private": False},
],
"response_metadata": {"next_cursor": ""},
}
)
def conversations_history(self, cursor=None, limit=None, channel=None, oldest=None, latest=None, **kwargs):
return _FakeResponse(
{
"messages": [
{"ts": "1.0", "user": "U1", "text": "Hello world"},
{"ts": "2.0", "user": "U2", "text": "Question?", "thread_ts": "2.0"},
],
"response_metadata": {"next_cursor": ""},
}
)
def conversations_replies(self, cursor=None, limit=None, channel=None, ts=None, **kwargs):
return _FakeResponse(
{
"messages": [
{"ts": "2.0", "user": "U2", "text": "Question?", "thread_ts": "2.0"},
{"ts": "3.0", "user": "U3", "text": "Answer!", "thread_ts": "2.0"},
],
"response_metadata": {"next_cursor": ""},
}
)
def users_info(self, user=None, **kwargs):
return _FakeResponse({"ok": True, "user": self._users.get(user, {})})
def _connector_with_fake_client(client, batch_size=10):
connector = SlackConnector(batch_size=batch_size)
connector.client = client
connector.text_cleaner = SlackTextCleaner(client=client)
return connector
# --- credential loading -----------------------------------------------------
@pytest.mark.p2
def test_load_credentials_is_not_supported():
connector = SlackConnector()
with pytest.raises(NotImplementedError):
connector.load_credentials({"slack_bot_token": "xoxb-abc"})
@pytest.mark.p2
def test_set_credentials_provider_initializes_clients():
connector = SlackConnector()
class _Provider:
def get_credentials(self):
return {"slack_bot_token": "xoxb-abc"}
connector.set_credentials_provider(_Provider())
assert connector.client is not None
assert connector.fast_client is not None
assert connector.text_cleaner is not None
@pytest.mark.p2
def test_fetch_without_credentials_raises():
connector = SlackConnector()
with pytest.raises(slack_connector.ConnectorMissingCredentialError):
list(connector.load_from_state())
# --- document generation ----------------------------------------------------
@pytest.mark.p1
def test_load_from_state_generates_thread_documents():
connector = _connector_with_fake_client(_FakeSlackClient())
batches = list(connector.load_from_state())
docs = [doc for batch in batches for doc in batch]
# Standalone message + one thread (parent + reply collapsed into one doc).
assert [doc.id for doc in docs] == ["C1__1.0", "C1__2.0"]
standalone, thread_doc = docs
assert standalone.source == "slack"
assert standalone.extension == ".txt"
assert standalone.blob == b"Hello world"
assert standalone.size_bytes == len(b"Hello world")
assert standalone.metadata == {"Channel": "general"}
# get_semantic_name() prefers real_name ("Alice") over the display_name.
assert standalone.semantic_identifier == "Alice in #general: Hello world"
# Thread messages are flattened into a single blob, joined by blank lines.
assert thread_doc.blob == "Question?\n\nAnswer!".encode("utf-8")
assert thread_doc.size_bytes == len(thread_doc.blob)
assert thread_doc.semantic_identifier == "Bob in #general: Question?"
@pytest.mark.p1
def test_poll_source_passes_time_window():
client = _FakeSlackClient()
captured = {}
def _history(cursor=None, limit=None, channel=None, oldest=None, latest=None, **kwargs):
captured["oldest"] = oldest
captured["latest"] = latest
return _FakeResponse({"messages": [], "response_metadata": {"next_cursor": ""}})
client.conversations_history = _history
connector = _connector_with_fake_client(client)
list(connector.poll_source(100.0, 200.0))
assert captured == {"oldest": "100.0", "latest": "200.0"}