zlei6/ragflow
1
0
Fork 0
mirror of https://github.com/infiniflow/ragflow.git synced 2026-07-20 06:31:02 +08:00
Code Issues Packages Projects Releases Wiki Activity

Refactor: reformat all code for lefthook using ruff and gofmt (#16585)

Browse Source
This commit is contained in:
Wang Qi
2026-07-03 12:53:39 +08:00
committed by GitHub
parent 19fcb4a981
commit 6a4b9be426
588 changed files with 11123 additions and 15412 deletions
Download Patch File Download Diff File Expand all files Collapse all files

48
test/unit_test/data_source/test_bigquery_connector.py
View File

@@ -174,17 +174,12 @@ def test_missing_table_and_query_raises():
@pytest.mark.p2
def test_time_filtered_query_compound_cursor_with_id_column():
client = _FakeClient(table_schema=[
_FakeSchemaField("updated_at", "TIMESTAMP"),
_FakeSchemaField("id", "STRING")
])
client = _FakeClient(table_schema=[_FakeSchemaField("updated_at", "TIMESTAMP"), _FakeSchemaField("id", "STRING")])
connector = _make_connector(client=client, timestamp_column="updated_at", id_column="id")
start = datetime(2026, 1, 1, tzinfo=timezone.utc)
end = datetime(2026, 2, 1, tzinfo=timezone.utc)
query, params = connector._build_time_filtered_query(
connector._build_base_query(), start, end, start_id="last-id"
)
query, params = connector._build_time_filtered_query(connector._build_base_query(), start, end, start_id="last-id")
assert "(ragflow_src.updated_at > @start_cursor OR (ragflow_src.updated_at = @start_cursor AND ragflow_src.id > @start_cursor_id))" in query
assert "ragflow_src.updated_at <= @end_cursor" in query
@@ -194,6 +189,7 @@ def test_time_filtered_query_compound_cursor_with_id_column():
("end_cursor", "TIMESTAMP", end),
]
@pytest.mark.p2
def test_time_filtered_query_uses_gte_without_id_column():
client = _FakeClient(table_schema=[_FakeSchemaField("updated_at", "TIMESTAMP")])
@@ -201,9 +197,7 @@ def test_time_filtered_query_uses_gte_without_id_column():
start = datetime(2026, 1, 1, tzinfo=timezone.utc)
end = datetime(2026, 2, 1, tzinfo=timezone.utc)
query, params = connector._build_time_filtered_query(
connector._build_base_query(), start, end
)
query, params = connector._build_time_filtered_query(connector._build_base_query(), start, end)
assert "ragflow_src.updated_at >= @start_cursor" in query
assert "ragflow_src.updated_at <= @end_cursor" in query
@@ -251,13 +245,8 @@ def test_value_serialization_types():
assert BigQueryConnector._render_content_value(b"binary") is None
assert BigQueryConnector._render_content_value(date(2026, 1, 2)) == "2026-01-02"
assert BigQueryConnector._render_content_value(time(3, 4, 5)) == "03:04:05"
assert (
BigQueryConnector._render_content_value({"a": 1, "b": [1, 2]})
== '{"a": 1, "b": [1, 2]}'
)
assert (
BigQueryConnector._render_content_value("POINT(1 2)") == "POINT(1 2)"
) # GEOGRAPHY WKT passes through
assert BigQueryConnector._render_content_value({"a": 1, "b": [1, 2]}) == '{"a": 1, "b": [1, 2]}'
assert BigQueryConnector._render_content_value("POINT(1 2)") == "POINT(1 2)" # GEOGRAPHY WKT passes through
# Metadata base64-encodes bytes instead of skipping.
assert BigQueryConnector._render_metadata_value(b"hi") == "aGk="
@@ -310,10 +299,7 @@ def test_custom_query_mode_id_prefix():
@pytest.mark.p2
def test_batches_accumulate_to_batch_size():
rows = [
{"id": i, "name": f"n{i}", "description": "d", "status": "s"}
for i in range(5)
]
rows = [{"id": i, "name": f"n{i}", "description": "d", "status": "s"} for i in range(5)]
client = _FakeClient(rows=rows)
connector = _make_connector(client=client, batch_size=2)
@@ -329,18 +315,10 @@ def test_cursor_serialize_deserialize_roundtrip():
t = time(12, 0)
dec = Decimal("1.23")
assert BigQueryConnector.deserialize_cursor_value(
BigQueryConnector.serialize_cursor_value(dt)
) == dt
assert BigQueryConnector.deserialize_cursor_value(
BigQueryConnector.serialize_cursor_value(d)
) == d
assert BigQueryConnector.deserialize_cursor_value(
BigQueryConnector.serialize_cursor_value(t)
) == t
assert BigQueryConnector.deserialize_cursor_value(
BigQueryConnector.serialize_cursor_value(dec)
) == dec
assert BigQueryConnector.deserialize_cursor_value(BigQueryConnector.serialize_cursor_value(dt)) == dt
assert BigQueryConnector.deserialize_cursor_value(BigQueryConnector.serialize_cursor_value(d)) == d
assert BigQueryConnector.deserialize_cursor_value(BigQueryConnector.serialize_cursor_value(t)) == t
assert BigQueryConnector.deserialize_cursor_value(BigQueryConnector.serialize_cursor_value(dec)) == dec
assert BigQueryConnector.serialize_cursor_value(42) == 42
assert BigQueryConnector.deserialize_cursor_value(42) == 42
@@ -383,6 +361,7 @@ def test_validation_detects_missing_content_column():
with pytest.raises(ConnectorValidationError, match="name"):
connector.validate_connector_settings()
@pytest.mark.p2
def test_validation_detects_missing_metadata_column():
client = _FakeClient(table_schema=[_FakeSchemaField("name", "STRING")])
@@ -390,6 +369,7 @@ def test_validation_detects_missing_metadata_column():
with pytest.raises(ConnectorValidationError, match="status"):
connector.validate_connector_settings()
@pytest.mark.p2
def test_validation_detects_missing_id_column():
client = _FakeClient(table_schema=[_FakeSchemaField("name", "STRING")])
@@ -397,6 +377,7 @@ def test_validation_detects_missing_id_column():
with pytest.raises(ConnectorValidationError, match="id"):
connector.validate_connector_settings()
@pytest.mark.p2
def test_validation_detects_missing_timestamp_column():
client = _FakeClient(table_schema=[_FakeSchemaField("name", "STRING")])
@@ -404,6 +385,7 @@ def test_validation_detects_missing_timestamp_column():
with pytest.raises(ConnectorValidationError, match="ts"):
connector.validate_connector_settings()
@pytest.mark.p2
def test_validation_detects_unsupported_cursor_type_early():
client = _FakeClient(table_schema=[_FakeSchemaField("name", "STRING"), _FakeSchemaField("ts", "BOOL")])

16
test/unit_test/data_source/test_imap_connector_addr_parsing.py
View File

@@ -42,9 +42,7 @@ class TestParseAddrs:
assert _parse_addrs("user@example.com") == [("", "user@example.com")]
def test_address_with_display_name(self):
assert _parse_addrs("Alice <alice@example.com>") == [
("Alice", "alice@example.com")
]
assert _parse_addrs("Alice <alice@example.com>") == [("Alice", "alice@example.com")]
def test_quoted_display_name_with_comma_returns_single_address(self):
# #14963: the bug was that ``split(",")`` produced two bogus tuples.
@@ -57,9 +55,7 @@ class TestParseAddrs:
assert result == [("", "a@example.com"), ("", "b@example.com")]
def test_multiple_addresses_with_quoted_comma_in_name(self):
result = _parse_addrs(
'"Wilkens, Michael" <m@example.com>, "Müller, Hans" <h@example.com>'
)
result = _parse_addrs('"Wilkens, Michael" <m@example.com>, "Müller, Hans" <h@example.com>')
assert result == [
("Wilkens, Michael", "m@example.com"),
("Müller, Hans", "h@example.com"),
@@ -79,9 +75,7 @@ class TestParseSingularAddr:
def test_quoted_comma_display_name_does_not_raise(self):
# #14963 cascade: before the fix, ``_parse_addrs`` returned two bogus
# tuples and ``_parse_singular_addr`` then raised RuntimeError.
assert _parse_singular_addr(
'"Schlüter, Sabine" <sabine.schlueter@ihklw.de>'
) == ("Schlüter, Sabine", "sabine.schlueter@ihklw.de")
assert _parse_singular_addr('"Schlüter, Sabine" <sabine.schlueter@ihklw.de>') == ("Schlüter, Sabine", "sabine.schlueter@ihklw.de")
def test_multi_address_header_warns_and_returns_first(self, caplog):
# #14964: a legitimately multi-address From header must not crash sync.
@@ -89,9 +83,7 @@ class TestParseSingularAddr:
with caplog.at_level(logging.WARNING):
result = _parse_singular_addr(header)
assert result == ("User A", "a@example.com")
assert any(
"Multiple addresses" in rec.message for rec in caplog.records
), f"expected warning about multiple addresses, got: {caplog.records}"
assert any("Multiple addresses" in rec.message for rec in caplog.records), f"expected warning about multiple addresses, got: {caplog.records}"
def test_multi_address_header_does_not_raise(self):
# Explicit guard: no RuntimeError should propagate.

130
test/unit_test/data_source/test_onedrive_connector_unit.py
View File

@@ -26,22 +26,19 @@ _GRAPH_BASE = "https://graph.microsoft.com/v1.0"
# folder_path / _delta_url
# ---------------------------------------------------------------------------
@pytest.mark.p2
def test_folder_path_prepends_leading_slash_for_delta_url():
connector = OneDriveConnector(folder_path="Documents/Reports")
assert connector.folder_path == "/Documents/Reports"
assert connector._delta_url("drive-1") == (
f"{_GRAPH_BASE}/drives/drive-1/root:/Documents/Reports:/delta"
)
assert connector._delta_url("drive-1") == (f"{_GRAPH_BASE}/drives/drive-1/root:/Documents/Reports:/delta")
@pytest.mark.p2
def test_folder_path_preserves_leading_slash():
connector = OneDriveConnector(folder_path="/Documents/Reports/")
assert connector.folder_path == "/Documents/Reports"
assert connector._delta_url("drive-1") == (
f"{_GRAPH_BASE}/drives/drive-1/root:/Documents/Reports:/delta"
)
assert connector._delta_url("drive-1") == (f"{_GRAPH_BASE}/drives/drive-1/root:/Documents/Reports:/delta")
@pytest.mark.p2
@@ -54,9 +51,7 @@ def test_folder_path_rejects_parent_segments():
def test_folder_path_normalizes_consecutive_slashes():
connector = OneDriveConnector(folder_path="//Documents//Reports")
assert connector.folder_path == "/Documents/Reports"
assert connector._delta_url("drive-1") == (
f"{_GRAPH_BASE}/drives/drive-1/root:/Documents/Reports:/delta"
)
assert connector._delta_url("drive-1") == (f"{_GRAPH_BASE}/drives/drive-1/root:/Documents/Reports:/delta")
@pytest.mark.p2
@@ -83,6 +78,7 @@ def test_folder_path_double_slash_only_uses_drive_root_delta():
# load_credentials
# ---------------------------------------------------------------------------
@pytest.mark.p2
def test_load_credentials_missing_fields_raises():
connector = OneDriveConnector()
@@ -122,6 +118,7 @@ def test_load_credentials_msal_failure_raises():
# validate_connector_settings
# ---------------------------------------------------------------------------
@pytest.mark.p2
def test_validate_without_credentials_raises():
connector = OneDriveConnector()
@@ -190,6 +187,7 @@ def test_validate_unexpected_status_raises():
# Checkpoint helpers
# ---------------------------------------------------------------------------
@pytest.mark.p2
def test_build_dummy_checkpoint():
connector = OneDriveConnector()
@@ -210,6 +208,7 @@ def test_validate_checkpoint_json_invalid_returns_dummy():
# _iter_documents (via poll_source)
# ---------------------------------------------------------------------------
def _ok(json_value):
"""Tiny helper: build a successful MagicMock response with .ok / .json()."""
resp = MagicMock()
@@ -234,20 +233,22 @@ def test_poll_source_yields_supported_files():
connector._access_token = "tok"
drives_resp = _ok({"value": [{"id": "drive-1"}]})
delta_resp = _ok({
"value": [
{
"id": "file-1",
"name": "report.docx",
"file": {},
"lastModifiedDateTime": "2026-05-20T10:00:00Z",
"webUrl": "https://example.com/report.docx",
"size": 1024,
"createdBy": {"user": {"displayName": "Alice"}},
}
],
"@odata.deltaLink": "https://graph.microsoft.com/delta-link",
})
delta_resp = _ok(
{
"value": [
{
"id": "file-1",
"name": "report.docx",
"file": {},
"lastModifiedDateTime": "2026-05-20T10:00:00Z",
"webUrl": "https://example.com/report.docx",
"size": 1024,
"createdBy": {"user": {"displayName": "Alice"}},
}
],
"@odata.deltaLink": "https://graph.microsoft.com/delta-link",
}
)
with patch.object(connector, "_get", side_effect=[drives_resp, delta_resp]):
batches = list(connector.poll_source(0.0, 9999999999.0))
@@ -263,18 +264,20 @@ def test_poll_source_skips_unsupported_extensions():
connector._access_token = "tok"
drives_resp = _ok({"value": [{"id": "drive-1"}]})
delta_resp = _ok({
"value": [
{
"id": "img-1",
"name": "photo.png", # not in _SUPPORTED_EXTENSIONS
"file": {},
"lastModifiedDateTime": "2026-05-20T10:00:00Z",
"webUrl": "https://example.com/photo.png",
"size": 512,
}
],
})
delta_resp = _ok(
{
"value": [
{
"id": "img-1",
"name": "photo.png", # not in _SUPPORTED_EXTENSIONS
"file": {},
"lastModifiedDateTime": "2026-05-20T10:00:00Z",
"webUrl": "https://example.com/photo.png",
"size": 512,
}
],
}
)
with patch.object(connector, "_get", side_effect=[drives_resp, delta_resp]):
batches = list(connector.poll_source(0.0, 9999999999.0))
@@ -288,16 +291,18 @@ def test_poll_source_skips_deleted_items():
connector._access_token = "tok"
drives_resp = _ok({"value": [{"id": "drive-1"}]})
delta_resp = _ok({
"value": [
{
"id": "file-del",
"name": "gone.docx",
"file": {},
"deleted": {"state": "deleted"},
}
],
})
delta_resp = _ok(
{
"value": [
{
"id": "file-del",
"name": "gone.docx",
"file": {},
"deleted": {"state": "deleted"},
}
],
}
)
with patch.object(connector, "_get", side_effect=[drives_resp, delta_resp]):
batches = list(connector.poll_source(0.0, 9999999999.0))
@@ -309,6 +314,7 @@ def test_poll_source_skips_deleted_items():
# Non-2xx Graph responses must raise (no silent partial syncs)
# ---------------------------------------------------------------------------
@pytest.mark.p1
def test_iter_documents_raises_on_graph_http_500():
"""A 500 from the delta endpoint must surface — silently breaking would
@@ -353,6 +359,7 @@ def test_list_drive_ids_raises_on_http_error():
# retrieve_all_slim_docs_perm_sync: yields SlimDocument batches for prune
# ---------------------------------------------------------------------------
@pytest.mark.p1
def test_retrieve_slim_docs_yields_slimdocument_batches():
"""The prune collector does file_list.extend(batch) and reads .id on each
@@ -362,13 +369,15 @@ def test_retrieve_slim_docs_yields_slimdocument_batches():
connector._access_token = "tok"
drives_resp = _ok({"value": [{"id": "drive-1"}]})
delta_resp = _ok({
"value": [
{"id": "f1", "name": "a.docx", "file": {}},
{"id": "f2", "name": "b.pdf", "file": {}},
{"id": "f3", "name": "c.txt", "file": {}},
],
})
delta_resp = _ok(
{
"value": [
{"id": "f1", "name": "a.docx", "file": {}},
{"id": "f2", "name": "b.pdf", "file": {}},
{"id": "f3", "name": "c.txt", "file": {}},
],
}
)
with patch.object(connector, "_get", side_effect=[drives_resp, delta_resp]):
batches = list(connector.retrieve_all_slim_docs_perm_sync())
@@ -387,13 +396,15 @@ def test_retrieve_slim_docs_skips_folders_and_deleted():
connector._access_token = "tok"
drives_resp = _ok({"value": [{"id": "drive-1"}]})
delta_resp = _ok({
"value": [
{"id": "folder-1", "name": "Docs", "folder": {}}, # folder, no "file"
{"id": "del-1", "name": "gone.pdf", "file": {}, "deleted": {"state": "deleted"}},
{"id": "ok-1", "name": "keep.pdf", "file": {}},
],
})
delta_resp = _ok(
{
"value": [
{"id": "folder-1", "name": "Docs", "folder": {}}, # folder, no "file"
{"id": "del-1", "name": "gone.pdf", "file": {}, "deleted": {"state": "deleted"}},
{"id": "ok-1", "name": "keep.pdf", "file": {}},
],
}
)
with patch.object(connector, "_get", side_effect=[drives_resp, delta_resp]):
batches = list(connector.retrieve_all_slim_docs_perm_sync())
@@ -427,6 +438,7 @@ def test_retrieve_slim_docs_requires_credentials():
# load_from_checkpoint: resumes from delta_links and honors start floor
# ---------------------------------------------------------------------------
@pytest.mark.p1
def test_load_from_checkpoint_uses_persisted_delta_link():
"""When the checkpoint carries a delta_link for a drive, the connector

64
test/unit_test/data_source/test_outlook_connector_unit.py
View File

@@ -29,6 +29,7 @@ _GOOD_CREDS = {
# _strip_html
# ---------------------------------------------------------------------------
@pytest.mark.p3
def test_strip_html_removes_tags_and_script():
html = "<html><body><script>evil()</script><p>Hello <b>world</b></p></body></html>"
@@ -45,6 +46,7 @@ def test_strip_html_empty_returns_empty():
# load_credentials
# ---------------------------------------------------------------------------
@pytest.mark.p2
def test_load_credentials_missing_fields_raises():
connector = OutlookConnector()
@@ -90,6 +92,7 @@ def test_load_credentials_msal_failure_raises():
# validate_connector_settings
# ---------------------------------------------------------------------------
@pytest.mark.p2
def test_validate_without_credentials_raises():
connector = OutlookConnector()
@@ -170,6 +173,7 @@ def test_validate_5xx_raises_unexpected():
# Checkpoint helpers
# ---------------------------------------------------------------------------
@pytest.mark.p2
def test_build_dummy_checkpoint():
connector = OutlookConnector()
@@ -190,6 +194,7 @@ def test_validate_checkpoint_json_invalid_returns_dummy():
# _list_user_ids
# ---------------------------------------------------------------------------
@pytest.mark.p2
def test_list_user_ids_returns_configured_ids():
connector = OutlookConnector(user_ids=["a@x.com", "b@x.com"])
@@ -224,11 +229,10 @@ def test_list_user_ids_paginates_when_unset():
# _iter_documents (via poll_source)
# ---------------------------------------------------------------------------
@pytest.mark.p1
def test_poll_source_yields_messages():
connector = OutlookConnector(
batch_size=10, user_ids=["alice@example.com"]
)
connector = OutlookConnector(batch_size=10, user_ids=["alice@example.com"])
connector._access_token = "tok"
delta_resp = MagicMock(ok=True)
@@ -240,12 +244,8 @@ def test_poll_source_yields_messages():
"body": {"contentType": "text", "content": "Body text"},
"receivedDateTime": "2026-05-20T10:00:00Z",
"webLink": "https://outlook.office.com/mail/1",
"from": {
"emailAddress": {"name": "Bob", "address": "bob@example.com"}
},
"toRecipients": [
{"emailAddress": {"address": "alice@example.com"}}
],
"from": {"emailAddress": {"name": "Bob", "address": "bob@example.com"}},
"toRecipients": [{"emailAddress": {"address": "alice@example.com"}}],
"ccRecipients": [],
"hasAttachments": False,
"conversationId": "conv-1",
@@ -268,9 +268,7 @@ def test_poll_source_yields_messages():
@pytest.mark.p2
def test_poll_source_filters_old_messages():
connector = OutlookConnector(
batch_size=10, user_ids=["alice@example.com"]
)
connector = OutlookConnector(batch_size=10, user_ids=["alice@example.com"])
connector._access_token = "tok"
delta_resp = MagicMock(ok=True)
@@ -293,9 +291,7 @@ def test_poll_source_filters_old_messages():
@pytest.mark.p2
def test_poll_source_skips_removed_messages():
connector = OutlookConnector(
batch_size=10, user_ids=["alice@example.com"]
)
connector = OutlookConnector(batch_size=10, user_ids=["alice@example.com"])
connector._access_token = "tok"
delta_resp = MagicMock(ok=True)
@@ -320,9 +316,7 @@ def test_poll_source_skips_removed_messages():
@pytest.mark.p2
def test_poll_source_html_body_is_stripped():
connector = OutlookConnector(
batch_size=10, user_ids=["alice@example.com"]
)
connector = OutlookConnector(batch_size=10, user_ids=["alice@example.com"])
connector._access_token = "tok"
delta_resp = MagicMock(ok=True)
@@ -352,6 +346,7 @@ def test_poll_source_html_body_is_stripped():
# Non-2xx Graph responses must raise (no silent partial syncs)
# ---------------------------------------------------------------------------
def _ok(json_value):
resp = MagicMock(ok=True, status_code=200)
resp.json.return_value = json_value
@@ -399,6 +394,7 @@ def test_list_user_ids_raises_on_http_error():
# retrieve_all_slim_docs_perm_sync: yields list[SlimDocument] for prune
# ---------------------------------------------------------------------------
@pytest.mark.p1
def test_retrieve_slim_docs_yields_slimdocument_batches():
"""The prune collector calls file_list.extend(batch) and reads `.id` on
@@ -407,13 +403,15 @@ def test_retrieve_slim_docs_yields_slimdocument_batches():
connector = OutlookConnector(batch_size=2, user_ids=["alice@example.com"])
connector._access_token = "tok"
delta_resp = _ok({
"value": [
{"id": "m1", "subject": "a"},
{"id": "m2", "subject": "b"},
{"id": "m3", "subject": "c"},
],
})
delta_resp = _ok(
{
"value": [
{"id": "m1", "subject": "a"},
{"id": "m2", "subject": "b"},
{"id": "m3", "subject": "c"},
],
}
)
with patch.object(connector, "_get", return_value=delta_resp):
batches = list(connector.retrieve_all_slim_docs_perm_sync())
@@ -430,12 +428,14 @@ def test_retrieve_slim_docs_skips_removed():
connector = OutlookConnector(batch_size=10, user_ids=["alice@example.com"])
connector._access_token = "tok"
delta_resp = _ok({
"value": [
{"id": "del", "@removed": {"reason": "deleted"}},
{"id": "keep", "subject": "kept"},
],
})
delta_resp = _ok(
{
"value": [
{"id": "del", "@removed": {"reason": "deleted"}},
{"id": "keep", "subject": "kept"},
],
}
)
with patch.object(connector, "_get", return_value=delta_resp):
batches = list(connector.retrieve_all_slim_docs_perm_sync())
flat = [item for batch in batches for item in batch]
@@ -462,6 +462,7 @@ def test_retrieve_slim_docs_requires_credentials():
# load_from_checkpoint: resumes from delta_links
# ---------------------------------------------------------------------------
@pytest.mark.p1
def test_load_from_checkpoint_uses_persisted_delta_link():
"""With a delta_link for a user the connector must hit that URL — not
@@ -490,6 +491,7 @@ def test_load_from_checkpoint_uses_persisted_delta_link():
# _redact: keep debugging hint, drop PII
# ---------------------------------------------------------------------------
@pytest.mark.p3
def test_redact_email_masks_local_and_domain():
assert _redact("alice@example.com") == "al***@***"

69
test/unit_test/data_source/test_rest_api_connector.py
View File

@@ -53,13 +53,17 @@ def _mocked_rest_api_requests_and_dns():
that wrap `requests.get` / `requests.post` and avoid retry backoff delays.
"""
mock_rl = MagicMock()
with patch(
"common.data_source.rest_api_connector.socket.getaddrinfo",
return_value=_MOCK_DNS_ADDRINFO,
), patch.object(_ds_utils._RateLimitedRequest, "get", mock_rl.get), patch.object(
_ds_utils._RateLimitedRequest,
"post",
mock_rl.post,
with (
patch(
"common.data_source.rest_api_connector.socket.getaddrinfo",
return_value=_MOCK_DNS_ADDRINFO,
),
patch.object(_ds_utils._RateLimitedRequest, "get", mock_rl.get),
patch.object(
_ds_utils._RateLimitedRequest,
"post",
mock_rl.post,
),
):
yield mock_rl
@@ -108,6 +112,7 @@ def _mock_response(json_data, status_code=200):
# 1. Config schema validation #
# ===================================================================== #
class TestRestAPIConfig:
"""Test Pydantic RestAPIConnectorConfig schema validation."""
@@ -143,9 +148,7 @@ class TestRestAPIConfig:
def test_string_to_dict_coercion_for_headers(self):
"""A key=value string should be coerced to a dict."""
cfg = RestAPIConnectorConfig(
url=VALID_URL, content_fields=["t"], headers="X-Custom=hello"
)
cfg = RestAPIConnectorConfig(url=VALID_URL, content_fields=["t"], headers="X-Custom=hello")
assert cfg.headers == {"X-Custom": "hello"}
def test_string_to_list_coercion_for_content_fields(self):
@@ -158,6 +161,7 @@ class TestRestAPIConfig:
# 2. SSRF URL validation #
# ===================================================================== #
class TestSSRFValidation:
"""Test that unsafe URLs are blocked before any HTTP request is made."""
@@ -249,6 +253,7 @@ class TestSSRFValidation:
# redirect target.
import common.data_source.rest_api_connector as rc_module
from unittest.mock import patch as _patch
with _patch.object(rc_module.socket, "getaddrinfo", side_effect=_dns_for_host):
# Coderabbit MAJOR #3486038795: SSRF validation failures inside
# _safe_request are now wrapped to raise ConnectorValidationError
@@ -305,11 +310,11 @@ class TestSSRFValidation:
# 3. Authentication setup #
# ===================================================================== #
class TestAuthSetup:
"""Test _build_auth produces the correct headers / auth objects."""
@patch("common.data_source.rest_api_connector.socket.getaddrinfo",
return_value=[(2, 1, 6, "", ("93.184.216.34", 0))])
@patch("common.data_source.rest_api_connector.socket.getaddrinfo", return_value=[(2, 1, 6, "", ("93.184.216.34", 0))])
def test_auth_none(self, _dns):
"""auth_type=none should produce no auth headers."""
c = _make_connector(auth_type=AuthType.NONE)
@@ -317,8 +322,7 @@ class TestAuthSetup:
assert c._auth_headers == {}
assert c._basic_auth is None
@patch("common.data_source.rest_api_connector.socket.getaddrinfo",
return_value=[(2, 1, 6, "", ("93.184.216.34", 0))])
@patch("common.data_source.rest_api_connector.socket.getaddrinfo", return_value=[(2, 1, 6, "", ("93.184.216.34", 0))])
def test_api_key_header(self, _dns):
"""api_key_header should set the specified header."""
c = _make_connector(
@@ -328,16 +332,14 @@ class TestAuthSetup:
c.load_credentials({"api_key": "secret123"})
assert c._auth_headers == {"X-API-Key": "secret123"}
@patch("common.data_source.rest_api_connector.socket.getaddrinfo",
return_value=[(2, 1, 6, "", ("93.184.216.34", 0))])
@patch("common.data_source.rest_api_connector.socket.getaddrinfo", return_value=[(2, 1, 6, "", ("93.184.216.34", 0))])
def test_bearer_token(self, _dns):
"""bearer should set Authorization: Bearer <token>."""
c = _make_connector(auth_type=AuthType.BEARER)
c.load_credentials({"token": "tok_abc"})
assert c._auth_headers == {"Authorization": "Bearer tok_abc"}
@patch("common.data_source.rest_api_connector.socket.getaddrinfo",
return_value=[(2, 1, 6, "", ("93.184.216.34", 0))])
@patch("common.data_source.rest_api_connector.socket.getaddrinfo", return_value=[(2, 1, 6, "", ("93.184.216.34", 0))])
def test_basic_auth(self, _dns):
"""basic should produce an HTTPBasicAuth object."""
c = _make_connector(auth_type=AuthType.BASIC)
@@ -351,14 +353,13 @@ class TestAuthSetup:
# 4. Field extraction #
# ===================================================================== #
class TestFieldExtraction:
"""Test _extract_field / _extract_field_values dot-notation paths."""
@patch("common.data_source.rest_api_connector.socket.getaddrinfo",
return_value=[(2, 1, 6, "", ("93.184.216.34", 0))])
@patch("common.data_source.rest_api_connector.socket.getaddrinfo", return_value=[(2, 1, 6, "", ("93.184.216.34", 0))])
def setup_method(self, method, _dns=None):
with patch("common.data_source.rest_api_connector.socket.getaddrinfo",
return_value=[(2, 1, 6, "", ("93.184.216.34", 0))]):
with patch("common.data_source.rest_api_connector.socket.getaddrinfo", return_value=[(2, 1, 6, "", ("93.184.216.34", 0))]):
self.connector = _make_connector()
def test_simple_field(self):
@@ -382,8 +383,7 @@ class TestFieldExtraction:
def test_missing_field_with_default(self):
"""Missing field returns configured default value."""
with patch("common.data_source.rest_api_connector.socket.getaddrinfo",
return_value=[(2, 1, 6, "", ("93.184.216.34", 0))]):
with patch("common.data_source.rest_api_connector.socket.getaddrinfo", return_value=[(2, 1, 6, "", ("93.184.216.34", 0))]):
c = _make_connector(field_default_values={"missing": "fallback"})
result = c._get_typed_field_value("missing", {"other": 1})
assert result == "fallback"
@@ -398,14 +398,13 @@ class TestFieldExtraction:
# 5. Items array detection #
# ===================================================================== #
class TestItemsArrayDetection:
"""Test _extract_items auto-detection of the items array."""
@patch("common.data_source.rest_api_connector.socket.getaddrinfo",
return_value=[(2, 1, 6, "", ("93.184.216.34", 0))])
@patch("common.data_source.rest_api_connector.socket.getaddrinfo", return_value=[(2, 1, 6, "", ("93.184.216.34", 0))])
def setup_method(self, method, _dns=None):
with patch("common.data_source.rest_api_connector.socket.getaddrinfo",
return_value=[(2, 1, 6, "", ("93.184.216.34", 0))]):
with patch("common.data_source.rest_api_connector.socket.getaddrinfo", return_value=[(2, 1, 6, "", ("93.184.216.34", 0))]):
self.connector = _make_connector()
def test_items_key(self):
@@ -451,6 +450,7 @@ class TestItemsArrayDetection:
# 6. HTML stripping #
# ===================================================================== #
class TestHTMLStripping:
"""Test the _strip_html static method."""
@@ -485,14 +485,13 @@ class TestHTMLStripping:
# 7. Document creation #
# ===================================================================== #
class TestDocumentCreation:
"""Test _item_to_document mapping."""
@patch("common.data_source.rest_api_connector.socket.getaddrinfo",
return_value=[(2, 1, 6, "", ("93.184.216.34", 0))])
@patch("common.data_source.rest_api_connector.socket.getaddrinfo", return_value=[(2, 1, 6, "", ("93.184.216.34", 0))])
def setup_method(self, method, _dns=None):
with patch("common.data_source.rest_api_connector.socket.getaddrinfo",
return_value=[(2, 1, 6, "", ("93.184.216.34", 0))]):
with patch("common.data_source.rest_api_connector.socket.getaddrinfo", return_value=[(2, 1, 6, "", ("93.184.216.34", 0))]):
self.connector = _make_connector(
id_field="id",
content_fields=["title", "body"],
@@ -551,6 +550,7 @@ class TestDocumentCreation:
# 8. Pagination behaviour #
# ===================================================================== #
class TestPaginationBehavior:
"""Test pagination iteration with mocked HTTP responses."""
@@ -611,9 +611,7 @@ class TestPaginationBehavior:
def test_max_pages_cap(self):
"""Pagination respects the max_pages safety cap."""
with _mocked_rest_api_requests_and_dns() as mock_rl:
mock_rl.get.return_value = _mock_response(
{"items": [{"title": "A"}, {"title": "B"}]}
)
mock_rl.get.return_value = _mock_response({"items": [{"title": "A"}, {"title": "B"}]})
c = _make_paged_connector(
max_pages=3,
@@ -642,6 +640,7 @@ class TestPaginationBehavior:
# 9. Non-retriable HTTP errors #
# ===================================================================== #
class TestNonRetriableErrors:
"""Test that HTTP errors are classified correctly in _fetch_page."""

18
test/unit_test/data_source/test_sharepoint_connector_unit.py
View File

@@ -26,11 +26,7 @@ def _load_sharepoint_connector_module():
"""Load sharepoint_connector.py in isolation (avoid the package __init__)."""
repo_root = Path(__file__).resolve().parents[3]
package_name = "common.data_source"
saved_modules = {
name: module
for name, module in sys.modules.items()
if name == package_name or name.startswith(f"{package_name}.")
}
saved_modules = {name: module for name, module in sys.modules.items() if name == package_name or name.startswith(f"{package_name}.")}
package_stub = ModuleType(package_name)
package_stub.__path__ = [str(repo_root / "common" / "data_source")]
sys.modules[package_name] = package_stub
@@ -214,9 +210,7 @@ def _collect(generator):
def test_load_from_checkpoint_walks_libraries_and_downloads():
connector, _jan, _feb = _build_connector_with_tree()
docs, checkpoint = _collect(
connector.load_from_checkpoint(0.0, 9e12, connector.build_dummy_checkpoint())
)
docs, checkpoint = _collect(connector.load_from_checkpoint(0.0, 9e12, connector.build_dummy_checkpoint()))
assert checkpoint.has_more is False
assert {doc.id for doc in docs} == {"drv-A:f1", "drv-A:f2"}
@@ -240,9 +234,7 @@ def test_load_from_checkpoint_filters_by_modified_window():
start = datetime(2026, 1, 15, tzinfo=timezone.utc).timestamp()
end = datetime(2026, 3, 1, tzinfo=timezone.utc).timestamp()
docs, _ = _collect(
connector.load_from_checkpoint(start, end, connector.build_dummy_checkpoint())
)
docs, _ = _collect(connector.load_from_checkpoint(start, end, connector.build_dummy_checkpoint()))
assert [doc.id for doc in docs] == ["drv-A:f2"]
@@ -274,9 +266,7 @@ def test_document_ids_are_unique_across_drives_with_colliding_item_ids():
connector.graph_client = _FakeGraphClient(site)
connector._site_url = "https://contoso.sharepoint.com/sites/MySite"
docs, _ = _collect(
connector.load_from_checkpoint(0.0, 9e12, connector.build_dummy_checkpoint())
)
docs, _ = _collect(connector.load_from_checkpoint(0.0, 9e12, connector.build_dummy_checkpoint()))
ids = {doc.id for doc in docs}
assert ids == {"drv-A:same-id", "drv-B:same-id"}

6
test/unit_test/data_source/test_slack_connector_unit.py
View File

@@ -30,11 +30,7 @@ def _load_slack_connector_module():
"""
repo_root = Path(__file__).resolve().parents[3]
package_name = "common.data_source"
saved_modules = {
name: module
for name, module in sys.modules.items()
if name == package_name or name.startswith(f"{package_name}.")
}
saved_modules = {name: module for name, module in sys.modules.items() if name == package_name or name.startswith(f"{package_name}.")}
package_stub = ModuleType(package_name)
package_stub.__path__ = [str(repo_root / "common" / "data_source")]
sys.modules[package_name] = package_stub

18
test/unit_test/data_source/test_teams_connector_unit.py
View File

@@ -26,11 +26,7 @@ def _load_teams_connector_module():
"""Load teams_connector.py in isolation (avoid the package __init__)."""
repo_root = Path(__file__).resolve().parents[3]
package_name = "common.data_source"
saved_modules = {
name: module
for name, module in sys.modules.items()
if name == package_name or name.startswith(f"{package_name}.")
}
saved_modules = {name: module for name, module in sys.modules.items() if name == package_name or name.startswith(f"{package_name}.")}
package_stub = ModuleType(package_name)
package_stub.__path__ = [str(repo_root / "common" / "data_source")]
sys.modules[package_name] = package_stub
@@ -175,9 +171,7 @@ def test_load_credentials_sets_graph_client(monkeypatch):
monkeypatch.setattr(teams_connector, "GraphClient", lambda token_callback: SimpleNamespace(cb=token_callback))
connector = TeamsConnector()
result = connector.load_credentials(
{"tenant_id": "tenant", "client_id": "client", "client_secret": "secret"}
)
result = connector.load_credentials({"tenant_id": "tenant", "client_id": "client", "client_secret": "secret"})
assert result is None
assert connector.graph_client is not None
@@ -217,9 +211,7 @@ def test_validate_maps_permission_error():
def test_load_from_checkpoint_flattens_posts_and_replies():
connector = _build_connector()
docs, checkpoint = _collect(
connector.load_from_checkpoint(0.0, 9e12, connector.build_dummy_checkpoint())
)
docs, checkpoint = _collect(connector.load_from_checkpoint(0.0, 9e12, connector.build_dummy_checkpoint()))
assert checkpoint.has_more is False
assert {doc.id for doc in docs} == {"t1__c1__m1", "t1__c1__m2"}
@@ -245,9 +237,7 @@ def test_load_from_checkpoint_filters_by_modified_window():
start = datetime(2026, 1, 15, tzinfo=timezone.utc).timestamp()
end = datetime(2026, 3, 1, tzinfo=timezone.utc).timestamp()
docs, _ = _collect(
connector.load_from_checkpoint(start, end, connector.build_dummy_checkpoint())
)
docs, _ = _collect(connector.load_from_checkpoint(start, end, connector.build_dummy_checkpoint()))
assert [doc.id for doc in docs] == ["t1__c1__m2"]

10
test/unit_test/data_source/test_webdav_connector_unit.py
View File

@@ -256,10 +256,7 @@ def test_yield_webdav_documents_skips_missing_size_metadata(caplog):
"webdav:https://webdav.example:/small.txt",
]
assert connector.client.downloaded_paths == ["/small.txt"]
assert (
"missing.txt: size metadata missing from WebDAV server response, "
"skipping to avoid processing potentially large files."
) in caplog.text
assert ("missing.txt: size metadata missing from WebDAV server response, skipping to avoid processing potentially large files.") in caplog.text
@pytest.mark.p1
@@ -301,7 +298,4 @@ def test_retrieve_all_slim_docs_skips_missing_size_metadata(caplog):
assert [doc.id for batch in batches for doc in batch] == [
"webdav:https://webdav.example:/small.txt",
]
assert (
"missing.txt: size metadata missing from WebDAV server response, "
"skipping to avoid processing potentially large files."
) in caplog.text
assert ("missing.txt: size metadata missing from WebDAV server response, skipping to avoid processing potentially large files.") in caplog.text