zlei6/ragflow
1
0
Fork 0
mirror of https://github.com/infiniflow/ragflow.git synced 2026-08-01 13:33:48 +08:00
Code Issues Packages Projects Releases Wiki Activity

Refactor: Consolidation WEB API & HTTP API for document list_docs (#14176)

Browse Source 
### What problem does this PR solve?

Before consolidation
Web API: POST /v1/document/list
Http API - GET /api/v1/datasets/<dataset_id>/documents

After consolidation, Restful API -- GET
/api/v1/datasets/<dataset_id>/documents

### Type of change

- [x] Refactoring
This commit is contained in:
Jack
2026-04-20 14:54:40 +08:00
committed by GitHub
parent d053317c4d
commit 939933649a
28 changed files with 627 additions and 712 deletions
Download Patch File Download Diff File Expand all files Collapse all files

59
test/testcases/test_http_api/test_file_management_within_dataset/test_doc_sdk_routes_unit.py
View File

@@ -388,65 +388,6 @@ class TestDocRoutesUnit:
res = _run(module.download_doc("doc-1"))
assert res["filename"] == "doc.txt"
def test_list_docs_metadata_filters(self, monkeypatch):
module = _load_doc_module(monkeypatch)
monkeypatch.setattr(module.KnowledgebaseService, "accessible", lambda **_kwargs: False)
monkeypatch.setattr(module, "request", SimpleNamespace(args=_DummyArgs()))
res = module.list_docs.__wrapped__("ds-1", "tenant-1")
assert "don't own the dataset" in res["message"]
monkeypatch.setattr(module.KnowledgebaseService, "accessible", lambda **_kwargs: True)
monkeypatch.setattr(
module,
"request",
SimpleNamespace(
args=_DummyArgs(
{
"metadata_condition": "{bad json",
}
)
),
)
res = module.list_docs.__wrapped__("ds-1", "tenant-1")
assert res["message"] == "metadata_condition must be valid JSON."
monkeypatch.setattr(module, "request", SimpleNamespace(args=_DummyArgs({"metadata_condition": "[1]"})))
res = module.list_docs.__wrapped__("ds-1", "tenant-1")
assert res["message"] == "metadata_condition must be an object."
monkeypatch.setattr(module.DocMetadataService, "get_flatted_meta_by_kbs", lambda _kbs: [{"doc_id": "x"}])
monkeypatch.setattr(module, "meta_filter", lambda *_args, **_kwargs: [])
monkeypatch.setattr(module, "convert_conditions", lambda cond: cond)
monkeypatch.setattr(
module,
"request",
SimpleNamespace(args=_DummyArgs({"metadata_condition": '{"conditions":[{"field":"x","op":"eq","value":"y"}]}'})),
)
res = module.list_docs.__wrapped__("ds-1", "tenant-1")
assert res["code"] == module.RetCode.SUCCESS
assert res["data"]["total"] == 0
monkeypatch.setattr(
module.DocumentService,
"get_list",
lambda *_args, **_kwargs: ([{"id": "doc-1", "create_time": 100, "run": "0"}], 1),
)
monkeypatch.setattr(
module,
"request",
SimpleNamespace(
args=_DummyArgs(
{
"create_time_from": "101",
"create_time_to": "200",
}
)
),
)
res = module.list_docs.__wrapped__("ds-1", "tenant-1")
assert res["code"] == 0
assert res["data"]["docs"] == []
def test_metadata_batch_update(self, monkeypatch):
module = _load_doc_module(monkeypatch)
monkeypatch.setattr(module, "convert_conditions", lambda cond: cond)

160
test/testcases/test_http_api/test_file_management_within_dataset/test_list_documents.py
View File

@@ -27,11 +27,11 @@ class TestAuthorization:
@pytest.mark.parametrize(
"invalid_auth, expected_code, expected_message",
[
(None, 0, "`Authorization` can't be empty"),
(None, 401, "<Unauthorized '401: Unauthorized'>"),
(
RAGFlowHttpApiAuth(INVALID_API_TOKEN),
109,
"Authentication error: API key is invalid!",
401,
"<Unauthorized '401: Unauthorized'>",
),
],
)
@@ -72,7 +72,7 @@ class TestDocumentsList:
"params, expected_code, expected_page_size, expected_message",
[
({"page": None, "page_size": 2}, 0, 2, ""),
({"page": 0, "page_size": 2}, 0, 2, ""),
({"page": 1, "page_size": 2}, 0, 2, ""),
({"page": 2, "page_size": 2}, 0, 2, ""),
({"page": 3, "page_size": 2}, 0, 1, ""),
({"page": "3", "page_size": 2}, 0, 1, ""),
@@ -115,7 +115,6 @@ class TestDocumentsList:
"params, expected_code, expected_page_size, expected_message",
[
({"page_size": None}, 0, 5, ""),
({"page_size": 0}, 0, 0, ""),
({"page_size": 1}, 0, 1, ""),
({"page_size": 6}, 0, 5, ""),
({"page_size": "1"}, 0, 1, ""),
@@ -232,6 +231,7 @@ class TestDocumentsList:
assert len(res["data"]["docs"]) == expected_num
assert res["data"]["total"] == expected_num
@pytest.mark.p1
@pytest.mark.parametrize(
"params, expected_code, expected_num, expected_message",
@@ -240,21 +240,21 @@ class TestDocumentsList:
({"name": ""}, 0, 5, ""),
({"name": "ragflow_test_upload_0.txt"}, 0, 1, ""),
(
{"name": "unknown.txt"},
102,
0,
"You don't own the document unknown.txt.",
{"name": "unknown.txt"},
102,
0,
"You don't own the document unknown.txt.",
),
],
)
def test_name(
self,
HttpApiAuth,
add_documents,
params,
expected_code,
expected_num,
expected_message,
self,
HttpApiAuth,
add_documents,
params,
expected_code,
expected_num,
expected_message,
):
dataset_id, _ = add_documents
res = list_documents(HttpApiAuth, dataset_id, params=params)
@@ -267,6 +267,7 @@ class TestDocumentsList:
else:
assert res["message"] == expected_message
@pytest.mark.p1
@pytest.mark.parametrize(
"document_id, expected_code, expected_num, expected_message",
@@ -278,13 +279,13 @@ class TestDocumentsList:
],
)
def test_id(
self,
HttpApiAuth,
add_documents,
document_id,
expected_code,
expected_num,
expected_message,
self,
HttpApiAuth,
add_documents,
document_id,
expected_code,
expected_num,
expected_message,
):
dataset_id, document_ids = add_documents
if callable(document_id):
@@ -298,11 +299,13 @@ class TestDocumentsList:
if params["id"] in [None, ""]:
assert len(res["data"]["docs"]) == expected_num
else:
assert res["data"]["docs"][0]["id"] == params["id"]
doc = res["data"]["docs"][0]
assert doc["id"] == params["id"]
else:
assert res["message"] == expected_message
@pytest.mark.p3
@pytest.mark.p2
@pytest.mark.parametrize(
"document_id, name, expected_code, expected_num, expected_message",
[
@@ -310,23 +313,23 @@ class TestDocumentsList:
(lambda r: r[0], "ragflow_test_upload_1.txt", 0, 0, ""),
(lambda r: r[0], "unknown", 102, 0, "You don't own the document unknown."),
(
"id",
"ragflow_test_upload_0.txt",
102,
0,
"You don't own the document id.",
"id",
"ragflow_test_upload_0.txt",
102,
0,
"You don't own the document id.",
),
],
)
def test_name_and_id(
self,
HttpApiAuth,
add_documents,
document_id,
name,
expected_code,
expected_num,
expected_message,
self,
HttpApiAuth,
add_documents,
document_id,
name,
expected_code,
expected_num,
expected_message,
):
dataset_id, document_ids = add_documents
if callable(document_id):
@@ -340,6 +343,7 @@ class TestDocumentsList:
else:
assert res["message"] == expected_message
@pytest.mark.p3
def test_concurrent_list(self, HttpApiAuth, add_documents):
dataset_id, _ = add_documents
@@ -358,3 +362,83 @@ class TestDocumentsList:
res = list_documents(HttpApiAuth, dataset_id, params=params)
assert res["code"] == 0
assert len(res["data"]["docs"]) == 5
@pytest.mark.p2
@pytest.mark.parametrize(
"params, expected_code, expected_message",
[
(
{"metadata_condition": "{bad json"},
102,
"metadata_condition must be valid JSON",
),
(
{"metadata_condition": "[1]"},
102,
"metadata_condition must be an object",
),
],
)
def test_metadata_condition_validation(
self, HttpApiAuth, add_documents, params, expected_code, expected_message
):
dataset_id, _ = add_documents
res = list_documents(HttpApiAuth, dataset_id, params=params)
assert res["code"] == expected_code
assert expected_message in res["message"]
@pytest.mark.p2
@pytest.mark.parametrize(
"params, expected_code, expected_total",
[
# Filter with create_time_from in the future - should return 0 results
({"create_time_from": "9999999999000"}, 0, 0),
# Filter with create_time_to in the past - should return 0 results
({"create_time_to": "1"}, 0, 0),
# Filter with create_time_from and create_time_to covering all time
({"create_time_from": "0", "create_time_to": "9999999999000"}, 0, 5),
],
)
def test_create_time_filter(
self, HttpApiAuth, add_documents, params, expected_code, expected_total
):
dataset_id, _ = add_documents
res = list_documents(HttpApiAuth, dataset_id, params=params)
assert res["code"] == expected_code
assert len(res["data"]["docs"]) == expected_total
assert res["data"]["total"] == 5
@pytest.mark.p2
@pytest.mark.parametrize(
"params, expected_code, expected_message",
[
# Invalid run status - should return error
({"run": ["INVALID_STATUS"]}, 102, "Invalid filter run status conditions: INVALID_STATUS"),
],
)
def test_run_status_filter_invalid(
self, HttpApiAuth, add_documents, params, expected_code, expected_message
):
dataset_id, _ = add_documents
res = list_documents(HttpApiAuth, dataset_id, params=params)
assert res["code"] == expected_code
assert expected_message in res["message"]
@pytest.mark.p2
@pytest.mark.parametrize(
"params, expected_size",
[
# Invalid run status - should return error
({"run": ["UNSTART"]}, 5),
],
)
def test_run_status_filter_unstart(
self, HttpApiAuth, add_documents, params, expected_size
):
dataset_id, _ = add_documents
res = list_documents(HttpApiAuth, dataset_id, params=params)
assert res["code"] == 0
assert res["data"]["total"] == expected_size

1
test/testcases/test_http_api/test_file_management_within_dataset/test_parse_documents.py
View File

@@ -42,6 +42,7 @@ def condition(_auth, _dataset_id, _document_ids=None):
def validate_document_details(auth, dataset_id, document_ids):
# currently list_documents not support search by document id
for document_id in document_ids:
res = list_documents(auth, dataset_id, params={"id": document_id})
doc = res["data"]["docs"][0]

18
test/testcases/test_http_api/test_file_management_within_dataset/test_update_document.py
View File

@@ -42,7 +42,8 @@ class TestAuthorization:
class TestDocumentsUpdated:
@pytest.mark.p1
# GET /api/v1/datasets/<dataset_id>/documents no longer support find by id/name
@pytest.mark.p3
@pytest.mark.parametrize(
"name, expected_code, expected_message",
[
@@ -94,7 +95,8 @@ class TestDocumentsUpdated:
else:
assert res["message"] == expected_message
@pytest.mark.p2
# GET /api/v1/datasets/<dataset_id>/documents no longer support find by id/name
@pytest.mark.p3
@pytest.mark.parametrize(
"document_id, expected_code, expected_message",
[
@@ -206,10 +208,12 @@ class TestDocumentsUpdated:
assert res["code"] == expected_code
if expected_code == 0:
res = list_documents(HttpApiAuth, dataset_id, {"id": document_ids[0]})
doc_of_id = res["data"]["docs"][0]
if chunk_method == "":
assert res["data"]["docs"][0]["chunk_method"] == "naive"
assert doc_of_id["chunk_method"] == "naive"
else:
assert res["data"]["docs"][0]["chunk_method"] == chunk_method
print(f"doc:{doc_of_id}")
assert doc_of_id["chunk_method"] == chunk_method
else:
assert res["message"] == expected_message
@@ -597,10 +601,12 @@ class TestUpdateDocumentParserConfig:
assert res["code"] == expected_code
if expected_code == 0:
res = list_documents(HttpApiAuth, dataset_id, {"id": document_ids[0]})
doc_of_id = res["data"]["docs"][0]
if parser_config == {}:
assert res["data"]["docs"][0]["parser_config"] == DEFAULT_PARSER_CONFIG
assert doc_of_id["parser_config"] == DEFAULT_PARSER_CONFIG
else:
for k, v in parser_config.items():
assert res["data"]["docs"][0]["parser_config"][k] == v
assert doc_of_id["parser_config"][k] == v
if expected_code != 0 or expected_message:
assert res["message"] == expected_message

2
test/testcases/test_sdk_api/test_file_management_within_dataset/conftest.py
View File

@@ -37,7 +37,7 @@ def add_document_func(request: FixtureRequest, add_dataset: DataSet, ragflow_tmp
def add_documents(request: FixtureRequest, add_dataset: DataSet, ragflow_tmp_dir) -> tuple[DataSet, list[Document]]:
dataset = add_dataset
documents = bulk_upload_documents(dataset, 5, ragflow_tmp_dir)
def cleanup():
delete_all_documents(dataset)

6
test/testcases/test_sdk_api/test_file_management_within_dataset/test_list_documents.py
View File

@@ -30,7 +30,7 @@ class TestDocumentsList:
"params, expected_page_size, expected_message",
[
({"page": None, "page_size": 2}, 2, "not instance of"),
({"page": 0, "page_size": 2}, 2, ""),
({"page": 1, "page_size": 2}, 2, ""),
({"page": 2, "page_size": 2}, 2, ""),
({"page": 3, "page_size": 2}, 1, ""),
({"page": "3", "page_size": 2}, 1, "not instance of"),
@@ -63,7 +63,7 @@ class TestDocumentsList:
"params, expected_page_size, expected_message",
[
({"page_size": None}, 5, "not instance of"),
({"page_size": 0}, 0, ""),
({"page_size": 2}, 2, ""),
({"page_size": 1}, 1, ""),
({"page_size": 6}, 5, ""),
({"page_size": "1"}, 1, "not instance of"),
@@ -151,6 +151,7 @@ class TestDocumentsList:
documents = dataset.list_documents(**params)
assert len(documents) == expected_num, str(documents)
@pytest.mark.p1
@pytest.mark.parametrize(
"params, expected_num, expected_message",
@@ -222,6 +223,7 @@ class TestDocumentsList:
documents = dataset.list_documents(**params)
assert len(documents) == expected_num, str(documents)
@pytest.mark.p3
def test_concurrent_list(self, add_documents):
dataset, _ = add_documents

9
test/testcases/test_sdk_api/test_file_management_within_dataset/test_update_document.py
View File

@@ -64,7 +64,8 @@ class TestDocumentsUpdated:
assert expected_message in str(exception_info.value), str(exception_info.value)
else:
document.update({"name": name})
updated_doc = dataset.list_documents(id=document.id)[0]
docs = dataset.list_documents(id=document.id)
updated_doc = [doc for doc in docs if doc.id == document.id][0]
assert updated_doc.name == name, str(updated_doc)
@pytest.mark.p2
@@ -138,7 +139,8 @@ class TestDocumentsUpdated:
assert expected_message in str(exception_info.value), str(exception_info.value)
else:
document.update({"chunk_method": chunk_method})
updated_doc = dataset.list_documents(id=document.id)[0]
docs = dataset.list_documents()
updated_doc = [doc for doc in docs if doc.id == document.id][0]
assert updated_doc.chunk_method == chunk_method, str(updated_doc)
@pytest.mark.p3
@@ -479,7 +481,8 @@ class TestUpdateDocumentParserConfig:
assert expected_message in str(exception_info.value), str(exception_info.value)
else:
document.update(update_data)
updated_doc = dataset.list_documents(id=document.id)[0]
docs = dataset.list_documents(id=document.id)
updated_doc = [doc for doc in docs if doc.id == document.id][0]
if parser_config:
for k, v in parser_config.items():
if isinstance(v, dict):

4
test/testcases/test_web_api/conftest.py
View File

@@ -49,9 +49,9 @@ from utils.file_utils import (
@wait_for(30, 1, "Document parsing timeout")
def condition(_auth, _kb_id):
res = list_documents(_auth, {"id": _kb_id})
res = list_documents(_auth, {"kb_id": _kb_id})
for doc in res["data"]["docs"]:
if doc["run"] != "3":
if doc["run"] != "DONE":
return False
return True

4
test/testcases/test_web_api/test_chunk_app/conftest.py
View File

@@ -24,9 +24,9 @@ from utils import wait_for
@wait_for(30, 1, "Document parsing timeout")
def condition(_auth, _kb_id):
res = list_documents(_auth, {"id": _kb_id})
res = list_documents(_auth, {"kb_id": _kb_id})
for doc in res["data"]["docs"]:
if doc["run"] != "3":
if doc["run"] != "DONE":
return False
return True

4
test/testcases/test_web_api/test_common.py
View File

@@ -374,9 +374,11 @@ def create_document(auth, payload=None, *, headers=HEADERS, data=None):
def list_documents(auth, params=None, payload=None, *, headers=HEADERS, data=None):
kb_id = params.get("kb_id") if params else None
url = f"{HOST_ADDRESS}/api/{VERSION}/datasets/{kb_id}/documents"
if payload is None:
payload = {}
res = requests.post(url=f"{HOST_ADDRESS}{DOCUMENT_APP_URL}/list", headers=headers, auth=auth, params=params, json=payload, data=data)
res = requests.get(url=url, headers=headers, auth=auth, params=params, json=payload, data=data)
return res.json()

6
test/testcases/test_web_api/test_document_app/conftest.py
View File

@@ -34,7 +34,7 @@ class _DummyManager:
@pytest.fixture(scope="function")
def add_document_func(request, WebApiAuth, add_dataset, ragflow_tmp_dir):
def cleanup():
res = list_documents(WebApiAuth, {"id": dataset_id})
res = list_documents(WebApiAuth, {"kb_id": dataset_id})
for doc in res["data"]["docs"]:
delete_document(WebApiAuth, {"doc_id": doc["id"]})
@@ -47,7 +47,7 @@ def add_document_func(request, WebApiAuth, add_dataset, ragflow_tmp_dir):
@pytest.fixture(scope="class")
def add_documents(request, WebApiAuth, add_dataset, ragflow_tmp_dir):
def cleanup():
res = list_documents(WebApiAuth, {"id": dataset_id})
res = list_documents(WebApiAuth, {"kb_id": dataset_id})
for doc in res["data"]["docs"]:
delete_document(WebApiAuth, {"doc_id": doc["id"]})
@@ -60,7 +60,7 @@ def add_documents(request, WebApiAuth, add_dataset, ragflow_tmp_dir):
@pytest.fixture(scope="function")
def add_documents_func(request, WebApiAuth, add_dataset_func, ragflow_tmp_dir):
def cleanup():
res = list_documents(WebApiAuth, {"id": dataset_id})
res = list_documents(WebApiAuth, {"kb_id": dataset_id})
for doc in res["data"]["docs"]:
delete_document(WebApiAuth, {"doc_id": doc["id"]})

2
test/testcases/test_web_api/test_document_app/test_create_document.py
View File

@@ -75,7 +75,7 @@ class TestDocumentCreate:
res = create_document(WebApiAuth, {"name": filename, "kb_id": kb_id})
assert res["code"] == 0, res
assert res["data"]["kb_id"] == kb_id, res
assert res["data"]["dataset_id"] == kb_id, res
assert res["data"]["name"] == filename, f"Expected: {filename}, Got: {res['data']['name']}"
@pytest.mark.p3

280
test/testcases/test_web_api/test_document_app/test_list_documents.py
View File

@@ -13,9 +13,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
#
import asyncio
from concurrent.futures import ThreadPoolExecutor, as_completed
from types import SimpleNamespace
import pytest
from test_common import list_documents
@@ -43,30 +41,18 @@ class TestDocumentsList:
@pytest.mark.p1
def test_default(self, WebApiAuth, add_documents):
kb_id, _ = add_documents
res = list_documents(WebApiAuth, {"id": kb_id})
assert res["code"] == 0
res = list_documents(WebApiAuth, {"kb_id": kb_id})
assert res["code"] == 0, f", kb_id:{kb_id} +, res:{str(res)}"
assert len(res["data"]["docs"]) == 5
assert res["data"]["total"] == 5
@pytest.mark.p3
@pytest.mark.parametrize(
"kb_id, expected_code, expected_message",
[
("", 101, 'Lack of "KB ID"'),
("invalid_dataset_id", 103, "Only owner of dataset authorized for this operation."),
],
)
def test_invalid_dataset_id(self, WebApiAuth, kb_id, expected_code, expected_message):
res = list_documents(WebApiAuth, {"id": kb_id})
assert res["code"] == expected_code
assert res["message"] == expected_message
@pytest.mark.p1
@pytest.mark.parametrize(
"params, expected_code, expected_page_size, expected_message",
[
({"page": None, "page_size": 2}, 0, 5, ""),
({"page": 0, "page_size": 2}, 0, 5, ""),
({"page": None, "page_size": 5}, 0, 5, ""),
({"page": 0, "page_size": 5}, 0, 5, ""),
({"page": 2, "page_size": 2}, 0, 2, ""),
({"page": 3, "page_size": 2}, 0, 1, ""),
({"page": "3", "page_size": 2}, 0, 1, ""),
@@ -76,7 +62,7 @@ class TestDocumentsList:
)
def test_page(self, WebApiAuth, add_documents, params, expected_code, expected_page_size, expected_message):
kb_id, _ = add_documents
res = list_documents(WebApiAuth, {"id": kb_id, **params})
res = list_documents(WebApiAuth, {"kb_id": kb_id, **params})
assert res["code"] == expected_code, res
if expected_code == 0:
assert len(res["data"]["docs"]) == expected_page_size, res
@@ -89,17 +75,17 @@ class TestDocumentsList:
"params, expected_code, expected_page_size, expected_message",
[
({"page_size": None}, 0, 5, ""),
({"page_size": 0}, 0, 5, ""),
({"page_size": 1}, 0, 5, ""),
({"page_size": 5}, 0, 5, ""),
({"page_size": 1}, 0, 1, ""),
({"page_size": 6}, 0, 5, ""),
({"page_size": "1"}, 0, 5, ""),
({"page_size": "1"}, 0, 1, ""),
pytest.param({"page_size": -1}, 100, 0, "1064", marks=pytest.mark.skip(reason="issues/5851")),
pytest.param({"page_size": "a"}, 100, 0, """ValueError("invalid literal for int() with base 10: 'a'")""", marks=pytest.mark.skip(reason="issues/5851")),
],
)
def test_page_size(self, WebApiAuth, add_documents, params, expected_code, expected_page_size, expected_message):
kb_id, _ = add_documents
res = list_documents(WebApiAuth, {"id": kb_id, **params})
res = list_documents(WebApiAuth, {"kb_id": kb_id, **params})
assert res["code"] == expected_code, res
if expected_code == 0:
assert len(res["data"]["docs"]) == expected_page_size, res
@@ -119,7 +105,7 @@ class TestDocumentsList:
)
def test_orderby(self, WebApiAuth, add_documents, params, expected_code, assertions, expected_message):
kb_id, _ = add_documents
res = list_documents(WebApiAuth, {"id": kb_id, **params})
res = list_documents(WebApiAuth, {"kb_id": kb_id, **params})
assert res["code"] == expected_code, res
if expected_code == 0:
if callable(assertions):
@@ -144,7 +130,7 @@ class TestDocumentsList:
)
def test_desc(self, WebApiAuth, add_documents, params, expected_code, assertions, expected_message):
kb_id, _ = add_documents
res = list_documents(WebApiAuth, {"id": kb_id, **params})
res = list_documents(WebApiAuth, {"kb_id": kb_id, **params})
assert res["code"] == expected_code, res
if expected_code == 0:
if callable(assertions):
@@ -165,7 +151,7 @@ class TestDocumentsList:
)
def test_keywords(self, WebApiAuth, add_documents, params, expected_num):
kb_id, _ = add_documents
res = list_documents(WebApiAuth, {"id": kb_id, **params})
res = list_documents(WebApiAuth, {"kb_id": kb_id, **params})
assert res["code"] == 0, res
assert len(res["data"]["docs"]) == expected_num, res
assert res["data"]["total"] == expected_num, res
@@ -181,213 +167,53 @@ class TestDocumentsList:
assert len(responses) == count, responses
assert all(future.result()["code"] == 0 for future in futures), responses
# Tests moved from TestDocumentsListUnit
@pytest.mark.p2
def test_missing_kb_id(self, WebApiAuth):
"""Test missing KB ID returns error."""
res = list_documents(WebApiAuth, {"kb_id": ""})
assert res["code"] == 100
assert res["message"] == "<MethodNotAllowed '405: Method Not Allowed'>"
def _run(coro):
return asyncio.run(coro)
@pytest.mark.p2
def test_unauthorized_dataset(self, WebApiAuth):
"""Test unauthorized dataset returns error."""
res = list_documents(WebApiAuth, {"kb_id": "non_existent_kb_id"})
assert res["code"] == 102
assert "You don't own the dataset" in res["message"]
class _DummyArgs(dict):
def get(self, key, default=None):
return super().get(key, default)
@pytest.mark.p2
class TestDocumentsListUnit:
def _set_args(self, module, monkeypatch, **kwargs):
monkeypatch.setattr(module, "request", SimpleNamespace(args=_DummyArgs(kwargs)))
def _allow_kb(self, module, monkeypatch, kb_id="kb1", tenant_id="tenant1"):
monkeypatch.setattr(module.UserTenantService, "query", lambda **_kwargs: [SimpleNamespace(tenant_id=tenant_id)])
monkeypatch.setattr(module.KnowledgebaseService, "query", lambda **_kwargs: True if _kwargs.get("id") == kb_id else False)
def test_missing_kb_id(self, document_app_module, monkeypatch):
module = document_app_module
self._set_args(module, monkeypatch)
async def fake_request_json():
return {}
monkeypatch.setattr(module, "get_request_json", fake_request_json)
res = _run(module.list_docs())
assert res["code"] == 101
assert res["message"] == 'Dataset ID is required for listing files.'
def test_unauthorized_dataset(self, document_app_module, monkeypatch):
module = document_app_module
self._set_args(module, monkeypatch, id="kb1")
monkeypatch.setattr(module.UserTenantService, "query", lambda **_kwargs: [SimpleNamespace(tenant_id="tenant1")])
monkeypatch.setattr(module.KnowledgebaseService, "query", lambda **_kwargs: False)
async def fake_request_json():
return {}
monkeypatch.setattr(module, "get_request_json", fake_request_json)
res = _run(module.list_docs())
assert res["code"] == 103
assert "Only owner of dataset" in res["message"]
def test_return_empty_metadata_flags(self, document_app_module, monkeypatch):
module = document_app_module
self._set_args(module, monkeypatch, id="kb1")
self._allow_kb(module, monkeypatch)
monkeypatch.setattr(module.DocumentService, "get_by_kb_id", lambda *_args, **_kwargs: ([], 0))
async def fake_request_json():
return {"return_empty_metadata": "true", "metadata": {"author": "alice"}}
monkeypatch.setattr(module, "get_request_json", fake_request_json)
res = _run(module.list_docs())
assert res["code"] == 0
async def fake_request_json_empty():
return {"metadata": {"empty_metadata": True, "author": "alice"}}
monkeypatch.setattr(module, "get_request_json", fake_request_json_empty)
res = _run(module.list_docs())
assert res["code"] == 0
def test_invalid_filters(self, document_app_module, monkeypatch):
module = document_app_module
self._set_args(module, monkeypatch, id="kb1")
self._allow_kb(module, monkeypatch)
async def fake_request_json():
return {"run_status": ["INVALID"]}
monkeypatch.setattr(module, "get_request_json", fake_request_json)
res = _run(module.list_docs())
@pytest.mark.p3
def test_invalid_run_status_filter(self, WebApiAuth, add_documents):
"""Test invalid run status filter returns error."""
kb_id, _ = add_documents
res = list_documents(WebApiAuth, {"kb_id": kb_id, "run": "INVALID"})
assert res["code"] == 102
assert "Invalid filter run status" in res["message"]
async def fake_request_json_types():
return {"types": ["INVALID"]}
monkeypatch.setattr(module, "get_request_json", fake_request_json_types)
res = _run(module.list_docs())
@pytest.mark.p3
def test_invalid_document_id_filter(self, WebApiAuth, add_documents):
"""Test invalid document ID filter returns error."""
kb_id, _ = add_documents
# Use a non-existent document ID
res = list_documents(WebApiAuth, {"kb_id": kb_id, "id": "non_existent_doc_id"})
assert res["code"] == 102
assert "Invalid filter conditions" in res["message"]
assert "You don't own the document" in res["message"]
def test_invalid_metadata_types(self, document_app_module, monkeypatch):
module = document_app_module
self._set_args(module, monkeypatch, id="kb1")
self._allow_kb(module, monkeypatch)
async def fake_request_json():
return {"metadata_condition": "bad"}
monkeypatch.setattr(module, "get_request_json", fake_request_json)
res = _run(module.list_docs())
assert res["code"] == 102
assert "metadata_condition" in res["message"]
async def fake_request_json_meta():
return {"metadata": ["not", "object"]}
monkeypatch.setattr(module, "get_request_json", fake_request_json_meta)
res = _run(module.list_docs())
assert res["code"] == 102
assert "metadata must be an object" in res["message"]
def test_metadata_condition_empty_result(self, document_app_module, monkeypatch):
module = document_app_module
self._set_args(module, monkeypatch, id="kb1")
self._allow_kb(module, monkeypatch)
monkeypatch.setattr(module.DocMetadataService, "get_flatted_meta_by_kbs", lambda *_args, **_kwargs: {})
monkeypatch.setattr(module, "meta_filter", lambda *_args, **_kwargs: set())
async def fake_request_json():
return {"metadata_condition": {"conditions": [{"name": "author", "comparison_operator": "is", "value": "alice"}]}}
monkeypatch.setattr(module, "get_request_json", fake_request_json)
res = _run(module.list_docs())
@pytest.mark.p3
def test_create_time_filter(self, WebApiAuth, add_documents):
"""Test create time range filter."""
kb_id, _ = add_documents
# Get current time range
res = list_documents(WebApiAuth, {"kb_id": kb_id})
assert res["code"] == 0
assert res["data"]["total"] == 0
if res["data"]["docs"]:
create_time = res["data"]["docs"][0].get("create_time", 0)
# Test with time range that should include the document
res = list_documents(WebApiAuth, {"kb_id": kb_id, "create_time_from": 0, "create_time_to": create_time + 1000})
assert res["code"] == 0
assert len(res["data"]["docs"]) > 0
# Test with time range that should not include the document
res = list_documents(WebApiAuth, {"kb_id": kb_id, "create_time_from": create_time + 1000, "create_time_to": create_time + 2000})
assert res["code"] == 0
assert len(res["data"]["docs"]) == 0
def test_metadata_values_intersection(self, document_app_module, monkeypatch):
module = document_app_module
self._set_args(module, monkeypatch, id="kb1")
self._allow_kb(module, monkeypatch)
metas = {
"author": {"alice": ["doc1", "doc2"]},
"topic": {"rag": ["doc2"]},
}
monkeypatch.setattr(module.DocMetadataService, "get_flatted_meta_by_kbs", lambda *_args, **_kwargs: metas)
captured = {}
def fake_get_by_kb_id(*_args, **_kwargs):
if len(_args) >= 10:
captured["doc_ids_filter"] = _args[9]
else:
captured["doc_ids_filter"] = None
return ([{"id": "doc2", "thumbnail": "", "parser_config": {}}], 1)
monkeypatch.setattr(module.DocumentService, "get_by_kb_id", fake_get_by_kb_id)
async def fake_request_json():
return {"metadata": {"author": ["alice", " ", None], "topic": "rag"}}
monkeypatch.setattr(module, "get_request_json", fake_request_json)
res = _run(module.list_docs())
assert res["code"] == 0
assert captured["doc_ids_filter"] == ["doc2"]
def test_metadata_intersection_empty(self, document_app_module, monkeypatch):
module = document_app_module
self._set_args(module, monkeypatch, id="kb1")
self._allow_kb(module, monkeypatch)
metas = {
"author": {"alice": ["doc1"]},
"topic": {"rag": ["doc2"]},
}
monkeypatch.setattr(module.DocMetadataService, "get_flatted_meta_by_kbs", lambda *_args, **_kwargs: metas)
async def fake_request_json():
return {"metadata": {"author": "alice", "topic": "rag"}}
monkeypatch.setattr(module, "get_request_json", fake_request_json)
res = _run(module.list_docs())
assert res["code"] == 0
assert res["data"]["total"] == 0
def test_desc_time_and_schema(self, document_app_module, monkeypatch):
module = document_app_module
self._set_args(module, monkeypatch, id="kb1", desc="false", create_time_from="150", create_time_to="250")
self._allow_kb(module, monkeypatch)
docs = [
{"id": "doc1", "thumbnail": "", "parser_config": {"metadata": {"a": 1}}, "create_time": 100},
{"id": "doc2", "thumbnail": "", "parser_config": {"metadata": {"b": 2}}, "create_time": 200},
]
def fake_get_by_kb_id(*_args, **_kwargs):
return (docs, 2)
monkeypatch.setattr(module.DocumentService, "get_by_kb_id", fake_get_by_kb_id)
monkeypatch.setattr(module, "turn2jsonschema", lambda _meta: {"schema": True})
async def fake_request_json():
return {}
monkeypatch.setattr(module, "get_request_json", fake_request_json)
res = _run(module.list_docs())
assert res["code"] == 0
assert len(res["data"]["docs"]) == 1
assert res["data"]["docs"][0]["parser_config"]["metadata"] == {"schema": True}
def test_exception_path(self, document_app_module, monkeypatch):
module = document_app_module
self._set_args(module, monkeypatch, id="kb1")
self._allow_kb(module, monkeypatch)
def raise_error(*_args, **_kwargs):
raise RuntimeError("boom")
monkeypatch.setattr(module.DocumentService, "get_by_kb_id", raise_error)
async def fake_request_json():
return {}
monkeypatch.setattr(module, "get_request_json", fake_request_json)
res = _run(module.list_docs())
assert res["code"] == 100

28
test/testcases/test_web_api/test_document_app/test_paser_documents.py
View File

@@ -30,29 +30,28 @@ def _run(coro):
@wait_for(30, 1, "Document parsing timeout")
def condition(_auth, _kb_id, _document_ids=None):
res = list_documents(_auth, {"id": _kb_id})
res = list_documents(_auth, {"kb_id": _kb_id})
target_docs = res["data"]["docs"]
if _document_ids is None:
for doc in target_docs:
if doc["run"] != "3":
if doc["run"] != "DONE":
return False
return True
target_ids = set(_document_ids)
for doc in target_docs:
if doc["id"] in target_ids:
if doc.get("run") != "3":
if doc.get("run") != "DONE":
return False
return True
def validate_document_parse_done(auth, _kb_id, _document_ids):
res = list_documents(auth, {"id": _kb_id})
res = list_documents(auth, {"kb_id": _kb_id})
for doc in res["data"]["docs"]:
if doc["id"] not in _document_ids:
continue
assert doc["run"] == "3"
assert doc["run"] == "DONE"
assert len(doc["process_begin_at"]) > 0
assert doc["process_duration"] > 0
assert doc["progress"] > 0
@@ -60,11 +59,11 @@ def validate_document_parse_done(auth, _kb_id, _document_ids):
def validate_document_parse_cancel(auth, _kb_id, _document_ids):
res = list_documents(auth, {"id": _kb_id})
res = list_documents(auth, {"kb_id": _kb_id})
for doc in res["data"]["docs"]:
if doc["id"] not in _document_ids:
continue
assert doc["run"] == "2"
assert doc["run"] == "CANCEL"
assert len(doc["process_begin_at"]) > 0
assert doc["progress"] == 0.0
@@ -151,9 +150,9 @@ class TestDocumentsParse:
def test_parse_100_files(WebApiAuth, add_dataset_func, tmp_path):
@wait_for(100, 1, "Document parsing timeout")
def condition(_auth, _kb_id, _document_num):
res = list_documents(_auth, {"id": _kb_id, "page_size": _document_num})
res = list_documents(_auth, {"kb_id": _kb_id, "page_size": _document_num})
for doc in res["data"]["docs"]:
if doc["run"] != "3":
if doc["run"] != "DONE":
return False
return True
@@ -172,7 +171,7 @@ def test_parse_100_files(WebApiAuth, add_dataset_func, tmp_path):
def test_concurrent_parse(WebApiAuth, add_dataset_func, tmp_path):
@wait_for(120, 1, "Document parsing timeout")
def condition(_auth, _kb_id, _document_num):
res = list_documents(_auth, {"id": _kb_id, "page_size": _document_num})
res = list_documents(_auth, {"kb_id": _kb_id, "page_size": _document_num})
for doc in res["data"]["docs"]:
if doc["run"] != "3":
return False
@@ -305,15 +304,16 @@ class TestDocumentsParseStop:
def test_basic_scenarios(self, WebApiAuth, add_documents_func, payload, expected_code, expected_message):
@wait_for(30, 1, "Document parsing timeout")
def condition(_auth, _kb_id, _doc_ids):
res = list_documents(_auth, {"id": _kb_id})
res = list_documents(_auth, {"kb_id": _kb_id})
for doc in res["data"]["docs"]:
if doc["id"] in _doc_ids:
if doc["run"] != "3":
if doc["run"] != "DONE":
return False
return True
kb_id, document_ids = add_documents_func
parse_documents(WebApiAuth, {"doc_ids": document_ids, "run": "1"})
parse_documents(WebApiAuth, {"doc_ids": document_ids, "run":
"1"})
if callable(payload):
payload = payload(document_ids)

6
test/testcases/test_web_api/test_document_app/test_rm_documents.py
View File

@@ -63,7 +63,7 @@ class TestDocumentsDeletion:
if res["code"] != 0:
assert res["message"] == expected_message, res
res = list_documents(WebApiAuth, {"id": kb_id})
res = list_documents(WebApiAuth, {"kb_id": kb_id})
assert len(res["data"]["docs"]) == remaining, res
assert res["data"]["total"] == remaining, res
@@ -124,12 +124,12 @@ def test_delete_100(WebApiAuth, add_dataset, tmp_path):
documents_num = 100
kb_id = add_dataset
document_ids = bulk_upload_documents(WebApiAuth, kb_id, documents_num, tmp_path)
res = list_documents(WebApiAuth, {"id": kb_id})
res = list_documents(WebApiAuth, {"kb_id": kb_id})
assert res["data"]["total"] == documents_num, res
for doc_id in document_ids:
res = delete_document(WebApiAuth, {"doc_id": doc_id})
assert res["code"] == 0, res
res = list_documents(WebApiAuth, {"id": kb_id})
res = list_documents(WebApiAuth, {"kb_id": kb_id})
assert res["data"]["total"] == 0, res

2
test/testcases/test_web_api/test_kb_app/test_kb_pipeline_tasks.py
View File

@@ -75,7 +75,7 @@ def _wait_for_task(trace_func, auth, kb_id, task_id, timeout=60, use_params_payl
def _wait_for_docs_parsed(auth, kb_id, timeout=60):
@wait_for(timeout, 2, "Document parsing timeout")
def _condition():
res = list_documents(auth, {"id": kb_id})
res = list_documents(auth, {"kb_id": kb_id})
if res["code"] != 0:
return False
for doc in res["data"]["docs"]: