mirror of
https://github.com/infiniflow/ragflow.git
synced 2026-07-27 02:42:00 +08:00
Refactor: Consolidation WEB API & HTTP API for document list_docs (#14176)
### What problem does this PR solve? Before consolidation Web API: POST /v1/document/list Http API - GET /api/v1/datasets/<dataset_id>/documents After consolidation, Restful API -- GET /api/v1/datasets/<dataset_id>/documents ### Type of change - [x] Refactoring
This commit is contained in:
@@ -49,9 +49,9 @@ from utils.file_utils import (
|
||||
|
||||
@wait_for(30, 1, "Document parsing timeout")
|
||||
def condition(_auth, _kb_id):
|
||||
res = list_documents(_auth, {"id": _kb_id})
|
||||
res = list_documents(_auth, {"kb_id": _kb_id})
|
||||
for doc in res["data"]["docs"]:
|
||||
if doc["run"] != "3":
|
||||
if doc["run"] != "DONE":
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
@@ -24,9 +24,9 @@ from utils import wait_for
|
||||
|
||||
@wait_for(30, 1, "Document parsing timeout")
|
||||
def condition(_auth, _kb_id):
|
||||
res = list_documents(_auth, {"id": _kb_id})
|
||||
res = list_documents(_auth, {"kb_id": _kb_id})
|
||||
for doc in res["data"]["docs"]:
|
||||
if doc["run"] != "3":
|
||||
if doc["run"] != "DONE":
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
@@ -374,9 +374,11 @@ def create_document(auth, payload=None, *, headers=HEADERS, data=None):
|
||||
|
||||
|
||||
def list_documents(auth, params=None, payload=None, *, headers=HEADERS, data=None):
|
||||
kb_id = params.get("kb_id") if params else None
|
||||
url = f"{HOST_ADDRESS}/api/{VERSION}/datasets/{kb_id}/documents"
|
||||
if payload is None:
|
||||
payload = {}
|
||||
res = requests.post(url=f"{HOST_ADDRESS}{DOCUMENT_APP_URL}/list", headers=headers, auth=auth, params=params, json=payload, data=data)
|
||||
res = requests.get(url=url, headers=headers, auth=auth, params=params, json=payload, data=data)
|
||||
return res.json()
|
||||
|
||||
|
||||
|
||||
@@ -34,7 +34,7 @@ class _DummyManager:
|
||||
@pytest.fixture(scope="function")
|
||||
def add_document_func(request, WebApiAuth, add_dataset, ragflow_tmp_dir):
|
||||
def cleanup():
|
||||
res = list_documents(WebApiAuth, {"id": dataset_id})
|
||||
res = list_documents(WebApiAuth, {"kb_id": dataset_id})
|
||||
for doc in res["data"]["docs"]:
|
||||
delete_document(WebApiAuth, {"doc_id": doc["id"]})
|
||||
|
||||
@@ -47,7 +47,7 @@ def add_document_func(request, WebApiAuth, add_dataset, ragflow_tmp_dir):
|
||||
@pytest.fixture(scope="class")
|
||||
def add_documents(request, WebApiAuth, add_dataset, ragflow_tmp_dir):
|
||||
def cleanup():
|
||||
res = list_documents(WebApiAuth, {"id": dataset_id})
|
||||
res = list_documents(WebApiAuth, {"kb_id": dataset_id})
|
||||
for doc in res["data"]["docs"]:
|
||||
delete_document(WebApiAuth, {"doc_id": doc["id"]})
|
||||
|
||||
@@ -60,7 +60,7 @@ def add_documents(request, WebApiAuth, add_dataset, ragflow_tmp_dir):
|
||||
@pytest.fixture(scope="function")
|
||||
def add_documents_func(request, WebApiAuth, add_dataset_func, ragflow_tmp_dir):
|
||||
def cleanup():
|
||||
res = list_documents(WebApiAuth, {"id": dataset_id})
|
||||
res = list_documents(WebApiAuth, {"kb_id": dataset_id})
|
||||
for doc in res["data"]["docs"]:
|
||||
delete_document(WebApiAuth, {"doc_id": doc["id"]})
|
||||
|
||||
|
||||
@@ -75,7 +75,7 @@ class TestDocumentCreate:
|
||||
|
||||
res = create_document(WebApiAuth, {"name": filename, "kb_id": kb_id})
|
||||
assert res["code"] == 0, res
|
||||
assert res["data"]["kb_id"] == kb_id, res
|
||||
assert res["data"]["dataset_id"] == kb_id, res
|
||||
assert res["data"]["name"] == filename, f"Expected: {filename}, Got: {res['data']['name']}"
|
||||
|
||||
@pytest.mark.p3
|
||||
|
||||
@@ -13,9 +13,7 @@
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
#
|
||||
import asyncio
|
||||
from concurrent.futures import ThreadPoolExecutor, as_completed
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
from test_common import list_documents
|
||||
@@ -43,30 +41,18 @@ class TestDocumentsList:
|
||||
@pytest.mark.p1
|
||||
def test_default(self, WebApiAuth, add_documents):
|
||||
kb_id, _ = add_documents
|
||||
res = list_documents(WebApiAuth, {"id": kb_id})
|
||||
assert res["code"] == 0
|
||||
res = list_documents(WebApiAuth, {"kb_id": kb_id})
|
||||
assert res["code"] == 0, f", kb_id:{kb_id} +, res:{str(res)}"
|
||||
assert len(res["data"]["docs"]) == 5
|
||||
assert res["data"]["total"] == 5
|
||||
|
||||
@pytest.mark.p3
|
||||
@pytest.mark.parametrize(
|
||||
"kb_id, expected_code, expected_message",
|
||||
[
|
||||
("", 101, 'Lack of "KB ID"'),
|
||||
("invalid_dataset_id", 103, "Only owner of dataset authorized for this operation."),
|
||||
],
|
||||
)
|
||||
def test_invalid_dataset_id(self, WebApiAuth, kb_id, expected_code, expected_message):
|
||||
res = list_documents(WebApiAuth, {"id": kb_id})
|
||||
assert res["code"] == expected_code
|
||||
assert res["message"] == expected_message
|
||||
|
||||
@pytest.mark.p1
|
||||
@pytest.mark.parametrize(
|
||||
"params, expected_code, expected_page_size, expected_message",
|
||||
[
|
||||
({"page": None, "page_size": 2}, 0, 5, ""),
|
||||
({"page": 0, "page_size": 2}, 0, 5, ""),
|
||||
({"page": None, "page_size": 5}, 0, 5, ""),
|
||||
({"page": 0, "page_size": 5}, 0, 5, ""),
|
||||
({"page": 2, "page_size": 2}, 0, 2, ""),
|
||||
({"page": 3, "page_size": 2}, 0, 1, ""),
|
||||
({"page": "3", "page_size": 2}, 0, 1, ""),
|
||||
@@ -76,7 +62,7 @@ class TestDocumentsList:
|
||||
)
|
||||
def test_page(self, WebApiAuth, add_documents, params, expected_code, expected_page_size, expected_message):
|
||||
kb_id, _ = add_documents
|
||||
res = list_documents(WebApiAuth, {"id": kb_id, **params})
|
||||
res = list_documents(WebApiAuth, {"kb_id": kb_id, **params})
|
||||
assert res["code"] == expected_code, res
|
||||
if expected_code == 0:
|
||||
assert len(res["data"]["docs"]) == expected_page_size, res
|
||||
@@ -89,17 +75,17 @@ class TestDocumentsList:
|
||||
"params, expected_code, expected_page_size, expected_message",
|
||||
[
|
||||
({"page_size": None}, 0, 5, ""),
|
||||
({"page_size": 0}, 0, 5, ""),
|
||||
({"page_size": 1}, 0, 5, ""),
|
||||
({"page_size": 5}, 0, 5, ""),
|
||||
({"page_size": 1}, 0, 1, ""),
|
||||
({"page_size": 6}, 0, 5, ""),
|
||||
({"page_size": "1"}, 0, 5, ""),
|
||||
({"page_size": "1"}, 0, 1, ""),
|
||||
pytest.param({"page_size": -1}, 100, 0, "1064", marks=pytest.mark.skip(reason="issues/5851")),
|
||||
pytest.param({"page_size": "a"}, 100, 0, """ValueError("invalid literal for int() with base 10: 'a'")""", marks=pytest.mark.skip(reason="issues/5851")),
|
||||
],
|
||||
)
|
||||
def test_page_size(self, WebApiAuth, add_documents, params, expected_code, expected_page_size, expected_message):
|
||||
kb_id, _ = add_documents
|
||||
res = list_documents(WebApiAuth, {"id": kb_id, **params})
|
||||
res = list_documents(WebApiAuth, {"kb_id": kb_id, **params})
|
||||
assert res["code"] == expected_code, res
|
||||
if expected_code == 0:
|
||||
assert len(res["data"]["docs"]) == expected_page_size, res
|
||||
@@ -119,7 +105,7 @@ class TestDocumentsList:
|
||||
)
|
||||
def test_orderby(self, WebApiAuth, add_documents, params, expected_code, assertions, expected_message):
|
||||
kb_id, _ = add_documents
|
||||
res = list_documents(WebApiAuth, {"id": kb_id, **params})
|
||||
res = list_documents(WebApiAuth, {"kb_id": kb_id, **params})
|
||||
assert res["code"] == expected_code, res
|
||||
if expected_code == 0:
|
||||
if callable(assertions):
|
||||
@@ -144,7 +130,7 @@ class TestDocumentsList:
|
||||
)
|
||||
def test_desc(self, WebApiAuth, add_documents, params, expected_code, assertions, expected_message):
|
||||
kb_id, _ = add_documents
|
||||
res = list_documents(WebApiAuth, {"id": kb_id, **params})
|
||||
res = list_documents(WebApiAuth, {"kb_id": kb_id, **params})
|
||||
assert res["code"] == expected_code, res
|
||||
if expected_code == 0:
|
||||
if callable(assertions):
|
||||
@@ -165,7 +151,7 @@ class TestDocumentsList:
|
||||
)
|
||||
def test_keywords(self, WebApiAuth, add_documents, params, expected_num):
|
||||
kb_id, _ = add_documents
|
||||
res = list_documents(WebApiAuth, {"id": kb_id, **params})
|
||||
res = list_documents(WebApiAuth, {"kb_id": kb_id, **params})
|
||||
assert res["code"] == 0, res
|
||||
assert len(res["data"]["docs"]) == expected_num, res
|
||||
assert res["data"]["total"] == expected_num, res
|
||||
@@ -181,213 +167,53 @@ class TestDocumentsList:
|
||||
assert len(responses) == count, responses
|
||||
assert all(future.result()["code"] == 0 for future in futures), responses
|
||||
|
||||
# Tests moved from TestDocumentsListUnit
|
||||
@pytest.mark.p2
|
||||
def test_missing_kb_id(self, WebApiAuth):
|
||||
"""Test missing KB ID returns error."""
|
||||
res = list_documents(WebApiAuth, {"kb_id": ""})
|
||||
assert res["code"] == 100
|
||||
assert res["message"] == "<MethodNotAllowed '405: Method Not Allowed'>"
|
||||
|
||||
def _run(coro):
|
||||
return asyncio.run(coro)
|
||||
@pytest.mark.p2
|
||||
def test_unauthorized_dataset(self, WebApiAuth):
|
||||
"""Test unauthorized dataset returns error."""
|
||||
res = list_documents(WebApiAuth, {"kb_id": "non_existent_kb_id"})
|
||||
assert res["code"] == 102
|
||||
assert "You don't own the dataset" in res["message"]
|
||||
|
||||
|
||||
class _DummyArgs(dict):
|
||||
def get(self, key, default=None):
|
||||
return super().get(key, default)
|
||||
|
||||
|
||||
@pytest.mark.p2
|
||||
class TestDocumentsListUnit:
|
||||
def _set_args(self, module, monkeypatch, **kwargs):
|
||||
monkeypatch.setattr(module, "request", SimpleNamespace(args=_DummyArgs(kwargs)))
|
||||
|
||||
def _allow_kb(self, module, monkeypatch, kb_id="kb1", tenant_id="tenant1"):
|
||||
monkeypatch.setattr(module.UserTenantService, "query", lambda **_kwargs: [SimpleNamespace(tenant_id=tenant_id)])
|
||||
monkeypatch.setattr(module.KnowledgebaseService, "query", lambda **_kwargs: True if _kwargs.get("id") == kb_id else False)
|
||||
|
||||
def test_missing_kb_id(self, document_app_module, monkeypatch):
|
||||
module = document_app_module
|
||||
self._set_args(module, monkeypatch)
|
||||
|
||||
async def fake_request_json():
|
||||
return {}
|
||||
|
||||
monkeypatch.setattr(module, "get_request_json", fake_request_json)
|
||||
res = _run(module.list_docs())
|
||||
assert res["code"] == 101
|
||||
assert res["message"] == 'Dataset ID is required for listing files.'
|
||||
|
||||
def test_unauthorized_dataset(self, document_app_module, monkeypatch):
|
||||
module = document_app_module
|
||||
self._set_args(module, monkeypatch, id="kb1")
|
||||
monkeypatch.setattr(module.UserTenantService, "query", lambda **_kwargs: [SimpleNamespace(tenant_id="tenant1")])
|
||||
monkeypatch.setattr(module.KnowledgebaseService, "query", lambda **_kwargs: False)
|
||||
|
||||
async def fake_request_json():
|
||||
return {}
|
||||
|
||||
monkeypatch.setattr(module, "get_request_json", fake_request_json)
|
||||
res = _run(module.list_docs())
|
||||
assert res["code"] == 103
|
||||
assert "Only owner of dataset" in res["message"]
|
||||
|
||||
def test_return_empty_metadata_flags(self, document_app_module, monkeypatch):
|
||||
module = document_app_module
|
||||
self._set_args(module, monkeypatch, id="kb1")
|
||||
self._allow_kb(module, monkeypatch)
|
||||
monkeypatch.setattr(module.DocumentService, "get_by_kb_id", lambda *_args, **_kwargs: ([], 0))
|
||||
|
||||
async def fake_request_json():
|
||||
return {"return_empty_metadata": "true", "metadata": {"author": "alice"}}
|
||||
|
||||
monkeypatch.setattr(module, "get_request_json", fake_request_json)
|
||||
res = _run(module.list_docs())
|
||||
assert res["code"] == 0
|
||||
|
||||
async def fake_request_json_empty():
|
||||
return {"metadata": {"empty_metadata": True, "author": "alice"}}
|
||||
|
||||
monkeypatch.setattr(module, "get_request_json", fake_request_json_empty)
|
||||
res = _run(module.list_docs())
|
||||
assert res["code"] == 0
|
||||
|
||||
def test_invalid_filters(self, document_app_module, monkeypatch):
|
||||
module = document_app_module
|
||||
self._set_args(module, monkeypatch, id="kb1")
|
||||
self._allow_kb(module, monkeypatch)
|
||||
|
||||
async def fake_request_json():
|
||||
return {"run_status": ["INVALID"]}
|
||||
|
||||
monkeypatch.setattr(module, "get_request_json", fake_request_json)
|
||||
res = _run(module.list_docs())
|
||||
@pytest.mark.p3
|
||||
def test_invalid_run_status_filter(self, WebApiAuth, add_documents):
|
||||
"""Test invalid run status filter returns error."""
|
||||
kb_id, _ = add_documents
|
||||
res = list_documents(WebApiAuth, {"kb_id": kb_id, "run": "INVALID"})
|
||||
assert res["code"] == 102
|
||||
assert "Invalid filter run status" in res["message"]
|
||||
|
||||
async def fake_request_json_types():
|
||||
return {"types": ["INVALID"]}
|
||||
|
||||
monkeypatch.setattr(module, "get_request_json", fake_request_json_types)
|
||||
res = _run(module.list_docs())
|
||||
@pytest.mark.p3
|
||||
def test_invalid_document_id_filter(self, WebApiAuth, add_documents):
|
||||
"""Test invalid document ID filter returns error."""
|
||||
kb_id, _ = add_documents
|
||||
# Use a non-existent document ID
|
||||
res = list_documents(WebApiAuth, {"kb_id": kb_id, "id": "non_existent_doc_id"})
|
||||
assert res["code"] == 102
|
||||
assert "Invalid filter conditions" in res["message"]
|
||||
assert "You don't own the document" in res["message"]
|
||||
|
||||
def test_invalid_metadata_types(self, document_app_module, monkeypatch):
|
||||
module = document_app_module
|
||||
self._set_args(module, monkeypatch, id="kb1")
|
||||
self._allow_kb(module, monkeypatch)
|
||||
|
||||
async def fake_request_json():
|
||||
return {"metadata_condition": "bad"}
|
||||
|
||||
monkeypatch.setattr(module, "get_request_json", fake_request_json)
|
||||
res = _run(module.list_docs())
|
||||
assert res["code"] == 102
|
||||
assert "metadata_condition" in res["message"]
|
||||
|
||||
async def fake_request_json_meta():
|
||||
return {"metadata": ["not", "object"]}
|
||||
|
||||
monkeypatch.setattr(module, "get_request_json", fake_request_json_meta)
|
||||
res = _run(module.list_docs())
|
||||
assert res["code"] == 102
|
||||
assert "metadata must be an object" in res["message"]
|
||||
|
||||
def test_metadata_condition_empty_result(self, document_app_module, monkeypatch):
|
||||
module = document_app_module
|
||||
self._set_args(module, monkeypatch, id="kb1")
|
||||
self._allow_kb(module, monkeypatch)
|
||||
monkeypatch.setattr(module.DocMetadataService, "get_flatted_meta_by_kbs", lambda *_args, **_kwargs: {})
|
||||
monkeypatch.setattr(module, "meta_filter", lambda *_args, **_kwargs: set())
|
||||
|
||||
async def fake_request_json():
|
||||
return {"metadata_condition": {"conditions": [{"name": "author", "comparison_operator": "is", "value": "alice"}]}}
|
||||
|
||||
monkeypatch.setattr(module, "get_request_json", fake_request_json)
|
||||
res = _run(module.list_docs())
|
||||
@pytest.mark.p3
|
||||
def test_create_time_filter(self, WebApiAuth, add_documents):
|
||||
"""Test create time range filter."""
|
||||
kb_id, _ = add_documents
|
||||
# Get current time range
|
||||
res = list_documents(WebApiAuth, {"kb_id": kb_id})
|
||||
assert res["code"] == 0
|
||||
assert res["data"]["total"] == 0
|
||||
if res["data"]["docs"]:
|
||||
create_time = res["data"]["docs"][0].get("create_time", 0)
|
||||
# Test with time range that should include the document
|
||||
res = list_documents(WebApiAuth, {"kb_id": kb_id, "create_time_from": 0, "create_time_to": create_time + 1000})
|
||||
assert res["code"] == 0
|
||||
assert len(res["data"]["docs"]) > 0
|
||||
# Test with time range that should not include the document
|
||||
res = list_documents(WebApiAuth, {"kb_id": kb_id, "create_time_from": create_time + 1000, "create_time_to": create_time + 2000})
|
||||
assert res["code"] == 0
|
||||
assert len(res["data"]["docs"]) == 0
|
||||
|
||||
def test_metadata_values_intersection(self, document_app_module, monkeypatch):
|
||||
module = document_app_module
|
||||
self._set_args(module, monkeypatch, id="kb1")
|
||||
self._allow_kb(module, monkeypatch)
|
||||
metas = {
|
||||
"author": {"alice": ["doc1", "doc2"]},
|
||||
"topic": {"rag": ["doc2"]},
|
||||
}
|
||||
monkeypatch.setattr(module.DocMetadataService, "get_flatted_meta_by_kbs", lambda *_args, **_kwargs: metas)
|
||||
|
||||
captured = {}
|
||||
|
||||
def fake_get_by_kb_id(*_args, **_kwargs):
|
||||
if len(_args) >= 10:
|
||||
captured["doc_ids_filter"] = _args[9]
|
||||
else:
|
||||
captured["doc_ids_filter"] = None
|
||||
return ([{"id": "doc2", "thumbnail": "", "parser_config": {}}], 1)
|
||||
|
||||
monkeypatch.setattr(module.DocumentService, "get_by_kb_id", fake_get_by_kb_id)
|
||||
|
||||
async def fake_request_json():
|
||||
return {"metadata": {"author": ["alice", " ", None], "topic": "rag"}}
|
||||
|
||||
monkeypatch.setattr(module, "get_request_json", fake_request_json)
|
||||
res = _run(module.list_docs())
|
||||
assert res["code"] == 0
|
||||
assert captured["doc_ids_filter"] == ["doc2"]
|
||||
|
||||
def test_metadata_intersection_empty(self, document_app_module, monkeypatch):
|
||||
module = document_app_module
|
||||
self._set_args(module, monkeypatch, id="kb1")
|
||||
self._allow_kb(module, monkeypatch)
|
||||
metas = {
|
||||
"author": {"alice": ["doc1"]},
|
||||
"topic": {"rag": ["doc2"]},
|
||||
}
|
||||
monkeypatch.setattr(module.DocMetadataService, "get_flatted_meta_by_kbs", lambda *_args, **_kwargs: metas)
|
||||
|
||||
async def fake_request_json():
|
||||
return {"metadata": {"author": "alice", "topic": "rag"}}
|
||||
|
||||
monkeypatch.setattr(module, "get_request_json", fake_request_json)
|
||||
res = _run(module.list_docs())
|
||||
assert res["code"] == 0
|
||||
assert res["data"]["total"] == 0
|
||||
|
||||
def test_desc_time_and_schema(self, document_app_module, monkeypatch):
|
||||
module = document_app_module
|
||||
self._set_args(module, monkeypatch, id="kb1", desc="false", create_time_from="150", create_time_to="250")
|
||||
self._allow_kb(module, monkeypatch)
|
||||
|
||||
docs = [
|
||||
{"id": "doc1", "thumbnail": "", "parser_config": {"metadata": {"a": 1}}, "create_time": 100},
|
||||
{"id": "doc2", "thumbnail": "", "parser_config": {"metadata": {"b": 2}}, "create_time": 200},
|
||||
]
|
||||
|
||||
def fake_get_by_kb_id(*_args, **_kwargs):
|
||||
return (docs, 2)
|
||||
|
||||
monkeypatch.setattr(module.DocumentService, "get_by_kb_id", fake_get_by_kb_id)
|
||||
monkeypatch.setattr(module, "turn2jsonschema", lambda _meta: {"schema": True})
|
||||
|
||||
async def fake_request_json():
|
||||
return {}
|
||||
|
||||
monkeypatch.setattr(module, "get_request_json", fake_request_json)
|
||||
res = _run(module.list_docs())
|
||||
assert res["code"] == 0
|
||||
assert len(res["data"]["docs"]) == 1
|
||||
assert res["data"]["docs"][0]["parser_config"]["metadata"] == {"schema": True}
|
||||
|
||||
def test_exception_path(self, document_app_module, monkeypatch):
|
||||
module = document_app_module
|
||||
self._set_args(module, monkeypatch, id="kb1")
|
||||
self._allow_kb(module, monkeypatch)
|
||||
|
||||
def raise_error(*_args, **_kwargs):
|
||||
raise RuntimeError("boom")
|
||||
|
||||
monkeypatch.setattr(module.DocumentService, "get_by_kb_id", raise_error)
|
||||
|
||||
async def fake_request_json():
|
||||
return {}
|
||||
|
||||
monkeypatch.setattr(module, "get_request_json", fake_request_json)
|
||||
res = _run(module.list_docs())
|
||||
assert res["code"] == 100
|
||||
|
||||
@@ -30,29 +30,28 @@ def _run(coro):
|
||||
|
||||
@wait_for(30, 1, "Document parsing timeout")
|
||||
def condition(_auth, _kb_id, _document_ids=None):
|
||||
res = list_documents(_auth, {"id": _kb_id})
|
||||
res = list_documents(_auth, {"kb_id": _kb_id})
|
||||
target_docs = res["data"]["docs"]
|
||||
|
||||
if _document_ids is None:
|
||||
for doc in target_docs:
|
||||
if doc["run"] != "3":
|
||||
if doc["run"] != "DONE":
|
||||
return False
|
||||
return True
|
||||
|
||||
target_ids = set(_document_ids)
|
||||
for doc in target_docs:
|
||||
if doc["id"] in target_ids:
|
||||
if doc.get("run") != "3":
|
||||
if doc.get("run") != "DONE":
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
def validate_document_parse_done(auth, _kb_id, _document_ids):
|
||||
res = list_documents(auth, {"id": _kb_id})
|
||||
res = list_documents(auth, {"kb_id": _kb_id})
|
||||
for doc in res["data"]["docs"]:
|
||||
if doc["id"] not in _document_ids:
|
||||
continue
|
||||
assert doc["run"] == "3"
|
||||
assert doc["run"] == "DONE"
|
||||
assert len(doc["process_begin_at"]) > 0
|
||||
assert doc["process_duration"] > 0
|
||||
assert doc["progress"] > 0
|
||||
@@ -60,11 +59,11 @@ def validate_document_parse_done(auth, _kb_id, _document_ids):
|
||||
|
||||
|
||||
def validate_document_parse_cancel(auth, _kb_id, _document_ids):
|
||||
res = list_documents(auth, {"id": _kb_id})
|
||||
res = list_documents(auth, {"kb_id": _kb_id})
|
||||
for doc in res["data"]["docs"]:
|
||||
if doc["id"] not in _document_ids:
|
||||
continue
|
||||
assert doc["run"] == "2"
|
||||
assert doc["run"] == "CANCEL"
|
||||
assert len(doc["process_begin_at"]) > 0
|
||||
assert doc["progress"] == 0.0
|
||||
|
||||
@@ -151,9 +150,9 @@ class TestDocumentsParse:
|
||||
def test_parse_100_files(WebApiAuth, add_dataset_func, tmp_path):
|
||||
@wait_for(100, 1, "Document parsing timeout")
|
||||
def condition(_auth, _kb_id, _document_num):
|
||||
res = list_documents(_auth, {"id": _kb_id, "page_size": _document_num})
|
||||
res = list_documents(_auth, {"kb_id": _kb_id, "page_size": _document_num})
|
||||
for doc in res["data"]["docs"]:
|
||||
if doc["run"] != "3":
|
||||
if doc["run"] != "DONE":
|
||||
return False
|
||||
return True
|
||||
|
||||
@@ -172,7 +171,7 @@ def test_parse_100_files(WebApiAuth, add_dataset_func, tmp_path):
|
||||
def test_concurrent_parse(WebApiAuth, add_dataset_func, tmp_path):
|
||||
@wait_for(120, 1, "Document parsing timeout")
|
||||
def condition(_auth, _kb_id, _document_num):
|
||||
res = list_documents(_auth, {"id": _kb_id, "page_size": _document_num})
|
||||
res = list_documents(_auth, {"kb_id": _kb_id, "page_size": _document_num})
|
||||
for doc in res["data"]["docs"]:
|
||||
if doc["run"] != "3":
|
||||
return False
|
||||
@@ -305,15 +304,16 @@ class TestDocumentsParseStop:
|
||||
def test_basic_scenarios(self, WebApiAuth, add_documents_func, payload, expected_code, expected_message):
|
||||
@wait_for(30, 1, "Document parsing timeout")
|
||||
def condition(_auth, _kb_id, _doc_ids):
|
||||
res = list_documents(_auth, {"id": _kb_id})
|
||||
res = list_documents(_auth, {"kb_id": _kb_id})
|
||||
for doc in res["data"]["docs"]:
|
||||
if doc["id"] in _doc_ids:
|
||||
if doc["run"] != "3":
|
||||
if doc["run"] != "DONE":
|
||||
return False
|
||||
return True
|
||||
|
||||
kb_id, document_ids = add_documents_func
|
||||
parse_documents(WebApiAuth, {"doc_ids": document_ids, "run": "1"})
|
||||
parse_documents(WebApiAuth, {"doc_ids": document_ids, "run":
|
||||
"1"})
|
||||
|
||||
if callable(payload):
|
||||
payload = payload(document_ids)
|
||||
|
||||
@@ -63,7 +63,7 @@ class TestDocumentsDeletion:
|
||||
if res["code"] != 0:
|
||||
assert res["message"] == expected_message, res
|
||||
|
||||
res = list_documents(WebApiAuth, {"id": kb_id})
|
||||
res = list_documents(WebApiAuth, {"kb_id": kb_id})
|
||||
assert len(res["data"]["docs"]) == remaining, res
|
||||
assert res["data"]["total"] == remaining, res
|
||||
|
||||
@@ -124,12 +124,12 @@ def test_delete_100(WebApiAuth, add_dataset, tmp_path):
|
||||
documents_num = 100
|
||||
kb_id = add_dataset
|
||||
document_ids = bulk_upload_documents(WebApiAuth, kb_id, documents_num, tmp_path)
|
||||
res = list_documents(WebApiAuth, {"id": kb_id})
|
||||
res = list_documents(WebApiAuth, {"kb_id": kb_id})
|
||||
assert res["data"]["total"] == documents_num, res
|
||||
|
||||
for doc_id in document_ids:
|
||||
res = delete_document(WebApiAuth, {"doc_id": doc_id})
|
||||
assert res["code"] == 0, res
|
||||
|
||||
res = list_documents(WebApiAuth, {"id": kb_id})
|
||||
res = list_documents(WebApiAuth, {"kb_id": kb_id})
|
||||
assert res["data"]["total"] == 0, res
|
||||
|
||||
@@ -75,7 +75,7 @@ def _wait_for_task(trace_func, auth, kb_id, task_id, timeout=60, use_params_payl
|
||||
def _wait_for_docs_parsed(auth, kb_id, timeout=60):
|
||||
@wait_for(timeout, 2, "Document parsing timeout")
|
||||
def _condition():
|
||||
res = list_documents(auth, {"id": kb_id})
|
||||
res = list_documents(auth, {"kb_id": kb_id})
|
||||
if res["code"] != 0:
|
||||
return False
|
||||
for doc in res["data"]["docs"]:
|
||||
|
||||
Reference in New Issue
Block a user