Refactor: Consolidation WEB API & HTTP API for document list_docs (#14176)

### What problem does this PR solve?

Before consolidation
Web API: POST /v1/document/list
Http API - GET /api/v1/datasets/<dataset_id>/documents

After consolidation, Restful API -- GET
/api/v1/datasets/<dataset_id>/documents

### Type of change

- [x] Refactoring
This commit is contained in:
Jack
2026-04-20 14:54:40 +08:00
committed by GitHub
parent d053317c4d
commit 939933649a
28 changed files with 627 additions and 712 deletions

View File

@@ -49,9 +49,9 @@ from utils.file_utils import (
@wait_for(30, 1, "Document parsing timeout")
def condition(_auth, _kb_id):
res = list_documents(_auth, {"id": _kb_id})
res = list_documents(_auth, {"kb_id": _kb_id})
for doc in res["data"]["docs"]:
if doc["run"] != "3":
if doc["run"] != "DONE":
return False
return True

View File

@@ -24,9 +24,9 @@ from utils import wait_for
@wait_for(30, 1, "Document parsing timeout")
def condition(_auth, _kb_id):
res = list_documents(_auth, {"id": _kb_id})
res = list_documents(_auth, {"kb_id": _kb_id})
for doc in res["data"]["docs"]:
if doc["run"] != "3":
if doc["run"] != "DONE":
return False
return True

View File

@@ -374,9 +374,11 @@ def create_document(auth, payload=None, *, headers=HEADERS, data=None):
def list_documents(auth, params=None, payload=None, *, headers=HEADERS, data=None):
kb_id = params.get("kb_id") if params else None
url = f"{HOST_ADDRESS}/api/{VERSION}/datasets/{kb_id}/documents"
if payload is None:
payload = {}
res = requests.post(url=f"{HOST_ADDRESS}{DOCUMENT_APP_URL}/list", headers=headers, auth=auth, params=params, json=payload, data=data)
res = requests.get(url=url, headers=headers, auth=auth, params=params, json=payload, data=data)
return res.json()

View File

@@ -34,7 +34,7 @@ class _DummyManager:
@pytest.fixture(scope="function")
def add_document_func(request, WebApiAuth, add_dataset, ragflow_tmp_dir):
def cleanup():
res = list_documents(WebApiAuth, {"id": dataset_id})
res = list_documents(WebApiAuth, {"kb_id": dataset_id})
for doc in res["data"]["docs"]:
delete_document(WebApiAuth, {"doc_id": doc["id"]})
@@ -47,7 +47,7 @@ def add_document_func(request, WebApiAuth, add_dataset, ragflow_tmp_dir):
@pytest.fixture(scope="class")
def add_documents(request, WebApiAuth, add_dataset, ragflow_tmp_dir):
def cleanup():
res = list_documents(WebApiAuth, {"id": dataset_id})
res = list_documents(WebApiAuth, {"kb_id": dataset_id})
for doc in res["data"]["docs"]:
delete_document(WebApiAuth, {"doc_id": doc["id"]})
@@ -60,7 +60,7 @@ def add_documents(request, WebApiAuth, add_dataset, ragflow_tmp_dir):
@pytest.fixture(scope="function")
def add_documents_func(request, WebApiAuth, add_dataset_func, ragflow_tmp_dir):
def cleanup():
res = list_documents(WebApiAuth, {"id": dataset_id})
res = list_documents(WebApiAuth, {"kb_id": dataset_id})
for doc in res["data"]["docs"]:
delete_document(WebApiAuth, {"doc_id": doc["id"]})

View File

@@ -75,7 +75,7 @@ class TestDocumentCreate:
res = create_document(WebApiAuth, {"name": filename, "kb_id": kb_id})
assert res["code"] == 0, res
assert res["data"]["kb_id"] == kb_id, res
assert res["data"]["dataset_id"] == kb_id, res
assert res["data"]["name"] == filename, f"Expected: {filename}, Got: {res['data']['name']}"
@pytest.mark.p3

View File

@@ -13,9 +13,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
#
import asyncio
from concurrent.futures import ThreadPoolExecutor, as_completed
from types import SimpleNamespace
import pytest
from test_common import list_documents
@@ -43,30 +41,18 @@ class TestDocumentsList:
@pytest.mark.p1
def test_default(self, WebApiAuth, add_documents):
kb_id, _ = add_documents
res = list_documents(WebApiAuth, {"id": kb_id})
assert res["code"] == 0
res = list_documents(WebApiAuth, {"kb_id": kb_id})
assert res["code"] == 0, f", kb_id:{kb_id} +, res:{str(res)}"
assert len(res["data"]["docs"]) == 5
assert res["data"]["total"] == 5
@pytest.mark.p3
@pytest.mark.parametrize(
"kb_id, expected_code, expected_message",
[
("", 101, 'Lack of "KB ID"'),
("invalid_dataset_id", 103, "Only owner of dataset authorized for this operation."),
],
)
def test_invalid_dataset_id(self, WebApiAuth, kb_id, expected_code, expected_message):
res = list_documents(WebApiAuth, {"id": kb_id})
assert res["code"] == expected_code
assert res["message"] == expected_message
@pytest.mark.p1
@pytest.mark.parametrize(
"params, expected_code, expected_page_size, expected_message",
[
({"page": None, "page_size": 2}, 0, 5, ""),
({"page": 0, "page_size": 2}, 0, 5, ""),
({"page": None, "page_size": 5}, 0, 5, ""),
({"page": 0, "page_size": 5}, 0, 5, ""),
({"page": 2, "page_size": 2}, 0, 2, ""),
({"page": 3, "page_size": 2}, 0, 1, ""),
({"page": "3", "page_size": 2}, 0, 1, ""),
@@ -76,7 +62,7 @@ class TestDocumentsList:
)
def test_page(self, WebApiAuth, add_documents, params, expected_code, expected_page_size, expected_message):
kb_id, _ = add_documents
res = list_documents(WebApiAuth, {"id": kb_id, **params})
res = list_documents(WebApiAuth, {"kb_id": kb_id, **params})
assert res["code"] == expected_code, res
if expected_code == 0:
assert len(res["data"]["docs"]) == expected_page_size, res
@@ -89,17 +75,17 @@ class TestDocumentsList:
"params, expected_code, expected_page_size, expected_message",
[
({"page_size": None}, 0, 5, ""),
({"page_size": 0}, 0, 5, ""),
({"page_size": 1}, 0, 5, ""),
({"page_size": 5}, 0, 5, ""),
({"page_size": 1}, 0, 1, ""),
({"page_size": 6}, 0, 5, ""),
({"page_size": "1"}, 0, 5, ""),
({"page_size": "1"}, 0, 1, ""),
pytest.param({"page_size": -1}, 100, 0, "1064", marks=pytest.mark.skip(reason="issues/5851")),
pytest.param({"page_size": "a"}, 100, 0, """ValueError("invalid literal for int() with base 10: 'a'")""", marks=pytest.mark.skip(reason="issues/5851")),
],
)
def test_page_size(self, WebApiAuth, add_documents, params, expected_code, expected_page_size, expected_message):
kb_id, _ = add_documents
res = list_documents(WebApiAuth, {"id": kb_id, **params})
res = list_documents(WebApiAuth, {"kb_id": kb_id, **params})
assert res["code"] == expected_code, res
if expected_code == 0:
assert len(res["data"]["docs"]) == expected_page_size, res
@@ -119,7 +105,7 @@ class TestDocumentsList:
)
def test_orderby(self, WebApiAuth, add_documents, params, expected_code, assertions, expected_message):
kb_id, _ = add_documents
res = list_documents(WebApiAuth, {"id": kb_id, **params})
res = list_documents(WebApiAuth, {"kb_id": kb_id, **params})
assert res["code"] == expected_code, res
if expected_code == 0:
if callable(assertions):
@@ -144,7 +130,7 @@ class TestDocumentsList:
)
def test_desc(self, WebApiAuth, add_documents, params, expected_code, assertions, expected_message):
kb_id, _ = add_documents
res = list_documents(WebApiAuth, {"id": kb_id, **params})
res = list_documents(WebApiAuth, {"kb_id": kb_id, **params})
assert res["code"] == expected_code, res
if expected_code == 0:
if callable(assertions):
@@ -165,7 +151,7 @@ class TestDocumentsList:
)
def test_keywords(self, WebApiAuth, add_documents, params, expected_num):
kb_id, _ = add_documents
res = list_documents(WebApiAuth, {"id": kb_id, **params})
res = list_documents(WebApiAuth, {"kb_id": kb_id, **params})
assert res["code"] == 0, res
assert len(res["data"]["docs"]) == expected_num, res
assert res["data"]["total"] == expected_num, res
@@ -181,213 +167,53 @@ class TestDocumentsList:
assert len(responses) == count, responses
assert all(future.result()["code"] == 0 for future in futures), responses
# Tests moved from TestDocumentsListUnit
@pytest.mark.p2
def test_missing_kb_id(self, WebApiAuth):
"""Test missing KB ID returns error."""
res = list_documents(WebApiAuth, {"kb_id": ""})
assert res["code"] == 100
assert res["message"] == "<MethodNotAllowed '405: Method Not Allowed'>"
def _run(coro):
return asyncio.run(coro)
@pytest.mark.p2
def test_unauthorized_dataset(self, WebApiAuth):
"""Test unauthorized dataset returns error."""
res = list_documents(WebApiAuth, {"kb_id": "non_existent_kb_id"})
assert res["code"] == 102
assert "You don't own the dataset" in res["message"]
class _DummyArgs(dict):
def get(self, key, default=None):
return super().get(key, default)
@pytest.mark.p2
class TestDocumentsListUnit:
def _set_args(self, module, monkeypatch, **kwargs):
monkeypatch.setattr(module, "request", SimpleNamespace(args=_DummyArgs(kwargs)))
def _allow_kb(self, module, monkeypatch, kb_id="kb1", tenant_id="tenant1"):
monkeypatch.setattr(module.UserTenantService, "query", lambda **_kwargs: [SimpleNamespace(tenant_id=tenant_id)])
monkeypatch.setattr(module.KnowledgebaseService, "query", lambda **_kwargs: True if _kwargs.get("id") == kb_id else False)
def test_missing_kb_id(self, document_app_module, monkeypatch):
module = document_app_module
self._set_args(module, monkeypatch)
async def fake_request_json():
return {}
monkeypatch.setattr(module, "get_request_json", fake_request_json)
res = _run(module.list_docs())
assert res["code"] == 101
assert res["message"] == 'Dataset ID is required for listing files.'
def test_unauthorized_dataset(self, document_app_module, monkeypatch):
module = document_app_module
self._set_args(module, monkeypatch, id="kb1")
monkeypatch.setattr(module.UserTenantService, "query", lambda **_kwargs: [SimpleNamespace(tenant_id="tenant1")])
monkeypatch.setattr(module.KnowledgebaseService, "query", lambda **_kwargs: False)
async def fake_request_json():
return {}
monkeypatch.setattr(module, "get_request_json", fake_request_json)
res = _run(module.list_docs())
assert res["code"] == 103
assert "Only owner of dataset" in res["message"]
def test_return_empty_metadata_flags(self, document_app_module, monkeypatch):
module = document_app_module
self._set_args(module, monkeypatch, id="kb1")
self._allow_kb(module, monkeypatch)
monkeypatch.setattr(module.DocumentService, "get_by_kb_id", lambda *_args, **_kwargs: ([], 0))
async def fake_request_json():
return {"return_empty_metadata": "true", "metadata": {"author": "alice"}}
monkeypatch.setattr(module, "get_request_json", fake_request_json)
res = _run(module.list_docs())
assert res["code"] == 0
async def fake_request_json_empty():
return {"metadata": {"empty_metadata": True, "author": "alice"}}
monkeypatch.setattr(module, "get_request_json", fake_request_json_empty)
res = _run(module.list_docs())
assert res["code"] == 0
def test_invalid_filters(self, document_app_module, monkeypatch):
module = document_app_module
self._set_args(module, monkeypatch, id="kb1")
self._allow_kb(module, monkeypatch)
async def fake_request_json():
return {"run_status": ["INVALID"]}
monkeypatch.setattr(module, "get_request_json", fake_request_json)
res = _run(module.list_docs())
@pytest.mark.p3
def test_invalid_run_status_filter(self, WebApiAuth, add_documents):
"""Test invalid run status filter returns error."""
kb_id, _ = add_documents
res = list_documents(WebApiAuth, {"kb_id": kb_id, "run": "INVALID"})
assert res["code"] == 102
assert "Invalid filter run status" in res["message"]
async def fake_request_json_types():
return {"types": ["INVALID"]}
monkeypatch.setattr(module, "get_request_json", fake_request_json_types)
res = _run(module.list_docs())
@pytest.mark.p3
def test_invalid_document_id_filter(self, WebApiAuth, add_documents):
"""Test invalid document ID filter returns error."""
kb_id, _ = add_documents
# Use a non-existent document ID
res = list_documents(WebApiAuth, {"kb_id": kb_id, "id": "non_existent_doc_id"})
assert res["code"] == 102
assert "Invalid filter conditions" in res["message"]
assert "You don't own the document" in res["message"]
def test_invalid_metadata_types(self, document_app_module, monkeypatch):
module = document_app_module
self._set_args(module, monkeypatch, id="kb1")
self._allow_kb(module, monkeypatch)
async def fake_request_json():
return {"metadata_condition": "bad"}
monkeypatch.setattr(module, "get_request_json", fake_request_json)
res = _run(module.list_docs())
assert res["code"] == 102
assert "metadata_condition" in res["message"]
async def fake_request_json_meta():
return {"metadata": ["not", "object"]}
monkeypatch.setattr(module, "get_request_json", fake_request_json_meta)
res = _run(module.list_docs())
assert res["code"] == 102
assert "metadata must be an object" in res["message"]
def test_metadata_condition_empty_result(self, document_app_module, monkeypatch):
module = document_app_module
self._set_args(module, monkeypatch, id="kb1")
self._allow_kb(module, monkeypatch)
monkeypatch.setattr(module.DocMetadataService, "get_flatted_meta_by_kbs", lambda *_args, **_kwargs: {})
monkeypatch.setattr(module, "meta_filter", lambda *_args, **_kwargs: set())
async def fake_request_json():
return {"metadata_condition": {"conditions": [{"name": "author", "comparison_operator": "is", "value": "alice"}]}}
monkeypatch.setattr(module, "get_request_json", fake_request_json)
res = _run(module.list_docs())
@pytest.mark.p3
def test_create_time_filter(self, WebApiAuth, add_documents):
"""Test create time range filter."""
kb_id, _ = add_documents
# Get current time range
res = list_documents(WebApiAuth, {"kb_id": kb_id})
assert res["code"] == 0
assert res["data"]["total"] == 0
if res["data"]["docs"]:
create_time = res["data"]["docs"][0].get("create_time", 0)
# Test with time range that should include the document
res = list_documents(WebApiAuth, {"kb_id": kb_id, "create_time_from": 0, "create_time_to": create_time + 1000})
assert res["code"] == 0
assert len(res["data"]["docs"]) > 0
# Test with time range that should not include the document
res = list_documents(WebApiAuth, {"kb_id": kb_id, "create_time_from": create_time + 1000, "create_time_to": create_time + 2000})
assert res["code"] == 0
assert len(res["data"]["docs"]) == 0
def test_metadata_values_intersection(self, document_app_module, monkeypatch):
module = document_app_module
self._set_args(module, monkeypatch, id="kb1")
self._allow_kb(module, monkeypatch)
metas = {
"author": {"alice": ["doc1", "doc2"]},
"topic": {"rag": ["doc2"]},
}
monkeypatch.setattr(module.DocMetadataService, "get_flatted_meta_by_kbs", lambda *_args, **_kwargs: metas)
captured = {}
def fake_get_by_kb_id(*_args, **_kwargs):
if len(_args) >= 10:
captured["doc_ids_filter"] = _args[9]
else:
captured["doc_ids_filter"] = None
return ([{"id": "doc2", "thumbnail": "", "parser_config": {}}], 1)
monkeypatch.setattr(module.DocumentService, "get_by_kb_id", fake_get_by_kb_id)
async def fake_request_json():
return {"metadata": {"author": ["alice", " ", None], "topic": "rag"}}
monkeypatch.setattr(module, "get_request_json", fake_request_json)
res = _run(module.list_docs())
assert res["code"] == 0
assert captured["doc_ids_filter"] == ["doc2"]
def test_metadata_intersection_empty(self, document_app_module, monkeypatch):
module = document_app_module
self._set_args(module, monkeypatch, id="kb1")
self._allow_kb(module, monkeypatch)
metas = {
"author": {"alice": ["doc1"]},
"topic": {"rag": ["doc2"]},
}
monkeypatch.setattr(module.DocMetadataService, "get_flatted_meta_by_kbs", lambda *_args, **_kwargs: metas)
async def fake_request_json():
return {"metadata": {"author": "alice", "topic": "rag"}}
monkeypatch.setattr(module, "get_request_json", fake_request_json)
res = _run(module.list_docs())
assert res["code"] == 0
assert res["data"]["total"] == 0
def test_desc_time_and_schema(self, document_app_module, monkeypatch):
module = document_app_module
self._set_args(module, monkeypatch, id="kb1", desc="false", create_time_from="150", create_time_to="250")
self._allow_kb(module, monkeypatch)
docs = [
{"id": "doc1", "thumbnail": "", "parser_config": {"metadata": {"a": 1}}, "create_time": 100},
{"id": "doc2", "thumbnail": "", "parser_config": {"metadata": {"b": 2}}, "create_time": 200},
]
def fake_get_by_kb_id(*_args, **_kwargs):
return (docs, 2)
monkeypatch.setattr(module.DocumentService, "get_by_kb_id", fake_get_by_kb_id)
monkeypatch.setattr(module, "turn2jsonschema", lambda _meta: {"schema": True})
async def fake_request_json():
return {}
monkeypatch.setattr(module, "get_request_json", fake_request_json)
res = _run(module.list_docs())
assert res["code"] == 0
assert len(res["data"]["docs"]) == 1
assert res["data"]["docs"][0]["parser_config"]["metadata"] == {"schema": True}
def test_exception_path(self, document_app_module, monkeypatch):
module = document_app_module
self._set_args(module, monkeypatch, id="kb1")
self._allow_kb(module, monkeypatch)
def raise_error(*_args, **_kwargs):
raise RuntimeError("boom")
monkeypatch.setattr(module.DocumentService, "get_by_kb_id", raise_error)
async def fake_request_json():
return {}
monkeypatch.setattr(module, "get_request_json", fake_request_json)
res = _run(module.list_docs())
assert res["code"] == 100

View File

@@ -30,29 +30,28 @@ def _run(coro):
@wait_for(30, 1, "Document parsing timeout")
def condition(_auth, _kb_id, _document_ids=None):
res = list_documents(_auth, {"id": _kb_id})
res = list_documents(_auth, {"kb_id": _kb_id})
target_docs = res["data"]["docs"]
if _document_ids is None:
for doc in target_docs:
if doc["run"] != "3":
if doc["run"] != "DONE":
return False
return True
target_ids = set(_document_ids)
for doc in target_docs:
if doc["id"] in target_ids:
if doc.get("run") != "3":
if doc.get("run") != "DONE":
return False
return True
def validate_document_parse_done(auth, _kb_id, _document_ids):
res = list_documents(auth, {"id": _kb_id})
res = list_documents(auth, {"kb_id": _kb_id})
for doc in res["data"]["docs"]:
if doc["id"] not in _document_ids:
continue
assert doc["run"] == "3"
assert doc["run"] == "DONE"
assert len(doc["process_begin_at"]) > 0
assert doc["process_duration"] > 0
assert doc["progress"] > 0
@@ -60,11 +59,11 @@ def validate_document_parse_done(auth, _kb_id, _document_ids):
def validate_document_parse_cancel(auth, _kb_id, _document_ids):
res = list_documents(auth, {"id": _kb_id})
res = list_documents(auth, {"kb_id": _kb_id})
for doc in res["data"]["docs"]:
if doc["id"] not in _document_ids:
continue
assert doc["run"] == "2"
assert doc["run"] == "CANCEL"
assert len(doc["process_begin_at"]) > 0
assert doc["progress"] == 0.0
@@ -151,9 +150,9 @@ class TestDocumentsParse:
def test_parse_100_files(WebApiAuth, add_dataset_func, tmp_path):
@wait_for(100, 1, "Document parsing timeout")
def condition(_auth, _kb_id, _document_num):
res = list_documents(_auth, {"id": _kb_id, "page_size": _document_num})
res = list_documents(_auth, {"kb_id": _kb_id, "page_size": _document_num})
for doc in res["data"]["docs"]:
if doc["run"] != "3":
if doc["run"] != "DONE":
return False
return True
@@ -172,7 +171,7 @@ def test_parse_100_files(WebApiAuth, add_dataset_func, tmp_path):
def test_concurrent_parse(WebApiAuth, add_dataset_func, tmp_path):
@wait_for(120, 1, "Document parsing timeout")
def condition(_auth, _kb_id, _document_num):
res = list_documents(_auth, {"id": _kb_id, "page_size": _document_num})
res = list_documents(_auth, {"kb_id": _kb_id, "page_size": _document_num})
for doc in res["data"]["docs"]:
if doc["run"] != "3":
return False
@@ -305,15 +304,16 @@ class TestDocumentsParseStop:
def test_basic_scenarios(self, WebApiAuth, add_documents_func, payload, expected_code, expected_message):
@wait_for(30, 1, "Document parsing timeout")
def condition(_auth, _kb_id, _doc_ids):
res = list_documents(_auth, {"id": _kb_id})
res = list_documents(_auth, {"kb_id": _kb_id})
for doc in res["data"]["docs"]:
if doc["id"] in _doc_ids:
if doc["run"] != "3":
if doc["run"] != "DONE":
return False
return True
kb_id, document_ids = add_documents_func
parse_documents(WebApiAuth, {"doc_ids": document_ids, "run": "1"})
parse_documents(WebApiAuth, {"doc_ids": document_ids, "run":
"1"})
if callable(payload):
payload = payload(document_ids)

View File

@@ -63,7 +63,7 @@ class TestDocumentsDeletion:
if res["code"] != 0:
assert res["message"] == expected_message, res
res = list_documents(WebApiAuth, {"id": kb_id})
res = list_documents(WebApiAuth, {"kb_id": kb_id})
assert len(res["data"]["docs"]) == remaining, res
assert res["data"]["total"] == remaining, res
@@ -124,12 +124,12 @@ def test_delete_100(WebApiAuth, add_dataset, tmp_path):
documents_num = 100
kb_id = add_dataset
document_ids = bulk_upload_documents(WebApiAuth, kb_id, documents_num, tmp_path)
res = list_documents(WebApiAuth, {"id": kb_id})
res = list_documents(WebApiAuth, {"kb_id": kb_id})
assert res["data"]["total"] == documents_num, res
for doc_id in document_ids:
res = delete_document(WebApiAuth, {"doc_id": doc_id})
assert res["code"] == 0, res
res = list_documents(WebApiAuth, {"id": kb_id})
res = list_documents(WebApiAuth, {"kb_id": kb_id})
assert res["data"]["total"] == 0, res

View File

@@ -75,7 +75,7 @@ def _wait_for_task(trace_func, auth, kb_id, task_id, timeout=60, use_params_payl
def _wait_for_docs_parsed(auth, kb_id, timeout=60):
@wait_for(timeout, 2, "Document parsing timeout")
def _condition():
res = list_documents(auth, {"id": kb_id})
res = list_documents(auth, {"kb_id": kb_id})
if res["code"] != 0:
return False
for doc in res["data"]["docs"]: