zlei6/ragflow
1
0
Fork 0
mirror of https://github.com/infiniflow/ragflow.git synced 2026-07-21 07:01:04 +08:00
Code Issues Packages Projects Releases Wiki Activity

Consolidateion metadata summary API (#14031)

Browse Source 
### What problem does this PR solve?

Consolidation WEB API & HTTP API for document metadata summary

Before consolidation
Web API: POST /api/v1/document/metadata/summary
Http API - GET /v1/datasets/<dataset_id>/metadata/summary

After consolidation, Restful API -- GET
/v1/datasets/<dataset_id>/metadata/summary

### Type of change

- [x] Refactoring
This commit is contained in:
Jack
2026-04-10 18:41:30 +08:00
committed by GitHub
parent 11c89d87da
commit 4046a4cfb6
8 changed files with 108 additions and 126 deletions
Download Patch File Download Diff File Expand all files Collapse all files

18
test/testcases/test_http_api/test_file_management_within_dataset/test_doc_sdk_routes_unit.py
View File

@@ -472,7 +472,7 @@ class TestDocRoutesUnit:
assert res["code"] == 0
assert res["data"]["docs"] == []
def test_metadata_summary_and_batch_update(self, monkeypatch):
def test_metadata_batch_update(self, monkeypatch):
module = _load_doc_module(monkeypatch)
monkeypatch.setattr(module, "convert_conditions", lambda cond: cond)
monkeypatch.setattr(module.KnowledgebaseService, "accessible", lambda **_kwargs: False)
@@ -480,22 +480,7 @@ class TestDocRoutesUnit:
res = _run(module.metadata_batch_update.__wrapped__("ds-1", "tenant-1"))
assert "don't own the dataset" in res["message"]
monkeypatch.setattr(module.KnowledgebaseService, "accessible", lambda **_kwargs: False)
res = _run(module.metadata_summary.__wrapped__("ds-1", "tenant-1"))
assert "don't own the dataset" in res["message"]
monkeypatch.setattr(module.KnowledgebaseService, "accessible", lambda **_kwargs: True)
monkeypatch.setattr(module, "get_request_json", lambda: _AwaitableValue({"doc_ids": ["d1"]}))
monkeypatch.setattr(module.DocMetadataService, "get_metadata_summary", lambda *_args, **_kwargs: {"k": 1})
res = _run(module.metadata_summary.__wrapped__("ds-1", "tenant-1"))
assert res["code"] == 0
assert res["data"]["summary"] == {"k": 1}
monkeypatch.setattr(module.DocMetadataService, "get_metadata_summary", lambda *_args, **_kwargs: (_ for _ in ()).throw(RuntimeError("x")))
monkeypatch.setattr(module, "server_error_response", lambda e: {"code": 500, "message": str(e)})
res = _run(module.metadata_summary.__wrapped__("ds-1", "tenant-1"))
assert res["code"] == 500
monkeypatch.setattr(module, "get_request_json", lambda: _AwaitableValue({"selector": [1]}))
res = _run(module.metadata_batch_update.__wrapped__("ds-1", "tenant-1"))
assert res["message"] == "selector must be an object."
@@ -576,6 +561,7 @@ class TestDocRoutesUnit:
assert res["data"]["updated"] == 1
assert res["data"]["matched_docs"] == 1
def test_delete_branches(self, monkeypatch):
module = _load_doc_module(monkeypatch)
monkeypatch.setattr(module.KnowledgebaseService, "accessible", lambda **_kwargs: False)

89
test/testcases/test_http_api/test_file_management_within_dataset/test_metadata_summary.py
View File

@@ -13,11 +13,8 @@
# See the License for the specific language governing permissions and
# limitations under the License.
#
# Although the docs group this under "chunk management," the backend aggregates
# Document.meta_fields via document_service#get_metadata_summary and the test
# uses update_document, so it belongs with file/document management tests.
# import pytest
#from common import metadata_summary, update_document
import pytest
from common import metadata_summary, update_document
def _summary_to_counts(summary):
@@ -30,29 +27,63 @@ def _summary_to_counts(summary):
class TestMetadataSummary:
pass
@pytest.mark.p2
def test_metadata_summary_missing_kb_id(self, HttpApiAuth, add_document_func):
"""
Call with non-existent dataset
:param HttpApiAuth:
:param add_document_func:
:return:
"""
res = metadata_summary(HttpApiAuth, "")
assert res["code"] == 404, res
assert res["message"] == "Not Found: /api/v1/datasets//metadata/summary", res
# Alteration of API
# TODO
#@pytest.mark.p2
#def test_metadata_summary_counts(self, HttpApiAuth, add_documents_func):
# dataset_id, document_ids = add_documents_func
# payloads = [
# {"tags": ["foo", "bar"], "author": "alice"},
# {"tags": ["foo"], "author": "bob"},
# {"tags": ["bar", "baz"], "author": None},
# ]
# for doc_id, meta_fields in zip(document_ids, payloads):
# res = update_document(HttpApiAuth, dataset_id, doc_id, {"meta_fields": meta_fields})
# assert res["code"] == 0, res
@pytest.mark.p2
def test_metadata_summary_invalid_kb_id(self, HttpApiAuth, add_document_func):
"""Test metadata summary when user doesn't have access to the dataset."""
kb_id, doc_id = add_document_func
invalid_kb_id = "invalid_" + kb_id
# Call with a dataset that the user doesn't have access to
res = metadata_summary(HttpApiAuth, invalid_kb_id)
assert res["code"] == 102, res
assert res["message"] == f"You don't own the dataset {invalid_kb_id}. "
# res = metadata_summary(HttpApiAuth, dataset_id)
# assert res["code"] == 0, res
# summary = res["data"]["summary"]
# counts = _summary_to_counts(summary)
# assert counts["tags"]["foo"] == 2, counts
# assert counts["tags"]["bar"] == 2, counts
# assert counts["tags"]["baz"] == 1, counts
# assert counts["author"]["alice"] == 1, counts
# assert counts["author"]["bob"] == 1, counts
# assert "None" not in counts["author"], counts
@pytest.mark.p2
def test_metadata_summary_success(self, HttpApiAuth, add_document_func):
"""Test metadata summary success case"""
kb_id, doc_id = add_document_func
# Test successful case
res = metadata_summary(HttpApiAuth, kb_id)
assert res["code"] == 0, res
assert "summary" in res["data"], res
@pytest.mark.p2
def test_metadata_summary_counts(self, HttpApiAuth, add_documents_func):
"""
test normal cases
:param HttpApiAuth:
:param add_documents_func:
:return:
"""
dataset_id, document_ids = add_documents_func
payloads = [
{"tags": ["foo", "bar"], "author": "alice"},
{"tags": ["foo"], "author": "bob"},
{"tags": ["bar", "baz"], "author": ""},
]
for doc_id, meta_fields in zip(document_ids, payloads):
res = update_document(HttpApiAuth, dataset_id, doc_id, {"meta_fields": meta_fields})
assert res["code"] == 0, res
res = metadata_summary(HttpApiAuth, dataset_id)
assert res["code"] == 0, res
summary = res["data"]["summary"]
counts = _summary_to_counts(summary)
assert counts["tags"]["foo"] == 2, counts
assert counts["tags"]["bar"] == 2, counts
assert counts["tags"]["baz"] == 1, counts
assert counts["author"]["alice"] == 1, counts
assert counts["author"]["bob"] == 1, counts
assert "None" not in counts["author"], counts

42
test/testcases/test_web_api/test_document_app/test_document_metadata.py
View File

@@ -319,48 +319,6 @@ class TestDocumentMetadataUnit:
assert res["code"] == 0
assert res["data"][0]["meta_fields"]["author"] == "alice"
def test_metadata_summary_missing_kb_id(self, document_app_module, monkeypatch):
module = document_app_module
async def fake_request_json():
return {"doc_ids": ["doc1"]}
monkeypatch.setattr(module, "get_request_json", fake_request_json)
res = _run(module.metadata_summary())
assert res["code"] == 101
def test_metadata_summary_unauthorized(self, document_app_module, monkeypatch):
module = document_app_module
monkeypatch.setattr(module.UserTenantService, "query", lambda **_kwargs: [SimpleNamespace(tenant_id="tenant1")])
monkeypatch.setattr(module.KnowledgebaseService, "query", lambda **_kwargs: False)
async def fake_request_json():
return {"kb_id": "kb1", "doc_ids": ["doc1"]}
monkeypatch.setattr(module, "get_request_json", fake_request_json)
res = _run(module.metadata_summary())
assert res["code"] == 103
def test_metadata_summary_success_and_exception(self, document_app_module, monkeypatch):
module = document_app_module
self._allow_kb(module, monkeypatch)
monkeypatch.setattr(module.DocMetadataService, "get_metadata_summary", lambda *_args, **_kwargs: {"author": {"alice": 1}})
async def fake_request_json():
return {"kb_id": "kb1", "doc_ids": ["doc1"]}
monkeypatch.setattr(module, "get_request_json", fake_request_json)
res = _run(module.metadata_summary())
assert res["code"] == 0
assert "summary" in res["data"]
def raise_error(*_args, **_kwargs):
raise RuntimeError("boom")
monkeypatch.setattr(module.DocMetadataService, "get_metadata_summary", raise_error)
res = _run(module.metadata_summary())
assert res["code"] == 100
def test_metadata_update_missing_kb_id(self, document_app_module, monkeypatch):
module = document_app_module