zlei6/ragflow
1
0
Fork 0
mirror of https://github.com/infiniflow/ragflow.git synced 2026-07-29 04:08:12 +08:00
Code Issues Packages Projects Releases Wiki Activity

Refactor: Merge document update API (#13962)

Browse Source 
### What problem does this PR solve?

Refactor: merge document.rename into document.update_document

### Type of change

- [x] Refactoring


<!-- This is an auto-generated comment: release notes by coderabbit.ai
-->
## Summary by CodeRabbit

* **New Features**
* Added a unified document update API (PUT) supporting name, metadata,
parser/chunk settings, and status changes.

* **Breaking Changes**
* Legacy single-parameter rename endpoint removed; renames now require
dataset + document identifiers.
  * `/list` now reads dataset id from a different query parameter.

* **Validation / Bug Fixes**
* Stricter meta_fields and parser-config validation; unauthenticated
requests return 401.

* **Frontend**
  * UI now sends dataset id when saving document names.

* **Tests**
* Numerous unit and HTTP tests adjusted or removed to match new API and
validations.
<!-- end of auto-generated comment: release notes by coderabbit.ai -->

---------

Signed-off-by: Jin Hai <haijin.chn@gmail.com>
Co-authored-by: Jin Hai <haijin.chn@gmail.com>
Co-authored-by: MkDev11 <94194147+MkDev11@users.noreply.github.com>
Co-authored-by: mkdev11 <YOUR_GITHUB_ID+MkDev11@users.noreply.github.com>
Co-authored-by: mkdev11 <MkDev11@users.noreply.github.com>
Co-authored-by: Qi Wang <wangq8@outlook.com>
Co-authored-by: dataCenter430 <161712630+dataCenter430@users.noreply.github.com>
Co-authored-by: balibabu <cike8899@users.noreply.github.com>
This commit is contained in:
Jack
2026-04-09 11:17:38 +08:00
committed by GitHub
parent c13f8856a1
commit 577c96bf2a
14 changed files with 651 additions and 710 deletions
Download Patch File Download Diff File Expand all files Collapse all files

117
test/testcases/test_http_api/test_file_management_within_dataset/test_doc_sdk_routes_unit.py
View File

@@ -366,123 +366,6 @@ class TestDocRoutesUnit:
assert res["code"] == module.RetCode.SERVER_ERROR
assert res["message"] == "upload failed"
def test_update_doc_guards_and_error_paths(self, monkeypatch):
module = _load_doc_module(monkeypatch)
doc = _DummyDoc()
monkeypatch.setattr(module.KnowledgebaseService, "query", lambda **_kwargs: [])
monkeypatch.setattr(module, "get_request_json", lambda: _AwaitableValue({}))
res = _run(module.update_doc.__wrapped__("tenant-1", "ds-1", "doc-1"))
assert res["message"] == "You don't own the dataset."
monkeypatch.setattr(module.KnowledgebaseService, "query", lambda **_kwargs: [1])
monkeypatch.setattr(module.KnowledgebaseService, "get_by_id", lambda _id: (False, None))
res = _run(module.update_doc.__wrapped__("tenant-1", "ds-1", "doc-1"))
assert res["message"] == "Can't find this dataset!"
monkeypatch.setattr(module.KnowledgebaseService, "get_by_id", lambda _id: (True, SimpleNamespace(tenant_id="tenant-1")))
monkeypatch.setattr(module.DocumentService, "query", lambda **_kwargs: [])
res = _run(module.update_doc.__wrapped__("tenant-1", "ds-1", "doc-1"))
assert "doesn't own the document" in res["message"]
monkeypatch.setattr(module.DocumentService, "query", lambda **_kwargs: [doc])
monkeypatch.setattr(module, "get_request_json", lambda: _AwaitableValue({"chunk_count": 100}))
res = _run(module.update_doc.__wrapped__("tenant-1", "ds-1", "doc-1"))
assert "chunk_count" in res["message"]
monkeypatch.setattr(module, "get_request_json", lambda: _AwaitableValue({"token_count": 100}))
res = _run(module.update_doc.__wrapped__("tenant-1", "ds-1", "doc-1"))
assert "token_count" in res["message"]
monkeypatch.setattr(module, "get_request_json", lambda: _AwaitableValue({"progress": 100}))
res = _run(module.update_doc.__wrapped__("tenant-1", "ds-1", "doc-1"))
assert "progress" in res["message"]
monkeypatch.setattr(module, "get_request_json", lambda: _AwaitableValue({"meta_fields": []}))
res = _run(module.update_doc.__wrapped__("tenant-1", "ds-1", "doc-1"))
assert res["message"] == "Field: <meta_fields> - Message: <Input should be a valid dictionary> - Value: <[]>"
monkeypatch.setattr(module, "get_request_json", lambda: _AwaitableValue({"meta_fields": {"k": "v"}}))
monkeypatch.setattr(module.DocMetadataService, "update_document_metadata", lambda *_args, **_kwargs: False)
res = _run(module.update_doc.__wrapped__("tenant-1", "ds-1", "doc-1"))
assert res["message"] == "Failed to update metadata"
monkeypatch.setattr(module, "get_request_json", lambda: _AwaitableValue({"name": "a" * (module.FILE_NAME_LEN_LIMIT + 1)}))
res = _run(module.update_doc.__wrapped__("tenant-1", "ds-1", "doc-1"))
assert res["code"] == module.RetCode.ARGUMENT_ERROR
assert "bytes or less" in res["message"]
monkeypatch.setattr(module.DocumentService, "update_by_id", lambda *_args, **_kwargs: False)
monkeypatch.setattr(module, "get_request_json", lambda: _AwaitableValue({"name": "new.txt"}))
res = _run(module.update_doc.__wrapped__("tenant-1", "ds-1", "doc-1"))
assert "Document rename" in res["message"]
def test_update_doc_chunk_method_enabled_and_db_error(self, monkeypatch):
module = _load_doc_module(monkeypatch)
from api.db import FileType
visual_doc = _DummyDoc(parser_id="naive", doc_type=FileType.VISUAL)
kb = SimpleNamespace(tenant_id="tenant-1")
monkeypatch.setattr(module.KnowledgebaseService, "query", lambda **_kwargs: [1])
monkeypatch.setattr(module.KnowledgebaseService, "get_by_id", lambda _id: (True, kb))
monkeypatch.setattr(module.DocumentService, "query", lambda **_kwargs: [visual_doc])
monkeypatch.setattr(module, "get_request_json", lambda: _AwaitableValue({"chunk_method": "naive"}))
res = _run(module.update_doc.__wrapped__("tenant-1", "ds-1", "doc-1"))
assert res["message"] == "Not supported yet!"
doc = _DummyDoc(token_num=2, chunk_num=1, parser_id="naive")
monkeypatch.setattr(module.DocumentService, "query", lambda **_kwargs: [doc])
monkeypatch.setattr(module.DocumentService, "update_by_id", lambda *_args, **_kwargs: False)
monkeypatch.setattr(module, "get_request_json", lambda: _AwaitableValue({"chunk_method": "manual"}))
res = _run(module.update_doc.__wrapped__("tenant-1", "ds-1", "doc-1"))
assert res["message"] == "Document not found!"
monkeypatch.setattr(module.DocumentService, "update_by_id", lambda *_args, **_kwargs: True)
monkeypatch.setattr(module.DocumentService, "update_parser_config", lambda *_args, **_kwargs: None)
monkeypatch.setattr(module.DocumentService, "increment_chunk_num", lambda *_args, **_kwargs: False)
_patch_docstore(monkeypatch, module, delete=lambda *_args, **_kwargs: None)
monkeypatch.setattr(module, "get_request_json", lambda: _AwaitableValue({"chunk_method": "manual"}))
res = _run(module.update_doc.__wrapped__("tenant-1", "ds-1", "doc-1"))
assert res["message"] == "Document not found!"
monkeypatch.setattr(module.DocumentService, "increment_chunk_num", lambda *_args, **_kwargs: True)
doc_for_enabled = _DummyDoc(status=False)
monkeypatch.setattr(module.DocumentService, "query", lambda **_kwargs: [doc_for_enabled])
monkeypatch.setattr(module.DocumentService, "get_by_id", lambda _id: (True, doc_for_enabled))
monkeypatch.setattr(module.DocumentService, "update_by_id", lambda *_args, **_kwargs: False)
_patch_docstore(monkeypatch, module, update=lambda *_args, **_kwargs: None, delete=lambda *_args, **_kwargs: None)
monkeypatch.setattr(module, "get_request_json", lambda: _AwaitableValue({"enabled": 1}))
res = _run(module.update_doc.__wrapped__("tenant-1", "ds-1", "doc-1"))
assert "Document update" in res["message"]
monkeypatch.setattr(module.DocumentService, "update_by_id", lambda *_args, **_kwargs: True)
_patch_docstore(monkeypatch, module, update=lambda *_args, **_kwargs: (_ for _ in ()).throw(RuntimeError("boom")), delete=lambda *_args, **_kwargs: None)
monkeypatch.setattr(module, "server_error_response", lambda e: {"code": 500, "message": str(e)})
res = _run(module.update_doc.__wrapped__("tenant-1", "ds-1", "doc-1"))
assert res["code"] == 500
assert "boom" in res["message"]
monkeypatch.setattr(module.DocumentService, "get_by_id", lambda _id: (False, None))
_patch_docstore(monkeypatch, module, update=lambda *_args, **_kwargs: None, delete=lambda *_args, **_kwargs: None)
res = _run(module.update_doc.__wrapped__("tenant-1", "ds-1", "doc-1"))
assert res["message"] == "Dataset created failed"
# cover token reset + docStore deletion branch
doc_reset = _DummyDoc(token_num=3, chunk_num=2, parser_id="naive", run=0)
monkeypatch.setattr(module.DocumentService, "query", lambda **_kwargs: [doc_reset])
monkeypatch.setattr(module.DocumentService, "update_by_id", lambda *_args, **_kwargs: True)
monkeypatch.setattr(module.DocumentService, "increment_chunk_num", lambda *_args, **_kwargs: True)
monkeypatch.setattr(module.DocumentService, "get_by_id", lambda _id: (True, doc_reset))
_patch_docstore(monkeypatch, module, delete=lambda *_args, **_kwargs: None, update=lambda *_args, **_kwargs: None)
monkeypatch.setattr(module, "get_request_json", lambda: _AwaitableValue({"chunk_method": "manual"}))
res = _run(module.update_doc.__wrapped__("tenant-1", "ds-1", "doc-1"))
assert res["code"] == 0
def _raise_operational_error(_id):
raise module.OperationalError("db down")
monkeypatch.setattr(module.DocumentService, "get_by_id", _raise_operational_error)
res = _run(module.update_doc.__wrapped__("tenant-1", "ds-1", "doc-1"))
assert res["message"] == "Database operation failed"
def test_download_and_download_doc_errors(self, monkeypatch):
module = _load_doc_module(monkeypatch)
_patch_send_file(monkeypatch, module)

223
test/testcases/test_http_api/test_file_management_within_dataset/test_update_document.py
View File

@@ -21,16 +21,17 @@ from configs import DOCUMENT_NAME_LIMIT, INVALID_API_TOKEN, INVALID_ID_32
from libs.auth import RAGFlowHttpApiAuth
from configs import DEFAULT_PARSER_CONFIG
@pytest.mark.p1
class TestAuthorization:
@pytest.mark.parametrize(
"invalid_auth, expected_code, expected_message",
[
(None, 0, "`Authorization` can't be empty"),
(None, 401, "<Unauthorized '401: Unauthorized'>"),
(
RAGFlowHttpApiAuth(INVALID_API_TOKEN),
109,
"Authentication error: API key is invalid!",
401,
"<Unauthorized '401: Unauthorized'>",
),
],
)
@@ -53,13 +54,13 @@ class TestDocumentsUpdated:
),
(
0,
100,
"""AttributeError(\'int\' object has no attribute \'encode\')""",
102,
"Field: <name> - Message: <Input should be a valid string> - Value: <0>",
),
(
None,
100,
"""AttributeError(\'NoneType\' object has no attribute \'encode\')""",
"AttributeError('NoneType' object has no attribute 'encode')",
),
(
"",
@@ -93,7 +94,7 @@ class TestDocumentsUpdated:
else:
assert res["message"] == expected_message
@pytest.mark.p3
@pytest.mark.p2
@pytest.mark.parametrize(
"document_id, expected_code, expected_message",
[
@@ -110,7 +111,7 @@ class TestDocumentsUpdated:
assert res["code"] == expected_code
assert res["message"] == expected_message
@pytest.mark.p3
@pytest.mark.p2
@pytest.mark.parametrize(
"dataset_id, expected_code, expected_message",
[
@@ -127,10 +128,28 @@ class TestDocumentsUpdated:
assert res["code"] == expected_code
assert res["message"] == expected_message
@pytest.mark.p3
@pytest.mark.p2
@pytest.mark.parametrize(
"meta_fields, expected_code, expected_message",
[({"test": "test"}, 0, ""), ("test", 102, "meta_fields must be a dictionary")],
[
# Valid meta_fields
({"test": "test"}, 0, ""),
# Valid meta_fields with various types
({"author": "alice", "year": 2024}, 0, ""),
({"tags": ["tag1", "tag2"]}, 0, ""),
({"count": 42, "price": 19.99}, 0, ""),
# Invalid type - string instead of dict
("test", 102, "Field: <meta_fields> - Message: <Input should be a valid dictionary> - Value: <test>"),
# Invalid type - list instead of dict
([], 102, "Field: <meta_fields> - Message: <Input should be a valid dictionary> - Value: <[]>"),
# Invalid - list containing objects (unsupported type in list)
({"tags": [{"x": {"a": "b"}}]}, 102, "Field: <meta_fields> - Message: <The type is not supported in list: [{'x': {'a': 'b'}}]> - Value: <{'tags': [{'x': {'a': 'b'}}]}>"),
({"tags": [{"x": 1}]}, 102, "Field: <meta_fields> - Message: <The type is not supported in list: [{'x': 1}]> - Value: <{'tags': [{'x': 1}]}>"),
# Invalid - nested object with unsupported type
({"obj": {"x": 1}}, 102, "Field: <meta_fields> - Message: <The type is not supported: {'x': 1}> - Value: <{'obj': {'x': 1}}>"),
# Valid types of list
({"tags": [2, 1]}, 0, ""),
],
)
def test_meta_fields(self, HttpApiAuth, add_documents, meta_fields, expected_code, expected_message):
dataset_id, document_ids = add_documents
@@ -139,7 +158,22 @@ class TestDocumentsUpdated:
res = list_documents(HttpApiAuth, dataset_id, {"id": document_ids[0]})
assert res["data"]["docs"][0]["meta_fields"] == meta_fields
else:
assert res["message"] == expected_message
assert expected_message in res["message"] or res["message"] == expected_message
@pytest.mark.p2
@pytest.mark.parametrize(
"meta_fields, expected_code, expected_message",
[
# Test with invalid document ID (not owned by dataset)
({"author": "alice"}, 102, "The dataset doesn't own the document."),
],
)
def test_meta_fields_invalid_document(self, HttpApiAuth, add_documents, meta_fields, expected_code, expected_message):
"""Test meta_fields update with invalid document ID"""
dataset_id, _ = add_documents
res = update_document(HttpApiAuth, dataset_id, "invalid_doc_id_12345678901234567890", {"meta_fields": meta_fields})
assert res["code"] == expected_code
assert expected_message in res["message"]
@pytest.mark.p2
@pytest.mark.parametrize(
@@ -297,6 +331,30 @@ class TestDocumentsUpdated:
assert res["code"] == expected_code
assert res["message"] == expected_message
@pytest.mark.p2
@pytest.mark.parametrize(
"payload, expected_code, expected_message",
[
({"chunk_count": 100}, 102, "Can't change `chunk_count`."),
({"token_count": 100}, 102, "Can't change `token_count`."),
({"progress": 2.0}, 102, "Field: <progress> - Message: <Input should be less than or equal to 1> - Value: <2.0>"),
({"progress": 1.0}, 102, "Can't change `progress`."),
({"meta_fields": []}, 102, "Field: <meta_fields> - Message: <Input should be a valid dictionary> - Value: <[]>"),
],
)
def test_update_doc_guards_and_error_paths(self, HttpApiAuth, add_documents, payload, expected_code, expected_message):
"""
Test various guard conditions and error paths for document update functionality.
This includes testing for invalid dataset ownership, document ownership,
immutable fields, and validation errors.
"""
dataset_id, document_ids = add_documents
document_id = document_ids[0]
res = update_document(HttpApiAuth, dataset_id, document_id, payload)
assert res["code"] == expected_code
if expected_message:
assert expected_message in res["message"] or res["message"] == expected_message
DEFAULT_PARSER_CONFIG_FOR_TEST = {
@@ -328,6 +386,7 @@ DEFAULT_PARSER_CONFIG_FOR_TEST = {
},
}
class TestUpdateDocumentParserConfig:
@pytest.mark.p2
@pytest.mark.parametrize(
@@ -343,37 +402,32 @@ class TestUpdateDocumentParserConfig:
pytest.param(
"naive",
{"chunk_token_num": -1},
100,
"AssertionError('chunk_token_num should be in range from 1 to 100000000')",
marks=pytest.mark.skip(reason="issues/6098"),
102,
"Field: <parser_config.chunk_token_num> - Message: <Input should be greater than or equal to 1> - Value: <-1>",
),
pytest.param(
"naive",
{"chunk_token_num": 0},
100,
"AssertionError('chunk_token_num should be in range from 1 to 100000000')",
marks=pytest.mark.skip(reason="issues/6098"),
102,
"Field: <parser_config.chunk_token_num> - Message: <Input should be greater than or equal to 1> - Value: <0>",
),
pytest.param(
"naive",
{"chunk_token_num": 100000000},
100,
"AssertionError('chunk_token_num should be in range from 1 to 100000000')",
marks=pytest.mark.skip(reason="issues/6098"),
102,
"Field: <parser_config.chunk_token_num> - Message: <Input should be less than or equal to 2048> - Value: <100000000>",
),
pytest.param(
"naive",
{"chunk_token_num": 3.14},
102,
"",
marks=pytest.mark.skip(reason="issues/6098"),
"Field: <parser_config.chunk_token_num> - Message: <Input should be a valid integer> - Value: <3.14>",
),
pytest.param(
"naive",
{"chunk_token_num": "1024"},
100,
"",
marks=pytest.mark.skip(reason="issues/6098"),
102,
"Field: <parser_config.chunk_token_num> - Message: <Input should be a valid integer> - Value: <1024>",
),
(
"naive",
@@ -392,148 +446,135 @@ class TestUpdateDocumentParserConfig:
pytest.param(
"naive",
{"html4excel": 1},
100,
"AssertionError('html4excel should be True or False')",
marks=pytest.mark.skip(reason="issues/6098"),
102,
"Field: <parser_config.html4excel> - Message: <Input should be a valid boolean> - Value: <1>",
),
("naive", {"delimiter": ""}, 102, "Field: <parser_config.delimiter> - Message: <String should have at least 1 character> - Value: <>"),
("naive", {"delimiter": "`##`"}, 0, ""),
pytest.param(
"naive",
{"delimiter": 1},
100,
"",
marks=pytest.mark.skip(reason="issues/6098"),
102,
"Field: <parser_config.delimiter> - Message: <Input should be a valid string> - Value: <1>",
),
pytest.param(
"naive",
{"task_page_size": -1},
100,
"AssertionError('task_page_size should be in range from 1 to 100000000')",
marks=pytest.mark.skip(reason="issues/6098"),
102,
"Field: <parser_config.task_page_size> - Message: <Input should be greater than or equal to 1> - Value: <-1>",
),
pytest.param(
"naive",
{"task_page_size": 0},
100,
"AssertionError('task_page_size should be in range from 1 to 100000000')",
marks=pytest.mark.skip(reason="issues/6098"),
102,
"Field: <parser_config.task_page_size> - Message: <Input should be greater than or equal to 1> - Value: <0>",
),
pytest.param(
"naive",
{"task_page_size": 100000000},
100,
"AssertionError('task_page_size should be in range from 1 to 100000000')",
marks=pytest.mark.skip(reason="issues/6098"),
0,
"",
),
pytest.param(
"naive",
{"task_page_size": 3.14},
100,
"",
marks=pytest.mark.skip(reason="issues/6098"),
102,
"Field: <parser_config.task_page_size> - Message: <Input should be a valid integer> - Value: <3.14>",
),
pytest.param(
"naive",
{"task_page_size": "1024"},
100,
"",
marks=pytest.mark.skip(reason="issues/6098"),
102,
"Field: <parser_config.task_page_size> - Message: <Input should be a valid integer> - Value: <1024>",
),
(
"naive",
{
"raptor": {
"use_raptor": {"a": "b"},
}
},
102,
"Field: <parser_config.raptor.use_raptor> - Message: <Input should be a valid boolean> - Value: <{'a': 'b'}>",
),
("naive", {"raptor": {"use_raptor": {
"a": "b"
},}}, 102, "Field: <parser_config.raptor.use_raptor> - Message: <Input should be a valid boolean> - Value: <{'a': 'b'}>"),
("naive", {"raptor": {"use_raptor": False}}, 0, ""),
pytest.param(
"naive",
{"invalid_key": "invalid_value"},
100,
"""AssertionError("Abnormal \'parser_config\'. Invalid key: invalid_key")""",
marks=pytest.mark.skip(reason="issues/6098"),
102,
"Field: <parser_config.invalid_key> - Message: <Extra inputs are not permitted> - Value: <invalid_value>",
),
pytest.param(
"naive",
{"auto_keywords": -1},
100,
"AssertionError('auto_keywords should be in range from 0 to 32')",
marks=pytest.mark.skip(reason="issues/6098"),
102,
"Field: <parser_config.auto_keywords> - Message: <Input should be greater than or equal to 0> - Value: <-1>",
),
pytest.param(
"naive",
{"auto_keywords": 32},
100,
"AssertionError('auto_keywords should be in range from 0 to 32')",
marks=pytest.mark.skip(reason="issues/6098"),
),
pytest.param(
"naive",
{"auto_questions": 3.14},
100,
0,
"",
marks=pytest.mark.skip(reason="issues/6098"),
),
pytest.param(
"naive",
{"auto_keywords": "1024"},
100,
"",
marks=pytest.mark.skip(reason="issues/6098"),
102,
"Field: <parser_config.auto_keywords> - Message: <Input should be a valid integer> - Value: <1024>",
),
pytest.param(
"naive",
{"auto_keywords": 3.14},
102,
"Field: <parser_config.auto_keywords> - Message: <Input should be a valid integer> - Value: <3.14>",
),
pytest.param(
"naive",
{"auto_questions": -1},
100,
"AssertionError('auto_questions should be in range from 0 to 10')",
marks=pytest.mark.skip(reason="issues/6098"),
102,
"Field: <parser_config.auto_questions> - Message: <Input should be greater than or equal to 0> - Value: <-1>",
),
pytest.param(
"naive",
{"auto_questions": 10},
100,
"AssertionError('auto_questions should be in range from 0 to 10')",
marks=pytest.mark.skip(reason="issues/6098"),
0,
"",
),
pytest.param(
"naive",
{"auto_questions": 3.14},
100,
"",
marks=pytest.mark.skip(reason="issues/6098"),
102,
"Field: <parser_config.auto_questions> - Message: <Input should be a valid integer> - Value: <3.14>",
),
pytest.param(
"naive",
{"auto_questions": "1024"},
100,
"",
marks=pytest.mark.skip(reason="issues/6098"),
102,
"Field: <parser_config.auto_questions> - Message: <Input should be a valid integer> - Value: <1024>",
),
pytest.param(
"naive",
{"topn_tags": -1},
100,
"AssertionError('topn_tags should be in range from 0 to 10')",
marks=pytest.mark.skip(reason="issues/6098"),
102,
"Field: <parser_config.topn_tags> - Message: <Input should be greater than or equal to 1> - Value: <-1>",
),
pytest.param(
"naive",
{"topn_tags": 10},
100,
"AssertionError('topn_tags should be in range from 0 to 10')",
marks=pytest.mark.skip(reason="issues/6098"),
0,
"",
),
pytest.param(
"naive",
{"topn_tags": 3.14},
100,
"",
marks=pytest.mark.skip(reason="issues/6098"),
102,
"Field: <parser_config.topn_tags> - Message: <Input should be a valid integer> - Value: <3.14>",
),
pytest.param(
"naive",
{"topn_tags": "1024"},
100,
"",
marks=pytest.mark.skip(reason="issues/6098"),
102,
"Field: <parser_config.topn_tags> - Message: <Input should be a valid integer> - Value: <1024>",
),
],
)

105
test/testcases/test_sdk_api/test_file_management_within_dataset/test_update_document.py
View File

@@ -67,7 +67,7 @@ class TestDocumentsUpdated:
updated_doc = dataset.list_documents(id=document.id)[0]
assert updated_doc.name == name, str(updated_doc)
@pytest.mark.p3
@pytest.mark.p2
@pytest.mark.parametrize(
"meta_fields, expected_message",
[
@@ -238,27 +238,81 @@ class TestDocumentsUpdated:
document.update(payload)
assert expected_message in str(exception_info.value), str(exception_info.value)
@pytest.mark.p3
def test_immutable_fields_chunk_count(self, add_document):
document, _ = add_document # Unpack the tuple to get the document object
with pytest.raises(Exception) as exception_info:
document.update({"chunk_count": 999}) # Attempt to change immutable field
assert "Can't change `chunk_count`" in str(exception_info.value), str(exception_info.value)
@pytest.mark.p2
@pytest.mark.parametrize(
"payload, expected_message",
[
({"chunk_count": 1}, "Can't change `chunk_count`"),
],
)
def test_immutable_fields_chunk_count(self, add_documents, payload, expected_message):
_, documents = add_documents
document = documents[0]
@pytest.mark.p3
def test_immutable_fields_token_count(self, add_document):
document, _ = add_document # Unpack the tuple to get the document object
with pytest.raises(Exception) as exception_info:
document.update({"token_count": 9999}) # Attempt to change immutable field
assert "Can't change `token_num`" in str(exception_info.value), str(exception_info.value)
document.update(payload)
assert expected_message in str(exception_info.value), str(exception_info.value)
@pytest.mark.p2
@pytest.mark.parametrize(
"payload, expected_message",
[
({"token_count": 9999}, "Can't change `token_count`"), # Attempt to change immutable field
],
)
def test_immutable_fields_token_count(self, add_documents, payload, expected_message):
_, documents = add_documents
document = documents[0]
@pytest.mark.p3
def test_immutable_fields_progress(self, add_document):
document, _ = add_document # Unpack the tuple to get the document object
with pytest.raises(Exception) as exception_info:
document.update({"progress": 0.5}) # Attempt to change immutable field
assert "Can't change `progress`" in str(exception_info.value), str(exception_info.value)
document.update(payload)
assert expected_message in str(exception_info.value), str(exception_info.value)
@pytest.mark.p2
@pytest.mark.parametrize(
"payload, expected_message",
[
({"progress": 0.5}, "Can't change `progress`"), # Attempt to change immutable field
({"progress": 1.5}, "Field: <progress> - Message: <Input should be less than or equal to 1> - Value: <1.5>"), # Attempt to change immutable field
],
)
def test_immutable_fields_progress(self, add_documents, payload, expected_message):
_, documents = add_documents
document = documents[0]
with pytest.raises(Exception) as exception_info:
document.update(payload)
assert expected_message in str(exception_info.value), str(exception_info.value)
DEFAULT_PARSER_CONFIG_FOR_TEST = {
"layout_recognize": "DeepDOC",
"chunk_token_num": 512,
"delimiter": "\n",
"auto_keywords": 0,
"auto_questions": 0,
"html4excel": False,
"topn_tags": 3,
"raptor": {
"use_raptor": True,
"prompt": "Please summarize the following paragraphs. Be careful with the numbers, do not make things up. Paragraphs as following:\n {cluster_content}\nThe above is the content you need to summarize.",
"max_token": 256,
"threshold": 0.1,
"max_cluster": 64,
"random_seed": 0,
},
"graphrag": {
"use_graphrag": True,
"entity_types": [
"organization",
"person",
"geo",
"event",
"category",
],
"method": "light",
},
}
class TestUpdateDocumentParserConfig:
@pytest.mark.p2
@@ -268,15 +322,14 @@ class TestUpdateDocumentParserConfig:
("naive", {}, ""),
pytest.param(
"naive",
DEFAULT_PARSER_CONFIG,
DEFAULT_PARSER_CONFIG_FOR_TEST,
"",
marks=pytest.mark.skip(reason="DEFAULT_PARSER_CONFIG contains fields not allowed in document update API"),
),
pytest.param(
"naive",
{"chunk_token_num": -1},
"chunk_token_num should be in range from 1 to 100000000",
marks=pytest.mark.skip(reason="issues/6098"),
"Field: <parser_config.chunk_token_num> - Message: <Input should be greater than or equal to 1> - Value: <-1>",
),
(
"naive",
@@ -327,8 +380,7 @@ class TestUpdateDocumentParserConfig:
pytest.param(
"naive",
{"task_page_size": 100000000},
"task_page_size should be in range from 1 to 100000000",
marks=pytest.mark.skip(reason="API validation differs from expected message"),
"",
),
(
"naive",
@@ -360,8 +412,7 @@ class TestUpdateDocumentParserConfig:
pytest.param(
"naive",
{"auto_keywords": 32},
"auto_keywords should be in range from 0 to 32",
marks=pytest.mark.skip(reason="API validation differs from expected message"),
"",
),
(
"naive",
@@ -381,8 +432,7 @@ class TestUpdateDocumentParserConfig:
pytest.param(
"naive",
{"auto_questions": 10},
"auto_questions should be in range from 0 to 10",
marks=pytest.mark.skip(reason="API validation differs from expected message"),
"",
),
(
"naive",
@@ -402,8 +452,7 @@ class TestUpdateDocumentParserConfig:
pytest.param(
"naive",
{"topn_tags": 10},
"topn_tags should be in range from 0 to 10",
marks=pytest.mark.skip(reason="API validation differs from expected message"),
"",
),
(
"naive",

10
test/testcases/test_web_api/common.py
View File

@@ -435,16 +435,6 @@ def document_change_status(auth, payload=None, *, headers=HEADERS, data=None):
return res.json()
def document_rename(auth, payload=None, *, headers=HEADERS, data=None):
res = requests.post(url=f"{HOST_ADDRESS}{DOCUMENT_APP_URL}/rename", headers=headers, auth=auth, json=payload, data=data)
return res.json()
def document_set_meta(auth, payload=None, *, headers=HEADERS, data=None):
res = requests.post(url=f"{HOST_ADDRESS}{DOCUMENT_APP_URL}/set_meta", headers=headers, auth=auth, json=payload, data=data)
return res.json()
def bulk_upload_documents(auth, kb_id, num, tmp_path):
fps = []
for i in range(num):

181
test/testcases/test_web_api/test_document_app/test_document_metadata.py
View File

@@ -22,8 +22,6 @@ from common import (
document_filter,
document_infos,
document_metadata_summary,
document_rename,
document_set_meta,
document_update_metadata_setting,
)
from configs import INVALID_API_TOKEN
@@ -82,21 +80,6 @@ class TestAuthorization:
assert res["code"] == expected_code, res
assert expected_fragment in res["message"], res
@pytest.mark.p2
@pytest.mark.parametrize("invalid_auth, expected_code, expected_fragment", INVALID_AUTH_CASES)
def test_rename_auth_invalid(self, invalid_auth, expected_code, expected_fragment):
res = document_rename(invalid_auth, {"doc_id": "doc_id", "name": "rename.txt"})
assert res["code"] == expected_code, res
assert expected_fragment in res["message"], res
@pytest.mark.p2
@pytest.mark.parametrize("invalid_auth, expected_code, expected_fragment", INVALID_AUTH_CASES)
def test_set_meta_auth_invalid(self, invalid_auth, expected_code, expected_fragment):
res = document_set_meta(invalid_auth, {"doc_id": "doc_id", "meta": "{}"})
assert res["code"] == expected_code, res
assert expected_fragment in res["message"], res
class TestDocumentMetadata:
@pytest.mark.p2
def test_filter(self, WebApiAuth, add_dataset_func):
@@ -163,28 +146,6 @@ class TestDocumentMetadata:
assert info_res["code"] == 0, info_res
assert info_res["data"][0]["status"] == "1", info_res
@pytest.mark.p2
def test_rename(self, WebApiAuth, add_document_func):
_, doc_id = add_document_func
name = f"renamed_{doc_id}.txt"
res = document_rename(WebApiAuth, {"doc_id": doc_id, "name": name})
assert res["code"] == 0, res
assert res["data"] is True, res
info_res = document_infos(WebApiAuth, {"doc_ids": [doc_id]})
assert info_res["code"] == 0, info_res
assert info_res["data"][0]["name"] == name, info_res
@pytest.mark.p2
def test_set_meta(self, WebApiAuth, add_document_func):
_, doc_id = add_document_func
res = document_set_meta(WebApiAuth, {"doc_id": doc_id, "meta": "{\"author\": \"alice\"}"})
assert res["code"] == 0, res
assert res["data"] is True, res
info_res = document_infos(WebApiAuth, {"doc_ids": [doc_id]})
assert info_res["code"] == 0, info_res
meta_fields = info_res["data"][0].get("meta_fields", {})
assert meta_fields.get("author") == "alice", info_res
class TestDocumentMetadataNegative:
@pytest.mark.p3
@@ -231,20 +192,6 @@ class TestDocumentMetadataNegative:
assert res["code"] == 101, res
assert "Status" in res["message"], res
@pytest.mark.p3
def test_rename_extension_mismatch(self, WebApiAuth, add_document_func):
_, doc_id = add_document_func
res = document_rename(WebApiAuth, {"doc_id": doc_id, "name": "renamed.pdf"})
assert res["code"] == 101, res
assert "extension" in res["message"], res
@pytest.mark.p3
def test_set_meta_invalid_type(self, WebApiAuth, add_document_func):
_, doc_id = add_document_func
res = document_set_meta(WebApiAuth, {"doc_id": doc_id, "meta": "[]"})
assert res["code"] == 101, res
assert "dictionary" in res["message"], res
def _run(coro):
return asyncio.run(coro)
@@ -590,87 +537,6 @@ class TestDocumentMetadataUnit:
assert res["code"] == 0
assert res["data"]["doc1"]["status"] == "1"
def test_rename_branch_matrix_and_exception_unit(self, document_app_module, monkeypatch):
module = document_app_module
file_updates = []
es_updates = []
async def fake_thread_pool_exec(func, *_args, **_kwargs):
return func()
monkeypatch.setattr(module, "thread_pool_exec", fake_thread_pool_exec)
monkeypatch.setattr(module.DocumentService, "get_tenant_id", lambda _doc_id: "tenant1")
monkeypatch.setattr(module.rag_tokenizer, "tokenize", lambda _name: ["token"])
monkeypatch.setattr(module.rag_tokenizer, "fine_grained_tokenize", lambda _tokens: ["fine"])
monkeypatch.setattr(module, "server_error_response", lambda e: {"code": 500, "message": str(e)})
class _DocStore:
def index_exist(self, _index_name, _kb_id):
return True
def update(self, where, payload, _index_name, _kb_id):
es_updates.append((where, payload))
monkeypatch.setattr(module.settings, "docStoreConn", _DocStore())
monkeypatch.setattr(module.search, "index_name", lambda tenant_id: f"idx_{tenant_id}")
def set_req(name):
async def fake_request_json():
return {"doc_id": "doc1", "name": name}
monkeypatch.setattr(module, "get_request_json", fake_request_json)
set_req("renamed.txt")
monkeypatch.setattr(module.DocumentService, "accessible", lambda *_args, **_kwargs: False)
res = _run(module.rename.__wrapped__())
assert res["code"] == module.RetCode.AUTHENTICATION_ERROR
monkeypatch.setattr(module.DocumentService, "accessible", lambda *_args, **_kwargs: True)
monkeypatch.setattr(module.DocumentService, "get_by_id", lambda _doc_id: (False, None))
res = _run(module.rename.__wrapped__())
assert res["code"] == module.RetCode.DATA_ERROR
assert "Document not found!" in res["message"]
monkeypatch.setattr(module.DocumentService, "get_by_id", lambda _doc_id: (True, SimpleNamespace(id="doc1", name="origin.txt", kb_id="kb1")))
set_req("renamed.pdf")
res = _run(module.rename.__wrapped__())
assert res["code"] == module.RetCode.ARGUMENT_ERROR
assert "extension" in res["message"]
too_long = "a" * (module.FILE_NAME_LEN_LIMIT + 1) + ".txt"
set_req(too_long)
res = _run(module.rename.__wrapped__())
assert res["code"] == module.RetCode.ARGUMENT_ERROR
assert "bytes or less" in res["message"]
set_req("dup.txt")
monkeypatch.setattr(module.DocumentService, "query", lambda **_kwargs: [SimpleNamespace(name="dup.txt")])
res = _run(module.rename.__wrapped__())
assert res["code"] == module.RetCode.DATA_ERROR
assert "Duplicated document name" in res["message"]
set_req("ok.txt")
monkeypatch.setattr(module.DocumentService, "query", lambda **_kwargs: [])
monkeypatch.setattr(module.DocumentService, "update_by_id", lambda *_args, **_kwargs: True)
monkeypatch.setattr(module.File2DocumentService, "get_by_document_id", lambda _doc_id: [SimpleNamespace(file_id="file1")])
monkeypatch.setattr(module.FileService, "get_by_id", lambda _file_id: (True, SimpleNamespace(id="file1")))
monkeypatch.setattr(module.FileService, "update_by_id", lambda file_id, payload: file_updates.append((file_id, payload)))
res = _run(module.rename.__wrapped__())
assert res["code"] == 0
assert file_updates == [("file1", {"name": "ok.txt"})]
assert es_updates[0][0] == {"doc_id": "doc1"}
assert es_updates[0][1]["docnm_kwd"] == "ok.txt"
assert es_updates[0][1]["title_tks"] == ["token"]
assert es_updates[0][1]["title_sm_tks"] == ["fine"]
def raise_db_error(*_args, **_kwargs):
raise RuntimeError("rename boom")
monkeypatch.setattr(module.DocumentService, "update_by_id", raise_db_error)
res = _run(module.rename.__wrapped__())
assert res["code"] == 500
assert "rename boom" in res["message"]
def test_get_route_not_found_success_and_exception_unit(self, document_app_module, monkeypatch):
module = document_app_module
monkeypatch.setattr(module.DocumentService, "get_by_id", lambda _doc_id: (False, None))
@@ -932,50 +798,3 @@ class TestDocumentMetadataUnit:
res = _run(module.get_image("bucket-name"))
assert res["code"] == 500
assert "image boom" in res["message"]
def test_set_meta_validation_and_persistence_matrix_unit(self, document_app_module, monkeypatch):
module = document_app_module
def set_req(payload):
async def fake_request_json():
return payload
monkeypatch.setattr(module, "get_request_json", fake_request_json)
set_req({"doc_id": "doc1", "meta": "{}"})
monkeypatch.setattr(module.DocumentService, "accessible", lambda *_args, **_kwargs: False)
res = _run(module.set_meta.__wrapped__())
assert res["code"] == module.RetCode.AUTHENTICATION_ERROR
monkeypatch.setattr(module.DocumentService, "accessible", lambda *_args, **_kwargs: True)
set_req({"doc_id": "doc1", "meta": "[]"})
res = _run(module.set_meta.__wrapped__())
assert res["code"] == module.RetCode.ARGUMENT_ERROR
assert "Only dictionary type supported." in res["message"]
set_req({"doc_id": "doc1", "meta": '{"tags":[{"x":1}]}'})
res = _run(module.set_meta.__wrapped__())
assert res["code"] == module.RetCode.ARGUMENT_ERROR
assert "The type is not supported in list" in res["message"]
set_req({"doc_id": "doc1", "meta": '{"obj":{"x":1}}'})
res = _run(module.set_meta.__wrapped__())
assert res["code"] == module.RetCode.ARGUMENT_ERROR
assert "The type is not supported" in res["message"]
set_req({"doc_id": "doc1", "meta": "{"})
res = _run(module.set_meta.__wrapped__())
assert res["code"] == module.RetCode.ARGUMENT_ERROR
assert "Json syntax error:" in res["message"]
set_req({"doc_id": "doc1", "meta": '{"author":"alice"}'})
monkeypatch.setattr(module.DocumentService, "get_by_id", lambda _doc_id: (False, None))
res = _run(module.set_meta.__wrapped__())
assert res["code"] == module.RetCode.DATA_ERROR
assert "Document not found!" in res["message"]
monkeypatch.setattr(module.DocumentService, "get_by_id", lambda _doc_id: (True, SimpleNamespace(id="doc1")))
monkeypatch.setattr(module.DocMetadataService, "update_document_metadata", lambda *_args, **_kwargs: False)
res = _run(module.set_meta.__wrapped__())
assert res["code"] == module.RetCode.DATA_ERROR
assert "Database error (meta updates)!" in res["message"]