zlei6/ragflow
1
0
Fork 0
mirror of https://github.com/infiniflow/ragflow.git synced 2026-08-15 05:04:27 +08:00
Code Issues Packages Projects Releases Wiki Activity

Refactor: Doc change parser (#14327)

Browse Source 
### What problem does this PR solve?

Before migration
Web API: POST /v1/document/change_parser
HTTP API: PATCH /api/v1/datasets/<dataset_id>/documents

After consolidation, Restful API
PATCH /api/v1/datasets/<dataset_id>/documents

### Type of change

- [x] Refactoring
This commit is contained in:
Jack
2026-04-27 23:42:57 +08:00
committed by GitHub
parent 872ff08304
commit c81081f8ef
14 changed files with 272 additions and 326 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
test/testcases/test_web_api/test_common.py
View File

@@ -451,6 +451,12 @@ def document_change_status(auth, dataset_id, payload=None, *, headers=HEADERS, d
return res.json()
def document_update(auth, dataset_id, doc_id, payload=None, *, headers=HEADERS, data=None):
"""Update document via PATCH /api/v1/datasets/<dataset_id>/documents/<doc_id>"""
res = requests.patch(url=f"{HOST_ADDRESS}{DATASETS_URL}/{dataset_id}/documents/{doc_id}", headers=headers, auth=auth, json=payload, data=data)
return res.json()
def document_thumbnails(auth, params=None, *, headers=HEADERS, data=None):
"""Get document thumbnails.

3
test/testcases/test_web_api/test_document_app/conftest.py
View File

@@ -204,8 +204,9 @@ def document_rest_api_module(monkeypatch):
document_api_service_mod.map_doc_keys_with_run_status = _map_doc_keys_with_run_status
document_api_service_mod.update_document_name_only = lambda *_args, **_kwargs: None
document_api_service_mod.update_chunk_method_only = lambda *_args, **_kwargs: None
document_api_service_mod.update_chunk_method = lambda *_args, **_kwargs: None
document_api_service_mod.update_document_status_only = lambda *_args, **_kwargs: None
document_api_service_mod.reset_document_for_reparse = lambda *_args, **_kwargs: None
monkeypatch.setitem(sys.modules, "api.apps.services.document_api_service", document_api_service_mod)
module_path = repo_root / "api" / "apps" / "restful_apis" / "document_api.py"

213
test/testcases/test_web_api/test_document_app/test_document_metadata.py
View File

@@ -26,8 +26,10 @@ from test_common import (
document_update_metadata_setting,
bulk_upload_documents,
delete_document,
document_update,
)
from common.constants import RetCode
from configs import INVALID_API_TOKEN
from libs.auth import RAGFlowWebApiAuth
@@ -155,6 +157,57 @@ class TestDocumentMetadata:
assert info_res["data"]["docs"][0]["status"] == "1", info_res
@pytest.mark.p2
def test_update_document_change_parser(self, WebApiAuth, add_document_func):
"""Test updating document chunk_method via PATCH /api/v1/datasets/<dataset_id>/documents/<doc_id>."""
dataset_id, doc_id = add_document_func
# Get initial document info
res = document_infos(WebApiAuth, dataset_id, {"doc_ids": [doc_id]})
assert res["code"] == 0, res
original_parser_id = res["data"]["docs"][0].get("parser_id")
res = document_update(WebApiAuth, dataset_id, doc_id, {"chunk_method": "invalid_chunk_method"})
assert res["code"] == 102, res
assert res["message"] == "Field: <chunk_method> - Message: <`chunk_method` invalid_chunk_method doesn't exist> - Value: <invalid_chunk_method>", res
# Change to a different parser (naive bayes)
# valid_chunk_method = {"naive", "manual", "qa", "table", "paper", "book", "laws", "presentation", "picture", "one", "knowledge_graph", "email", "tag"}
new_parser_id = "naive"
if original_parser_id == new_parser_id:
new_parser_id = "paper"
document_update(WebApiAuth, dataset_id, doc_id, {"chunk_method": new_parser_id})
# Verify the document was updated
res = document_infos(WebApiAuth, dataset_id, {"doc_ids": [doc_id]})
assert res["code"] == 0, res
assert res["data"]["docs"][0]["chunk_method"] == new_parser_id, res
@pytest.mark.p2
def test_update_document_change_pipeline(self, WebApiAuth, add_document_func):
"""Test updating document pipeline via PATCH /api/v1/datasets/<dataset_id>/documents/<doc_id>."""
dataset_id, doc_id = add_document_func
# Get initial document info
res = document_infos(WebApiAuth, dataset_id, {"doc_ids": [doc_id]})
assert res["code"] == 0, res
original_pipeline_id = res["data"]["docs"][0].get("pipeline_id")
# Change to a different pipeline (if available)
# Note: This test assumes there's at least one other pipeline available
new_pipeline_id = "general" if original_pipeline_id != "general" else "resume"
res = document_update(WebApiAuth, dataset_id, doc_id, {"pipeline_id": new_pipeline_id})
assert res["code"] == 0, res
# Verify the document was updated
res = document_infos(WebApiAuth, dataset_id, {"doc_ids": [doc_id]})
assert res["code"] == 0, res
assert res["data"]["docs"][0]["pipeline_id"] == new_pipeline_id, res
class TestDocumentMetadataNegative:
@pytest.mark.p2
def test_filter_missing_kb_id(self, WebApiAuth, add_document_func):
@@ -292,7 +345,7 @@ class TestDocumentMetadataUnit:
module = document_app_module
monkeypatch.setattr(module.DocumentService, "get_by_id", lambda _doc_id: (False, None))
res = _run(module.get("doc1"))
assert res["code"] == module.RetCode.DATA_ERROR
assert res["code"] == RetCode.DATA_ERROR
assert "Document not found!" in res["message"]
async def fake_thread_pool_exec(*_args, **_kwargs):
@@ -356,164 +409,6 @@ class TestDocumentMetadataUnit:
assert res["code"] == 500
assert "download boom" in res["message"]
def test_change_parser_guards_and_reset_update_failure_unit(self, document_app_module, monkeypatch):
module = document_app_module
monkeypatch.setattr(module, "server_error_response", lambda e: {"code": 500, "message": str(e)})
async def req_auth_fail():
return {"doc_id": "doc1", "parser_id": "naive", "pipeline_id": "pipe2"}
monkeypatch.setattr(module, "get_request_json", req_auth_fail)
monkeypatch.setattr(module.DocumentService, "accessible", lambda *_args, **_kwargs: False)
res = _run(module.change_parser.__wrapped__())
assert res["code"] == module.RetCode.AUTHENTICATION_ERROR
monkeypatch.setattr(module.DocumentService, "accessible", lambda *_args, **_kwargs: True)
monkeypatch.setattr(module.DocumentService, "get_by_id", lambda _doc_id: (False, None))
res = _run(module.change_parser.__wrapped__())
assert res["code"] == module.RetCode.DATA_ERROR
assert "Document not found!" in res["message"]
async def req_same_pipeline():
return {"doc_id": "doc1", "parser_id": "naive", "pipeline_id": "pipe1"}
doc_same = SimpleNamespace(
id="doc1",
pipeline_id="pipe1",
parser_id="naive",
parser_config={"k": "v"},
token_num=0,
chunk_num=0,
process_duration=0,
kb_id="kb1",
type="doc",
name="doc.txt",
)
monkeypatch.setattr(module, "get_request_json", req_same_pipeline)
monkeypatch.setattr(module.DocumentService, "get_by_id", lambda _doc_id: (True, doc_same))
res = _run(module.change_parser.__wrapped__())
assert res["code"] == 0
calls = []
async def req_pipeline_change():
return {"doc_id": "doc1", "parser_id": "naive", "pipeline_id": "pipe2"}
doc = SimpleNamespace(
id="doc1",
pipeline_id="pipe1",
parser_id="naive",
parser_config={},
token_num=0,
chunk_num=0,
process_duration=0,
kb_id="kb1",
type="doc",
name="doc.txt",
)
def fake_update_by_id(doc_id, payload):
calls.append((doc_id, payload))
return True
monkeypatch.setattr(module, "get_request_json", req_pipeline_change)
monkeypatch.setattr(module.DocumentService, "get_by_id", lambda _doc_id: (True, doc))
monkeypatch.setattr(module.DocumentService, "update_by_id", fake_update_by_id)
res = _run(module.change_parser.__wrapped__())
assert res["code"] == 0
assert calls[0][1] == {"pipeline_id": "pipe2"}
assert calls[1][1]["run"] == module.TaskStatus.UNSTART.value
doc.token_num = 3
doc.chunk_num = 2
doc.process_duration = 9
monkeypatch.setattr(module.DocumentService, "increment_chunk_num", lambda *_args, **_kwargs: False)
res = _run(module.change_parser.__wrapped__())
assert res["code"] == 0
monkeypatch.setattr(module.DocumentService, "increment_chunk_num", lambda *_args, **_kwargs: True)
monkeypatch.setattr(module.DocumentService, "get_tenant_id", lambda _doc_id: None)
res = _run(module.change_parser.__wrapped__())
assert res["code"] == 0
side_effects = {"img": [], "delete": []}
class _DocStore:
def index_exist(self, _idx, _kb_id):
return True
def delete(self, where, _idx, kb_id):
side_effects["delete"].append((where["doc_id"], kb_id))
monkeypatch.setattr(module.DocumentService, "get_tenant_id", lambda _doc_id: "tenant1")
monkeypatch.setattr(module.DocumentService, "delete_chunk_images", lambda _doc, _tenant: side_effects["img"].append((_doc.id, _tenant)))
monkeypatch.setattr(module.search, "index_name", lambda tenant_id: f"idx_{tenant_id}")
monkeypatch.setattr(module.settings, "docStoreConn", _DocStore())
res = _run(module.change_parser.__wrapped__())
assert res["code"] == 0
assert ("doc1", "tenant1") in side_effects["img"]
assert ("doc1", "kb1") in side_effects["delete"]
async def req_same_parser_with_cfg():
return {"doc_id": "doc1", "parser_id": "naive", "parser_config": {"a": 1}}
doc_same_parser = SimpleNamespace(
id="doc1",
pipeline_id="pipe1",
parser_id="naive",
parser_config={"a": 1},
token_num=0,
chunk_num=0,
process_duration=0,
kb_id="kb1",
type="doc",
name="doc.txt",
)
monkeypatch.setattr(module, "get_request_json", req_same_parser_with_cfg)
monkeypatch.setattr(module.DocumentService, "get_by_id", lambda _doc_id: (True, doc_same_parser))
res = _run(module.change_parser.__wrapped__())
assert res["code"] == 0
async def req_same_parser_no_cfg():
return {"doc_id": "doc1", "parser_id": "naive"}
monkeypatch.setattr(module, "get_request_json", req_same_parser_no_cfg)
res = _run(module.change_parser.__wrapped__())
assert res["code"] == 0
parser_cfg_updates = []
async def req_parser_update():
return {"doc_id": "doc1", "parser_id": "paper", "pipeline_id": "", "parser_config": {"beta": True}}
doc_parser_update = SimpleNamespace(
id="doc1",
pipeline_id="pipe1",
parser_id="naive",
parser_config={"alpha": 1},
token_num=0,
chunk_num=0,
process_duration=0,
kb_id="kb1",
type="doc",
name="doc.txt",
)
monkeypatch.setattr(module, "get_request_json", req_parser_update)
monkeypatch.setattr(module.DocumentService, "get_by_id", lambda _doc_id: (True, doc_parser_update))
monkeypatch.setattr(module.DocumentService, "update_parser_config", lambda doc_id, cfg: parser_cfg_updates.append((doc_id, cfg)))
monkeypatch.setattr(module.DocumentService, "update_by_id", lambda *_args, **_kwargs: True)
res = _run(module.change_parser.__wrapped__())
assert res["code"] == 0
assert parser_cfg_updates == [("doc1", {"beta": True})]
def raise_parser_config(*_args, **_kwargs):
raise RuntimeError("parser boom")
monkeypatch.setattr(module.DocumentService, "update_parser_config", raise_parser_config)
res = _run(module.change_parser.__wrapped__())
assert res["code"] == 500
assert "parser boom" in res["message"]
@pytest.mark.skip(reason="Moved to /api/v1/documents/images/<image_id>")
def test_get_image_success_and_exception_unit(self, document_app_module, monkeypatch):