Fix: #6098 - Add validation logic for parser_config when update document (#13911)

### What problem does this PR solve?

Add validation logic for parser_config.
Refactor the processing flow. Before change, validation logics and
update logics are mixed up - some validation logis executes followed by
some update logic executes and then another such
"validation-and-then-update" which is not good. After change, all
validation logic executes firstly. Update logic will be executed after
ALL validation logic executed.
Validation logic for parameters (that come from front end) will be
checked using Pydantic. For validation logic that depends on data from
DB, they will be in separate methods.

### Type of change

- [x] Bug Fix (non-breaking change which fixes an issue)
- [x] Refactoring

test/testcases/test_http_api/test_file_management_within_dataset/test_doc_sdk_routes_unit.py

@@ -22,6 +22,8 @@ from types import ModuleType, SimpleNamespace
 import numpy as np
 import pytest
 
+from api.db import FileType
+
 
 class _DummyManager:
     def route(self, *_args, **_kwargs):
@@ -69,7 +71,7 @@ class _DummyDoc:
         progress=0,
         process_duration=0,
         parser_id="naive",
-        doc_type=1,
+        doc_type=FileType.OTHER,
         status=True,
         run=0,
     ):
@@ -397,7 +399,7 @@ class TestDocRoutesUnit:
 
         monkeypatch.setattr(module, "get_request_json", lambda: _AwaitableValue({"meta_fields": []}))
         res = _run(module.update_doc.__wrapped__("tenant-1", "ds-1", "doc-1"))
-        assert res["message"] == "meta_fields must be a dictionary"
+        assert res["message"] == "Field: <meta_fields> - Message: <Input should be a valid dictionary> - Value: <[]>"
 
         monkeypatch.setattr(module, "get_request_json", lambda: _AwaitableValue({"meta_fields": {"k": "v"}}))
         monkeypatch.setattr(module.DocMetadataService, "update_document_metadata", lambda *_args, **_kwargs: False)
@@ -416,7 +418,8 @@ class TestDocRoutesUnit:
 
     def test_update_doc_chunk_method_enabled_and_db_error(self, monkeypatch):
         module = _load_doc_module(monkeypatch)
-        visual_doc = _DummyDoc(parser_id="naive", doc_type=module.FileType.VISUAL)
+        from api.db import FileType
+        visual_doc = _DummyDoc(parser_id="naive", doc_type=FileType.VISUAL)
         kb = SimpleNamespace(tenant_id="tenant-1")
         monkeypatch.setattr(module.KnowledgebaseService, "query", lambda **_kwargs: [1])
         monkeypatch.setattr(module.KnowledgebaseService, "get_by_id", lambda _id: (True, kb))
@@ -446,7 +449,7 @@ class TestDocRoutesUnit:
         monkeypatch.setattr(module.DocumentService, "get_by_id", lambda _id: (True, doc_for_enabled))
         monkeypatch.setattr(module.DocumentService, "update_by_id", lambda *_args, **_kwargs: False)
         _patch_docstore(monkeypatch, module, update=lambda *_args, **_kwargs: None, delete=lambda *_args, **_kwargs: None)
-        monkeypatch.setattr(module, "get_request_json", lambda: _AwaitableValue({"enabled": True}))
+        monkeypatch.setattr(module, "get_request_json", lambda: _AwaitableValue({"enabled": 1}))
         res = _run(module.update_doc.__wrapped__("tenant-1", "ds-1", "doc-1"))
         assert "Document update" in res["message"]
 

test/testcases/test_http_api/test_file_management_within_dataset/test_update_document.py

@@ -54,12 +54,12 @@ class TestDocumentsUpdated:
             (
                 0,
                 100,
-                """AttributeError("\'int\' object has no attribute \'encode\'")""",
+                """AttributeError(\'int\' object has no attribute \'encode\')""",
             ),
             (
                 None,
                 100,
-                """AttributeError("\'NoneType\' object has no attribute \'encode\'")""",
+                """AttributeError(\'NoneType\' object has no attribute \'encode\')""",
             ),
             (
                 "",
@@ -158,11 +158,11 @@ class TestDocumentsUpdated:
             ("knowledge_graph", 0, ""),
             ("email", 0, ""),
             ("tag", 0, ""),
-            ("", 102, "`chunk_method`  doesn't exist"),
+            ("", 102, "`chunk_method` (empty string) is not valid"),
             (
                 "other_chunk_method",
                 102,
-                "`chunk_method` other_chunk_method doesn't exist",
+                "Field: <chunk_method> - Message: <`chunk_method` other_chunk_method doesn't exist> - Value: <other_chunk_method>",
             ),
         ],
     )
@@ -298,6 +298,36 @@ class TestDocumentsUpdated:
         assert res["message"] == expected_message
 
 
+
+DEFAULT_PARSER_CONFIG_FOR_TEST = {
+    "layout_recognize": "DeepDOC",
+    "chunk_token_num": 512,
+    "delimiter": "\n",
+    "auto_keywords": 0,
+    "auto_questions": 0,
+    "html4excel": False,
+    "topn_tags": 3,
+    "raptor": {
+        "use_raptor": True,
+        "prompt": "Please summarize the following paragraphs. Be careful with the numbers, do not make things up. Paragraphs as following:\n      {cluster_content}\nThe above is the content you need to summarize.",
+        "max_token": 256,
+        "threshold": 0.1,
+        "max_cluster": 64,
+        "random_seed": 0,
+    },
+    "graphrag": {
+        "use_graphrag": True,
+        "entity_types": [
+            "organization",
+            "person",
+            "geo",
+            "event",
+            "category",
+        ],
+        "method": "light",
+    },
+}
+
 class TestUpdateDocumentParserConfig:
     @pytest.mark.p2
     @pytest.mark.parametrize(
@@ -306,7 +336,7 @@ class TestUpdateDocumentParserConfig:
             ("naive", {}, 0, ""),
             (
                 "naive",
-                DEFAULT_PARSER_CONFIG,
+                DEFAULT_PARSER_CONFIG_FOR_TEST,
                 0,
                 "",
             ),
@@ -366,7 +396,7 @@ class TestUpdateDocumentParserConfig:
                 "AssertionError('html4excel should be True or False')",
                 marks=pytest.mark.skip(reason="issues/6098"),
             ),
-            ("naive", {"delimiter": ""}, 0, ""),
+            ("naive", {"delimiter": ""}, 102, "Field: <parser_config.delimiter> - Message: <String should have at least 1 character> - Value: <>"),
             ("naive", {"delimiter": "`##`"}, 0, ""),
             pytest.param(
                 "naive",
@@ -411,13 +441,8 @@ class TestUpdateDocumentParserConfig:
                 marks=pytest.mark.skip(reason="issues/6098"),
             ),
             ("naive", {"raptor": {"use_raptor": {
-                "use_raptor": True,
-                "prompt": "Please summarize the following paragraphs. Be careful with the numbers, do not make things up. Paragraphs as following:\n      {cluster_content}\nThe above is the content you need to summarize.",
-                "max_token": 256,
-                "threshold": 0.1,
-                "max_cluster": 64,
-                "random_seed": 0,
-            },}}, 0, ""),
+                "a": "b"
+            },}}, 102, "Field: <parser_config.raptor.use_raptor> - Message: <Input should be a valid boolean> - Value: <{'a': 'b'}>"),
             ("naive", {"raptor": {"use_raptor": False}}, 0, ""),
             pytest.param(
                 "naive",