Fix link file to dataset check the duplicate name (#846) (#16975)

api/apps/restful_apis/file2document_api.py

@@ -21,6 +21,7 @@ from pathlib import Path
 from quart import request
 
 from api.common.check_team_permission import check_file_team_permission, check_kb_team_permission
+from api.db.services import duplicate_name
 from api.db.services.file2document_service import File2DocumentService
 from api.db.services.file_service import FileService
 
@@ -70,17 +71,18 @@ def _convert_files(file_ids, kb_ids, user_id, mode):
             e, kb = KnowledgebaseService.get_by_id(kb_id)
             if not e:
                 continue
+            filename = duplicate_name(DocumentService.query, name=file.name, kb_id=kb.id)
             doc = DocumentService.insert(
                 {
                     "id": get_uuid(),
                     "kb_id": kb.id,
-                    "parser_id": FileService.get_parser(file.type, file.name, kb.parser_id),
+                    "parser_id": FileService.get_parser(file.type, filename, kb.parser_id),
                     "pipeline_id": kb.pipeline_id,
                     "parser_config": kb.parser_config,
                     "created_by": user_id,
                     "type": file.type,
-                    "name": file.name,
-                    "suffix": Path(file.name).suffix.lstrip("."),
+                    "name": filename,
+                    "suffix": Path(filename).suffix.lstrip("."),
                     "location": file.location,
                     "size": file.size,
                 }

test/testcases/restful_api/test_file_routes_unit.py

@@ -443,6 +443,7 @@ def _load_file2document_module(monkeypatch):
 
     services_pkg = ModuleType("api.db.services")
     services_pkg.__path__ = []
+    services_pkg.duplicate_name = lambda _query_func, **kwargs: "file(1).txt" if kwargs.get("name") == "file.txt" and kwargs.get("kb_id") == "kb-dup" else kwargs.get("name")
     monkeypatch.setitem(sys.modules, "api.db.services", services_pkg)
 
     common_pkg = ModuleType("api.common")
@@ -527,6 +528,10 @@ def _load_file2document_module(monkeypatch):
         def remove_document(*_args, **_kwargs):
             return True
 
+        @staticmethod
+        def query(*_args, **_kwargs):
+            return False
+
         @staticmethod
         def insert(_payload):
             return SimpleNamespace(id="doc-1")
@@ -684,6 +689,24 @@ def test_convert_files_mode_add_and_replace_unit(monkeypatch):
     assert deleted_file_links == ["f1", "f2"]
 
 
+@pytest.mark.p2
+def test_convert_files_renames_duplicate_document_name_unit(monkeypatch):
+    module = _load_file2document_module(monkeypatch)
+    inserted_docs = []
+    file = _DummyFile("f1", module.FileType.DOC.value, name="file.txt")
+    kb = SimpleNamespace(id="kb-dup", parser_id="naive", pipeline_id="p1", parser_config={})
+
+    monkeypatch.setattr(module.FileService, "get_by_id", lambda _file_id: (True, file))
+    monkeypatch.setattr(module.KnowledgebaseService, "get_by_id", lambda _kb_id: (True, kb))
+    monkeypatch.setattr(module.DocumentService, "insert", lambda payload: inserted_docs.append(payload) or SimpleNamespace(id="doc-1"))
+
+    module._convert_files(["f1"], ["kb-dup"], "user-1", "add")
+
+    assert inserted_docs[0]["name"] == "file(1).txt"
+    assert inserted_docs[0]["suffix"] == "txt"
+    assert inserted_docs[0]["location"] == "loc"
+
+
 def _load_file_api_service(monkeypatch):
     LOGGER.debug("_load_file_api_service: entry")
     repo_root = Path(__file__).resolve().parents[3]

test/testcases/test_web_api/test_file_app/test_file2document_routes_unit.py

@@ -100,6 +100,7 @@ def _load_file2document_module(monkeypatch):
 
     services_pkg = ModuleType("api.db.services")
     services_pkg.__path__ = []
+    services_pkg.duplicate_name = lambda _query_func, **kwargs: "file(1).txt" if kwargs.get("name") == "file.txt" and kwargs.get("kb_id") == "kb-dup" else kwargs.get("name")
     monkeypatch.setitem(sys.modules, "api.db.services", services_pkg)
 
     common_pkg = ModuleType("api.common")
@@ -184,6 +185,10 @@ def _load_file2document_module(monkeypatch):
         def remove_document(*_args, **_kwargs):
             return True
 
+        @staticmethod
+        def query(*_args, **_kwargs):
+            return False
+
         @staticmethod
         def insert(_payload):
             return SimpleNamespace(id="doc-1")
@@ -300,3 +305,21 @@ def test_convert_branch_matrix_unit(monkeypatch):
     res = _run(module.convert())
     assert res["code"] == 500
     assert "convert boom" in res["message"]
+
+
+@pytest.mark.p2
+def test_convert_files_renames_duplicate_document_name_unit(monkeypatch):
+    module = _load_file2document_module(monkeypatch)
+    inserted_docs = []
+    file = _DummyFile("f1", module.FileType.DOC.value, name="file.txt")
+    kb = SimpleNamespace(id="kb-dup", parser_id="naive", pipeline_id="p1", parser_config={})
+
+    monkeypatch.setattr(module.FileService, "get_by_id", lambda _file_id: (True, file))
+    monkeypatch.setattr(module.KnowledgebaseService, "get_by_id", lambda _kb_id: (True, kb))
+    monkeypatch.setattr(module.DocumentService, "insert", lambda payload: inserted_docs.append(payload) or SimpleNamespace(id="doc-1"))
+
+    module._convert_files(["f1"], ["kb-dup"], "user-1", "add")
+
+    assert inserted_docs[0]["name"] == "file(1).txt"
+    assert inserted_docs[0]["suffix"] == "txt"
+    assert inserted_docs[0]["location"] == "loc"