From aae45b959b1200f6e39ce9287a2057077018c630 Mon Sep 17 00:00:00 2001 From: Wang Qi Date: Thu, 23 Apr 2026 11:40:45 +0800 Subject: [PATCH] Refactor: API file2document (#14306) Refactor: API file2document --- api/apps/file_app.py | 464 ------------------ .../file2document_api.py} | 36 +- .../test_file2document_routes_unit.py | 64 +-- web/src/utils/api.ts | 4 +- 4 files changed, 4 insertions(+), 564 deletions(-) delete mode 100644 api/apps/file_app.py rename api/apps/{file2document_app.py => restful_apis/file2document_api.py} (75%) diff --git a/api/apps/file_app.py b/api/apps/file_app.py deleted file mode 100644 index 172b49ff85..0000000000 --- a/api/apps/file_app.py +++ /dev/null @@ -1,464 +0,0 @@ -# # -# # Copyright 2024 The InfiniFlow Authors. All Rights Reserved. -# # -# # Licensed under the Apache License, Version 2.0 (the "License"); -# # you may not use this file except in compliance with the License. -# # You may obtain a copy of the License at -# # -# # http://www.apache.org/licenses/LICENSE-2.0 -# # -# # Unless required by applicable law or agreed to in writing, software -# # distributed under the License is distributed on an "AS IS" BASIS, -# # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# # See the License for the specific language governing permissions and -# # limitations under the License -# # -# import logging -# import os -# import pathlib -# import re -# from quart import request, make_response -# from api.apps import login_required, current_user -# -# from api.common.check_team_permission import check_file_team_permission -# from api.db.services.document_service import DocumentService -# from api.db.services.file2document_service import File2DocumentService -# from api.utils.api_utils import server_error_response, get_data_error_result, validate_request -# from common.misc_utils import get_uuid, thread_pool_exec -# from common.constants import RetCode, FileSource -# from api.db import FileType -# from api.db.services import duplicate_name -# from api.db.services.file_service import FileService -# from api.utils.api_utils import get_json_result, get_request_json -# from api.utils.file_utils import filename_type -# from api.utils.web_utils import CONTENT_TYPE_MAP, apply_safe_file_response_headers -# from common import settings -# -# @manager.route('/upload', methods=['POST']) # noqa: F821 -# @login_required -# # @validate_request("parent_id") -# async def upload(): -# form = await request.form -# pf_id = form.get("parent_id") -# -# if not pf_id: -# root_folder = FileService.get_root_folder(current_user.id) -# pf_id = root_folder["id"] -# -# files = await request.files -# if 'file' not in files: -# return get_json_result( -# data=False, message='No file part!', code=RetCode.ARGUMENT_ERROR) -# file_objs = files.getlist('file') -# -# for file_obj in file_objs: -# if file_obj.filename == '': -# return get_json_result( -# data=False, message='No file selected!', code=RetCode.ARGUMENT_ERROR) -# file_res = [] -# try: -# e, pf_folder = FileService.get_by_id(pf_id) -# if not e: -# return get_data_error_result( message="Can't find this folder!") -# -# async def _handle_single_file(file_obj): -# MAX_FILE_NUM_PER_USER: int = int(os.environ.get('MAX_FILE_NUM_PER_USER', 0)) -# if 0 < MAX_FILE_NUM_PER_USER <= await thread_pool_exec(DocumentService.get_doc_count, current_user.id): -# return get_data_error_result( message="Exceed the maximum file number of a free user!") -# -# # split file name path -# if not file_obj.filename: -# file_obj_names = [pf_folder.name, file_obj.filename] -# else: -# full_path = '/' + file_obj.filename -# file_obj_names = full_path.split('/') -# file_len = len(file_obj_names) -# -# # get folder -# file_id_list = await thread_pool_exec(FileService.get_id_list_by_id, pf_id, file_obj_names, 1, [pf_id]) -# len_id_list = len(file_id_list) -# -# # create folder -# if file_len != len_id_list: -# e, file = await thread_pool_exec(FileService.get_by_id, file_id_list[len_id_list - 1]) -# if not e: -# return get_data_error_result(message="Folder not found!") -# last_folder = await thread_pool_exec(FileService.create_folder, file, file_id_list[len_id_list - 1], file_obj_names, -# len_id_list) -# else: -# e, file = await thread_pool_exec(FileService.get_by_id, file_id_list[len_id_list - 2]) -# if not e: -# return get_data_error_result(message="Folder not found!") -# last_folder = await thread_pool_exec(FileService.create_folder, file, file_id_list[len_id_list - 2], file_obj_names, -# len_id_list) -# -# # file type -# filetype = filename_type(file_obj_names[file_len - 1]) -# location = file_obj_names[file_len - 1] -# while await thread_pool_exec(settings.STORAGE_IMPL.obj_exist, last_folder.id, location): -# location += "_" -# blob = await thread_pool_exec(file_obj.read) -# filename = await thread_pool_exec( -# duplicate_name, -# FileService.query, -# name=file_obj_names[file_len - 1], -# parent_id=last_folder.id) -# await thread_pool_exec(settings.STORAGE_IMPL.put, last_folder.id, location, blob) -# file_data = { -# "id": get_uuid(), -# "parent_id": last_folder.id, -# "tenant_id": current_user.id, -# "created_by": current_user.id, -# "type": filetype, -# "name": filename, -# "location": location, -# "size": len(blob), -# } -# inserted = await thread_pool_exec(FileService.insert, file_data) -# return inserted.to_json() -# -# for file_obj in file_objs: -# res = await _handle_single_file(file_obj) -# file_res.append(res) -# -# return get_json_result(data=file_res) -# except Exception as e: -# return server_error_response(e) -# -# -# @manager.route('/create', methods=['POST']) # noqa: F821 -# @login_required -# @validate_request("name") -# async def create(): -# req = await get_request_json() -# pf_id = req.get("parent_id") -# input_file_type = req.get("type") -# if not pf_id: -# root_folder = FileService.get_root_folder(current_user.id) -# pf_id = root_folder["id"] -# -# try: -# if not FileService.is_parent_folder_exist(pf_id): -# return get_json_result( -# data=False, message="Parent Folder Doesn't Exist!", code=RetCode.OPERATING_ERROR) -# if FileService.query(name=req["name"], parent_id=pf_id): -# return get_data_error_result( -# message="Duplicated folder name in the same folder.") -# -# if input_file_type == FileType.FOLDER.value: -# file_type = FileType.FOLDER.value -# else: -# file_type = FileType.VIRTUAL.value -# -# file = FileService.insert({ -# "id": get_uuid(), -# "parent_id": pf_id, -# "tenant_id": current_user.id, -# "created_by": current_user.id, -# "name": req["name"], -# "location": "", -# "size": 0, -# "type": file_type -# }) -# -# return get_json_result(data=file.to_json()) -# except Exception as e: -# return server_error_response(e) -# -# -# @manager.route('/list', methods=['GET']) # noqa: F821 -# @login_required -# def list_files(): -# pf_id = request.args.get("parent_id") -# -# keywords = request.args.get("keywords", "") -# -# page_number = int(request.args.get("page", 1)) -# items_per_page = int(request.args.get("page_size", 15)) -# orderby = request.args.get("orderby", "create_time") -# desc = request.args.get("desc", True) -# if not pf_id: -# root_folder = FileService.get_root_folder(current_user.id) -# pf_id = root_folder["id"] -# FileService.init_knowledgebase_docs(pf_id, current_user.id) -# try: -# e, file = FileService.get_by_id(pf_id) -# if not e: -# return get_data_error_result(message="Folder not found!") -# -# files, total = FileService.get_by_pf_id( -# current_user.id, pf_id, page_number, items_per_page, orderby, desc, keywords) -# -# parent_folder = FileService.get_parent_folder(pf_id) -# if not parent_folder: -# return get_json_result(message="File not found!") -# -# return get_json_result(data={"total": total, "files": files, "parent_folder": parent_folder.to_json()}) -# except Exception as e: -# return server_error_response(e) -# -# -# @manager.route('/root_folder', methods=['GET']) # noqa: F821 -# @login_required -# def get_root_folder(): -# try: -# root_folder = FileService.get_root_folder(current_user.id) -# return get_json_result(data={"root_folder": root_folder}) -# except Exception as e: -# return server_error_response(e) -# -# -# @manager.route('/parent_folder', methods=['GET']) # noqa: F821 -# @login_required -# def get_parent_folder(): -# file_id = request.args.get("file_id") -# try: -# e, file = FileService.get_by_id(file_id) -# if not e: -# return get_data_error_result(message="Folder not found!") -# -# parent_folder = FileService.get_parent_folder(file_id) -# return get_json_result(data={"parent_folder": parent_folder.to_json()}) -# except Exception as e: -# return server_error_response(e) -# -# -# @manager.route('/all_parent_folder', methods=['GET']) # noqa: F821 -# @login_required -# def get_all_parent_folders(): -# file_id = request.args.get("file_id") -# try: -# e, file = FileService.get_by_id(file_id) -# if not e: -# return get_data_error_result(message="Folder not found!") -# -# parent_folders = FileService.get_all_parent_folders(file_id) -# parent_folders_res = [] -# for parent_folder in parent_folders: -# parent_folders_res.append(parent_folder.to_json()) -# return get_json_result(data={"parent_folders": parent_folders_res}) -# except Exception as e: -# return server_error_response(e) -# -# -# @manager.route("/rm", methods=["POST"]) # noqa: F821 -# @login_required -# @validate_request("file_ids") -# async def rm(): -# req = await get_request_json() -# file_ids = req["file_ids"] -# uid = current_user.id -# -# try: -# def _delete_single_file(file): -# try: -# if file.location: -# settings.STORAGE_IMPL.rm(file.parent_id, file.location) -# except Exception as e: -# logging.exception(f"Fail to remove object: {file.parent_id}/{file.location}, error: {e}") -# -# informs = File2DocumentService.get_by_file_id(file.id) -# for inform in informs: -# doc_id = inform.document_id -# e, doc = DocumentService.get_by_id(doc_id) -# if e and doc: -# tenant_id = DocumentService.get_tenant_id(doc_id) -# if tenant_id: -# DocumentService.remove_document(doc, tenant_id) -# File2DocumentService.delete_by_file_id(file.id) -# -# FileService.delete(file) -# -# def _delete_folder_recursive(folder, tenant_id): -# sub_files = FileService.list_all_files_by_parent_id(folder.id) -# for sub_file in sub_files: -# if sub_file.type == FileType.FOLDER.value: -# _delete_folder_recursive(sub_file, tenant_id) -# else: -# _delete_single_file(sub_file) -# -# FileService.delete(folder) -# -# def _rm_sync(): -# for file_id in file_ids: -# e, file = FileService.get_by_id(file_id) -# if not e or not file: -# return get_data_error_result(message="File or Folder not found!") -# if not file.tenant_id: -# return get_data_error_result(message="Tenant not found!") -# if not check_file_team_permission(file, uid): -# return get_json_result(data=False, message="No authorization.", code=RetCode.AUTHENTICATION_ERROR) -# -# if file.source_type == FileSource.KNOWLEDGEBASE: -# continue -# -# if file.type == FileType.FOLDER.value: -# _delete_folder_recursive(file, uid) -# continue -# -# _delete_single_file(file) -# -# return get_json_result(data=True) -# -# return await thread_pool_exec(_rm_sync) -# -# except Exception as e: -# return server_error_response(e) -# -# -# @manager.route('/rename', methods=['POST']) # noqa: F821 -# @login_required -# @validate_request("file_id", "name") -# async def rename(): -# req = await get_request_json() -# try: -# e, file = FileService.get_by_id(req["file_id"]) -# if not e: -# return get_data_error_result(message="File not found!") -# if not check_file_team_permission(file, current_user.id): -# return get_json_result(data=False, message='No authorization.', code=RetCode.AUTHENTICATION_ERROR) -# if file.type != FileType.FOLDER.value \ -# and pathlib.Path(req["name"].lower()).suffix != pathlib.Path( -# file.name.lower()).suffix: -# return get_json_result( -# data=False, -# message="The extension of file can't be changed", -# code=RetCode.ARGUMENT_ERROR) -# for file in FileService.query(name=req["name"], pf_id=file.parent_id): -# if file.name == req["name"]: -# return get_data_error_result( -# message="Duplicated file name in the same folder.") -# -# if not FileService.update_by_id( -# req["file_id"], {"name": req["name"]}): -# return get_data_error_result( -# message="Database error (File rename)!") -# -# informs = File2DocumentService.get_by_file_id(req["file_id"]) -# if informs: -# if not DocumentService.update_by_id( -# informs[0].document_id, {"name": req["name"]}): -# return get_data_error_result( -# message="Database error (Document rename)!") -# -# return get_json_result(data=True) -# except Exception as e: -# return server_error_response(e) -# -# -# @manager.route('/get/', methods=['GET']) # noqa: F821 -# @login_required -# async def get(file_id): -# try: -# e, file = FileService.get_by_id(file_id) -# if not e: -# return get_data_error_result(message="Document not found!") -# if not check_file_team_permission(file, current_user.id): -# return get_json_result(data=False, message='No authorization.', code=RetCode.AUTHENTICATION_ERROR) -# -# blob = await thread_pool_exec(settings.STORAGE_IMPL.get, file.parent_id, file.location) -# if not blob: -# b, n = File2DocumentService.get_storage_address(file_id=file_id) -# blob = await thread_pool_exec(settings.STORAGE_IMPL.get, b, n) -# -# response = await make_response(blob) -# ext = re.search(r"\.([^.]+)$", file.name.lower()) -# ext = ext.group(1) if ext else None -# content_type = None -# if ext: -# fallback_prefix = "image" if file.type == FileType.VISUAL.value else "application" -# content_type = CONTENT_TYPE_MAP.get(ext, f"{fallback_prefix}/{ext}") -# apply_safe_file_response_headers(response, content_type, ext) -# return response -# except Exception as e: -# return server_error_response(e) -# -# -# @manager.route("/mv", methods=["POST"]) # noqa: F821 -# @login_required -# @validate_request("src_file_ids", "dest_file_id") -# async def move(): -# req = await get_request_json() -# try: -# file_ids = req["src_file_ids"] -# dest_parent_id = req["dest_file_id"] -# -# ok, dest_folder = FileService.get_by_id(dest_parent_id) -# if not ok or not dest_folder: -# return get_data_error_result(message="Parent folder not found!") -# -# files = FileService.get_by_ids(file_ids) -# if not files: -# return get_data_error_result(message="Source files not found!") -# -# files_dict = {f.id: f for f in files} -# -# for file_id in file_ids: -# file = files_dict.get(file_id) -# if not file: -# return get_data_error_result(message="File or folder not found!") -# if not file.tenant_id: -# return get_data_error_result(message="Tenant not found!") -# if not check_file_team_permission(file, current_user.id): -# return get_json_result( -# data=False, -# message="No authorization.", -# code=RetCode.AUTHENTICATION_ERROR, -# ) -# -# def _move_entry_recursive(source_file_entry, dest_folder): -# if source_file_entry.type == FileType.FOLDER.value: -# existing_folder = FileService.query(name=source_file_entry.name, parent_id=dest_folder.id) -# if existing_folder: -# new_folder = existing_folder[0] -# else: -# new_folder = FileService.insert( -# { -# "id": get_uuid(), -# "parent_id": dest_folder.id, -# "tenant_id": source_file_entry.tenant_id, -# "created_by": current_user.id, -# "name": source_file_entry.name, -# "location": "", -# "size": 0, -# "type": FileType.FOLDER.value, -# } -# ) -# -# sub_files = FileService.list_all_files_by_parent_id(source_file_entry.id) -# for sub_file in sub_files: -# _move_entry_recursive(sub_file, new_folder) -# -# FileService.delete_by_id(source_file_entry.id) -# return -# -# old_parent_id = source_file_entry.parent_id -# old_location = source_file_entry.location -# filename = source_file_entry.name -# -# new_location = filename -# while settings.STORAGE_IMPL.obj_exist(dest_folder.id, new_location): -# new_location += "_" -# -# try: -# settings.STORAGE_IMPL.move(old_parent_id, old_location, dest_folder.id, new_location) -# except Exception as storage_err: -# raise RuntimeError(f"Move file failed at storage layer: {str(storage_err)}") -# -# FileService.update_by_id( -# source_file_entry.id, -# { -# "parent_id": dest_folder.id, -# "location": new_location, -# }, -# ) -# -# def _move_sync(): -# for file in files: -# _move_entry_recursive(file, dest_folder) -# return get_json_result(data=True) -# -# return await thread_pool_exec(_move_sync) -# -# except Exception as e: -# return server_error_response(e) diff --git a/api/apps/file2document_app.py b/api/apps/restful_apis/file2document_api.py similarity index 75% rename from api/apps/file2document_app.py rename to api/apps/restful_apis/file2document_api.py index c82207ab73..e599eb04ad 100644 --- a/api/apps/file2document_app.py +++ b/api/apps/restful_apis/file2document_api.py @@ -25,7 +25,6 @@ from api.apps import login_required, current_user from api.db.services.knowledgebase_service import KnowledgebaseService from api.utils.api_utils import get_data_error_result, get_json_result, get_request_json, server_error_response, validate_request from common.misc_utils import get_uuid -from common.constants import RetCode from api.db import FileType from api.db.services.document_service import DocumentService @@ -74,7 +73,7 @@ def _convert_files(file_ids, kb_ids, user_id): }) -@manager.route('/convert', methods=['POST']) # noqa: F821 +@manager.route('/files/link-to-datasets', methods=['POST']) # noqa: F821 @login_required @validate_request("file_ids", "kb_ids") async def convert(): @@ -118,36 +117,3 @@ async def convert(): return get_json_result(data=True) except Exception as e: return server_error_response(e) - - -@manager.route('/rm', methods=['POST']) # noqa: F821 -@login_required -@validate_request("file_ids") -async def rm(): - req = await get_request_json() - file_ids = req["file_ids"] - if not file_ids: - return get_json_result( - data=False, message='Lack of "Files ID"', code=RetCode.ARGUMENT_ERROR) - try: - for file_id in file_ids: - informs = File2DocumentService.get_by_file_id(file_id) - if not informs: - return get_data_error_result(message="Inform not found!") - for inform in informs: - if not inform: - return get_data_error_result(message="Inform not found!") - File2DocumentService.delete_by_file_id(file_id) - doc_id = inform.document_id - e, doc = DocumentService.get_by_id(doc_id) - if not e: - return get_data_error_result(message="Document not found!") - tenant_id = DocumentService.get_tenant_id(doc_id) - if not tenant_id: - return get_data_error_result(message="Tenant not found!") - if not DocumentService.remove_document(doc, tenant_id): - return get_data_error_result( - message="Database error (Document removal)!") - return get_json_result(data=True) - except Exception as e: - return server_error_response(e) diff --git a/test/testcases/test_web_api/test_file_app/test_file2document_routes_unit.py b/test/testcases/test_web_api/test_file_app/test_file2document_routes_unit.py index a81414829c..cd9de79260 100644 --- a/test/testcases/test_web_api/test_file_app/test_file2document_routes_unit.py +++ b/test/testcases/test_web_api/test_file_app/test_file2document_routes_unit.py @@ -229,7 +229,7 @@ def _load_file2document_module(monkeypatch): monkeypatch.setitem(sys.modules, "common.constants", constants_mod) module_name = "test_file2document_routes_unit_module" - module_path = repo_root / "api" / "apps" / "file2document_app.py" + module_path = repo_root / "api" / "apps" / "restful_apis" / "file2document_api.py" spec = importlib.util.spec_from_file_location(module_name, module_path) module = importlib.util.module_from_spec(spec) module.manager = _DummyManager() @@ -280,65 +280,3 @@ def test_convert_branch_matrix_unit(monkeypatch): res = _run(module.convert()) assert res["code"] == 500 assert "convert boom" in res["message"] - - -@pytest.mark.p2 -def test_rm_branch_matrix_unit(monkeypatch): - module = _load_file2document_module(monkeypatch) - req_state = {"file_ids": []} - _set_request_json(monkeypatch, module, req_state) - - deleted = [] - - res = _run(module.rm()) - assert res["code"] == module.RetCode.ARGUMENT_ERROR - assert 'Lack of "Files ID"' in res["message"] - - req_state["file_ids"] = ["f1"] - monkeypatch.setattr(module.File2DocumentService, "get_by_file_id", lambda _file_id: []) - res = _run(module.rm()) - assert res["message"] == "Inform not found!" - - monkeypatch.setattr(module.File2DocumentService, "get_by_file_id", lambda _file_id: [None]) - res = _run(module.rm()) - assert res["message"] == "Inform not found!" - - monkeypatch.setattr(module.File2DocumentService, "get_by_file_id", lambda _file_id: [SimpleNamespace(document_id="doc-1")]) - monkeypatch.setattr(module.File2DocumentService, "delete_by_file_id", lambda file_id: deleted.append(file_id)) - monkeypatch.setattr(module.DocumentService, "get_by_id", lambda _doc_id: (False, None)) - res = _run(module.rm()) - assert res["message"] == "Document not found!" - assert deleted == ["f1"] - - monkeypatch.setattr(module.DocumentService, "get_by_id", lambda _doc_id: (True, SimpleNamespace(id=_doc_id))) - monkeypatch.setattr(module.DocumentService, "get_tenant_id", lambda _doc_id: None) - res = _run(module.rm()) - assert res["message"] == "Tenant not found!" - - monkeypatch.setattr(module.DocumentService, "get_tenant_id", lambda _doc_id: "tenant-1") - monkeypatch.setattr(module.DocumentService, "remove_document", lambda *_args, **_kwargs: False) - res = _run(module.rm()) - assert "Document removal" in res["message"] - - req_state["file_ids"] = ["f1", "f2"] - monkeypatch.setattr( - module.File2DocumentService, - "get_by_file_id", - lambda file_id: [SimpleNamespace(document_id=f"doc-{file_id}")], - ) - monkeypatch.setattr(module.DocumentService, "get_by_id", lambda doc_id: (True, SimpleNamespace(id=doc_id))) - monkeypatch.setattr(module.DocumentService, "get_tenant_id", lambda _doc_id: "tenant-1") - monkeypatch.setattr(module.DocumentService, "remove_document", lambda *_args, **_kwargs: True) - res = _run(module.rm()) - assert res["code"] == 0 - assert res["data"] is True - - monkeypatch.setattr( - module.File2DocumentService, - "get_by_file_id", - lambda _file_id: (_ for _ in ()).throw(RuntimeError("rm boom")), - ) - req_state["file_ids"] = ["boom"] - res = _run(module.rm()) - assert res["code"] == 500 - assert "rm boom" in res["message"] diff --git a/web/src/utils/api.ts b/web/src/utils/api.ts index e300293964..8da592cfaf 100644 --- a/web/src/utils/api.ts +++ b/web/src/utils/api.ts @@ -35,7 +35,7 @@ export default { deleteFactory: `${webAPI}/llm/delete_factory`, // data source - dataSourceUpdate: (id:string) => `${restAPIv1}/connectors/${id}`, + dataSourceUpdate: (id: string) => `${restAPIv1}/connectors/${id}`, dataSourceSet: `${restAPIv1}/connectors`, dataSourceList: `${restAPIv1}/connectors`, dataSourceDel: (id: string) => `${restAPIv1}/connectors/${id}`, @@ -165,7 +165,7 @@ export default { removeFile: `${restAPIv1}/files`, getAllParentFolder: `${restAPIv1}/files`, createFolder: `${restAPIv1}/files`, - connectFileToKnowledge: `${webAPI}/file2document/convert`, + connectFileToKnowledge: `${restAPIv1}/files/link-to-datasets`, getFile: `${restAPIv1}/files`, moveFile: `${restAPIv1}/files/move`,