Refactor: deco doc-parse API that is not used any more (#14367)

### What problem does this PR solve?

Delete un-used API "POST /v1/document/parse"

### Type of change

- [x] Refactoring

api/apps/document_app.py

@@ -14,9 +14,7 @@
 #  limitations under the License
 #
 import logging
-import os.path
 import re
-from pathlib import PurePosixPath, PureWindowsPath
 
 from quart import make_response, request
 
@@ -26,7 +24,6 @@ from api.db import FileType
 from api.db.db_models import Task
 from api.db.services.document_service import DocumentService, doc_upload_and_parse
 from api.db.services.file2document_service import File2DocumentService
-from api.db.services.file_service import FileService
 from api.db.services.knowledgebase_service import KnowledgebaseService
 from api.db.services.task_service import TaskService, cancel_all_task_of
 from api.utils.api_utils import (
@@ -36,28 +33,13 @@ from api.utils.api_utils import (
     server_error_response,
     validate_request,
 )
-from api.utils.web_utils import CONTENT_TYPE_MAP, apply_safe_file_response_headers, is_valid_url
+from api.utils.web_utils import CONTENT_TYPE_MAP, apply_safe_file_response_headers
 from common import settings
 from common.constants import RetCode, TaskStatus
-from common.file_utils import get_project_base_directory
 from common.misc_utils import thread_pool_exec
-from deepdoc.parser.html_parser import RAGFlowHtmlParser
 from rag.nlp import search
 
 
-def _is_safe_download_filename(name: str) -> bool:
-    if not name or name in {".", ".."}:
-        return False
-    if "\x00" in name or len(name) > 255:
-        return False
-    if name != PurePosixPath(name).name:
-        return False
-    if name != PureWindowsPath(name).name:
-        return False
-    return True
-
-
-
 @manager.route("/thumbnails", methods=["GET"])  # noqa: F821
 # @login_required
 def thumbnails():
@@ -339,62 +321,3 @@ async def upload_and_parse():
     form = await request.form
     doc_ids = doc_upload_and_parse(form.get("conversation_id"), file_objs, current_user.id)
     return get_json_result(data=doc_ids)
-
-
-@manager.route("/parse", methods=["POST"])  # noqa: F821
-@login_required
-async def parse():
-    req = await get_request_json()
-    url = req.get("url", "")
-    if url:
-        if not is_valid_url(url):
-            return get_json_result(data=False, message="The URL format is invalid", code=RetCode.ARGUMENT_ERROR)
-        download_path = os.path.join(get_project_base_directory(), "logs/downloads")
-        os.makedirs(download_path, exist_ok=True)
-        from seleniumwire.webdriver import Chrome, ChromeOptions
-
-        options = ChromeOptions()
-        options.add_argument("--headless")
-        options.add_argument("--disable-gpu")
-        options.add_argument("--no-sandbox")
-        options.add_argument("--disable-dev-shm-usage")
-        options.add_experimental_option("prefs", {"download.default_directory": download_path, "download.prompt_for_download": False, "download.directory_upgrade": True, "safebrowsing.enabled": True})
-        driver = Chrome(options=options)
-        driver.get(url)
-        res_headers = [r.response.headers for r in driver.requests if r and r.response]
-        if len(res_headers) > 1:
-            sections = RAGFlowHtmlParser().parser_txt(driver.page_source)
-            driver.quit()
-            return get_json_result(data="\n".join(sections))
-
-        class File:
-            filename: str
-            filepath: str
-
-            def __init__(self, filename, filepath):
-                self.filename = filename
-                self.filepath = filepath
-
-            def read(self):
-                with open(self.filepath, "rb") as f:
-                    return f.read()
-
-        r = re.search(r"filename=\"([^\"]+)\"", str(res_headers))
-        if not r or not r.group(1):
-            return get_json_result(data=False, message="Can't not identify downloaded file", code=RetCode.ARGUMENT_ERROR)
-        filename = r.group(1).strip()
-        if not _is_safe_download_filename(filename):
-            return get_json_result(data=False, message="Invalid downloaded filename", code=RetCode.ARGUMENT_ERROR)
-        filepath = os.path.join(download_path, filename)
-        f = File(filename, filepath)
-        txt = FileService.parse_docs([f], current_user.id)
-        return get_json_result(data=txt)
-
-    files = await request.files
-    if "file" not in files:
-        return get_json_result(data=False, message="No file part!", code=RetCode.ARGUMENT_ERROR)
-
-    file_objs = files.getlist("file")
-    txt = FileService.parse_docs(file_objs, current_user.id)
-
-    return get_json_result(data=txt)