Fix: validate URL scheme and resolved IP before crawling to prevent SSRF (#14090)

### What problem does this PR solve?

The POST /upload_info?url=<url> endpoint accepted a user-supplied URL
and passed it directly to AsyncWebCrawler without any validation. There
were no restrictions on URL scheme, destination hostname, or resolved IP
address. This allowed any authenticated user to instruct the server to
make outbound HTTP requests to internal infrastructure — including RFC
1918 private networks, loopback addresses, and cloud metadata services
such as http://169.254.169.254 — effectively using the server as a proxy
for internal network reconnaissance or credential theft.

This PR adds an SSRF guard (_validate_url_for_crawl) that runs before
any crawl is initiated. It enforces an allowlist of safe schemes
(http/https), resolves the hostname at validation time, and rejects any
URL whose resolved IP falls within a private or reserved network range.

### Type of change

- [x] Bug Fix (non-breaking change which fixes an issue)

api/apps/document_app.py

@@ -43,6 +43,7 @@ from common import settings
 from common.constants import SANDBOX_ARTIFACT_BUCKET, ParserType, RetCode, TaskStatus
 from common.file_utils import get_project_base_directory
 from common.misc_utils import get_uuid, thread_pool_exec
+from common.ssrf_guard import assert_url_is_safe
 from deepdoc.parser.html_parser import RAGFlowHtmlParser
 from rag.nlp import search
 
@@ -333,6 +334,7 @@ async def run():
     except Exception as e:
         return server_error_response(e)
 
+
 @manager.route("/get/<doc_id>", methods=["GET"])  # noqa: F821
 @login_required
 async def get(doc_id):
@@ -581,6 +583,7 @@ async def upload_info():
 
     try:
         if url and not file_objs:
+            assert_url_is_safe(url)
             return get_json_result(data=FileService.upload_info(current_user.id, None, url))
 
         if len(file_objs) == 1:

api/db/services/file_service.py

@@ -23,6 +23,8 @@ from concurrent.futures import ThreadPoolExecutor
 from pathlib import Path
 from typing import Union
 
+logger = logging.getLogger(__name__)
+
 import xxhash
 from peewee import fn
 
@@ -33,6 +35,7 @@ from api.db.services.common_service import CommonService
 from api.db.services.document_service import DocumentService
 from api.db.services.file2document_service import File2DocumentService
 from common.misc_utils import get_uuid
+from common.ssrf_guard import assert_url_is_safe
 from common.constants import TaskStatus, FileSource, ParserType
 from api.db.services.knowledgebase_service import KnowledgebaseService
 from api.db.services.task_service import TaskService
@@ -624,6 +627,26 @@ class FileService(CommonService):
 
         return errors
 
+    _ALLOWED_SCHEMES = {"http", "https"}
+
+    @staticmethod
+    def _validate_url_for_crawl(url: str) -> tuple[str, str]:
+        """Raise ValueError if the URL is not safe to crawl (SSRF guard).
+
+        Delegates to :func:`common.ssrf_guard.assert_url_is_safe`, which
+        validates the scheme, hostname, and every DNS-resolved address, and
+        returns ``(hostname, resolved_ip)`` for DNS pinning.
+
+        Only the scheme and host (and port when present) are forwarded to the
+        guard so that credentials or query parameters in *url* are never
+        written to the log.
+        """
+        from urllib.parse import urlparse
+        parsed = urlparse(url)
+        port_suffix = f":{parsed.port}" if parsed.port else ""
+        redacted = f"{parsed.scheme}://{parsed.hostname}{port_suffix}"
+        return assert_url_is_safe(redacted, allowed_schemes=FileService._ALLOWED_SCHEMES)
+
     @staticmethod
     def upload_info(user_id, file, url: str|None=None):
         def structured(filename, filetype, blob, content_type):
@@ -646,6 +669,53 @@ class FileService(CommonService):
             }
 
         if url:
+            import requests as _requests
+            from urllib.parse import urljoin as _urljoin
+
+            _MAX_CRAWL_REDIRECTS = 10
+
+            # Pre-resolve the full redirect chain so that AsyncWebCrawler never
+            # follows a server-sent redirect to an unvalidated (potentially
+            # internal) host.  Each hop is SSRF-checked before being followed;
+            # the validated (hostname, ip) pairs are pinned via Chromium's
+            # --host-resolver-rules so the browser cannot re-resolve any of them
+            # through a fresh DNS query.
+            current_url = url
+            current_hostname, current_ip = FileService._validate_url_for_crawl(current_url)
+            # Accumulate MAP rules for every hostname we encounter in the chain.
+            host_pins: dict[str, str] = {current_hostname: current_ip}
+
+            for _ in range(_MAX_CRAWL_REDIRECTS):
+                try:
+                    _resp = _requests.get(
+                        current_url,
+                        timeout=10,
+                        allow_redirects=False,
+                    )
+                except _requests.RequestException as _exc:
+                    raise ValueError(f"Failed to fetch {current_url!r}: {_exc}") from _exc
+
+                if _resp.status_code not in (301, 302, 303, 307, 308):
+                    break
+
+                _location = _resp.headers.get("Location")
+                if not _location:
+                    break
+
+                _next_url = _urljoin(current_url, _location)
+                _next_hostname, _next_ip = FileService._validate_url_for_crawl(_next_url)
+                host_pins[_next_hostname] = _next_ip
+                current_url = _next_url
+            else:
+                raise ValueError(
+                    f"Exceeded {_MAX_CRAWL_REDIRECTS} redirects fetching {url!r}"
+                )
+
+            # Build a single MAP rule string covering every validated hostname
+            # in the redirect chain.  Chromium uses the pinned IP for each,
+            # skipping DNS entirely and eliminating the rebinding window.
+            _map_rules = ",".join(f"MAP {h} {ip}" for h, ip in host_pins.items())
+
             from crawl4ai import (
                 AsyncWebCrawler,
                 BrowserConfig,
@@ -659,6 +729,7 @@ class FileService(CommonService):
                 browser_config = BrowserConfig(
                     headless=True,
                     verbose=False,
+                    extra_args=[f"--host-resolver-rules={_map_rules}"],
                 )
                 async with AsyncWebCrawler(config=browser_config) as crawler:
                     crawler_config = CrawlerRunConfig(
@@ -668,8 +739,10 @@ class FileService(CommonService):
                         pdf=True,
                         screenshot=False
                     )
+                    # Use the final resolved URL so the browser starts at the
+                    # redirect destination rather than re-following the chain.
                     result: CrawlResult = await crawler.arun(
-                        url=url,
+                        url=current_url,
                         config=crawler_config
                     )
                     return result
@@ -679,7 +752,7 @@ class FileService(CommonService):
                     filename += ".pdf"
                 return structured(filename, "pdf", page.pdf, page.response_headers["content-type"])
 
-            return structured(filename, "html", str(page.markdown).encode("utf-8"), page.response_headers["content-type"], user_id)
+            return structured(filename, "html", str(page.markdown).encode("utf-8"), page.response_headers["content-type"])
 
         DocumentService.check_doc_health(user_id, file.filename)
         return structured(file.filename, filename_type(file.filename), file.read(), file.content_type)

api/utils/web_utils.py

@@ -15,11 +15,8 @@
 #
 
 import base64
-import ipaddress
 import json
 import re
-import socket
-from urllib.parse import urlparse
 import aiosmtplib
 from email.mime.text import MIMEText
 from email.header import Header
@@ -37,10 +34,10 @@ from webdriver_manager.chrome import ChromeDriverManager
 
 
 OTP_LENGTH = 4
-OTP_TTL_SECONDS = 5 * 60 # valid for 5 minutes
-ATTEMPT_LIMIT = 5 # maximum attempts
-ATTEMPT_LOCK_SECONDS = 30 * 60 # lock for 30 minutes
-RESEND_COOLDOWN_SECONDS = 60 # cooldown for 1 minute
+OTP_TTL_SECONDS = 5 * 60  # valid for 5 minutes
+ATTEMPT_LIMIT = 5  # maximum attempts
+ATTEMPT_LOCK_SECONDS = 30 * 60  # lock for 30 minutes
+RESEND_COOLDOWN_SECONDS = 60  # cooldown for 1 minute
 
 
 CONTENT_TYPE_MAP = {
@@ -188,29 +185,16 @@ def __get_pdf_from_html(path: str, timeout: int, install_driver: bool, print_opt
         return base64.b64decode(result["data"])
 
 
-def is_private_ip(ip: str) -> bool:
-    try:
-        ip_obj = ipaddress.ip_address(ip)
-        return ip_obj.is_private
-    except ValueError:
-        return False
-
-
 def is_valid_url(url: str) -> bool:
     if not re.match(r"(https?)://[-A-Za-z0-9+&@#/%?=~_|!:,.;]+[-A-Za-z0-9+&@#/%=~_|]", url):
         return False
-    parsed_url = urlparse(url)
-    hostname = parsed_url.hostname
+    from common.ssrf_guard import assert_url_is_safe
 
-    if not hostname:
-        return False
     try:
-        ip = socket.gethostbyname(hostname)
-        if is_private_ip(ip):
-            return False
-    except socket.gaierror:
+        assert_url_is_safe(url)
+        return True
+    except ValueError:
         return False
-    return True
 
 
 def safe_json_parse(data: str | dict) -> dict: