Fix: validate URL scheme and resolved IP before crawling to prevent SSRF (#14090)

### What problem does this PR solve?

The POST /upload_info?url=<url> endpoint accepted a user-supplied URL
and passed it directly to AsyncWebCrawler without any validation. There
were no restrictions on URL scheme, destination hostname, or resolved IP
address. This allowed any authenticated user to instruct the server to
make outbound HTTP requests to internal infrastructure — including RFC
1918 private networks, loopback addresses, and cloud metadata services
such as http://169.254.169.254 — effectively using the server as a proxy
for internal network reconnaissance or credential theft.

This PR adds an SSRF guard (_validate_url_for_crawl) that runs before
any crawl is initiated. It enforces an allowlist of safe schemes
(http/https), resolves the hostname at validation time, and rejects any
URL whose resolved IP falls within a private or reserved network range.

### Type of change

- [x] Bug Fix (non-breaking change which fixes an issue)

test/testcases/test_web_api/test_document_app/test_upload_info_unit.py

@@ -79,6 +79,7 @@ def _load_document_app_module(monkeypatch):
 @pytest.mark.p2
 def test_upload_info_rejects_mixed_inputs(monkeypatch):
     module = _load_document_app_module(monkeypatch)
+    monkeypatch.setattr(module, "assert_url_is_safe", lambda url: ("example.com", "93.184.216.34"))
     files = _DummyFiles({"file": [_DummyFile("a.txt")]})
     monkeypatch.setattr(module, "request", _DummyRequest(files=files, args={"url": "https://example.com/a.txt"}))
 
@@ -100,6 +101,7 @@ def test_upload_info_requires_file_or_url(monkeypatch):
 @pytest.mark.p2
 def test_upload_info_supports_url_single_and_multiple_files(monkeypatch):
     module = _load_document_app_module(monkeypatch)
+    monkeypatch.setattr(module, "assert_url_is_safe", lambda url: ("example.com", "93.184.216.34"))
     captured = []
 
     def fake_upload_info(user_id, file_obj, url=None):

test/unit_test/api/db/services/test_file_service_upload_document.py

@@ -14,6 +14,7 @@
 #  limitations under the License.
 #
 import importlib.util
+import socket
 import sys
 import types
 import warnings
@@ -120,3 +121,158 @@ def test_upload_document_skips_cross_kb_document_id_collision(monkeypatch):
     assert len(err) == 1
     assert err[0].startswith("collision.txt: ")
     assert "Existing document id collision with another knowledge base; skipping update." in err[0]
+
+
+# ---------------------------------------------------------------------------
+# Helpers shared by TestValidateUrlForCrawl
+# ---------------------------------------------------------------------------
+
+def _addrinfo(ip_str: str) -> list:
+    """Build a minimal getaddrinfo-style result for a single address string."""
+    family = socket.AF_INET6 if ":" in ip_str else socket.AF_INET
+    return [(family, socket.SOCK_STREAM, 6, "", (ip_str, 0))]
+
+
+# ---------------------------------------------------------------------------
+# _validate_url_for_crawl SSRF-guard tests
+# ---------------------------------------------------------------------------
+
+@pytest.mark.p2
+class TestValidateUrlForCrawl:
+    """Focused regression suite for the SSRF guard on the URL-crawl path.
+
+    All DNS lookups are monkeypatched so the tests are deterministic and
+    require no network access.
+    """
+
+    # -- scheme checks -------------------------------------------------------
+
+    def test_rejects_ftp_scheme(self):
+        with pytest.raises(ValueError, match="scheme"):
+            FileService._validate_url_for_crawl("ftp://example.com/file.txt")
+
+    def test_rejects_file_scheme(self):
+        with pytest.raises(ValueError, match="scheme"):
+            FileService._validate_url_for_crawl("file:///etc/passwd")
+
+    def test_rejects_javascript_scheme(self):
+        with pytest.raises(ValueError, match="scheme"):
+            FileService._validate_url_for_crawl("javascript:alert(1)")
+
+    # -- host checks ---------------------------------------------------------
+
+    def test_rejects_missing_host(self):
+        with pytest.raises(ValueError, match="host"):
+            FileService._validate_url_for_crawl("http:///path")
+
+    def test_rejects_dns_resolution_failure(self, monkeypatch):
+        def _raise(h, p):
+            raise socket.gaierror("NXDOMAIN")
+
+        monkeypatch.setattr(socket, "getaddrinfo", _raise)
+        with pytest.raises(ValueError, match="Could not resolve"):
+            FileService._validate_url_for_crawl("http://nxdomain.invalid/")
+
+    # -- blocked address families --------------------------------------------
+
+    def test_rejects_loopback_ipv4(self, monkeypatch):
+        monkeypatch.setattr(socket, "getaddrinfo", lambda h, p: _addrinfo("127.0.0.1"))
+        with pytest.raises(ValueError, match="non-public"):
+            FileService._validate_url_for_crawl("http://localhost/")
+
+    def test_rejects_private_class_a(self, monkeypatch):
+        monkeypatch.setattr(socket, "getaddrinfo", lambda h, p: _addrinfo("10.0.0.1"))
+        with pytest.raises(ValueError, match="non-public"):
+            FileService._validate_url_for_crawl("http://internal.example/")
+
+    def test_rejects_private_class_b(self, monkeypatch):
+        monkeypatch.setattr(socket, "getaddrinfo", lambda h, p: _addrinfo("172.16.0.1"))
+        with pytest.raises(ValueError, match="non-public"):
+            FileService._validate_url_for_crawl("http://internal.example/")
+
+    def test_rejects_private_class_c(self, monkeypatch):
+        monkeypatch.setattr(socket, "getaddrinfo", lambda h, p: _addrinfo("192.168.1.100"))
+        with pytest.raises(ValueError, match="non-public"):
+            FileService._validate_url_for_crawl("http://internal.example/")
+
+    def test_rejects_link_local_ipv4(self, monkeypatch):
+        monkeypatch.setattr(socket, "getaddrinfo", lambda h, p: _addrinfo("169.254.0.1"))
+        with pytest.raises(ValueError, match="non-public"):
+            FileService._validate_url_for_crawl("http://link-local.example/")
+
+    def test_rejects_reserved_ipv4(self, monkeypatch):
+        # 240.0.0.0/4 is IANA reserved — not globally routable
+        monkeypatch.setattr(socket, "getaddrinfo", lambda h, p: _addrinfo("240.0.0.1"))
+        with pytest.raises(ValueError, match="non-public"):
+            FileService._validate_url_for_crawl("http://reserved.example/")
+
+    def test_rejects_ipv4_mapped_loopback(self, monkeypatch):
+        """::ffff:127.0.0.1 must not bypass the loopback check."""
+        monkeypatch.setattr(socket, "getaddrinfo", lambda h, p: _addrinfo("::ffff:127.0.0.1"))
+        with pytest.raises(ValueError, match="non-public"):
+            FileService._validate_url_for_crawl("http://mapped-loopback.example/")
+
+    def test_rejects_ipv4_mapped_private(self, monkeypatch):
+        """::ffff:192.168.1.1 must not bypass the private-range check."""
+        monkeypatch.setattr(socket, "getaddrinfo", lambda h, p: _addrinfo("::ffff:192.168.1.1"))
+        with pytest.raises(ValueError, match="non-public"):
+            FileService._validate_url_for_crawl("http://mapped-private.example/")
+
+    def test_rejects_when_any_record_is_private(self, monkeypatch):
+        """All DNS records must pass; one private record is enough to block."""
+        monkeypatch.setattr(
+            socket,
+            "getaddrinfo",
+            lambda h, p: _addrinfo("93.184.216.34") + _addrinfo("10.0.0.1"),
+        )
+        with pytest.raises(ValueError, match="non-public"):
+            FileService._validate_url_for_crawl("http://mixed.example/")
+
+    # -- allowed cases -------------------------------------------------------
+
+    def test_allows_public_ipv4(self, monkeypatch):
+        monkeypatch.setattr(socket, "getaddrinfo", lambda h, p: _addrinfo("93.184.216.34"))
+        hostname, resolved_ip = FileService._validate_url_for_crawl("https://example.com/doc.pdf")
+        assert hostname == "example.com"
+        assert resolved_ip == "93.184.216.34"
+
+    def test_allows_public_ipv6(self, monkeypatch):
+        monkeypatch.setattr(
+            socket,
+            "getaddrinfo",
+            lambda h, p: _addrinfo("2606:2800:220:1:248:1893:25c8:1946"),
+        )
+        hostname, resolved_ip = FileService._validate_url_for_crawl("https://example.com/")
+        assert hostname == "example.com"
+        assert resolved_ip == "2606:2800:220:1:248:1893:25c8:1946"
+
+    def test_allows_http_scheme(self, monkeypatch):
+        monkeypatch.setattr(socket, "getaddrinfo", lambda h, p: _addrinfo("1.2.3.4"))
+        hostname, _ = FileService._validate_url_for_crawl("http://example.com/")
+        assert hostname == "example.com"
+
+    # -- multi-record behaviour ----------------------------------------------
+
+    def test_returns_first_ip_for_multi_record_host(self, monkeypatch):
+        """The first public IP is returned as the DNS pin value."""
+        monkeypatch.setattr(
+            socket,
+            "getaddrinfo",
+            lambda h, p: _addrinfo("1.2.3.4") + _addrinfo("5.6.7.8"),
+        )
+        _, resolved_ip = FileService._validate_url_for_crawl("http://multi.example/")
+        assert resolved_ip == "1.2.3.4"
+
+    def test_allows_dual_stack_host(self, monkeypatch):
+        """A host with both public IPv4 and public IPv6 records is allowed."""
+        monkeypatch.setattr(
+            socket,
+            "getaddrinfo",
+            lambda h, p: (
+                _addrinfo("93.184.216.34")
+                + _addrinfo("2606:2800:220:1:248:1893:25c8:1946")
+            ),
+        )
+        hostname, resolved_ip = FileService._validate_url_for_crawl("https://example.com/")
+        assert hostname == "example.com"
+        assert resolved_ip == "93.184.216.34"