Fix: validate URL scheme and resolved IP before crawling to prevent SSRF (#14090)

### What problem does this PR solve?

The POST /upload_info?url=<url> endpoint accepted a user-supplied URL
and passed it directly to AsyncWebCrawler without any validation. There
were no restrictions on URL scheme, destination hostname, or resolved IP
address. This allowed any authenticated user to instruct the server to
make outbound HTTP requests to internal infrastructure — including RFC
1918 private networks, loopback addresses, and cloud metadata services
such as http://169.254.169.254 — effectively using the server as a proxy
for internal network reconnaissance or credential theft.

This PR adds an SSRF guard (_validate_url_for_crawl) that runs before
any crawl is initiated. It enforces an allowlist of safe schemes
(http/https), resolves the hostname at validation time, and rejects any
URL whose resolved IP falls within a private or reserved network range.

### Type of change

- [x] Bug Fix (non-breaking change which fixes an issue)

common/data_source/rss_connector.py

@@ -1,11 +1,9 @@
 import hashlib
-import ipaddress
-import socket
 from datetime import datetime, timezone
 from email.utils import parsedate_to_datetime
 from time import struct_time
 from typing import Any
-from urllib.parse import urlparse
+from urllib.parse import urljoin, urlparse
 
 import bs4
 import feedparser
@@ -14,28 +12,9 @@ import requests
 from common.data_source.config import INDEX_BATCH_SIZE, REQUEST_TIMEOUT_SECONDS, DocumentSource
 from common.data_source.interfaces import LoadConnector, PollConnector
 from common.data_source.models import Document, GenerateDocumentsOutput, SecondsSinceUnixEpoch
+from common.ssrf_guard import assert_url_is_safe, pin_dns as _pin_dns
 
-
-def _is_private_ip(ip: str) -> bool:
-    try:
-        ip_obj = ipaddress.ip_address(ip)
-        return ip_obj.is_private or ip_obj.is_link_local or ip_obj.is_loopback
-    except ValueError:
-        return False
-
-
-def _validate_url_no_ssrf(url: str) -> None:
-    parsed = urlparse(url)
-    hostname = parsed.hostname
-    if not hostname:
-        raise ValueError("URL must have a valid hostname")
-
-    try:
-        ip = socket.gethostbyname(hostname)
-        if _is_private_ip(ip):
-            raise ValueError(f"URL resolves to private/internal IP address: {ip}")
-    except socket.gaierror as e:
-        raise ValueError(f"Failed to resolve hostname: {hostname}") from e
+_MAX_REDIRECTS = 10
 
 
 class RSSConnector(LoadConnector, PollConnector):
@@ -87,7 +66,8 @@ class RSSConnector(LoadConnector, PollConnector):
         if batch:
             yield batch
 
-    def _validate_feed_url(self) -> None:
+    def _validate_feed_url(self) -> tuple[str, str]:
+        """Validate ``self.feed_url`` and return ``(hostname, resolved_ip)``."""
         if not self.feed_url:
             raise ValueError("feed_url is required")
 
@@ -95,7 +75,7 @@ class RSSConnector(LoadConnector, PollConnector):
         if parsed.scheme not in {"http", "https"} or not parsed.netloc:
             raise ValueError("feed_url must be a valid http or https URL")
 
-        _validate_url_no_ssrf(self.feed_url)
+        return assert_url_is_safe(self.feed_url)
 
     def _read_feed(self, require_entries: bool) -> Any:
         if self._cached_feed is not None:
@@ -103,15 +83,38 @@ class RSSConnector(LoadConnector, PollConnector):
                 raise ValueError("RSS feed contains no entries")
             return self._cached_feed
 
-        self._validate_feed_url()
+        # Validate once to get the pinned IP for the initial request.
+        current_hostname, current_ip = self._validate_feed_url()
+        current_url = self.feed_url
+
+        # Follow redirects manually: each hop is validated and DNS-pinned
+        # *before* the connection is made, closing the TOCTOU rebinding window
+        # that existed when allow_redirects=True was used with post-hoc checks.
+        response: requests.Response | None = None
+        for _ in range(_MAX_REDIRECTS + 1):
+            with _pin_dns(current_hostname, current_ip):
+                response = requests.get(
+                    current_url,
+                    timeout=REQUEST_TIMEOUT_SECONDS,
+                    allow_redirects=False,
+                )
+
+            if response.status_code not in (301, 302, 303, 307, 308):
+                break
+
+            location = response.headers.get("Location")
+            if not location:
+                break  # broken redirect; let raise_for_status() handle it
+
+            redirect_url = urljoin(current_url, location)
+            # Validate redirect target before following it.
+            current_hostname, current_ip = assert_url_is_safe(redirect_url)
+            current_url = redirect_url
+        else:
+            raise ValueError(f"Exceeded {_MAX_REDIRECTS} redirects fetching {self.feed_url!r}")
 
-        response = requests.get(self.feed_url, timeout=REQUEST_TIMEOUT_SECONDS, allow_redirects=True)
         response.raise_for_status()
 
-        final_url = getattr(response, "url", self.feed_url)
-        if final_url != self.feed_url and urlparse(final_url).hostname:
-            _validate_url_no_ssrf(final_url)
-
         feed = feedparser.parse(response.content)
         if getattr(feed, "bozo", False) and not feed.entries:
             error = getattr(feed, "bozo_exception", None)