zlei6/ragflow
1
0
Fork 0
mirror of https://github.com/infiniflow/ragflow.git synced 2026-07-21 15:11:08 +08:00
Code Issues Packages Projects Releases Wiki Activity

Fix: validate URL scheme and resolved IP before crawling to prevent SSRF (#14090)

Browse Source 
### What problem does this PR solve?

The POST /upload_info?url=<url> endpoint accepted a user-supplied URL
and passed it directly to AsyncWebCrawler without any validation. There
were no restrictions on URL scheme, destination hostname, or resolved IP
address. This allowed any authenticated user to instruct the server to
make outbound HTTP requests to internal infrastructure — including RFC
1918 private networks, loopback addresses, and cloud metadata services
such as http://169.254.169.254 — effectively using the server as a proxy
for internal network reconnaissance or credential theft.

This PR adds an SSRF guard (_validate_url_for_crawl) that runs before
any crawl is initiated. It enforces an allowlist of safe schemes
(http/https), resolves the hostname at validation time, and rejects any
URL whose resolved IP falls within a private or reserved network range.

### Type of change

- [x] Bug Fix (non-breaking change which fixes an issue)
This commit is contained in:
Xing Hong
2026-04-25 15:30:15 +09:00
committed by GitHub
parent 78188ce9e9
commit fb95136f39
10 changed files with 485 additions and 109 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
agent/component/invoke.py
View File

@@ -179,10 +179,7 @@ class Invoke(ComponentBase, ABC):
if not isinstance(headers, dict):
raise ValueError("Invoke headers must be a JSON object.")
return {
key: self._resolve_header_text(value, kwargs) if isinstance(value, str) else value
for key, value in headers.items()
}
return {key: self._resolve_header_text(value, kwargs) if isinstance(value, str) else value for key, value in headers.items()}
def _build_proxies(self) -> dict | None:
if not re.sub(r"https?:?/?/?", "", self._param.proxy):
@@ -215,7 +212,7 @@ class Invoke(ComponentBase, ABC):
# HtmlParser keeps the Invoke output text-focused when the endpoint returns HTML.
sections = HtmlParser()(None, response.content)
return "\n".join(sections)
@timeout(int(os.environ.get("COMPONENT_EXEC_TIMEOUT", 3)))
def _invoke(self, **kwargs):
if self.check_if_canceled("Invoke processing"):

26
agent/tools/crawler.py
View File

@@ -19,7 +19,6 @@ from crawl4ai import AsyncWebCrawler
from agent.tools.base import ToolParamBase, ToolBase
class CrawlerParam(ToolParamBase):
"""
Define the Crawler component parameters.
@@ -31,20 +30,26 @@ class CrawlerParam(ToolParamBase):
self.extract_type = "markdown"
def check(self):
self.check_valid_value(self.extract_type, "Type of content from the crawler", ['html', 'markdown', 'content'])
self.check_valid_value(self.extract_type, "Type of content from the crawler", ["html", "markdown", "content"])
class Crawler(ToolBase, ABC):
component_name = "Crawler"
def _run(self, history, **kwargs):
from api.utils.web_utils import is_valid_url
from common.ssrf_guard import assert_url_is_safe, pin_dns_global
ans = self.get_input()
ans = " - ".join(ans["content"]) if "content" in ans else ""
if not is_valid_url(ans):
try:
_ssrf_hostname, _ssrf_ip = assert_url_is_safe(ans)
except ValueError:
return Crawler.be_output("URL not valid")
try:
result = asyncio.run(self.get_web(ans))
# pin_dns_global is used (not thread-local) because crawl4ai resolves
# DNS in asyncio executor threads that don't share thread-local state.
with pin_dns_global(_ssrf_hostname, _ssrf_ip):
result = asyncio.run(self.get_web(ans))
return Crawler.be_output(result)
@@ -57,18 +62,15 @@ class Crawler(ToolBase, ABC):
proxy = self._param.proxy if self._param.proxy else None
async with AsyncWebCrawler(verbose=True, proxy=proxy) as crawler:
result = await crawler.arun(
url=url,
bypass_cache=True
)
result = await crawler.arun(url=url, bypass_cache=True)
if self.check_if_canceled("Crawler async operation"):
return
if self._param.extract_type == 'html':
if self._param.extract_type == "html":
return result.cleaned_html
elif self._param.extract_type == 'markdown':
elif self._param.extract_type == "markdown":
return result.markdown
elif self._param.extract_type == 'content':
elif self._param.extract_type == "content":
return result.extracted_content
return result.markdown

52
agent/tools/searxng.py
View File

@@ -20,6 +20,7 @@ from abc import ABC
import requests
from agent.tools.base import ToolMeta, ToolParamBase, ToolBase
from common.connection_utils import timeout
from common.ssrf_guard import assert_url_is_safe, pin_dns
class SearXNGParam(ToolParamBase):
@@ -36,15 +37,15 @@ class SearXNGParam(ToolParamBase):
"type": "string",
"description": "The search keywords to execute with SearXNG. The keywords should be the most important words/terms(includes synonyms) from the original request.",
"default": "{sys.query}",
"required": True
"required": True,
},
"searxng_url": {
"type": "string",
"description": "The base URL of your SearXNG instance (e.g., http://localhost:4000). This is required to connect to your SearXNG server.",
"required": False,
"default": ""
}
}
"default": "",
},
},
}
super().__init__()
self.top_n = 10
@@ -61,17 +62,7 @@ class SearXNGParam(ToolParamBase):
self.check_positive_integer(self.top_n, "Top N")
def get_input_form(self) -> dict[str, dict]:
return {
"query": {
"name": "Query",
"type": "line"
},
"searxng_url": {
"name": "SearXNG URL",
"type": "line",
"placeholder": "http://localhost:4000"
}
}
return {"query": {"name": "Query", "type": "line"}, "searxng_url": {"name": "SearXNG URL", "type": "line", "placeholder": "http://localhost:4000"}}
class SearXNG(ToolBase, ABC):
@@ -94,26 +85,22 @@ class SearXNG(ToolBase, ABC):
self.set_output("formalized_content", "")
return ""
try:
_ssrf_hostname, _ssrf_ip = assert_url_is_safe(searxng_url)
except ValueError as e:
self.set_output("_ERROR", str(e))
return f"SearXNG error: SSRF guard blocked {searxng_url!r}: {e}"
last_e = ""
for _ in range(self._param.max_retries+1):
for _ in range(self._param.max_retries + 1):
if self.check_if_canceled("SearXNG processing"):
return
try:
search_params = {
'q': query,
'format': 'json',
'categories': 'general',
'language': 'auto',
'safesearch': 1,
'pageno': 1
}
search_params = {"q": query, "format": "json", "categories": "general", "language": "auto", "safesearch": 1, "pageno": 1}
response = requests.get(
f"{searxng_url}/search",
params=search_params,
timeout=10
)
with pin_dns(_ssrf_hostname, _ssrf_ip):
response = requests.get(f"{searxng_url}/search", params=search_params, timeout=10)
response.raise_for_status()
if self.check_if_canceled("SearXNG processing"):
@@ -128,15 +115,12 @@ class SearXNG(ToolBase, ABC):
if not isinstance(results, list):
raise ValueError("Invalid results format from SearXNG")
results = results[:self._param.top_n]
results = results[: self._param.top_n]
if self.check_if_canceled("SearXNG processing"):
return
self._retrieve_chunks(results,
get_title=lambda r: r.get("title", ""),
get_url=lambda r: r.get("url", ""),
get_content=lambda r: r.get("content", ""))
self._retrieve_chunks(results, get_title=lambda r: r.get("title", ""), get_url=lambda r: r.get("url", ""), get_content=lambda r: r.get("content", ""))
self.set_output("json", results)
return self.output("formalized_content")