diff --git a/evalbench/generators/models/tools/fetch_url.py b/evalbench/generators/models/tools/fetch_url.py
index 76ab9061f..3da09f903 100644
--- a/evalbench/generators/models/tools/fetch_url.py
+++ b/evalbench/generators/models/tools/fetch_url.py
@@ -4,11 +4,11 @@
judge can ground rubric questions in external state.
Defense in depth: HTTPS only, SSRF guard against private/loopback/link-local
-hosts, 10s timeout, 50KB body cap.
+hosts (also for each redirect target), 10s timeout, 50KB body cap.
"""
from html.parser import HTMLParser
-from typing import Any, Dict
+from typing import Any, Dict, Optional
from urllib.parse import urlparse
import ipaddress
import logging
@@ -66,25 +66,49 @@ def _is_blocked_host(host: str) -> bool:
return False
-def fetch_url(args: Dict[str, Any]) -> str:
- url = args.get("url", "")
- if not isinstance(url, str) or not url:
- return "Error: missing required arg 'url' (string)"
-
+def _url_error(url: str) -> Optional[str]:
+ """Returns the reason to refuse url, or None if the URL is allowed."""
parsed = urlparse(url)
if parsed.scheme != "https":
- return f"Error: only https URLs are allowed (got scheme '{parsed.scheme}')"
+ return f"only https URLs are allowed (got scheme '{parsed.scheme}')"
if not parsed.hostname:
- return "Error: URL has no hostname"
+ return "URL has no hostname"
if _is_blocked_host(parsed.hostname):
return (
- f"Error: refusing to fetch '{parsed.hostname}' "
+ f"refusing to fetch '{parsed.hostname}' "
"(private, loopback, link-local, or unresolvable)"
)
+ return None
+
+
+class _CheckedRedirectHandler(urllib.request.HTTPRedirectHandler):
+ """Applies the URL checks to each redirect target."""
+
+ def redirect_request(self, req, fp, code, msg, headers, newurl):
+ error = _url_error(newurl)
+ if error:
+ raise urllib.error.URLError(f"redirect refused: {error}")
+ return super().redirect_request(req, fp, code, msg, headers, newurl)
+
+
+def _open(request):
+ """Opens request. A redirect target gets the same checks as the URL."""
+ opener = urllib.request.build_opener(_CheckedRedirectHandler)
+ return opener.open(request, timeout=_TIMEOUT_SECONDS)
+
+
+def fetch_url(args: Dict[str, Any]) -> str:
+ url = args.get("url", "")
+ if not isinstance(url, str) or not url:
+ return "Error: missing required arg 'url' (string)"
+
+ error = _url_error(url)
+ if error:
+ return f"Error: {error}"
request = urllib.request.Request(url, headers={"User-Agent": _USER_AGENT})
try:
- with urllib.request.urlopen(request, timeout=_TIMEOUT_SECONDS) as response:
+ with _open(request) as response:
raw = response.read(_MAX_BYTES + 1)
content_type = response.headers.get("Content-Type", "")
except urllib.error.HTTPError as e:
diff --git a/evalbench/test/tools_fetch_url_test.py b/evalbench/test/tools_fetch_url_test.py
index d7486f8c6..5df9aa5ac 100644
--- a/evalbench/test/tools_fetch_url_test.py
+++ b/evalbench/test/tools_fetch_url_test.py
@@ -1,13 +1,22 @@
+import http.server
import io
import os
import socket
import sys
+import threading
import urllib.error
+import urllib.request
from unittest.mock import MagicMock, patch
+import pytest
+
sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
-from generators.models.tools.fetch_url import fetch_url # noqa: E402
+from generators.models.tools.fetch_url import ( # noqa: E402
+ _CheckedRedirectHandler,
+ _open,
+ fetch_url,
+)
def _public_addrinfo():
@@ -70,7 +79,7 @@ def test_unresolvable_host_rejected(mock_getaddrinfo):
assert "private, loopback, link-local, or unresolvable" in result
-@patch("generators.models.tools.fetch_url.urllib.request.urlopen")
+@patch("generators.models.tools.fetch_url._open")
@patch("generators.models.tools.fetch_url.socket.getaddrinfo")
def test_plain_text_response_returned(mock_getaddrinfo, mock_urlopen):
mock_getaddrinfo.return_value = _public_addrinfo()
@@ -82,7 +91,7 @@ def test_plain_text_response_returned(mock_getaddrinfo, mock_urlopen):
assert result == "Apache Beam 2.99.0 released"
-@patch("generators.models.tools.fetch_url.urllib.request.urlopen")
+@patch("generators.models.tools.fetch_url._open")
@patch("generators.models.tools.fetch_url.socket.getaddrinfo")
def test_html_response_is_stripped(mock_getaddrinfo, mock_urlopen):
mock_getaddrinfo.return_value = _public_addrinfo()
@@ -99,7 +108,7 @@ def test_html_response_is_stripped(mock_getaddrinfo, mock_urlopen):
assert "
" not in result
-@patch("generators.models.tools.fetch_url.urllib.request.urlopen")
+@patch("generators.models.tools.fetch_url._open")
@patch("generators.models.tools.fetch_url.socket.getaddrinfo")
def test_oversized_body_is_truncated(mock_getaddrinfo, mock_urlopen):
mock_getaddrinfo.return_value = _public_addrinfo()
@@ -110,7 +119,7 @@ def test_oversized_body_is_truncated(mock_getaddrinfo, mock_urlopen):
assert len(result) <= 50_000 + len("\n\n[truncated at 50000 bytes]")
-@patch("generators.models.tools.fetch_url.urllib.request.urlopen")
+@patch("generators.models.tools.fetch_url._open")
@patch("generators.models.tools.fetch_url.socket.getaddrinfo")
def test_http_error_returned_as_string(mock_getaddrinfo, mock_urlopen):
mock_getaddrinfo.return_value = _public_addrinfo()
@@ -121,7 +130,7 @@ def test_http_error_returned_as_string(mock_getaddrinfo, mock_urlopen):
assert "HTTP 404" in result
-@patch("generators.models.tools.fetch_url.urllib.request.urlopen")
+@patch("generators.models.tools.fetch_url._open")
@patch("generators.models.tools.fetch_url.socket.getaddrinfo")
def test_timeout_returned_as_string(mock_getaddrinfo, mock_urlopen):
mock_getaddrinfo.return_value = _public_addrinfo()
@@ -130,7 +139,7 @@ def test_timeout_returned_as_string(mock_getaddrinfo, mock_urlopen):
assert "timeout after" in result
-@patch("generators.models.tools.fetch_url.urllib.request.urlopen")
+@patch("generators.models.tools.fetch_url._open")
@patch("generators.models.tools.fetch_url.socket.getaddrinfo")
def test_url_error_returned_as_string(mock_getaddrinfo, mock_urlopen):
mock_getaddrinfo.return_value = _public_addrinfo()
@@ -138,3 +147,87 @@ def test_url_error_returned_as_string(mock_getaddrinfo, mock_urlopen):
result = fetch_url({"url": "https://example.com/down"})
assert "URL error" in result
assert "connection refused" in result
+
+
+# --- Redirect targets get the same checks as the first URL ---
+
+def _redirect(newurl):
+ request = urllib.request.Request("https://example.com/start")
+ return _CheckedRedirectHandler().redirect_request(
+ request, None, 302, "Found", {}, newurl)
+
+
+@patch("generators.models.tools.fetch_url.socket.getaddrinfo")
+def test_redirect_to_public_https_host_is_followed(mock_getaddrinfo):
+ mock_getaddrinfo.return_value = _public_addrinfo()
+
+ new_request = _redirect("https://example.org/next")
+
+ assert new_request.full_url == "https://example.org/next"
+
+
+def test_redirect_to_http_is_refused():
+ with pytest.raises(urllib.error.URLError) as error:
+ _redirect("http://example.org/next")
+
+ assert "only https URLs are allowed" in str(error.value.reason)
+
+
+@patch("generators.models.tools.fetch_url.socket.getaddrinfo")
+def test_redirect_to_private_host_is_refused(mock_getaddrinfo):
+ for addrinfo in (_private_addrinfo(), _loopback_addrinfo()):
+ mock_getaddrinfo.return_value = addrinfo
+
+ with pytest.raises(urllib.error.URLError) as error:
+ _redirect("https://internal.example/secret")
+
+ assert "refusing to fetch 'internal.example'" in str(error.value.reason)
+
+
+def test_opener_refuses_a_redirect_before_the_second_request():
+ """Runs the real opener against a local server that redirects."""
+ paths = []
+
+ class Handler(http.server.BaseHTTPRequestHandler):
+ def do_GET(self):
+ paths.append(self.path)
+ if self.path == "/start":
+ self.send_response(302)
+ self.send_header(
+ "Location", f"http://127.0.0.1:{self.server.server_port}/target")
+ self.end_headers()
+ else:
+ self.send_response(200)
+ self.end_headers()
+ self.wfile.write(b"secret")
+
+ def log_message(self, *args):
+ pass
+
+ server = http.server.HTTPServer(("127.0.0.1", 0), Handler)
+ thread = threading.Thread(target=server.serve_forever, daemon=True)
+ thread.start()
+ try:
+ request = urllib.request.Request(
+ f"http://127.0.0.1:{server.server_port}/start")
+ with pytest.raises(urllib.error.URLError) as error:
+ _open(request)
+ finally:
+ server.shutdown()
+ server.server_close()
+
+ assert "redirect refused" in str(error.value.reason)
+ assert paths == ["/start"]
+
+
+@patch("generators.models.tools.fetch_url._open")
+@patch("generators.models.tools.fetch_url.socket.getaddrinfo")
+def test_refused_redirect_is_returned_as_string(mock_getaddrinfo, mock_open):
+ mock_getaddrinfo.return_value = _public_addrinfo()
+ mock_open.side_effect = urllib.error.URLError(
+ "redirect refused: URL has no hostname")
+
+ result = fetch_url({"url": "https://example.com/start"})
+
+ assert result.startswith("Error: URL error fetching")
+ assert "redirect refused" in result