From 8e5426387e68360f94ba5fc6b15408087b5e4770 Mon Sep 17 00:00:00 2001 From: Omkar Gaikwad Date: Fri, 20 Nov 2026 07:44:29 +0000 Subject: [PATCH] fix(tools): apply the fetch_url checks to redirect targets fetch_url checked only the first URL. A redirect could then send the request to a private host or to a URL that is not HTTPS. Check each redirect target with the same rules. --- .../generators/models/tools/fetch_url.py | 46 ++++++-- evalbench/test/tools_fetch_url_test.py | 107 ++++++++++++++++-- 2 files changed, 135 insertions(+), 18 deletions(-) diff --git a/evalbench/generators/models/tools/fetch_url.py b/evalbench/generators/models/tools/fetch_url.py index 76ab9061f..3da09f903 100644 --- a/evalbench/generators/models/tools/fetch_url.py +++ b/evalbench/generators/models/tools/fetch_url.py @@ -4,11 +4,11 @@ judge can ground rubric questions in external state. Defense in depth: HTTPS only, SSRF guard against private/loopback/link-local -hosts, 10s timeout, 50KB body cap. +hosts (also for each redirect target), 10s timeout, 50KB body cap. """ from html.parser import HTMLParser -from typing import Any, Dict +from typing import Any, Dict, Optional from urllib.parse import urlparse import ipaddress import logging @@ -66,25 +66,49 @@ def _is_blocked_host(host: str) -> bool: return False -def fetch_url(args: Dict[str, Any]) -> str: - url = args.get("url", "") - if not isinstance(url, str) or not url: - return "Error: missing required arg 'url' (string)" - +def _url_error(url: str) -> Optional[str]: + """Returns the reason to refuse url, or None if the URL is allowed.""" parsed = urlparse(url) if parsed.scheme != "https": - return f"Error: only https URLs are allowed (got scheme '{parsed.scheme}')" + return f"only https URLs are allowed (got scheme '{parsed.scheme}')" if not parsed.hostname: - return "Error: URL has no hostname" + return "URL has no hostname" if _is_blocked_host(parsed.hostname): return ( - f"Error: refusing to fetch '{parsed.hostname}' " + f"refusing to fetch '{parsed.hostname}' " "(private, loopback, link-local, or unresolvable)" ) + return None + + +class _CheckedRedirectHandler(urllib.request.HTTPRedirectHandler): + """Applies the URL checks to each redirect target.""" + + def redirect_request(self, req, fp, code, msg, headers, newurl): + error = _url_error(newurl) + if error: + raise urllib.error.URLError(f"redirect refused: {error}") + return super().redirect_request(req, fp, code, msg, headers, newurl) + + +def _open(request): + """Opens request. A redirect target gets the same checks as the URL.""" + opener = urllib.request.build_opener(_CheckedRedirectHandler) + return opener.open(request, timeout=_TIMEOUT_SECONDS) + + +def fetch_url(args: Dict[str, Any]) -> str: + url = args.get("url", "") + if not isinstance(url, str) or not url: + return "Error: missing required arg 'url' (string)" + + error = _url_error(url) + if error: + return f"Error: {error}" request = urllib.request.Request(url, headers={"User-Agent": _USER_AGENT}) try: - with urllib.request.urlopen(request, timeout=_TIMEOUT_SECONDS) as response: + with _open(request) as response: raw = response.read(_MAX_BYTES + 1) content_type = response.headers.get("Content-Type", "") except urllib.error.HTTPError as e: diff --git a/evalbench/test/tools_fetch_url_test.py b/evalbench/test/tools_fetch_url_test.py index d7486f8c6..5df9aa5ac 100644 --- a/evalbench/test/tools_fetch_url_test.py +++ b/evalbench/test/tools_fetch_url_test.py @@ -1,13 +1,22 @@ +import http.server import io import os import socket import sys +import threading import urllib.error +import urllib.request from unittest.mock import MagicMock, patch +import pytest + sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) -from generators.models.tools.fetch_url import fetch_url # noqa: E402 +from generators.models.tools.fetch_url import ( # noqa: E402 + _CheckedRedirectHandler, + _open, + fetch_url, +) def _public_addrinfo(): @@ -70,7 +79,7 @@ def test_unresolvable_host_rejected(mock_getaddrinfo): assert "private, loopback, link-local, or unresolvable" in result -@patch("generators.models.tools.fetch_url.urllib.request.urlopen") +@patch("generators.models.tools.fetch_url._open") @patch("generators.models.tools.fetch_url.socket.getaddrinfo") def test_plain_text_response_returned(mock_getaddrinfo, mock_urlopen): mock_getaddrinfo.return_value = _public_addrinfo() @@ -82,7 +91,7 @@ def test_plain_text_response_returned(mock_getaddrinfo, mock_urlopen): assert result == "Apache Beam 2.99.0 released" -@patch("generators.models.tools.fetch_url.urllib.request.urlopen") +@patch("generators.models.tools.fetch_url._open") @patch("generators.models.tools.fetch_url.socket.getaddrinfo") def test_html_response_is_stripped(mock_getaddrinfo, mock_urlopen): mock_getaddrinfo.return_value = _public_addrinfo() @@ -99,7 +108,7 @@ def test_html_response_is_stripped(mock_getaddrinfo, mock_urlopen): assert "

" not in result -@patch("generators.models.tools.fetch_url.urllib.request.urlopen") +@patch("generators.models.tools.fetch_url._open") @patch("generators.models.tools.fetch_url.socket.getaddrinfo") def test_oversized_body_is_truncated(mock_getaddrinfo, mock_urlopen): mock_getaddrinfo.return_value = _public_addrinfo() @@ -110,7 +119,7 @@ def test_oversized_body_is_truncated(mock_getaddrinfo, mock_urlopen): assert len(result) <= 50_000 + len("\n\n[truncated at 50000 bytes]") -@patch("generators.models.tools.fetch_url.urllib.request.urlopen") +@patch("generators.models.tools.fetch_url._open") @patch("generators.models.tools.fetch_url.socket.getaddrinfo") def test_http_error_returned_as_string(mock_getaddrinfo, mock_urlopen): mock_getaddrinfo.return_value = _public_addrinfo() @@ -121,7 +130,7 @@ def test_http_error_returned_as_string(mock_getaddrinfo, mock_urlopen): assert "HTTP 404" in result -@patch("generators.models.tools.fetch_url.urllib.request.urlopen") +@patch("generators.models.tools.fetch_url._open") @patch("generators.models.tools.fetch_url.socket.getaddrinfo") def test_timeout_returned_as_string(mock_getaddrinfo, mock_urlopen): mock_getaddrinfo.return_value = _public_addrinfo() @@ -130,7 +139,7 @@ def test_timeout_returned_as_string(mock_getaddrinfo, mock_urlopen): assert "timeout after" in result -@patch("generators.models.tools.fetch_url.urllib.request.urlopen") +@patch("generators.models.tools.fetch_url._open") @patch("generators.models.tools.fetch_url.socket.getaddrinfo") def test_url_error_returned_as_string(mock_getaddrinfo, mock_urlopen): mock_getaddrinfo.return_value = _public_addrinfo() @@ -138,3 +147,87 @@ def test_url_error_returned_as_string(mock_getaddrinfo, mock_urlopen): result = fetch_url({"url": "https://example.com/down"}) assert "URL error" in result assert "connection refused" in result + + +# --- Redirect targets get the same checks as the first URL --- + +def _redirect(newurl): + request = urllib.request.Request("https://example.com/start") + return _CheckedRedirectHandler().redirect_request( + request, None, 302, "Found", {}, newurl) + + +@patch("generators.models.tools.fetch_url.socket.getaddrinfo") +def test_redirect_to_public_https_host_is_followed(mock_getaddrinfo): + mock_getaddrinfo.return_value = _public_addrinfo() + + new_request = _redirect("https://example.org/next") + + assert new_request.full_url == "https://example.org/next" + + +def test_redirect_to_http_is_refused(): + with pytest.raises(urllib.error.URLError) as error: + _redirect("http://example.org/next") + + assert "only https URLs are allowed" in str(error.value.reason) + + +@patch("generators.models.tools.fetch_url.socket.getaddrinfo") +def test_redirect_to_private_host_is_refused(mock_getaddrinfo): + for addrinfo in (_private_addrinfo(), _loopback_addrinfo()): + mock_getaddrinfo.return_value = addrinfo + + with pytest.raises(urllib.error.URLError) as error: + _redirect("https://internal.example/secret") + + assert "refusing to fetch 'internal.example'" in str(error.value.reason) + + +def test_opener_refuses_a_redirect_before_the_second_request(): + """Runs the real opener against a local server that redirects.""" + paths = [] + + class Handler(http.server.BaseHTTPRequestHandler): + def do_GET(self): + paths.append(self.path) + if self.path == "/start": + self.send_response(302) + self.send_header( + "Location", f"http://127.0.0.1:{self.server.server_port}/target") + self.end_headers() + else: + self.send_response(200) + self.end_headers() + self.wfile.write(b"secret") + + def log_message(self, *args): + pass + + server = http.server.HTTPServer(("127.0.0.1", 0), Handler) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + try: + request = urllib.request.Request( + f"http://127.0.0.1:{server.server_port}/start") + with pytest.raises(urllib.error.URLError) as error: + _open(request) + finally: + server.shutdown() + server.server_close() + + assert "redirect refused" in str(error.value.reason) + assert paths == ["/start"] + + +@patch("generators.models.tools.fetch_url._open") +@patch("generators.models.tools.fetch_url.socket.getaddrinfo") +def test_refused_redirect_is_returned_as_string(mock_getaddrinfo, mock_open): + mock_getaddrinfo.return_value = _public_addrinfo() + mock_open.side_effect = urllib.error.URLError( + "redirect refused: URL has no hostname") + + result = fetch_url({"url": "https://example.com/start"}) + + assert result.startswith("Error: URL error fetching") + assert "redirect refused" in result