Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
46 changes: 35 additions & 11 deletions evalbench/generators/models/tools/fetch_url.py
Original file line number Diff line number Diff line change
Expand Up @@ -4,11 +4,11 @@
judge can ground rubric questions in external state.

Defense in depth: HTTPS only, SSRF guard against private/loopback/link-local
hosts, 10s timeout, 50KB body cap.
hosts (also for each redirect target), 10s timeout, 50KB body cap.
"""

from html.parser import HTMLParser
from typing import Any, Dict
from typing import Any, Dict, Optional
from urllib.parse import urlparse
import ipaddress
import logging
Expand Down Expand Up @@ -66,25 +66,49 @@ def _is_blocked_host(host: str) -> bool:
return False


def fetch_url(args: Dict[str, Any]) -> str:
url = args.get("url", "")
if not isinstance(url, str) or not url:
return "Error: missing required arg 'url' (string)"

def _url_error(url: str) -> Optional[str]:
"""Returns the reason to refuse url, or None if the URL is allowed."""
parsed = urlparse(url)
if parsed.scheme != "https":
return f"Error: only https URLs are allowed (got scheme '{parsed.scheme}')"
return f"only https URLs are allowed (got scheme '{parsed.scheme}')"
if not parsed.hostname:
return "Error: URL has no hostname"
return "URL has no hostname"
if _is_blocked_host(parsed.hostname):
return (
f"Error: refusing to fetch '{parsed.hostname}' "
f"refusing to fetch '{parsed.hostname}' "
"(private, loopback, link-local, or unresolvable)"
)
return None


class _CheckedRedirectHandler(urllib.request.HTTPRedirectHandler):
"""Applies the URL checks to each redirect target."""

def redirect_request(self, req, fp, code, msg, headers, newurl):
error = _url_error(newurl)
if error:
raise urllib.error.URLError(f"redirect refused: {error}")
return super().redirect_request(req, fp, code, msg, headers, newurl)


def _open(request):
"""Opens request. A redirect target gets the same checks as the URL."""
opener = urllib.request.build_opener(_CheckedRedirectHandler)
return opener.open(request, timeout=_TIMEOUT_SECONDS)


def fetch_url(args: Dict[str, Any]) -> str:
url = args.get("url", "")
if not isinstance(url, str) or not url:
return "Error: missing required arg 'url' (string)"

error = _url_error(url)
if error:
return f"Error: {error}"

request = urllib.request.Request(url, headers={"User-Agent": _USER_AGENT})
try:
with urllib.request.urlopen(request, timeout=_TIMEOUT_SECONDS) as response:
with _open(request) as response:
raw = response.read(_MAX_BYTES + 1)
content_type = response.headers.get("Content-Type", "")
except urllib.error.HTTPError as e:
Expand Down
107 changes: 100 additions & 7 deletions evalbench/test/tools_fetch_url_test.py
Original file line number Diff line number Diff line change
@@ -1,13 +1,22 @@
import http.server
import io
import os
import socket
import sys
import threading
import urllib.error
import urllib.request
from unittest.mock import MagicMock, patch

import pytest

sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))

from generators.models.tools.fetch_url import fetch_url # noqa: E402
from generators.models.tools.fetch_url import ( # noqa: E402
_CheckedRedirectHandler,
_open,
fetch_url,
)


def _public_addrinfo():
Expand Down Expand Up @@ -70,7 +79,7 @@ def test_unresolvable_host_rejected(mock_getaddrinfo):
assert "private, loopback, link-local, or unresolvable" in result


@patch("generators.models.tools.fetch_url.urllib.request.urlopen")
@patch("generators.models.tools.fetch_url._open")
@patch("generators.models.tools.fetch_url.socket.getaddrinfo")
def test_plain_text_response_returned(mock_getaddrinfo, mock_urlopen):
mock_getaddrinfo.return_value = _public_addrinfo()
Expand All @@ -82,7 +91,7 @@ def test_plain_text_response_returned(mock_getaddrinfo, mock_urlopen):
assert result == "Apache Beam 2.99.0 released"


@patch("generators.models.tools.fetch_url.urllib.request.urlopen")
@patch("generators.models.tools.fetch_url._open")
@patch("generators.models.tools.fetch_url.socket.getaddrinfo")
def test_html_response_is_stripped(mock_getaddrinfo, mock_urlopen):
mock_getaddrinfo.return_value = _public_addrinfo()
Expand All @@ -99,7 +108,7 @@ def test_html_response_is_stripped(mock_getaddrinfo, mock_urlopen):
assert "<h1>" not in result


@patch("generators.models.tools.fetch_url.urllib.request.urlopen")
@patch("generators.models.tools.fetch_url._open")
@patch("generators.models.tools.fetch_url.socket.getaddrinfo")
def test_oversized_body_is_truncated(mock_getaddrinfo, mock_urlopen):
mock_getaddrinfo.return_value = _public_addrinfo()
Expand All @@ -110,7 +119,7 @@ def test_oversized_body_is_truncated(mock_getaddrinfo, mock_urlopen):
assert len(result) <= 50_000 + len("\n\n[truncated at 50000 bytes]")


@patch("generators.models.tools.fetch_url.urllib.request.urlopen")
@patch("generators.models.tools.fetch_url._open")
@patch("generators.models.tools.fetch_url.socket.getaddrinfo")
def test_http_error_returned_as_string(mock_getaddrinfo, mock_urlopen):
mock_getaddrinfo.return_value = _public_addrinfo()
Expand All @@ -121,7 +130,7 @@ def test_http_error_returned_as_string(mock_getaddrinfo, mock_urlopen):
assert "HTTP 404" in result


@patch("generators.models.tools.fetch_url.urllib.request.urlopen")
@patch("generators.models.tools.fetch_url._open")
@patch("generators.models.tools.fetch_url.socket.getaddrinfo")
def test_timeout_returned_as_string(mock_getaddrinfo, mock_urlopen):
mock_getaddrinfo.return_value = _public_addrinfo()
Expand All @@ -130,11 +139,95 @@ def test_timeout_returned_as_string(mock_getaddrinfo, mock_urlopen):
assert "timeout after" in result


@patch("generators.models.tools.fetch_url.urllib.request.urlopen")
@patch("generators.models.tools.fetch_url._open")
@patch("generators.models.tools.fetch_url.socket.getaddrinfo")
def test_url_error_returned_as_string(mock_getaddrinfo, mock_urlopen):
mock_getaddrinfo.return_value = _public_addrinfo()
mock_urlopen.side_effect = urllib.error.URLError("connection refused")
result = fetch_url({"url": "https://example.com/down"})
assert "URL error" in result
assert "connection refused" in result


# --- Redirect targets get the same checks as the first URL ---

def _redirect(newurl):
request = urllib.request.Request("https://example.com/start")
return _CheckedRedirectHandler().redirect_request(
request, None, 302, "Found", {}, newurl)


@patch("generators.models.tools.fetch_url.socket.getaddrinfo")
def test_redirect_to_public_https_host_is_followed(mock_getaddrinfo):
mock_getaddrinfo.return_value = _public_addrinfo()

new_request = _redirect("https://example.org/next")

assert new_request.full_url == "https://example.org/next"


def test_redirect_to_http_is_refused():
with pytest.raises(urllib.error.URLError) as error:
_redirect("http://example.org/next")

assert "only https URLs are allowed" in str(error.value.reason)


@patch("generators.models.tools.fetch_url.socket.getaddrinfo")
def test_redirect_to_private_host_is_refused(mock_getaddrinfo):
for addrinfo in (_private_addrinfo(), _loopback_addrinfo()):
mock_getaddrinfo.return_value = addrinfo

with pytest.raises(urllib.error.URLError) as error:
_redirect("https://internal.example/secret")

assert "refusing to fetch 'internal.example'" in str(error.value.reason)


def test_opener_refuses_a_redirect_before_the_second_request():
"""Runs the real opener against a local server that redirects."""
paths = []

class Handler(http.server.BaseHTTPRequestHandler):
def do_GET(self):
paths.append(self.path)
if self.path == "/start":
self.send_response(302)
self.send_header(
"Location", f"http://127.0.0.1:{self.server.server_port}/target")
self.end_headers()
else:
self.send_response(200)
self.end_headers()
self.wfile.write(b"secret")

def log_message(self, *args):
pass

server = http.server.HTTPServer(("127.0.0.1", 0), Handler)
thread = threading.Thread(target=server.serve_forever, daemon=True)
thread.start()
try:
request = urllib.request.Request(
f"http://127.0.0.1:{server.server_port}/start")
with pytest.raises(urllib.error.URLError) as error:
_open(request)
finally:
server.shutdown()
server.server_close()

assert "redirect refused" in str(error.value.reason)
assert paths == ["/start"]


@patch("generators.models.tools.fetch_url._open")
@patch("generators.models.tools.fetch_url.socket.getaddrinfo")
def test_refused_redirect_is_returned_as_string(mock_getaddrinfo, mock_open):
mock_getaddrinfo.return_value = _public_addrinfo()
mock_open.side_effect = urllib.error.URLError(
"redirect refused: URL has no hostname")

result = fetch_url({"url": "https://example.com/start"})

assert result.startswith("Error: URL error fetching")
assert "redirect refused" in result
Loading