From 3db0789f17c0a09da075ea3e41c744ea4a87cbc5 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 26 Aug 2026 08:33:13 +0900 Subject: [PATCH 1/9] fix(security): require OS isolation for web e2e commands --- scripts/ci/sandboxed_web_e2e.py | 125 +++++++++++++++++- ...ory_branch_coverage_execution_sandboxes.py | 2 + tests/test_sandboxed_web_e2e.py | 67 ++++++++++ 3 files changed, 191 insertions(+), 3 deletions(-) diff --git a/scripts/ci/sandboxed_web_e2e.py b/scripts/ci/sandboxed_web_e2e.py index ae0c3105a..5e0b635ce 100644 --- a/scripts/ci/sandboxed_web_e2e.py +++ b/scripts/ci/sandboxed_web_e2e.py @@ -5,6 +5,7 @@ import argparse import json import os +import platform import signal import shutil import shlex @@ -25,6 +26,7 @@ RESULT_MARKER = "SANDBOXED_WEB_E2E_RESULT" +SANDBOX_MOUNT = "/workspace" class NoRedirectHandler(urllib.request.HTTPRedirectHandler): @@ -63,6 +65,15 @@ def parse_args(argv: Sequence[str] | None = None) -> argparse.Namespace: parser.add_argument("--startup-timeout", type=int, default=120, help="Seconds to wait for readiness URLs.") parser.add_argument("--e2e-timeout", type=int, default=600, help="Seconds to allow the E2E command to run.") parser.add_argument("--keep-sandbox", action="store_true", help="Keep the temporary sandbox after execution.") + parser.add_argument( + "--isolation", + choices=("required", "disabled"), + default="required", + help=( + "Require a bubblewrap OS sandbox (the default). Use disabled only for " + "trusted local debugging when bubblewrap is unavailable." + ), + ) parser.add_argument( "--allow-env", action="append", @@ -98,6 +109,70 @@ def parse_args(argv: Sequence[str] | None = None) -> argparse.Namespace: return args +def isolation_backend(mode: str) -> str | None: + """Resolve the requested OS isolation backend without silently downgrading.""" + if mode == "disabled": + return None + if platform.system() != "Linux": + raise RuntimeError("required isolation is only supported on Linux with bubblewrap") + backend = shutil.which("bwrap") + if backend is None: + raise RuntimeError("required isolation needs bubblewrap (bwrap) on PATH") + return backend + + +def _sandbox_environment(env: dict[str, str], sandbox_root: Path) -> dict[str, str]: + """Map host sandbox paths to the path exposed inside the bubblewrap mount.""" + source = str(sandbox_root) + mapped = dict(env) + for key in ("HOME", "TMPDIR", "XDG_CACHE_HOME", "XDG_CONFIG_HOME", "XDG_DATA_HOME"): + value = mapped.get(key) + if value: + mapped[key] = value.replace(source, SANDBOX_MOUNT, 1) + return mapped + + +def isolated_command( + command: str, + *, + backend: str, + cwd: Path, + sandbox_root: Path, + env: dict[str, str], +) -> str: + """Wrap one command in a read-only-root bubblewrap workspace.""" + argv = shlex.split(command) + if not argv: + raise ValueError("command must not be empty") + executable = shutil.which(argv[0], path=env.get("PATH")) + if executable is not None and Path(executable).is_relative_to(Path.home()): + raise RuntimeError("commands from the host home directory are not allowed in isolation") + bind_roots = [Path(path) for path in ("/usr", "/bin", "/sbin", "/lib", "/lib64", "/opt") if Path(path).exists()] + args = [backend, "--die-with-parent", "--new-session", "--unshare-pid"] + for root in bind_roots: + args.extend(("--ro-bind", str(root), str(root))) + for path in ("/etc/ssl", "/etc/hosts", "/etc/resolv.conf", "/etc/localtime"): + if Path(path).exists(): + args.extend(("--ro-bind", path, path)) + args.extend( + ( + "--proc", + "/proc", + "--dev", + "/dev", + "--tmpfs", + "/tmp", + "--bind", + str(sandbox_root), + SANDBOX_MOUNT, + "--chdir", + f"{SANDBOX_MOUNT}/{cwd.relative_to(sandbox_root)}", + "--", + ) + ) + return shlex.join([*args, *argv]) + + def start_service(label: str, command: str, cwd: Path, env: dict[str, str], logs_dir: Path) -> Service: """Start a service command in its own process group.""" log_path = logs_dir / f"{label}.log" @@ -195,6 +270,8 @@ def emit_result( "frontend_cmd": args.frontend_cmd, "frontend_ready": frontend_ready, "network": args.network, + "isolation": args.isolation, + "isolation_backend": getattr(args, "isolation_backend", "unknown"), "sandbox": str(sandbox_root) if args.keep_sandbox else "(removed)", "sandboxed": True, } @@ -216,13 +293,55 @@ def main(argv: Sequence[str] | None = None) -> int: try: copied_repo = sandboxed_verify.copy_workspace(Path(args.repo_root), sandbox, args.ignore) env = sandboxed_verify.scrubbed_env(sandbox, args.allow_env) + try: + backend = isolation_backend(args.isolation) + except RuntimeError as exc: + print(f"sandboxed-web-e2e: {exc}", file=sys.stderr) + args.isolation_backend = "unavailable" + exit_code = 126 + return exit_code + args.isolation_backend = backend or "disabled" print(f"sandboxed-web-e2e: cwd={copied_repo}") if args.allow_env: print(f"sandboxed-web-e2e: allowed env names={','.join(sorted(set(args.allow_env)))}") if args.network != "default": print(f"sandboxed-web-e2e: network={args.network}") - services.append(start_service("backend", args.backend_cmd, copied_repo, env, logs_dir)) - services.append(start_service("frontend", args.frontend_cmd, copied_repo, env, logs_dir)) + command_env = _sandbox_environment(env, sandbox) if backend else env + backend_cmd = ( + isolated_command( + args.backend_cmd, + backend=backend, + cwd=copied_repo, + sandbox_root=sandbox, + env=env, + ) + if backend + else args.backend_cmd + ) + frontend_cmd = ( + isolated_command( + args.frontend_cmd, + backend=backend, + cwd=copied_repo, + sandbox_root=sandbox, + env=env, + ) + if backend + else args.frontend_cmd + ) + e2e_cmd = ( + isolated_command( + args.e2e_cmd, + backend=backend, + cwd=copied_repo, + sandbox_root=sandbox, + env=env, + ) + if backend + else args.e2e_cmd + ) + services.append(start_service("backend", backend_cmd, copied_repo, command_env, logs_dir)) + services.append(start_service("frontend", frontend_cmd, copied_repo, command_env, logs_dir)) backend_ready = wait_for_url(args.backend_ready_url, args.startup_timeout, services[0]) frontend_ready = wait_for_url(args.frontend_ready_url, args.startup_timeout, services[1]) if not backend_ready or not frontend_ready: @@ -230,7 +349,7 @@ def main(argv: Sequence[str] | None = None) -> int: exit_code = 125 return exit_code try: - completed = run_shell(args.e2e_cmd, copied_repo, env, args.e2e_timeout) + completed = run_shell(e2e_cmd, copied_repo, command_env, args.e2e_timeout) if completed.stdout: print(completed.stdout, end="") if completed.stderr: diff --git a/tests/test_repository_branch_coverage_execution_sandboxes.py b/tests/test_repository_branch_coverage_execution_sandboxes.py index f8912272a..7d1ec0a43 100644 --- a/tests/test_repository_branch_coverage_execution_sandboxes.py +++ b/tests/test_repository_branch_coverage_execution_sandboxes.py @@ -216,6 +216,8 @@ def timeout_runner( [ "--repo-root", str(repo), + "--isolation", + "disabled", "--backend-cmd", "backend", "--frontend-cmd", diff --git a/tests/test_sandboxed_web_e2e.py b/tests/test_sandboxed_web_e2e.py index 6e092c293..e015b0cdb 100644 --- a/tests/test_sandboxed_web_e2e.py +++ b/tests/test_sandboxed_web_e2e.py @@ -56,6 +56,8 @@ def test_sandboxed_web_e2e_runs_services_and_does_not_mutate_source(tmp_path, ca [ "--repo-root", str(repo), + "--isolation", + "disabled", "--backend-cmd", http_server_command(backend_port, "backend"), "--frontend-cmd", @@ -298,6 +300,8 @@ def fake_start(label, command, cwd, env, logs_dir): [ "--repo-root", str(repo), + "--isolation", + "disabled", "--backend-cmd", "backend", "--frontend-cmd", @@ -361,6 +365,8 @@ def fake_wait(url, timeout, service): [ "--repo-root", str(repo), + "--isolation", + "disabled", "--backend-cmd", "backend", "--frontend-cmd", @@ -410,6 +416,8 @@ def fake_run_shell(command, cwd, env, timeout): [ "--repo-root", str(repo), + "--isolation", + "disabled", "--backend-cmd", "backend", "--frontend-cmd", @@ -440,6 +448,8 @@ def test_sandboxed_web_e2e_reports_readiness_failure(tmp_path, capsys): [ "--repo-root", str(repo), + "--isolation", + "disabled", "--backend-cmd", http_server_command(backend_port, "backend"), "--frontend-cmd", @@ -478,6 +488,8 @@ def fake_run_shell(command, cwd, env, timeout): [ "--repo-root", str(repo), + "--isolation", + "disabled", "--backend-cmd", f"{sys.executable} -c \"import time; time.sleep(3)\"", "--frontend-cmd", @@ -497,6 +509,59 @@ def fake_run_shell(command, cwd, env, timeout): assert "SANDBOXED_WEB_E2E_RESULT" in captured.out +def test_isolation_backend_fails_closed_outside_linux(monkeypatch): + """Required isolation never silently falls back to a host process.""" + monkeypatch.setattr(sandboxed_web_e2e.platform, "system", lambda: "Darwin") + with pytest.raises(RuntimeError, match="only supported on Linux"): + sandboxed_web_e2e.isolation_backend("required") + assert sandboxed_web_e2e.isolation_backend("disabled") is None + + +def test_isolated_command_mounts_only_workspace(monkeypatch, tmp_path): + """Bubblewrap commands expose the copied workspace and not the host root.""" + monkeypatch.setattr(sandboxed_web_e2e.platform, "system", lambda: "Linux") + monkeypatch.setattr( + sandboxed_web_e2e.shutil, + "which", + lambda name, path=None: "/usr/bin/bwrap" if name == "bwrap" else "/usr/bin/python3", + ) + sandbox = tmp_path / "sandbox" + repo = sandbox / "repo" + repo.mkdir(parents=True) + env = {"PATH": "/usr/bin", "HOME": str(sandbox / "home")} + command = sandboxed_web_e2e.isolated_command( + "python3 -c 'print(1)'", + backend="/usr/bin/bwrap", + cwd=repo, + sandbox_root=sandbox, + env=env, + ) + assert command.startswith("/usr/bin/bwrap") + assert "--bind" in command + assert "--chdir /workspace/repo" in command + assert "--ro-bind / /" not in command + + +def test_isolated_command_rejects_host_home_executable(monkeypatch, tmp_path): + """Executable paths from a user's home cannot enter the isolated runner.""" + monkeypatch.setattr( + sandboxed_web_e2e.shutil, + "which", + lambda *_args, **_kwargs: str(Path.home() / "bin/tool"), + ) + sandbox = tmp_path / "sandbox" + repo = sandbox / "repo" + repo.mkdir(parents=True) + with pytest.raises(RuntimeError, match="host home directory"): + sandboxed_web_e2e.isolated_command( + "tool", + backend="/usr/bin/bwrap", + cwd=repo, + sandbox_root=sandbox, + env={"PATH": "/usr/bin"}, + ) + + def test_parse_args_rejects_invalid_inputs(): """The CLI rejects unusable timeout and environment values.""" with pytest.raises(SystemExit): @@ -582,6 +647,8 @@ def test_module_import_and_main_entrypoint(monkeypatch, tmp_path): "sandboxed_web_e2e.py", "--repo-root", str(repo), + "--isolation", + "disabled", "--backend-cmd", f"{sys.executable} -c \"import time; time.sleep(0.2)\"", "--frontend-cmd", From d42a97f196ede04517e286e839b596cd570ac1da Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 26 Aug 2026 08:36:09 +0900 Subject: [PATCH 2/9] fix(security): harden loopback and document isolation --- .../doctoring/sandboxed-web-command-isolation.md | 16 ++++++++++++++++ scripts/ci/sandboxed_web_e2e.py | 12 ++++++++++++ tests/test_sandboxed_web_e2e.py | 10 ++++++++++ 3 files changed, 38 insertions(+) create mode 100644 docs/doctoring/sandboxed-web-command-isolation.md diff --git a/docs/doctoring/sandboxed-web-command-isolation.md b/docs/doctoring/sandboxed-web-command-isolation.md new file mode 100644 index 000000000..5dbf40b73 --- /dev/null +++ b/docs/doctoring/sandboxed-web-command-isolation.md @@ -0,0 +1,16 @@ +# Sandboxed web command isolation + +`sandboxed_web_e2e.py` requires Linux `bubblewrap` (`bwrap`) by default. Each +backend, frontend, and E2E command runs with a read-only runtime root and a +single writable mount at `/workspace`; the copied repository and temporary +homes are mapped there. The host filesystem is therefore not reachable through +absolute paths or `..` traversal. + +Use `--isolation disabled` only for trusted local debugging. The result marker +records the requested mode and resolved backend so CI evidence cannot be +mistaken for an OS-isolated run. If required isolation is unavailable, the +command exits with code `126` before starting any service. + +Readiness polling remains loopback-only and does not follow redirects. The +network declaration is evidence metadata; callers that need stronger network +policy must run this helper inside a network-restricted runner or container. diff --git a/scripts/ci/sandboxed_web_e2e.py b/scripts/ci/sandboxed_web_e2e.py index 5e0b635ce..7bec55c33 100644 --- a/scripts/ci/sandboxed_web_e2e.py +++ b/scripts/ci/sandboxed_web_e2e.py @@ -3,6 +3,7 @@ from __future__ import annotations import argparse +import ipaddress import json import os import platform @@ -14,6 +15,7 @@ import tempfile import time import urllib.error +import urllib.parse import urllib.request from collections.abc import Sequence from dataclasses import dataclass @@ -196,6 +198,16 @@ def wait_for_url(url: str, timeout: int, service: Service) -> bool: return True if not (url.startswith("http://") or url.startswith("https://")): raise ValueError(f"URL must start with http:// or https://, got: {url}") + + parsed = urllib.parse.urlparse(url) + hostname = (parsed.hostname or "").lower() + try: + is_loopback = hostname == "localhost" or ipaddress.ip_address(hostname).is_loopback + except ValueError: + is_loopback = False + if not is_loopback: + raise ValueError(f"URL cannot target external hostname: {hostname}") + deadline = time.monotonic() + timeout opener = urllib.request.build_opener(NoRedirectHandler()) while time.monotonic() < deadline: diff --git a/tests/test_sandboxed_web_e2e.py b/tests/test_sandboxed_web_e2e.py index e015b0cdb..0365feaf9 100644 --- a/tests/test_sandboxed_web_e2e.py +++ b/tests/test_sandboxed_web_e2e.py @@ -113,9 +113,19 @@ def test_wait_helpers_and_service_cleanup_edges(monkeypatch, tmp_path): exited_service = sandboxed_web_e2e.Service("done", "true", exited, tmp_path / "missing.log") assert sandboxed_web_e2e.wait_for_url("", 1, exited_service) is True + assert sandboxed_web_e2e.wait_for_url("http://localhost:1/", 1, exited_service) is False assert sandboxed_web_e2e.wait_for_url("http://127.0.0.1:1/", 1, exited_service) is False + assert sandboxed_web_e2e.wait_for_url("http://127.0.0.2:1/", 1, exited_service) is False with pytest.raises(ValueError, match="URL must start with http:// or https://"): sandboxed_web_e2e.wait_for_url("file:///etc/passwd", 1, exited_service) + with pytest.raises(ValueError, match="URL cannot target external hostname: external.example.com"): + sandboxed_web_e2e.wait_for_url("http://external.example.com/ready", 1, exited_service) + with pytest.raises(ValueError, match="URL cannot target external hostname: app.localhost"): + sandboxed_web_e2e.wait_for_url("http://app.localhost:8000/health", 1, exited_service) + with pytest.raises(ValueError, match="URL cannot target external hostname: 169.254.169.254"): + sandboxed_web_e2e.wait_for_url("http://169.254.169.254/latest/meta-data/", 1, exited_service) + with pytest.raises(ValueError, match="URL cannot target external hostname: 0.0.0.0"): + sandboxed_web_e2e.wait_for_url("http://0.0.0.0:8000/health", 1, exited_service) sandboxed_web_e2e.stop_service(exited_service) assert sandboxed_web_e2e.tail_text(tmp_path / "missing.log") == "" From 823cd89a8698918fb303b6bab6f0330c8b7684ea Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 26 Aug 2026 08:38:05 +0900 Subject: [PATCH 3/9] fix(security): mount isolated root filesystem --- scripts/ci/sandboxed_web_e2e.py | 2 +- tests/test_sandboxed_web_e2e.py | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/scripts/ci/sandboxed_web_e2e.py b/scripts/ci/sandboxed_web_e2e.py index 7bec55c33..c37b188c5 100644 --- a/scripts/ci/sandboxed_web_e2e.py +++ b/scripts/ci/sandboxed_web_e2e.py @@ -150,7 +150,7 @@ def isolated_command( if executable is not None and Path(executable).is_relative_to(Path.home()): raise RuntimeError("commands from the host home directory are not allowed in isolation") bind_roots = [Path(path) for path in ("/usr", "/bin", "/sbin", "/lib", "/lib64", "/opt") if Path(path).exists()] - args = [backend, "--die-with-parent", "--new-session", "--unshare-pid"] + args = [backend, "--die-with-parent", "--new-session", "--unshare-pid", "--tmpfs", "/"] for root in bind_roots: args.extend(("--ro-bind", str(root), str(root))) for path in ("/etc/ssl", "/etc/hosts", "/etc/resolv.conf", "/etc/localtime"): diff --git a/tests/test_sandboxed_web_e2e.py b/tests/test_sandboxed_web_e2e.py index 0365feaf9..985100dd0 100644 --- a/tests/test_sandboxed_web_e2e.py +++ b/tests/test_sandboxed_web_e2e.py @@ -547,6 +547,7 @@ def test_isolated_command_mounts_only_workspace(monkeypatch, tmp_path): env=env, ) assert command.startswith("/usr/bin/bwrap") + assert "--tmpfs /" in command assert "--bind" in command assert "--chdir /workspace/repo" in command assert "--ro-bind / /" not in command From ce93f556b35edc9665ac78dd374266a774b76f26 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 26 Aug 2026 09:06:59 +0900 Subject: [PATCH 4/9] docs(security): document isolated web verification --- CHANGELOG.md | 3 +++ docs/doctoring/sandboxed-web-command-isolation.md | 5 +++++ 2 files changed, 8 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index cef0acda6..e88bf9678 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,9 @@ this file. The format follows Keep a Changelog, and versioned releases follow Semantic Versioning where the repository publishes a release. ## [Unreleased] +- Require Linux bubblewrap isolation for backend, frontend, and E2E commands + in the web verification helper, mount only a writable workspace, and reject + non-loopback readiness URLs or redirects so SSRF probes fail closed. - Route Strix cross-provider fallbacks to explicit direct-OpenAI models (`openai-direct/...`) through the OpenAI inference endpoint instead of inheriting a provider-specific primary base: the workflow now provisions diff --git a/docs/doctoring/sandboxed-web-command-isolation.md b/docs/doctoring/sandboxed-web-command-isolation.md index 5dbf40b73..3f8467d5b 100644 --- a/docs/doctoring/sandboxed-web-command-isolation.md +++ b/docs/doctoring/sandboxed-web-command-isolation.md @@ -14,3 +14,8 @@ command exits with code `126` before starting any service. Readiness polling remains loopback-only and does not follow redirects. The network declaration is evidence metadata; callers that need stronger network policy must run this helper inside a network-restricted runner or container. + +## References + +MITRE. (2026). *CWE-918: Server-side request forgery (SSRF)*. +https://cwe.mitre.org/data/definitions/918.html From 871d50bb14fd88bfa2d14e725b70e6bc61bedffd Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 26 Aug 2026 09:39:15 +0900 Subject: [PATCH 5/9] fix(e2e): cover and harden isolated command paths --- CHANGELOG.md | 3 + .../sandboxed-web-command-isolation.md | 13 +- scripts/ci/sandboxed_web_e2e.py | 25 +- tests/test_sandboxed_web_e2e.py | 240 ++++++++++++++++++ 4 files changed, 273 insertions(+), 8 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e88bf9678..278175d3a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,9 @@ Semantic Versioning where the repository publishes a release. - Require Linux bubblewrap isolation for backend, frontend, and E2E commands in the web verification helper, mount only a writable workspace, and reject non-loopback readiness URLs or redirects so SSRF probes fail closed. +- Fail closed when an isolated command resolves outside the mounted system + roots, return a coded readiness failure for invalid URLs, and cover required + isolation plus sandbox-environment path mapping in the 100% branch contract. - Route Strix cross-provider fallbacks to explicit direct-OpenAI models (`openai-direct/...`) through the OpenAI inference endpoint instead of inheriting a provider-specific primary base: the workflow now provisions diff --git a/docs/doctoring/sandboxed-web-command-isolation.md b/docs/doctoring/sandboxed-web-command-isolation.md index 3f8467d5b..1e862cfc3 100644 --- a/docs/doctoring/sandboxed-web-command-isolation.md +++ b/docs/doctoring/sandboxed-web-command-isolation.md @@ -6,14 +6,21 @@ single writable mount at `/workspace`; the copied repository and temporary homes are mapped there. The host filesystem is therefore not reachable through absolute paths or `..` traversal. +Before wrapping a command, the helper resolves its executable and rejects paths +outside the read-only system roots mounted by bubblewrap. A tool installed in a +host-only location must be installed into one of those roots or the run exits +before any service starts. + Use `--isolation disabled` only for trusted local debugging. The result marker records the requested mode and resolved backend so CI evidence cannot be mistaken for an OS-isolated run. If required isolation is unavailable, the command exits with code `126` before starting any service. -Readiness polling remains loopback-only and does not follow redirects. The -network declaration is evidence metadata; callers that need stronger network -policy must run this helper inside a network-restricted runner or container. +Readiness polling remains loopback-only and does not follow redirects. Invalid +readiness URLs are reported as a coded readiness failure (`125`) rather than an +uncaught traceback. The network declaration is evidence metadata; callers that +need stronger network policy must run this helper inside a network-restricted +runner or container. ## References diff --git a/scripts/ci/sandboxed_web_e2e.py b/scripts/ci/sandboxed_web_e2e.py index c37b188c5..a99ebee45 100644 --- a/scripts/ci/sandboxed_web_e2e.py +++ b/scripts/ci/sandboxed_web_e2e.py @@ -146,10 +146,20 @@ def isolated_command( argv = shlex.split(command) if not argv: raise ValueError("command must not be empty") + bind_roots = [ + Path(path) + for path in ("/usr", "/bin", "/sbin", "/lib", "/lib64", "/opt") + if Path(path).exists() + ] executable = shutil.which(argv[0], path=env.get("PATH")) - if executable is not None and Path(executable).is_relative_to(Path.home()): - raise RuntimeError("commands from the host home directory are not allowed in isolation") - bind_roots = [Path(path) for path in ("/usr", "/bin", "/sbin", "/lib", "/lib64", "/opt") if Path(path).exists()] + if executable is not None: + executable_path = Path(executable) + if executable_path.is_relative_to(Path.home()): + raise RuntimeError("commands from the host home directory are not allowed in isolation") + if not any(executable_path.is_relative_to(root) for root in bind_roots): + raise RuntimeError( + f"executable is outside the isolated bind roots: {executable_path}" + ) args = [backend, "--die-with-parent", "--new-session", "--unshare-pid", "--tmpfs", "/"] for root in bind_roots: args.extend(("--ro-bind", str(root), str(root))) @@ -354,8 +364,13 @@ def main(argv: Sequence[str] | None = None) -> int: ) services.append(start_service("backend", backend_cmd, copied_repo, command_env, logs_dir)) services.append(start_service("frontend", frontend_cmd, copied_repo, command_env, logs_dir)) - backend_ready = wait_for_url(args.backend_ready_url, args.startup_timeout, services[0]) - frontend_ready = wait_for_url(args.frontend_ready_url, args.startup_timeout, services[1]) + try: + backend_ready = wait_for_url(args.backend_ready_url, args.startup_timeout, services[0]) + frontend_ready = wait_for_url(args.frontend_ready_url, args.startup_timeout, services[1]) + except ValueError as exc: + print(f"sandboxed-web-e2e: invalid readiness URL: {exc}", file=sys.stderr) + exit_code = 125 + return exit_code if not backend_ready or not frontend_ready: print("sandboxed-web-e2e: service readiness failed", file=sys.stderr) exit_code = 125 diff --git a/tests/test_sandboxed_web_e2e.py b/tests/test_sandboxed_web_e2e.py index 985100dd0..3fb2fbfb8 100644 --- a/tests/test_sandboxed_web_e2e.py +++ b/tests/test_sandboxed_web_e2e.py @@ -350,6 +350,92 @@ def fake_start(label, command, cwd, env, logs_dir): assert payload["evidence_note"] == "needs browser auth" +def test_main_runs_required_isolation_with_mapped_environment(monkeypatch, tmp_path, capsys): + """Required isolation wraps every command and maps sandbox paths into /workspace.""" + repo = tmp_path / "repo" + repo.mkdir() + wrapped = [] + started = [] + + class DoneProcess: + def poll(self): + return 0 + + def fake_isolated(command, **kwargs): + wrapped.append((command, kwargs)) + return f"wrapped {command}" + + def fake_start(label, command, cwd, env, logs_dir): + log_path = logs_dir / f"{label}.log" + log_path.write_text(f"{label} ready\n", encoding="utf-8") + started.append((label, command, cwd, env)) + return sandboxed_web_e2e.Service(label, command, DoneProcess(), log_path) + + monkeypatch.setattr(sandboxed_web_e2e, "isolation_backend", lambda mode: "/usr/bin/bwrap") + monkeypatch.setattr(sandboxed_web_e2e, "isolated_command", fake_isolated) + monkeypatch.setattr(sandboxed_web_e2e, "start_service", fake_start) + monkeypatch.setattr(sandboxed_web_e2e, "wait_for_url", lambda url, timeout, service: True) + monkeypatch.setattr( + sandboxed_web_e2e, + "run_shell", + lambda command, cwd, env, timeout: subprocess.CompletedProcess(command, 0), + ) + monkeypatch.setattr(sandboxed_web_e2e, "stop_service", lambda service: None) + + exit_code = sandboxed_web_e2e.main( + [ + "--repo-root", + str(repo), + "--backend-cmd", + "backend", + "--frontend-cmd", + "frontend", + "--e2e-cmd", + "e2e", + ] + ) + captured = capsys.readouterr() + + assert exit_code == 0 + assert [item[0] for item in wrapped] == ["backend", "frontend", "e2e"] + assert [item[0] for item in started] == ["backend", "frontend"] + assert all(item[1].startswith("wrapped ") for item in started) + assert started[0][3]["HOME"].startswith("/workspace/") + assert "isolation_backend=unknown" not in captured.out + + +def test_main_reports_unavailable_required_isolation(monkeypatch, tmp_path, capsys): + """Required isolation errors exit before starting services with code 126.""" + repo = tmp_path / "repo" + repo.mkdir() + monkeypatch.setattr( + sandboxed_web_e2e, + "isolation_backend", + lambda mode: (_ for _ in ()).throw(RuntimeError("bwrap unavailable")), + ) + + exit_code = sandboxed_web_e2e.main( + [ + "--repo-root", + str(repo), + "--backend-cmd", + "backend", + "--frontend-cmd", + "frontend", + "--e2e-cmd", + "e2e", + ] + ) + captured = capsys.readouterr() + + assert exit_code == 126 + assert "bwrap unavailable" in captured.err + result_line = [line for line in captured.out.splitlines() if line.startswith(sandboxed_web_e2e.RESULT_MARKER)][-1] + payload = json.loads(result_line.removeprefix(sandboxed_web_e2e.RESULT_MARKER).strip()) + assert payload["exit_code"] == 126 + assert payload["isolation_backend"] == "unavailable" + + def test_main_reports_stubbed_readiness_failure(monkeypatch, tmp_path, capsys): """Main exits distinctly when a stubbed service never becomes ready.""" repo = tmp_path / "repo" @@ -400,6 +486,55 @@ def fake_wait(url, timeout, service): assert payload["exit_code"] == 125 +def test_main_reports_invalid_readiness_url(monkeypatch, tmp_path, capsys): + """Invalid readiness input exits with the same clean readiness failure code.""" + repo = tmp_path / "repo" + repo.mkdir() + + class DoneProcess: + def poll(self): + return 0 + + def fake_start(label, command, cwd, env, logs_dir): + log_path = logs_dir / f"{label}.log" + log_path.write_text("ready\n", encoding="utf-8") + return sandboxed_web_e2e.Service(label, command, DoneProcess(), log_path) + + monkeypatch.setattr(sandboxed_web_e2e, "start_service", fake_start) + monkeypatch.setattr( + sandboxed_web_e2e, + "wait_for_url", + lambda url, timeout, service: (_ for _ in ()).throw(ValueError("bad host")), + ) + monkeypatch.setattr(sandboxed_web_e2e, "stop_service", lambda service: None) + + exit_code = sandboxed_web_e2e.main( + [ + "--repo-root", + str(repo), + "--isolation", + "disabled", + "--backend-cmd", + "backend", + "--frontend-cmd", + "frontend", + "--backend-ready-url", + "http://external.example/health", + "--frontend-ready-url", + "http://127.0.0.1:3000/", + "--e2e-cmd", + "e2e", + ] + ) + captured = capsys.readouterr() + + assert exit_code == 125 + assert "invalid readiness URL: bad host" in captured.err + result_line = [line for line in captured.out.splitlines() if line.startswith(sandboxed_web_e2e.RESULT_MARKER)][-1] + payload = json.loads(result_line.removeprefix(sandboxed_web_e2e.RESULT_MARKER).strip()) + assert payload["exit_code"] == 125 + + def test_main_reports_stubbed_e2e_timeout(monkeypatch, tmp_path, capsys): """Main preserves timeout output from stubbed E2E execution.""" repo = tmp_path / "repo" @@ -527,6 +662,21 @@ def test_isolation_backend_fails_closed_outside_linux(monkeypatch): assert sandboxed_web_e2e.isolation_backend("disabled") is None +def test_isolation_backend_fails_closed_without_bwrap(monkeypatch): + """Linux isolation refuses to continue when bubblewrap is not installed.""" + monkeypatch.setattr(sandboxed_web_e2e.platform, "system", lambda: "Linux") + monkeypatch.setattr(sandboxed_web_e2e.shutil, "which", lambda name, path=None: None) + with pytest.raises(RuntimeError, match="needs bubblewrap"): + sandboxed_web_e2e.isolation_backend("required") + + +def test_isolation_backend_returns_bwrap_path_on_linux(monkeypatch): + """Linux isolation returns the resolved bubblewrap executable.""" + monkeypatch.setattr(sandboxed_web_e2e.platform, "system", lambda: "Linux") + monkeypatch.setattr(sandboxed_web_e2e.shutil, "which", lambda name, path=None: "/usr/bin/bwrap") + assert sandboxed_web_e2e.isolation_backend("required") == "/usr/bin/bwrap" + + def test_isolated_command_mounts_only_workspace(monkeypatch, tmp_path): """Bubblewrap commands expose the copied workspace and not the host root.""" monkeypatch.setattr(sandboxed_web_e2e.platform, "system", lambda: "Linux") @@ -573,6 +723,96 @@ def test_isolated_command_rejects_host_home_executable(monkeypatch, tmp_path): ) +def test_isolated_command_rejects_executable_outside_bound_roots(monkeypatch, tmp_path): + """Resolved tools outside read-only mounts fail before entering bubblewrap.""" + monkeypatch.setattr(sandboxed_web_e2e.shutil, "which", lambda *_args, **_kwargs: "/snap/bin/tool") + sandbox = tmp_path / "sandbox" + repo = sandbox / "repo" + repo.mkdir(parents=True) + with pytest.raises(RuntimeError, match="outside the isolated bind roots"): + sandboxed_web_e2e.isolated_command( + "tool", + backend="/usr/bin/bwrap", + cwd=repo, + sandbox_root=sandbox, + env={"PATH": "/usr/bin"}, + ) + + +def test_isolated_command_allows_unresolved_executable_for_bwrap(monkeypatch, tmp_path): + """Commands with shell-resolved executables still receive the isolated wrapper.""" + monkeypatch.setattr(sandboxed_web_e2e.shutil, "which", lambda *_args, **_kwargs: None) + sandbox = tmp_path / "sandbox" + repo = sandbox / "repo" + repo.mkdir(parents=True) + command = sandboxed_web_e2e.isolated_command( + "tool", + backend="/usr/bin/bwrap", + cwd=repo, + sandbox_root=sandbox, + env={"PATH": "/usr/bin"}, + ) + assert command.startswith("/usr/bin/bwrap") + + +def test_isolated_command_skips_unavailable_optional_mount(monkeypatch, tmp_path): + """Optional runtime mounts are omitted when a host path is unavailable.""" + original_exists = Path.exists + + def fake_exists(path): + if str(path) == "/etc/ssl": + return False + return original_exists(path) + + monkeypatch.setattr(Path, "exists", fake_exists) + monkeypatch.setattr( + sandboxed_web_e2e.shutil, + "which", + lambda name, path=None: "/usr/bin/python3", + ) + sandbox = tmp_path / "sandbox" + repo = sandbox / "repo" + repo.mkdir(parents=True) + command = sandboxed_web_e2e.isolated_command( + "python3", + backend="/usr/bin/bwrap", + cwd=repo, + sandbox_root=sandbox, + env={"PATH": "/usr/bin"}, + ) + assert "--ro-bind /etc/ssl /etc/ssl" not in command + + +def test_sandbox_environment_maps_host_paths_to_workspace(tmp_path): + """Only configured sandbox paths are rewritten for the mounted workspace.""" + sandbox = tmp_path / "sandbox" + env = { + "HOME": str(sandbox / "home"), + "TMPDIR": str(sandbox / "tmp"), + "PATH": "/usr/bin", + } + + mapped = sandboxed_web_e2e._sandbox_environment(env, sandbox) + + assert mapped is not env + assert mapped["HOME"] == "/workspace/home" + assert mapped["TMPDIR"] == "/workspace/tmp" + assert mapped["PATH"] == "/usr/bin" + assert "XDG_CACHE_HOME" not in mapped + + +def test_isolated_command_rejects_empty_command(tmp_path): + """Empty commands fail before bubblewrap arguments are constructed.""" + with pytest.raises(ValueError, match="command must not be empty"): + sandboxed_web_e2e.isolated_command( + " ", + backend="/usr/bin/bwrap", + cwd=tmp_path, + sandbox_root=tmp_path, + env={"PATH": "/usr/bin"}, + ) + + def test_parse_args_rejects_invalid_inputs(): """The CLI rejects unusable timeout and environment values.""" with pytest.raises(SystemExit): From 8b9d884f6db7788847253eb9496046b6ead532d9 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 26 Aug 2026 10:00:12 +0900 Subject: [PATCH 6/9] docs: make sandbox changelog actionable --- CHANGELOG.md | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 278175d3a..742362707 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,12 +5,12 @@ this file. The format follows Keep a Changelog, and versioned releases follow Semantic Versioning where the repository publishes a release. ## [Unreleased] -- Require Linux bubblewrap isolation for backend, frontend, and E2E commands - in the web verification helper, mount only a writable workspace, and reject - non-loopback readiness URLs or redirects so SSRF probes fail closed. -- Fail closed when an isolated command resolves outside the mounted system - roots, return a coded readiness failure for invalid URLs, and cover required - isolation plus sandbox-environment path mapping in the 100% branch contract. +- Web verification now runs backend, frontend, and E2E commands in an isolated + workspace and accepts only local readiness URLs. Run it on a supported Linux + runner; trusted local debugging may opt out with `--isolation disabled`. +- Invalid readiness URLs and unavailable isolation now fail with clear + diagnostics before services start, so update the URL or runner instead of + retrying the same setup. - Route Strix cross-provider fallbacks to explicit direct-OpenAI models (`openai-direct/...`) through the OpenAI inference endpoint instead of inheriting a provider-specific primary base: the workflow now provisions From c50e26be529f473e6cdbce6dd9a7540cb750e7a0 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 26 Aug 2026 10:10:41 +0900 Subject: [PATCH 7/9] fix(e2e): fail closed on rejected sandbox commands --- .../sandboxed-web-command-isolation.md | 12 +- scripts/ci/sandboxed_web_e2e.py | 75 +++++++----- tests/test_sandboxed_web_e2e.py | 112 ++++++++++++++++-- 3 files changed, 151 insertions(+), 48 deletions(-) diff --git a/docs/doctoring/sandboxed-web-command-isolation.md b/docs/doctoring/sandboxed-web-command-isolation.md index 1e862cfc3..ee2b6a334 100644 --- a/docs/doctoring/sandboxed-web-command-isolation.md +++ b/docs/doctoring/sandboxed-web-command-isolation.md @@ -1,15 +1,17 @@ # Sandboxed web command isolation `sandboxed_web_e2e.py` requires Linux `bubblewrap` (`bwrap`) by default. Each -backend, frontend, and E2E command runs with a read-only runtime root and a -single writable mount at `/workspace`; the copied repository and temporary -homes are mapped there. The host filesystem is therefore not reachable through -absolute paths or `..` traversal. +backend, frontend, and E2E command runs with a fresh writable `tmpfs` root and +`/tmp`, plus one writable copied-repository bind at `/workspace`; the copied +repository and temporary homes are mapped there. Host runtime roots and the +minimal `/etc` identity, DNS, and time files are mounted read-only, so the host +filesystem is not reachable through absolute paths or `..` traversal. Before wrapping a command, the helper resolves its executable and rejects paths outside the read-only system roots mounted by bubblewrap. A tool installed in a host-only location must be installed into one of those roots or the run exits -before any service starts. +with code `126` before any service starts; the result marker records that code +and the selected backend. Use `--isolation disabled` only for trusted local debugging. The result marker records the requested mode and resolved backend so CI evidence cannot be diff --git a/scripts/ci/sandboxed_web_e2e.py b/scripts/ci/sandboxed_web_e2e.py index a99ebee45..beddb52c3 100644 --- a/scripts/ci/sandboxed_web_e2e.py +++ b/scripts/ci/sandboxed_web_e2e.py @@ -163,7 +163,15 @@ def isolated_command( args = [backend, "--die-with-parent", "--new-session", "--unshare-pid", "--tmpfs", "/"] for root in bind_roots: args.extend(("--ro-bind", str(root), str(root))) - for path in ("/etc/ssl", "/etc/hosts", "/etc/resolv.conf", "/etc/localtime"): + for path in ( + "/etc/ssl", + "/etc/hosts", + "/etc/resolv.conf", + "/etc/localtime", + "/etc/passwd", + "/etc/group", + "/etc/nsswitch.conf", + ): if Path(path).exists(): args.extend(("--ro-bind", path, path)) args.extend( @@ -329,39 +337,44 @@ def main(argv: Sequence[str] | None = None) -> int: if args.network != "default": print(f"sandboxed-web-e2e: network={args.network}") command_env = _sandbox_environment(env, sandbox) if backend else env - backend_cmd = ( - isolated_command( - args.backend_cmd, - backend=backend, - cwd=copied_repo, - sandbox_root=sandbox, - env=env, + try: + backend_cmd = ( + isolated_command( + args.backend_cmd, + backend=backend, + cwd=copied_repo, + sandbox_root=sandbox, + env=env, + ) + if backend + else args.backend_cmd ) - if backend - else args.backend_cmd - ) - frontend_cmd = ( - isolated_command( - args.frontend_cmd, - backend=backend, - cwd=copied_repo, - sandbox_root=sandbox, - env=env, + frontend_cmd = ( + isolated_command( + args.frontend_cmd, + backend=backend, + cwd=copied_repo, + sandbox_root=sandbox, + env=env, + ) + if backend + else args.frontend_cmd ) - if backend - else args.frontend_cmd - ) - e2e_cmd = ( - isolated_command( - args.e2e_cmd, - backend=backend, - cwd=copied_repo, - sandbox_root=sandbox, - env=env, + e2e_cmd = ( + isolated_command( + args.e2e_cmd, + backend=backend, + cwd=copied_repo, + sandbox_root=sandbox, + env=env, + ) + if backend + else args.e2e_cmd ) - if backend - else args.e2e_cmd - ) + except RuntimeError as exc: + print(f"sandboxed-web-e2e: isolation rejected command: {exc}", file=sys.stderr) + exit_code = 126 + return exit_code services.append(start_service("backend", backend_cmd, copied_repo, command_env, logs_dir)) services.append(start_service("frontend", frontend_cmd, copied_repo, command_env, logs_dir)) try: diff --git a/tests/test_sandboxed_web_e2e.py b/tests/test_sandboxed_web_e2e.py index 3fb2fbfb8..a14926321 100644 --- a/tests/test_sandboxed_web_e2e.py +++ b/tests/test_sandboxed_web_e2e.py @@ -1,5 +1,6 @@ import json import os +import re import runpy import socket import subprocess @@ -116,15 +117,15 @@ def test_wait_helpers_and_service_cleanup_edges(monkeypatch, tmp_path): assert sandboxed_web_e2e.wait_for_url("http://localhost:1/", 1, exited_service) is False assert sandboxed_web_e2e.wait_for_url("http://127.0.0.1:1/", 1, exited_service) is False assert sandboxed_web_e2e.wait_for_url("http://127.0.0.2:1/", 1, exited_service) is False - with pytest.raises(ValueError, match="URL must start with http:// or https://"): + with pytest.raises(ValueError, match=re.escape("URL must start with http:// or https://")): sandboxed_web_e2e.wait_for_url("file:///etc/passwd", 1, exited_service) - with pytest.raises(ValueError, match="URL cannot target external hostname: external.example.com"): + with pytest.raises(ValueError, match=re.escape("URL cannot target external hostname: external.example.com")): sandboxed_web_e2e.wait_for_url("http://external.example.com/ready", 1, exited_service) - with pytest.raises(ValueError, match="URL cannot target external hostname: app.localhost"): + with pytest.raises(ValueError, match=re.escape("URL cannot target external hostname: app.localhost")): sandboxed_web_e2e.wait_for_url("http://app.localhost:8000/health", 1, exited_service) - with pytest.raises(ValueError, match="URL cannot target external hostname: 169.254.169.254"): + with pytest.raises(ValueError, match=re.escape("URL cannot target external hostname: 169.254.169.254")): sandboxed_web_e2e.wait_for_url("http://169.254.169.254/latest/meta-data/", 1, exited_service) - with pytest.raises(ValueError, match="URL cannot target external hostname: 0.0.0.0"): + with pytest.raises(ValueError, match=re.escape("URL cannot target external hostname: 0.0.0.0")): sandboxed_web_e2e.wait_for_url("http://0.0.0.0:8000/health", 1, exited_service) sandboxed_web_e2e.stop_service(exited_service) assert sandboxed_web_e2e.tail_text(tmp_path / "missing.log") == "" @@ -211,7 +212,8 @@ def poll(self): return None class Response: - status = 204 + def __init__(self, status): + self.status = status def __enter__(self): return self @@ -226,7 +228,7 @@ def open(self, url, timeout): attempts.append((url, timeout)) if len(attempts) == 1: raise sandboxed_web_e2e.urllib.error.URLError("not ready") - return Response() + return Response(500 if len(attempts) == 2 else 204) monkeypatch.setattr(sandboxed_web_e2e.urllib.request, "build_opener", lambda *args: FakeOpener()) monkeypatch.setattr(sandboxed_web_e2e.time, "sleep", lambda seconds: None) @@ -236,7 +238,7 @@ def open(self, url, timeout): service = sandboxed_web_e2e.Service("web", "serve", RunningProcess(), log_path) assert sandboxed_web_e2e.wait_for_url("http://127.0.0.1:8000/health", 10, service) is True - assert len(attempts) == 2 + assert len(attempts) == 3 assert sandboxed_web_e2e.tail_text(log_path).splitlines()[0] == "line-10" @@ -401,7 +403,47 @@ def fake_start(label, command, cwd, env, logs_dir): assert [item[0] for item in started] == ["backend", "frontend"] assert all(item[1].startswith("wrapped ") for item in started) assert started[0][3]["HOME"].startswith("/workspace/") - assert "isolation_backend=unknown" not in captured.out + result_line = [line for line in captured.out.splitlines() if line.startswith(sandboxed_web_e2e.RESULT_MARKER)][-1] + payload = json.loads(result_line.removeprefix(sandboxed_web_e2e.RESULT_MARKER).strip()) + assert payload["isolation"] == "required" + assert payload["isolation_backend"] == "/usr/bin/bwrap" + + +def test_main_reports_rejected_isolated_command(monkeypatch, tmp_path, capsys): + """Rejected commands fail before services start and emit coded evidence.""" + repo = tmp_path / "repo" + repo.mkdir() + started = [] + + monkeypatch.setattr(sandboxed_web_e2e, "isolation_backend", lambda mode: "/usr/bin/bwrap") + monkeypatch.setattr( + sandboxed_web_e2e, + "isolated_command", + lambda command, **kwargs: (_ for _ in ()).throw(RuntimeError("host-only tool")), + ) + monkeypatch.setattr(sandboxed_web_e2e, "start_service", lambda *args: started.append(args)) + + exit_code = sandboxed_web_e2e.main( + [ + "--repo-root", + str(repo), + "--backend-cmd", + "backend", + "--frontend-cmd", + "frontend", + "--e2e-cmd", + "e2e", + ] + ) + captured = capsys.readouterr() + + assert exit_code == 126 + assert not started + assert "isolation rejected command: host-only tool" in captured.err + result_line = [line for line in captured.out.splitlines() if line.startswith(sandboxed_web_e2e.RESULT_MARKER)][-1] + payload = json.loads(result_line.removeprefix(sandboxed_web_e2e.RESULT_MARKER).strip()) + assert payload["exit_code"] == 126 + assert payload["isolation_backend"] == "/usr/bin/bwrap" def test_main_reports_unavailable_required_isolation(monkeypatch, tmp_path, capsys): @@ -580,6 +622,48 @@ def fake_run_shell(command, cwd, env, timeout): assert "e2e-err" in captured.err assert "e2e command timed out after 3s" in captured.err + def fake_run_shell_with_newlines(command, cwd, env, timeout): + raise subprocess.TimeoutExpired(command, timeout, output=b"e2e-out\n", stderr=b"e2e-err\n") + + monkeypatch.setattr(sandboxed_web_e2e, "run_shell", fake_run_shell_with_newlines) + assert sandboxed_web_e2e.main( + [ + "--repo-root", + str(repo), + "--isolation", + "disabled", + "--backend-cmd", + "backend", + "--frontend-cmd", + "frontend", + "--e2e-timeout", + "3", + "--e2e-cmd", + "e2e", + ] + ) == 124 + + def fake_run_shell_without_output(command, cwd, env, timeout): + raise subprocess.TimeoutExpired(command, timeout) + + monkeypatch.setattr(sandboxed_web_e2e, "run_shell", fake_run_shell_without_output) + assert sandboxed_web_e2e.main( + [ + "--repo-root", + str(repo), + "--isolation", + "disabled", + "--backend-cmd", + "backend", + "--frontend-cmd", + "frontend", + "--e2e-timeout", + "3", + "--e2e-cmd", + "e2e", + ] + ) == 124 + @POSIX_PROCESS_GROUPS def test_sandboxed_web_e2e_reports_readiness_failure(tmp_path, capsys): @@ -701,6 +785,10 @@ def test_isolated_command_mounts_only_workspace(monkeypatch, tmp_path): assert "--bind" in command assert "--chdir /workspace/repo" in command assert "--ro-bind / /" not in command + assert "--ro-bind /etc/passwd /etc/passwd" in command + assert "--ro-bind /etc/group /etc/group" in command + if Path("/etc/nsswitch.conf").exists(): + assert "--ro-bind /etc/nsswitch.conf /etc/nsswitch.conf" in command def test_isolated_command_rejects_host_home_executable(monkeypatch, tmp_path): @@ -713,7 +801,7 @@ def test_isolated_command_rejects_host_home_executable(monkeypatch, tmp_path): sandbox = tmp_path / "sandbox" repo = sandbox / "repo" repo.mkdir(parents=True) - with pytest.raises(RuntimeError, match="host home directory"): + with pytest.raises(RuntimeError, match=re.escape("host home directory")): sandboxed_web_e2e.isolated_command( "tool", backend="/usr/bin/bwrap", @@ -729,7 +817,7 @@ def test_isolated_command_rejects_executable_outside_bound_roots(monkeypatch, tm sandbox = tmp_path / "sandbox" repo = sandbox / "repo" repo.mkdir(parents=True) - with pytest.raises(RuntimeError, match="outside the isolated bind roots"): + with pytest.raises(RuntimeError, match=re.escape("outside the isolated bind roots")): sandboxed_web_e2e.isolated_command( "tool", backend="/usr/bin/bwrap", @@ -803,7 +891,7 @@ def test_sandbox_environment_maps_host_paths_to_workspace(tmp_path): def test_isolated_command_rejects_empty_command(tmp_path): """Empty commands fail before bubblewrap arguments are constructed.""" - with pytest.raises(ValueError, match="command must not be empty"): + with pytest.raises(ValueError, match=re.escape("command must not be empty")): sandboxed_web_e2e.isolated_command( " ", backend="/usr/bin/bwrap", From 391233f13c9f57d365f19868c4e7ae2b8e9ac79d Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 26 Aug 2026 10:34:20 +0900 Subject: [PATCH 8/9] fix(e2e): validate readiness before launch --- scripts/ci/sandboxed_web_e2e.py | 20 +++++++++++-- tests/test_sandboxed_web_e2e.py | 53 ++++++++++++++++++++++++++++++++- 2 files changed, 69 insertions(+), 4 deletions(-) diff --git a/scripts/ci/sandboxed_web_e2e.py b/scripts/ci/sandboxed_web_e2e.py index beddb52c3..9bdf0820a 100644 --- a/scripts/ci/sandboxed_web_e2e.py +++ b/scripts/ci/sandboxed_web_e2e.py @@ -210,10 +210,10 @@ def start_service(label: str, command: str, cwd: Path, env: dict[str, str], logs return Service(label=label, command=command, process=process, log_path=log_path) -def wait_for_url(url: str, timeout: int, service: Service) -> bool: - """Poll a readiness URL until it responds or the service exits.""" +def validate_readiness_url(url: str) -> None: + """Reject a readiness URL that is not an HTTP(S) loopback target.""" if not url: - return True + return if not (url.startswith("http://") or url.startswith("https://")): raise ValueError(f"URL must start with http:// or https://, got: {url}") @@ -226,6 +226,13 @@ def wait_for_url(url: str, timeout: int, service: Service) -> bool: if not is_loopback: raise ValueError(f"URL cannot target external hostname: {hostname}") + +def wait_for_url(url: str, timeout: int, service: Service) -> bool: + """Poll a validated readiness URL until it responds or the service exits.""" + validate_readiness_url(url) + if not url: + return True + deadline = time.monotonic() + timeout opener = urllib.request.build_opener(NoRedirectHandler()) while time.monotonic() < deadline: @@ -375,6 +382,13 @@ def main(argv: Sequence[str] | None = None) -> int: print(f"sandboxed-web-e2e: isolation rejected command: {exc}", file=sys.stderr) exit_code = 126 return exit_code + try: + validate_readiness_url(args.backend_ready_url) + validate_readiness_url(args.frontend_ready_url) + except ValueError as exc: + print(f"sandboxed-web-e2e: invalid readiness URL: {exc}", file=sys.stderr) + exit_code = 125 + return exit_code services.append(start_service("backend", backend_cmd, copied_repo, command_env, logs_dir)) services.append(start_service("frontend", frontend_cmd, copied_repo, command_env, logs_dir)) try: diff --git a/tests/test_sandboxed_web_e2e.py b/tests/test_sandboxed_web_e2e.py index a14926321..a229fdd41 100644 --- a/tests/test_sandboxed_web_e2e.py +++ b/tests/test_sandboxed_web_e2e.py @@ -532,12 +532,14 @@ def test_main_reports_invalid_readiness_url(monkeypatch, tmp_path, capsys): """Invalid readiness input exits with the same clean readiness failure code.""" repo = tmp_path / "repo" repo.mkdir() + started = [] class DoneProcess: def poll(self): return 0 def fake_start(label, command, cwd, env, logs_dir): + started.append(label) log_path = logs_dir / f"{label}.log" log_path.write_text("ready\n", encoding="utf-8") return sandboxed_web_e2e.Service(label, command, DoneProcess(), log_path) @@ -571,12 +573,61 @@ def fake_start(label, command, cwd, env, logs_dir): captured = capsys.readouterr() assert exit_code == 125 - assert "invalid readiness URL: bad host" in captured.err + assert not started + assert "invalid readiness URL: URL cannot target external hostname" in captured.err result_line = [line for line in captured.out.splitlines() if line.startswith(sandboxed_web_e2e.RESULT_MARKER)][-1] payload = json.loads(result_line.removeprefix(sandboxed_web_e2e.RESULT_MARKER).strip()) assert payload["exit_code"] == 125 +def test_main_reports_readiness_exception_after_start(monkeypatch, tmp_path, capsys): + """Unexpected readiness errors after launch still clean up services.""" + repo = tmp_path / "repo" + repo.mkdir() + started = [] + + class DoneProcess: + def poll(self): + return 0 + + def fake_start(label, command, cwd, env, logs_dir): + started.append(label) + log_path = logs_dir / f"{label}.log" + return sandboxed_web_e2e.Service(label, command, DoneProcess(), log_path) + + monkeypatch.setattr(sandboxed_web_e2e, "start_service", fake_start) + monkeypatch.setattr( + sandboxed_web_e2e, + "wait_for_url", + lambda url, timeout, service: (_ for _ in ()).throw(ValueError("bad host")), + ) + monkeypatch.setattr(sandboxed_web_e2e, "stop_service", lambda service: None) + + exit_code = sandboxed_web_e2e.main( + [ + "--repo-root", + str(repo), + "--isolation", + "disabled", + "--backend-cmd", + "backend", + "--frontend-cmd", + "frontend", + "--backend-ready-url", + "http://127.0.0.1:8000/health", + "--frontend-ready-url", + "http://127.0.0.1:3000/", + "--e2e-cmd", + "e2e", + ] + ) + captured = capsys.readouterr() + + assert exit_code == 125 + assert started == ["backend", "frontend"] + assert "invalid readiness URL: bad host" in captured.err + + def test_main_reports_stubbed_e2e_timeout(monkeypatch, tmp_path, capsys): """Main preserves timeout output from stubbed E2E execution.""" repo = tmp_path / "repo" From 524093e130176b6bd86e2e6e4730bb182c5897ba Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 26 Aug 2026 10:42:39 +0900 Subject: [PATCH 9/9] fix(e2e): back off after server readiness errors --- scripts/ci/sandboxed_web_e2e.py | 1 + tests/test_sandboxed_web_e2e.py | 4 +++- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/scripts/ci/sandboxed_web_e2e.py b/scripts/ci/sandboxed_web_e2e.py index 9bdf0820a..5dbdbf2b8 100644 --- a/scripts/ci/sandboxed_web_e2e.py +++ b/scripts/ci/sandboxed_web_e2e.py @@ -242,6 +242,7 @@ def wait_for_url(url: str, timeout: int, service: Service) -> bool: with opener.open(url, timeout=2) as response: # nosec B310 if 200 <= response.status < 500: return True + time.sleep(1) except (urllib.error.URLError, TimeoutError): time.sleep(1) return False diff --git a/tests/test_sandboxed_web_e2e.py b/tests/test_sandboxed_web_e2e.py index a229fdd41..eff8e3133 100644 --- a/tests/test_sandboxed_web_e2e.py +++ b/tests/test_sandboxed_web_e2e.py @@ -231,7 +231,8 @@ def open(self, url, timeout): return Response(500 if len(attempts) == 2 else 204) monkeypatch.setattr(sandboxed_web_e2e.urllib.request, "build_opener", lambda *args: FakeOpener()) - monkeypatch.setattr(sandboxed_web_e2e.time, "sleep", lambda seconds: None) + sleeps = [] + monkeypatch.setattr(sandboxed_web_e2e.time, "sleep", lambda seconds: sleeps.append(seconds)) log_path = tmp_path / "service.log" log_path.write_text("\n".join(f"line-{index}" for index in range(90)), encoding="utf-8") @@ -239,6 +240,7 @@ def open(self, url, timeout): assert sandboxed_web_e2e.wait_for_url("http://127.0.0.1:8000/health", 10, service) is True assert len(attempts) == 3 + assert sleeps == [1, 1] assert sandboxed_web_e2e.tail_text(log_path).splitlines()[0] == "line-10"