diff --git a/CHANGELOG.md b/CHANGELOG.md index 6c7978fa5..fb2fca850 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -47,10 +47,20 @@ Semantic Versioning where the repository publishes a release. ### Changed +- Bound backend and frontend combined service logs plus E2E command output, + stop a service before running E2E when readiness evidence overflows, preserve + timeout and prior-failure precedence, and publish separate resource and + capture-finalization fields with bounded log tails. + - Route sandboxed verification commands through the bounded subprocess layer, reject copied-tree symlinks that leave the sandbox, and publish distinct output-limit, unsupported-platform, missing/non-executable command, and path-boundary evidence without exposing host paths or uncaught tracebacks. +- Classify missing or non-executable backend, frontend, and E2E commands with + stable exit codes and operator recovery actions while still cleaning up + services that started before the failure. +- Reject non-HTTP readiness URLs during argument parsing with the exact option + to correct, instead of starting services and exposing a runtime traceback. - Emit completed repository pull-list requests as they finish in the five-minute agent-mention sweep, while retaining the four-worker ceiling, rotation, and diff --git a/docs/doctoring/sandboxed-output-resource-bounds.md b/docs/doctoring/sandboxed-output-resource-bounds.md index d9b7d1b69..3e2e91969 100644 --- a/docs/doctoring/sandboxed-output-resource-bounds.md +++ b/docs/doctoring/sandboxed-output-resource-bounds.md @@ -54,7 +54,7 @@ POSIX file-size resource limits apply to every regular file written by the child A truncation marker is included inside, not in addition to, the declared retained byte budget. Reader errors and reader-join timeouts are explicit failures. -## Deferred consumer integration +## Long-running service boundary The second stack layer adopts the library in `sandboxed_verify.py` for short-lived verification commands. Long-running `sandboxed_web_e2e.py` service @@ -62,6 +62,26 @@ evidence remains a separate layer. Keeping those integrations separate prevents a shared process primitive, workspace symlink policy, and E2E result schema from becoming one monolithic review. +The third stack layer adopts the same bounded drainer for each backend and +frontend combined stdout/stderr pipe. A capture retains the final suffix in +memory and writes only its bounded rendered form to the private sandbox log when +the stream closes, so the evidence file cannot exceed its declared budget. + +Service overflow is checked during readiness, after E2E execution, and after +service shutdown. It takes precedence over an otherwise successful command or +readiness result, while a true E2E timeout remains `124`. A verbose but healthy +service is intentionally stopped once it exceeds the default 4 MiB combined-log +contract; projects that need more evidence must select an explicit supported +budget. `tail_text()` reads no more than 65,536 bytes from the end of the already +bounded file and keeps the truncation marker visible. + +The result separates `output_limited`, `output_limit_unsupported`, and +`service_capture_failed`. A nonzero E2E or readiness code remains authoritative +even when a late service overflow also sets `output_limited=true`; the Boolean +retains the secondary resource evidence without erasing the original failure. +If service finalization fails, the wrapper performs another best-effort group +kill, bounded reap, and capture join before publishing failure evidence. + The verification consumer maps an executable lookup failure to exit code `127`, publishes its normal machine-readable failed result, and tells the operator to install the executable or correct `PATH`. Provider and host path details do not @@ -72,6 +92,15 @@ the consumer returns exit code `126` and tells the operator to select an executable file or correct its permissions. The stable failed result remains available without exposing the operating-system exception traceback. +The web E2E consumer applies the same `126`/`127` machine-readable failure +boundary to backend, frontend, and E2E command launch. Services that started +before a later launch failure still pass through the ordinary bounded cleanup +path. + +Backend and frontend readiness URLs are rejected during argument parsing unless +they are empty or use `http://` or `https://`. The parser names the option the +operator must correct before any workspace copy or service launch occurs. + ## Security and availability properties - Parent retained memory is bounded independently for stdout and stderr. @@ -103,13 +132,17 @@ Real subprocess tests exercise: - a real same-group descendant that inherits the pipes, outlives the direct child, and is prevented from writing a delayed sentinel; - final-suffix retention and one overflow callback; +- bounded persisted service evidence; +- service overflow before or during readiness/E2E, including a sentinel proof + that E2E never ran; +- ordinary backend/frontend/E2E success and cleanup; - partial UTF-8 suffix decoding; - UTF-8 replacement expansion within the declared byte budget; - bounded file reads; - unsupported-platform failure; - invalid budgets; - reader exceptions, stuck-reader joins, a common finite join bound, and sibling finalization after the first failure; -- deterministic return and timeout evidence. +- deterministic result fields and exit-code precedence. The exact pull-request head must additionally pass the complete central test suite, 100% production statement and branch coverage for the changed surface, production docstrings, Secret Scan, CodeQL, Semgrep, Python Security, dependency and supply-chain checks, OpenCode, Noema, CodeRabbit, independent current-head approval, and branch protection. diff --git a/scripts/ci/sandboxed_web_e2e.py b/scripts/ci/sandboxed_web_e2e.py index ae0c3105a..23b6ae417 100644 --- a/scripts/ci/sandboxed_web_e2e.py +++ b/scripts/ci/sandboxed_web_e2e.py @@ -3,6 +3,7 @@ from __future__ import annotations import argparse +import contextlib import json import os import signal @@ -17,14 +18,16 @@ from collections.abc import Sequence from dataclasses import dataclass from pathlib import Path +from typing import BinaryIO if __package__ in (None, ""): sys.path.insert(0, str(Path(__file__).resolve().parents[2])) -from scripts.ci import sandboxed_verify +from scripts.ci import bounded_subprocess, sandboxed_verify RESULT_MARKER = "SANDBOXED_WEB_E2E_RESULT" +DEFAULT_TAIL_BYTES = 65_536 class NoRedirectHandler(urllib.request.HTTPRedirectHandler): @@ -35,14 +38,24 @@ def redirect_request(self, req, fp, code, msg, headers, newurl): raise urllib.error.HTTPError(req.full_url, code, msg, headers, fp) +class CommandExecutableNotFoundError(RuntimeError): + """Report that one declared service or E2E executable is unavailable.""" + + +class CommandNotExecutableError(RuntimeError): + """Report that one declared service or E2E path cannot be executed.""" + + @dataclass class Service: - """A long-running web service process and its log file.""" + """A long-running web service process and its bounded combined log capture.""" label: str command: str - process: subprocess.Popen[str] + process: subprocess.Popen[bytes] log_path: Path + capture: bounded_subprocess.BoundedOutputCapture | None = None + log_limit_bytes: int = bounded_subprocess.DEFAULT_SERVICE_LOG_LIMIT_BYTES def parse_args(argv: Sequence[str] | None = None) -> argparse.Namespace: @@ -62,6 +75,18 @@ def parse_args(argv: Sequence[str] | None = None) -> argparse.Namespace: parser.add_argument("--frontend-ready-url", default="", help="Frontend readiness URL to poll before E2E.") parser.add_argument("--startup-timeout", type=int, default=120, help="Seconds to wait for readiness URLs.") parser.add_argument("--e2e-timeout", type=int, default=600, help="Seconds to allow the E2E command to run.") + parser.add_argument( + "--output-limit-bytes", + type=int, + default=bounded_subprocess.DEFAULT_COMMAND_OUTPUT_LIMIT_BYTES, + help="Maximum retained stdout and stderr bytes for the E2E command.", + ) + parser.add_argument( + "--service-log-limit-bytes", + type=int, + default=bounded_subprocess.DEFAULT_SERVICE_LOG_LIMIT_BYTES, + help="Maximum retained combined log bytes for each long-running service.", + ) parser.add_argument("--keep-sandbox", action="store_true", help="Keep the temporary sandbox after execution.") parser.add_argument( "--allow-env", @@ -92,31 +117,108 @@ def parse_args(argv: Sequence[str] | None = None) -> argparse.Namespace: parser.error("--startup-timeout must be positive") if args.e2e_timeout <= 0: parser.error("--e2e-timeout must be positive") + for option, url in ( + ("--backend-ready-url", args.backend_ready_url), + ("--frontend-ready-url", args.frontend_ready_url), + ): + if url and not (url.startswith("http://") or url.startswith("https://")): + parser.error(f"{option} must start with http:// or https://") + try: + args.output_limit_bytes = bounded_subprocess.validate_output_limit( + args.output_limit_bytes, + "--output-limit-bytes", + ) + args.service_log_limit_bytes = bounded_subprocess.validate_output_limit( + args.service_log_limit_bytes, + "--service-log-limit-bytes", + ) + except ValueError as error: + parser.error(str(error)) for name in args.allow_env: if not sandboxed_verify.ENV_NAME_RE.match(name): parser.error(f"--allow-env must be an environment variable name: {name}") return args -def start_service(label: str, command: str, cwd: Path, env: dict[str, str], logs_dir: Path) -> Service: - """Start a service command in its own process group.""" +def _cleanup_failed_service_start( + process: subprocess.Popen[bytes], + stream: BinaryIO, +) -> None: + """Best-effort stop, reap, and close after bounded capture startup fails.""" + with contextlib.suppress(OSError, subprocess.SubprocessError): + bounded_subprocess.kill_process_group(process) + with contextlib.suppress(OSError, subprocess.SubprocessError): + process.wait(timeout=10) + with contextlib.suppress(OSError): + stream.close() + + +def start_service( + label: str, + command: str, + cwd: Path, + env: dict[str, str], + logs_dir: Path, + log_limit_bytes: int = bounded_subprocess.DEFAULT_SERVICE_LOG_LIMIT_BYTES, +) -> Service: + """Start one service group and continuously drain its combined bounded log.""" + bounded_subprocess.require_supported_platform() + log_limit = bounded_subprocess.validate_output_limit( + log_limit_bytes, + "service log limit", + ) log_path = logs_dir / f"{label}.log" - log_file = log_path.open("w", encoding="utf-8") - process = subprocess.Popen( - shlex.split(command), - cwd=cwd, - env=env, - text=True, - stdout=log_file, - stderr=subprocess.STDOUT, - start_new_session=True, + try: + process = subprocess.Popen( + shlex.split(command), + cwd=cwd, + env=env, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + bufsize=0, + start_new_session=True, + shell=False, + ) + except FileNotFoundError as error: + raise CommandExecutableNotFoundError from error + except (PermissionError, IsADirectoryError) as error: + raise CommandNotExecutableError from error + if process.stdout is None: + bounded_subprocess.kill_process_group(process) + process.wait() + raise RuntimeError("service output pipe was not created") + try: + capture = bounded_subprocess.start_bounded_capture( + process.stdout, + evidence_limit_bytes=log_limit, + on_limit=lambda: bounded_subprocess.kill_process_group(process), + destination=log_path, + ) + except BaseException: + _cleanup_failed_service_start(process, process.stdout) + raise + return Service( + label=label, + command=command, + process=process, + log_path=log_path, + capture=capture, + log_limit_bytes=log_limit, + ) + + +def service_output_limited(service: Service) -> bool: + """Return whether one service exceeded its declared combined log budget.""" + if service.capture is not None: + return service.capture.output_limited + return ( + service.log_path.exists() + and service.log_path.stat().st_size > service.log_limit_bytes ) - log_file.close() - return Service(label=label, command=command, process=process, log_path=log_path) def wait_for_url(url: str, timeout: int, service: Service) -> bool: - """Poll a readiness URL until it responds or the service exits.""" + """Poll a readiness URL until it responds, exits, or exceeds its log budget.""" if not url: return True if not (url.startswith("http://") or url.startswith("https://")): @@ -124,7 +226,7 @@ def wait_for_url(url: str, timeout: int, service: Service) -> bool: deadline = time.monotonic() + timeout opener = urllib.request.build_opener(NoRedirectHandler()) while time.monotonic() < deadline: - if service.process.poll() is not None: + if service_output_limited(service) or service.process.poll() is not None: return False try: with opener.open(url, timeout=2) as response: # nosec B310 @@ -135,41 +237,59 @@ def wait_for_url(url: str, timeout: int, service: Service) -> bool: return False -def run_shell(command: str, cwd: Path, env: dict[str, str], timeout: int) -> subprocess.CompletedProcess[str]: - """Run a shell command and capture its output.""" - return subprocess.run( - shlex.split(command), - cwd=cwd, - env=env, - text=True, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - timeout=timeout, - check=False, - ) +def run_shell( + command: str, + cwd: Path, + env: dict[str, str], + timeout: int, + output_limit_bytes: int = bounded_subprocess.DEFAULT_COMMAND_OUTPUT_LIMIT_BYTES, +) -> bounded_subprocess.BoundedCompletedProcess: + """Run one shell-style command without a shell and with bounded pipe drains.""" + try: + return bounded_subprocess.run_bounded_command( + shlex.split(command), + cwd=cwd, + env=env, + timeout=timeout, + evidence_limit_bytes=output_limit_bytes, + ) + except FileNotFoundError as error: + raise CommandExecutableNotFoundError from error + except (PermissionError, IsADirectoryError) as error: + raise CommandNotExecutableError from error def stop_service(service: Service) -> None: - """Terminate a service process group and wait briefly for cleanup.""" - if service.process.poll() is not None: - return - try: - os.killpg(service.process.pid, signal.SIGTERM) - service.process.wait(timeout=10) - except (ProcessLookupError, subprocess.TimeoutExpired): + """Terminate a service process group and finalize its bounded log evidence.""" + if service.process.poll() is None: try: - os.killpg(service.process.pid, signal.SIGKILL) + os.killpg(service.process.pid, signal.SIGTERM) + service.process.wait(timeout=10) except ProcessLookupError: - return - service.process.wait(timeout=10) + pass + except subprocess.TimeoutExpired: + bounded_subprocess.kill_process_group(service.process) + service.process.wait(timeout=10) + if service.capture is not None: + service.capture.join(timeout=10) -def tail_text(path: Path, max_lines: int = 80) -> str: - """Return the final lines of a service log.""" +def tail_text( + path: Path, + max_lines: int = 80, + max_bytes: int = DEFAULT_TAIL_BYTES, +) -> str: + """Return final lines after a byte-bounded service evidence read.""" + if max_lines <= 0: + raise ValueError("max_lines must be positive") if not path.exists(): return "" - lines = path.read_text(encoding="utf-8", errors="replace").splitlines() - return "\n".join(lines[-max_lines:]) + bounded_text = bounded_subprocess.read_bounded_suffix(path, max_bytes) + lines = bounded_text.text.splitlines() + tail = "\n".join(lines[-max_lines:]) + if bounded_text.truncated and bounded_subprocess.TRUNCATION_MARKER.strip() not in tail: + return f"{bounded_subprocess.TRUNCATION_MARKER.strip()}\n{tail}" + return tail def emit_result( @@ -181,6 +301,10 @@ def emit_result( frontend_ready: bool, exit_code: int, elapsed_seconds: float, + output_limited: bool, + output_limit_unsupported: bool, + service_capture_failed: bool, + path_boundary_rejected: bool = False, ) -> None: """Print a machine-readable web E2E execution evidence summary.""" payload = { @@ -195,12 +319,24 @@ def emit_result( "frontend_cmd": args.frontend_cmd, "frontend_ready": frontend_ready, "network": args.network, + "output_limit_bytes": args.output_limit_bytes, + "output_limited": output_limited, + "output_limit_unsupported": output_limit_unsupported, + "path_boundary_rejected": path_boundary_rejected, "sandbox": str(sandbox_root) if args.keep_sandbox else "(removed)", "sandboxed": True, + "service_capture_failed": service_capture_failed, + "service_log_limit_bytes": args.service_log_limit_bytes, } + print() print(f"{RESULT_MARKER} {json.dumps(payload, sort_keys=True)}") +def _services_output_limited(services: Sequence[Service]) -> bool: + """Return whether any started service exceeded its combined log budget.""" + return any(service_output_limited(service) for service in services) + + def main(argv: Sequence[str] | None = None) -> int: """Run backend, frontend, and E2E commands inside a sandbox copy.""" args = parse_args(argv) @@ -212,44 +348,160 @@ def main(argv: Sequence[str] | None = None) -> int: backend_ready = False frontend_ready = False exit_code = 1 + output_limited = False + output_limit_unsupported = False + service_capture_failed = False + service_limit_reported = False + path_boundary_rejected = False start = time.monotonic() try: - copied_repo = sandboxed_verify.copy_workspace(Path(args.repo_root), sandbox, args.ignore) - env = sandboxed_verify.scrubbed_env(sandbox, args.allow_env) - print(f"sandboxed-web-e2e: cwd={copied_repo}") - if args.allow_env: - print(f"sandboxed-web-e2e: allowed env names={','.join(sorted(set(args.allow_env)))}") - if args.network != "default": - print(f"sandboxed-web-e2e: network={args.network}") - services.append(start_service("backend", args.backend_cmd, copied_repo, env, logs_dir)) - services.append(start_service("frontend", args.frontend_cmd, copied_repo, env, logs_dir)) - backend_ready = wait_for_url(args.backend_ready_url, args.startup_timeout, services[0]) - frontend_ready = wait_for_url(args.frontend_ready_url, args.startup_timeout, services[1]) - if not backend_ready or not frontend_ready: - print("sandboxed-web-e2e: service readiness failed", file=sys.stderr) - exit_code = 125 - return exit_code try: - completed = run_shell(args.e2e_cmd, copied_repo, env, args.e2e_timeout) - if completed.stdout: - print(completed.stdout, end="") - if completed.stderr: - print(completed.stderr, end="", file=sys.stderr) - exit_code = completed.returncode - return exit_code - except subprocess.TimeoutExpired as exc: - stdout = sandboxed_verify.timeout_output_text(exc.stdout) - stderr = sandboxed_verify.timeout_output_text(exc.stderr) - if stdout: - print(stdout, end="" if stdout.endswith("\n") else "\n") - if stderr: - print(stderr, end="" if stderr.endswith("\n") else "\n", file=sys.stderr) - print(f"sandboxed-web-e2e: e2e command timed out after {args.e2e_timeout}s", file=sys.stderr) - exit_code = 124 - return exit_code + copied_repo = sandboxed_verify.copy_workspace(Path(args.repo_root), sandbox, args.ignore) + env = sandboxed_verify.scrubbed_env(sandbox, args.allow_env) + print(f"sandboxed-web-e2e: cwd={copied_repo}") + if args.allow_env: + print(f"sandboxed-web-e2e: allowed env names={','.join(sorted(set(args.allow_env)))}") + if args.network != "default": + print(f"sandboxed-web-e2e: network={args.network}") + services.append( + start_service( + "backend", + args.backend_cmd, + copied_repo, + env, + logs_dir, + args.service_log_limit_bytes, + ) + ) + services.append( + start_service( + "frontend", + args.frontend_cmd, + copied_repo, + env, + logs_dir, + args.service_log_limit_bytes, + ) + ) + backend_ready = wait_for_url(args.backend_ready_url, args.startup_timeout, services[0]) + frontend_ready = wait_for_url(args.frontend_ready_url, args.startup_timeout, services[1]) + if _services_output_limited(services): + output_limited = True + service_limit_reported = True + print( + "sandboxed-web-e2e: service output exceeded " + f"{args.service_log_limit_bytes} bytes", + file=sys.stderr, + ) + exit_code = bounded_subprocess.OUTPUT_LIMIT_EXIT_CODE + elif not backend_ready or not frontend_ready: + print("sandboxed-web-e2e: service readiness failed", file=sys.stderr) + exit_code = 125 + else: + try: + completed = run_shell( + args.e2e_cmd, + copied_repo, + env, + args.e2e_timeout, + args.output_limit_bytes, + ) + if completed.stdout: + print(completed.stdout, end="") + if completed.stderr: + print(completed.stderr, end="", file=sys.stderr) + output_limited = bool(getattr(completed, "output_limited", False)) + if output_limited: + print( + "sandboxed-web-e2e: E2E output exceeded " + f"{args.output_limit_bytes} bytes", + file=sys.stderr, + ) + exit_code = bounded_subprocess.OUTPUT_LIMIT_EXIT_CODE + else: + exit_code = completed.returncode + except subprocess.TimeoutExpired as exc: + stdout = sandboxed_verify.timeout_output_text(exc.stdout) + stderr = sandboxed_verify.timeout_output_text(exc.stderr) + if stdout: + print(stdout, end="" if stdout.endswith("\n") else "\n") + if stderr: + print(stderr, end="" if stderr.endswith("\n") else "\n", file=sys.stderr) + output_limited = bool(getattr(exc, "output_limited", False)) + print(f"sandboxed-web-e2e: e2e command timed out after {args.e2e_timeout}s", file=sys.stderr) + exit_code = 124 + except CommandExecutableNotFoundError: + print( + "sandboxed-web-e2e: install each executable or correct command PATH", + file=sys.stderr, + ) + exit_code = sandboxed_verify.COMMAND_NOT_FOUND_EXIT_CODE + except CommandNotExecutableError: + print( + "sandboxed-web-e2e: select executable files or correct their permissions", + file=sys.stderr, + ) + exit_code = sandboxed_verify.COMMAND_NOT_EXECUTABLE_EXIT_CODE + except bounded_subprocess.OutputLimitUnsupportedError: + output_limit_unsupported = True + print( + "sandboxed-web-e2e: bounded child output is unavailable on this platform", + file=sys.stderr, + ) + exit_code = bounded_subprocess.OUTPUT_LIMIT_EXIT_CODE + except sandboxed_verify.RepositoryPathBoundaryError: + path_boundary_rejected = True + copied_repo = Path("(not-created)") + print( + "sandboxed-web-e2e: repository path boundary rejected", + file=sys.stderr, + ) + exit_code = sandboxed_verify.PATH_BOUNDARY_EXIT_CODE + except sandboxed_verify.RepositoryRootError: + copied_repo = Path("(not-created)") + print( + "sandboxed-web-e2e: repository root is not a directory", + file=sys.stderr, + ) + exit_code = 1 + except RuntimeError: + print( + "sandboxed-web-e2e: bounded output capture failed", + file=sys.stderr, + ) + exit_code = bounded_subprocess.OUTPUT_LIMIT_EXIT_CODE finally: for service in reversed(services): - stop_service(service) + try: + stop_service(service) + except (OSError, RuntimeError, subprocess.SubprocessError): + with contextlib.suppress(OSError, subprocess.SubprocessError): + bounded_subprocess.kill_process_group(service.process) + with contextlib.suppress(OSError, subprocess.SubprocessError): + wait = getattr(service.process, "wait", None) + if wait is not None: + wait(timeout=10) + if service.capture is not None: + with contextlib.suppress(OSError, RuntimeError, subprocess.SubprocessError): + service.capture.join(timeout=10) + service_capture_failed = True + if exit_code == 0: + exit_code = bounded_subprocess.OUTPUT_LIMIT_EXIT_CODE + print( + "sandboxed-web-e2e: bounded service capture failed", + file=sys.stderr, + ) + if _services_output_limited(services): + output_limited = True + if exit_code == 0: + exit_code = bounded_subprocess.OUTPUT_LIMIT_EXIT_CODE + if not service_limit_reported: + print( + "sandboxed-web-e2e: service output exceeded " + f"{args.service_log_limit_bytes} bytes", + file=sys.stderr, + ) + for service in reversed(services): log_tail = tail_text(service.log_path) if log_tail: print(f"--- {service.label} log tail ---") @@ -262,9 +514,14 @@ def main(argv: Sequence[str] | None = None) -> int: frontend_ready=frontend_ready, exit_code=exit_code, elapsed_seconds=time.monotonic() - start, + output_limited=output_limited, + output_limit_unsupported=output_limit_unsupported, + service_capture_failed=service_capture_failed, + path_boundary_rejected=path_boundary_rejected, ) if not args.keep_sandbox: shutil.rmtree(sandbox, ignore_errors=True) + return exit_code if __name__ == "__main__": diff --git a/tests/test_repository_branch_coverage_execution_sandboxes.py b/tests/test_repository_branch_coverage_execution_sandboxes.py index 65de0a774..be4666d9a 100644 --- a/tests/test_repository_branch_coverage_execution_sandboxes.py +++ b/tests/test_repository_branch_coverage_execution_sandboxes.py @@ -200,6 +200,7 @@ def start_service( _cwd: Path, _env: dict[str, str], logs_dir: Path, + _log_limit_bytes: int, ) -> sandboxed_web_e2e.Service: log_path = logs_dir / f"{label}.log" log_path.write_text("", encoding="utf-8") @@ -208,7 +209,11 @@ def start_service( ) def timeout_runner( - command: str, _cwd: Path, _env: dict[str, str], timeout: int + command: str, + _cwd: Path, + _env: dict[str, str], + timeout: int, + _output_limit_bytes: int, ) -> subprocess.CompletedProcess[str]: raise subprocess.TimeoutExpired(command, timeout, output=None, stderr=None) diff --git a/tests/test_sandboxed_entrypoint_and_cleanup_coverage.py b/tests/test_sandboxed_entrypoint_and_cleanup_coverage.py new file mode 100644 index 000000000..c1164e287 --- /dev/null +++ b/tests/test_sandboxed_entrypoint_and_cleanup_coverage.py @@ -0,0 +1,91 @@ +import subprocess + +from scripts.ci import bounded_subprocess, sandboxed_web_e2e + + +def test_web_e2e_reports_bounded_capture_finalization_failure( + monkeypatch, + tmp_path, + capsys, +): + """A service-capture finalization failure remains a bounded hard failure.""" + + repo = tmp_path / "repo" + repo.mkdir() + + class DoneProcess: + pid = 12345 + + def poll(self): + return 0 + + def fake_start( + label, + command, + cwd, + env, + logs_dir, + log_limit_bytes=bounded_subprocess.DEFAULT_SERVICE_LOG_LIMIT_BYTES, + ): + del cwd, env + log_path = logs_dir / f"{label}.log" + log_path.write_text(f"{label} ready\n", encoding="utf-8") + return sandboxed_web_e2e.Service( + label=label, + command=command, + process=DoneProcess(), + log_path=log_path, + log_limit_bytes=log_limit_bytes, + ) + + monkeypatch.setattr(sandboxed_web_e2e, "start_service", fake_start) + monkeypatch.setattr(sandboxed_web_e2e, "wait_for_url", lambda *args: True) + monkeypatch.setattr( + sandboxed_web_e2e, + "run_shell", + lambda *args, **kwargs: subprocess.CompletedProcess( + args=["e2e"], + returncode=0, + stdout="ok\n", + stderr="", + ), + ) + + def fail_capture_finalization(service): + raise OSError(f"cannot finalize {service.label}") + + monkeypatch.setattr( + sandboxed_web_e2e, + "stop_service", + fail_capture_finalization, + ) + monkeypatch.setattr( + bounded_subprocess, + "kill_process_group", + lambda _process: None, + ) + + exit_code = sandboxed_web_e2e.main( + [ + "--repo-root", + str(repo), + "--backend-cmd", + "backend", + "--frontend-cmd", + "frontend", + "--backend-ready-url", + "http://127.0.0.1:8000/health", + "--frontend-ready-url", + "http://127.0.0.1:3000/", + "--e2e-cmd", + "e2e", + ] + ) + captured = capsys.readouterr() + + assert exit_code == bounded_subprocess.OUTPUT_LIMIT_EXIT_CODE + assert captured.err.count("bounded service capture failed") == 2 + assert f'"exit_code": {bounded_subprocess.OUTPUT_LIMIT_EXIT_CODE}' in captured.out + assert '"output_limited": false' in captured.out + assert '"output_limit_unsupported": false' in captured.out + assert '"service_capture_failed": true' in captured.out diff --git a/tests/test_sandboxed_service_capture_startup.py b/tests/test_sandboxed_service_capture_startup.py new file mode 100644 index 000000000..b8c323b87 --- /dev/null +++ b/tests/test_sandboxed_service_capture_startup.py @@ -0,0 +1,77 @@ +"""Failure contracts for bounded sandbox service capture startup.""" + +from __future__ import annotations + +import io +from pathlib import Path + +import pytest + +from scripts.ci import bounded_subprocess as bounded +from scripts.ci import sandboxed_web_e2e + + +class _RunningProcess: + """Minimal process double active until explicitly stopped and waited.""" + + pid = 200 + + def __init__(self) -> None: + self.stdout = io.BytesIO(b"") + self.returncode: int | None = None + self.waited = False + + def poll(self) -> int | None: + """Return the current process state.""" + + return self.returncode + + def wait(self, timeout=None) -> int: + """Record reaping and return the terminal status.""" + + del timeout + self.waited = True + self.returncode = -9 + return self.returncode + + +def test_capture_startup_failure_stops_reaps_and_closes_the_service_pipe( + monkeypatch, + tmp_path: Path, +) -> None: + """A failed drainer cannot leave a child or parent-side pipe uncollected.""" + + process = _RunningProcess() + monkeypatch.setattr(bounded, "require_supported_platform", lambda: None) + monkeypatch.setattr( + sandboxed_web_e2e.subprocess, + "Popen", + lambda *args, **kwargs: process, + ) + monkeypatch.setattr( + bounded, + "start_bounded_capture", + lambda *args, **kwargs: (_ for _ in ()).throw( + RuntimeError("capture startup failed") + ), + ) + stopped: list[object] = [] + monkeypatch.setattr( + bounded, + "kill_process_group", + lambda candidate: stopped.append(candidate), + ) + + with pytest.raises(RuntimeError, match="capture startup failed"): + sandboxed_web_e2e.start_service( + "backend", + "tool", + tmp_path, + {}, + tmp_path, + 4096, + ) + + assert stopped == [process] + assert process.waited is True + assert process.stdout.closed is True diff --git a/tests/test_sandboxed_web_e2e.py b/tests/test_sandboxed_web_e2e.py index 6e092c293..37034c2f2 100644 --- a/tests/test_sandboxed_web_e2e.py +++ b/tests/test_sandboxed_web_e2e.py @@ -1,4 +1,5 @@ import json +import io import os import runpy import socket @@ -159,6 +160,7 @@ def test_start_service_and_run_shell_capture_bash_contract(monkeypatch, tmp_path class FakeProcess: pid = 42 + stdout = io.BytesIO(b"") def poll(self): return 0 @@ -167,12 +169,22 @@ def fake_popen(*args, **kwargs): popen_calls.append((args, kwargs)) return FakeProcess() - def fake_run(*args, **kwargs): + def fake_bounded_run(*args, **kwargs): run_calls.append((args, kwargs)) - return subprocess.CompletedProcess(args[0], 7, stdout="out", stderr="err") + return sandboxed_web_e2e.bounded_subprocess.BoundedCompletedProcess( + args=("npm", "test"), + returncode=7, + stdout="out", + stderr="err", + output_limited=False, + ) monkeypatch.setattr(sandboxed_web_e2e.subprocess, "Popen", fake_popen) - monkeypatch.setattr(sandboxed_web_e2e.subprocess, "run", fake_run) + monkeypatch.setattr( + sandboxed_web_e2e.bounded_subprocess, + "run_bounded_command", + fake_bounded_run, + ) service = sandboxed_web_e2e.start_service("backend", "npm run dev", tmp_path, {"PATH": "/bin"}, tmp_path) completed = sandboxed_web_e2e.run_shell("npm test", tmp_path, {"PATH": "/bin"}, 5) @@ -181,14 +193,14 @@ def fake_run(*args, **kwargs): assert service.command == "npm run dev" assert service.log_path == tmp_path / "backend.log" assert popen_calls[0][0] == (["npm", "run", "dev"],) - assert "shell" not in popen_calls[0][1] assert "executable" not in popen_calls[0][1] assert popen_calls[0][1]["start_new_session"] is True + assert popen_calls[0][1]["shell"] is False + service.capture.join(timeout=5) assert completed.returncode == 7 assert run_calls[0][0] == (["npm", "test"],) assert run_calls[0][1]["timeout"] == 5 - assert "shell" not in run_calls[0][1] - assert "executable" not in run_calls[0][1] + assert run_calls[0][1]["evidence_limit_bytes"] > 0 def test_wait_for_url_handles_success_retry_and_log_tail(monkeypatch, tmp_path): @@ -273,11 +285,11 @@ class DoneProcess: def poll(self): return 0 - def fake_start(label, command, cwd, env, logs_dir): + def fake_start(label, command, cwd, env, logs_dir, log_limit_bytes): log_path = logs_dir / f"{label}.log" log_path.write_text(f"{label} ready\n", encoding="utf-8") service = sandboxed_web_e2e.Service(label, command, DoneProcess(), log_path) - started.append((label, command, cwd, "SANDBOXED_VERIFY" in env)) + started.append((label, command, cwd, "SANDBOXED_VERIFY" in env, log_limit_bytes)) return service monkeypatch.setattr(sandboxed_web_e2e, "start_service", fake_start) @@ -285,11 +297,14 @@ def fake_start(label, command, cwd, env, logs_dir): monkeypatch.setattr( sandboxed_web_e2e, "run_shell", - lambda command, cwd, env, timeout: subprocess.CompletedProcess( - command, - 0, - stdout="e2e-out\n", - stderr="e2e-err\n", + lambda command, cwd, env, timeout, output_limit_bytes: ( + sandboxed_web_e2e.bounded_subprocess.BoundedCompletedProcess( + args=(command,), + returncode=0, + stdout="e2e-out\n", + stderr="e2e-err\n", + output_limited=False, + ) ), ) monkeypatch.setattr(sandboxed_web_e2e, "stop_service", lambda service: stopped.append(service.label)) @@ -345,7 +360,8 @@ class DoneProcess: def poll(self): return 0 - def fake_start(label, command, cwd, env, logs_dir): + def fake_start(label, command, cwd, env, logs_dir, log_limit_bytes): + del log_limit_bytes log_path = logs_dir / f"{label}.log" log_path.write_text(f"{label} not ready\n", encoding="utf-8") return sandboxed_web_e2e.Service(label, command, DoneProcess(), log_path) @@ -393,12 +409,14 @@ class DoneProcess: def poll(self): return 0 - def fake_start(label, command, cwd, env, logs_dir): + def fake_start(label, command, cwd, env, logs_dir, log_limit_bytes): + del log_limit_bytes log_path = logs_dir / f"{label}.log" log_path.write_text(f"{label} tail\n", encoding="utf-8") return sandboxed_web_e2e.Service(label, command, DoneProcess(), log_path) - def fake_run_shell(command, cwd, env, timeout): + def fake_run_shell(command, cwd, env, timeout, output_limit_bytes): + del output_limit_bytes raise subprocess.TimeoutExpired(command, timeout, output=b"e2e-out", stderr=b"e2e-err") monkeypatch.setattr(sandboxed_web_e2e, "start_service", fake_start) @@ -469,7 +487,8 @@ def test_sandboxed_web_e2e_reports_e2e_timeout(monkeypatch, tmp_path, capsys): repo = tmp_path / "repo" repo.mkdir() - def fake_run_shell(command, cwd, env, timeout): + def fake_run_shell(command, cwd, env, timeout, output_limit_bytes): + del output_limit_bytes raise subprocess.TimeoutExpired(command, timeout, output="e2e-out", stderr="e2e-err") monkeypatch.setattr(sandboxed_web_e2e, "run_shell", fake_run_shell) @@ -540,6 +559,33 @@ def test_parse_args_rejects_invalid_inputs(): ) +@pytest.mark.parametrize( + "option", + ["--backend-ready-url", "--frontend-ready-url"], +) +def test_parse_args_rejects_non_http_readiness_urls(option, capsys): + """Invalid readiness schemes fail in argument parsing without a traceback.""" + + with pytest.raises(SystemExit) as raised: + sandboxed_web_e2e.parse_args( + [ + "--backend-cmd", + "backend", + "--frontend-cmd", + "frontend", + "--e2e-cmd", + "e2e", + option, + "file:///runner/private", + ] + ) + captured = capsys.readouterr() + + assert raised.value.code == 2 + assert f"{option} must start with http:// or https://" in captured.err + assert "Traceback" not in captured.err + + def test_module_main_entrypoint_parse_error(monkeypatch): """The module entrypoint reaches main and propagates argument errors.""" runpy.run_path(str(Path(sandboxed_web_e2e.__file__)), run_name="not_main") diff --git a/tests/test_sandboxed_web_e2e_branch_contract.py b/tests/test_sandboxed_web_e2e_branch_contract.py new file mode 100644 index 000000000..2a9dede6b --- /dev/null +++ b/tests/test_sandboxed_web_e2e_branch_contract.py @@ -0,0 +1,680 @@ +"""Branch-complete contracts for bounded sandbox web E2E orchestration.""" + +from __future__ import annotations + +import json +import subprocess +import urllib.error +from pathlib import Path +from typing import cast + +import pytest + +from scripts.ci import bounded_subprocess as bounded +from scripts.ci import sandboxed_verify +from scripts.ci import sandboxed_web_e2e + + +def _result(output: str) -> dict[str, object]: + """Parse one final web E2E result marker.""" + + marker = f"{sandboxed_web_e2e.RESULT_MARKER} " + line = next(line for line in output.splitlines() if line.startswith(marker)) + return json.loads(line.removeprefix(marker)) + + +class _DoneProcess: + """Minimal process double that has already completed.""" + + pid = 100 + returncode = 0 + + def poll(self) -> int: + """Return the completed status.""" + + return self.returncode + + def wait(self, timeout=None) -> int: + """Return immediately.""" + + del timeout + return self.returncode + + +class _RunningProcess: + """Minimal running process double for cleanup branches.""" + + pid = 101 + returncode = None + + def poll(self): + """Report that the process remains active.""" + + return self.returncode + + def wait(self, timeout=None) -> int: + """Complete when the fake process is explicitly waited.""" + + del timeout + self.returncode = 0 + return 0 + + +def _service(tmp_path: Path, *, process=None, log_limit_bytes: int = 4096): + """Create one service double with no background capture.""" + + return sandboxed_web_e2e.Service( + label="service", + command="service", + process=cast(subprocess.Popen[bytes], process or _DoneProcess()), + log_path=tmp_path / "service.log", + log_limit_bytes=log_limit_bytes, + ) + + +def test_start_service_rejects_missing_output_pipe( + monkeypatch, + tmp_path: Path, +) -> None: + """A broken Popen pipe contract is killed and rejected.""" + + class MissingPipeProcess(_RunningProcess): + """Return no stdout despite the requested PIPE configuration.""" + + stdout = None + + process = MissingPipeProcess() + monkeypatch.setattr(bounded, "require_supported_platform", lambda: None) + monkeypatch.setattr( + sandboxed_web_e2e.subprocess, + "Popen", + lambda *args, **kwargs: process, + ) + killed: list[object] = [] + monkeypatch.setattr( + bounded, + "kill_process_group", + lambda candidate: killed.append(candidate), + ) + + with pytest.raises(RuntimeError, match="pipe"): + sandboxed_web_e2e.start_service( + "backend", + "tool", + tmp_path, + {}, + tmp_path, + 4096, + ) + assert killed == [process] + + +def test_service_limit_fallback_handles_missing_small_and_large_files( + tmp_path: Path, +) -> None: + """Legacy/fake services classify file-only evidence deterministically.""" + + service = _service(tmp_path) + assert not sandboxed_web_e2e.service_output_limited(service) + service.log_path.write_bytes(b"safe") + assert not sandboxed_web_e2e.service_output_limited(service) + service.log_path.write_bytes(b"x" * 4096) + assert not sandboxed_web_e2e.service_output_limited(service) + service.log_path.write_bytes(b"x" * 4097) + assert sandboxed_web_e2e.service_output_limited(service) + + +def test_wait_for_url_handles_empty_invalid_exited_limited_and_success( + monkeypatch, + tmp_path: Path, +) -> None: + """Readiness polling preserves every validation and termination branch.""" + + service = _service(tmp_path) + assert sandboxed_web_e2e.wait_for_url("", 1, service) + with pytest.raises(ValueError, match="http"): + sandboxed_web_e2e.wait_for_url("file:///tmp/ready", 1, service) + assert not sandboxed_web_e2e.wait_for_url( + "https://example.invalid/ready", + 1, + service, + ) + + running = _service(tmp_path, process=_RunningProcess()) + running.log_path.write_bytes(b"x" * 4097) + assert not sandboxed_web_e2e.wait_for_url( + "https://example.invalid/ready", + 1, + running, + ) + running.log_path.unlink() + + class Response: + """Context-managed readiness response.""" + + status = 204 + + def __enter__(self): + """Return the response.""" + + return self + + def __exit__(self, exc_type, exc, traceback) -> None: + """Close without suppressing exceptions.""" + + del exc_type, exc, traceback + + class Opener: + """Return one successful response.""" + + def open(self, url: str, timeout: int): + """Validate the poll request and return readiness.""" + + assert url == "https://ready.example/health" + assert timeout == 2 + return Response() + + clean_running = _service(tmp_path, process=_RunningProcess()) + monkeypatch.setattr( + sandboxed_web_e2e.urllib.request, + "build_opener", + lambda handler: Opener(), + ) + assert sandboxed_web_e2e.wait_for_url( + "https://ready.example/health", + 1, + clean_running, + ) + + +def test_wait_for_url_retries_url_errors_until_deadline( + monkeypatch, + tmp_path: Path, +) -> None: + """Transient URL errors sleep and eventually produce a bounded false result.""" + + class FailingOpener: + """Raise one deterministic URL error per poll.""" + + def open(self, url: str, timeout: int): + """Reject the readiness request.""" + + del url, timeout + raise urllib.error.URLError("not ready") + + timeline = iter([0.0, 0.0, 2.0]) + sleeps: list[int] = [] + monkeypatch.setattr(sandboxed_web_e2e.time, "monotonic", lambda: next(timeline)) + monkeypatch.setattr(sandboxed_web_e2e.time, "sleep", lambda seconds: sleeps.append(seconds)) + monkeypatch.setattr( + sandboxed_web_e2e.urllib.request, + "build_opener", + lambda handler: FailingOpener(), + ) + + assert not sandboxed_web_e2e.wait_for_url( + "https://ready.example/health", + 1, + _service(tmp_path, process=_RunningProcess()), + ) + assert sleeps == [1] + + +def test_redirect_handler_raises_http_error() -> None: + """Readiness redirects are never followed.""" + + handler = sandboxed_web_e2e.NoRedirectHandler() + request = type("Request", (), {"full_url": "https://ready.example"})() + with pytest.raises(urllib.error.HTTPError): + handler.redirect_request(request, None, 302, "redirect", {}, "https://other") + + +def test_stop_service_handles_finished_lookup_race_timeout_and_capture( + monkeypatch, + tmp_path: Path, +) -> None: + """Cleanup covers normal, disappearing, force-kill, and capture-finalization paths.""" + + joined: list[float | None] = [] + + class Capture: + """Record finalization of one fake background drain.""" + + output_limited = False + + def join(self, timeout=None) -> None: + """Record the requested join timeout.""" + + joined.append(timeout) + + finished = _service(tmp_path) + finished.capture = cast(bounded.BoundedOutputCapture, Capture()) + sandboxed_web_e2e.stop_service(finished) + assert joined == [10] + + disappearing = _service(tmp_path, process=_RunningProcess()) + monkeypatch.setattr( + sandboxed_web_e2e.os, + "killpg", + lambda pid, signal_number: (_ for _ in ()).throw(ProcessLookupError()), + ) + sandboxed_web_e2e.stop_service(disappearing) + + class TimeoutProcess(_RunningProcess): + """Timeout once before completing after force kill.""" + + def __init__(self) -> None: + self.waits = 0 + + def wait(self, timeout=None) -> int: + """Raise once, then return the terminal status.""" + + del timeout + self.waits += 1 + if self.waits == 1: + raise subprocess.TimeoutExpired("service", 10) + self.returncode = -9 + return self.returncode + + timeout_process = TimeoutProcess() + timed = _service(tmp_path, process=timeout_process) + monkeypatch.setattr(sandboxed_web_e2e.os, "killpg", lambda pid, sig: None) + forced: list[object] = [] + monkeypatch.setattr( + bounded, + "kill_process_group", + lambda process: forced.append(process), + ) + sandboxed_web_e2e.stop_service(timed) + assert forced == [timeout_process] + + +def test_tail_text_rejects_nonpositive_line_count(tmp_path: Path) -> None: + """A caller cannot request an ambiguous or unbounded line selection.""" + + log_path = tmp_path / "service.log" + log_path.write_text("line\n", encoding="utf-8") + with pytest.raises(ValueError, match="max_lines"): + sandboxed_web_e2e.tail_text(log_path, max_lines=0) + + +def test_tail_text_validates_line_count_before_missing_file(tmp_path: Path) -> None: + """A missing evidence file cannot bypass the configured line-budget contract.""" + + with pytest.raises(ValueError, match="max_lines"): + sandboxed_web_e2e.tail_text(tmp_path / "missing.log", max_lines=0) + + +def test_timeout_precedence_survives_limited_partial_output( + monkeypatch, + tmp_path: Path, + capsys, +) -> None: + """A timed-out E2E remains 124 even when its bounded stream was truncated.""" + + repository = tmp_path / "repository" + repository.mkdir() + + def fake_start(label, command, cwd, env, logs_dir, log_limit_bytes): + """Return already-running service doubles without real children.""" + + del command, cwd, env + return sandboxed_web_e2e.Service( + label=label, + command=label, + process=cast(subprocess.Popen[bytes], _RunningProcess()), + log_path=logs_dir / f"{label}.log", + log_limit_bytes=log_limit_bytes, + ) + + def timeout_run(command, cwd, env, timeout, output_limit_bytes): + """Raise bounded timeout evidence.""" + + del command, cwd, env, output_limit_bytes + raise bounded.BoundedTimeoutExpired( + ["e2e"], + timeout, + stdout=bounded.TRUNCATION_MARKER, + stderr="", + output_limited=True, + ) + + monkeypatch.setattr(sandboxed_web_e2e, "start_service", fake_start) + monkeypatch.setattr(sandboxed_web_e2e, "wait_for_url", lambda *args: True) + monkeypatch.setattr(sandboxed_web_e2e, "run_shell", timeout_run) + monkeypatch.setattr(sandboxed_web_e2e, "stop_service", lambda service: None) + + exit_code = sandboxed_web_e2e.main( + [ + "--repo-root", + str(repository), + "--backend-cmd", + "backend", + "--frontend-cmd", + "frontend", + "--e2e-cmd", + "e2e", + "--e2e-timeout", + "1", + ] + ) + captured = capsys.readouterr() + + assert exit_code == 124 + assert "timed out after 1s" in captured.err + assert _result(captured.out)["output_limited"] is True + + +def test_capture_finalization_failure_maps_to_resource_exit( + monkeypatch, + tmp_path: Path, + capsys, +) -> None: + """A service capture failure cannot leave a successful result envelope.""" + + repository = tmp_path / "repository" + repository.mkdir() + captures = [] + + class Capture: + """Record the cleanup retry after service termination fails.""" + + output_limited = False + + def __init__(self) -> None: + self.join_calls = 0 + + def join(self, timeout=None) -> None: + """Record the bounded retry timeout.""" + + del timeout + self.join_calls += 1 + + def fake_start(label, command, cwd, env, logs_dir, log_limit_bytes): + """Return completed service doubles.""" + + del command, cwd, env + capture = Capture() + captures.append(capture) + return sandboxed_web_e2e.Service( + label=label, + command=label, + process=cast(subprocess.Popen[bytes], _DoneProcess()), + log_path=logs_dir / f"{label}.log", + capture=capture, + log_limit_bytes=log_limit_bytes, + ) + + monkeypatch.setattr(sandboxed_web_e2e, "start_service", fake_start) + monkeypatch.setattr(sandboxed_web_e2e, "wait_for_url", lambda *args: True) + monkeypatch.setattr( + sandboxed_web_e2e, + "run_shell", + lambda *args: bounded.BoundedCompletedProcess( + args=("e2e",), + returncode=0, + stdout="", + stderr="", + output_limited=False, + ), + ) + monkeypatch.setattr( + sandboxed_web_e2e, + "stop_service", + lambda service: (_ for _ in ()).throw(RuntimeError("capture failed")), + ) + forced: list[object] = [] + monkeypatch.setattr( + bounded, + "kill_process_group", + lambda process: forced.append(process), + ) + + exit_code = sandboxed_web_e2e.main( + [ + "--repo-root", + str(repository), + "--backend-cmd", + "backend", + "--frontend-cmd", + "frontend", + "--e2e-cmd", + "e2e", + ] + ) + captured = capsys.readouterr() + + assert exit_code == bounded.OUTPUT_LIMIT_EXIT_CODE + assert "bounded service capture failed" in captured.err + assert _result(captured.out)["output_limited"] is False + assert _result(captured.out)["output_limit_unsupported"] is False + assert _result(captured.out)["service_capture_failed"] is True + assert len(forced) == 2 + assert [capture.join_calls for capture in captures] == [1, 1] + + +def test_main_classifies_web_symlink_boundary_without_host_target( + tmp_path: Path, + capsys, +) -> None: + """Web E2E rejects a copied symlink without disclosing its host target.""" + repository = tmp_path / "repository" + repository.mkdir() + sensitive_target = tmp_path / "runner-secret.txt" + sensitive_target.write_text("host-only", encoding="utf-8") + (repository / "escape").symlink_to(sensitive_target) + + exit_code = sandboxed_web_e2e.main( + [ + "--repo-root", + str(repository), + "--backend-cmd", + "backend", + "--frontend-cmd", + "frontend", + "--e2e-cmd", + "e2e", + ] + ) + captured = capsys.readouterr() + result = _result(captured.out) + + assert exit_code == sandboxed_verify.PATH_BOUNDARY_EXIT_CODE + assert result["path_boundary_rejected"] is True + assert result["cwd"] == "(not-created)" + assert "repository path boundary rejected" in captured.err + assert str(sensitive_target) not in captured.err + assert str(repository) not in captured.err + assert "Traceback" not in captured.err + + +def test_main_reports_web_invalid_root_without_boundary_evidence( + tmp_path: Path, + capsys, +) -> None: + """Web E2E distinguishes an absent repository root from a path escape.""" + missing = tmp_path / "missing-repository" + + exit_code = sandboxed_web_e2e.main( + [ + "--repo-root", + str(missing), + "--backend-cmd", + "backend", + "--frontend-cmd", + "frontend", + "--e2e-cmd", + "e2e", + ] + ) + captured = capsys.readouterr() + result = _result(captured.out) + + assert exit_code == 1 + assert result["path_boundary_rejected"] is False + assert result["cwd"] == "(not-created)" + assert "repository root is not a directory" in captured.err + assert str(missing) not in captured.err + + +def test_main_maps_command_capture_failure_to_bounded_resource_evidence( + monkeypatch, + tmp_path: Path, + capsys, +) -> None: + """A command-side stuck reader is reported without leaking its exception.""" + repository = tmp_path / "repository" + repository.mkdir() + + def fake_start(label, command, cwd, env, logs_dir, log_limit_bytes): + """Return completed services so the E2E command path is reached.""" + + del cwd, env + return sandboxed_web_e2e.Service( + label=label, + command=command, + process=cast(subprocess.Popen[bytes], _DoneProcess()), + log_path=logs_dir / f"{label}.log", + log_limit_bytes=log_limit_bytes, + ) + + monkeypatch.setattr(sandboxed_web_e2e, "start_service", fake_start) + monkeypatch.setattr(sandboxed_web_e2e, "wait_for_url", lambda *args: True) + monkeypatch.setattr( + sandboxed_web_e2e, + "run_shell", + lambda *args, **kwargs: (_ for _ in ()).throw( + RuntimeError("host descriptor detail") + ), + ) + + exit_code = sandboxed_web_e2e.main( + [ + "--repo-root", + str(repository), + "--backend-cmd", + "backend", + "--frontend-cmd", + "frontend", + "--e2e-cmd", + "e2e", + ] + ) + captured = capsys.readouterr() + + assert exit_code == bounded.OUTPUT_LIMIT_EXIT_CODE + assert "bounded output capture failed" in captured.err + assert "host descriptor detail" not in captured.err + assert "Traceback" not in captured.err + + +def test_timeout_precedence_survives_cleanup_and_late_service_limit( + monkeypatch, + tmp_path: Path, +) -> None: + """Timeout 124 remains authoritative through late capture failures and overflow.""" + repository = tmp_path / "repository" + repository.mkdir() + + def fake_start(label, command, cwd, env, logs_dir, log_limit_bytes): + del command, cwd, env + return sandboxed_web_e2e.Service( + label=label, + command=label, + process=cast(subprocess.Popen[bytes], _DoneProcess()), + log_path=logs_dir / f"{label}.log", + log_limit_bytes=log_limit_bytes, + ) + + def timeout_run(*args): + del args + raise subprocess.TimeoutExpired(["e2e"], 1) + + limit_checks = iter([False, True]) + monkeypatch.setattr(sandboxed_web_e2e, "start_service", fake_start) + monkeypatch.setattr(sandboxed_web_e2e, "wait_for_url", lambda *args: True) + monkeypatch.setattr(sandboxed_web_e2e, "run_shell", timeout_run) + monkeypatch.setattr( + sandboxed_web_e2e, + "stop_service", + lambda service: (_ for _ in ()).throw(RuntimeError(service.label)), + ) + monkeypatch.setattr( + sandboxed_web_e2e, + "_services_output_limited", + lambda services: next(limit_checks), + ) + + assert sandboxed_web_e2e.main( + [ + "--repo-root", + str(repository), + "--backend-cmd", + "backend", + "--frontend-cmd", + "frontend", + "--e2e-cmd", + "e2e", + "--e2e-timeout", + "1", + ] + ) == 124 + + +def test_late_service_overflow_preserves_nonzero_e2e_exit( + monkeypatch, + tmp_path: Path, + capsys, +) -> None: + """A late service overflow must not hide an earlier command failure.""" + repository = tmp_path / "repository" + repository.mkdir() + + def fake_start(label, command, cwd, env, logs_dir, log_limit_bytes): + """Return completed service doubles for the late-limit branch.""" + + del command, cwd, env, log_limit_bytes + return sandboxed_web_e2e.Service( + label=label, + command=label, + process=cast(subprocess.Popen[bytes], _DoneProcess()), + log_path=logs_dir / f"{label}.log", + log_limit_bytes=4096, + ) + + monkeypatch.setattr(sandboxed_web_e2e, "start_service", fake_start) + monkeypatch.setattr(sandboxed_web_e2e, "wait_for_url", lambda *args: True) + monkeypatch.setattr( + sandboxed_web_e2e, + "run_shell", + lambda *args: bounded.BoundedCompletedProcess( + args=("e2e",), + returncode=7, + stdout="", + stderr="", + output_limited=False, + ), + ) + monkeypatch.setattr(sandboxed_web_e2e, "stop_service", lambda service: None) + limit_checks = iter([False, True]) + monkeypatch.setattr( + sandboxed_web_e2e, + "_services_output_limited", + lambda services: next(limit_checks), + ) + + assert sandboxed_web_e2e.main( + [ + "--repo-root", + str(repository), + "--backend-cmd", + "backend", + "--frontend-cmd", + "frontend", + "--e2e-cmd", + "e2e", + ] + ) == 7 + payload = _result(capsys.readouterr().out) + assert payload["output_limited"] is True + assert payload["service_capture_failed"] is False diff --git a/tests/test_sandboxed_web_e2e_output_limits.py b/tests/test_sandboxed_web_e2e_output_limits.py new file mode 100644 index 000000000..f2a38a4d2 --- /dev/null +++ b/tests/test_sandboxed_web_e2e_output_limits.py @@ -0,0 +1,434 @@ +"""Real-process contracts for bounded sandbox web E2E output.""" + +from __future__ import annotations + +import json +import socket +import shlex +import shutil +import sys +from pathlib import Path + +import pytest + +from scripts.ci import bounded_subprocess as bounded +from scripts.ci import sandboxed_verify +from scripts.ci import sandboxed_web_e2e + + +def _command(source: str) -> str: + """Return one shell-style command that safely launches the current Python.""" + + return shlex.join([sys.executable, "-c", source]) + + +def _repository(tmp_path: Path) -> Path: + """Create one minimal repository accepted by the sandbox copy boundary.""" + + repository = tmp_path / "repository" + repository.mkdir() + (repository / "README.md").write_text("web E2E fixture\n", encoding="utf-8") + return repository + + +def _free_ports(count: int) -> list[int]: + """Return distinct localhost ports reserved at the same time.""" + + listeners = [socket.socket() for _ in range(count)] + try: + ports = [] + for listener in listeners: + listener.bind(("127.0.0.1", 0)) + ports.append(int(listener.getsockname()[1])) + return ports + finally: + for listener in listeners: + listener.close() + + +def _http_service_command(port: int, label: str) -> str: + """Return a bounded-test HTTP service that emits its readiness label.""" + + return _command( + "import http.server\n" + "import socketserver\n" + "socketserver.TCPServer.allow_reuse_address=True\n" + f"server=socketserver.TCPServer(('127.0.0.1',{port})," + "http.server.SimpleHTTPRequestHandler)\n" + f"print({label!r},flush=True)\n" + "server.serve_forever()\n" + ) + + +def _result_payload(output: str) -> dict[str, object]: + """Parse the final machine-readable web E2E result marker.""" + + marker = f"{sandboxed_web_e2e.RESULT_MARKER} " + line = next( + item for item in reversed(output.splitlines()) if item.startswith(marker) + ) + return json.loads(line.removeprefix(marker)) + + +def test_start_service_enforces_real_log_file_ceiling(tmp_path: Path) -> None: + """A long-running child cannot grow its combined service log past the ceiling.""" + + logs_directory = tmp_path / "logs" + logs_directory.mkdir() + log_limit_bytes = 4096 + service = sandboxed_web_e2e.start_service( + "backend", + _command( + "import os\n" + "chunk=b'x'*1024\n" + "while True:\n" + " os.write(1,chunk)\n" + ), + tmp_path, + {"PATH": ""}, + logs_directory, + log_limit_bytes, + ) + try: + service.process.wait(timeout=10) + assert service.log_path.stat().st_size <= log_limit_bytes + assert sandboxed_web_e2e.service_output_limited(service) + finally: + sandboxed_web_e2e.stop_service(service) + + +def test_service_log_overflow_returns_resource_limit_before_e2e( + tmp_path: Path, + capsys, +) -> None: + """Readiness cannot convert a backend log flood into an ordinary E2E run.""" + + sentinel = tmp_path / "e2e-ran" + exit_code = sandboxed_web_e2e.main( + [ + "--repo-root", + str(_repository(tmp_path)), + "--backend-cmd", + _command( + "import os\n" + "chunk=b'x'*1024\n" + "while True:\n" + " os.write(1,chunk)\n" + ), + "--backend-ready-url", + "http://127.0.0.1:1/ready", + "--frontend-cmd", + _command("import time; time.sleep(30)"), + "--e2e-cmd", + _command( + f"from pathlib import Path; Path({str(sentinel)!r}).touch()" + ), + "--service-log-limit-bytes", + "4096", + ] + ) + captured = capsys.readouterr() + payload = _result_payload(captured.out) + + assert exit_code == bounded.OUTPUT_LIMIT_EXIT_CODE + assert "service output exceeded 4096 bytes" in captured.err + assert payload["output_limited"] is True + assert payload["output_limit_unsupported"] is False + assert payload["service_capture_failed"] is False + assert payload["service_log_limit_bytes"] == 4096 + assert not sentinel.exists() + + +def test_e2e_output_overflow_is_bounded_and_returns_123( + tmp_path: Path, + capsys, +) -> None: + """The short-lived E2E command uses the same kernel-enforced output boundary.""" + + exit_code = sandboxed_web_e2e.main( + [ + "--repo-root", + str(_repository(tmp_path)), + "--backend-cmd", + _command("import time; time.sleep(30)"), + "--frontend-cmd", + _command("import time; time.sleep(30)"), + "--e2e-cmd", + _command( + "import os\n" + "chunk=b'y'*1024\n" + "while True:\n" + " os.write(2,chunk)\n" + ), + "--output-limit-bytes", + "4096", + ] + ) + captured = capsys.readouterr() + payload = _result_payload(captured.out) + + assert exit_code == bounded.OUTPUT_LIMIT_EXIT_CODE + assert bounded.TRUNCATION_MARKER.strip() in captured.err + assert "E2E output exceeded 4096 bytes" in captured.err + assert payload["output_limit_bytes"] == 4096 + assert payload["output_limited"] is True + assert payload["output_limit_unsupported"] is False + assert payload["service_capture_failed"] is False + assert len((captured.out + captured.err).encode("utf-8")) < 25_000 + + +def test_normal_services_and_e2e_preserve_existing_success_contract( + tmp_path: Path, + capsys, +) -> None: + """Ordinary services, Unicode output, cleanup, and evidence remain unchanged.""" + + backend_port, frontend_port = _free_ports(2) + exit_code = sandboxed_web_e2e.main( + [ + "--repo-root", + str(_repository(tmp_path)), + "--backend-cmd", + _http_service_command(backend_port, "backend-ready"), + "--frontend-cmd", + _http_service_command(frontend_port, "frontend-ready"), + "--backend-ready-url", + f"http://127.0.0.1:{backend_port}/README.md", + "--frontend-ready-url", + f"http://127.0.0.1:{frontend_port}/README.md", + "--e2e-cmd", + _command("print('통합 성공')"), + "--output-limit-bytes", + "4096", + "--service-log-limit-bytes", + "4096", + ] + ) + captured = capsys.readouterr() + payload = _result_payload(captured.out) + + assert exit_code == 0 + assert "통합 성공" in captured.out + assert "backend-ready" in captured.out + assert "frontend-ready" in captured.out + assert payload["output_limited"] is False + assert payload["output_limit_unsupported"] is False + assert payload["service_capture_failed"] is False + assert payload["output_limit_bytes"] == 4096 + assert payload["service_log_limit_bytes"] == 4096 + + +def test_tail_text_uses_bounded_suffix_and_tolerates_partial_utf8( + monkeypatch, + tmp_path: Path, +) -> None: + """Service evidence delegates to a byte-bounded suffix before line selection.""" + + log_path = tmp_path / "service.log" + log_path.write_bytes(b"ignored" + "가".encode("utf-8")) + observed: dict[str, object] = {} + + def fake_suffix(path: Path, maximum_bytes: int) -> bounded.BoundedText: + observed["path"] = path + observed["maximum_bytes"] = maximum_bytes + return bounded.BoundedText( + text=f"{bounded.TRUNCATION_MARKER}�\nlast-line\n", + truncated=True, + stored_bytes=10_000, + ) + + monkeypatch.setattr(bounded, "read_bounded_suffix", fake_suffix) + + tail = sandboxed_web_e2e.tail_text( + log_path, + max_lines=2, + max_bytes=4096, + ) + + assert observed == {"path": log_path, "maximum_bytes": 4096} + assert tail == f"{bounded.TRUNCATION_MARKER.strip()}\n�\nlast-line" + + +def test_unsupported_resource_boundary_fails_closed( + monkeypatch, + tmp_path: Path, + capsys, +) -> None: + """Service startup cannot silently continue without file-size enforcement.""" + + def fail_start(*args, **kwargs): + del args, kwargs + raise bounded.OutputLimitUnsupportedError("unsupported") + + monkeypatch.setattr(sandboxed_web_e2e, "start_service", fail_start) + exit_code = sandboxed_web_e2e.main( + [ + "--repo-root", + str(_repository(tmp_path)), + "--backend-cmd", + _command("pass"), + "--frontend-cmd", + _command("pass"), + "--e2e-cmd", + _command("pass"), + ] + ) + captured = capsys.readouterr() + payload = _result_payload(captured.out) + + assert exit_code == bounded.OUTPUT_LIMIT_EXIT_CODE + assert "bounded child output is unavailable" in captured.err + assert payload["output_limited"] is False + assert payload["output_limit_unsupported"] is True + assert payload["service_capture_failed"] is False + + +@pytest.mark.parametrize("missing_role", ["backend", "frontend", "e2e"]) +def test_missing_executable_returns_stable_failed_evidence( + tmp_path: Path, + capsys, + missing_role: str, +) -> None: + """Every command role reports a missing executable without a traceback.""" + + commands = { + "backend": _command("import time; time.sleep(30)"), + "frontend": _command("import time; time.sleep(30)"), + "e2e": _command("print('ready')"), + } + commands[missing_role] = "missing-web-e2e-executable" + exit_code = sandboxed_web_e2e.main( + [ + "--repo-root", + str(_repository(tmp_path)), + "--backend-cmd", + commands["backend"], + "--frontend-cmd", + commands["frontend"], + "--e2e-cmd", + commands["e2e"], + ] + ) + captured = capsys.readouterr() + payload = _result_payload(captured.out) + + assert exit_code == sandboxed_verify.COMMAND_NOT_FOUND_EXIT_CODE + assert payload["exit_code"] == sandboxed_verify.COMMAND_NOT_FOUND_EXIT_CODE + assert "install each executable or correct command PATH" in captured.err + assert "Traceback" not in captured.err + + +@pytest.mark.parametrize("command_role", ["backend", "frontend", "e2e"]) +@pytest.mark.parametrize("candidate_kind", ["file", "directory"]) +def test_non_executable_command_returns_stable_failed_evidence( + tmp_path: Path, + capsys, + command_role: str, + candidate_kind: str, +) -> None: + """Every web command role classifies a present but unusable executable.""" + + candidate = tmp_path / "web-command-candidate" + if candidate_kind == "file": + candidate.write_text("not executable\n", encoding="utf-8") + else: + candidate.mkdir() + commands = { + "backend": _command("import time; time.sleep(30)"), + "frontend": _command("import time; time.sleep(30)"), + "e2e": _command("print('ready')"), + } + commands[command_role] = str(candidate) + + exit_code = sandboxed_web_e2e.main( + [ + "--repo-root", + str(_repository(tmp_path)), + "--backend-cmd", + commands["backend"], + "--frontend-cmd", + commands["frontend"], + "--e2e-cmd", + commands["e2e"], + ] + ) + captured = capsys.readouterr() + payload = _result_payload(captured.out) + + assert exit_code == sandboxed_verify.COMMAND_NOT_EXECUTABLE_EXIT_CODE + assert payload["exit_code"] == sandboxed_verify.COMMAND_NOT_EXECUTABLE_EXIT_CODE + assert "select executable files or correct their permissions" in captured.err + assert "Traceback" not in captured.err + + +@pytest.mark.parametrize( + ("option", "value"), + [ + ("--output-limit-bytes", "4095"), + ( + "--service-log-limit-bytes", + str(bounded.MAXIMUM_OUTPUT_LIMIT_BYTES + 1), + ), + ], +) +def test_cli_rejects_unsafe_command_and_service_budgets( + tmp_path: Path, + option: str, + value: str, +) -> None: + """Both output budgets fail parsing outside the explicit safe range.""" + + repository = _repository(tmp_path) + base = [ + "--repo-root", + str(repository), + "--backend-cmd", + _command("pass"), + "--frontend-cmd", + _command("pass"), + "--e2e-cmd", + _command("pass"), + ] + with pytest.raises(SystemExit) as raised: + sandboxed_web_e2e.parse_args([*base, option, value]) + assert raised.value.code == 2 + + +def test_kept_sandbox_service_file_never_exceeds_kernel_ceiling( + tmp_path: Path, + capsys, +) -> None: + """Persisted debugging sandboxes retain only the bounded service artifact.""" + + log_limit_bytes = 4096 + exit_code = sandboxed_web_e2e.main( + [ + "--repo-root", + str(_repository(tmp_path)), + "--backend-cmd", + _command( + "import os\n" + "chunk=b'z'*1024\n" + "while True:\n" + " os.write(1,chunk)\n" + ), + "--frontend-cmd", + _command("import time; time.sleep(30)"), + "--e2e-cmd", + _command("pass"), + "--service-log-limit-bytes", + str(log_limit_bytes), + "--keep-sandbox", + ] + ) + captured = capsys.readouterr() + payload = _result_payload(captured.out) + sandbox_path = Path(str(payload["sandbox"])) + + try: + assert exit_code == bounded.OUTPUT_LIMIT_EXIT_CODE + assert ( + sandbox_path / "logs" / "backend.log" + ).stat().st_size <= log_limit_bytes + finally: + shutil.rmtree(sandbox_path, ignore_errors=True)