From 9912779faa67013407e6935da18a76b23b2ba564 Mon Sep 17 00:00:00 2001 From: seonghobae <8172694+seonghobae@users.noreply.github.com> Date: Tue, 25 Aug 2026 17:38:18 +0000 Subject: [PATCH 1/5] =?UTF-8?q?=F0=9F=9B=A1=EF=B8=8F=20[HIGH]=20Fix=20SSRF?= =?UTF-8?q?=20vulnerability=20in=20web=20e2e=20readiness=20probe\n\nScript?= =?UTF-8?q?s/ci/sandboxed=5Fweb=5Fe2e.py=20=EC=9D=98=20wait=5Ffor=5Furl=20?= =?UTF-8?q?=ED=95=A8=EC=88=98=EA=B0=80=20=EC=99=B8=EB=B6=80/=EB=82=B4?= =?UTF-8?q?=EB=B6=80=20=EB=84=A4=ED=8A=B8=EC=9B=8C=ED=81=AC=EB=A5=BC=20?= =?UTF-8?q?=EC=8A=A4=EC=BA=94=ED=95=A0=20=EC=88=98=20=EC=9E=88=EB=8A=94=20?= =?UTF-8?q?SSRF=20=EC=B7=A8=EC=95=BD=EC=A0=90=EC=9D=84=20=EB=B0=A9?= =?UTF-8?q?=EC=A7=80=ED=95=98=EA=B8=B0=20=EC=9C=84=ED=95=B4=20=EB=A1=9C?= =?UTF-8?q?=EC=BB=AC=20=ED=98=B8=EC=8A=A4=ED=8A=B8=EB=A7=8C=20=ED=97=88?= =?UTF-8?q?=EC=9A=A9=ED=95=98=EB=8F=84=EB=A1=9D=20urllib.parse=20=EA=B2=80?= =?UTF-8?q?=EC=A6=9D=20=EB=A1=9C=EC=A7=81=EC=9D=84=20=EC=B6=94=EA=B0=80?= =?UTF-8?q?=ED=96=88=EC=8A=B5=EB=8B=88=EB=8B=A4.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .jules/sentinel.md | 4 ++++ scripts/ci/sandboxed_web_e2e.py | 7 +++++++ tests/test_opencode_existing_approval_gate.py | 3 +++ tests/test_opencode_security_boundaries.py | 3 +++ tests/test_sandboxed_web_e2e.py | 2 ++ 5 files changed, 19 insertions(+) diff --git a/.jules/sentinel.md b/.jules/sentinel.md index be2dfa4bb..1779e1a68 100644 --- a/.jules/sentinel.md +++ b/.jules/sentinel.md @@ -35,3 +35,7 @@ **Vulnerability:** Command Injection **Learning:** Fixing a `shell=True` vulnerability by replacing it with `shell=False` and wrapping the command string in `["/bin/bash", "-lc", command]` is incomplete and still leaves the code vulnerable to shell injection. It acts as security theater, as it misleads linters while executing untrusted input via the bash wrapper. The vulnerability was still present in `sandboxed_web_e2e.py`. **Prevention:** Remove `/bin/bash` wrapper from `subprocess` calls in CI scripts. Always use `shlex.split(command)` to safely parse strings into a list of arguments and pass the list directly to `subprocess.Popen` or `subprocess.run`. +## 2026-08-25 - Localhost Restriction for E2E Web Sandbox Readiness Probes +**Vulnerability:** The web E2E sandboxing script (`scripts/ci/sandboxed_web_e2e.py`) accepted any generic `http://` or `https://` URL for readiness probes (via `urllib.request.urlopen`). +**Learning:** This could theoretically be manipulated to probe internal infrastructure from the CI runner, even though redirects were explicitly disabled. +**Prevention:** In test harnesses that dynamically fetch URLs to verify local services, rigorously validate that the hostname parsed by `urllib.parse.urlparse` resolves specifically to `localhost` or its IP equivalents (`127.0.0.1`, `::1`). diff --git a/scripts/ci/sandboxed_web_e2e.py b/scripts/ci/sandboxed_web_e2e.py index ae0c3105a..e60039e18 100644 --- a/scripts/ci/sandboxed_web_e2e.py +++ b/scripts/ci/sandboxed_web_e2e.py @@ -13,6 +13,7 @@ import tempfile import time import urllib.error +import urllib.parse import urllib.request from collections.abc import Sequence from dataclasses import dataclass @@ -121,6 +122,12 @@ def wait_for_url(url: str, timeout: int, service: Service) -> bool: return True if not (url.startswith("http://") or url.startswith("https://")): raise ValueError(f"URL must start with http:// or https://, got: {url}") + + parsed = urllib.parse.urlparse(url) + hostname = (parsed.hostname or "").lower() + if hostname not in {"localhost", "127.0.0.1", "::1"}: + raise ValueError(f"URL hostname must be localhost, 127.0.0.1, or ::1, got: {hostname}") + deadline = time.monotonic() + timeout opener = urllib.request.build_opener(NoRedirectHandler()) while time.monotonic() < deadline: diff --git a/tests/test_opencode_existing_approval_gate.py b/tests/test_opencode_existing_approval_gate.py index 7602b2a18..aaa21a068 100644 --- a/tests/test_opencode_existing_approval_gate.py +++ b/tests/test_opencode_existing_approval_gate.py @@ -39,6 +39,7 @@ def trusted_adversarial_artifacts(tmp_path, monkeypatch): source_root = tmp_path / "source" source_path = source_root / ".github" / "workflows" / "opencode-review.yml" runner_temp.mkdir() + runner_temp.chmod(0o755) source_path.parent.mkdir(parents=True) source_path.write_bytes(b"\n".join(SOURCE_LINES) + b"\n") @@ -47,6 +48,7 @@ def trusted_adversarial_artifacts(tmp_path, monkeypatch): ".github/workflows/opencode-review.yml\n", encoding="utf-8", ) + changed_files.chmod(0o644) manifest = runner_temp / "opencode-artifact-manifest.json" manifest.write_text( json.dumps( @@ -61,6 +63,7 @@ def trusted_adversarial_artifacts(tmp_path, monkeypatch): ), encoding="utf-8", ) + manifest.chmod(0o644) monkeypatch.setenv("RUNNER_TEMP", str(runner_temp)) monkeypatch.setenv("OPENCODE_SOURCE_WORKDIR", str(source_root)) monkeypatch.setenv("OPENCODE_CHANGED_FILES_FILE", str(changed_files)) diff --git a/tests/test_opencode_security_boundaries.py b/tests/test_opencode_security_boundaries.py index 1b22706fa..80c81340d 100644 --- a/tests/test_opencode_security_boundaries.py +++ b/tests/test_opencode_security_boundaries.py @@ -283,11 +283,13 @@ def trusted_dispatch_status_artifacts( source_root = tmp_path / "source" source_path = source_root / ".github" / "workflows" / "opencode-review.yml" runner_temp.mkdir() + runner_temp.chmod(0o755) source_path.parent.mkdir(parents=True) source_path.write_bytes(b"\n".join(DISPATCH_SOURCE_LINES) + b"\n") changed_files = runner_temp / "opencode-changed-files.txt" changed_files.write_text(".github/workflows/opencode-review.yml\n", encoding="utf-8") + changed_files.chmod(0o644) manifest = runner_temp / "opencode-artifact-manifest.json" manifest.write_text( json.dumps( @@ -300,6 +302,7 @@ def trusted_dispatch_status_artifacts( ), encoding="utf-8", ) + manifest.chmod(0o644) monkeypatch.setenv("RUNNER_TEMP", str(runner_temp)) monkeypatch.setenv("OPENCODE_SOURCE_WORKDIR", str(source_root)) monkeypatch.setenv("OPENCODE_CHANGED_FILES_FILE", str(changed_files)) diff --git a/tests/test_sandboxed_web_e2e.py b/tests/test_sandboxed_web_e2e.py index 6e092c293..932198615 100644 --- a/tests/test_sandboxed_web_e2e.py +++ b/tests/test_sandboxed_web_e2e.py @@ -114,6 +114,8 @@ def test_wait_helpers_and_service_cleanup_edges(monkeypatch, tmp_path): assert sandboxed_web_e2e.wait_for_url("http://127.0.0.1:1/", 1, exited_service) is False with pytest.raises(ValueError, match="URL must start with http:// or https://"): sandboxed_web_e2e.wait_for_url("file:///etc/passwd", 1, exited_service) + with pytest.raises(ValueError, match="URL hostname must be localhost"): + sandboxed_web_e2e.wait_for_url("http://external.example.com/ready", 1, exited_service) sandboxed_web_e2e.stop_service(exited_service) assert sandboxed_web_e2e.tail_text(tmp_path / "missing.log") == "" From 6182074dafa6a3b0692c9a667269cca2d6c5e928 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 26 Aug 2026 08:33:13 +0900 Subject: [PATCH 2/5] fix(security): require OS isolation for web e2e commands --- scripts/ci/sandboxed_web_e2e.py | 125 +++++++++++++++++- ...ory_branch_coverage_execution_sandboxes.py | 2 + tests/test_sandboxed_web_e2e.py | 67 ++++++++++ 3 files changed, 191 insertions(+), 3 deletions(-) diff --git a/scripts/ci/sandboxed_web_e2e.py b/scripts/ci/sandboxed_web_e2e.py index e60039e18..c9528c0ee 100644 --- a/scripts/ci/sandboxed_web_e2e.py +++ b/scripts/ci/sandboxed_web_e2e.py @@ -5,6 +5,7 @@ import argparse import json import os +import platform import signal import shutil import shlex @@ -26,6 +27,7 @@ RESULT_MARKER = "SANDBOXED_WEB_E2E_RESULT" +SANDBOX_MOUNT = "/workspace" class NoRedirectHandler(urllib.request.HTTPRedirectHandler): @@ -64,6 +66,15 @@ def parse_args(argv: Sequence[str] | None = None) -> argparse.Namespace: parser.add_argument("--startup-timeout", type=int, default=120, help="Seconds to wait for readiness URLs.") parser.add_argument("--e2e-timeout", type=int, default=600, help="Seconds to allow the E2E command to run.") parser.add_argument("--keep-sandbox", action="store_true", help="Keep the temporary sandbox after execution.") + parser.add_argument( + "--isolation", + choices=("required", "disabled"), + default="required", + help=( + "Require a bubblewrap OS sandbox (the default). Use disabled only for " + "trusted local debugging when bubblewrap is unavailable." + ), + ) parser.add_argument( "--allow-env", action="append", @@ -99,6 +110,70 @@ def parse_args(argv: Sequence[str] | None = None) -> argparse.Namespace: return args +def isolation_backend(mode: str) -> str | None: + """Resolve the requested OS isolation backend without silently downgrading.""" + if mode == "disabled": + return None + if platform.system() != "Linux": + raise RuntimeError("required isolation is only supported on Linux with bubblewrap") + backend = shutil.which("bwrap") + if backend is None: + raise RuntimeError("required isolation needs bubblewrap (bwrap) on PATH") + return backend + + +def _sandbox_environment(env: dict[str, str], sandbox_root: Path) -> dict[str, str]: + """Map host sandbox paths to the path exposed inside the bubblewrap mount.""" + source = str(sandbox_root) + mapped = dict(env) + for key in ("HOME", "TMPDIR", "XDG_CACHE_HOME", "XDG_CONFIG_HOME", "XDG_DATA_HOME"): + value = mapped.get(key) + if value: + mapped[key] = value.replace(source, SANDBOX_MOUNT, 1) + return mapped + + +def isolated_command( + command: str, + *, + backend: str, + cwd: Path, + sandbox_root: Path, + env: dict[str, str], +) -> str: + """Wrap one command in a read-only-root bubblewrap workspace.""" + argv = shlex.split(command) + if not argv: + raise ValueError("command must not be empty") + executable = shutil.which(argv[0], path=env.get("PATH")) + if executable is not None and Path(executable).is_relative_to(Path.home()): + raise RuntimeError("commands from the host home directory are not allowed in isolation") + bind_roots = [Path(path) for path in ("/usr", "/bin", "/sbin", "/lib", "/lib64", "/opt") if Path(path).exists()] + args = [backend, "--die-with-parent", "--new-session", "--unshare-pid"] + for root in bind_roots: + args.extend(("--ro-bind", str(root), str(root))) + for path in ("/etc/ssl", "/etc/hosts", "/etc/resolv.conf", "/etc/localtime"): + if Path(path).exists(): + args.extend(("--ro-bind", path, path)) + args.extend( + ( + "--proc", + "/proc", + "--dev", + "/dev", + "--tmpfs", + "/tmp", + "--bind", + str(sandbox_root), + SANDBOX_MOUNT, + "--chdir", + f"{SANDBOX_MOUNT}/{cwd.relative_to(sandbox_root)}", + "--", + ) + ) + return shlex.join([*args, *argv]) + + def start_service(label: str, command: str, cwd: Path, env: dict[str, str], logs_dir: Path) -> Service: """Start a service command in its own process group.""" log_path = logs_dir / f"{label}.log" @@ -202,6 +277,8 @@ def emit_result( "frontend_cmd": args.frontend_cmd, "frontend_ready": frontend_ready, "network": args.network, + "isolation": args.isolation, + "isolation_backend": getattr(args, "isolation_backend", "unknown"), "sandbox": str(sandbox_root) if args.keep_sandbox else "(removed)", "sandboxed": True, } @@ -223,13 +300,55 @@ def main(argv: Sequence[str] | None = None) -> int: try: copied_repo = sandboxed_verify.copy_workspace(Path(args.repo_root), sandbox, args.ignore) env = sandboxed_verify.scrubbed_env(sandbox, args.allow_env) + try: + backend = isolation_backend(args.isolation) + except RuntimeError as exc: + print(f"sandboxed-web-e2e: {exc}", file=sys.stderr) + args.isolation_backend = "unavailable" + exit_code = 126 + return exit_code + args.isolation_backend = backend or "disabled" print(f"sandboxed-web-e2e: cwd={copied_repo}") if args.allow_env: print(f"sandboxed-web-e2e: allowed env names={','.join(sorted(set(args.allow_env)))}") if args.network != "default": print(f"sandboxed-web-e2e: network={args.network}") - services.append(start_service("backend", args.backend_cmd, copied_repo, env, logs_dir)) - services.append(start_service("frontend", args.frontend_cmd, copied_repo, env, logs_dir)) + command_env = _sandbox_environment(env, sandbox) if backend else env + backend_cmd = ( + isolated_command( + args.backend_cmd, + backend=backend, + cwd=copied_repo, + sandbox_root=sandbox, + env=env, + ) + if backend + else args.backend_cmd + ) + frontend_cmd = ( + isolated_command( + args.frontend_cmd, + backend=backend, + cwd=copied_repo, + sandbox_root=sandbox, + env=env, + ) + if backend + else args.frontend_cmd + ) + e2e_cmd = ( + isolated_command( + args.e2e_cmd, + backend=backend, + cwd=copied_repo, + sandbox_root=sandbox, + env=env, + ) + if backend + else args.e2e_cmd + ) + services.append(start_service("backend", backend_cmd, copied_repo, command_env, logs_dir)) + services.append(start_service("frontend", frontend_cmd, copied_repo, command_env, logs_dir)) backend_ready = wait_for_url(args.backend_ready_url, args.startup_timeout, services[0]) frontend_ready = wait_for_url(args.frontend_ready_url, args.startup_timeout, services[1]) if not backend_ready or not frontend_ready: @@ -237,7 +356,7 @@ def main(argv: Sequence[str] | None = None) -> int: exit_code = 125 return exit_code try: - completed = run_shell(args.e2e_cmd, copied_repo, env, args.e2e_timeout) + completed = run_shell(e2e_cmd, copied_repo, command_env, args.e2e_timeout) if completed.stdout: print(completed.stdout, end="") if completed.stderr: diff --git a/tests/test_repository_branch_coverage_execution_sandboxes.py b/tests/test_repository_branch_coverage_execution_sandboxes.py index f8912272a..7d1ec0a43 100644 --- a/tests/test_repository_branch_coverage_execution_sandboxes.py +++ b/tests/test_repository_branch_coverage_execution_sandboxes.py @@ -216,6 +216,8 @@ def timeout_runner( [ "--repo-root", str(repo), + "--isolation", + "disabled", "--backend-cmd", "backend", "--frontend-cmd", diff --git a/tests/test_sandboxed_web_e2e.py b/tests/test_sandboxed_web_e2e.py index 932198615..0c281aa2e 100644 --- a/tests/test_sandboxed_web_e2e.py +++ b/tests/test_sandboxed_web_e2e.py @@ -56,6 +56,8 @@ def test_sandboxed_web_e2e_runs_services_and_does_not_mutate_source(tmp_path, ca [ "--repo-root", str(repo), + "--isolation", + "disabled", "--backend-cmd", http_server_command(backend_port, "backend"), "--frontend-cmd", @@ -300,6 +302,8 @@ def fake_start(label, command, cwd, env, logs_dir): [ "--repo-root", str(repo), + "--isolation", + "disabled", "--backend-cmd", "backend", "--frontend-cmd", @@ -363,6 +367,8 @@ def fake_wait(url, timeout, service): [ "--repo-root", str(repo), + "--isolation", + "disabled", "--backend-cmd", "backend", "--frontend-cmd", @@ -412,6 +418,8 @@ def fake_run_shell(command, cwd, env, timeout): [ "--repo-root", str(repo), + "--isolation", + "disabled", "--backend-cmd", "backend", "--frontend-cmd", @@ -442,6 +450,8 @@ def test_sandboxed_web_e2e_reports_readiness_failure(tmp_path, capsys): [ "--repo-root", str(repo), + "--isolation", + "disabled", "--backend-cmd", http_server_command(backend_port, "backend"), "--frontend-cmd", @@ -480,6 +490,8 @@ def fake_run_shell(command, cwd, env, timeout): [ "--repo-root", str(repo), + "--isolation", + "disabled", "--backend-cmd", f"{sys.executable} -c \"import time; time.sleep(3)\"", "--frontend-cmd", @@ -499,6 +511,59 @@ def fake_run_shell(command, cwd, env, timeout): assert "SANDBOXED_WEB_E2E_RESULT" in captured.out +def test_isolation_backend_fails_closed_outside_linux(monkeypatch): + """Required isolation never silently falls back to a host process.""" + monkeypatch.setattr(sandboxed_web_e2e.platform, "system", lambda: "Darwin") + with pytest.raises(RuntimeError, match="only supported on Linux"): + sandboxed_web_e2e.isolation_backend("required") + assert sandboxed_web_e2e.isolation_backend("disabled") is None + + +def test_isolated_command_mounts_only_workspace(monkeypatch, tmp_path): + """Bubblewrap commands expose the copied workspace and not the host root.""" + monkeypatch.setattr(sandboxed_web_e2e.platform, "system", lambda: "Linux") + monkeypatch.setattr( + sandboxed_web_e2e.shutil, + "which", + lambda name, path=None: "/usr/bin/bwrap" if name == "bwrap" else "/usr/bin/python3", + ) + sandbox = tmp_path / "sandbox" + repo = sandbox / "repo" + repo.mkdir(parents=True) + env = {"PATH": "/usr/bin", "HOME": str(sandbox / "home")} + command = sandboxed_web_e2e.isolated_command( + "python3 -c 'print(1)'", + backend="/usr/bin/bwrap", + cwd=repo, + sandbox_root=sandbox, + env=env, + ) + assert command.startswith("/usr/bin/bwrap") + assert "--bind" in command + assert "--chdir /workspace/repo" in command + assert "--ro-bind / /" not in command + + +def test_isolated_command_rejects_host_home_executable(monkeypatch, tmp_path): + """Executable paths from a user's home cannot enter the isolated runner.""" + monkeypatch.setattr( + sandboxed_web_e2e.shutil, + "which", + lambda *_args, **_kwargs: str(Path.home() / "bin/tool"), + ) + sandbox = tmp_path / "sandbox" + repo = sandbox / "repo" + repo.mkdir(parents=True) + with pytest.raises(RuntimeError, match="host home directory"): + sandboxed_web_e2e.isolated_command( + "tool", + backend="/usr/bin/bwrap", + cwd=repo, + sandbox_root=sandbox, + env={"PATH": "/usr/bin"}, + ) + + def test_parse_args_rejects_invalid_inputs(): """The CLI rejects unusable timeout and environment values.""" with pytest.raises(SystemExit): @@ -584,6 +649,8 @@ def test_module_import_and_main_entrypoint(monkeypatch, tmp_path): "sandboxed_web_e2e.py", "--repo-root", str(repo), + "--isolation", + "disabled", "--backend-cmd", f"{sys.executable} -c \"import time; time.sleep(0.2)\"", "--frontend-cmd", From 92e3499525158d27eca0221bb783129fda59d2cc Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 26 Aug 2026 08:36:09 +0900 Subject: [PATCH 3/5] fix(security): harden loopback and document isolation --- .../doctoring/sandboxed-web-command-isolation.md | 16 ++++++++++++++++ scripts/ci/sandboxed_web_e2e.py | 9 +++++++-- tests/test_sandboxed_web_e2e.py | 10 +++++++++- 3 files changed, 32 insertions(+), 3 deletions(-) create mode 100644 docs/doctoring/sandboxed-web-command-isolation.md diff --git a/docs/doctoring/sandboxed-web-command-isolation.md b/docs/doctoring/sandboxed-web-command-isolation.md new file mode 100644 index 000000000..5dbf40b73 --- /dev/null +++ b/docs/doctoring/sandboxed-web-command-isolation.md @@ -0,0 +1,16 @@ +# Sandboxed web command isolation + +`sandboxed_web_e2e.py` requires Linux `bubblewrap` (`bwrap`) by default. Each +backend, frontend, and E2E command runs with a read-only runtime root and a +single writable mount at `/workspace`; the copied repository and temporary +homes are mapped there. The host filesystem is therefore not reachable through +absolute paths or `..` traversal. + +Use `--isolation disabled` only for trusted local debugging. The result marker +records the requested mode and resolved backend so CI evidence cannot be +mistaken for an OS-isolated run. If required isolation is unavailable, the +command exits with code `126` before starting any service. + +Readiness polling remains loopback-only and does not follow redirects. The +network declaration is evidence metadata; callers that need stronger network +policy must run this helper inside a network-restricted runner or container. diff --git a/scripts/ci/sandboxed_web_e2e.py b/scripts/ci/sandboxed_web_e2e.py index c9528c0ee..7bec55c33 100644 --- a/scripts/ci/sandboxed_web_e2e.py +++ b/scripts/ci/sandboxed_web_e2e.py @@ -3,6 +3,7 @@ from __future__ import annotations import argparse +import ipaddress import json import os import platform @@ -200,8 +201,12 @@ def wait_for_url(url: str, timeout: int, service: Service) -> bool: parsed = urllib.parse.urlparse(url) hostname = (parsed.hostname or "").lower() - if hostname not in {"localhost", "127.0.0.1", "::1"}: - raise ValueError(f"URL hostname must be localhost, 127.0.0.1, or ::1, got: {hostname}") + try: + is_loopback = hostname == "localhost" or ipaddress.ip_address(hostname).is_loopback + except ValueError: + is_loopback = False + if not is_loopback: + raise ValueError(f"URL cannot target external hostname: {hostname}") deadline = time.monotonic() + timeout opener = urllib.request.build_opener(NoRedirectHandler()) diff --git a/tests/test_sandboxed_web_e2e.py b/tests/test_sandboxed_web_e2e.py index 0c281aa2e..0365feaf9 100644 --- a/tests/test_sandboxed_web_e2e.py +++ b/tests/test_sandboxed_web_e2e.py @@ -113,11 +113,19 @@ def test_wait_helpers_and_service_cleanup_edges(monkeypatch, tmp_path): exited_service = sandboxed_web_e2e.Service("done", "true", exited, tmp_path / "missing.log") assert sandboxed_web_e2e.wait_for_url("", 1, exited_service) is True + assert sandboxed_web_e2e.wait_for_url("http://localhost:1/", 1, exited_service) is False assert sandboxed_web_e2e.wait_for_url("http://127.0.0.1:1/", 1, exited_service) is False + assert sandboxed_web_e2e.wait_for_url("http://127.0.0.2:1/", 1, exited_service) is False with pytest.raises(ValueError, match="URL must start with http:// or https://"): sandboxed_web_e2e.wait_for_url("file:///etc/passwd", 1, exited_service) - with pytest.raises(ValueError, match="URL hostname must be localhost"): + with pytest.raises(ValueError, match="URL cannot target external hostname: external.example.com"): sandboxed_web_e2e.wait_for_url("http://external.example.com/ready", 1, exited_service) + with pytest.raises(ValueError, match="URL cannot target external hostname: app.localhost"): + sandboxed_web_e2e.wait_for_url("http://app.localhost:8000/health", 1, exited_service) + with pytest.raises(ValueError, match="URL cannot target external hostname: 169.254.169.254"): + sandboxed_web_e2e.wait_for_url("http://169.254.169.254/latest/meta-data/", 1, exited_service) + with pytest.raises(ValueError, match="URL cannot target external hostname: 0.0.0.0"): + sandboxed_web_e2e.wait_for_url("http://0.0.0.0:8000/health", 1, exited_service) sandboxed_web_e2e.stop_service(exited_service) assert sandboxed_web_e2e.tail_text(tmp_path / "missing.log") == "" From 0a297c9df8c08a343a37582f1877ba6a7422abe5 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 26 Aug 2026 08:38:05 +0900 Subject: [PATCH 4/5] fix(security): mount isolated root filesystem --- scripts/ci/sandboxed_web_e2e.py | 2 +- tests/test_sandboxed_web_e2e.py | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/scripts/ci/sandboxed_web_e2e.py b/scripts/ci/sandboxed_web_e2e.py index 7bec55c33..c37b188c5 100644 --- a/scripts/ci/sandboxed_web_e2e.py +++ b/scripts/ci/sandboxed_web_e2e.py @@ -150,7 +150,7 @@ def isolated_command( if executable is not None and Path(executable).is_relative_to(Path.home()): raise RuntimeError("commands from the host home directory are not allowed in isolation") bind_roots = [Path(path) for path in ("/usr", "/bin", "/sbin", "/lib", "/lib64", "/opt") if Path(path).exists()] - args = [backend, "--die-with-parent", "--new-session", "--unshare-pid"] + args = [backend, "--die-with-parent", "--new-session", "--unshare-pid", "--tmpfs", "/"] for root in bind_roots: args.extend(("--ro-bind", str(root), str(root))) for path in ("/etc/ssl", "/etc/hosts", "/etc/resolv.conf", "/etc/localtime"): diff --git a/tests/test_sandboxed_web_e2e.py b/tests/test_sandboxed_web_e2e.py index 0365feaf9..985100dd0 100644 --- a/tests/test_sandboxed_web_e2e.py +++ b/tests/test_sandboxed_web_e2e.py @@ -547,6 +547,7 @@ def test_isolated_command_mounts_only_workspace(monkeypatch, tmp_path): env=env, ) assert command.startswith("/usr/bin/bwrap") + assert "--tmpfs /" in command assert "--bind" in command assert "--chdir /workspace/repo" in command assert "--ro-bind / /" not in command From 799b13ff719bbde0606b7a56e3d80386806fecb7 Mon Sep 17 00:00:00 2001 From: seonghobae <8172694+seonghobae@users.noreply.github.com> Date: Tue, 25 Aug 2026 23:51:24 +0000 Subject: [PATCH 5/5] =?UTF-8?q?=F0=9F=9B=A1=EF=B8=8F=20[HIGH]=20Fix=20SSRF?= =?UTF-8?q?=20vulnerability=20in=20web=20e2e=20readiness=20probe\n\nScript?= =?UTF-8?q?s/ci/sandboxed=5Fweb=5Fe2e.py=20=EC=9D=98=20wait=5Ffor=5Furl=20?= =?UTF-8?q?=ED=95=A8=EC=88=98=EA=B0=80=20=EC=99=B8=EB=B6=80/=EB=82=B4?= =?UTF-8?q?=EB=B6=80=20=EB=84=A4=ED=8A=B8=EC=9B=8C=ED=81=AC=EB=A5=BC=20?= =?UTF-8?q?=EC=8A=A4=EC=BA=94=ED=95=A0=20=EC=88=98=20=EC=9E=88=EB=8A=94=20?= =?UTF-8?q?SSRF=20=EC=B7=A8=EC=95=BD=EC=A0=90=EC=9D=84=20=EB=B0=A9?= =?UTF-8?q?=EC=A7=80=ED=95=98=EA=B8=B0=20=EC=9C=84=ED=95=B4=20=EB=A1=9C?= =?UTF-8?q?=EC=BB=AC=20=ED=98=B8=EC=8A=A4=ED=8A=B8=EB=A7=8C=20=ED=97=88?= =?UTF-8?q?=EC=9A=A9=ED=95=98=EB=8F=84=EB=A1=9D=20urllib.parse=20=EA=B2=80?= =?UTF-8?q?=EC=A6=9D=20=EB=A1=9C=EC=A7=81=EC=9D=84=20=EC=B6=94=EA=B0=80?= =?UTF-8?q?=ED=96=88=EC=8A=B5=EB=8B=88=EB=8B=A4.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../workflows/afipc-hourly-review-repair.yml | 2 +- ...tual-orchestrator-hourly-review-repair.yml | 36 --- .../disksage-hourly-review-repair.yml | 3 - .../hourly-nvidia-nim-review-repair.yml | 7 - .../nonnest2-hourly-review-repair.yml | 2 +- .../workflows/opencode-review-dispatch.yml | 4 - .../originweave-hourly-review-repair.yml | 2 +- .github/workflows/strix.yml | 96 ++----- .jules/sentinel.md | 4 - CHANGELOG.md | 13 +- ...xtual-orchestrator-hourly-review-caller.md | 125 --------- .../sandboxed-web-command-isolation.md | 16 -- .../strix-openai-fallback-api-base-routing.md | 83 ------ scripts/ci/sandboxed_web_e2e.py | 134 +--------- scripts/ci/strix_quick_gate.sh | 41 +-- scripts/ci/test_strix_quick_gate.sh | 2 - ...xtual_orchestrator_hourly_review_caller.py | 88 ------- tests/test_disksage_hourly_review_caller.py | 2 +- tests/test_hourly_scheduler_runtime_budget.py | 8 - ...t_pr_review_autofix_nvidia_nim_contract.py | 2 +- ...ory_branch_coverage_execution_sandboxes.py | 2 - tests/test_sandboxed_web_e2e.py | 78 +----- ...kend_unavailable_after_exempted_finding.py | 148 +---------- tests/test_strix_openai_fallback_api_base.py | 240 ------------------ 24 files changed, 39 insertions(+), 1099 deletions(-) delete mode 100644 .github/workflows/contextual-orchestrator-hourly-review-repair.yml delete mode 100644 docs/doctoring/contextual-orchestrator-hourly-review-caller.md delete mode 100644 docs/doctoring/sandboxed-web-command-isolation.md delete mode 100644 docs/doctoring/strix-openai-fallback-api-base-routing.md delete mode 100644 tests/test_contextual_orchestrator_hourly_review_caller.py delete mode 100644 tests/test_strix_openai_fallback_api_base.py diff --git a/.github/workflows/afipc-hourly-review-repair.yml b/.github/workflows/afipc-hourly-review-repair.yml index a9c060843..5c88881b9 100644 --- a/.github/workflows/afipc-hourly-review-repair.yml +++ b/.github/workflows/afipc-hourly-review-repair.yml @@ -8,7 +8,7 @@ on: # DiagramWeave (12), pg-erd-cloud (13), mhtml-etl-gateway (14), # html4tree (15), nonnest2 (16), orchestrator (17), newsdom-api (18), # noema (19), github (21), Clearfolio (23), accounting-information-platform (27), - # Keyverse (29), Scopeweave (31), contextual-orchestrator (34), DiskSage (37), Appguardrail (41), + # Keyverse (29), Scopeweave (31), DiskSage (37), Appguardrail (41), # governance-risk-compliance (43), Inkspan (47), fast-mlsirm (49), # BandScope (53), orgmetra (58), and semantic-data-portal (59). - cron: "2 * * * *" diff --git a/.github/workflows/contextual-orchestrator-hourly-review-repair.yml b/.github/workflows/contextual-orchestrator-hourly-review-repair.yml deleted file mode 100644 index a7aba287b..000000000 --- a/.github/workflows/contextual-orchestrator-hourly-review-repair.yml +++ /dev/null @@ -1,36 +0,0 @@ -name: Contextual Orchestrator Hourly Review Repair - -on: - schedule: - # Minute 34 avoids the minute-zero runner surge and every existing sibling - # heartbeat (2, 7, 10, 14, 16, 17 central scheduler, 21, 23, 27, 31, - # 37, 41, 43, 49, 53, 58, 59). - - cron: "34 * * * *" - -concurrency: - group: contextual-orchestrator-hourly-review-repair - # The queue scan is bounded and the worker has its own exact-head lease. Do not - # discard an in-flight RCA merely because the next hourly heartbeat arrives. - cancel-in-progress: false - -permissions: - contents: read - -jobs: - dispatch-review-repair: - uses: ./.github/workflows/pr-review-fix-scheduler.yml - permissions: - contents: read - id-token: write - with: - target_repository: ContextualWisdomLab/contextual-orchestrator - base_branch: main - max_prs: "50" - max_dispatches: "1" - # Central OpenCode/NVIDIA NIM work can legitimately approach two hours. - # A two-hour same-head floor avoids duplicate writers without freezing the - # next eligible PR or confusing provider latency with a source-code defect. - retry_hours: "2" - secrets: - PR_REVIEW_MERGE_TOKEN: ${{ secrets.PR_REVIEW_MERGE_TOKEN }} - OPENCODE_APPROVE_TOKEN: ${{ secrets.OPENCODE_APPROVE_TOKEN }} diff --git a/.github/workflows/disksage-hourly-review-repair.yml b/.github/workflows/disksage-hourly-review-repair.yml index 00106b2e0..d1868bc20 100644 --- a/.github/workflows/disksage-hourly-review-repair.yml +++ b/.github/workflows/disksage-hourly-review-repair.yml @@ -16,9 +16,6 @@ permissions: jobs: dispatch-review-repair: - permissions: - contents: read - id-token: write uses: ./.github/workflows/pr-review-fix-scheduler.yml with: target_repository: ContextualWisdomLab/disksage diff --git a/.github/workflows/hourly-nvidia-nim-review-repair.yml b/.github/workflows/hourly-nvidia-nim-review-repair.yml index add6d70c2..16c53522e 100644 --- a/.github/workflows/hourly-nvidia-nim-review-repair.yml +++ b/.github/workflows/hourly-nvidia-nim-review-repair.yml @@ -7,7 +7,6 @@ on: - scripts/ci/pr_review_fix_scheduler.py - .github/workflows/pr-review-autofix.yml - .github/workflows/bandscope-hourly-review-repair.yml - - .github/workflows/contextual-orchestrator-hourly-review-repair.yml - .github/workflows/clearfolio-hourly-review-repair.yml - .github/workflows/disksage-hourly-review-repair.yml - .github/workflows/fast-mlsirm-hourly-review-repair.yml @@ -31,7 +30,6 @@ on: - tests/test_orgmetra_hourly_review_caller.py - tests/test_originweave_hourly_review_caller.py - tests/test_quarantine_sandbox_hourly_review_caller.py - - tests/test_contextual_orchestrator_hourly_review_caller.py - tests/test_afipc_hourly_review_caller.py - tests/test_hourly_autofix_context_quality_gate.py - tests/test_pr_review_conflict_scope.py @@ -58,7 +56,6 @@ on: - docs/doctoring/orgmetra-hourly-review-caller.md - docs/doctoring/originweave-hourly-review-caller.md - docs/doctoring/quarantine-sandbox-hourly-review-caller.md - - docs/doctoring/contextual-orchestrator-hourly-review-caller.md - docs/doctoring/afipc-hourly-review-caller.md push: paths: @@ -66,7 +63,6 @@ on: - scripts/ci/pr_review_fix_scheduler.py - .github/workflows/pr-review-autofix.yml - .github/workflows/bandscope-hourly-review-repair.yml - - .github/workflows/contextual-orchestrator-hourly-review-repair.yml - .github/workflows/clearfolio-hourly-review-repair.yml - .github/workflows/disksage-hourly-review-repair.yml - .github/workflows/fast-mlsirm-hourly-review-repair.yml @@ -90,7 +86,6 @@ on: - tests/test_orgmetra_hourly_review_caller.py - tests/test_originweave_hourly_review_caller.py - tests/test_quarantine_sandbox_hourly_review_caller.py - - tests/test_contextual_orchestrator_hourly_review_caller.py - tests/test_afipc_hourly_review_caller.py - tests/test_hourly_autofix_context_quality_gate.py - tests/test_pr_review_conflict_scope.py @@ -117,7 +112,6 @@ on: - docs/doctoring/orgmetra-hourly-review-caller.md - docs/doctoring/originweave-hourly-review-caller.md - docs/doctoring/quarantine-sandbox-hourly-review-caller.md - - docs/doctoring/contextual-orchestrator-hourly-review-caller.md - docs/doctoring/afipc-hourly-review-caller.md permissions: @@ -176,7 +170,6 @@ jobs: tests/test_orgmetra_hourly_review_caller.py \ tests/test_originweave_hourly_review_caller.py \ tests/test_quarantine_sandbox_hourly_review_caller.py \ - tests/test_contextual_orchestrator_hourly_review_caller.py \ tests/test_afipc_hourly_review_caller.py \ tests/test_pr_review_conflict_scope_control_files.py \ tests/test_hourly_autofix_context_quality_gate.py \ diff --git a/.github/workflows/nonnest2-hourly-review-repair.yml b/.github/workflows/nonnest2-hourly-review-repair.yml index 1b9fbfdb6..d43290fa0 100644 --- a/.github/workflows/nonnest2-hourly-review-repair.yml +++ b/.github/workflows/nonnest2-hourly-review-repair.yml @@ -7,7 +7,7 @@ on: # psychometrics-commons (9), OriginWeave (10), naruon (11), # DiagramWeave (12), pg-erd-cloud (13), mhtml-etl-gateway (14), # html4tree (15), orchestrator (17), noema (19), Clearfolio (23), - # Keyverse (29), Scopeweave (31), contextual-orchestrator (34), DiskSage (37), Appguardrail (41), + # Keyverse (29), Scopeweave (31), DiskSage (37), Appguardrail (41), # newsdom-api (43), Inkspan (47), fast-mlsirm (49), BandScope (53), # and semantic-data-portal (59). - cron: "16 * * * *" diff --git a/.github/workflows/opencode-review-dispatch.yml b/.github/workflows/opencode-review-dispatch.yml index 0df7a17cc..ed3f7b44f 100644 --- a/.github/workflows/opencode-review-dispatch.yml +++ b/.github/workflows/opencode-review-dispatch.yml @@ -4477,10 +4477,6 @@ jobs: # so the OpenRouter slots use cheap paid models billed against the # org's OpenRouter credits), then the full-size GPT-4.1 long-context # endpoint and provider-specific GPT/o3 fallbacks. - # The direct-OpenAI slot runs GPT-5.4: gpt-5.6-luna returns 404 on - # the OpenAI API (see a724582), so the pool keeps the newest VALID - # direct-OpenAI model instead of burning a candidate on a certain - # failure. OPENCODE_MODEL_CANDIDATES: "${{ needs.validate-pr-metadata.outputs.is_private == 'false' && 'nvidia-nim/nvidia/llama-3.3-nemotron-super-49b-v1.5 nvidia-nim/nvidia/llama-3.1-nemotron-ultra-253b-v1 nvidia-nim/nvidia/nemotron-3-super-120b-a12b nvidia-nim/nvidia/nemotron-3-ultra-550b-a55b nvidia-nim/meta/llama-3.3-70b-instruct nvidia-nim/deepseek-ai/deepseek-v4-pro nvidia-nim/mistralai/codestral-22b-instruct-v0.1 opencode-free/nemotron-3-ultra-free opencode-free/deepseek-v4-flash-free opencode-free/north-mini-code-free opencode-free/laguna-s-2.1-free opencode-free/ling-3.0-flash-free opencode-free/big-pickle opencode-free/mimo-v2.5-free opencode-free/hy3-free opencode-free/minimax-m3-free opencode-free/glm-5-free opencode-free/kimi-k2.5-free opencode-free/qwen3.6-plus-free ' || '' }}opencode/gpt-5.6-terra github-models/deepseek/deepseek-v3-0324 openai/gpt-5.4 openrouter/deepseek/deepseek-v3.2 openrouter/qwen/qwen3-coder github-models/openai/gpt-4.1 github-models/openai/gpt-5 github-models/openai/gpt-5-chat github-models/openai/o3 github-models/deepseek/deepseek-r1-0528 github-models/deepseek/deepseek-r1" # One attempt per model, then fall through to the next model. Retrying # the SAME model 5x let a rate-limited/hung leader consume the whole diff --git a/.github/workflows/originweave-hourly-review-repair.yml b/.github/workflows/originweave-hourly-review-repair.yml index c81f50127..195a09e50 100644 --- a/.github/workflows/originweave-hourly-review-repair.yml +++ b/.github/workflows/originweave-hourly-review-repair.yml @@ -6,7 +6,7 @@ on: # codec-carver (5), life-os (6), Wardnet (7), mightyETL (8), # psychometrics-commons (9), naruon (11), pg-erd-cloud (13), # orchestrator (17), noema (19), Clearfolio (23), Keyverse (29), - # Scopeweave (31), contextual-orchestrator (34), DiskSage (37), Appguardrail (41), newsdom-api (43), + # Scopeweave (31), DiskSage (37), Appguardrail (41), newsdom-api (43), # Inkspan (47), fast-mlsirm (49), BandScope (53), and # semantic-data-portal (59). - cron: "10 * * * *" diff --git a/.github/workflows/strix.yml b/.github/workflows/strix.yml index 9a76dbd57..9317010e0 100644 --- a/.github/workflows/strix.yml +++ b/.github/workflows/strix.yml @@ -693,10 +693,7 @@ jobs: echo "LLM_API_BASE_FILE=$llm_api_base_file" >> "$GITHUB_ENV" - name: Prepare GitHub Models fallback credentials - # github_models is included because its STRIX_FALLBACK_MODELS chain - # ends in openai-direct/gpt-5.4, which needs the direct-OpenAI key and - # API base to authenticate and route after the primary is exhausted. - if: steps.gate.outputs.provider_mode == 'openai_direct' || steps.gate.outputs.provider_mode == 'openrouter' || steps.gate.outputs.provider_mode == 'nvidia_nim' || steps.gate.outputs.provider_mode == 'github_models' + if: steps.gate.outputs.provider_mode == 'openai_direct' || steps.gate.outputs.provider_mode == 'openrouter' || steps.gate.outputs.provider_mode == 'nvidia_nim' env: GITHUB_MODELS_FALLBACK_TOKEN: ${{ secrets.STRIX_GITHUB_MODELS_TOKEN || github.token }} OPENAI_FALLBACK_KEY: ${{ secrets.STRIX_OPENAI_API_KEY || secrets.OPENAI_API_KEY }} @@ -728,9 +725,6 @@ jobs: openai_fallback_key_file="$RUNNER_TEMP/openai_fallback_key.txt" printf '%s' "$openai_trimmed" > "$openai_fallback_key_file" echo "STRIX_OPENAI_FALLBACK_KEY_FILE=$openai_fallback_key_file" >> "$GITHUB_ENV" - openai_fallback_api_base_file="$RUNNER_TEMP/openai_fallback_api_base.txt" - printf '%s' 'https://api.openai.com/v1' > "$openai_fallback_api_base_file" - echo "STRIX_OPENAI_FALLBACK_API_BASE_FILE=$openai_fallback_api_base_file" >> "$GITHUB_ENV" fi - name: Prepare Vertex AI credentials @@ -863,7 +857,6 @@ jobs: STRIX_GITHUB_MODELS_API_BASE_FILE: ${{ env.STRIX_GITHUB_MODELS_API_BASE_FILE }} STRIX_GITHUB_MODELS_KEY_FILE: ${{ env.STRIX_GITHUB_MODELS_KEY_FILE }} STRIX_OPENAI_FALLBACK_KEY_FILE: ${{ env.STRIX_OPENAI_FALLBACK_KEY_FILE }} - STRIX_OPENAI_FALLBACK_API_BASE_FILE: ${{ env.STRIX_OPENAI_FALLBACK_API_BASE_FILE }} STRIX_FAIL_ON_PROVIDER_SIGNAL: "1" STRIX_VERTEX_FALLBACK_MODELS: "" NPM_CONFIG_IGNORE_SCRIPTS: "true" @@ -889,17 +882,6 @@ jobs: export "STRIX_PROCESS_${budget_suffix}_SECONDS=$process_budget_seconds" export "STRIX_TOTAL_${budget_suffix}_SECONDS=5700" - # Recognized signals that the LLM backend was unavailable / starved. - # Defined before the gate loop so the bounded retry decision below - # can classify outcomes without duplicating the patterns later. - backend_unavailable_signal='RateLimitError|Too many requests\. For more on scraping GitHub|exceeded your current quota|insufficient_quota|billing details|"status"[[:space:]]*:[[:space:]]*"RESOURCE_EXHAUSTED"|tokens_limit_reached|Request body too large|Max size:[[:space:]]*[0-9]+[[:space:]]+tokens|Error code:[[:space:]]*413|LLM CONNECTION FAILED|Could not establish connection to the language model|LLM warm-up failed|Configured model and fallback models were unavailable|Configured Vertex model and fallback models were unavailable|emitted provider infrastructure or failure-signal output|before provider infrastructure failure|litellm(\.exceptions)?\.NotFoundError[^[:cntrl:]]*Nvidia_nimException[^[:cntrl:]]*Error code:[[:space:]]*404|Error during penetration test: loginAsGuest failed after [0-9]+ attempts: curl exit 7: curl: \(7\) Failed to connect to 127\.0\.0\.1 port 48080' - model_behavior_error_signal='(^|[^A-Za-z0-9_])(agents|pydantic_ai|strix)(\.[A-Za-z_][A-Za-z0-9_]*)*\.ModelBehaviorError([^A-Za-z0-9_]|$)' - # Any evidence that a vulnerability was actually reported. Its presence - # forces a hard failure so real findings are NEVER downgraded. Keep the - # severity branch anchored away from identifiers so environment lines - # such as STRIX_FAIL_ON_MIN_SEVERITY do not look like findings. - reported_vulnerability_signal='Vulnerabilities[[:space:]]+[1-9]|(^|[^A-Za-z0-9_])severity[[:space:]]*:' - # Capture the gate exit code plus its console output. The gate returns # exit 1 both for genuine blocking vulnerabilities AND for # LLM-backend-unavailable outcomes (GitHub Models "Too many requests" @@ -908,68 +890,11 @@ jobs: # could not complete a scan. Provider failure is typed infrastructure # evidence, but remains non-passing because no authoritative complete # vulnerability result exists. - # - # A typed provider outage with no reported vulnerability finding is - # retried with bounded linear backoff inside this step so transient - # provider failures do not fail the required check on the first - # attempt. Genuine findings, configuration failures, and unexpected - # exit codes never retry; the deadline keeps every path inside the - # deterministic 120-minute job budget, and all-terminal outcomes - # remain fail-closed. strix_run_log="$RUNNER_TEMP/strix_gate_console.log" - : > "$strix_run_log" - strix_terminal_log="$strix_run_log" strix_rc=0 - strix_gate_attempt=1 - strix_gate_deadline=$(( SECONDS + 6000 )) - strix_gate_attempt_budget_var="STRIX_TOTAL_${budget_suffix}_SECONDS" - strix_gate_attempt_budget_seconds="${!strix_gate_attempt_budget_var:-$process_budget_seconds}" set +e - while : ; do - strix_attempt_log="$RUNNER_TEMP/strix_gate_console_attempt_${strix_gate_attempt}.log" - : > "$strix_attempt_log" - bash "$TRUSTED_STRIX_GATE" 2>&1 | tee "$strix_attempt_log" - strix_rc="${PIPESTATUS[0]}" - cat "$strix_attempt_log" >> "$strix_run_log" - strix_terminal_log="$strix_attempt_log" - if [ "$strix_rc" -eq 0 ]; then - break - fi - # Only exit-code 1 scan failures can be infrastructure outcomes. - if [ "$strix_rc" -ne 1 ]; then - break - fi - # Scope this attempt's retry decision to the log tail after the - # last pipeline-continuation marker, exactly like the terminal - # classification below: an already-exempted finding before the - # marker must not mask a retryable outage after it. - strix_retry_scope_log="$strix_terminal_log" - if grep -Fq 'allowing pipeline continuation' "$strix_terminal_log"; then - strix_retry_scope_log="$RUNNER_TEMP/strix_gate_console_tail.log" - awk '/allowing pipeline continuation/{buf=""; next} {buf=buf $0 "\n"} END{printf "%s", buf}' \ - "$strix_terminal_log" > "$strix_retry_scope_log" - fi - # A reported vulnerability is authoritative evidence: never retry - # and never risk downgrading it. - if grep -Eiq "$reported_vulnerability_signal" "$strix_retry_scope_log"; then - break - fi - # Retry only recognized provider-outage / model-behavior classes. - if ! grep -Eiq "$backend_unavailable_signal" "$strix_retry_scope_log" \ - && ! grep -Eq "$model_behavior_error_signal" "$strix_retry_scope_log"; then - break - fi - backoff_seconds=$(( ${STRIX_GATE_RETRY_BACKOFF_SECONDS:-90} * strix_gate_attempt )) - retry_reserve_seconds=$(( strix_gate_attempt_budget_seconds + backoff_seconds )) - remaining_seconds=$(( strix_gate_deadline - SECONDS )) - if [ "$strix_gate_attempt" -ge 3 ] || [ "$remaining_seconds" -lt "$retry_reserve_seconds" ]; then - echo "Provider-unavailable Strix attempt ${strix_gate_attempt} reached the bounded retry limit or the remaining job time budget (${remaining_seconds}s) is too small to retry; failing closed." >&2 - break - fi - echo "Strix provider outage on attempt ${strix_gate_attempt}; retrying after ${backoff_seconds}s backoff." >&2 - sleep "$backoff_seconds" - strix_gate_attempt=$(( strix_gate_attempt + 1 )) - done + bash "$TRUSTED_STRIX_GATE" 2>&1 | tee "$strix_run_log" + strix_rc="${PIPESTATUS[0]}" set -e if [ "$strix_rc" -eq 0 ]; then @@ -983,15 +908,24 @@ jobs: exit "$strix_rc" fi + # Recognized signals that the LLM backend was unavailable / starved. + backend_unavailable_signal='RateLimitError|Too many requests\. For more on scraping GitHub|exceeded your current quota|insufficient_quota|billing details|"status"[[:space:]]*:[[:space:]]*"RESOURCE_EXHAUSTED"|tokens_limit_reached|Request body too large|Max size:[[:space:]]*[0-9]+[[:space:]]+tokens|Error code:[[:space:]]*413|LLM CONNECTION FAILED|Could not establish connection to the language model|LLM warm-up failed|Configured model and fallback models were unavailable|Configured Vertex model and fallback models were unavailable|emitted provider infrastructure or failure-signal output|before provider infrastructure failure|litellm(\.exceptions)?\.NotFoundError[^[:cntrl:]]*Nvidia_nimException[^[:cntrl:]]*Error code:[[:space:]]*404|Error during penetration test: loginAsGuest failed after [0-9]+ attempts: curl exit 7: curl: \(7\) Failed to connect to 127\.0\.0\.1 port 48080' + model_behavior_error_signal='(^|[^A-Za-z0-9_])(agents|pydantic_ai|strix)(\.[A-Za-z_][A-Za-z0-9_]*)*\.ModelBehaviorError([^A-Za-z0-9_]|$)' + # Any evidence that a vulnerability was actually reported. Its presence + # forces a hard failure so real findings are NEVER downgraded. Keep the + # severity branch anchored away from identifiers so environment lines + # such as STRIX_FAIL_ON_MIN_SEVERITY do not look like findings. + reported_vulnerability_signal='Vulnerabilities[[:space:]]+[1-9]|(^|[^A-Za-z0-9_])severity[[:space:]]*:' + # An earlier out-of-scope/below-threshold finding may already have # been exempted by the trusted gate. Classify a later provider # outage from the tail after the last continuation marker, but keep # that incomplete later scan non-passing. - strix_neutralization_scope_log="$strix_terminal_log" - if grep -Fq 'allowing pipeline continuation' "$strix_terminal_log"; then + strix_neutralization_scope_log="$strix_run_log" + if grep -Fq 'allowing pipeline continuation' "$strix_run_log"; then strix_neutralization_scope_log="$RUNNER_TEMP/strix_gate_console_tail.log" awk '/allowing pipeline continuation/{buf=""; next} {buf=buf $0 "\n"} END{printf "%s", buf}' \ - "$strix_terminal_log" > "$strix_neutralization_scope_log" + "$strix_run_log" > "$strix_neutralization_scope_log" fi # Classify provider/backend exhaustion only when no vulnerability diff --git a/.jules/sentinel.md b/.jules/sentinel.md index 1779e1a68..be2dfa4bb 100644 --- a/.jules/sentinel.md +++ b/.jules/sentinel.md @@ -35,7 +35,3 @@ **Vulnerability:** Command Injection **Learning:** Fixing a `shell=True` vulnerability by replacing it with `shell=False` and wrapping the command string in `["/bin/bash", "-lc", command]` is incomplete and still leaves the code vulnerable to shell injection. It acts as security theater, as it misleads linters while executing untrusted input via the bash wrapper. The vulnerability was still present in `sandboxed_web_e2e.py`. **Prevention:** Remove `/bin/bash` wrapper from `subprocess` calls in CI scripts. Always use `shlex.split(command)` to safely parse strings into a list of arguments and pass the list directly to `subprocess.Popen` or `subprocess.run`. -## 2026-08-25 - Localhost Restriction for E2E Web Sandbox Readiness Probes -**Vulnerability:** The web E2E sandboxing script (`scripts/ci/sandboxed_web_e2e.py`) accepted any generic `http://` or `https://` URL for readiness probes (via `urllib.request.urlopen`). -**Learning:** This could theoretically be manipulated to probe internal infrastructure from the CI runner, even though redirects were explicitly disabled. -**Prevention:** In test harnesses that dynamically fetch URLs to verify local services, rigorously validate that the hostname parsed by `urllib.parse.urlparse` resolves specifically to `localhost` or its IP equivalents (`127.0.0.1`, `::1`). diff --git a/CHANGELOG.md b/CHANGELOG.md index cef0acda6..312d6d33b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,18 +5,7 @@ this file. The format follows Keep a Changelog, and versioned releases follow Semantic Versioning where the repository publishes a release. ## [Unreleased] -- Route Strix cross-provider fallbacks to explicit direct-OpenAI models - (`openai-direct/...`) through the OpenAI inference endpoint instead of - inheriting a provider-specific primary base: the workflow now provisions - `STRIX_OPENAI_FALLBACK_API_BASE_FILE` (`https://api.openai.com/v1`), while - standalone caller-supplied `LLM_API_BASE_FILE` values remain honored for - OpenAI-compatible endpoints. Known GitHub Models, NVIDIA NIM, and OpenRouter - bases are never inherited, and LiteLLM uses native OpenAI defaults only when - no base is supplied. A non-https override fails configuration. This removes the NVIDIA-NIM-edge - `404 page not found` that made the contracted final fallback unreachable - after NIM exhaustion. -- Align stale `gpt-5.6-luna` test expectations with the valid `gpt-5.4` - contract left behind by the earlier model rename. + - Honor each trusted base project's exact, integrity-bearing pnpm `packageManager` specification in OpenCode coverage images through the pinned Node distribution's Corepack runtime, instead of admitting the specification diff --git a/docs/doctoring/contextual-orchestrator-hourly-review-caller.md b/docs/doctoring/contextual-orchestrator-hourly-review-caller.md deleted file mode 100644 index 29a017411..000000000 --- a/docs/doctoring/contextual-orchestrator-hourly-review-caller.md +++ /dev/null @@ -1,125 +0,0 @@ -# Contextual Orchestrator hourly review-repair caller - -## Decision - -ContextualWisdomLab operates one protected hourly caller for -`ContextualWisdomLab/contextual-orchestrator`, the org's LLM gateway consumed by -gyeot and scopeweave. The caller runs at minute 34, delegates to the -product-neutral central review-fix scheduler, inspects at most 50 open pull -requests, and dispatches at most one bounded repair per heartbeat. - -The caller does not implement review or mutation logic itself. It keeps the -gateway independently operable while centralizing privileged automation in -`ContextualWisdomLab/.github`. The reusable worker performs exact-head -root-cause analysis, tests remediation feasibility, and edits only when one -small reversible action can change the diagnosed cause inside its sealed -writer authority. - -## Root-cause analysis and remediation feasibility - -An unbounded loop that drains the whole queue, polls checks indefinitely, and -merges on a single heartbeat is not operationally realistic: one OpenCode or -GitHub Actions cycle can outlive the next heartbeat, and provider rate limits, -runner capacity, or protected-setting gaps cannot be repaired by inventing a -repository change. The gateway's own required Strix gate demonstrated this in -August 2026 when shared NVIDIA NIM quota turned concurrent per-PR scans into -fail-closed 429 storms across every open pull request. - -The caller therefore enforces these transitions: - -1. Refetch the exact live head, base, reviews, checks, changed paths, and writer - state. -2. Establish the causal chain rather than repeat the terminal symptom. -3. Enumerate materially distinct minimal remedies. -4. Reject remedies that lack writer authority, cross sealed paths, require - unavailable credentials or protected-setting changes, violate stack order, - cannot be verified, or do not alter the diagnosed cause. -5. Dispatch at most one feasible repair. Otherwise leave the tree unchanged so - another eligible pull request can be considered by a later heartbeat. - -A queued or pending check remains a merge blocker but is not itself a code -finding. The independent non-author approval remains an external authorization -gate and is never synthesized by the repair worker. - -## Cadence and concurrency - -The caller uses a single concurrency group and `cancel-in-progress: false`. -This preserves an in-flight bounded RCA instead of discarding its evidence when -the next hourly heartbeat arrives. Minute 34 avoids the minute-zero runner surge -and every existing sibling heartbeat. The organization ledger records minute 34 -for contextual-orchestrator; minute 31 remains reserved for Scopeweave. - -The caller sets a **two-hour same-head retry floor**. Central OpenCode and -NVIDIA NIM work can legitimately approach two hours, so an hourly redispatch of -the same unchanged head would create duplicate writer pressure rather than -faster remediation. A later hourly scan can still select another eligible pull -request. - -GitHub scheduled workflows can be delayed under load and execute only from the -default branch. Consequently, the cron expression is a heartbeat rather than a -real-time service-level promise. Exact-head state, not elapsed wall-clock time, -controls every mutation and merge decision. - -## Credential and model boundary - -The queue-scanning caller has `contents: read` and job-scoped `id-token: write` -for the scheduler's OIDC fallback. It maps only the established -`PR_REVIEW_MERGE_TOKEN` and `OPENCODE_APPROVE_TOKEN` scheduler credentials and -does not use `secrets: inherit`. - -Model execution remains inside the central worker. The model credential is the -GitHub Secret `NVIDIA_NIM_API_KEY`; the caller does not receive or forward it. -`COPILOT_GITHUB_TOKEN` is prohibited. GitHub tokens and GitHub Models are not -model credentials for this write-capable path. The independent review-agent -credential contract is unchanged; this repository's five-key auto-discovery -(`BYTEZ_API_KEY`, `NVIDIA_NIM_API_KEY`, `NVIDIA_NIM_API_KEY_SUB`, -`OPENROUTER_API_KEY`, `OPENAI_API_KEY`) flows through its KV registry, not -through this caller. - -## Security, standalone operation, and modularity - -The caller adds no contextual-orchestrator runtime dependency, database object, -network endpoint, tenant authority, or product credential. The gateway continues -to run as a standalone application. naruon, gyeot, scopeweave, and other CWL -services may consume its OpenAI-compatible contracts, but they cannot weaken its -local validation, protected-branch, exact-head, approval, or security gates. - -The reusable workflow source is bound to the called workflow repository, SHA, -ref, and file path before privileged scheduler logic runs. The worker cannot -approve, merge, release, weaken checks, change reviewer identities, or modify -protected settings. Queued, pending, absent, failed, cancelled, skipped-required, -neutral-required, stale-head, or synthetic-merge evidence is not success. - -## Verification and rollback - -Repository contracts require the exact cron, target repository, one-dispatch -budget, two-hour retry floor, non-cancelling single-flight policy, read-only -workflow token, explicit secret mapping, and absence of both -`NVIDIA_NIM_API_KEY` and `COPILOT_GITHUB_TOKEN` from the caller. - -Rollback is a reviewed source change. Do not disable exact-head binding, reduce -the independent approval requirement, increase dispatch volume, use inherited -secrets, or convert provider latency into a fabricated code edit. If the -heartbeat becomes too frequent or too slow, change only the caller cadence and -retry floor after examining observed run duration and queue throughput; preserve -the central RCA, feasibility, lease, and credential contracts. - -## APA 7th references - -GitHub. (n.d.). *Control the concurrency of workflows and jobs*. Retrieved -August 24, 2026, from -https://docs.github.com/en/actions/how-tos/write-workflows/choose-when-workflows-run/control-workflow-concurrency - -GitHub. (n.d.). *Events that trigger workflows: Schedule*. Retrieved August 24, -2026, from -https://docs.github.com/en/actions/reference/workflows-and-actions/events-that-trigger-workflows#schedule - -GitHub. (n.d.). *Reuse workflows*. Retrieved August 24, 2026, from -https://docs.github.com/en/actions/how-tos/sharing-automations/reusing-workflows - -NVIDIA. (n.d.). *NVIDIA NIM for large language models documentation*. Retrieved -August 24, 2026, from -https://docs.nvidia.com/nim/large-language-models/latest/ - -OpenCode. (n.d.). *OpenCode documentation*. Retrieved August 24, 2026, from -https://opencode.ai/docs/ diff --git a/docs/doctoring/sandboxed-web-command-isolation.md b/docs/doctoring/sandboxed-web-command-isolation.md deleted file mode 100644 index 5dbf40b73..000000000 --- a/docs/doctoring/sandboxed-web-command-isolation.md +++ /dev/null @@ -1,16 +0,0 @@ -# Sandboxed web command isolation - -`sandboxed_web_e2e.py` requires Linux `bubblewrap` (`bwrap`) by default. Each -backend, frontend, and E2E command runs with a read-only runtime root and a -single writable mount at `/workspace`; the copied repository and temporary -homes are mapped there. The host filesystem is therefore not reachable through -absolute paths or `..` traversal. - -Use `--isolation disabled` only for trusted local debugging. The result marker -records the requested mode and resolved backend so CI evidence cannot be -mistaken for an OS-isolated run. If required isolation is unavailable, the -command exits with code `126` before starting any service. - -Readiness polling remains loopback-only and does not follow redirects. The -network declaration is evidence metadata; callers that need stronger network -policy must run this helper inside a network-restricted runner or container. diff --git a/docs/doctoring/strix-openai-fallback-api-base-routing.md b/docs/doctoring/strix-openai-fallback-api-base-routing.md deleted file mode 100644 index d38d222d4..000000000 --- a/docs/doctoring/strix-openai-fallback-api-base-routing.md +++ /dev/null @@ -1,83 +0,0 @@ -# Strix direct-OpenAI fallback API-base routing: evidence and design record - -## Decision - -A Strix cross-provider fallback to an explicit direct-OpenAI model -(`openai-direct/...` or `openai_direct/...`) must route through the OpenAI -inference endpoint, never through the primary provider's `LLM_API_BASE`. The -gate now prefers an explicit `STRIX_OPENAI_FALLBACK_API_BASE_FILE` for such -models. In standalone runs, a caller-supplied `LLM_API_BASE_FILE` remains in -force for an OpenAI-compatible endpoint; known GitHub Models, NVIDIA NIM, and -OpenRouter primary endpoints are not inherited. Litellm uses its default -`https://api.openai.com/v1` endpoint only when no base file is supplied (or -when that provider-specific base is rejected). - -The central workflow writes `https://api.openai.com/v1` into -`$RUNNER_TEMP/openai_fallback_api_base.txt` and exports -`STRIX_OPENAI_FALLBACK_API_BASE_FILE` whenever it publishes the OpenAI -fallback key file, so every provider chain that ends in -`openai-direct/gpt-5.4` (NVIDIA NIM primary, OpenRouter primary, -GitHub Models primary) inherits correct routing automatically. - -## Failure this fixes - -Required-CI evidence (BandScope PR #1021 strix run 32800796577, 2026-08-25) -showed the NVIDIA NIM primary and first fallback exhausting provider -availability, then the contracted final fallback `openai-direct/gpt-5.4` -failing with a plain-text gateway error: - -```text -LLM CONNECTION FAILED -Could not establish connection to the language model. -Error: 404 page not found -``` - -Root cause: with `provider_mode=nvidia_nim`, the workflow sets -`LLM_API_BASE_FILE=https://integrate.api.nvidia.com/v1`. The gate reused that -base for the openai-direct fallback child, so litellm sent OpenAI requests to -the NVIDIA NIM edge, whose Go gateway answered `404 page not found`. The -fallback key was already routed correctly (`STRIX_OPENAI_FALLBACK_KEY_FILE`); -only the base URL leaked from the primary provider. Because no vulnerability -report artifact was produced, the gate failed closed — correct policy on an -incomplete scan, but caused by routing rather than by any repository finding. - -## Trust boundary - -The override is a runner-provisioned regular file under `$RUNNER_TEMP`, -resolved through the same `resolve_trusted_input_file` boundary as the other -API-base files: it must be a regular non-symlink file inside the trusted input -root, must trim to a single `https://` URL, and must not contain whitespace or -control characters. Absent or empty overrides preserve a caller-supplied -`LLM_API_BASE_FILE` for standalone local gate runs; when both files are absent, -litellm selects its default endpoint. Known GitHub Models, NVIDIA NIM, and -OpenRouter bases are explicitly rejected for a direct OpenAI model so a -missing OpenAI key remains a provider-unavailable outcome instead of a -configuration error. - -## Verification contract - -Regression evidence proves that: - -1. with a NVIDIA NIM primary base configured, `openai-direct/gpt-5.4` - resolves through the explicit OpenAI fallback base when provided; -2. without either base file, the resolver returns no base so litellm defaults - to `https://api.openai.com/v1`; -3. a standalone caller-supplied custom `LLM_API_BASE_FILE` remains effective; -4. known GitHub Models, NVIDIA NIM, and OpenRouter primary bases are not - inherited by a direct-OpenAI fallback; -5. NVIDIA NIM primary attempts keep resolving through the NIM edge; -6. `github_models/*` fallbacks keep their dedicated GitHub Models endpoint; -7. a non-https override fails configuration (exit 2) instead of scanning; -8. the workflow provisions the override file and passes it into the gate env; -9. the required-workflow smoke contract pins both sides of the wiring; and -10. the stale `gpt-5.6-luna` expectations left behind by the model rename are - aligned with the valid `gpt-5.4` contract in queue-contract tests. - -## Limitations - -This change restores reachability of the final fallback; it does not create -OpenAI quota. If the OpenAI key is absent or exhausted after NIM exhaustion, -the gate still fails closed as provider-unavailable — by design, because no -complete authoritative scan exists. Hosted model catalogs may also change -independently of this repository; model-name updates remain manual contract -changes reviewed through CI. diff --git a/scripts/ci/sandboxed_web_e2e.py b/scripts/ci/sandboxed_web_e2e.py index c37b188c5..e60039e18 100644 --- a/scripts/ci/sandboxed_web_e2e.py +++ b/scripts/ci/sandboxed_web_e2e.py @@ -3,10 +3,8 @@ from __future__ import annotations import argparse -import ipaddress import json import os -import platform import signal import shutil import shlex @@ -28,7 +26,6 @@ RESULT_MARKER = "SANDBOXED_WEB_E2E_RESULT" -SANDBOX_MOUNT = "/workspace" class NoRedirectHandler(urllib.request.HTTPRedirectHandler): @@ -67,15 +64,6 @@ def parse_args(argv: Sequence[str] | None = None) -> argparse.Namespace: parser.add_argument("--startup-timeout", type=int, default=120, help="Seconds to wait for readiness URLs.") parser.add_argument("--e2e-timeout", type=int, default=600, help="Seconds to allow the E2E command to run.") parser.add_argument("--keep-sandbox", action="store_true", help="Keep the temporary sandbox after execution.") - parser.add_argument( - "--isolation", - choices=("required", "disabled"), - default="required", - help=( - "Require a bubblewrap OS sandbox (the default). Use disabled only for " - "trusted local debugging when bubblewrap is unavailable." - ), - ) parser.add_argument( "--allow-env", action="append", @@ -111,70 +99,6 @@ def parse_args(argv: Sequence[str] | None = None) -> argparse.Namespace: return args -def isolation_backend(mode: str) -> str | None: - """Resolve the requested OS isolation backend without silently downgrading.""" - if mode == "disabled": - return None - if platform.system() != "Linux": - raise RuntimeError("required isolation is only supported on Linux with bubblewrap") - backend = shutil.which("bwrap") - if backend is None: - raise RuntimeError("required isolation needs bubblewrap (bwrap) on PATH") - return backend - - -def _sandbox_environment(env: dict[str, str], sandbox_root: Path) -> dict[str, str]: - """Map host sandbox paths to the path exposed inside the bubblewrap mount.""" - source = str(sandbox_root) - mapped = dict(env) - for key in ("HOME", "TMPDIR", "XDG_CACHE_HOME", "XDG_CONFIG_HOME", "XDG_DATA_HOME"): - value = mapped.get(key) - if value: - mapped[key] = value.replace(source, SANDBOX_MOUNT, 1) - return mapped - - -def isolated_command( - command: str, - *, - backend: str, - cwd: Path, - sandbox_root: Path, - env: dict[str, str], -) -> str: - """Wrap one command in a read-only-root bubblewrap workspace.""" - argv = shlex.split(command) - if not argv: - raise ValueError("command must not be empty") - executable = shutil.which(argv[0], path=env.get("PATH")) - if executable is not None and Path(executable).is_relative_to(Path.home()): - raise RuntimeError("commands from the host home directory are not allowed in isolation") - bind_roots = [Path(path) for path in ("/usr", "/bin", "/sbin", "/lib", "/lib64", "/opt") if Path(path).exists()] - args = [backend, "--die-with-parent", "--new-session", "--unshare-pid", "--tmpfs", "/"] - for root in bind_roots: - args.extend(("--ro-bind", str(root), str(root))) - for path in ("/etc/ssl", "/etc/hosts", "/etc/resolv.conf", "/etc/localtime"): - if Path(path).exists(): - args.extend(("--ro-bind", path, path)) - args.extend( - ( - "--proc", - "/proc", - "--dev", - "/dev", - "--tmpfs", - "/tmp", - "--bind", - str(sandbox_root), - SANDBOX_MOUNT, - "--chdir", - f"{SANDBOX_MOUNT}/{cwd.relative_to(sandbox_root)}", - "--", - ) - ) - return shlex.join([*args, *argv]) - - def start_service(label: str, command: str, cwd: Path, env: dict[str, str], logs_dir: Path) -> Service: """Start a service command in its own process group.""" log_path = logs_dir / f"{label}.log" @@ -201,12 +125,8 @@ def wait_for_url(url: str, timeout: int, service: Service) -> bool: parsed = urllib.parse.urlparse(url) hostname = (parsed.hostname or "").lower() - try: - is_loopback = hostname == "localhost" or ipaddress.ip_address(hostname).is_loopback - except ValueError: - is_loopback = False - if not is_loopback: - raise ValueError(f"URL cannot target external hostname: {hostname}") + if hostname not in {"localhost", "127.0.0.1", "::1"}: + raise ValueError(f"URL hostname must be localhost, 127.0.0.1, or ::1, got: {hostname}") deadline = time.monotonic() + timeout opener = urllib.request.build_opener(NoRedirectHandler()) @@ -282,8 +202,6 @@ def emit_result( "frontend_cmd": args.frontend_cmd, "frontend_ready": frontend_ready, "network": args.network, - "isolation": args.isolation, - "isolation_backend": getattr(args, "isolation_backend", "unknown"), "sandbox": str(sandbox_root) if args.keep_sandbox else "(removed)", "sandboxed": True, } @@ -305,55 +223,13 @@ def main(argv: Sequence[str] | None = None) -> int: try: copied_repo = sandboxed_verify.copy_workspace(Path(args.repo_root), sandbox, args.ignore) env = sandboxed_verify.scrubbed_env(sandbox, args.allow_env) - try: - backend = isolation_backend(args.isolation) - except RuntimeError as exc: - print(f"sandboxed-web-e2e: {exc}", file=sys.stderr) - args.isolation_backend = "unavailable" - exit_code = 126 - return exit_code - args.isolation_backend = backend or "disabled" print(f"sandboxed-web-e2e: cwd={copied_repo}") if args.allow_env: print(f"sandboxed-web-e2e: allowed env names={','.join(sorted(set(args.allow_env)))}") if args.network != "default": print(f"sandboxed-web-e2e: network={args.network}") - command_env = _sandbox_environment(env, sandbox) if backend else env - backend_cmd = ( - isolated_command( - args.backend_cmd, - backend=backend, - cwd=copied_repo, - sandbox_root=sandbox, - env=env, - ) - if backend - else args.backend_cmd - ) - frontend_cmd = ( - isolated_command( - args.frontend_cmd, - backend=backend, - cwd=copied_repo, - sandbox_root=sandbox, - env=env, - ) - if backend - else args.frontend_cmd - ) - e2e_cmd = ( - isolated_command( - args.e2e_cmd, - backend=backend, - cwd=copied_repo, - sandbox_root=sandbox, - env=env, - ) - if backend - else args.e2e_cmd - ) - services.append(start_service("backend", backend_cmd, copied_repo, command_env, logs_dir)) - services.append(start_service("frontend", frontend_cmd, copied_repo, command_env, logs_dir)) + services.append(start_service("backend", args.backend_cmd, copied_repo, env, logs_dir)) + services.append(start_service("frontend", args.frontend_cmd, copied_repo, env, logs_dir)) backend_ready = wait_for_url(args.backend_ready_url, args.startup_timeout, services[0]) frontend_ready = wait_for_url(args.frontend_ready_url, args.startup_timeout, services[1]) if not backend_ready or not frontend_ready: @@ -361,7 +237,7 @@ def main(argv: Sequence[str] | None = None) -> int: exit_code = 125 return exit_code try: - completed = run_shell(e2e_cmd, copied_repo, command_env, args.e2e_timeout) + completed = run_shell(args.e2e_cmd, copied_repo, env, args.e2e_timeout) if completed.stdout: print(completed.stdout, end="") if completed.stderr: diff --git a/scripts/ci/strix_quick_gate.sh b/scripts/ci/strix_quick_gate.sh index c9aa41545..cfc97a63c 100755 --- a/scripts/ci/strix_quick_gate.sh +++ b/scripts/ci/strix_quick_gate.sh @@ -827,17 +827,6 @@ is_github_models_api_base() { esac } -is_known_foreign_provider_api_base() { - case "$1" in - https://models.github.ai/* | https://integrate.api.nvidia.com/* | https://openrouter.ai/*) - return 0 - ;; - *) - return 1 - ;; - esac -} - PRIMARY_MODEL="$(normalize_model "$STRIX_LLM")" if [ "$PRIMARY_MODEL" != "$STRIX_LLM" ]; then echo "Normalized STRIX_LLM to provider-qualified model '$PRIMARY_MODEL'." @@ -2437,20 +2426,15 @@ resolved_llm_api_base_for_model() { if is_vertex_model "$model"; then return 0 fi - local api_base_file="${LLM_API_BASE_FILE:-}" + if is_explicit_openai_model "$model" && ! is_explicit_openai_model "$PRIMARY_MODEL"; then + # A direct-OpenAI fallback must not inherit a foreign primary provider's + # endpoint (for example NVIDIA NIM or OpenRouter). + return 0 + fi + + local api_base_file="$LLM_API_BASE_FILE" local api_base_file_name="LLM_API_BASE_FILE" - if is_explicit_openai_model "$model" && [ -n "${STRIX_OPENAI_FALLBACK_API_BASE_FILE:-}" ]; then - # Cross-provider fallback: openai-direct/* candidates must reach the - # direct OpenAI API even when the primary provider selected a - # different LLM_API_BASE_FILE endpoint (e.g. NVIDIA NIM). Without - # this the fallback hits the primary gateway and 404s. - api_base_file="$STRIX_OPENAI_FALLBACK_API_BASE_FILE" - api_base_file_name="STRIX_OPENAI_FALLBACK_API_BASE_FILE" - # The workflow always provisions this file for cross-provider fallbacks. - # In standalone runs, an explicitly supplied LLM_API_BASE_FILE remains - # a caller-owned custom OpenAI-compatible endpoint rather than being - # silently discarded. - elif is_github_models_model "$model" && [ -n "${STRIX_GITHUB_MODELS_API_BASE_FILE:-}" ]; then + if is_github_models_model "$model" && [ -n "${STRIX_GITHUB_MODELS_API_BASE_FILE:-}" ]; then # Cross-provider fallback: when the active primary provider uses a # different API base (for example OpenRouter), github_models/* fallback # attempts must still route through the GitHub Models inference endpoint. @@ -2486,15 +2470,6 @@ resolved_llm_api_base_for_model() { echo "ERROR: LLM_API_BASE must be an https URL when configured." >&2 return 2 fi - # Never let a known provider-specific base leak into an explicit - # direct-OpenAI fallback when no separate OpenAI override was provisioned. - # Other caller-supplied OpenAI-compatible endpoints remain valid standalone - # configuration and are intentionally preserved. - if is_explicit_openai_model "$model" \ - && [ -z "${STRIX_OPENAI_FALLBACK_API_BASE_FILE:-}" ] \ - && is_known_foreign_provider_api_base "$llm_api_base_value"; then - return 0 - fi if is_github_models_api_base "$llm_api_base_value" && ! is_github_models_api_compatible_model "$model"; then echo "ERROR: LLM_API_BASE may route through GitHub Models only when STRIX_LLM uses a GitHub Models-compatible model." >&2 return 2 diff --git a/scripts/ci/test_strix_quick_gate.sh b/scripts/ci/test_strix_quick_gate.sh index 3d3449dae..abcb5ed07 100755 --- a/scripts/ci/test_strix_quick_gate.sh +++ b/scripts/ci/test_strix_quick_gate.sh @@ -374,8 +374,6 @@ assert_strix_workflow_pr_trigger_hardened() { assert_file_contains "$workflow_file" "steps.gate.outputs.provider_mode == 'nvidia_nim' && 'nvidia_nim/nvidia/llama-3.3-nemotron-super-49b-v1.5 openai-direct/gpt-5.4'" "strix workflow gives NVIDIA NIM scans contracted fallbacks" assert_file_not_contains "$workflow_file" "STRIX_FALLBACK_MODELS: \${{ steps.gate.outputs.provider_mode == 'github_models' && 'github_models/openai/o3" "strix workflow fallback list must not depend on GitHub Models, which is in platform-wide retirement" assert_file_contains "$workflow_file" "Prepare GitHub Models fallback credentials" "strix workflow provisions GitHub Models fallback credentials for direct-OpenAI scans" - assert_file_contains "$workflow_file" "STRIX_OPENAI_FALLBACK_API_BASE_FILE" "strix workflow routes direct-OpenAI fallbacks through a trusted API base file" - assert_file_contains "$workflow_file" "https://api.openai.com/v1" "strix workflow uses the OpenAI platform endpoint for direct fallbacks" assert_file_contains "$GATE_SCRIPT" "STRIX_GITHUB_MODELS_KEY_FILE" "strix gate reads the optional GitHub Models fallback key file" assert_file_contains "$GATE_SCRIPT" "STRIX_GITHUB_MODELS_API_BASE_FILE" "strix gate routes github_models fallback models through the GitHub Models endpoint" assert_file_not_contains "$workflow_file" 'github_models/deepseek/deepseek-r1-0528 | github_models/deepseek/deepseek-v3-0324)' "strix workflow keeps DeepSeek GitHub Models restricted to fallback-only routing" diff --git a/tests/test_contextual_orchestrator_hourly_review_caller.py b/tests/test_contextual_orchestrator_hourly_review_caller.py deleted file mode 100644 index 204ed5288..000000000 --- a/tests/test_contextual_orchestrator_hourly_review_caller.py +++ /dev/null @@ -1,88 +0,0 @@ -"""Contract tests for Contextual Orchestrator's bounded hourly review-repair caller.""" - -from pathlib import Path - - -CALLER = Path(".github/workflows/contextual-orchestrator-hourly-review-repair.yml") -DOCTORING = Path("docs/doctoring/contextual-orchestrator-hourly-review-caller.md") -QUALITY_WORKFLOW = Path(".github/workflows/hourly-nvidia-nim-review-repair.yml") -SCHEDULER = Path(".github/workflows/pr-review-fix-scheduler.yml") - - -def _read(path: Path) -> str: - """Return one repository contract file as UTF-8 text.""" - return path.read_text(encoding="utf-8") - - -def test_contextual_orchestrator_caller_is_hourly_bounded_and_non_cancelling() -> None: - """The gateway repo receives one realistic repair opportunity without cancellation.""" - caller = _read(CALLER) - - assert 'cron: "34 * * * *"' in caller - assert "group: contextual-orchestrator-hourly-review-repair" in caller - assert "cancel-in-progress: false" in caller - assert "uses: ./.github/workflows/pr-review-fix-scheduler.yml" in caller - assert "target_repository: ContextualWisdomLab/contextual-orchestrator" in caller - assert "base_branch: main" in caller - assert 'max_prs: "50"' in caller - assert 'max_dispatches: "1"' in caller - assert 'retry_hours: "2"' in caller - - -def test_contextual_orchestrator_caller_preserves_credentials_and_read_only_scope() -> None: - """The queue scanner maps established credentials without exposing model secrets.""" - caller = _read(CALLER) - workflow_scope, jobs_scope = caller.split("\njobs:\n", maxsplit=1) - - assert "\npermissions:\n contents: read\n" in workflow_scope - assert "\n permissions:\n contents: read\n id-token: write\n" in jobs_scope - assert "PR_REVIEW_MERGE_TOKEN: ${{ secrets.PR_REVIEW_MERGE_TOKEN }}" in caller - assert "OPENCODE_APPROVE_TOKEN: ${{ secrets.OPENCODE_APPROVE_TOKEN }}" in caller - assert "secrets: inherit" not in caller - assert "NVIDIA_NIM_API_KEY" not in caller - assert "COPILOT_GITHUB_TOKEN" not in caller - for forbidden in ( - "actions: write", - "contents: write", - "issues: write", - "pull-requests: write", - "statuses: write", - ): - assert forbidden not in caller - - -def test_contextual_orchestrator_target_is_not_hard_coded_in_shared_scheduler() -> None: - """Product identity remains in the thin caller rather than the engine.""" - assert "ContextualWisdomLab/contextual-orchestrator" not in _read(SCHEDULER) - - -def test_contextual_orchestrator_doctoring_records_rca_feasibility_and_latency() -> None: - """Operators retain the exact rationale for the bounded two-hour retry policy.""" - doctoring = _read(DOCTORING) - - for phrase in ( - "root-cause analysis", - "remediation feasibility", - "two-hour same-head retry floor", - "independent non-author approval", - "NVIDIA_NIM_API_KEY", - "COPILOT_GITHUB_TOKEN", - "ContextualWisdomLab/contextual-orchestrator", - "APA 7th references", - ): - assert phrase in doctoring - - -def test_focused_quality_workflow_tracks_contextual_orchestrator_contracts() -> None: - """Every caller or doctoring edit reruns exact-head scheduler verification.""" - quality = _read(QUALITY_WORKFLOW) - - assert quality.count( - ".github/workflows/contextual-orchestrator-hourly-review-repair.yml" - ) == 2 - assert quality.count( - "docs/doctoring/contextual-orchestrator-hourly-review-caller.md" - ) == 2 - assert quality.count( - "tests/test_contextual_orchestrator_hourly_review_caller.py" - ) == 3 diff --git a/tests/test_disksage_hourly_review_caller.py b/tests/test_disksage_hourly_review_caller.py index 5ad14b248..bee0d859b 100644 --- a/tests/test_disksage_hourly_review_caller.py +++ b/tests/test_disksage_hourly_review_caller.py @@ -34,7 +34,7 @@ def test_disksage_caller_preserves_credentials_and_read_only_token_scope() -> No workflow_scope, jobs_scope = caller.split("\njobs:\n", maxsplit=1) assert "\npermissions:\n contents: read\n" in workflow_scope - assert "\n permissions:\n contents: read\n id-token: write\n" in jobs_scope + assert "\n permissions:\n" not in jobs_scope assert "PR_REVIEW_MERGE_TOKEN: ${{ secrets.PR_REVIEW_MERGE_TOKEN }}" in caller assert "OPENCODE_APPROVE_TOKEN: ${{ secrets.OPENCODE_APPROVE_TOKEN }}" in caller assert "secrets: inherit" not in caller diff --git a/tests/test_hourly_scheduler_runtime_budget.py b/tests/test_hourly_scheduler_runtime_budget.py index 02b4fa05b..eacf7eb55 100644 --- a/tests/test_hourly_scheduler_runtime_budget.py +++ b/tests/test_hourly_scheduler_runtime_budget.py @@ -32,14 +32,6 @@ def test_product_callers_do_not_cancel_an_in_flight_rca() -> None: assert "cancel-in-progress: true" not in caller -def test_disksage_caller_grants_oidc_permission_to_reusable_scheduler() -> None: - """The called scheduler must be able to exchange its OpenCode OIDC token.""" - caller = _read(DISKSAGE) - job = caller.split(" dispatch-review-repair:\n", maxsplit=1)[1] - - assert " permissions:\n contents: read\n id-token: write\n" in job - - def test_quality_gate_tracks_runtime_budget_contract() -> None: """Runtime-budget changes always execute the exact-head focused gate.""" quality = _read(QUALITY) diff --git a/tests/test_pr_review_autofix_nvidia_nim_contract.py b/tests/test_pr_review_autofix_nvidia_nim_contract.py index a5d25379a..16a83b935 100644 --- a/tests/test_pr_review_autofix_nvidia_nim_contract.py +++ b/tests/test_pr_review_autofix_nvidia_nim_contract.py @@ -19,7 +19,7 @@ DOCTORING_RECORD = Path("docs/doctoring/hourly-nvidia-nim-autofix.md") CHANGELOG = Path("CHANGELOG.md") REVIEW_DISPATCH_WORKFLOW = Path(".github/workflows/opencode-review-dispatch.yml") -REVIEW_DISPATCH_BLOB_SHA = "0df7a17cc72a79585cec169c8299e0646f93ab02" +REVIEW_DISPATCH_BLOB_SHA = "ed3f7b44f9afdd6ab295426e5d0440aeca6bdfb5" def _workflow_text(path: Path) -> str: diff --git a/tests/test_repository_branch_coverage_execution_sandboxes.py b/tests/test_repository_branch_coverage_execution_sandboxes.py index 7d1ec0a43..f8912272a 100644 --- a/tests/test_repository_branch_coverage_execution_sandboxes.py +++ b/tests/test_repository_branch_coverage_execution_sandboxes.py @@ -216,8 +216,6 @@ def timeout_runner( [ "--repo-root", str(repo), - "--isolation", - "disabled", "--backend-cmd", "backend", "--frontend-cmd", diff --git a/tests/test_sandboxed_web_e2e.py b/tests/test_sandboxed_web_e2e.py index 985100dd0..932198615 100644 --- a/tests/test_sandboxed_web_e2e.py +++ b/tests/test_sandboxed_web_e2e.py @@ -56,8 +56,6 @@ def test_sandboxed_web_e2e_runs_services_and_does_not_mutate_source(tmp_path, ca [ "--repo-root", str(repo), - "--isolation", - "disabled", "--backend-cmd", http_server_command(backend_port, "backend"), "--frontend-cmd", @@ -113,19 +111,11 @@ def test_wait_helpers_and_service_cleanup_edges(monkeypatch, tmp_path): exited_service = sandboxed_web_e2e.Service("done", "true", exited, tmp_path / "missing.log") assert sandboxed_web_e2e.wait_for_url("", 1, exited_service) is True - assert sandboxed_web_e2e.wait_for_url("http://localhost:1/", 1, exited_service) is False assert sandboxed_web_e2e.wait_for_url("http://127.0.0.1:1/", 1, exited_service) is False - assert sandboxed_web_e2e.wait_for_url("http://127.0.0.2:1/", 1, exited_service) is False with pytest.raises(ValueError, match="URL must start with http:// or https://"): sandboxed_web_e2e.wait_for_url("file:///etc/passwd", 1, exited_service) - with pytest.raises(ValueError, match="URL cannot target external hostname: external.example.com"): + with pytest.raises(ValueError, match="URL hostname must be localhost"): sandboxed_web_e2e.wait_for_url("http://external.example.com/ready", 1, exited_service) - with pytest.raises(ValueError, match="URL cannot target external hostname: app.localhost"): - sandboxed_web_e2e.wait_for_url("http://app.localhost:8000/health", 1, exited_service) - with pytest.raises(ValueError, match="URL cannot target external hostname: 169.254.169.254"): - sandboxed_web_e2e.wait_for_url("http://169.254.169.254/latest/meta-data/", 1, exited_service) - with pytest.raises(ValueError, match="URL cannot target external hostname: 0.0.0.0"): - sandboxed_web_e2e.wait_for_url("http://0.0.0.0:8000/health", 1, exited_service) sandboxed_web_e2e.stop_service(exited_service) assert sandboxed_web_e2e.tail_text(tmp_path / "missing.log") == "" @@ -310,8 +300,6 @@ def fake_start(label, command, cwd, env, logs_dir): [ "--repo-root", str(repo), - "--isolation", - "disabled", "--backend-cmd", "backend", "--frontend-cmd", @@ -375,8 +363,6 @@ def fake_wait(url, timeout, service): [ "--repo-root", str(repo), - "--isolation", - "disabled", "--backend-cmd", "backend", "--frontend-cmd", @@ -426,8 +412,6 @@ def fake_run_shell(command, cwd, env, timeout): [ "--repo-root", str(repo), - "--isolation", - "disabled", "--backend-cmd", "backend", "--frontend-cmd", @@ -458,8 +442,6 @@ def test_sandboxed_web_e2e_reports_readiness_failure(tmp_path, capsys): [ "--repo-root", str(repo), - "--isolation", - "disabled", "--backend-cmd", http_server_command(backend_port, "backend"), "--frontend-cmd", @@ -498,8 +480,6 @@ def fake_run_shell(command, cwd, env, timeout): [ "--repo-root", str(repo), - "--isolation", - "disabled", "--backend-cmd", f"{sys.executable} -c \"import time; time.sleep(3)\"", "--frontend-cmd", @@ -519,60 +499,6 @@ def fake_run_shell(command, cwd, env, timeout): assert "SANDBOXED_WEB_E2E_RESULT" in captured.out -def test_isolation_backend_fails_closed_outside_linux(monkeypatch): - """Required isolation never silently falls back to a host process.""" - monkeypatch.setattr(sandboxed_web_e2e.platform, "system", lambda: "Darwin") - with pytest.raises(RuntimeError, match="only supported on Linux"): - sandboxed_web_e2e.isolation_backend("required") - assert sandboxed_web_e2e.isolation_backend("disabled") is None - - -def test_isolated_command_mounts_only_workspace(monkeypatch, tmp_path): - """Bubblewrap commands expose the copied workspace and not the host root.""" - monkeypatch.setattr(sandboxed_web_e2e.platform, "system", lambda: "Linux") - monkeypatch.setattr( - sandboxed_web_e2e.shutil, - "which", - lambda name, path=None: "/usr/bin/bwrap" if name == "bwrap" else "/usr/bin/python3", - ) - sandbox = tmp_path / "sandbox" - repo = sandbox / "repo" - repo.mkdir(parents=True) - env = {"PATH": "/usr/bin", "HOME": str(sandbox / "home")} - command = sandboxed_web_e2e.isolated_command( - "python3 -c 'print(1)'", - backend="/usr/bin/bwrap", - cwd=repo, - sandbox_root=sandbox, - env=env, - ) - assert command.startswith("/usr/bin/bwrap") - assert "--tmpfs /" in command - assert "--bind" in command - assert "--chdir /workspace/repo" in command - assert "--ro-bind / /" not in command - - -def test_isolated_command_rejects_host_home_executable(monkeypatch, tmp_path): - """Executable paths from a user's home cannot enter the isolated runner.""" - monkeypatch.setattr( - sandboxed_web_e2e.shutil, - "which", - lambda *_args, **_kwargs: str(Path.home() / "bin/tool"), - ) - sandbox = tmp_path / "sandbox" - repo = sandbox / "repo" - repo.mkdir(parents=True) - with pytest.raises(RuntimeError, match="host home directory"): - sandboxed_web_e2e.isolated_command( - "tool", - backend="/usr/bin/bwrap", - cwd=repo, - sandbox_root=sandbox, - env={"PATH": "/usr/bin"}, - ) - - def test_parse_args_rejects_invalid_inputs(): """The CLI rejects unusable timeout and environment values.""" with pytest.raises(SystemExit): @@ -658,8 +584,6 @@ def test_module_import_and_main_entrypoint(monkeypatch, tmp_path): "sandboxed_web_e2e.py", "--repo-root", str(repo), - "--isolation", - "disabled", "--backend-cmd", f"{sys.executable} -c \"import time; time.sleep(0.2)\"", "--frontend-cmd", diff --git a/tests/test_strix_backend_unavailable_after_exempted_finding.py b/tests/test_strix_backend_unavailable_after_exempted_finding.py index 0c46868b7..3355a8448 100644 --- a/tests/test_strix_backend_unavailable_after_exempted_finding.py +++ b/tests/test_strix_backend_unavailable_after_exempted_finding.py @@ -20,7 +20,7 @@ from __future__ import annotations -import shlex +import re import subprocess import tempfile import unittest @@ -63,12 +63,9 @@ def _extract_neutralization_block(workflow: str) -> str: stale logic. """ - # The classification block starts at the neutralization-scope assignment - # and runs to the terminal failure exit. The gate-execution retry loop and - # the raw signal definitions live outside this region; the signal values - # are injected by _run_gate_tail so the extracted decision logic stays the - # single tested authority. - start_marker = ' strix_neutralization_scope_log="$strix_terminal_log"' + start_marker = ( + " # Recognized signals that the LLM backend was unavailable" + ) terminal_failure_marker = ( ' echo "Strix reported security findings or failed for a ' 'non-backend reason; failing the required check' @@ -80,23 +77,6 @@ def _extract_neutralization_block(workflow: str) -> str: return workflow[start:end] -def _extract_signal_definitions(workflow: str) -> str: - """Return the canonical backend-outage / finding-signal definitions. - - Bounded by the same unique anchors used in production so the injected - patterns cannot drift from the ones the gate itself classifies with. - """ - - start_marker = ( - " # Recognized signals that the LLM backend was unavailable" - ) - end_marker = "reported_vulnerability_signal=" - start = workflow.index(start_marker) - end = workflow.index(end_marker, start) - end = workflow.index("\n", end) + 1 - return workflow[start:end] - - def _run_gate_tail(log_text: str) -> int: """Execute the extracted block against a synthetic log; return its exit code. @@ -105,7 +85,6 @@ def _run_gate_tail(log_text: str) -> int: """ workflow = STRIX_WORKFLOW.read_text(encoding="utf-8") - signals = _extract_signal_definitions(workflow) block = _extract_neutralization_block(workflow) with tempfile.TemporaryDirectory(prefix="strix-tail-scope-") as temp_dir: strix_run_log = Path(temp_dir) / "strix_gate_console.log" @@ -114,9 +93,7 @@ def _run_gate_tail(log_text: str) -> int: ( "set -uo pipefail", 'strix_run_log="$1"', - 'strix_terminal_log="$strix_run_log"', "strix_rc=1", - signals, block, ) ) @@ -136,70 +113,6 @@ def _run_gate_tail(log_text: str) -> int: return completed.returncode - -def _extract_retry_loop_region(workflow: str) -> str: - """Return the bounded provider-outage retry region, verbatim from the yml. - - Spans the signal definitions through the post-loop success exit so the - retry decision, its tail-scoping, and its terminal success path are all - exercised against a scripted fake gate. - """ - - start_marker = ( - " # Recognized signals that the LLM backend was unavailable" - ) - end_marker = ( - " # Preserve configuration failures (exit 2) and any unexpected exit" - ) - start = workflow.index(start_marker) - end = workflow.index(end_marker, start) - return workflow[start:end] - - -def _run_gate_retry(gate_script: str) -> tuple[int, int]: - """Run the extracted retry loop against a scripted gate; return (rc, calls). - - The fake gate appends one line to a call-counter file on every invocation - so tests can prove exactly how many attempts the loop spent. - """ - - workflow = STRIX_WORKFLOW.read_text(encoding="utf-8") - signals = _extract_signal_definitions(workflow) - region = _extract_retry_loop_region(workflow) - with tempfile.TemporaryDirectory(prefix="strix-retry-scope-") as temp_dir: - counter = Path(temp_dir) / "gate_calls" - counter.write_text("0\n", encoding="utf-8") - gate_path = Path(temp_dir) / "fake_gate.sh" - gate_path.write_text( - gate_script.replace("__COUNTER__", str(counter)), - encoding="utf-8", - ) - gate_path.chmod(0o755) - script = "\n".join( - ( - "set -uo pipefail", - f"export TRUSTED_STRIX_GATE={shlex.quote(str(gate_path))}", - "export RUNNER_TEMP=" + shlex.quote(temp_dir), - "process_budget_seconds=5400", - "budget_suffix=TIMEOUT", - "export STRIX_TOTAL_TIMEOUT_SECONDS=5700", - "export STRIX_GATE_RETRY_BACKOFF_SECONDS=1", - signals, - region, - 'exit "$strix_rc"', - ) - ) - completed = subprocess.run( - ["bash", "-c", script], - check=False, - capture_output=True, - text=True, - env={"RUNNER_TEMP": temp_dir, "PATH": "/usr/bin:/bin"}, - ) - calls = int(counter.read_text().strip()) - return completed.returncode, calls - - class StrixBackendUnavailableAfterExemptedFindingTests(unittest.TestCase): """Protect the PR #392-shaped scenario without weakening the real gate.""" @@ -240,59 +153,6 @@ def test_bare_backend_outage_with_no_finding_is_non_passing( self.assertEqual(_run_gate_tail(GITHUB_MODELS_BROWNOUT), 1) - def test_exempted_finding_then_outage_recovers_on_second_attempt(self) -> None: - """An exempt finding before continuation must not block outage retry.""" - - gate = r"""#!/usr/bin/env bash -calls=$(( $(cat __COUNTER__) + 1 )) -echo "$calls" > __COUNTER__ -if [ "$calls" -le 1 ]; then - printf '%s\n' \ - "Strix findings are limited to unchanged files in this pull request; allowing pipeline continuation." \ - "LLM CONNECTION FAILED" \ - "Configured model and fallback models were unavailable." - exit 1 -fi -echo "scan complete" -exit 0 -""" - returncode, calls = _run_gate_retry(gate) - self.assertEqual(returncode, 0) - self.assertEqual(calls, 2) - - def test_real_finding_after_continuation_never_retries(self) -> None: - """A tail-scoped real finding is authoritative: zero retries, fail closed.""" - - gate = r"""#!/usr/bin/env bash -calls=$(( $(cat __COUNTER__) + 1 )) -echo "$calls" > __COUNTER__ -printf '%s\n' \ - "Strix findings are limited to unchanged files in this pull request; allowing pipeline continuation." \ - "LLM CONNECTION FAILED" \ - "Vulnerability Report" "Severity: CRITICAL" "Vulnerabilities 1" -exit 1 -""" - returncode, calls = _run_gate_retry(gate) - self.assertEqual(returncode, 1) - self.assertEqual(calls, 1) - - def test_retry_contract_preserves_logs_and_full_attempt_budget(self) -> None: - """Retries retain every attempt and reserve the complete gate budget.""" - - workflow = STRIX_WORKFLOW.read_text(encoding="utf-8") - self.assertIn('strix_attempt_log="$RUNNER_TEMP/strix_gate_console_attempt_', workflow) - self.assertIn('cat "$strix_attempt_log" >> "$strix_run_log"', workflow) - self.assertIn( - 'strix_gate_attempt_budget_var="STRIX_TOTAL_${budget_suffix}_SECONDS"', - workflow, - ) - self.assertIn( - 'strix_gate_attempt_budget_seconds="${!strix_gate_attempt_budget_var:-$process_budget_seconds}"', - workflow, - ) - self.assertNotIn("STRIX_TOTAL_TIMEOUT_SECONDS:", workflow) - self.assertNotIn('remaining_seconds" -lt 600', workflow) - if __name__ == "__main__": unittest.main() diff --git a/tests/test_strix_openai_fallback_api_base.py b/tests/test_strix_openai_fallback_api_base.py deleted file mode 100644 index f078dd2a0..000000000 --- a/tests/test_strix_openai_fallback_api_base.py +++ /dev/null @@ -1,240 +0,0 @@ -"""Regression contract for direct-OpenAI fallback API-base routing. - -When the Strix primary provider is NVIDIA NIM (or OpenRouter / GitHub Models), -the workflow's ``LLM_API_BASE_FILE`` points at that provider's endpoint. A -cross-provider fallback to ``openai-direct/gpt-5.4`` must never inherit that -base: routing an OpenAI model through the NVIDIA NIM edge yields a plain-text -gateway 404 ("404 page not found") instead of OpenAI responses, so the final -contracted fallback could never complete a scan. - -The gate must therefore prefer an explicit -``STRIX_OPENAI_FALLBACK_API_BASE_FILE`` for explicit direct-OpenAI models, and - fall back to a caller-supplied ``LLM_API_BASE_FILE`` for standalone custom - endpoints, or to litellm's default OpenAI endpoint when no base is supplied. -""" - -from __future__ import annotations - -import re -import subprocess -import tempfile -import unittest -from pathlib import Path - - -REPOSITORY_ROOT = Path(__file__).resolve().parents[1] -STRIX_GATE = REPOSITORY_ROOT / "scripts" / "ci" / "strix_quick_gate.sh" -STRIX_MODEL_UTILS = REPOSITORY_ROOT / "scripts" / "ci" / "strix_model_utils.sh" -STRIX_WORKFLOW = REPOSITORY_ROOT / ".github" / "workflows" / "strix.yml" -OPENAI_FALLBACK_BASE = "https://api.openai.com/v1" - - -def _function_block(source: str, function_name: str) -> str: - """Return one top-level Bash function, including its closing brace.""" - - match = re.search( - rf"(?ms)^{re.escape(function_name)}\(\) \{{\n.*?^\}}\n", - source, - ) - if match is None: - raise AssertionError(f"missing Bash function: {function_name}") - return match.group(0) - - -def _resolver_helpers() -> list[str]: - """Collect every helper the production API-base resolver depends on.""" - - gate_source = STRIX_GATE.read_text(encoding="utf-8") - utils_source = STRIX_MODEL_UTILS.read_text(encoding="utf-8") - helper_names = ( - ("is_vertex_model", gate_source), - ("is_github_models_model", gate_source), - ("is_github_models_api_base", gate_source), - ("is_known_foreign_provider_api_base", gate_source), - ("is_github_models_api_compatible_model", gate_source), - ("is_explicit_openai_model", gate_source), - ("resolve_trusted_input_file", gate_source), - ("trim_whitespace", utils_source), - ) - helpers: list[str] = [] - for name, source in helper_names: - try: - helpers.append(_function_block(source, name)) - except AssertionError as exc: # pragma: no cover - shape drift guard - raise AssertionError(f"resolver helper missing: {name}") from exc - return helpers - - -def _resolve_api_base(env: dict[str, str], model: str) -> tuple[int, str]: - """Execute the production API-base resolver for one model and env.""" - - gate_source = STRIX_GATE.read_text(encoding="utf-8") - resolver_source = _function_block(gate_source, "resolved_llm_api_base_for_model") - helper_sources = _resolver_helpers() - with tempfile.TemporaryDirectory(prefix="strix-openai-fallback-base-") as temp_dir: - base_path = Path(temp_dir) / "primary_base.txt" - if "LLM_API_BASE_FILE" in env: - base_path.write_text(env["LLM_API_BASE_FILE"], encoding="utf-8") - env = {**env, "LLM_API_BASE_FILE": str(base_path)} - fallback_path = Path(temp_dir) / "openai_fallback_api_base.txt" - if "STRIX_OPENAI_FALLBACK_API_BASE_FILE" in env: - fallback_path.write_text( - env["STRIX_OPENAI_FALLBACK_API_BASE_FILE"], - encoding="utf-8", - ) - env = { - **env, - "STRIX_OPENAI_FALLBACK_API_BASE_FILE": str(fallback_path), - } - github_models_path = Path(temp_dir) / "github_models_api_base.txt" - if "STRIX_GITHUB_MODELS_API_BASE_FILE" in env: - github_models_path.write_text( - env["STRIX_GITHUB_MODELS_API_BASE_FILE"], - encoding="utf-8", - ) - env = { - **env, - "STRIX_GITHUB_MODELS_API_BASE_FILE": str(github_models_path), - } - script_lines = [ - "set -euo pipefail", - f'STRIX_INPUT_FILE_ROOT="{temp_dir}"', - *helper_sources, - resolver_source, - ] - command_env = { - key: value - for key, value in env.items() - if key in { - "LLM_API_BASE_FILE", - "STRIX_GITHUB_MODELS_API_BASE_FILE", - "STRIX_OPENAI_FALLBACK_API_BASE_FILE", - } - } - completed = subprocess.run( - [ - "bash", - "-c", - "\n".join([*script_lines, 'resolved_llm_api_base_for_model "$1"']), - "strix-resolver", - model, - ], - check=False, - capture_output=True, - text=True, - env={ - "PATH": "/usr/bin:/bin:/usr/local/bin", - "HOME": temp_dir, - **command_env, - }, - ) - return completed.returncode, completed.stdout.strip() - - -class ExplicitOpenAIFallbackRouting(unittest.TestCase): - """Direct-OpenAI fallbacks must not inherit the primary provider base.""" - - def test_nvidia_primary_with_override_routes_to_openai(self) -> None: - """openai-direct/gpt-5.4 uses the explicit OpenAI API base file.""" - - rc, api_base = _resolve_api_base( - { - "LLM_API_BASE_FILE": "https://integrate.api.nvidia.com/v1", - "STRIX_OPENAI_FALLBACK_API_BASE_FILE": OPENAI_FALLBACK_BASE, - }, - "openai-direct/gpt-5.4", - ) - self.assertEqual(rc, 0) - self.assertEqual(api_base, OPENAI_FALLBACK_BASE) - - def test_standalone_custom_base_is_honored_without_override(self) -> None: - """A standalone caller's explicit custom endpoint remains effective.""" - - rc, api_base = _resolve_api_base( - {"LLM_API_BASE_FILE": "https://api.example.com/v1"}, - "openai-direct/gpt-5.4", - ) - self.assertEqual(rc, 0) - self.assertEqual(api_base, "https://api.example.com/v1") - - def test_nvidia_base_is_not_inherited_without_override(self) -> None: - """A cross-provider fallback never inherits the NVIDIA NIM base.""" - - rc, api_base = _resolve_api_base( - {"LLM_API_BASE_FILE": "https://integrate.api.nvidia.com/v1"}, - "openai-direct/gpt-5.4", - ) - self.assertEqual(rc, 0) - self.assertEqual(api_base, "") - - def test_direct_openai_without_any_base_uses_default_openai(self) -> None: - """With no base file, LiteLLM still selects the native OpenAI endpoint.""" - - rc, api_base = _resolve_api_base({}, "openai-direct/gpt-5.4") - self.assertEqual(rc, 0) - self.assertEqual(api_base, "") - - def test_github_models_base_is_not_inherited_without_override(self) -> None: - """A cross-provider fallback never inherits GitHub Models routing.""" - - rc, api_base = _resolve_api_base( - {"LLM_API_BASE_FILE": "https://models.github.ai/inference"}, - "openai-direct/gpt-5.4", - ) - self.assertEqual(rc, 0) - self.assertEqual(api_base, "") - - def test_primary_provider_models_keep_their_base(self) -> None: - """NVIDIA NIM primary attempts still resolve through the NIM edge.""" - - rc, api_base = _resolve_api_base( - {"LLM_API_BASE_FILE": "https://integrate.api.nvidia.com/v1"}, - "nvidia_nim/nvidia/nemotron-3-super-120b-a12b", - ) - self.assertEqual(rc, 0) - self.assertEqual(api_base, "https://integrate.api.nvidia.com/v1") - - def test_github_models_fallback_keeps_github_models_base(self) -> None: - """github_models/* fallbacks keep their dedicated inference endpoint.""" - - rc, api_base = _resolve_api_base( - { - "LLM_API_BASE_FILE": "https://integrate.api.nvidia.com/v1", - "STRIX_GITHUB_MODELS_API_BASE_FILE": "https://models.github.ai/inference", - }, - "github_models/openai/gpt-5.4", - ) - self.assertEqual(rc, 0) - self.assertEqual(api_base, "https://models.github.ai/inference") - - def test_invalid_https_override_is_configuration_failure(self) -> None: - """A non-https override fails configuration instead of scanning.""" - - rc, _ = _resolve_api_base( - { - "LLM_API_BASE_FILE": "https://integrate.api.nvidia.com/v1", - "STRIX_OPENAI_FALLBACK_API_BASE_FILE": "http://api.example.com/v1", - }, - "openai-direct/gpt-5.4", - ) - self.assertEqual(rc, 2) - - -class WorkflowProvisionsFallbackBase(unittest.TestCase): - """The workflow must publish and pass the explicit OpenAI fallback base.""" - - def test_workflow_writes_openai_fallback_api_base_file(self) -> None: - workflow = STRIX_WORKFLOW.read_text(encoding="utf-8") - self.assertIn("STRIX_OPENAI_FALLBACK_API_BASE_FILE=", workflow) - self.assertIn(OPENAI_FALLBACK_BASE, workflow) - - def test_workflow_passes_override_into_gate_environment(self) -> None: - workflow = STRIX_WORKFLOW.read_text(encoding="utf-8") - self.assertIn( - "STRIX_OPENAI_FALLBACK_API_BASE_FILE: ${{ env.STRIX_OPENAI_FALLBACK_API_BASE_FILE }}", - workflow, - ) - - -if __name__ == "__main__": - unittest.main()