From ff967f4c21d1b9678eac05db7957e778e5593b57 Mon Sep 17 00:00:00 2001 From: Dan Morosanu Date: Thu, 6 Aug 2026 19:19:26 +0300 Subject: [PATCH 1/4] feat(sandbox): driver-independent protected mock service for CLI fixtures --- docs/TASK_DEFINITION_GUIDE.md | 54 ++ src/coder_eval/models/__init__.py | 4 + src/coder_eval/models/sandbox.py | 87 +++ src/coder_eval/orchestrator.py | 66 ++ src/coder_eval/protected_mock/__init__.py | 1 + src/coder_eval/protected_mock/client.py | 130 ++++ src/coder_eval/protected_mock/protocol.py | 71 +++ src/coder_eval/protected_mock/runtime.py | 230 +++++++ src/coder_eval/protected_mock/server.py | 425 +++++++++++++ src/coder_eval/sandbox.py | 88 +++ tests/test_protected_mock.py | 707 ++++++++++++++++++++++ 11 files changed, 1863 insertions(+) create mode 100644 src/coder_eval/protected_mock/__init__.py create mode 100644 src/coder_eval/protected_mock/client.py create mode 100644 src/coder_eval/protected_mock/protocol.py create mode 100644 src/coder_eval/protected_mock/runtime.py create mode 100644 src/coder_eval/protected_mock/server.py create mode 100644 tests/test_protected_mock.py diff --git a/docs/TASK_DEFINITION_GUIDE.md b/docs/TASK_DEFINITION_GUIDE.md index ef97b474..2537d796 100644 --- a/docs/TASK_DEFINITION_GUIDE.md +++ b/docs/TASK_DEFINITION_GUIDE.md @@ -541,6 +541,60 @@ Notes: - **Collisions are rejected.** If a `mock_path_dirs` entry already provides an executable of the same name, setup raises rather than letting directory order decide which one runs. - **It stubs a tool; it does not proxy one, and it does not serve per-invocation responses.** Recording a *real* executable on the way through, or returning different output per invocation, stays a hand-written mock under `mock_path_dirs` — both depend on state the harness cannot guarantee (the tool being installed, PATH order, live credentials, a fixture set). +### Protected Fixture-Backed CLIs + +Use `protected_mocks` when `uip` or another mock needs different fixture-backed responses per invocation and the fixture itself must not be readable by the evaluated agent. The fixture is loaded host-side by a small per-run server; the agent's workspace only ever contains a thin client shim, so no encoding or sealing of grading material is involved: + +```yaml +sandbox: + driver: tempdir + protected_mocks: + - tool: uip + fixture: ./fixtures/uip-troubleshoot.json + max_requests: 100 + passthrough_argv_prefixes: + - [docsai, ask] +``` + +Notes: + +- **Drivers.** Supported under `driver: tempdir` (the default). Under `driver: docker` the task fails validation: protected mocks under the docker driver require the UID/GID isolation layer, which is not yet available — that combination fails closed rather than running without the isolation it assumes. +- **Endpoint.** The server binds a per-run endpoint at start: an AF_UNIX socket in a run-scoped scratch directory when the platform supports it (probed with a real bind), else TCP on `127.0.0.1` with an ephemeral port. Every request must carry the run's random token, which the generated shim bakes in. The token keeps other local processes from casually querying the service; it is same-user hygiene, not a security boundary — the protection comes from the server only ever answering with configured command responses, never fixture contents. +- **Fixture paths.** `fixture` resolves against the task YAML's directory (like `uipath_eval.eval_set`). The file is read host-side only and is never copied into the sandbox. +- **Shims.** Each entry generates a `protected_mocks/` shim (plus a `.cmd` twin for Windows PATHEXT lookup) that is PATH-prepended for the agent exactly like `mock_path_dirs` entries. The shim carries its endpoint, token, and call-log path itself; it does not rely on the agent's environment. +- **Call log.** Every invocation is appended, in the `cli_called` JSON Lines schema, to `protected_mock_calls.jsonl` next to `task.json` in the run directory — outside the sandbox. This is a diagnostic surface: the [`cli_called`](#cli_called) criterion resolves its `log` field sandbox-relative and cannot read this host-side file today. +- **Budget and audit.** `max_requests` caps calls per tool per run (exceeded calls get exit 75). The run's `environment_info` records the endpoint kind and a SHA-256 digest of the fixture contents. +- `protected_mocks` and `record_cli` cannot claim the same tool name. + +Fixture files map argument lists to responses: + +```json +{ + "version": 1, + "responses": [ + { + "argv": ["rpa", "get-errors", "--output", "json"], + "exit_code": 0, + "stdout": "{\"errors\":[]}\n", + "stderr": "" + } + ], + "default": { + "exit_code": 2, + "stderr": "command not configured for this scenario\n" + } +} +``` + +Matching defaults to exact argv equality. Two further modes exist, selected per response via `match_mode`: + +- `"normalized"` still selects from a finite command map but ignores `--output `, treats `--flag=value` like `--flag value`, and permits token reordering. Duplicate keys are rejected at load for both finite modes. +- `"subset"` matches when every rule token appears in the invocation's normalized token set, regardless of order or extra arguments. Subset rules are evaluated in fixture-file order and the first match wins; exact and normalized matches always take precedence over subset scanning. Duplicate subset rules are allowed (an earlier rule shadows a later one); an empty subset `argv` is rejected at load. + +Malformed responses, oversized output, request-budget exhaustion, and service startup failures all fail loudly. + +`passthrough_argv_prefixes` is for deliberately public live operations such as `uip docsai ask`. The server invokes the real tool only when argv begins with one of these typed prefixes, caches the response in memory for the run, and never reveals the executable path to the agent. Do not use a broad prefix such as `[or]` or `[auth]`. + ## Template Sources Tasks can start with preset files instead of an empty sandbox. Multiple sources are applied sequentially (last wins for conflicts). diff --git a/src/coder_eval/models/__init__.py b/src/coder_eval/models/__init__.py index 51504e92..d079bd9a 100644 --- a/src/coder_eval/models/__init__.py +++ b/src/coder_eval/models/__init__.py @@ -155,11 +155,13 @@ # Sandbox from coder_eval.models.sandbox import ( + PROTECTED_MOCK_DIR, RECORD_CLI_DIR, RECORD_CLI_LOG, DockerBuildConfig, DockerDriverConfig, NodeEnvConfig, + ProtectedMockConfig, PythonEnvConfig, RecordedCli, ResourceLimits, @@ -274,6 +276,8 @@ "NodeEnvConfig", "PythonEnvConfig", "SandboxConfig", + "ProtectedMockConfig", + "PROTECTED_MOCK_DIR", "RecordedCli", "RECORD_CLI_DIR", "RECORD_CLI_LOG", diff --git a/src/coder_eval/models/sandbox.py b/src/coder_eval/models/sandbox.py index d2083d09..b186c1e7 100644 --- a/src/coder_eval/models/sandbox.py +++ b/src/coder_eval/models/sandbox.py @@ -398,6 +398,69 @@ def validate_tool_name(cls, v: str) -> str: return v +# Sandbox-relative directory the protected-mock client shims are generated into. +# Separate from RECORD_CLI_DIR so a wipe-and-regenerate of either feature can +# never delete the other's shims. Not dot-prefixed for the same artifact-upload +# reason as RECORD_CLI_DIR. +PROTECTED_MOCK_DIR = "protected_mocks" + + +class ProtectedMockConfig(BaseModel): + """Fixture-backed CLI served by the host-side protected mock service. + + ``fixture`` is read host-side by the service process, never by the + evaluated agent, and is never copied into the agent workspace. The fixture + schema maps argv lists to bounded stdout/stderr/exit-code responses; it + exposes no general file or search operation. Relative ``fixture`` paths + resolve against the task YAML's directory. + """ + + model_config = ConfigDict(extra="forbid") + + tool: str = Field(description="Bare executable name presented to the agent (for example, 'uip')") + fixture: str = Field( + description=( + "Path to the protected command-response fixture, resolved against the task YAML's " + "directory when relative. Read host-side only; never copied into the sandbox." + ) + ) + max_requests: int = Field(default=100, ge=1, le=10_000, description="Per-run request budget for this tool") + passthrough_argv_prefixes: list[list[str]] = Field( + default_factory=list, + max_length=16, + description=( + "Public argv prefixes the service may proxy to the real tool, for example [['docsai', 'ask']]. " + "All other invocations remain fixture-backed or receive the fixed default response." + ), + ) + + @field_validator("tool") + @classmethod + def validate_tool_name(cls, value: str) -> str: + if not value or value != value.strip() or value in {".", ".."} or "/" in value or "\\" in value: + raise ValueError("protected mock tool must be a non-empty bare executable name") + if value.lower().endswith((".cmd", ".bat", ".exe")): + raise ValueError("protected mock tool must not include a platform executable suffix") + return value + + @field_validator("passthrough_argv_prefixes") + @classmethod + def validate_passthrough_prefixes(cls, prefixes: list[list[str]]) -> list[list[str]]: + normalized: list[list[str]] = [] + seen: set[tuple[str, ...]] = set() + for prefix in prefixes: + if not prefix or len(prefix) > 8: + raise ValueError("protected mock passthrough prefixes must contain 1 to 8 argv tokens") + if any(not isinstance(token, str) or not token or len(token) > 256 for token in prefix): + raise ValueError("protected mock passthrough prefix tokens must be non-empty strings up to 256 chars") + key = tuple(prefix) + if key in seen: + raise ValueError("protected mock passthrough prefixes must be unique") + seen.add(key) + normalized.append(list(prefix)) + return normalized + + class SandboxConfig(BaseModel): """Configuration for the sandboxed execution environment. @@ -451,6 +514,17 @@ class SandboxConfig(BaseModel): ), ) + protected_mocks: list[ProtectedMockConfig] | None = MergeField( + strategy="replace", + default=None, + description=( + "Fixture-backed mock CLIs served by a host-side per-run service. The agent receives a thin " + "client shim; fixture bytes stay host-side and are never copied into its workspace. " + "Supported under driver: tempdir. Under driver: docker this fails validation until the " + "UID/GID isolation layer lands. Replaced (not merged) across config layers." + ), + ) + record_cli: list[RecordedCli] | None = MergeField( strategy="replace", default=None, @@ -489,4 +563,17 @@ def validate_template_sources(self) -> SandboxConfig: """Validate template sources configuration.""" if self.template_sources: validate_template_sources_list(self.template_sources) + if self.protected_mocks: + if self.driver == "docker": + raise ValueError( + "sandbox.protected_mocks under the docker driver requires the UID/GID isolation " + + "layer; not yet available. Use driver: tempdir." + ) + tools = [mock.tool for mock in self.protected_mocks] + if len(tools) != len(set(tools)): + raise ValueError("sandbox.protected_mocks tool names must be unique") + recorded = {spec.tool for spec in self.record_cli or []} + overlap = sorted(recorded & set(tools)) + if overlap: + raise ValueError(f"protected_mocks and record_cli cannot both provide: {overlap}") return self diff --git a/src/coder_eval/orchestrator.py b/src/coder_eval/orchestrator.py index 8640dd7f..3e306fb2 100644 --- a/src/coder_eval/orchestrator.py +++ b/src/coder_eval/orchestrator.py @@ -59,6 +59,8 @@ from .orchestration.early_stop import EarlyStopWatcher, early_stop_active, validate_early_stop from .orchestration.evaluation import load_reference from .path_utils import format_task_log_id, task_log_path +from .protected_mock.runtime import CALL_LOG_NAME as PROTECTED_MOCK_CALL_LOG_NAME +from .protected_mock.runtime import ProtectedMockRuntime from .sandbox import Sandbox from .simulation import DialogStopReason, SimulatorResult, UserSimulator, evaluate_stop from .streaming.callbacks import CompositeStreamCallback, StreamCallback, TaskScopedCallback, safe_emit @@ -395,6 +397,10 @@ def __init__( # so the default path is entirely unaffected). self._early_stop_watcher: EarlyStopWatcher | None = None + # Protected mock service (started in _setup only when the task declares + # sandbox.protected_mocks; stopped unconditionally in _cleanup). + self._protected_mock_runtime: ProtectedMockRuntime | None = None + # One-shot flag: emit the "cost budget configured but no cost data" warning # exactly once per task even if _check_run_limits fires every turn. self._cost_budget_skipped_logged: bool = False @@ -1077,6 +1083,12 @@ async def _setup_sandbox() -> Any: logger.info("API routing: %s", _format_routing(self.route, self.task.agent.model if self.task.agent else None)) self.success_checker = SuccessChecker(self.sandbox, route=self.eval_route) + # Start the protected mock service (host-side fixture server) and write + # its client shims into the sandbox BEFORE the agent's PATH is assembled + # below. Teardown is unconditional in _cleanup, which runs on every exit + # path (success, crash, timeout, early stop) via run()'s finally block. + await self._start_protected_mocks() + # Create and start the agent. For a no-op (type: none) task this dispatches # to NoOpAgent, whose start/communicate/stop are no-ops — the orchestrator # runs the normal lifecycle without any agentless branching, and the @@ -1116,6 +1128,51 @@ async def _start_agent() -> None: if self.sandbox and self.sandbox.installed_tool_versions: self.result.environment_info["installed_tools"] = self.sandbox.installed_tool_versions + # After the environment_info re-capture above, which would otherwise + # discard keys written earlier in _setup. + self._record_protected_mock_environment_info() + + async def _start_protected_mocks(self) -> None: + """Start the host-side fixture service and generate its sandbox shims. + + No-op unless the task declares ``sandbox.protected_mocks``. Fixture + paths resolve against the task YAML's directory host-side; fixture + bytes never enter the sandbox. The per-task invocation log lands next + to task.json in the run dir, outside the sandbox. + """ + mocks = self.task.sandbox.protected_mocks + if not mocks: + return + assert self.sandbox is not None + task_dir = self.task_file.parent.resolve() if self.task_file else None + runtime = ProtectedMockRuntime(mocks, task_dir=task_dir) + await asyncio.to_thread(runtime.start) + # Stored before shim generation so _cleanup stops the server even if + # generation fails. + self._protected_mock_runtime = runtime + call_log = self.run_dir / PROTECTED_MOCK_CALL_LOG_NAME + # Seed the log so the diagnostic surface always exists, even for a run + # that never called the tool. + await asyncio.to_thread(call_log.write_text, "", encoding="utf-8") + await asyncio.to_thread( + self.sandbox.generate_protected_mock_shims, + endpoint=runtime.endpoint, + token=runtime.token, + call_log=call_log, + ) + logger.info( + "Protected mock service up (%s endpoint) for tool(s): %s", + runtime.endpoint_kind, + ", ".join(mock.tool for mock in mocks), + ) + + def _record_protected_mock_environment_info(self) -> None: + """Persist the protected mock audit record into ``result.environment_info``.""" + if self._protected_mock_runtime is None or self.result is None: + return + self.result.environment_info["protected_mock_endpoint_kind"] = self._protected_mock_runtime.endpoint_kind + self.result.environment_info["protected_mock_fixture_digest"] = self._protected_mock_runtime.fixture_digest + def _sync_sandbox_command_path_with_agent(self) -> None: """Align criteria command PATH with the PATH used for the last agent query. @@ -2269,6 +2326,15 @@ async def _cleanup(self) -> None: except Exception as e: logger.warning(f"Failed to stop agent: {e}") + # Stop the protected mock service. After the agent (no in-flight calls), + # before sandbox teardown; the server must never outlive the task. + if self._protected_mock_runtime is not None: + try: + await asyncio.to_thread(self._protected_mock_runtime.stop) + except Exception as e: + logger.warning(f"Failed to stop protected mock service: {e}") + self._protected_mock_runtime = None + # Cleanup sandbox. Preservation and cleanup() are SIBLING try blocks: # a preservation failure (e.g. disk full during preserve_to) must never # skip cleanup(), or the tempdir leaks. diff --git a/src/coder_eval/protected_mock/__init__.py b/src/coder_eval/protected_mock/__init__.py new file mode 100644 index 00000000..68ff5f7a --- /dev/null +++ b/src/coder_eval/protected_mock/__init__.py @@ -0,0 +1 @@ +"""Protected fixture-backed CLI service: host-side server, thin sandbox client shim.""" diff --git a/src/coder_eval/protected_mock/client.py b/src/coder_eval/protected_mock/client.py new file mode 100644 index 00000000..efb02e1a --- /dev/null +++ b/src/coder_eval/protected_mock/client.py @@ -0,0 +1,130 @@ +"""Thin, agent-visible client for the protected fixture-backed CLI service. + +Connection details (endpoint, token, call-log path) come from environment +variables the generated shim bakes in and sets itself -- the client never +depends on the agent process environment carrying them. +""" + +from __future__ import annotations + +import json +import os +import socket +import sys +import time +from pathlib import Path + +from .protocol import ( + CALL_LOG_ENV, + CLIENT_TIMEOUT_SECONDS, + ENDPOINT_ENV, + MAX_REQUEST_BYTES, + MAX_RESPONSE_BYTES, + PROTOCOL_VERSION, + TOKEN_ENV, + parse_endpoint, +) + + +def _record(tool: str, argv: list[str], exit_code: int) -> None: + """Best-effort compatibility with the existing cli_called JSONL schema.""" + + raw_path = os.environ.get(CALL_LOG_ENV) + if not raw_path: + return + entry = {"ts": round(time.time(), 3), "tool": tool, "argv": argv, "exit": exit_code} + try: + with Path(raw_path).open("a", encoding="utf-8", newline="\n") as stream: + stream.write(json.dumps(entry, ensure_ascii=True, separators=(",", ":")) + "\n") + except OSError as exc: + sys.stderr.write(f"protected mock client: invocation log failed: {exc!r}\n") + + +def _receive_line(connection: socket.socket) -> bytes: + chunks: list[bytes] = [] + total = 0 + while True: + chunk = connection.recv(min(65536, MAX_RESPONSE_BYTES + 1 - total)) + if not chunk: + break + chunks.append(chunk) + total += len(chunk) + if total > MAX_RESPONSE_BYTES: + raise RuntimeError("response exceeded size limit") + if b"\n" in chunk: + break + return b"".join(chunks).split(b"\n", 1)[0] + + +def _connect(endpoint: str) -> socket.socket: + parsed = parse_endpoint(endpoint) + if parsed[0] == "unix": + af_unix = getattr(socket, "AF_UNIX", None) + if af_unix is None: + raise RuntimeError("Unix-domain sockets are unavailable") + connection = socket.socket(af_unix, socket.SOCK_STREAM) + try: + connection.settimeout(CLIENT_TIMEOUT_SECONDS) + connection.connect(parsed[1]) + except OSError: + connection.close() + raise + return connection + host, port = parsed[1] + return socket.create_connection((host, port), timeout=CLIENT_TIMEOUT_SECONDS) + + +def invoke(tool: str, argv: list[str]) -> int: + endpoint = os.environ.get(ENDPOINT_ENV, "") + token = os.environ.get(TOKEN_ENV, "") + request = ( + json.dumps( + {"version": PROTOCOL_VERSION, "token": token, "tool": tool, "argv": argv}, + ensure_ascii=True, + separators=(",", ":"), + ).encode("utf-8") + + b"\n" + ) + if len(request) > MAX_REQUEST_BYTES: + sys.stderr.write("protected mock client: request exceeds size limit\n") + _record(tool, argv, 125) + return 125 + + try: + if not endpoint: + raise RuntimeError(f"{ENDPOINT_ENV} is not set") + with _connect(endpoint) as connection: + connection.sendall(request) + raw_response = _receive_line(connection) + response = json.loads(raw_response.decode("utf-8")) + if not isinstance(response, dict) or response.get("version") != PROTOCOL_VERSION: + raise RuntimeError("invalid response envelope") + exit_code = response.get("exit_code") + stdout = response.get("stdout") + stderr = response.get("stderr") + if not isinstance(exit_code, int) or not 0 <= exit_code <= 255: + raise RuntimeError("invalid response exit_code") + if not isinstance(stdout, str) or not isinstance(stderr, str): + raise RuntimeError("invalid response streams") + except (OSError, UnicodeError, ValueError, RuntimeError) as exc: + sys.stderr.write(f"protected mock client: service unavailable or invalid response: {exc}\n") + _record(tool, argv, 125) + return 125 + + if stdout: + sys.stdout.write(stdout) + if stderr: + sys.stderr.write(stderr) + _record(tool, argv, exit_code) + return exit_code + + +def main() -> int: + if len(sys.argv) < 2: + sys.stderr.write("protected mock client: missing tool name\n") + return 64 + return invoke(sys.argv[1], sys.argv[2:]) + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/src/coder_eval/protected_mock/protocol.py b/src/coder_eval/protected_mock/protocol.py new file mode 100644 index 00000000..a40f4273 --- /dev/null +++ b/src/coder_eval/protected_mock/protocol.py @@ -0,0 +1,71 @@ +"""Constants and endpoint helpers shared by the protected mock client, server, and runtime. + +The service endpoint is chosen per run at server start: an AF_UNIX socket in the +run's scratch directory when the platform supports it (probed with an actual +bind), else TCP on 127.0.0.1 with an ephemeral port. Either way the client must +present the run's random token in each request. The token is same-user hygiene +(it keeps other local processes from casually querying the service), not a +security boundary: the agent can read it out of its own shim. +""" + +from __future__ import annotations + +from typing import Literal + + +PROTOCOL_VERSION = 1 +MAX_REQUEST_BYTES = 64 * 1024 +MAX_RESPONSE_BYTES = 1024 * 1024 +CLIENT_TIMEOUT_SECONDS = 5.0 + +# Environment variables the generated shim sets for the client process. The shim +# bakes the values in itself -- the client never relies on the agent process +# environment for its connection details. +ENDPOINT_ENV = "CODER_EVAL_PROTECTED_MOCK_ENDPOINT" +TOKEN_ENV = "CODER_EVAL_PROTECTED_MOCK_TOKEN" +CALL_LOG_ENV = "CODER_EVAL_MOCK_CALL_LOG" + +# Files inside the per-run runtime (scratch) directory. The token file is written +# by the runtime before the server starts; the endpoint file is written by the +# server once bound (atomically, via rename), and doubles as the readiness signal. +TOKEN_FILE_NAME = "token" +ENDPOINT_FILE_NAME = "endpoint" +SOCKET_FILE_NAME = "mock.sock" + + +def format_unix_endpoint(path: str) -> str: + """Render an AF_UNIX endpoint string (``unix:``).""" + return f"unix:{path}" + + +def format_tcp_endpoint(host: str, port: int) -> str: + """Render a TCP loopback endpoint string (``tcp::``).""" + return f"tcp:{host}:{port}" + + +def parse_endpoint(value: str) -> tuple[Literal["unix"], str] | tuple[Literal["tcp"], tuple[str, int]]: + """Parse an endpoint string into ``("unix", path)`` or ``("tcp", (host, port))``. + + The ``unix:`` payload is taken verbatim (Windows socket paths contain a + drive colon); the ``tcp:`` payload splits on the last colon. + + Raises: + ValueError: The string is not a recognized endpoint. + """ + if value.startswith("unix:"): + path = value[len("unix:") :] + if not path: + raise ValueError(f"invalid unix endpoint: {value!r}") + return ("unix", path) + if value.startswith("tcp:"): + host, sep, port_text = value[len("tcp:") :].rpartition(":") + if not sep or not host: + raise ValueError(f"invalid tcp endpoint: {value!r}") + try: + port = int(port_text) + except ValueError as exc: + raise ValueError(f"invalid tcp endpoint port: {value!r}") from exc + if not 0 < port <= 65535: + raise ValueError(f"invalid tcp endpoint port: {value!r}") + return ("tcp", (host, port)) + raise ValueError(f"unrecognized protected mock endpoint: {value!r}") diff --git a/src/coder_eval/protected_mock/runtime.py b/src/coder_eval/protected_mock/runtime.py new file mode 100644 index 00000000..b4933505 --- /dev/null +++ b/src/coder_eval/protected_mock/runtime.py @@ -0,0 +1,230 @@ +"""Host-side lifecycle for the protected mock service, plus the sandbox shim template. + +``ProtectedMockRuntime`` owns one per-run service: it resolves fixture paths +against the task directory, writes the server config and token into a fresh +scratch directory, spawns ``sys.executable -m coder_eval.protected_mock.server``, +waits for the endpoint file the server publishes once bound, and tears the +process down (terminate, then kill) on exit. Fixture bytes never enter the +agent workspace; only the generated shim does. +""" + +from __future__ import annotations + +import contextlib +import hashlib +import json +import secrets +import shutil +import subprocess +import sys +import tempfile +import time +from pathlib import Path + +from coder_eval.models import ProtectedMockConfig + +from .protocol import ( + CALL_LOG_ENV, + ENDPOINT_ENV, + ENDPOINT_FILE_NAME, + PROTOCOL_VERSION, + TOKEN_ENV, + TOKEN_FILE_NAME, +) + + +STARTUP_TIMEOUT_SECONDS = 10.0 + +# Host-side per-task invocation log, written next to task.json in the run dir +# (never inside the sandbox). Diagnostic surface: the `cli_called` criterion +# resolves its `log` field sandbox-relative, so it cannot read this file today. +CALL_LOG_NAME = "protected_mock_calls.jsonl" + + +def resolve_fixture_path(fixture: str, task_dir: Path | None) -> Path: + """Resolve a fixture path against the task YAML's directory (host-side). + + Mirrors how ``uipath_eval.eval_set`` resolves: relative paths join the task + directory; absolute paths are used as-is. The fixture must exist. + """ + path = Path(fixture) + if not path.is_absolute() and task_dir is not None: + path = task_dir / path + path = path.resolve() + if not path.is_file(): + raise RuntimeError(f"protected mock fixture not found: {path}") + return path + + +def fixture_digest(fixture_paths: list[Path]) -> str: + """SHA-256 over the fixture contents in config order, for the audit record.""" + digest = hashlib.sha256() + for path in fixture_paths: + digest.update(path.read_bytes()) + return digest.hexdigest() + + +class ProtectedMockRuntime: + """Context-manager lifecycle for one per-run protected mock server process.""" + + def __init__( + self, + mocks: list[ProtectedMockConfig], + *, + task_dir: Path | None, + transport: str = "auto", + ) -> None: + if not mocks: + raise ValueError("ProtectedMockRuntime requires at least one protected mock") + self._mocks = mocks + self._task_dir = task_dir + self._transport = transport + self._runtime_dir: Path | None = None + self._process: subprocess.Popen[bytes] | None = None + self.endpoint: str = "" + self.token: str = "" + self.fixture_digest: str = "" + + @property + def endpoint_kind(self) -> str: + """``"unix"`` or ``"tcp"`` (empty before start).""" + return self.endpoint.split(":", 1)[0] if self.endpoint else "" + + def start(self) -> None: + """Resolve fixtures, spawn the server, and wait for its endpoint file.""" + # Short prefix on purpose: the AF_UNIX socket lives in this directory and + # sun_path has a tight length limit on some platforms. + self._runtime_dir = Path(tempfile.mkdtemp(prefix="cepm-")) + try: + fixture_paths = [resolve_fixture_path(mock.fixture, self._task_dir) for mock in self._mocks] + self.fixture_digest = fixture_digest(fixture_paths) + config_path = self._runtime_dir / "mock-config.json" + config_path.write_text( + json.dumps( + { + "version": PROTOCOL_VERSION, + "tools": [ + { + "tool": mock.tool, + "fixture": str(path), + "max_requests": mock.max_requests, + "passthrough_argv_prefixes": mock.passthrough_argv_prefixes, + } + for mock, path in zip(self._mocks, fixture_paths, strict=True) + ], + } + ), + encoding="utf-8", + ) + self.token = secrets.token_hex(16) + (self._runtime_dir / TOKEN_FILE_NAME).write_text(self.token + "\n", encoding="utf-8") + self._process = subprocess.Popen( + [ + sys.executable, + "-m", + "coder_eval.protected_mock.server", + "--config", + str(config_path), + "--runtime-dir", + str(self._runtime_dir), + "--transport", + self._transport, + ], + stdin=subprocess.DEVNULL, + ) + self.endpoint = self._await_endpoint() + except Exception: + self.stop() + raise + + def _await_endpoint(self) -> str: + assert self._runtime_dir is not None and self._process is not None + endpoint_file = self._runtime_dir / ENDPOINT_FILE_NAME + deadline = time.monotonic() + STARTUP_TIMEOUT_SECONDS + while time.monotonic() < deadline: + if self._process.poll() is not None: + raise RuntimeError(f"protected mock server exited during startup with code {self._process.returncode}") + if endpoint_file.is_file(): + endpoint = endpoint_file.read_text(encoding="utf-8").strip() + if endpoint: + return endpoint + time.sleep(0.02) + raise RuntimeError(f"protected mock server did not publish its endpoint within {STARTUP_TIMEOUT_SECONDS}s") + + def stop(self) -> None: + """Terminate the server (kill on a slow exit) and remove the scratch dir.""" + if self._process is not None: + if self._process.poll() is None: + self._process.terminate() + try: + self._process.wait(timeout=3) + except subprocess.TimeoutExpired: + self._process.kill() + with contextlib.suppress(subprocess.TimeoutExpired): + self._process.wait(timeout=3) + self._process = None + if self._runtime_dir is not None: + shutil.rmtree(self._runtime_dir, ignore_errors=True) + self._runtime_dir = None + + def __enter__(self) -> ProtectedMockRuntime: + self.start() + return self + + def __exit__(self, *_exc_info: object) -> None: + self.stop() + + +_SHIM_TEMPLATE = '''\ +#!{interpreter} +"""Protected mock shim for `{tool}` - generated by coder_eval SandboxConfig.protected_mocks. + +Forwards the invocation to the host-side fixture service. No fixture data is +stored in this script or anywhere in the workspace. Do not edit: regenerated on +every sandbox setup. +""" + +import os +import subprocess +import sys + +TOOL = {tool!r} +INTERPRETER = {interpreter!r} + +# Baked in on purpose: the client must not depend on this process's inherited +# environment for its connection details. +SERVICE_ENV = {{ + {endpoint_env!r}: {endpoint!r}, + {token_env!r}: {token!r}, + {call_log_env!r}: {call_log!r}, +}} + +if __name__ == "__main__": + env = dict(os.environ) + env.update(SERVICE_ENV) + raise SystemExit( + subprocess.call( + [INTERPRETER, "-m", "coder_eval.protected_mock.client", TOOL, *sys.argv[1:]], + env=env, + ) + ) +''' + + +def render_shim(tool: str, *, interpreter: str, endpoint: str, token: str, call_log: str) -> str: + """Render the sandbox-visible shim source for one ``protected_mocks`` entry. + + The interpreter is the harness's own Python (where ``coder_eval`` is + importable); the shebang uses its absolute path so PATH order inside the + sandbox cannot change what runs. + """ + return _SHIM_TEMPLATE.format( + interpreter=interpreter, + tool=tool, + endpoint=endpoint, + token=token, + call_log=call_log, + endpoint_env=ENDPOINT_ENV, + token_env=TOKEN_ENV, + call_log_env=CALL_LOG_ENV, + ) diff --git a/src/coder_eval/protected_mock/server.py b/src/coder_eval/protected_mock/server.py new file mode 100644 index 00000000..9e5cf1ae --- /dev/null +++ b/src/coder_eval/protected_mock/server.py @@ -0,0 +1,425 @@ +"""Protected mock server: fixture-backed CLI service loaded host-side, queried by a thin client. + +Fixtures are read by this process only; the evaluated agent sees a generated +shim that forwards each invocation over a per-run socket. The endpoint is +chosen at bind time: an AF_UNIX socket in the run's scratch directory when a +real bind succeeds, else TCP on 127.0.0.1 with an ephemeral port. Requests must +carry the run's token; peer-credential checks (Unix ancillary credentials) run +only where available AND configured -- the Docker isolation layer will +configure them, the tempdir driver does not (same-user, nothing to verify +beyond the token). +""" + +from __future__ import annotations + +import argparse +import json +import os +import shutil +import socket +import socketserver +import struct +import subprocess +import threading +from dataclasses import dataclass +from pathlib import Path +from typing import Any, cast + +from .protocol import ( + ENDPOINT_FILE_NAME, + MAX_REQUEST_BYTES, + MAX_RESPONSE_BYTES, + PROTOCOL_VERSION, + SOCKET_FILE_NAME, + TOKEN_FILE_NAME, + format_tcp_endpoint, + format_unix_endpoint, +) + + +@dataclass(frozen=True) +class CommandResponse: + exit_code: int + stdout: str + stderr: str + + +@dataclass +class ToolState: + responses: dict[tuple[str, ...], CommandResponse] + normalized_responses: dict[tuple[str, ...], CommandResponse] + subset_responses: list[tuple[tuple[str, ...], CommandResponse]] + default: CommandResponse + remaining: int + passthrough_prefixes: tuple[tuple[str, ...], ...] + passthrough_executable: str | None + passthrough_cache: dict[tuple[str, ...], CommandResponse] + + +PASSTHROUGH_TIMEOUT_SECONDS = 60 +_NOISE_VALUE_FLAGS = frozenset({"--output"}) + + +def _expand_argv_tokens(argv: list[str]) -> list[str]: + """Flag-form-agnostic token stream: ``--flag=value`` split, noise flags dropped.""" + + expanded: list[str] = [] + for raw in argv: + if raw.startswith("-") and "=" in raw: + flag, value = raw.split("=", 1) + expanded.append(flag) + if value: + expanded.append(value) + else: + expanded.append(raw) + + cleaned: list[str] = [] + skip_next = False + for token in expanded: + if skip_next: + skip_next = False + continue + if token in _NOISE_VALUE_FLAGS: + skip_next = True + continue + cleaned.append(token) + return cleaned + + +def _normalized_argv(argv: list[str]) -> tuple[str, ...]: + """Canonical finite-command key: flag form/order agnostic, never subset matching.""" + + return tuple(sorted(_expand_argv_tokens(argv))) + + +def _response(raw: object, *, context: str) -> CommandResponse: + if not isinstance(raw, dict): + raise ValueError(f"{context} must be an object") + exit_code = raw.get("exit_code", 0) + stdout = raw.get("stdout", "") + stderr = raw.get("stderr", "") + if not isinstance(exit_code, int) or not 0 <= exit_code <= 255: + raise ValueError(f"{context}.exit_code must be an integer from 0 to 255") + if not isinstance(stdout, str) or not isinstance(stderr, str): + raise ValueError(f"{context} stdout/stderr must be strings") + encoded_size = len(stdout.encode("utf-8")) + len(stderr.encode("utf-8")) + if encoded_size > MAX_RESPONSE_BYTES // 2: + raise ValueError(f"{context} response exceeds the configured size limit") + return CommandResponse(exit_code=exit_code, stdout=stdout, stderr=stderr) + + +def _load_tool( + tool: str, + fixture_path: Path, + max_requests: int, + passthrough_prefixes: list[list[str]], +) -> ToolState: + raw = json.loads(fixture_path.read_text(encoding="utf-8")) + if not isinstance(raw, dict) or raw.get("version") != PROTOCOL_VERSION: + raise ValueError(f"fixture {fixture_path} must declare version {PROTOCOL_VERSION}") + entries = raw.get("responses") + if not isinstance(entries, list): + raise ValueError(f"fixture {fixture_path} responses must be a list") + responses: dict[tuple[str, ...], CommandResponse] = {} + normalized_responses: dict[tuple[str, ...], CommandResponse] = {} + # Ordered on purpose: subset rules are scanned in fixture-file order and the + # first match wins, so duplicates are legal (an earlier rule shadows a later + # one) -- the duplicate-key error applies to the finite match modes only. + subset_responses: list[tuple[tuple[str, ...], CommandResponse]] = [] + for index, entry in enumerate(entries): + if not isinstance(entry, dict): + raise ValueError(f"fixture {fixture_path} response {index} must be an object") + argv = entry.get("argv") + if not isinstance(argv, list) or not all(isinstance(item, str) for item in argv): + raise ValueError(f"fixture {fixture_path} response {index}.argv must be a string list") + match_mode = entry.get("match_mode", "exact") + if match_mode not in {"exact", "normalized", "subset"}: + raise ValueError(f"fixture {fixture_path} response {index}.match_mode must be exact, normalized, or subset") + if match_mode == "subset": + rule_tokens = tuple(_expand_argv_tokens(argv)) + if not argv or not rule_tokens: + raise ValueError(f"fixture {fixture_path} response {index}.argv must be non-empty for subset matching") + subset_responses.append((rule_tokens, _response(entry, context=f"response {index}"))) + continue + key = tuple(argv) + destination = responses if match_mode == "exact" else normalized_responses + command_key = key if match_mode == "exact" else _normalized_argv(argv) + if command_key in destination: + raise ValueError(f"fixture {fixture_path} contains duplicate argv {argv!r} for {match_mode} matching") + destination[command_key] = _response(entry, context=f"response {index}") + default = _response( + raw.get( + "default", + {"exit_code": 2, "stderr": "protected mock: command is not configured for this scenario\n"}, + ), + context="default", + ) + executable = shutil.which(tool) if passthrough_prefixes else None + if passthrough_prefixes and executable is None: + raise ValueError(f"protected mock passthrough tool is not installed: {tool}") + return ToolState( + responses=responses, + normalized_responses=normalized_responses, + subset_responses=subset_responses, + default=default, + remaining=max_requests, + passthrough_prefixes=tuple(tuple(prefix) for prefix in passthrough_prefixes), + passthrough_executable=executable, + passthrough_cache={}, + ) + + +def load_config(config_path: Path) -> dict[str, ToolState]: + raw = json.loads(config_path.read_text(encoding="utf-8")) + if not isinstance(raw, dict) or raw.get("version") != PROTOCOL_VERSION: + raise ValueError(f"mock config must declare version {PROTOCOL_VERSION}") + tools = raw.get("tools") + if not isinstance(tools, list) or not tools: + raise ValueError("mock config tools must be a non-empty list") + loaded: dict[str, ToolState] = {} + for entry in tools: + if not isinstance(entry, dict): + raise ValueError("mock config tool entries must be objects") + tool = entry.get("tool") + fixture = entry.get("fixture") + max_requests = entry.get("max_requests") + passthrough_prefixes = entry.get("passthrough_argv_prefixes", []) + if not isinstance(tool, str) or not tool or tool in loaded: + raise ValueError("mock config tools must have unique non-empty names") + if not isinstance(fixture, str) or not isinstance(max_requests, int) or max_requests < 1: + raise ValueError(f"mock config entry for {tool!r} has invalid fixture or max_requests") + if not isinstance(passthrough_prefixes, list) or not all( + isinstance(prefix, list) and prefix and all(isinstance(token, str) and token for token in prefix) + for prefix in passthrough_prefixes + ): + raise ValueError(f"mock config entry for {tool!r} has invalid passthrough prefixes") + loaded[tool] = _load_tool(tool, Path(fixture), max_requests, passthrough_prefixes) + return loaded + + +_UnixStreamServer: Any = getattr(socketserver, "UnixStreamServer", object) + + +class ProtectedMockServer: + """Transport-agnostic dispatch core; instantiated via a transport subclass. + + ``token`` gates every request (same-user hygiene). ``allowed_peer_uids`` + additionally gates by Unix socket peer credentials when set -- the Docker + isolation layer will set it; the tempdir driver leaves it ``None``. + """ + + tools: dict[str, ToolState] + budget_lock: threading.Lock + passthrough_lock: threading.Lock + token: str | None + allowed_peer_uids: frozenset[int] | None + + def _init_state(self, tools: dict[str, ToolState], token: str | None) -> None: + self.tools = tools + self.budget_lock = threading.Lock() + self.passthrough_lock = threading.Lock() + self.token = token + self.allowed_peer_uids = None + + def dispatch(self, tool: str, argv: list[str]) -> CommandResponse: + state = self.tools.get(tool) + if state is None: + return CommandResponse(127, "", "protected mock: unknown tool\n") + with self.budget_lock: + if state.remaining <= 0: + return CommandResponse(75, "", "protected mock: request budget exhausted\n") + state.remaining -= 1 + response = state.responses.get(tuple(argv)) + if response is None: + response = state.normalized_responses.get(_normalized_argv(argv)) + if response is None and state.subset_responses: + # Finite matches take precedence; subset rules scan in fixture-file + # order and the first whose tokens all appear in the invocation's + # normalized token set wins. + invocation_tokens = set(_expand_argv_tokens(argv)) + for rule_tokens, candidate in state.subset_responses: + if all(token in invocation_tokens for token in rule_tokens): + response = candidate + break + if response is not None: + return response + if any(tuple(argv[: len(prefix)]) == prefix for prefix in state.passthrough_prefixes): + return self._passthrough(state, argv) + return state.default + + def _passthrough(self, state: ToolState, argv: list[str]) -> CommandResponse: + key = tuple(argv) + with self.passthrough_lock: + cached = state.passthrough_cache.get(key) + if cached is not None: + return cached + if state.passthrough_executable is None: + return CommandResponse(69, "", "protected mock: passthrough is unavailable\n") + try: + result = subprocess.run( + [state.passthrough_executable, *argv], + stdin=subprocess.DEVNULL, + capture_output=True, + text=True, + encoding="utf-8", + errors="replace", + check=False, + timeout=PASSTHROUGH_TIMEOUT_SECONDS, + ) + exit_code = result.returncode if 0 <= result.returncode <= 255 else 70 + response = CommandResponse(exit_code, result.stdout, result.stderr) + except (OSError, subprocess.SubprocessError): + response = CommandResponse(70, "", "protected mock: passthrough failed\n") + encoded_size = len(response.stdout.encode("utf-8")) + len(response.stderr.encode("utf-8")) + if encoded_size > MAX_RESPONSE_BYTES // 2: + response = CommandResponse(70, "", "protected mock: passthrough response exceeds size limit\n") + state.passthrough_cache[key] = response + return response + + +class _TcpMockServer(ProtectedMockServer, socketserver.ThreadingMixIn, socketserver.TCPServer): + daemon_threads = True + allow_reuse_address = False + + def __init__(self, tools: dict[str, ToolState], token: str | None) -> None: + self._init_state(tools, token) + super().__init__(("127.0.0.1", 0), ProtectedMockHandler) + + +class _UnixMockServer(ProtectedMockServer, socketserver.ThreadingMixIn, _UnixStreamServer): + daemon_threads = True + + def __init__(self, path: str, tools: dict[str, ToolState], token: str | None) -> None: + self._init_state(tools, token) + super().__init__(path, ProtectedMockHandler) # pyright: ignore[reportCallIssue] + + +class ProtectedMockHandler(socketserver.StreamRequestHandler): + def handle(self) -> None: + server = cast(ProtectedMockServer, self.server) + line = self.rfile.readline(MAX_REQUEST_BYTES + 1) + if len(line) > MAX_REQUEST_BYTES or not line.endswith(b"\n"): + self._write(CommandResponse(64, "", "protected mock: invalid request size\n")) + return + try: + request: Any = json.loads(line.decode("utf-8")) + if not isinstance(request, dict) or request.get("version") != PROTOCOL_VERSION: + raise ValueError + tool = request.get("tool") + argv = request.get("argv") + if not isinstance(tool, str) or not isinstance(argv, list): + raise ValueError + if not all(isinstance(item, str) for item in argv): + raise ValueError + except (UnicodeError, ValueError): + self._write(CommandResponse(64, "", "protected mock: invalid request\n")) + return + if server.token is not None and request.get("token") != server.token: + self._write(CommandResponse(77, "", "protected mock: caller token rejected\n")) + return + if server.allowed_peer_uids is not None and self._peer_uid() not in server.allowed_peer_uids: + self._write(CommandResponse(77, "", "protected mock: caller identity rejected\n")) + return + self._write(server.dispatch(tool, argv)) + + def _peer_uid(self) -> int: + peer_cred = getattr(socket, "SO_PEERCRED", None) + if peer_cred is None: + raise RuntimeError("SO_PEERCRED is required for protected mock caller validation") + credentials = self.request.getsockopt(socket.SOL_SOCKET, peer_cred, struct.calcsize("3i")) + _pid, uid, _gid = struct.unpack("3i", credentials) + return uid + + def _write(self, response: CommandResponse) -> None: + payload = ( + json.dumps( + { + "version": PROTOCOL_VERSION, + "exit_code": response.exit_code, + "stdout": response.stdout, + "stderr": response.stderr, + }, + ensure_ascii=True, + separators=(",", ":"), + ).encode("utf-8") + + b"\n" + ) + if len(payload) > MAX_RESPONSE_BYTES: + payload = ( + json.dumps( + { + "version": PROTOCOL_VERSION, + "exit_code": 70, + "stdout": "", + "stderr": "protected mock: response exceeds size limit\n", + }, + separators=(",", ":"), + ).encode("utf-8") + + b"\n" + ) + self.wfile.write(payload) + + +def create_server( + tools: dict[str, ToolState], + token: str | None, + runtime_dir: Path, + transport: str = "auto", +) -> tuple[ProtectedMockServer, str]: + """Bind the service and return ``(server, endpoint string)``. + + ``transport``: ``"unix"`` forces AF_UNIX, ``"tcp"`` forces TCP loopback, + ``"auto"`` probes with a real AF_UNIX bind in ``runtime_dir`` and falls back + to TCP when the platform (or the path) does not support it. + """ + if transport not in {"auto", "unix", "tcp"}: + raise ValueError(f"unknown protected mock transport: {transport!r}") + if transport != "tcp" and getattr(socket, "AF_UNIX", None) is not None and _UnixStreamServer is not object: + socket_path = runtime_dir / SOCKET_FILE_NAME + socket_path.unlink(missing_ok=True) + try: + server: ProtectedMockServer = _UnixMockServer(str(socket_path), tools, token) + return server, format_unix_endpoint(str(socket_path)) + except (OSError, ValueError): + if transport == "unix": + raise + elif transport == "unix": + raise RuntimeError("AF_UNIX stream sockets are unavailable on this platform") + tcp_server = _TcpMockServer(tools, token) + host, port = tcp_server.server_address[:2] + return tcp_server, format_tcp_endpoint(str(host), int(port)) + + +def serve(config_path: Path, runtime_dir: Path, transport: str = "auto") -> None: + """Load fixtures, bind, publish the endpoint file, and serve until terminated. + + The endpoint file is written atomically (temp + rename) once the socket is + bound, so a reader that sees the file always sees a complete, live endpoint. + """ + token_file = runtime_dir / TOKEN_FILE_NAME + token = token_file.read_text(encoding="utf-8").strip() if token_file.is_file() else None + tools = load_config(config_path) + server, endpoint = create_server(tools, token, runtime_dir, transport) + endpoint_file = runtime_dir / ENDPOINT_FILE_NAME + endpoint_tmp = runtime_dir / (ENDPOINT_FILE_NAME + ".tmp") + try: + endpoint_tmp.write_text(endpoint + "\n", encoding="utf-8") + os.replace(endpoint_tmp, endpoint_file) + server.serve_forever(poll_interval=0.2) # pyright: ignore[reportAttributeAccessIssue] + finally: + server.server_close() # pyright: ignore[reportAttributeAccessIssue] + endpoint_file.unlink(missing_ok=True) + (runtime_dir / SOCKET_FILE_NAME).unlink(missing_ok=True) + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument("--config", required=True, type=Path) + parser.add_argument("--runtime-dir", required=True, type=Path) + parser.add_argument("--transport", default="auto", choices=["auto", "unix", "tcp"]) + args = parser.parse_args() + serve(args.config, args.runtime_dir, args.transport) + + +if __name__ == "__main__": + main() diff --git a/src/coder_eval/sandbox.py b/src/coder_eval/sandbox.py index 2748443f..bc408feb 100644 --- a/src/coder_eval/sandbox.py +++ b/src/coder_eval/sandbox.py @@ -12,6 +12,7 @@ from .invocation_log import render_recorder from .models import ( + PROTECTED_MOCK_DIR, RECORD_CLI_DIR, RECORD_CLI_LOG, RepoSource, @@ -19,6 +20,7 @@ StarterFilesSource, TemplateDirSource, ) +from .protected_mock.runtime import render_shim from .resources import get_ignore_patterns, should_ignore_path @@ -462,6 +464,13 @@ def resolved_mock_path_dirs(self) -> list[Path]: generated = self._resolve_within_sandbox(RECORD_CLI_DIR, field="record_cli directory") if generated.is_dir(): resolved.append(generated) + # Protected mock shims are generated by the orchestrator AFTER setup() + # (the service endpoint is only known once the server is up), so during + # setup this directory does not exist yet and is filtered out here. + if self.config.protected_mocks: + generated = self._resolve_within_sandbox(PROTECTED_MOCK_DIR, field="protected_mocks directory") + if generated.is_dir(): + resolved.append(generated) for rel in self.config.mock_path_dirs or []: candidate = self._resolve_within_sandbox(rel, field="mock_path_dirs entry") if candidate.is_dir(): @@ -559,6 +568,85 @@ def _generate_cli_recorders(self) -> None: + ", ".join(f"{s.tool}(exit {s.exit_code})" for s in self.config.record_cli) ) + def generate_protected_mock_shims(self, *, endpoint: str, token: str, call_log: Path) -> None: + """Write a client shim for every ``SandboxConfig.protected_mocks`` entry. + + Called by the orchestrator AFTER the protected mock server is up (the + per-run ``endpoint``/``token`` must be known) and before the agent + starts, so :attr:`resolved_mock_path_dirs` picks the directory up for + the agent's PATH. Each shim bakes the connection details in itself and + carries no fixture data. A ``.cmd`` twin is written beside it so a bare + tool name also resolves through Windows PATHEXT lookup. + + Raises: + RuntimeError: a task's own ``mock_path_dirs`` already provides an + executable with the same name (mirrors ``_generate_cli_recorders``). + """ + assert self.sandbox_dir is not None, "Sandbox directory not initialized" + if not self.config.protected_mocks: + return + + for rel in self.config.mock_path_dirs or []: + user_dir = self._resolve_within_sandbox(rel, field="mock_path_dirs entry") + if not user_dir.is_dir(): + continue + for spec in self.config.protected_mocks: + clash = next( + ( + user_dir / name + for name in (spec.tool, f"{spec.tool}.cmd", f"{spec.tool}.bat", f"{spec.tool}.exe") + if (user_dir / name).exists() + ), + None, + ) + if clash is not None: + msg = ( + f"protected_mocks would generate a '{spec.tool}' shim, but mock_path_dirs entry " + f"'{rel}' already provides one ({rel}/{clash.name}). " + "Remove the protected_mocks entry to keep your own mock, or drop the file to use " + "the protected mock service." + ) + raise RuntimeError(msg) + + shim_dir = self._resolve_within_sandbox(PROTECTED_MOCK_DIR, field="protected_mocks directory") + # Wipe rather than reuse for the same DIRECT_WRITE reason as record_cli: + # a reused --run-dir must not keep stale shims pointing at a dead endpoint. + if shim_dir.exists(): + shutil.rmtree(shim_dir, ignore_errors=True) + shim_dir.mkdir(parents=True, exist_ok=True) + + interpreter = os.path.realpath(sys.executable) + for spec in self.config.protected_mocks: + shim = shim_dir / spec.tool + shim.write_text( + render_shim( + spec.tool, + interpreter=interpreter, + endpoint=endpoint, + token=token, + call_log=str(call_log), + ), + encoding="utf-8", + newline="\n", + ) + shim.chmod(shim.stat().st_mode | 0o111) + cmd_lines = [ + "@echo off", + "REM Generated by coder_eval SandboxConfig.protected_mocks.", + "REM Windows PATHEXT lookup resolves this; POSIX uses the extensionless twin.", + f'"{interpreter}" "%~dp0{spec.tool}" %*', + ] + (shim_dir / f"{spec.tool}.cmd").write_text( + "\r\n".join(cmd_lines) + "\r\n", + encoding="utf-8", + newline="", + ) + + logger.info( + f"Generated {len(self.config.protected_mocks)} protected mock shim(s) in {PROTECTED_MOCK_DIR}/: " + + ", ".join(s.tool for s in self.config.protected_mocks) + ) + def _apply_starter_files_source(self, source: StarterFilesSource) -> None: """Create inline starter files in sandbox with overwrite tracking. diff --git a/tests/test_protected_mock.py b/tests/test_protected_mock.py new file mode 100644 index 00000000..96eed3eb --- /dev/null +++ b/tests/test_protected_mock.py @@ -0,0 +1,707 @@ +"""Tests for the protected fixture-backed CLI service and its thin wrappers.""" + +from __future__ import annotations + +import json +import socket +import socketserver +import subprocess +import sys +import threading +from pathlib import Path +from unittest.mock import MagicMock + +import pytest +from pydantic import ValidationError + +from coder_eval.models import ProtectedMockConfig, SandboxConfig +from coder_eval.protected_mock import client +from coder_eval.protected_mock.protocol import ( + CALL_LOG_ENV, + ENDPOINT_ENV, + TOKEN_ENV, + parse_endpoint, +) +from coder_eval.protected_mock.runtime import ( + ProtectedMockRuntime, + fixture_digest, + resolve_fixture_path, +) +from coder_eval.protected_mock.server import ProtectedMockServer, create_server, load_config +from coder_eval.sandbox import Sandbox + + +FIXTURE_MARKER = '{"errors":[]}' + + +def _fixture(path: Path) -> Path: + path.write_text( + json.dumps( + { + "version": 1, + "responses": [ + { + "argv": ["rpa", "get-errors", "--output", "json"], + "exit_code": 0, + "stdout": '{"errors":[]}\n', + } + ], + "default": {"exit_code": 2, "stderr": "not configured\n"}, + } + ), + encoding="utf-8", + ) + return path + + +def _write_config(config: Path, fixture: Path, max_requests: int = 10) -> Path: + config.write_text( + json.dumps( + { + "version": 1, + "tools": [{"tool": "uip", "fixture": str(fixture), "max_requests": max_requests}], + } + ), + encoding="utf-8", + ) + return config + + +def _fake_server(tools) -> MagicMock: + fake = MagicMock() + fake.tools = tools + fake.budget_lock = threading.Lock() + fake.passthrough_lock = threading.Lock() + return fake + + +def _unix_transport_usable(tmp_path: Path) -> bool: + af_unix = getattr(socket, "AF_UNIX", None) + if af_unix is None or not hasattr(socketserver, "UnixStreamServer"): + return False + probe = tmp_path / "probe.sock" + try: + with socket.socket(af_unix, socket.SOCK_STREAM) as sock: + sock.bind(str(probe)) + return True + except OSError: + return False + finally: + probe.unlink(missing_ok=True) + + +# --------------------------------------------------------------------------- # +# Model validation +# --------------------------------------------------------------------------- # + + +def test_protected_mocks_supported_under_tempdir(tmp_path: Path) -> None: + fixture = _fixture(tmp_path / "uip.json") + config = SandboxConfig( + driver="tempdir", + protected_mocks=[ProtectedMockConfig(tool="uip", fixture=str(fixture))], + ) + assert config.protected_mocks is not None and config.protected_mocks[0].tool == "uip" + + +def test_protected_mocks_fail_closed_under_docker(tmp_path: Path) -> None: + fixture = _fixture(tmp_path / "uip.json") + with pytest.raises(ValidationError, match="UID/GID isolation"): + SandboxConfig( + driver="docker", + protected_mocks=[ProtectedMockConfig(tool="uip", fixture=str(fixture))], + ) + + +def test_protected_mock_names_are_unique_and_do_not_collide_with_recorders(tmp_path: Path) -> None: + fixture = _fixture(tmp_path / "uip.json") + spec = {"tool": "uip", "fixture": str(fixture)} + with pytest.raises(ValidationError, match="must be unique"): + SandboxConfig(driver="tempdir", protected_mocks=[spec, spec]) # type: ignore[list-item] + with pytest.raises(ValidationError, match="cannot both provide"): + SandboxConfig( + driver="tempdir", + protected_mocks=[spec], # type: ignore[list-item] + record_cli=[{"tool": "uip"}], # type: ignore[list-item] + ) + + +def test_protected_mock_tool_name_is_validated() -> None: + with pytest.raises(ValidationError, match="bare executable name"): + ProtectedMockConfig(tool="dir/uip", fixture="f.json") + with pytest.raises(ValidationError, match="platform executable suffix"): + ProtectedMockConfig(tool="uip.exe", fixture="f.json") + + +def test_passthrough_prefixes_are_validated() -> None: + with pytest.raises(ValidationError, match="must be unique"): + ProtectedMockConfig( + tool="uip", + fixture="fixture.json", + passthrough_argv_prefixes=[["docsai", "ask"], ["docsai", "ask"]], + ) + with pytest.raises(ValidationError, match="1 to 8"): + ProtectedMockConfig(tool="uip", fixture="fixture.json", passthrough_argv_prefixes=[[]]) + + +# --------------------------------------------------------------------------- # +# Endpoint parsing +# --------------------------------------------------------------------------- # + + +def test_parse_endpoint_round_trips() -> None: + assert parse_endpoint("unix:/run/x/mock.sock") == ("unix", "/run/x/mock.sock") + # Windows socket paths carry a drive colon; the unix payload is verbatim. + assert parse_endpoint("unix:C:\\scratch\\mock.sock") == ("unix", "C:\\scratch\\mock.sock") + assert parse_endpoint("tcp:127.0.0.1:5001") == ("tcp", ("127.0.0.1", 5001)) + for bad in ["", "unix:", "tcp:127.0.0.1", "tcp:127.0.0.1:notaport", "tcp:127.0.0.1:0", "http://x"]: + with pytest.raises(ValueError, match="endpoint"): + parse_endpoint(bad) + + +# --------------------------------------------------------------------------- # +# Dispatch: exact / normalized / subset matching +# --------------------------------------------------------------------------- # + + +def test_fixture_service_matches_exact_argv_and_enforces_budget(tmp_path: Path) -> None: + fixture = _fixture(tmp_path / "uip.json") + config = _write_config(tmp_path / "config.json", fixture, max_requests=2) + fake_server = _fake_server(load_config(config)) + + expected = ProtectedMockServer.dispatch(fake_server, "uip", ["rpa", "get-errors", "--output", "json"]) + assert expected.exit_code == 0 + assert expected.stdout == '{"errors":[]}\n' + + # There is no generic file-read endpoint: an arbitrary path-bearing argv is + # merely an unmatched CLI command and receives the fixture's fixed default. + unmatched = ProtectedMockServer.dispatch(fake_server, "uip", ["read", "/etc/passwd"]) + assert unmatched.exit_code == 2 + assert unmatched.stderr == "not configured\n" + + exhausted = ProtectedMockServer.dispatch(fake_server, "uip", ["rpa", "get-errors", "--output", "json"]) + assert exhausted.exit_code == 75 + assert "budget exhausted" in exhausted.stderr + + +def test_unknown_tool_is_rejected(tmp_path: Path) -> None: + fixture = _fixture(tmp_path / "uip.json") + fake_server = _fake_server(load_config(_write_config(tmp_path / "config.json", fixture))) + response = ProtectedMockServer.dispatch(fake_server, "other", ["x"]) + assert response.exit_code == 127 + + +def test_fixture_rejects_duplicate_argv(tmp_path: Path) -> None: + fixture = tmp_path / "bad.json" + response = {"argv": ["same"], "exit_code": 0} + fixture.write_text(json.dumps({"version": 1, "responses": [response, response]}), encoding="utf-8") + config = _write_config(tmp_path / "config.json", fixture, max_requests=1) + + with pytest.raises(ValueError, match="duplicate argv"): + load_config(config) + + +def test_normalized_fixture_matching_remains_finite(tmp_path: Path) -> None: + fixture = tmp_path / "normalized.json" + fixture.write_text( + json.dumps( + { + "version": 1, + "responses": [ + { + "argv": ["rpa", "get-errors", "--job-id", "42"], + "match_mode": "normalized", + "exit_code": 0, + "stdout": "configured\n", + } + ], + "default": {"exit_code": 2, "stderr": "not configured\n"}, + } + ), + encoding="utf-8", + ) + fake_server = _fake_server(load_config(_write_config(tmp_path / "config.json", fixture, max_requests=2))) + + matched = ProtectedMockServer.dispatch( + fake_server, + "uip", + ["--job-id=42", "get-errors", "rpa", "--output", "json"], + ) + assert matched.stdout == "configured\n" + + extra_argument = ProtectedMockServer.dispatch( + fake_server, + "uip", + ["rpa", "get-errors", "--job-id", "42", "--include-secrets"], + ) + assert extra_argument.exit_code == 2 + + +def _subset_fixture(path: Path, responses: list[dict]) -> Path: + path.write_text( + json.dumps( + { + "version": 1, + "responses": responses, + "default": {"exit_code": 2, "stderr": "not configured\n"}, + } + ), + encoding="utf-8", + ) + return path + + +def test_subset_matching_is_order_independent_and_tolerates_extra_tokens(tmp_path: Path) -> None: + fixture = _subset_fixture( + tmp_path / "subset.json", + [{"argv": ["rpa", "get-errors"], "match_mode": "subset", "exit_code": 0, "stdout": "subset\n"}], + ) + fake_server = _fake_server(load_config(_write_config(tmp_path / "config.json", fixture))) + + # Extra tokens, reordering, and --flag=value form are all tolerated. + matched = ProtectedMockServer.dispatch(fake_server, "uip", ["get-errors", "--job-id=42", "rpa"]) + assert matched.stdout == "subset\n" + + # A rule token missing from the invocation is not a match. + missing = ProtectedMockServer.dispatch(fake_server, "uip", ["rpa", "list-jobs"]) + assert missing.exit_code == 2 + + +def test_subset_rules_scan_in_fixture_order_first_match_wins(tmp_path: Path) -> None: + fixture = _subset_fixture( + tmp_path / "subset.json", + [ + {"argv": ["rpa"], "match_mode": "subset", "exit_code": 0, "stdout": "broad\n"}, + {"argv": ["rpa", "get-errors"], "match_mode": "subset", "exit_code": 0, "stdout": "narrow\n"}, + ], + ) + fake_server = _fake_server(load_config(_write_config(tmp_path / "config.json", fixture))) + + # Both rules match; the earlier (broader) one wins because order decides. + response = ProtectedMockServer.dispatch(fake_server, "uip", ["rpa", "get-errors"]) + assert response.stdout == "broad\n" + + +def test_exact_and_normalized_take_precedence_over_subset(tmp_path: Path) -> None: + fixture = _subset_fixture( + tmp_path / "subset.json", + [ + {"argv": ["rpa", "get-errors"], "match_mode": "subset", "exit_code": 0, "stdout": "subset\n"}, + {"argv": ["rpa", "get-errors"], "exit_code": 0, "stdout": "exact\n"}, + { + "argv": ["rpa", "list-jobs", "--job-id", "42"], + "match_mode": "normalized", + "exit_code": 0, + "stdout": "normalized\n", + }, + {"argv": ["rpa", "list-jobs"], "match_mode": "subset", "exit_code": 0, "stdout": "subset-jobs\n"}, + ], + ) + fake_server = _fake_server(load_config(_write_config(tmp_path / "config.json", fixture))) + + assert ProtectedMockServer.dispatch(fake_server, "uip", ["rpa", "get-errors"]).stdout == "exact\n" + assert ( + ProtectedMockServer.dispatch(fake_server, "uip", ["--job-id=42", "list-jobs", "rpa"]).stdout == "normalized\n" + ) + # No exact/normalized hit -> the subset rule catches the variant. + assert ProtectedMockServer.dispatch(fake_server, "uip", ["rpa", "list-jobs", "--all"]).stdout == "subset-jobs\n" + + +def test_subset_duplicates_are_allowed_and_empty_argv_is_rejected(tmp_path: Path) -> None: + duplicate = {"argv": ["rpa"], "match_mode": "subset", "exit_code": 0, "stdout": "first\n"} + fixture = _subset_fixture(tmp_path / "dup.json", [duplicate, {**duplicate, "stdout": "second\n"}]) + fake_server = _fake_server(load_config(_write_config(tmp_path / "config.json", fixture))) + assert ProtectedMockServer.dispatch(fake_server, "uip", ["rpa"]).stdout == "first\n" + + empty = _subset_fixture(tmp_path / "empty.json", [{"argv": [], "match_mode": "subset", "exit_code": 0}]) + with pytest.raises(ValueError, match="non-empty for subset"): + load_config(_write_config(tmp_path / "config2.json", empty)) + + # Noise-flag-only argv normalizes to an empty token set: also rejected. + noise = _subset_fixture( + tmp_path / "noise.json", [{"argv": ["--output", "json"], "match_mode": "subset", "exit_code": 0}] + ) + with pytest.raises(ValueError, match="non-empty for subset"): + load_config(_write_config(tmp_path / "config3.json", noise)) + + +def test_passthrough_is_prefix_limited_and_cached(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + fixture = _fixture(tmp_path / "uip.json") + config = tmp_path / "config.json" + config.write_text( + json.dumps( + { + "version": 1, + "tools": [ + { + "tool": "uip", + "fixture": str(fixture), + "max_requests": 3, + "passthrough_argv_prefixes": [["docsai", "ask"]], + } + ], + } + ), + encoding="utf-8", + ) + monkeypatch.setattr("coder_eval.protected_mock.server.shutil.which", lambda _tool: "/usr/local/bin/uip") + run = MagicMock(return_value=subprocess.CompletedProcess([], 0, "answer\n", "")) + monkeypatch.setattr("coder_eval.protected_mock.server.subprocess.run", run) + fake_server = _fake_server(load_config(config)) + fake_server._passthrough.side_effect = lambda state, argv: ProtectedMockServer._passthrough( + fake_server, state, argv + ) + + argv = ["docsai", "ask", "what failed?"] + first = ProtectedMockServer.dispatch(fake_server, "uip", argv) + second = ProtectedMockServer.dispatch(fake_server, "uip", argv) + blocked = ProtectedMockServer.dispatch(fake_server, "uip", ["auth", "token"]) + + assert first.stdout == second.stdout == "answer\n" + assert blocked.exit_code == 2 + run.assert_called_once() + assert run.call_args.args[0] == ["/usr/local/bin/uip", *argv] + assert run.call_args.kwargs["stdin"] is subprocess.DEVNULL + + +# --------------------------------------------------------------------------- # +# Transport: in-process server + client over TCP loopback and AF_UNIX +# --------------------------------------------------------------------------- # + + +class _LiveServer: + """In-process threaded server around ``create_server`` for transport tests.""" + + def __init__(self, tmp_path: Path, transport: str, token: str | None = "test-token") -> None: + fixture = _fixture(tmp_path / "uip.json") + tools = load_config(_write_config(tmp_path / "config.json", fixture)) + self.token = token + self.server, self.endpoint = create_server(tools, token, tmp_path, transport) + self._thread = threading.Thread( + target=self.server.serve_forever, # pyright: ignore[reportAttributeAccessIssue] + kwargs={"poll_interval": 0.05}, + daemon=True, + ) + self._thread.start() + + def close(self) -> None: + self.server.shutdown() # pyright: ignore[reportAttributeAccessIssue] + self.server.server_close() # pyright: ignore[reportAttributeAccessIssue] + self._thread.join(timeout=3) + + +def _set_client_env(monkeypatch: pytest.MonkeyPatch, endpoint: str, token: str, call_log: Path) -> None: + monkeypatch.setenv(ENDPOINT_ENV, endpoint) + monkeypatch.setenv(TOKEN_ENV, token) + monkeypatch.setenv(CALL_LOG_ENV, str(call_log)) + + +def test_tcp_loopback_end_to_end_with_token( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str] +) -> None: + live = _LiveServer(tmp_path, transport="tcp") + try: + assert live.endpoint.startswith("tcp:127.0.0.1:") + call_log = tmp_path / "calls.jsonl" + _set_client_env(monkeypatch, live.endpoint, "test-token", call_log) + + exit_code = client.invoke("uip", ["rpa", "get-errors", "--output", "json"]) + assert exit_code == 0 + assert capsys.readouterr().out == '{"errors":[]}\n' + + record = json.loads(call_log.read_text(encoding="utf-8").splitlines()[0]) + assert record["tool"] == "uip" + assert record["argv"] == ["rpa", "get-errors", "--output", "json"] + assert record["exit"] == 0 + finally: + live.close() + + +def test_tcp_token_mismatch_is_rejected( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str] +) -> None: + live = _LiveServer(tmp_path, transport="tcp") + try: + _set_client_env(monkeypatch, live.endpoint, "wrong-token", tmp_path / "calls.jsonl") + exit_code = client.invoke("uip", ["rpa", "get-errors", "--output", "json"]) + assert exit_code == 77 + assert "token rejected" in capsys.readouterr().err + finally: + live.close() + + +def test_unix_socket_end_to_end_when_available( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str] +) -> None: + if not _unix_transport_usable(tmp_path): + pytest.skip("AF_UNIX stream sockets not usable on this platform") + live = _LiveServer(tmp_path, transport="unix") + try: + assert live.endpoint.startswith("unix:") + _set_client_env(monkeypatch, live.endpoint, "test-token", tmp_path / "calls.jsonl") + exit_code = client.invoke("uip", ["rpa", "get-errors", "--output", "json"]) + assert exit_code == 0 + assert capsys.readouterr().out == '{"errors":[]}\n' + finally: + live.close() + + +def test_client_reports_unreachable_service(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + call_log = tmp_path / "calls.jsonl" + _set_client_env(monkeypatch, "tcp:127.0.0.1:1", "t", call_log) + assert client.invoke("uip", ["anything"]) == 125 + record = json.loads(call_log.read_text(encoding="utf-8").splitlines()[0]) + assert record["exit"] == 125 + + +def test_client_main_and_size_guard(monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str]) -> None: + monkeypatch.delenv(CALL_LOG_ENV, raising=False) + monkeypatch.setenv(ENDPOINT_ENV, "tcp:127.0.0.1:1") + monkeypatch.setenv(TOKEN_ENV, "t") + + monkeypatch.setattr(sys, "argv", ["client"]) + assert client.main() == 64 + assert "missing tool name" in capsys.readouterr().err + + monkeypatch.setattr(sys, "argv", ["client", "uip", "x"]) + assert client.main() == 125 # unreachable endpoint + + # An oversized request is refused before any connection attempt. + assert client.invoke("uip", ["x" * (70 * 1024)]) == 125 + assert "request exceeds size limit" in capsys.readouterr().err + + +# --------------------------------------------------------------------------- # +# Runtime: subprocess lifecycle, per-run isolation, fixture resolution +# --------------------------------------------------------------------------- # + + +def test_runtime_end_to_end_and_teardown( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str] +) -> None: + fixture = _fixture(tmp_path / "uip.json") + mocks = [ProtectedMockConfig(tool="uip", fixture=str(fixture))] + runtime = ProtectedMockRuntime(mocks, task_dir=None) + with runtime: + assert runtime.endpoint_kind in {"unix", "tcp"} + assert runtime.fixture_digest == fixture_digest([fixture]) + _set_client_env(monkeypatch, runtime.endpoint, runtime.token, tmp_path / "calls.jsonl") + assert client.invoke("uip", ["rpa", "get-errors", "--output", "json"]) == 0 + assert capsys.readouterr().out == '{"errors":[]}\n' + process = runtime._process + runtime_dir = runtime._runtime_dir + assert process is not None and process.poll() is None + assert runtime_dir is not None and runtime_dir.is_dir() + # Teardown: the server process is gone and the scratch dir is removed. + assert process.poll() is not None + assert not runtime_dir.exists() + + +def test_two_runtimes_get_isolated_endpoints(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + fixture_a = _fixture(tmp_path / "a.json") + fixture_b = tmp_path / "b.json" + fixture_b.write_text( + json.dumps( + { + "version": 1, + "responses": [{"argv": ["ping"], "exit_code": 0, "stdout": "pong-b\n"}], + "default": {"exit_code": 2, "stderr": "not configured\n"}, + } + ), + encoding="utf-8", + ) + mocks_a = [ProtectedMockConfig(tool="uip", fixture=str(fixture_a))] + mocks_b = [ProtectedMockConfig(tool="uip", fixture=str(fixture_b))] + with ProtectedMockRuntime(mocks_a, task_dir=None) as run_a, ProtectedMockRuntime(mocks_b, task_dir=None) as run_b: + assert run_a.endpoint != run_b.endpoint + assert run_a.token != run_b.token + _set_client_env(monkeypatch, run_b.endpoint, run_b.token, tmp_path / "calls.jsonl") + assert client.invoke("uip", ["ping"]) == 0 + + +def test_runtime_start_fails_loudly_on_missing_fixture(tmp_path: Path) -> None: + mocks = [ProtectedMockConfig(tool="uip", fixture=str(tmp_path / "missing.json"))] + runtime = ProtectedMockRuntime(mocks, task_dir=None) + with pytest.raises(RuntimeError, match="fixture not found"): + runtime.start() + assert runtime._runtime_dir is None # scratch dir cleaned up on failure + + +def test_runtime_start_fails_loudly_on_bad_fixture(tmp_path: Path) -> None: + bad = tmp_path / "bad.json" + bad.write_text(json.dumps({"version": 999}), encoding="utf-8") + runtime = ProtectedMockRuntime([ProtectedMockConfig(tool="uip", fixture=str(bad))], task_dir=None) + with pytest.raises(RuntimeError, match="exited during startup"): + runtime.start() + + +def test_fixture_paths_resolve_against_task_dir(tmp_path: Path) -> None: + task_dir = tmp_path / "task" + (task_dir / "fixtures").mkdir(parents=True) + fixture = _fixture(task_dir / "fixtures" / "uip.json") + + resolved = resolve_fixture_path("./fixtures/uip.json", task_dir) + assert resolved == fixture.resolve() + + absolute = resolve_fixture_path(str(fixture), None) + assert absolute == fixture.resolve() + + with pytest.raises(RuntimeError, match="fixture not found"): + resolve_fixture_path("./fixtures/other.json", task_dir) + + +# --------------------------------------------------------------------------- # +# Sandbox shims +# --------------------------------------------------------------------------- # + + +def test_sandbox_generates_data_free_client_shims(tmp_path: Path) -> None: + fixture = _fixture(tmp_path / "uip.json") + config = SandboxConfig( + driver="tempdir", + python=None, + protected_mocks=[ProtectedMockConfig(tool="uip", fixture=str(fixture))], + ) + sandbox = Sandbox(config, task_id="protected-client") + workspace = tmp_path / "workspace" + try: + sandbox.setup(workspace) + # The shim dir does not exist until the orchestrator provides the + # endpoint, so PATH assembly at setup time skips it. + assert sandbox.resolved_mock_path_dirs == [] + + sandbox.generate_protected_mock_shims( + endpoint="tcp:127.0.0.1:5001", + token="run-token", + call_log=tmp_path / "run" / "protected_mock_calls.jsonl", + ) + shim = workspace / "protected_mocks" / "uip" + text = shim.read_text(encoding="utf-8") + assert "tcp:127.0.0.1:5001" in text + assert "run-token" in text + assert "coder_eval.protected_mock.client" in text + assert (workspace / "protected_mocks" / "uip.cmd").is_file() + assert sandbox.resolved_mock_path_dirs == [(workspace / "protected_mocks").resolve()] + + # No fixture bytes, and no fixture path, anywhere in the sandbox tree. + for path in workspace.rglob("*"): + if path.is_file(): + content = path.read_text(encoding="utf-8", errors="replace") + assert FIXTURE_MARKER not in content + assert str(fixture) not in content + finally: + sandbox.cleanup() + + +def test_shim_collision_with_mock_path_dirs_is_rejected(tmp_path: Path) -> None: + fixture = _fixture(tmp_path / "uip.json") + config = SandboxConfig( + driver="tempdir", + python=None, + mock_path_dirs=["mocks"], + protected_mocks=[ProtectedMockConfig(tool="uip", fixture=str(fixture))], + ) + sandbox = Sandbox(config, task_id="protected-clash") + workspace = tmp_path / "workspace" + try: + sandbox.setup(workspace) + mocks_dir = workspace / "mocks" + mocks_dir.mkdir() + (mocks_dir / "uip").write_text("#!/bin/sh\n", encoding="utf-8") + with pytest.raises(RuntimeError, match="already provides one"): + sandbox.generate_protected_mock_shims( + endpoint="tcp:127.0.0.1:5001", token="t", call_log=tmp_path / "calls.jsonl" + ) + finally: + sandbox.cleanup() + + +def test_shim_executes_against_live_service(tmp_path: Path) -> None: + """The generated shim is self-sufficient: no service env vars in the caller.""" + fixture = _fixture(tmp_path / "uip.json") + mocks = [ProtectedMockConfig(tool="uip", fixture=str(fixture))] + config = SandboxConfig(driver="tempdir", python=None, protected_mocks=mocks) + sandbox = Sandbox(config, task_id="protected-exec") + workspace = tmp_path / "workspace" + call_log = tmp_path / "run" / "protected_mock_calls.jsonl" + call_log.parent.mkdir(parents=True) + try: + sandbox.setup(workspace) + with ProtectedMockRuntime(mocks, task_dir=None) as runtime: + sandbox.generate_protected_mock_shims(endpoint=runtime.endpoint, token=runtime.token, call_log=call_log) + result = subprocess.run( + [sys.executable, str(workspace / "protected_mocks" / "uip"), "rpa", "get-errors", "--output", "json"], + capture_output=True, + text=True, + encoding="utf-8", + timeout=30, + ) + assert result.returncode == 0 + assert result.stdout == '{"errors":[]}\n' + record = json.loads(call_log.read_text(encoding="utf-8").splitlines()[0]) + assert record["argv"] == ["rpa", "get-errors", "--output", "json"] + finally: + sandbox.cleanup() + + +# --------------------------------------------------------------------------- # +# Orchestrator wiring (NoOpAgent, no network) +# --------------------------------------------------------------------------- # + + +async def test_orchestrator_starts_service_generates_shims_and_tears_down( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """Full lifecycle through Orchestrator.run(): start, audit record, teardown.""" + from coder_eval.config import settings + from coder_eval.models import AgentKind, ApiBackend, FileExistsCriterion, TaskDefinition, parse_agent_config + from coder_eval.orchestrator import Orchestrator + + fixture = _fixture(tmp_path / "uip.json") + task = TaskDefinition( + task_id="protected-orch", + description="d", + agent=parse_agent_config(type=AgentKind.NONE), + sandbox=SandboxConfig( + driver="tempdir", + python=None, + protected_mocks=[ProtectedMockConfig(tool="uip", fixture=str(fixture))], + ), + # The shim must be on disk while criteria run: proves the service was + # wired before the agent phase and the sandbox carries only the shim. + success_criteria=[FileExistsCriterion(description="shim exists", path="protected_mocks/uip")], + ) + + created: list[ProtectedMockRuntime] = [] + real_start = ProtectedMockRuntime.start + + def _recording_start(self: ProtectedMockRuntime) -> None: + created.append(self) + real_start(self) + + monkeypatch.setattr(ProtectedMockRuntime, "start", _recording_start) + monkeypatch.setattr(settings, "api_backend", ApiBackend.DIRECT) + + run_dir = tmp_path / "run" / task.task_id + run_dir.mkdir(parents=True) + orch = Orchestrator(task=task, run_dir=run_dir, variant_id="v") + result = await orch.run() + + from coder_eval.models import FinalStatus + + assert result.final_status == FinalStatus.SUCCESS + assert result.environment_info["protected_mock_endpoint_kind"] in {"unix", "tcp"} + assert result.environment_info["protected_mock_fixture_digest"] == fixture_digest([fixture]) + # Call log seeded next to task.json, outside the sandbox. + assert (run_dir / "protected_mock_calls.jsonl").is_file() + # The server never outlives the task: stopped and dereferenced in _cleanup. + assert orch._protected_mock_runtime is None + assert len(created) == 1 + assert created[0]._process is None + # The preserved sandbox carries the shim but no fixture bytes. + assert result.sandbox_path is not None + preserved = Path(result.sandbox_path) + assert (preserved / "protected_mocks" / "uip").is_file() + for path in preserved.rglob("*"): + if path.is_file(): + assert FIXTURE_MARKER not in path.read_text(encoding="utf-8", errors="replace") From 983c2ca4c69c18e6bfa40da633c31f4a45b4883e Mon Sep 17 00:00:00 2001 From: Dan Morosanu Date: Thu, 6 Aug 2026 19:46:27 +0300 Subject: [PATCH 2/4] fix(sandbox): bake sys.executable verbatim into protected mock shims to keep the venv --- src/coder_eval/sandbox.py | 5 ++++- tests/test_protected_mock.py | 26 ++++++++++++++++++++++++++ 2 files changed, 30 insertions(+), 1 deletion(-) diff --git a/src/coder_eval/sandbox.py b/src/coder_eval/sandbox.py index bc408feb..e931ad93 100644 --- a/src/coder_eval/sandbox.py +++ b/src/coder_eval/sandbox.py @@ -615,7 +615,10 @@ def generate_protected_mock_shims(self, *, endpoint: str, token: str, call_log: shutil.rmtree(shim_dir, ignore_errors=True) shim_dir.mkdir(parents=True, exist_ok=True) - interpreter = os.path.realpath(sys.executable) + # sys.executable verbatim, NOT realpath: on POSIX the venv python is a + # symlink to the base interpreter, and resolving it would drop the venv + # (and its site-packages) — the client must import coder_eval. + interpreter = sys.executable for spec in self.config.protected_mocks: shim = shim_dir / spec.tool shim.write_text( diff --git a/tests/test_protected_mock.py b/tests/test_protected_mock.py index 96eed3eb..e4db2c8e 100644 --- a/tests/test_protected_mock.py +++ b/tests/test_protected_mock.py @@ -593,6 +593,32 @@ def test_sandbox_generates_data_free_client_shims(tmp_path: Path) -> None: sandbox.cleanup() +def test_shim_bakes_harness_interpreter_verbatim(tmp_path: Path) -> None: + """The baked interpreter is ``sys.executable`` exactly, venv included. + + Regression: resolving it with realpath followed the POSIX venv symlink to + the base interpreter, where ``coder_eval`` is not installed, so the shim + died with ModuleNotFoundError on Linux. + """ + fixture = _fixture(tmp_path / "uip.json") + config = SandboxConfig( + driver="tempdir", + python=None, + protected_mocks=[ProtectedMockConfig(tool="uip", fixture=str(fixture))], + ) + sandbox = Sandbox(config, task_id="protected-interpreter") + workspace = tmp_path / "workspace" + try: + sandbox.setup(workspace) + sandbox.generate_protected_mock_shims( + endpoint="tcp:127.0.0.1:5001", token="t", call_log=tmp_path / "calls.jsonl" + ) + text = (workspace / "protected_mocks" / "uip").read_text(encoding="utf-8") + assert f"INTERPRETER = {sys.executable!r}" in text + finally: + sandbox.cleanup() + + def test_shim_collision_with_mock_path_dirs_is_rejected(tmp_path: Path) -> None: fixture = _fixture(tmp_path / "uip.json") config = SandboxConfig( From e834005007b86f2dedaeb3e2931c60caf9efab32 Mon Sep 17 00:00:00 2001 From: Dan Morosanu Date: Thu, 6 Aug 2026 20:09:50 +0300 Subject: [PATCH 3/4] fix(protected-mock): harden server startup and teardown against loaded-host timing --- src/coder_eval/protected_mock/runtime.py | 93 ++++++++++++++++++------ 1 file changed, 72 insertions(+), 21 deletions(-) diff --git a/src/coder_eval/protected_mock/runtime.py b/src/coder_eval/protected_mock/runtime.py index b4933505..ca0fc5eb 100644 --- a/src/coder_eval/protected_mock/runtime.py +++ b/src/coder_eval/protected_mock/runtime.py @@ -33,7 +33,15 @@ ) -STARTUP_TIMEOUT_SECONDS = 10.0 +# Generous on purpose: normal startup is well under a second, but a loaded box +# running many parallel test workers can stretch subprocess spawn + import + +# bind far past a tight deadline. The common case is unaffected -- the poll +# returns as soon as the endpoint file appears. +STARTUP_TIMEOUT_SECONDS = 30.0 + +# Child stderr is captured here (inside the runtime dir) so a startup failure +# or timeout can report what the server actually said. +SERVER_STDERR_NAME = "server-stderr.log" # Host-side per-task invocation log, written next to task.json in the run dir # (never inside the sandbox). Diagnostic surface: the `cli_called` criterion @@ -118,20 +126,24 @@ def start(self) -> None: ) self.token = secrets.token_hex(16) (self._runtime_dir / TOKEN_FILE_NAME).write_text(self.token + "\n", encoding="utf-8") - self._process = subprocess.Popen( - [ - sys.executable, - "-m", - "coder_eval.protected_mock.server", - "--config", - str(config_path), - "--runtime-dir", - str(self._runtime_dir), - "--transport", - self._transport, - ], - stdin=subprocess.DEVNULL, - ) + # Capture stderr to a file (not a pipe -- nothing drains a pipe, and a + # full one would deadlock the child) so failures are diagnosable. + with (self._runtime_dir / SERVER_STDERR_NAME).open("wb") as stderr_sink: + self._process = subprocess.Popen( + [ + sys.executable, + "-m", + "coder_eval.protected_mock.server", + "--config", + str(config_path), + "--runtime-dir", + str(self._runtime_dir), + "--transport", + self._transport, + ], + stdin=subprocess.DEVNULL, + stderr=stderr_sink, + ) self.endpoint = self._await_endpoint() except Exception: self.stop() @@ -140,16 +152,37 @@ def start(self) -> None: def _await_endpoint(self) -> str: assert self._runtime_dir is not None and self._process is not None endpoint_file = self._runtime_dir / ENDPOINT_FILE_NAME - deadline = time.monotonic() + STARTUP_TIMEOUT_SECONDS + started = time.monotonic() + deadline = started + STARTUP_TIMEOUT_SECONDS while time.monotonic() < deadline: if self._process.poll() is not None: - raise RuntimeError(f"protected mock server exited during startup with code {self._process.returncode}") + raise RuntimeError( + f"protected mock server exited during startup with code {self._process.returncode}" + + self._server_stderr_suffix() + ) if endpoint_file.is_file(): endpoint = endpoint_file.read_text(encoding="utf-8").strip() if endpoint: return endpoint time.sleep(0.02) - raise RuntimeError(f"protected mock server did not publish its endpoint within {STARTUP_TIMEOUT_SECONDS}s") + waited = time.monotonic() - started + raise RuntimeError( + f"protected mock server did not publish its endpoint within {waited:.1f}s " + f"(deadline {STARTUP_TIMEOUT_SECONDS}s)" + self._server_stderr_suffix() + ) + + def _server_stderr_suffix(self) -> str: + """Tail of the child's captured stderr, formatted for an error message.""" + if self._runtime_dir is None: + return "" + stderr_file = self._runtime_dir / SERVER_STDERR_NAME + try: + text = stderr_file.read_text(encoding="utf-8", errors="replace").strip() + except OSError: + return "" + if not text: + return "" + return f"; server stderr (tail): {text[-2000:]}" def stop(self) -> None: """Terminate the server (kill on a slow exit) and remove the scratch dir.""" @@ -157,16 +190,34 @@ def stop(self) -> None: if self._process.poll() is None: self._process.terminate() try: - self._process.wait(timeout=3) + # Generous under load: the graceful window only delays the + # slow path, and kill() below is unconditional force. + self._process.wait(timeout=5) except subprocess.TimeoutExpired: self._process.kill() with contextlib.suppress(subprocess.TimeoutExpired): - self._process.wait(timeout=3) + self._process.wait(timeout=5) self._process = None if self._runtime_dir is not None: - shutil.rmtree(self._runtime_dir, ignore_errors=True) + self._remove_runtime_dir(self._runtime_dir) self._runtime_dir = None + @staticmethod + def _remove_runtime_dir(runtime_dir: Path) -> None: + """Remove the scratch dir, retrying briefly. + + Windows releases a killed child's file handles asynchronously after + ``wait()`` returns, so an immediate rmtree can silently strand the + directory. Still best-effort: gives up without raising after the + deadline -- teardown must never take the run down. + """ + deadline = time.monotonic() + 10.0 + while True: + shutil.rmtree(runtime_dir, ignore_errors=True) + if not runtime_dir.exists() or time.monotonic() >= deadline: + return + time.sleep(0.05) + def __enter__(self) -> ProtectedMockRuntime: self.start() return self From 92e6e1174ce512e509fb7f110d79f4eb19173ffb Mon Sep 17 00:00:00 2001 From: Dan Morosanu Date: Thu, 6 Aug 2026 20:47:55 +0300 Subject: [PATCH 4/4] fix(sandbox): absolutize the protected mock call-log path before baking it into shims --- src/coder_eval/protected_mock/server.py | 4 ++++ src/coder_eval/sandbox.py | 6 +++++ tests/test_protected_mock.py | 31 +++++++++++++++++++++++++ 3 files changed, 41 insertions(+) diff --git a/src/coder_eval/protected_mock/server.py b/src/coder_eval/protected_mock/server.py index 9e5cf1ae..03bdc814 100644 --- a/src/coder_eval/protected_mock/server.py +++ b/src/coder_eval/protected_mock/server.py @@ -396,6 +396,10 @@ def serve(config_path: Path, runtime_dir: Path, transport: str = "auto") -> None The endpoint file is written atomically (temp + rename) once the socket is bound, so a reader that sees the file always sees a complete, live endpoint. """ + # The runtime always passes an absolute dir (tempfile.mkdtemp); resolve + # anyway so a hand-launched relative --runtime-dir cannot publish a + # relative socket path in the endpoint file. + runtime_dir = runtime_dir.resolve() token_file = runtime_dir / TOKEN_FILE_NAME token = token_file.read_text(encoding="utf-8").strip() if token_file.is_file() else None tools = load_config(config_path) diff --git a/src/coder_eval/sandbox.py b/src/coder_eval/sandbox.py index e931ad93..07dc64e6 100644 --- a/src/coder_eval/sandbox.py +++ b/src/coder_eval/sandbox.py @@ -586,6 +586,12 @@ def generate_protected_mock_shims(self, *, endpoint: str, token: str, call_log: if not self.config.protected_mocks: return + # Absolutize: a relative --run-dir would otherwise bake a relative log + # path that resolves against the AGENT's cwd (the sandbox) at invocation + # time, so every append fails. Resolved here, against the harness cwd, + # where the caller's intent (run-dir-relative) still holds. + call_log = call_log.resolve() + for rel in self.config.mock_path_dirs or []: user_dir = self._resolve_within_sandbox(rel, field="mock_path_dirs entry") if not user_dir.is_dir(): diff --git a/tests/test_protected_mock.py b/tests/test_protected_mock.py index e4db2c8e..08b25b2f 100644 --- a/tests/test_protected_mock.py +++ b/tests/test_protected_mock.py @@ -619,6 +619,37 @@ def test_shim_bakes_harness_interpreter_verbatim(tmp_path: Path) -> None: sandbox.cleanup() +def test_shim_bakes_absolute_call_log_path(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + """A relative call-log path (relative --run-dir) is absolutized before baking. + + Regression: baked relative, it resolved against the AGENT's cwd (the + sandbox) at invocation time, so every log append failed with + FileNotFoundError echoed into agent-visible stderr. + """ + fixture = _fixture(tmp_path / "uip.json") + config = SandboxConfig( + driver="tempdir", + python=None, + protected_mocks=[ProtectedMockConfig(tool="uip", fixture=str(fixture))], + ) + sandbox = Sandbox(config, task_id="protected-relative-log") + workspace = tmp_path / "workspace" + monkeypatch.chdir(tmp_path) # pin the cwd the relative path resolves against + try: + sandbox.setup(workspace) + sandbox.generate_protected_mock_shims( + endpoint="tcp:127.0.0.1:5001", + token="t", + call_log=Path("run/protected_mock_calls.jsonl"), + ) + text = (workspace / "protected_mocks" / "uip").read_text(encoding="utf-8") + expected = (tmp_path / "run" / "protected_mock_calls.jsonl").resolve() + assert f"{CALL_LOG_ENV!r}: {str(expected)!r}" in text + assert f"{CALL_LOG_ENV!r}: 'run" not in text # no relative form anywhere + finally: + sandbox.cleanup() + + def test_shim_collision_with_mock_path_dirs_is_rejected(tmp_path: Path) -> None: fixture = _fixture(tmp_path / "uip.json") config = SandboxConfig(