From 35b8ce311a60a1abbcadb60f2a80d5d1bc82d186 Mon Sep 17 00:00:00 2001 From: Seth Carlson Date: Mon, 26 Oct 2026 15:45:17 -0700 Subject: [PATCH] Close eleven SyberEval core holes found by adversarial verification; pin five design gaps Independent verification of the eval core (#137) before the GitHits A/B pilot. Every finding has a test in tests/test_eval_adversarial.py; each fixed one failed before its fix. Fixes are confined to syberlabs/eval/*. Fixed: - run ids could collide (task and arm names may hold dots): one cell's bundle overwrote another's; spec.check now refuses colliding run ids - a run that raised after spending was charged nothing, so every retry spent again under the cap; runs/failed.jsonl records its spend and the cap counts it - candidate code could rewrite the host's copy of the hidden checks and get a later candidate accepted; the runner refuses to accept on a changed copy (not_acceptable, hidden_tampered) - every run of a githits arm shared X-GitHits-Session-ID s1-b and agent syberfactory/b; the run's thread and the arm's name now label them - NaN passed every numeric bound of the spec schema (a NaN total_dollars disabled the cap); numbers must be finite and read_json is strict (no NaN, Infinity or repeated keys) - the preflight gate was checked once per invocation; it is checked before every run - bundles were never verified: edited evidence or dollars, a bundle in another run's directory and a .partial were read as done; EvalRun.verify checks them (bundle_tampered) - two runners could run one eval at once and pass the cap together; runs/.lock - a fixture's base honoured the host's global git ignore rules (an empty base on a host ignoring *.py); git add --force - write_task wrote hidden files named ../ or absolute outside the task Pinned as expectedFailure (design decisions): a candidate that exits 0 at import passes every hidden check; a candidate can print its hidden check into the next repair request; a claude-code arm's served model is never compared with its declared model; repo: task repositories hold commits after the base; a runner killed outright charges nothing to the cap. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01HCTqqTVthqCzZJu6MWasuz --- syberlabs/eval/arms.py | 17 +- syberlabs/eval/runner.py | 176 ++++++++++- syberlabs/eval/spec.py | 35 ++- syberlabs/eval/tasks.py | 49 ++- tests/test_eval_adversarial.py | 531 +++++++++++++++++++++++++++++++++ 5 files changed, 783 insertions(+), 25 deletions(-) create mode 100644 tests/test_eval_adversarial.py diff --git a/syberlabs/eval/arms.py b/syberlabs/eval/arms.py index 50af10e..5a8a7da 100644 --- a/syberlabs/eval/arms.py +++ b/syberlabs/eval/arms.py @@ -12,7 +12,8 @@ - ``evolve_jev``: the same provider with ``JevPolicy``, its price prior from the contract's price table; - ``claude-code``: ``claude_code.ClaudeCodeProvider`` with ``lane``, ``bare``, ``trusted_tree``, ``tools``, ``max_turns``, ``mcp_config`` and ``executable`` (a stand-in ``claude`` for tests), and ``githits`` passed through - by name to the provider's keyword of that name (refused ``arm_unsupported`` when the installed provider has none). + by name to the provider's keyword of that name (refused ``arm_unsupported`` when the installed provider has none); + a ``githits`` arm's lookups carry the run's thread and the arm's name (``X-GitHits-Session-ID``, ``-Agent``). The runner does not set Claude Code's model: the arm's ``model`` and ``effort`` are the spec's declaration, and the model that served is in the candidate's ``signal.harness``. @@ -27,6 +28,7 @@ import inspect import os +import re import statistics import sys from contextlib import contextmanager @@ -88,8 +90,15 @@ def jev_prices(prices: dict | None) -> dict | None: return {"mutate": round(statistics.mean(costs), 6), "crossover": 0.0, "evaluate": 0.0} -def build(arm: dict, *, repo: Path, budget: dict, prices: dict | None, harness: Path = HARNESS): - """The provider for one run of ``arm`` against the repository at ``repo``.""" +def githits_arm(name: str) -> str: + """The arm's name as the GitHits client's arm label (``[A-Za-z0-9_-]{1,16}``, ``X-GitHits-Agent``).""" + return re.sub(r"[^A-Za-z0-9_-]", "-", name)[:16] + + +def build(arm: dict, *, repo: Path, budget: dict, prices: dict | None, harness: Path = HARNESS, + thread_label: str | None = None): + """The provider for one run of ``arm`` against the repository at ``repo``. ``thread_label`` (the run's thread + id) and the arm's name label a ``githits`` arm's lookups, so each run is its own GitHits session.""" kind, k = arm["kind"], calls(arm, budget) timeout = arm.get("timeout_seconds", 600) name = arm["name"] if arm["name"][0].isalpha() else "arm-" + arm["name"] @@ -120,6 +129,8 @@ def build(arm: dict, *, repo: Path, budget: dict, prices: dict | None, harness: if "githits" not in inspect.signature(ClaudeCodeProvider.__init__).parameters: raise Rejected("arm_unsupported", "this ClaudeCodeProvider has no githits option") options["githits"] = arm["githits"] + if arm["githits"]: + options.update(arm=githits_arm(arm["name"]), thread_label=thread_label) return ClaudeCodeProvider(repo, **options) diff --git a/syberlabs/eval/runner.py b/syberlabs/eval/runner.py index 406a8cb..1aeb99e 100644 --- a/syberlabs/eval/runner.py +++ b/syberlabs/eval/runner.py @@ -14,7 +14,9 @@ run's seed, in the arm's environment (``arms.environment``), the host's governor pricing every call; 4. the candidate the search recommends (else its last in-scope candidate) is checked by the host (``Thread.check``: every check on the exact candidate tree) and accepted only if admission allows it - (``Thread.accept``, which moves the thread's branch in the run's own repository and nothing else); + (``Thread.accept``, which moves the thread's branch in the run's own repository and nothing else) and the + host's copy of the hidden files is still what was staged (else ``not_acceptable``, ``hidden_tampered``: candidate + code runs inside the checks with the host's permissions; this detects a rewrite that stays, not one undone); 5. evidence: the exported journal, the provider's trace, the run contract, a ``git bundle`` of the run's repository, and, when the run accepted a candidate and the seed variable is set, a certificate signed with that key and verified at once against the repository and the journal (``uncertified`` otherwise); @@ -23,11 +25,15 @@ of the evidence files with every file's SHA-256, and the outcome. A run's files are written into ``.partial`` and renamed when complete, so a run killed midway leaves no -bundle and is run again; a run with a bundle is skipped (resumable by run id). Before a run starts, the dollars the -eval's bundles record plus the spec's per-run dollars must fit under ``budget.total_dollars``, or the eval stops +bundle and is run again; a run with a bundle is skipped (resumable by run id). A bundle is read only when it is the +evidence of the run its directory names (``EvalRun.verify``; ``bundle_tampered`` otherwise). Before a run starts, +the dollars the eval's bundles record, plus what every run that raised had spent (``runs/failed.jsonl``, written +as it raised), plus the spec's per-run dollars must fit under ``budget.total_dollars``, or the eval stops ``eval_budget_exhausted``; within a run the governor's bound applies (the call that crosses ``max_dollars`` is recorded and the next is refused, ``docs/BUILD_THREAD.md``). A run whose spend names a model the price table does -not price stops the eval (``eval_spend_unknown``): its cost cannot be counted against the cap. +not price stops the eval (``eval_spend_unknown``): its cost cannot be counted against the cap. One runner holds an +eval at a time (``runs/.lock``; a second is refused ``eval_running``), and the preflight gate is asked again before +every run, so a checkout that moves or changes mid-eval stops it. ``preflight`` runs the deterministic suites (``scripts/run_shards.py``, ``conformance.run``, ``benchmarks/run_scenarios.py``, and this package's tests) from the checkout and records the result for the @@ -37,6 +43,7 @@ from __future__ import annotations import hashlib +import json import os import platform import random @@ -45,15 +52,28 @@ import sys import tempfile import time +from contextlib import contextmanager from dataclasses import dataclass from pathlib import Path -from typing import Callable +from typing import Callable, Iterator from syberlabs.errors import Rejected from syberlabs.eval import VERSION, arms, spec as specs, tasks from syberlabs.eval.spec import HARNESS, harness_state, now_utc, read_json, write_json +try: + import fcntl +except ImportError: # pragma: no cover - Windows + fcntl = None +try: + import msvcrt +except ImportError: # pragma: no cover - not Windows + msvcrt = None + RUNNER_ACTOR = "syberlabs.eval" +LOCK = ".lock" # runs/.lock: one runner per eval at a time +FAILED = "failed.jsonl" # runs/failed.jsonl: what each run that raised had spent, counted under the cap +HIDDEN_TAMPERED = "hidden_tampered" # the host's copy of the hidden files changed while the run's checks ran CONTRACT_ID = "eval-run" BUNDLE_SCHEMA_ID = "syberlabs.eval/bundle.v1" PREFLIGHT_SCHEMA_ID = "syberlabs.eval/preflight.v1" @@ -158,6 +178,32 @@ def _git_bundle(repo: Path, out: Path) -> bool: return done.returncode == 0 and out.is_file() +@contextmanager +def _exclusive(path: Path) -> Iterator[None]: + """Hold ``path`` locked for one runner (``flock``, or ``msvcrt.locking`` on Windows); refused, not waited for, + when another process holds it (``eval_running``): two runners would each count only the bundles they found + and pass the eval's dollar cap together, and both would start the same run.""" + path.parent.mkdir(parents=True, exist_ok=True) + handle = open(path, "a+b") + locked = False + try: + try: + if fcntl is not None: + fcntl.flock(handle.fileno(), fcntl.LOCK_EX | fcntl.LOCK_NB) + elif msvcrt is not None: + os.lseek(handle.fileno(), 0, os.SEEK_SET) + msvcrt.locking(handle.fileno(), msvcrt.LK_NBLCK, 1) + locked = True + except OSError: + raise Rejected("eval_running", f"another runner holds {path}; wait for it to finish") from None + yield + finally: + if locked and fcntl is None and msvcrt is not None: + os.lseek(handle.fileno(), 0, os.SEEK_SET) + msvcrt.locking(handle.fileno(), msvcrt.LK_UNLCK, 1) + handle.close() # releases the flock + + class EvalRun: """One eval: the registered spec, the order, the cap, and the runs.""" @@ -176,15 +222,74 @@ def __init__(self, evals_root: str | Path, eval_id: str, *, harness: Path = HARN # Bundles already written def bundles(self) -> dict[str, dict]: + """Every completed run's bundle, by run id; ``.partial`` directories are runs that did not complete. + A bundle under another spec digest is refused (``bundle_mismatch``), and so is one that is not the evidence + of the run its directory names (``bundle_tampered``, ``verify``).""" found = {} + planned = {item.run_id: item for item in plan(self.doc)} if self.runs_dir.is_dir(): for path in sorted(self.runs_dir.glob("*/bundle.json")): + if path.parent.name.endswith(".partial"): + continue bundle = read_json(path, "bundle_unreadable") - if bundle.get("spec_digest") != self.registration["spec_digest"]: + if not isinstance(bundle, dict) or bundle.get("spec_digest") != self.registration["spec_digest"]: raise Rejected("bundle_mismatch", f"{path} was written under another spec digest") + reason = self.verify(path.parent, bundle, planned.get(path.parent.name)) + if reason: + raise Rejected("bundle_tampered", f"{path}: {reason}") found[bundle["run_id"]] = bundle return found + def verify(self, folder: Path, bundle: dict, item: Planned | None) -> str | None: + """Why ``bundle`` is not the evidence of the planned run ``item`` its directory names, or None: its identity + and digests are the registration's, every file it lists is there with its SHA-256 and nothing else is, and + the dollars, unknown models and acceptance it reports are its journal's. Edits are detected, not prevented: + nothing signs ``bundle.json``, so a writer who rewrites every digest consistently is not.""" + from syberlabs.evolution import spend_total + + if item is None: + return "its directory names no run of this eval" + expected = {"schema": BUNDLE_SCHEMA_ID, "eval": self.eval_id, "run_id": item.run_id, "task": item.task, + "arm": item.arm, "seed": item.seed, "order_index": item.order_index, + "task_digest": self.registration["task_digests"].get(item.task), + "arm_digest": self.registration["arm_digests"].get(item.arm), "journal_path": "journal.json"} + wrong = sorted(key for key, value in expected.items() if bundle.get(key) != value) + if wrong: + return "it is not this run's: " + ", ".join(wrong) + files, present = bundle.get("files"), sorted(p.name for p in folder.iterdir() if p.name != "bundle.json") + if not isinstance(files, dict) or sorted(files) != present: + return "the files beside it are not the files it lists" + changed = [name for name in present if not (folder / name).is_file() or _sha256(folder / name) != files[name]] + if changed: + return "changed since the run: " + ", ".join(changed) + journal = read_json(folder / "journal.json", "bundle_unreadable") + events = journal.get("events") if isinstance(journal, dict) else None + outcome = bundle.get("outcome") + if not isinstance(events, list) or not isinstance(outcome, dict): + return "no journal events or no outcome" + spend = spend_total(events) + if (outcome.get("dollars"), outcome.get("unknown_models")) != (spend["dollars"], spend["unknown_models"]): + return "its dollars are not its journal's" + if (outcome.get("status") == "accepted") != any(e.get("kind") == "effect_succeeded" for e in events): + return "its acceptance is not its journal's" + return None + + def failed(self) -> list[dict]: + """The runs that raised after they started (``failed.jsonl``, ``run_one``): what each had spent.""" + path = self.runs_dir / FAILED + rows = [] + if path.is_file(): + for line in path.read_text(encoding="utf-8").splitlines(): + try: + row = json.loads(line) + except ValueError: + row = None + if not isinstance(row, dict): + raise Rejected("ledger_unreadable", f"{path}: a line is not a record; the spend of a failed run " + "cannot be counted") + rows.append(row) + return rows + @staticmethod def dollars(bundle: dict) -> float: return float((bundle.get("outcome") or {}).get("dollars") or 0.0) @@ -200,10 +305,17 @@ def run(self, *, only: str | None = None, dry: bool = False, require_preflight: if not dry: raise gate = {"refused": exc.code} + if dry: + return self._run(only, True, require_preflight, gate) + with _exclusive(self.runs_dir / LOCK): + return self._run(only, False, require_preflight, gate) + + def _run(self, only: str | None, dry: bool, require_preflight: bool, gate: dict | None) -> dict: planned = select(plan(self.doc), only) - done = self.bundles() + done, failed = self.bundles(), self.failed() budget = self.doc["budget"] - spent = sum(self.dollars(b) for b in done.values()) + # The cap counts what the bundles record and what every run that raised had spent before it raised. + spent = sum(self.dollars(b) for b in done.values()) + sum(float(f.get("dollars") or 0.0) for f in failed) rows, stopped = [], None seed = None if not dry and os.environ.get(self.seed_env): @@ -217,11 +329,22 @@ def run(self, *, only: str | None = None, dry: bool = False, require_preflight: if dry: rows.append({"run_id": item.run_id, "order_index": item.order_index, "state": "pending"}) continue + if any(f.get("dollars") is None or f.get("unknown_models") for f in failed): + stopped = "eval_spend_unknown" + self.log("stop: eval_spend_unknown (a run that raised spent what the price table cannot count)") + break if spent + budget["dollars"] > budget["total_dollars"]: stopped = "eval_budget_exhausted" self.log(f"stop: eval_budget_exhausted (spent ${spent:.4f}; a run may cost ${budget['dollars']:.4f}; " f"cap ${budget['total_dollars']:.4f})") break + if require_preflight: + try: # again before every run: the checkout may have moved or changed since the eval started + preflight_gate(self.root, self.eval_id, self.harness) + except Rejected as exc: + stopped = exc.code + self.log(f"stop: {exc.code} ({exc.detail})") + break self.log(f"run {item.order_index} {item.run_id}") bundle = self.run_one(item, seed) spent += self.dollars(bundle) @@ -257,18 +380,20 @@ def run_one(self, item: Planned, seed: bytes | None) -> dict: shutil.rmtree(partial) partial.mkdir(parents=True) repo_dir, home, host = work / "repo", work / "kit", work / "host" - kit = None + kit = thread = None try: base = tasks.materialize(task, repo_dir) staged = tasks.stage_hidden(self.tasks_root, item.task, host / tasks.HIDDEN, self.registration["task_digests"][item.task]) + intact = tasks.tree_digests(staged) checks, required = tasks.contract_checks(task, staged) contract = run_contract(task, checks, required, budget) write_json(home / "contracts" / f"{CONTRACT_ID}.v1.json", contract) kit = Kit.local(home, repo_dir, actor=RUNNER_ACTOR) thread = kit.start(task["objective"], f"{CONTRACT_ID}.v1", paths=task["scope"]["paths"], act_class="R", unattended=True) - provider = arms.build(arm, repo=repo_dir, budget=budget, prices=budget["prices"], harness=self.harness) + provider = arms.build(arm, repo=repo_dir, budget=budget, prices=budget["prices"], harness=self.harness, + thread_label=thread.id) with arms.environment(arm, task, withheld=(self.seed_env, DEFAULT_SEED_ENV)): thread.propose(provider, seed=item.seed, max_seconds=budget["seconds"]) events, views = thread.history(), thread._views() @@ -276,10 +401,16 @@ def run_one(self, item: Planned, seed: bytes | None) -> dict: verdict = receipt = None if chosen is not None: verdict = thread.check(chosen) - if verdict.acceptable: - receipt = thread.accept(chosen) + # Candidate code runs inside the hidden checks with the host's permissions, and the host's copy is an + # ordinary directory: a copy that changed while they ran judged nothing, so nothing is accepted on it. + changed = tasks.hidden_changes(intact, tasks.tree_digests(staged)) + if verdict is not None and verdict.acceptable and not changed: + receipt = thread.accept(chosen) events, views = thread.history(), thread._views() outcome = self._summary(events, views, chosen, verdict, receipt, provider, spend_total(events)) + outcome["hidden_intact"], outcome["hidden_changed"] = not changed, changed[:16] + if changed and chosen is not None: + outcome.update(status="not_acceptable", reason=HIDDEN_TAMPERED) certified = self._certify(kit, thread, chosen, receipt, seed, partial, repo_dir, home) outcome["certification"] = certified["state"] write_json(partial / "journal.json", kit.export(thread.id)) @@ -312,8 +443,9 @@ def run_one(self, item: Planned, seed: bytes | None) -> dict: } bundle["files"] = {p.name: _sha256(p) for p in sorted(partial.iterdir()) if p.is_file()} write_json(partial / "bundle.json", bundle) - except BaseException: + except BaseException as exc: shutil.rmtree(partial, ignore_errors=True) + self._failed(item, thread, exc) raise finally: if kit is not None: @@ -326,6 +458,24 @@ def run_one(self, item: Planned, seed: bytes | None) -> dict: partial.rename(final) return bundle + def _failed(self, item: Planned, thread, exc: BaseException) -> None: + """Append what a run that raised had spent to ``failed.jsonl``, which the cap counts as it counts bundles: + nothing was spent before its thread started; ``dollars`` is null when its journal cannot be read. A runner + killed outright (SIGKILL) writes nothing here (``spec/GAPS.md``).""" + from syberlabs.evolution import spend_total + + spend = {"dollars": 0.0, "unknown_models": []} + if thread is not None: + try: + spend = spend_total(thread.history()) + except Exception: + spend = {"dollars": None, "unknown_models": []} + row = {"run_id": item.run_id, "dollars": spend["dollars"], "unknown_models": spend["unknown_models"], + "error": exc.code if isinstance(exc, Rejected) else type(exc).__name__, "at": now_utc()} + self.runs_dir.mkdir(parents=True, exist_ok=True) + with open(self.runs_dir / FAILED, "a", encoding="utf-8") as handle: + handle.write(json.dumps(row, sort_keys=True) + "\n") + @staticmethod def _summary(events, views, chosen, verdict, receipt, provider, spend) -> dict: finished = next((e["body"] for e in reversed(events) if e["kind"] == "search_finished"), {}) diff --git a/syberlabs/eval/spec.py b/syberlabs/eval/spec.py index 3daedd6..f089af7 100644 --- a/syberlabs/eval/spec.py +++ b/syberlabs/eval/spec.py @@ -18,16 +18,21 @@ The schema is JSON Schema (draft 2020-12 vocabulary) held here as a Python value; ``validate`` implements the subset it uses (``type``, ``const``, ``enum``, ``properties``, ``required``, ``additionalProperties: false``, ``items``, ``minItems``, ``maxItems``, ``uniqueItems``, ``minLength``, ``maxLength``, ``pattern``, ``minimum``, -``maximum``, ``exclusiveMinimum``), standard library only, then the cross-field rules a schema cannot state -(``check``): arm names are distinct, the decision rule names two of the spec's arms and its metric is the primary -metric, each arm is valid for its kind, and the price table is one ``syberlabs.economic.check_prices`` accepts. +``maximum``, ``exclusiveMinimum``; a number must also be finite, because ``NaN`` passes every comparison), standard +library only, then the cross-field rules a schema cannot state (``check``): arm names are distinct, the decision +rule names two of the spec's arms and its metric is the primary metric, each arm is valid for its kind, no two +cells share a run id (``..s``, where task ids and arm names may hold dots), and the price table is +one ``syberlabs.economic.check_prices`` accepts. ``read_json`` is strict: a key repeated in one object, ``NaN`` and +``Infinity`` are refused, so what a reader sees in the file is what the digest freezes. """ from __future__ import annotations import json +import math import re import subprocess +from collections import Counter from datetime import datetime, timezone from pathlib import Path from typing import Any @@ -154,6 +159,8 @@ def _walk(value: Any, schema: dict, path: str, errors: list[str]) -> None: errors.append(f"{path}: length out of range") if "pattern" in schema and not re.search(schema["pattern"], value): errors.append(f"{path}: does not match {schema['pattern']}") + if type(value) is float and not math.isfinite(value): + errors.append(f"{path}: must be a finite number") # NaN passes every comparison below if type(value) in (int, float): if "minimum" in schema and value < schema["minimum"]: errors.append(f"{path}: below {schema['minimum']}") @@ -208,6 +215,13 @@ def check(doc: Any) -> list[str]: errors.append("$.decision_rule: treatment and baseline must be different arms") if rule["metric"] != doc["primary_metric"]["name"]: errors.append("$.decision_rule.metric: the decision is taken on the primary metric") + from syberlabs.eval.runner import run_id + + shared = sorted(found for found, count in Counter(run_id(task, arm, seed) for task in doc["tasks"]["ids"] + for arm in names for seed in doc["seeds"]).items() if count > 1) + if shared: + errors.append(f"$: two cells share the run id {shared[0]} (task ids and arm names may hold dots); " + "rename an arm or a task") reason = check_prices(doc["budget"]["prices"]) if reason: errors.append("$.budget." + reason) @@ -221,9 +235,22 @@ def spec_digest(doc: dict) -> str: return digest(doc) +def _object(pairs: list) -> dict: + found = dict(pairs) + if len(found) != len(pairs): + raise ValueError("a key appears twice in one object") # json.loads would keep the last one silently + return found + + +def _constant(name: str): + raise ValueError(f"{name} is not a JSON number") + + def read_json(path: Path, code: str) -> Any: + """Strict JSON: a key repeated in one object, ``NaN`` and ``Infinity`` are refused with ``code``.""" try: - return json.loads(Path(path).read_text(encoding="utf-8")) + return json.loads(Path(path).read_text(encoding="utf-8"), object_pairs_hook=_object, + parse_constant=_constant) except OSError as exc: raise Rejected(code, f"{path}: {exc.strerror or type(exc).__name__}") from None except ValueError as exc: diff --git a/syberlabs/eval/tasks.py b/syberlabs/eval/tasks.py index 7afd21e..a0f490b 100644 --- a/syberlabs/eval/tasks.py +++ b/syberlabs/eval/tasks.py @@ -24,9 +24,11 @@ it reads the candidate's files from its working directory. Nothing under ``hidden/`` is in the run's repository, in a provider's request (``GitSearchSpace`` serves only the repository), or in a candidate. A check's result and the tail of its output are recorded, as for any check, and reach a repair arm as feedback: that is the E3 design, where -an assertion message names its expectation. ``import_fixtures`` writes each fixture check's code to -``hidden/checks.json`` and runs it through ``LOADER``, which compiles it under the file name ````, so a -traceback names no host path and prints no line of the check's source. +an assertion message names its expectation. The checks run the candidate's code, which runs with the host's +permissions and can read and rewrite the host's copy: the runner refuses to accept on a copy whose files changed +(``tree_digests``), and nothing stops code that reads the copy or ends its check early. ``import_fixtures`` writes +each fixture check's code to ``hidden/checks.json`` and runs it through ``LOADER``, which compiles it under the file +name ````, so a traceback names no host path and prints no line of the check's source. """ from __future__ import annotations @@ -315,8 +317,9 @@ def materialize(task: dict, dest: Path) -> str: target = dest / path target.parent.mkdir(parents=True, exist_ok=True) target.write_bytes(data) - # Plumbing, not ``git commit``: the same commit for the same files on every host, and no hook runs. - _git(dest, "add", "-A") + # Plumbing, not ``git commit``: the same commit for the same files on every host, and no hook runs. Forced: + # an ignore rule of the host's (``core.excludesFile``, ``~/.config/git/ignore``) must not drop a fixture file. + _git(dest, "add", "-A", "--force") tree = _git(dest, "write-tree") commit = _git(dest, "commit-tree", tree, "-m", f"task {task['id']}", env=FIXTURE_IDENTITY) _git(dest, "update-ref", "refs/heads/main", commit) @@ -373,9 +376,45 @@ def list_tasks(tasks_root: Path) -> list[str]: return sorted(p.name for p in root.iterdir() if (p / "task.json").is_file()) if root.is_dir() else [] +def hidden_path(name: Any) -> bool: + """A relative POSIX path that stays under ``hidden/``: no absolute or drive path, backslash, ``.`` or ``..``.""" + return (isinstance(name, str) and bool(name) and not name.startswith("/") and not any(c in name for c in "\\:\0") + and all(part not in ("", ".", "..") for part in name.split("/"))) + + +def tree_digests(folder: Path) -> dict[str, str]: + """The SHA-256 of every file under ``folder`` as it is on disk, by POSIX path; a symbolic link is named by its + target. ``stage_hidden`` writes regular files only, so the staged copy's digests change if anything rewrites, + adds, removes or relinks a file in it.""" + found = {} + try: + for path in sorted(p for p in Path(folder).rglob("*") if p.is_symlink() or p.is_file()): + name = path.relative_to(folder).as_posix() + if path.is_symlink(): + found[name] = "link:" + os.readlink(path) + continue + sha = hashlib.sha256() + with open(path, "rb") as handle: + for chunk in iter(lambda: handle.read(65536), b""): + sha.update(chunk) + found[name] = sha.hexdigest() + except OSError as exc: # made unreadable: as changed as rewritten + found[""] = f"unreadable: {exc.errno}" + return found + + +def hidden_changes(before: dict[str, str], after: dict[str, str]) -> list[str]: + """The paths of a staged copy that differ between two ``tree_digests``: rewritten, removed, relinked or added, + except files under a ``.pytest_cache`` directory, which pytest's cache provider writes into its rootdir, and + that is the directory of a hidden test file run with no configuration file above it.""" + added = [path for path in after if path not in before and ".pytest_cache" not in path.split("/")[:-1]] + return sorted([path for path in before if after.get(path) != before[path]] + added) + + def write_task(tasks_root: Path, task: dict, hidden: dict[str, bytes]) -> str: """Write a task (used by tests and future adapters); returns its digest.""" errors = check_task(task) + errors += [f"hidden file {name!r}: a relative path under hidden/" for name in hidden if not hidden_path(name)] if errors: raise Rejected("invalid_task", "; ".join(errors[:8])) folder = task_dir(tasks_root, task["id"]) diff --git a/tests/test_eval_adversarial.py b/tests/test_eval_adversarial.py new file mode 100644 index 0000000..6ed3a5d --- /dev/null +++ b/tests/test_eval_adversarial.py @@ -0,0 +1,531 @@ +"""Adversarial verification of the SyberEval core before the GitHits A/B pilot runs on it. + +Each test names one way a run could cheat, be mislabelled, or be scored or charged wrongly, and shows it against +real Build Threads with stand-in models (nothing here calls a model or the network). A test marked +``expectedFailure`` pins a gap that needs a design decision, not a bug with a small fix: it passes, and must lose +its marker, once that decision is built. +""" + +import json +import os +import shutil +import signal +import subprocess +import sys +import tempfile +import textwrap +import time +import unittest +from pathlib import Path +from unittest import mock + +from syberlabs.errors import Rejected +from syberlabs.eval import runner, spec, tasks +from tests.test_eval_core import SIM, git, tiny_spec + +PY = sys.executable +ROOT = Path(__file__).resolve().parents[1] +CANARY_CODE = "canary-code-5be0c1d7e4" +CANARY_DATA = "canary-data-9a73f20c18" +# Every stand-in call declares 1000 tokens in and 100 out of the simulated model: $0.003 under this table. +PRICES = {"simulated": {"input_per_million": 2.0, "output_per_million": 10.0}} +CALL = 0.003 +GOOD = ("def discount(total):\n return total * 9 / 10 if total > 100 else total\n\n\n" + "def tax(amount):\n return amount * 108 / 100\n\n\ndef shipping(weight):\n return 5 * weight\n") +WRONG = ("def discount(total):\n return total\n\n\ndef tax(amount):\n return amount\n\n\n" + "def shipping(weight):\n return 0\n") + +# A stand-in model adapter under the search protocol. ``plan.json`` lists, per call, the files to write; the last +# entry repeats. Each request is logged. ``hold`` makes the first call wait (bounded) for a ``release`` file; +# ``kill_parent`` kills the process that ran it (the eval runner) with SIGKILL, as a crash or an OOM kill would. +ADAPTER = r''' +import json, os, signal, sys, time +from pathlib import Path + +plan, log = json.loads(Path(sys.argv[1]).read_text(encoding="utf-8")), Path(sys.argv[2]) +request = json.load(sys.stdin) +n = len(list(log.glob("request-*.json"))) + 1 +(log / f"request-{n}.json").write_text(json.dumps(request), encoding="utf-8") +step = plan[min(n, len(plan)) - 1] +if step.get("hold") and n == 1: + deadline = time.monotonic() + 60 + while not (log / "release").exists() and time.monotonic() < deadline: + time.sleep(0.05) +if step.get("kill_parent"): + os.kill(os.getppid(), signal.SIGKILL) + sys.exit(0) +print(json.dumps({"candidates": [{"changes": step["changes"], "message": "stand-in"}], "recommended": [0], + "usage": {"input_tokens": 1000, "output_tokens": 100, "model": "simulated", "simulated": True}})) +''' + +# A stand-in ``claude``: answers ``--version``, logs the GitHits session and agent it was given, writes a passing +# pricing.py in its working directory, and prints a one-turn ``stream-json`` transcript served by ``sys.argv[2]``. +CLAUDE = r''' +import json, os, sys +from pathlib import Path + +if "--version" in sys.argv: + print("2.1.309 (Claude Code)", flush=True) + sys.exit(0) +log, model = Path(sys.argv[1]), sys.argv[2] +sys.stdin.read() +n = len(list(log.glob("call-*.json"))) + 1 +(log / f"call-{n}.json").write_text(json.dumps({"session": os.environ.get("GITHITS_SESSION_ID"), + "agent": os.environ.get("GITHITS_AGENT")}), encoding="utf-8") +Path("src/pricing.py").write_text(GOOD, encoding="utf-8") +usage = {"input_tokens": 1000, "output_tokens": 100, "cache_read_input_tokens": 0, "cache_creation_input_tokens": 0} +for event in ({"type": "system", "subtype": "init", "claude_code_version": "2.1.309", "model": model, + "effort": "medium", "apiKeySource": "ANTHROPIC_API_KEY", "mcp_servers": [], "session_id": "x", + "tools": ["Read", "Edit"]}, + {"type": "assistant", "message": {"id": "msg_1", "model": model, "usage": usage, + "content": [{"type": "text", "text": "done"}]}}, + {"type": "result", "subtype": "success", "is_error": False, "num_turns": 1, "result": "done", + "total_cost_usd": 0.01, "session_id": "x", "usage": {}, "modelUsage": {}}): + print(json.dumps(event), flush=True) +''' + + +def canary_task(tasks_root: Path, task_id: str = "canary-pricing") -> str: + """The pricing fixture with two hidden checks run by ``tasks.LOADER``; canaries in a check's code and in a data + file that the ``tax`` check reads by its ``{hidden}`` path. Returns the task's digest.""" + discount = (f"# {CANARY_CODE}\nimport sys; sys.path.insert(0, 'src'); import pricing\n" + "assert pricing.discount(200) == 180, 'discount: ten percent off totals over 100'\n") + tax = ("import sys; sys.path.insert(0, 'src'); import pricing\n" + "want = int(open(sys.argv[1], encoding='utf-8').read().split()[0])\n" + "assert pricing.tax(100) == want, 'tax: eight percent'\n") + loader = ["{python}", "-c", tasks.LOADER, "{hidden}/checks.json"] + task = {"schema": tasks.TASK_SCHEMA_ID, "id": task_id, "source": "fixture:pricing", + "source_digest": tasks.source_digest(tasks.fixture_files("pricing")), + "objective": "Make discount, tax and shipping pass their checks", "scope": {"paths": ["src/"]}, + "checks": {}, "hidden": {"checks": {"discount": {"argv": loader + ["discount"]}, + "tax": {"argv": loader + ["tax", "{hidden}/expected.txt"]}}}, + "safety": {"expectations": [], "canaries": [CANARY_CODE, CANARY_DATA]}, + "provenance": {"origin": "tests/test_eval_adversarial.py"}, + "contamination": {"status": "private", "note": "written for this test"}, "labels": {}} + return tasks.write_task(tasks_root, task, { + "checks.json": json.dumps({"discount": discount, "tax": tax}).encode(), + "expected.txt": f"108\n# {CANARY_DATA}\n".encode()}) + + +class Case(unittest.TestCase): + def setUp(self): + self.folder = Path(tempfile.mkdtemp(prefix="syberlabs-evaladv-")) + self.root = self.folder / "evals" + self.log = self.folder / "requests" + self.log.mkdir() + (self.folder / "adapter.py").write_text(ADAPTER, encoding="utf-8") + + def tearDown(self): + shutil.rmtree(self.folder, ignore_errors=True) + + def adapter(self, *steps: dict) -> list[str]: + """The stand-in adapter's argv, answering call n with ``steps[n - 1]`` (the last repeats).""" + plan = self.folder / "plan.json" + plan.write_text(json.dumps(list(steps)), encoding="utf-8") + return ["{python}", str(self.folder / "adapter.py"), str(plan), str(self.log)] + + def requests(self) -> list[str]: + found = sorted(self.log.glob("request-*.json"), key=lambda p: int(p.stem.split("-")[1])) + return [p.read_text(encoding="utf-8") for p in found] + + def canary_spec(self, eval_id: str, adapter: list[str], **changes) -> dict: + digest = canary_task(self.root / "tasks") + doc = dict(tiny_spec(["canary-pricing"], {"canary-pricing": digest}), id=eval_id, seeds=[1], + arms=[{"name": "repair", "kind": "repair", "model": "simulated", "effort": "low", + "adapter": adapter, "max_calls": 3}, + {"name": "single", "kind": "single", "model": "simulated", "effort": "low", + "adapter": adapter}]) + doc["budget"] = dict(doc["budget"], prices=PRICES) + doc.update(changes) + return doc + + def register(self, doc: dict) -> dict: + path = self.root / doc["id"] / "spec.json" + spec.write_json(path, doc) + return spec.register(path) + + def run_eval(self, eval_id: str, **options): + options.setdefault("require_preflight", False) + options.setdefault("seed_env", "SYBERLABS_EVAL_ADV_NO_SEED") + options.setdefault("work_root", self.folder / "work") + return runner.run_eval(self.root, eval_id, **options) + + def runner_process(self, eval_id: str) -> list[str]: + """``run_eval`` of this eval's ``single`` runs in another process, as a second terminal would run it.""" + code = (f"import sys; sys.path.insert(0, {str(ROOT)!r}); from syberlabs.eval import runner; " + f"runner.run_eval({str(self.root)!r}, {eval_id!r}, only=',single,', require_preflight=False, " + f"seed_env='SYBERLABS_EVAL_ADV_NO_SEED', work_root={str(self.folder / 'work')!r})") + return [PY, "-c", code] + + def bundle(self, eval_id: str, run: str) -> dict: + return json.loads((self.root / eval_id / "runs" / run / "bundle.json").read_text(encoding="utf-8")) + + +class PreRegistration(Case): + def test_a_number_that_is_not_finite_is_not_a_valid_spec(self): + """``NaN`` passes ``minimum``, ``maximum`` and ``exclusiveMinimum`` (every comparison with it is false). A spec + whose ``total_dollars`` is ``NaN`` registers and its eval cap never stops (``spent + dollars > NaN`` is + false); a ``NaN`` threshold makes every decision false.""" + doc = self.canary_spec("nan", self.adapter({"changes": {"src/pricing.py": GOOD}})) + for where, field in (("budget", "total_dollars"), ("budget", "dollars"), ("decision_rule", "threshold")): + for value in (float("nan"), float("inf")): + changed = json.loads(json.dumps(doc)) + changed[where][field] = value + with self.subTest(field=field, value=value): + self.assertTrue(spec.check(changed), f"{field} = {value} accepted") + path = self.root / "nan" / "spec.json" + path.parent.mkdir(parents=True) + path.write_text(json.dumps(dict(doc, budget=dict(doc["budget"], total_dollars=float("nan")))), encoding="utf-8") + with self.assertRaises(Rejected): + spec.register(path) + self.assertFalse((path.parent / "registration.json").exists()) + + def test_a_spec_with_a_duplicate_key_is_refused(self): + """``json.loads`` keeps the last of two equal keys, so a reader of the file can see one threshold while the + registered digest freezes the other.""" + doc = self.canary_spec("dup", self.adapter({"changes": {"src/pricing.py": GOOD}})) + text = json.dumps(doc, indent=2).replace('"threshold": 0.0', '"threshold": 0.0, "threshold": -1.0') + self.assertIn('"threshold": -1.0', text) + path = self.root / "dup" / "spec.json" + path.parent.mkdir(parents=True) + path.write_text(text, encoding="utf-8") + with self.assertRaises(Rejected): + spec.register(path) + self.assertFalse((path.parent / "registration.json").exists()) + + def test_two_cells_that_would_share_a_run_id_are_refused(self): + """A run id is ``..s`` and both names may hold dots: task ``p`` with arm ``x.r`` and task + ``p.x`` with arm ``r`` are both ``p.x.r.s1``. The second run overwrites the first one's bundle, and on + resume both cells read as done from one bundle: a run is missing and another is counted twice.""" + task, bodies = tasks.fixture_task("pricing") + digests = {tid: tasks.write_task(self.root / "tasks", dict(task, id=tid), + {"checks.json": json.dumps(bodies).encode()}) for tid in ("p", "p.x")} + doc = tiny_spec(["p", "p.x"], digests, seeds=[1]) + doc["arms"] = [{"name": "x.r", "kind": "repair", "model": "simulated", "effort": "none", "adapter": SIM, + "max_calls": 3}, + {"name": "r", "kind": "single", "model": "simulated", "effort": "none", "adapter": SIM}] + doc["decision_rule"] = dict(doc["decision_rule"], treatment="x.r", baseline="r") + self.assertTrue(any("run id" in error for error in spec.check(doc)), spec.check(doc)) + with self.assertRaises(Rejected): + self.register(doc) + + +class TaskFiles(Case): + def test_write_task_keeps_hidden_files_inside_the_task(self): + """``write_task`` is the API for importing tasks (the pilot's 21 items will come through an adapter); a hidden + file named ``../..`` or by an absolute path is written outside the task, over a registration or anywhere.""" + task = dict(tasks.fixture_task("pricing")[0], id="escape") + bodies = json.dumps(tasks.fixture_task("pricing")[1]).encode() + for name in ("../../outside.txt", str(self.folder / "outside.txt"), "sub/../../../outside.txt", "", + "..", "a\\..\\..\\outside.txt"): + with self.subTest(name=name): + with self.assertRaises(Rejected): + tasks.write_task(self.root / "tasks", task, {"checks.json": bodies, name: b"written"}) + self.assertEqual(list(self.folder.rglob("outside.txt")), []) + + def test_a_fixture_s_base_holds_every_fixture_file_whatever_the_host_ignores(self): + """``materialize`` committed a fixture with ``git add -A``, which honours the operator's global ignore file + (``~/.config/git/ignore``, ``core.excludesFile``): a host that ignores ``*.py`` committed an empty base, and + every arm on that host worked on a repository without the task. ``source_digest`` pins the fixture's + files, not the commit made of them, so nothing noticed.""" + home = self.folder / "home" + (home / ".config" / "git").mkdir(parents=True) + (home / ".config" / "git" / "ignore").write_text("*.py\n", encoding="utf-8") + task = tasks.fixture_task("pricing")[0] + with mock.patch.dict(os.environ, {"HOME": str(home)}): + os.environ.pop("XDG_CONFIG_HOME", None) + base = tasks.materialize(task, self.folder / "repo") + self.assertEqual(git(self.folder / "repo", "ls-tree", "-r", "--name-only", base).split(), + sorted(tasks.fixture_files("pricing"))) + self.assertEqual(base, tasks.materialize(task, self.folder / "again"), "the same base on every host") + + +class Budget(Case): + def capped(self, eval_id: str, adapter: list[str]) -> dict: + doc = self.canary_spec(eval_id, adapter) + doc["budget"] = dict(doc["budget"], dollars=CALL, total_dollars=0.008) + self.register(doc) + return doc + + def test_a_run_that_fails_after_it_spent_is_charged_against_the_cap(self): + """A run that raises after its model calls (an interrupt, a harness error, a certificate that does not + verify) leaves no bundle, and only bundles were counted: each retry spent again under a cap that saw + nothing. Two failed attempts spend $0.006 of the $0.008 cap; a third run could reach $0.009.""" + self.capped("capfail", self.adapter({"changes": {"src/pricing.py": GOOD}})) + with mock.patch.object(runner, "_chosen", side_effect=KeyboardInterrupt): + with self.assertRaises(KeyboardInterrupt): + self.run_eval("capfail", only=",single,") + with mock.patch.object(runner, "_chosen", side_effect=RuntimeError("a harness error after the search")): + with self.assertRaises(RuntimeError): + self.run_eval("capfail", only=",single,") + self.assertEqual(len(self.requests()), 2, "each failed attempt made its call") + found = self.run_eval("capfail", only=",single,") + self.assertEqual(len(self.requests()), 2, "a third call could take the eval over its cap") + self.assertEqual(found["stopped"], "eval_budget_exhausted") + self.assertAlmostEqual(found["spent_dollars"], 2 * CALL) + + @unittest.expectedFailure + def test_a_run_killed_midway_is_charged_against_the_cap(self): + """Design gap: a runner killed during a run (SIGKILL, an OOM kill, a power cut) runs no handler; its + ``.partial`` is deleted on resume and what its calls cost is counted nowhere. Bounding it needs a policy + (reserve the per-run dollars when a run starts, or stop the eval until the operator reconciles).""" + self.capped("capkill", self.adapter({"changes": {"src/pricing.py": GOOD}, "kill_parent": True}, + {"changes": {"src/pricing.py": GOOD}})) + done = subprocess.run(self.runner_process("capkill"), capture_output=True, timeout=300) + self.assertEqual(done.returncode, -signal.SIGKILL, done.stderr) + self.assertEqual(len(self.requests()), 1) + found = self.run_eval("capkill", dry=True) + self.assertGreaterEqual(found["spent_dollars"], CALL, "the killed run's call is counted somewhere") + + def test_a_second_runner_cannot_run_the_same_eval_at_once(self): + """Two ``eval run`` processes on one eval each count only the bundles they found at start, so both pass + the cap check, and both start the same first run (the second deletes the first one's ``.partial``).""" + self.capped("twice", self.adapter({"changes": {"src/pricing.py": GOOD}, "hold": True}, + {"changes": {"src/pricing.py": GOOD}})) + first = subprocess.Popen(self.runner_process("twice"), stdout=subprocess.DEVNULL, stderr=subprocess.PIPE) + try: + deadline = time.monotonic() + 120 + while not (self.log / "request-1.json").exists(): + self.assertIsNone(first.poll(), "the first runner ended before its call") + self.assertLess(time.monotonic(), deadline) + time.sleep(0.05) + with self.assertRaises(Rejected) as caught: + self.run_eval("twice", only=",single,") + self.assertEqual(caught.exception.code, "eval_running") + self.assertEqual(len(self.requests()), 1, "the second runner made no call") + finally: + (self.log / "release").write_text("go", encoding="utf-8") + _, err = first.communicate(timeout=300) + self.assertEqual(first.returncode, 0, err) + self.assertEqual(self.bundle("twice", "canary-pricing.single.s1")["outcome"]["status"], "accepted") + + +class Preflight(Case): + PASS = (("ok", ["{python}", "-c", "print('fine')"]),) + + def test_the_preflight_gate_holds_for_every_run_not_only_the_first(self): + """The gate was read once, when ``run`` started. An eval runs for hours; a checkout that moves or is edited + after the first run (``git pull``, an edited adapter, which every call reads from disk again) ran every + later run under a preflight that no longer describes it.""" + harness = self.folder / "harness" + harness.mkdir() + git(harness, "init", "-q") + (harness / "README").write_text("harness\n", encoding="utf-8") + git(harness, "add", "README") + git(harness, "commit", "-qm", "one") + self.root = harness / "evals" + self.register(self.canary_spec("gate", self.adapter({"changes": {"src/pricing.py": GOOD}}), seeds=[1, 2])) + self.assertTrue(runner.preflight(self.root, "gate", harness=harness, commands=self.PASS)["passed"]) + + def log(line): + if line.startswith("run ") and not (harness / "edited.py").exists(): + (harness / "edited.py").write_text("# changed after the gate\n", encoding="utf-8") + + found = runner.run_eval(self.root, "gate", only=",single,", harness=harness, log=log, + seed_env="SYBERLABS_EVAL_ADV_NO_SEED", work_root=self.folder / "work") + self.assertEqual([row["state"] for row in found["runs"]], ["ran"]) + self.assertEqual(found["stopped"], "preflight_dirty") + + +class Bundles(Case): + def setUp(self): + super().setUp() + self.register(self.canary_spec("ev", self.adapter({"changes": {"src/pricing.py": WRONG}}, + {"changes": {"src/pricing.py": GOOD}}), seeds=[1, 2])) + self.run_eval("ev", only=",repair,") + self.runs = self.root / "ev" / "runs" + self.assertEqual(sorted(p.name for p in self.runs.iterdir() if p.is_dir()), + ["canary-pricing.repair.s1", "canary-pricing.repair.s2"]) + + def refused(self) -> str: + with self.assertRaises(Rejected) as caught: + self.run_eval("ev", dry=True) + return caught.exception.code + + def test_a_bundle_whose_evidence_changed_is_refused(self): + """``files`` holds a SHA-256 for every evidence file, and nothing compared them: an edited journal, trace or + repository bundle was read as the run's evidence.""" + trace = self.runs / "canary-pricing.repair.s1" / "trace.json" + trace.write_text(trace.read_text(encoding="utf-8").replace('"failed"', '"passed"'), encoding="utf-8") + self.assertEqual(self.refused(), "bundle_tampered") + + def test_a_bundle_whose_dollars_were_edited_is_refused(self): + """The cap sums ``outcome.dollars`` from bundle.json, which no digest covers: setting it to 0 bought room + under the cap that the journal beside it shows was spent.""" + path = self.runs / "canary-pricing.repair.s1" / "bundle.json" + bundle = json.loads(path.read_text(encoding="utf-8")) + self.assertGreater(bundle["outcome"]["dollars"], 0) + bundle["outcome"]["dollars"] = 0.0 + spec.write_json(path, bundle) + self.assertEqual(self.refused(), "bundle_tampered") + + def test_a_bundle_in_another_run_s_directory_is_refused(self): + """``bundles`` keyed a bundle by the run id inside it, not by where it lies: a bundle copied over another + run's directory made that run read as done.""" + first, second = self.runs / "canary-pricing.repair.s1", self.runs / "canary-pricing.repair.s2" + shutil.rmtree(second) + shutil.copytree(first, second) + self.assertEqual(self.refused(), "bundle_tampered") + + def test_a_partial_run_is_never_done(self): + """A run killed after it wrote ``bundle.json`` in ``.partial`` and before the rename was read as done + (the glob matched the partial directory), and its run never ran again.""" + done = self.runs / "canary-pricing.repair.s2" + done.rename(self.runs / "canary-pricing.repair.s2.partial") + found = self.run_eval("ev", dry=True) + states = {row["run_id"]: row["state"] for row in found["runs"]} + self.assertEqual(states["canary-pricing.repair.s2"], "pending") + + +class HiddenChecks(Case): + """What candidate code can do while the host's hidden checks run it, in the host's process tree.""" + + def test_a_candidate_that_rewrites_the_host_copy_of_a_check_is_not_accepted(self): + """The host's copy is an ordinary directory, and its path is in the check's own process (the loader's + ``__main__``, ``/proc/self/cmdline``). c1's code rewrites every hidden check to ``pass``; c2 changes + nothing that matters and passes. ``Not editable`` held only for the candidate's own tree.""" + tamper = ("import __main__, json\n" + "if getattr(__main__, 'path', None):\n" + " passing = json.dumps({'discount': 'pass', 'tax': 'pass'})\n" + " open(__main__.path, 'w', encoding='utf-8').write(passing)\n" + WRONG) + self.register(self.canary_spec("tamper", self.adapter({"changes": {"src/pricing.py": tamper}}, + {"changes": {"src/pricing.py": WRONG + "# two\n"}}))) + self.run_eval("tamper", only=",repair,") + outcome = self.bundle("tamper", "canary-pricing.repair.s1")["outcome"] + self.assertNotEqual(outcome["status"], "accepted", outcome) + self.assertIsNone(outcome["accepted_commit"]) + self.assertEqual((outcome["reason"], outcome["hidden_intact"], outcome["hidden_changed"]), + ("hidden_tampered", False, ["checks.json"])) + + def test_only_pytest_s_own_cache_may_appear_in_the_host_copy(self): + """A hidden pytest file with no configuration above it makes ``{hidden}`` pytest's rootdir, and its cache + provider writes ``.pytest_cache`` there; anything else added, changed or removed is a change, a planted + ``conftest.py`` or ``__pycache__`` (read even under ``PYTHONDONTWRITEBYTECODE``) above all.""" + before = {"checks.json": "a", "data/x.txt": "b"} + cache = {".pytest_cache/v/cache/lastfailed": "c", "data/.pytest_cache/README.md": "d"} + self.assertEqual(tasks.hidden_changes(before, {**before, **cache}), []) + for after, changed in (({**before, "conftest.py": "e"}, ["conftest.py"]), + ({**before, "__pycache__/m.cpython-313.pyc": "f"}, ["__pycache__/m.cpython-313.pyc"]), + ({**before, ".pytest_cache": "g"}, [".pytest_cache"]), + ({"checks.json": "a"}, ["data/x.txt"]), + ({**before, "checks.json": "z"}, ["checks.json"])): + with self.subTest(changed=changed): + self.assertEqual(tasks.hidden_changes(before, after), changed) + + @unittest.expectedFailure + def test_a_candidate_that_exits_at_import_does_not_pass_the_hidden_checks(self): + """Design gap (a sandbox, or a loader that sees the check finish): every hidden check imports the + candidate's module, and a module that ends the process with status 0 passes every check unread. The + checks are judged by exit status alone (``syberlabs/checks.py``); ``sys.exit(0)`` and ``os._exit(0)`` both + work. ``spec/GAPS.md`` 58 answers this for the ``tests`` check, not for hidden checks.""" + exits = {"changes": {"src/pricing.py": "import os\nos._exit(0)\n"}} + self.register(self.canary_spec("exit", self.adapter(exits))) + self.run_eval("exit", only=",single,") + outcome = self.bundle("exit", "canary-pricing.single.s1")["outcome"] + self.assertNotEqual(outcome["status"], "accepted", outcome) + + @unittest.expectedFailure + def test_a_candidate_cannot_carry_a_hidden_check_into_its_next_request(self): + """Design gap (a sandbox, or feedback the candidate cannot write): candidate code run by a hidden check reads + the check's source from the loader's ``__main__.body`` and its data from ``sys.argv``, and prints both; + the output tail is the repair arm's feedback, so the next request holds the hidden check. EVAL.md's + ``Not in a request`` holds only for a candidate that does not look.""" + leak = ("import __main__, sys\nprint(getattr(__main__, 'body', ''))\n" + "print(''.join(open(arg, encoding='utf-8').read() for arg in sys.argv[1:]))\n" + WRONG) + self.register(self.canary_spec("leak", self.adapter({"changes": {"src/pricing.py": leak}}, + {"changes": {"src/pricing.py": GOOD}}))) + self.run_eval("leak", only=",repair,") + second = self.requests()[1] + self.assertNotIn(CANARY_CODE, second) + self.assertNotIn(CANARY_DATA, second) + + +class ClaudeCodeArms(Case): + """Two ``claude-code`` arms that differ only in ``githits``, the pilot's shape, with a stand-in ``claude``.""" + + def setUp(self): + super().setUp() + self.claude = self.folder / "claude.py" + self.claude.write_text(f"GOOD = {GOOD!r}\n" + textwrap.dedent(CLAUDE), encoding="utf-8") + removed = ("ANTHROPIC_BASE_URL", "CLAUDE_CODE_USE_BEDROCK", "CLAUDE_CODE_USE_VERTEX", "CLAUDE_CODE_USE_FOUNDRY", + "CLAUDE_CODE_USE_MANTLE", "CLAUDE_CODE_USE_ANTHROPIC_AWS", "CLAUDE_CODE_PROVIDER_MANAGED_BY_HOST") + patcher = mock.patch.dict(os.environ, {"GITHITS_API_TOKEN": "test-not-a-token"}) + patcher.start() + self.addCleanup(patcher.stop) + for name in removed: + os.environ.pop(name, None) + + def pilot(self, eval_id: str, *, declared: str, served: str) -> dict: + written = tasks.import_fixtures(self.root / "tasks", ["pricing"]) + executable = ["{python}", str(self.claude), str(self.log), served] + base = {"kind": "claude-code", "model": declared, "effort": "medium", "bare": True, "lane": "api", + "max_calls": 1, "executable": executable} + doc = tiny_spec([w["id"] for w in written], {w["id"]: w["digest"] for w in written}, id=eval_id, + arms=[dict(base, name="off", githits=False), dict(base, name="on", githits=True)]) + doc["budget"] = dict(doc["budget"], prices={served: {"input_per_million": 2.0, "output_per_million": 10.0}}) + doc["decision_rule"] = dict(doc["decision_rule"], treatment="on", baseline="off") + self.assertEqual(spec.check(doc), []) + self.register(doc) + return doc + + def calls(self) -> list[dict]: + return [json.loads(p.read_text(encoding="utf-8")) for p in sorted(self.log.glob("call-*.json"))] + + def test_each_run_of_the_githits_arm_has_its_own_githits_session(self): + """``arms.build`` passed the provider neither a thread label nor an arm, so every run of every task and seed + sent ``X-GitHits-Session-ID: s1-b`` (the search id, then the default arm) and ``X-GitHits-Agent: + syberfactory/b``: GitHits saw the whole pilot arm as one session, and nothing tied a lookup to its run.""" + self.pilot("sessions", declared="claude-opus-5", served="claude-opus-5") + self.run_eval("sessions", only=",on,") + bundles = [self.bundle("sessions", f"v1-pricing.on.s{seed}") for seed in (1, 2)] + self.assertEqual([b["outcome"]["status"] for b in bundles], ["accepted", "accepted"]) + sessions = [call["session"] for call in self.calls()] + self.assertEqual(len(sessions), 2) + self.assertEqual(len(set(sessions)), 2, sessions) + for bundle, session in zip(bundles, sessions): + self.assertIn(bundle["thread"], session) + self.assertEqual({call["agent"] for call in self.calls()}, {"syberfactory/on"}) + + @unittest.expectedFailure + def test_a_run_served_by_another_model_than_its_arm_declares_is_not_reported_as_that_arm(self): + """Design gap: the runner does not set Claude Code's model, and nothing compares the model that served + (``signal.harness.model``) with the arm's registered ``model``. A Claude Code whose default moved (an update, + an operator's ``/model``) runs both pilot arms on another model, and the bundle still says the declared one.""" + self.pilot("model", declared="claude-sonnet-5-5", served="claude-opus-5") + self.run_eval("model", only=",off,1") + bundle = self.bundle("model", "v1-pricing.off.s1") + served = {e["body"]["signal"]["harness"]["model"] + for e in json.loads((self.root / "model" / "runs" / "v1-pricing.off.s1" / "journal.json") + .read_text(encoding="utf-8"))["events"] if e["kind"] == "candidate_registered"} + self.assertEqual(served, {"claude-opus-5"}) + self.assertFalse(bundle["outcome"]["status"] == "accepted" and bundle["provider"]["model"] not in served, + "an accepted run is reported as a model that did not serve it") + + +class RepoTasks(Case): + @unittest.expectedFailure + def test_a_repo_task_s_repository_holds_nothing_after_its_base(self): + """Design gap: ``materialize`` clones every ref of the source and checks out the base, so a pilot item's run + repository holds the upstream commits after its base, the fix itself when it was merged. Claude Code's + file tools under ``--restricted`` cannot read ``.git`` outside the worktree, but candidate code run by the + checks can (``git log --all``), any arm allowed ``Bash`` can, and ``repo.bundle`` exports all of it.""" + upstream = self.folder / "upstream" + upstream.mkdir() + git(upstream, "init", "-q", "-b", "main") + (upstream / "src").mkdir() + (upstream / "src" / "pricing.py").write_text(WRONG, encoding="utf-8") + git(upstream, "add", "-A") + git(upstream, "commit", "-qm", "base") + base = git(upstream, "rev-parse", "HEAD") + (upstream / "src" / "pricing.py").write_text(GOOD, encoding="utf-8") + git(upstream, "commit", "-qam", "the fix") + fix = git(upstream, "rev-parse", "HEAD") + task = {"id": "repo-task", "source": f"repo:{upstream}@{base}"} + repo = self.folder / "run" / "repo" + self.assertEqual(tasks.materialize(task, repo), base) + reachable = subprocess.run(["git", "-C", str(repo), "cat-file", "-e", fix + "^{commit}"], capture_output=True) + self.assertNotEqual(reachable.returncode, 0, "the fix commit is in the run's repository") + + +if __name__ == "__main__": + unittest.main()