From 346048aa4c00c1f4c4a3cfa3ae56a27a4e8ad0fd Mon Sep 17 00:00:00 2001 From: rysweet Date: Mon, 6 Jul 2026 22:43:31 +0000 Subject: [PATCH] =?UTF-8?q?feat(coin-gym):=20LOCAL=20COIN=20Gym=20harness?= =?UTF-8?q?=20scaffold=20=E2=80=94=20baseline-vs-team=20+=20overfit=20gate?= =?UTF-8?q?=20(#2713=20Phase=204)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Phase 4 of the LOCAL COIN Gym goal (issue #2713): a Rust `coin_gym` module and `coin-gym` CLI that run the COIN benchmark shape locally, score reach/precision against the published leaderboard, and A/B a single-model baseline vs. a multi-agent team — mirroring skwaq's failure-analysis + overfitting-reviewer gating. The whole pipeline runs offline against a mock oracle so it is exercised without a VM. Real `coin evaluate` grading (Docker/VM) is Phase 3; the live self-improvement verify/rollback loop is Phase 5 (both unchecked on #2713). Components (design doc Part 3.3): - target loader (pinned + held-out fresh slice; JSON snapshot manifest) - agent runner: baseline + team (reacher/skeptic/synthesizer threshold gate) - harness executor: mockable HarnessExecutor delegating to `coin evaluate` (CoinEvaluateExecutor.build_argv; Phase-3 gated) + MockHarnessExecutor - scorer: reach rate + precision, family split, R/W/A/T/N/E histogram - leaderboard comparator (published targeted-track numbers; material-deviation) - offline failure-analyst + overfitting-reviewer gate (the anti-overfit crux) - profiles: isolated per-model run state - CLI: coin-gym run|score|compare|improve|profiles Rust (not Python): the repo enforces a Rust-only policy (#2155) and the design calls the harness a Simard crate; `coin evaluate` stays an external oracle the harness delegates to via the mockable executor, never re-implemented. Docs: docs/howto/run-the-coin-gym-harness.md (wired into mkdocs nav); research doc phase table + CLI sketch reconciled. Test: 73 unit tests + gadugi scenario tests/gadugi/coin-gym-harness.{sh,yaml}. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- Cargo.toml | 4 + docs/howto/run-the-coin-gym-harness.md | 158 +++++++ .../coin-benchmark-and-skwaq-study.md | 19 +- mkdocs.yml | 1 + src/bin/coin_gym.rs | 8 + src/coin_gym/agent_runner.rs | 315 ++++++++++++++ src/coin_gym/executor.rs | 212 +++++++++ src/coin_gym/fixtures/sample_snapshot.json | 89 ++++ src/coin_gym/improve.rs | 286 +++++++++++++ src/coin_gym/leaderboard.rs | 213 +++++++++ src/coin_gym/mod.rs | 403 ++++++++++++++++++ src/coin_gym/profiles.rs | 215 ++++++++++ src/coin_gym/scorer.rs | 193 +++++++++ src/coin_gym/target_loader.rs | 210 +++++++++ src/coin_gym/tests_agent_runner.rs | 184 ++++++++ src/coin_gym/tests_cli.rs | 249 +++++++++++ src/coin_gym/tests_executor.rs | 107 +++++ src/coin_gym/tests_improve.rs | 176 ++++++++ src/coin_gym/tests_leaderboard.rs | 86 ++++ src/coin_gym/tests_profiles.rs | 117 +++++ src/coin_gym/tests_scorer.rs | 118 +++++ src/coin_gym/tests_target_loader.rs | 102 +++++ src/coin_gym/tests_types.rs | 111 +++++ src/coin_gym/types.rs | 259 +++++++++++ src/lib.rs | 10 + tests/gadugi/coin-gym-harness.sh | 87 ++++ tests/gadugi/coin-gym-harness.yaml | 55 +++ 27 files changed, 3986 insertions(+), 1 deletion(-) create mode 100644 docs/howto/run-the-coin-gym-harness.md create mode 100644 src/bin/coin_gym.rs create mode 100644 src/coin_gym/agent_runner.rs create mode 100644 src/coin_gym/executor.rs create mode 100644 src/coin_gym/fixtures/sample_snapshot.json create mode 100644 src/coin_gym/improve.rs create mode 100644 src/coin_gym/leaderboard.rs create mode 100644 src/coin_gym/mod.rs create mode 100644 src/coin_gym/profiles.rs create mode 100644 src/coin_gym/scorer.rs create mode 100644 src/coin_gym/target_loader.rs create mode 100644 src/coin_gym/tests_agent_runner.rs create mode 100644 src/coin_gym/tests_cli.rs create mode 100644 src/coin_gym/tests_executor.rs create mode 100644 src/coin_gym/tests_improve.rs create mode 100644 src/coin_gym/tests_leaderboard.rs create mode 100644 src/coin_gym/tests_profiles.rs create mode 100644 src/coin_gym/tests_scorer.rs create mode 100644 src/coin_gym/tests_target_loader.rs create mode 100644 src/coin_gym/tests_types.rs create mode 100644 src/coin_gym/types.rs create mode 100755 tests/gadugi/coin-gym-harness.sh create mode 100644 tests/gadugi/coin-gym-harness.yaml diff --git a/Cargo.toml b/Cargo.toml index 1825aed0..fb5233a0 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -8,6 +8,10 @@ default-run = "simard" name = "simard-gym" path = "src/bin/simard_gym.rs" +[[bin]] +name = "coin-gym" +path = "src/bin/coin_gym.rs" + [[bin]] name = "simard-rust-gym" path = "src/bin/simard_rust_gym.rs" diff --git a/docs/howto/run-the-coin-gym-harness.md b/docs/howto/run-the-coin-gym-harness.md new file mode 100644 index 00000000..f451bd7f --- /dev/null +++ b/docs/howto/run-the-coin-gym-harness.md @@ -0,0 +1,158 @@ +--- +title: Run the LOCAL COIN Gym harness +description: Operator guide for the coin-gym CLI — a local harness that runs the COIN benchmark shape, scores vs. the published leaderboard, and A/Bs a single-model baseline against a multi-agent team. +last_updated: 2026-07-06 +review_schedule: as-needed +owner: simard +doc_type: howto +related: + - ../research/coin-benchmark-and-skwaq-study.md +--- + +# Run the LOCAL COIN Gym harness + +The **COIN Gym** is a local harness for the [COIN](https://coin-bench.github.io/) +benchmark (COde → INput): it drives an agent to produce an input that reaches a +target line, grades the result, scores reach/precision against the published +leaderboard, and compares a **single-model baseline** against a **multi-agent +team** — mirroring skwaq's failure-analysis + overfitting-reviewer gating. + +This guide covers **Phase 4** (the local scaffold). The full design and the +phase plan live in +[COIN benchmark & skwaq gym study](../research/coin-benchmark-and-skwaq-study.md) +and are tracked in issue #2713. + +> **Offline by default.** In Phase 4 the harness grades against a **mock oracle** +> so the whole pipeline runs without a VM. Runs are clearly labelled +> `OFFLINE SCAFFOLD`. Real grading delegates to `coin evaluate` (Docker + +> instrumented replay) and needs a provisioned host — that is **Phase 3**. The +> live self-improvement loop (apply → verify on held-out fresh targets → +> keep-or-roll-back) is **Phase 5**. + +## Build the CLI + +```bash +cargo build --bin coin-gym +``` + +The binary reads/writes state under a home directory: `target/coin-gym` by +default, overridable with the `COIN_GYM_HOME` environment variable. + +## Commands + +```text +coin-gym run [--strategy baseline|team] [--profile ] [--targets ] +coin-gym score [--profile ] +coin-gym compare [--profile ] +coin-gym improve [--profile ] +coin-gym profiles +``` + +### `run` — evaluate a model on the target set + +```bash +coin-gym run claude-opus-4.6 --strategy baseline --profile opus +coin-gym run claude-opus-4.6 --strategy team --profile opus-team +``` + +`run` loads the bundled sample snapshot (or a JSON snapshot passed with +`--targets`), drives the chosen strategy over the pinned targets, grades each +submission, saves a run under the profile, and prints the score. The two +strategies make the design's central trade-off explicit: + +- **`baseline`** — a single model submits its candidate input directly. +- **`team`** — a *reacher* proposes an input, a *skeptic* challenges the + over-claim, and a *synthesizer* **submits or abstains** via a `threshold_hint` + gate. Because COIN **precision** punishes over-claiming, the team abstains on + low-confidence inputs instead of submitting wrong ones. + +On the bundled sample the two strategies reach the same number of targets, but +the team's abstention gate lifts precision from 60% to 100%: + +```text +baseline reach 60.0% (3/5) precision 60.0% (3/5) R:3/W:2/A:0/T:0/N:0/E:0 +team reach 60.0% (3/5) precision 100.0% (3/3) R:3/W:0/A:2/T:0/N:0/E:0 +``` + +### `score` — reach / precision + family split + +```bash +coin-gym score +``` + +Prints overall **reach rate** and **precision**, the split by family (frontier +vs. non-trivial reachable), and the `R/W/A/T/N/E` outcome histogram. + +- **reach rate** = reached / total targets. +- **precision** = reached / *submitted* inputs (abstain and no-submission are + excluded from the denominator). + +### `compare` — local vs. published leaderboard + +```bash +coin-gym compare +``` + +Diffs the run's reach/precision against COIN's published targeted-track numbers +for the same model. A gap beyond 10 percentage points is flagged as a **material +deviation** — a signal of a harness/config bug rather than a capability result. +Offline scaffold runs are labelled *illustrative only* (a mock oracle cannot +reproduce leaderboard numbers). + +### `improve` — offline failure analysis + overfitting gate + +```bash +coin-gym improve +``` + +Runs the **Phase-4 slice** of the self-improvement loop over a saved run: + +1. **Failure-analyst** turns each unreached target (`W`/`T`/`N`) into a + **general** reachability tactic (e.g. "for format-gated decoders, satisfy the + magic-byte/header validator before targeting deep lines"). +2. **Overfitting-reviewer gate** rejects any tactic that memorises a specific + input or keys off a specific target id / project / locator, accepting only + tactics that plausibly generalise. + +Applying an accepted tactic, re-running on held-out **fresh** targets, and +keeping it only if reach improves without a precision regression (else rolling +back) requires live grading and is **Phase 5**. + +### `profiles` — list isolated per-model run state + +```bash +coin-gym profiles +``` + +Each profile is an isolated directory (`/profiles//`) holding its own +metadata and saved runs, so baseline-vs-team and model-vs-model comparisons never +cross-contaminate. + +## Use a custom snapshot + +`--targets ` points `run` at a JSON snapshot manifest with this shape (see +`src/coin_gym/fixtures/sample_snapshot.json` for a complete example): + +```json +{ + "snapshot": "you/coin@v1", + "targets": { + "pinned": [ { "id": "…", "project": "…", "commit": "…", "harness": "…", "file": "…", "line": 1, "family": "frontier" } ], + "held_out_fresh": [ { "id": "…", "project": "…", "commit": "…", "harness": "…", "file": "…", "line": 2, "family": "non-trivial-reachable" } ] + }, + "oracle": { "": "" }, + "script": { "": { "input": "", "confidence": 0.8, "rationale": "…" } } +} +``` + +The `oracle` and `script` sections drive the **offline** demo run only. A real +run gets its oracle from `coin evaluate` and its candidates from a live model. + +## What is deferred + +| Phase | Work | Status | +|-------|------|--------| +| 3 | Provision an `azlin` VM + Docker host, pull a COIN snapshot, wire the real `coin evaluate` executor | follow-up (HIGH-RISK, operator-gated) | +| 5 | Live self-improvement loop: apply tactic → verify on held-out fresh targets → keep-or-roll-back; durable tactic memory | follow-up | + +Both remain unchecked on issue #2713. diff --git a/docs/research/coin-benchmark-and-skwaq-study.md b/docs/research/coin-benchmark-and-skwaq-study.md index e31d7b53..445e9d8a 100644 --- a/docs/research/coin-benchmark-and-skwaq-study.md +++ b/docs/research/coin-benchmark-and-skwaq-study.md @@ -505,6 +505,14 @@ coin-gym improve --holdout fresh # one self-improvement cycl coin-gym profiles # list per-model isolated state ``` +> **Implemented (Phase 4).** The Phase-4 CLI landed as +> `coin-gym run [--strategy baseline|team] [--profile ] [--targets ]`, +> `score|compare|improve [--profile ]`, and `profiles`. The +> `improve` command runs the **offline** failure-analyst + overfitting-reviewer +> gate over a saved run; the live `--holdout fresh` verify/rollback cycle +> sketched above needs live grading and is Phase 5. See +> [Run the LOCAL COIN Gym harness](../howto/run-the-coin-gym-harness.md). + ### 3.6 Anti-overfitting: the central design tension skwaq's whole loop is a fight against **building to the benchmark**. COIN gives @@ -536,9 +544,18 @@ that fight a stronger footing than skwaq's static suites: | 1 | LEARN COIN (this doc, Part 1) | ✅ done | | 2 | STUDY skwaq loop (this doc, Part 2) | ✅ done | | 3 | Provision compute (`azlin` VM) + pull COIN snapshot | ⏭ tracked (issue) | -| 4 | Build the LOCAL COIN Gym harness (Part 3) | ⏭ tracked (issue) | +| 4 | Build the LOCAL COIN Gym harness (Part 3) | ✅ done — Rust `coin_gym` module + `coin-gym` CLI; see [Run the LOCAL COIN Gym harness](../howto/run-the-coin-gym-harness.md) | | 5 | Iterative self-improve (baseline vs team; failure-analysis + overfit gate) | ⏭ tracked (issue) | +> **Phase 4 note (language).** The harness landed as a Rust module +> (`src/coin_gym/`) exposing a `coin-gym` CLI, not a standalone Python package: +> the Simard repo enforces a Rust-only policy (issue #2155, +> `scripts/check-rust-only-gate.sh`) and this design already called the harness a +> "Simard crate". COIN's own `coin evaluate` tool (Python/uv/Docker) stays an +> **external** oracle the harness delegates to via a mockable executor — never +> re-implemented. See +> [Run the LOCAL COIN Gym harness](../howto/run-the-coin-gym-harness.md). + Phases 3–5 are captured in the tracking issue **"Build LOCAL COIN Gym harness — phases 3-5"** in `rysweet/Simard`, which carries the harness design above plus the remaining work (VM provision, harness build, iterative self-improve). diff --git a/mkdocs.yml b/mkdocs.yml index fddb8b37..cd0934e2 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -133,6 +133,7 @@ nav: - First Benchmark Suite: tutorials/run-your-first-benchmark-gym.md - Read Simard's Daily Journal: tutorials/read-simards-daily-journal.md - How-To Guides: + - Run the LOCAL COIN Gym Harness: howto/run-the-coin-gym-harness.md - Terminal to Engineer: howto/move-from-terminal-recipes-into-engineer-runs.md - Use the Agent-Orchestration Engineer Loop: howto/use-agent-orchestration-engineer-loop.md - Bootstrap and Reflection: howto/configure-bootstrap-and-inspect-reflection.md diff --git a/src/bin/coin_gym.rs b/src/bin/coin_gym.rs new file mode 100644 index 00000000..6af87983 --- /dev/null +++ b/src/bin/coin_gym.rs @@ -0,0 +1,8 @@ +//! `coin-gym` — the LOCAL COIN Gym harness CLI (Phase 4 of issue #2713). +//! +//! A thin entry point; all logic lives in `simard::coin_gym`. See +//! `docs/howto/run-the-coin-gym-harness.md`. + +fn main() -> Result<(), Box> { + simard::dispatch_coin_gym_cli(std::env::args().skip(1)) +} diff --git a/src/coin_gym/agent_runner.rs b/src/coin_gym/agent_runner.rs new file mode 100644 index 00000000..d9cd11c1 --- /dev/null +++ b/src/coin_gym/agent_runner.rs @@ -0,0 +1,315 @@ +//! Agent-under-test runner (research doc Part 3.3, component 2). +//! +//! Two interchangeable strategies sit behind one [`AgentStrategy`] interface: +//! +//! - [`BaselineStrategy`] — a single model reads the harness + source and submits +//! its candidate input directly. +//! - [`TeamStrategy`] — a skwaq-style debate: a *reacher* proposes an input, a +//! *skeptic* challenges whether it truly reaches `ℓ`, and a *synthesizer* +//! submits-or-abstains via a `threshold_hint`-style gate. Because COIN +//! **precision** punishes over-claiming, abstaining on a low-confidence input +//! is often better than submitting a wrong one. +//! +//! The actual reasoning is abstracted behind [`Reasoner`] so a live LLM (via +//! LiteLLM, Phase 3+) and an offline scripted stand-in ([`FixtureReasoner`]) are +//! interchangeable. + +use std::collections::HashMap; +use std::sync::atomic::{AtomicU64, Ordering}; +use std::time::{SystemTime, UNIX_EPOCH}; + +use super::executor::HarnessExecutor; +use super::types::{CoinGymResult, Outcome, OutcomeCode, RunReport, Strategy, Target}; + +/// Default `threshold_hint` for the team synthesizer's submit/abstain gate. +pub const DEFAULT_THRESHOLD_HINT: f64 = 0.6; + +/// Process-local monotonic counter appended to run ids so two runs of the same +/// model+strategy in the same millisecond never collide. +static RUN_SEQ: AtomicU64 = AtomicU64::new(0); + +/// A candidate input proposed by the agent under test. +#[derive(Clone, Debug, PartialEq)] +pub struct Candidate { + /// The candidate input (a placeholder UTF-8 string in the scaffold; real + /// COIN inputs are raw bytes staged for `coin evaluate`). + pub input: String, + /// The agent's self-reported confidence in `0.0..=1.0`. + pub confidence: f64, + /// Free-text rationale (why this input should reach the line). + pub rationale: String, +} + +/// Produces candidate inputs for targets. A live implementation calls a model +/// through LiteLLM; [`FixtureReasoner`] replays scripted candidates offline. +pub trait Reasoner { + /// Propose a candidate input for `target`, or `None` if the agent produced + /// nothing (yields a `NoSubmission` outcome). + fn propose(&self, target: &Target) -> Option; + + /// Skeptic assessment of how likely `candidate` truly reaches the line, in + /// `0.0..=1.0`. Defaults to the candidate's own confidence; a real skeptic + /// agent lowers this when it spots over-claiming. + fn assess(&self, _target: &Target, candidate: &Candidate) -> f64 { + candidate.confidence + } +} + +/// A reasoner that replays scripted candidates keyed by target id. Targets +/// without a script entry yield `None` (→ `NoSubmission`). +#[derive(Clone, Debug, Default)] +pub struct FixtureReasoner { + script: HashMap, +} + +impl FixtureReasoner { + /// Build from a `target_id -> Candidate` script map. + #[must_use] + pub fn new(script: HashMap) -> Self { + Self { script } + } +} + +impl Reasoner for FixtureReasoner { + fn propose(&self, target: &Target) -> Option { + self.script.get(&target.id).cloned() + } +} + +/// The synthesizer's decision for one target. +#[derive(Clone, Debug, PartialEq)] +pub enum SubmissionDecision { + /// Submit `input` for grading. + Submit { + /// The input to grade. + input: String, + /// Confidence attached to the submission. + confidence: f64, + }, + /// Deliberately decline to submit (precision-preserving). + Abstain { + /// Why the agent abstained. + reason: String, + }, + /// The agent produced no candidate at all. + NoSubmission, +} + +/// The agent's submission for one target, with the rationale trail. +#[derive(Clone, Debug, PartialEq)] +pub struct Submission { + /// Target this submission is for. + pub target_id: String, + /// The submit/abstain/no-submission decision. + pub decision: SubmissionDecision, + /// Human-readable rationale. + pub rationale: String, +} + +/// A strategy for turning a target into a submission. +pub trait AgentStrategy { + /// Which strategy this is (recorded in the run report). + fn kind(&self) -> Strategy; + + /// Decide what to submit (or abstain) for `target`. + fn evaluate(&self, target: &Target) -> Submission; +} + +/// Single-model baseline: submit the proposed candidate as-is. +#[derive(Clone, Debug)] +pub struct BaselineStrategy { + reasoner: R, +} + +impl BaselineStrategy { + /// Create a baseline strategy over `reasoner`. + #[must_use] + pub fn new(reasoner: R) -> Self { + Self { reasoner } + } +} + +impl AgentStrategy for BaselineStrategy { + fn kind(&self) -> Strategy { + Strategy::Baseline + } + + fn evaluate(&self, target: &Target) -> Submission { + match self.reasoner.propose(target) { + Some(candidate) => Submission { + target_id: target.id.clone(), + decision: SubmissionDecision::Submit { + input: candidate.input, + confidence: candidate.confidence, + }, + rationale: candidate.rationale, + }, + None => Submission { + target_id: target.id.clone(), + decision: SubmissionDecision::NoSubmission, + rationale: "reasoner produced no candidate".to_string(), + }, + } + } +} + +/// Multi-agent team: reacher → skeptic → synthesizer submit/abstain gate. +#[derive(Clone, Debug)] +pub struct TeamStrategy { + reasoner: R, + threshold_hint: f64, +} + +impl TeamStrategy { + /// Create a team strategy with the default `threshold_hint`. + #[must_use] + pub fn new(reasoner: R) -> Self { + Self { + reasoner, + threshold_hint: DEFAULT_THRESHOLD_HINT, + } + } + + /// Create a team strategy with an explicit `threshold_hint` (clamped to + /// `0.0..=1.0`). + #[must_use] + pub fn with_threshold(reasoner: R, threshold_hint: f64) -> Self { + Self { + reasoner, + threshold_hint: threshold_hint.clamp(0.0, 1.0), + } + } + + /// The submit/abstain threshold in effect. + #[must_use] + pub fn threshold_hint(&self) -> f64 { + self.threshold_hint + } +} + +impl AgentStrategy for TeamStrategy { + fn kind(&self) -> Strategy { + Strategy::Team + } + + fn evaluate(&self, target: &Target) -> Submission { + let Some(candidate) = self.reasoner.propose(target) else { + return Submission { + target_id: target.id.clone(), + decision: SubmissionDecision::NoSubmission, + rationale: "reacher produced no candidate".to_string(), + }; + }; + // Skeptic challenges the reacher's over-claim; synthesizer gates on it. + let skeptic_score = self.reasoner.assess(target, &candidate); + if skeptic_score >= self.threshold_hint { + Submission { + target_id: target.id.clone(), + decision: SubmissionDecision::Submit { + input: candidate.input, + confidence: skeptic_score, + }, + rationale: format!( + "skeptic score {skeptic_score:.2} >= threshold {:.2}; {}", + self.threshold_hint, candidate.rationale + ), + } + } else { + Submission { + target_id: target.id.clone(), + decision: SubmissionDecision::Abstain { + reason: format!( + "skeptic score {skeptic_score:.2} < threshold {:.2} (precision-preserving abstention)", + self.threshold_hint + ), + }, + rationale: candidate.rationale, + } + } + } +} + +/// Drives a strategy over a target set and grades each submission through the +/// harness executor to produce a [`RunReport`]. +pub struct AgentRunner<'a, S: AgentStrategy, E: HarnessExecutor> { + strategy: &'a S, + executor: &'a E, + model: String, + snapshot: String, +} + +impl<'a, S: AgentStrategy, E: HarnessExecutor> AgentRunner<'a, S, E> { + /// Create a runner for `model` on `snapshot`. + #[must_use] + pub fn new( + strategy: &'a S, + executor: &'a E, + model: impl Into, + snapshot: impl Into, + ) -> Self { + Self { + strategy, + executor, + model: model.into(), + snapshot: snapshot.into(), + } + } + + /// Evaluate every target, grading submitted inputs through the executor. + /// + /// # Errors + /// Propagates [`crate::coin_gym::types::CoinGymError::Executor`] if the + /// executor cannot grade at all (e.g. the real `coin evaluate` delegate on a + /// host without Docker). Legitimate *timeout*/*error* grade verdicts are + /// recorded as outcomes, not surfaced as errors. + pub fn run(&self, targets: &[Target]) -> CoinGymResult { + let started_at_unix_ms = now_unix_ms(); + let mut outcomes = Vec::with_capacity(targets.len()); + for target in targets { + let submission = self.strategy.evaluate(target); + let code = match submission.decision { + SubmissionDecision::Submit { input, .. } => { + self.executor.grade(target, &input)?.to_outcome_code() + } + SubmissionDecision::Abstain { .. } => OutcomeCode::Abstained, + SubmissionDecision::NoSubmission => OutcomeCode::NoSubmission, + }; + outcomes.push(Outcome { + target_id: target.id.clone(), + family: target.family, + code, + cost_usd: 0.0, + }); + } + let run_id = format!( + "{}-{}-{}-{}", + sanitize(&self.model), + self.strategy.kind().label(), + started_at_unix_ms, + RUN_SEQ.fetch_add(1, Ordering::Relaxed) + ); + Ok(RunReport { + run_id, + model: self.model.clone(), + strategy: self.strategy.kind(), + snapshot: self.snapshot.clone(), + started_at_unix_ms, + outcomes, + offline_scaffold: self.executor.is_offline_scaffold(), + }) + } +} + +fn now_unix_ms() -> u128 { + SystemTime::now() + .duration_since(UNIX_EPOCH) + .map(|d| d.as_millis()) + .unwrap_or(0) +} + +fn sanitize(model: &str) -> String { + model + .chars() + .map(|c| if c.is_ascii_alphanumeric() { c } else { '-' }) + .collect() +} diff --git a/src/coin_gym/executor.rs b/src/coin_gym/executor.rs new file mode 100644 index 00000000..7405d970 --- /dev/null +++ b/src/coin_gym/executor.rs @@ -0,0 +1,212 @@ +//! Harness executor (research doc Part 3.3, component 3). +//! +//! The executor is the **objective oracle**: it decides whether a submitted +//! input actually reaches the target line. In production this delegates to +//! `coin evaluate` (Docker + instrumented replay) and is **never +//! re-implemented**. Real Docker wiring is gated behind Phase 3 (a provisioned +//! VM). For offline development, tests, and CI, a [`MockHarnessExecutor`] returns +//! deterministic verdicts from a ground-truth lookup table — a test double, not +//! a reachability engine. + +use std::collections::{HashMap, HashSet}; + +use super::types::{CoinGymError, CoinGymResult, OutcomeCode, Target}; + +/// The oracle's verdict for a single submitted input. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum GradeResult { + /// The input drove execution to the target line. + Reached, + /// The input was valid but did not reach the target line. + WrongInput, + /// Grading exceeded the per-target budget. + TimedOut, + /// Grading errored (harness crash, build failure, etc.). + Error, +} + +impl GradeResult { + /// Map a grade of a *submitted* input to its outcome code. + #[must_use] + pub fn to_outcome_code(self) -> OutcomeCode { + match self { + Self::Reached => OutcomeCode::Reached, + Self::WrongInput => OutcomeCode::WrongInput, + Self::TimedOut => OutcomeCode::TimedOut, + Self::Error => OutcomeCode::Error, + } + } +} + +/// Grades a submitted input against a target. Implementors are the objective +/// oracle; they must not be spoofed by the agent under test. +pub trait HarnessExecutor { + /// Grade `input` against `target`. + /// + /// # Errors + /// Returns [`CoinGymError::Executor`] when grading cannot be performed at all + /// (e.g. `coin evaluate` is unavailable because the Phase-3 VM is not + /// provisioned). A *reached/not-reached* verdict is returned as + /// `Ok(GradeResult)`, not an error. + fn grade(&self, target: &Target, input: &str) -> CoinGymResult; + + /// Whether this executor is an offline scaffold (mock) rather than the real + /// `coin evaluate` oracle. Recorded on the [`crate::coin_gym::types::RunReport`] + /// so offline runs are never mistaken for graded results. + fn is_offline_scaffold(&self) -> bool { + false + } +} + +// ── Real executor: delegates to `coin evaluate` (Phase 3 gated) ─────────────── + +/// Configuration for the real `coin evaluate` delegate. +#[derive(Clone, Debug)] +pub struct CoinEvaluateConfig { + /// Path/name of the `coin` binary (default `coin`). + pub binary: String, + /// Snapshot dataset reference (e.g. `you/coin@v1`). + pub dataset: String, +} + +impl CoinEvaluateConfig { + /// Build a config for a snapshot using the default `coin` binary. + #[must_use] + pub fn new(dataset: impl Into) -> Self { + Self { + binary: "coin".to_string(), + dataset: dataset.into(), + } + } +} + +/// Delegates grading to the COIN maintainer harness via `coin evaluate`. +/// +/// This is the production oracle path. It is intentionally a thin shell around +/// the external tool — the reachability judgement lives entirely in +/// `coin evaluate`'s instrumented replay, never here. Actually invoking it +/// requires Docker + a pulled snapshot (Phase 3); until then [`Self::grade`] +/// surfaces a clear Phase-3 gate error while [`Self::build_argv`] (the exact +/// delegation contract) stays unit-testable offline. +#[derive(Clone, Debug)] +pub struct CoinEvaluateExecutor { + config: CoinEvaluateConfig, +} + +impl CoinEvaluateExecutor { + /// Create the delegate from a config. + #[must_use] + pub fn new(config: CoinEvaluateConfig) -> Self { + Self { config } + } + + /// The argv this executor would pass to the `coin` binary to grade a target, + /// with the candidate input already staged at `input_path`. + /// + /// Exposed (and unit-tested) so the delegation contract is verified without a + /// Docker host. The reachability oracle remains `coin evaluate` itself. + #[must_use] + pub fn build_argv(&self, target: &Target, input_path: &str) -> Vec { + vec![ + self.config.binary.clone(), + "evaluate".to_string(), + "--dataset".to_string(), + self.config.dataset.clone(), + "--target".to_string(), + target.id.clone(), + "--input".to_string(), + input_path.to_string(), + ] + } +} + +impl HarnessExecutor for CoinEvaluateExecutor { + fn grade(&self, _target: &Target, _input: &str) -> CoinGymResult { + // Phase 3 gate: real grading needs `coin evaluate` on a Docker host with + // a pulled snapshot. Surfacing an explicit error (rather than a silent + // fake verdict) keeps the harness honest — an offline run must use + // MockHarnessExecutor and be flagged `offline_scaffold`. + Err(CoinGymError::Executor(format!( + "`{} evaluate` requires a Docker host + pulled snapshot (Phase 3, azlin VM); \ + use the offline MockHarnessExecutor for local scaffold runs", + self.config.binary + ))) + } +} + +// ── Mock executor: deterministic test double ───────────────────────────────── + +/// A deterministic oracle test double backed by a ground-truth lookup table. +/// +/// It does **not** compute reachability — it simply checks the submitted input +/// against a known reaching input per target (and optional injected +/// timeout/error sets). This lets the full pipeline run offline without a VM +/// while keeping the real oracle (`coin evaluate`) the only thing that ever +/// judges reachability for real. +#[derive(Clone, Debug, Default)] +pub struct MockHarnessExecutor { + reaching_input: HashMap, + timeout_ids: HashSet, + error_ids: HashSet, +} + +impl MockHarnessExecutor { + /// An empty mock (every submission grades as `WrongInput`). + #[must_use] + pub fn new() -> Self { + Self::default() + } + + /// Build a mock from a `target_id -> reaching_input` ground-truth map. + #[must_use] + pub fn from_oracle(reaching_input: HashMap) -> Self { + Self { + reaching_input, + ..Self::default() + } + } + + /// Register a reaching input for a target. + #[must_use] + pub fn with_reaching_input( + mut self, + target_id: impl Into, + input: impl Into, + ) -> Self { + self.reaching_input.insert(target_id.into(), input.into()); + self + } + + /// Force a target to grade as `TimedOut` regardless of input. + #[must_use] + pub fn with_timeout(mut self, target_id: impl Into) -> Self { + self.timeout_ids.insert(target_id.into()); + self + } + + /// Force a target to grade as `Error` regardless of input. + #[must_use] + pub fn with_error(mut self, target_id: impl Into) -> Self { + self.error_ids.insert(target_id.into()); + self + } +} + +impl HarnessExecutor for MockHarnessExecutor { + fn grade(&self, target: &Target, input: &str) -> CoinGymResult { + if self.error_ids.contains(&target.id) { + return Ok(GradeResult::Error); + } + if self.timeout_ids.contains(&target.id) { + return Ok(GradeResult::TimedOut); + } + match self.reaching_input.get(&target.id) { + Some(expected) if expected == input => Ok(GradeResult::Reached), + _ => Ok(GradeResult::WrongInput), + } + } + + fn is_offline_scaffold(&self) -> bool { + true + } +} diff --git a/src/coin_gym/fixtures/sample_snapshot.json b/src/coin_gym/fixtures/sample_snapshot.json new file mode 100644 index 00000000..2c0254aa --- /dev/null +++ b/src/coin_gym/fixtures/sample_snapshot.json @@ -0,0 +1,89 @@ +{ + "snapshot": "you/coin@v1-sample", + "note": "Illustrative Phase-4 scaffold fixture. Inputs are placeholder UTF-8 strings, not real COIN byte payloads; real grading requires `coin evaluate` on a Docker host (Phase 3). The `oracle` and `script` sections drive the offline demo run and the unit tests only.", + "targets": { + "pinned": [ + { + "id": "libraw-fuji-480", + "project": "libraw", + "commit": "d4c1f0a", + "harness": "libraw_raf_fuzzer", + "file": "src/metadata/fuji.cpp", + "line": 480, + "family": "frontier" + }, + { + "id": "libraw-crx-221", + "project": "libraw", + "commit": "d4c1f0a", + "harness": "libraw_crx_fuzzer", + "file": "src/decoders/crx.cpp", + "line": 221, + "family": "non-trivial-reachable" + }, + { + "id": "liboqs-kem-88", + "project": "liboqs", + "commit": "9a3be21", + "harness": "kem_fuzzer", + "file": "src/kem/kem.c", + "line": 88, + "family": "non-trivial-reachable" + }, + { + "id": "zstd-huf-1207", + "project": "zstd", + "commit": "0f1e2d3", + "harness": "decompress_fuzzer", + "file": "lib/decompress/huf_decompress.c", + "line": 1207, + "family": "frontier" + }, + { + "id": "harfbuzz-shape-540", + "project": "harfbuzz", + "commit": "77ac0de", + "harness": "hb_shape_fuzzer", + "file": "src/hb-ot-shape.cc", + "line": 540, + "family": "non-trivial-reachable" + } + ], + "held_out_fresh": [ + { + "id": "libpng-idat-903", + "project": "libpng", + "commit": "5b1e7f2", + "harness": "libpng_read_fuzzer", + "file": "pngrutil.c", + "line": 903, + "family": "frontier" + }, + { + "id": "openssl-asn1-1442", + "project": "openssl", + "commit": "e88a1c4", + "harness": "asn1_fuzzer", + "file": "crypto/asn1/tasn_dec.c", + "line": 1442, + "family": "non-trivial-reachable" + } + ] + }, + "oracle": { + "libraw-fuji-480": "FUJIFILMCCD-RAW\u0000header+deep-branch", + "libraw-crx-221": "CRX\u0000frame+huff-table", + "liboqs-kem-88": "kem-seed:decaps-branch", + "zstd-huf-1207": "\u0028\u00b5\u002f\u00fd+huffman-weights", + "harfbuzz-shape-540": "GSUB-lookup:contextual", + "libpng-idat-903": "\u0089PNG+malformed-idat", + "openssl-asn1-1442": "30:82:constructed-indefinite" + }, + "script": { + "libraw-fuji-480": { "input": "FUJIFILMCCD-RAW\u0000header+deep-branch", "confidence": 0.82, "rationale": "satisfied RAF magic header, then drove the Fuji metadata parser into the deep branch" }, + "libraw-crx-221": { "input": "CRX\u0000frame+huff-table", "confidence": 0.71, "rationale": "constructed a minimal CRX frame with a populated Huffman table" }, + "liboqs-kem-88": { "input": "kem-seed:wrong-branch", "confidence": 0.44, "rationale": "guessed a decapsulation seed; unsure it flips the target branch" }, + "zstd-huf-1207": { "input": "\u0028\u00b5\u002f\u00fd+wrong-weights", "confidence": 0.35, "rationale": "valid zstd magic but Huffman weights likely do not reach line 1207" }, + "harfbuzz-shape-540": { "input": "GSUB-lookup:contextual", "confidence": 0.66, "rationale": "contextual GSUB lookup should exercise the shaping path" } + } +} diff --git a/src/coin_gym/improve.rs b/src/coin_gym/improve.rs new file mode 100644 index 00000000..59efba48 --- /dev/null +++ b/src/coin_gym/improve.rs @@ -0,0 +1,286 @@ +//! Self-improvement scaffold (research doc Part 3.3, component 6). +//! +//! This is the **Phase-4** slice of the loop: the pure, offline, fully-testable +//! pieces — a **failure-analyst** that turns unreached targets into *general* +//! reachability tactics, and the **overfitting-reviewer GATE** that rejects any +//! tactic which memorises a specific input or keys off a specific target / +//! project. Both mirror skwaq's `failure-analyst` + `overfitting-reviewer`. +//! +//! The **live** part of the loop — apply an accepted tactic, re-run on held-out +//! *fresh* targets, and keep it iff reach improves without a precision +//! regression (else roll back) — needs real grading and is **Phase 5** +//! (tracked on issue #2713). It is intentionally NOT implemented here. + +use serde::Serialize; + +use super::target_loader::TargetSet; +use super::types::{OutcomeCode, RunReport, Target, TargetFamily}; + +/// A general reachability tactic proposed from a failing target. +#[derive(Clone, Debug, PartialEq, Serialize)] +pub struct TacticProposal { + /// Stable proposal id. + pub id: String, + /// The target whose failure motivated the proposal (diagnosis context). + pub target_id: String, + /// The proposed **general** tactic (must generalise across projects/harnesses). + pub tactic: String, + /// Evidence for the diagnosis (may be target-specific; the *tactic* may not). + pub evidence: String, +} + +/// The overfitting-reviewer's verdict. +#[derive(Clone, Copy, Debug, Eq, PartialEq, Serialize)] +#[serde(rename_all = "kebab-case")] +pub enum ReviewVerdict { + /// The tactic generalises — keep it. + Accept, + /// The tactic memorises a specific input / keys off a specific target — drop it. + Reject, +} + +/// A proposal after review. +#[derive(Clone, Debug, PartialEq, Serialize)] +pub struct ReviewedProposal { + /// The reviewed proposal. + pub proposal: TacticProposal, + /// Accept/Reject. + pub verdict: ReviewVerdict, + /// Why the reviewer decided this way. + pub reason: String, +} + +impl ReviewedProposal { + /// Uppercase label for the verdict (`ACCEPT` / `REJECT`). + #[must_use] + pub fn verdict_label(&self) -> &'static str { + match self.verdict { + ReviewVerdict::Accept => "ACCEPT", + ReviewVerdict::Reject => "REJECT", + } + } +} + +/// The result of one offline analysis cycle. +#[derive(Clone, Debug, PartialEq, Serialize)] +pub struct ImproveReport { + /// The run analysed. + pub run_id: String, + /// Number of unreached targets analysed. + pub analyzed: usize, + /// Reviewed proposals (accepted + rejected). + pub proposals: Vec, + /// Count accepted by the gate. + pub accepted: usize, + /// Count rejected by the gate. + pub rejected: usize, + /// Phase boundary note (live verify/rollback is Phase 5). + pub note: String, +} + +/// Broad heuristic class a target falls into, used to pick a *general* tactic. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +enum TacticCategory { + FormatGatedDecoder, + CryptoStateMachine, + Generic, +} + +fn categorize(target: &Target) -> TacticCategory { + let hay = format!( + "{} {} {}", + target.project.to_ascii_lowercase(), + target.harness.to_ascii_lowercase(), + target.file.to_ascii_lowercase() + ); + let is_decoder = [ + "raw", + "png", + "jpeg", + "zstd", + "decompress", + "decode", + "image", + "font", + "shape", + "harfbuzz", + "metadata", + ] + .iter() + .any(|k| hay.contains(k)); + let is_crypto = ["oqs", "kem", "asn1", "ssl", "crypto", "cipher", "tls"] + .iter() + .any(|k| hay.contains(k)); + if is_crypto { + TacticCategory::CryptoStateMachine + } else if is_decoder { + TacticCategory::FormatGatedDecoder + } else { + TacticCategory::Generic + } +} + +fn tactic_text(category: TacticCategory) -> &'static str { + match category { + TacticCategory::FormatGatedDecoder => { + "For format-gated decoders, first satisfy the container's magic-byte / header \ + validator, then construct the minimal well-formed structure that routes control \ + flow into the guarded deep branch." + } + TacticCategory::CryptoStateMachine => { + "For cryptographic state machines, drive the protocol into the specific state that \ + unlocks the target branch (e.g. a decapsulation or parse-error path) rather than \ + fuzzing random bytes." + } + TacticCategory::Generic => { + "Work backward from the target line's guarding predicate to the input constraints \ + that satisfy it, prioritising branch conditions over surface-level fuzzing." + } + } +} + +/// Failure-analyst: turn each **unreached** target (`W`/`T`/`N`) into a general +/// reachability tactic proposal. Abstentions (`A`) are deliberate no-claims and +/// are not treated as failures. Reached (`R`) targets need no tactic. +#[must_use] +pub fn analyze_failures(report: &RunReport, targets: &TargetSet) -> Vec { + let all: Vec<&Target> = targets + .pinned + .iter() + .chain(targets.held_out_fresh.iter()) + .collect(); + let mut proposals = Vec::new(); + for outcome in &report.outcomes { + let is_failure = matches!( + outcome.code, + OutcomeCode::WrongInput | OutcomeCode::TimedOut | OutcomeCode::NoSubmission + ); + if !is_failure { + continue; + } + let Some(target) = all.iter().find(|t| t.id == outcome.target_id) else { + continue; + }; + let category = categorize(target); + proposals.push(TacticProposal { + id: format!("tactic-{}", outcome.target_id), + target_id: outcome.target_id.clone(), + tactic: tactic_text(category).to_string(), + evidence: format!( + "target {} ({}) ended {}; the guarding predicate at line {} was not satisfied", + target.locator(), + family_label(target.family), + outcome.code.letter(), + target.line + ), + }); + } + proposals +} + +fn family_label(family: TargetFamily) -> &'static str { + family.label() +} + +/// Overfitting-reviewer GATE. Rejects a tactic that memorises a specific input +/// or keys off a specific target / project / locator; accepts tactics that +/// plausibly generalise. Mirrors skwaq rejecting benchmark-specific naming. +/// +/// The `targets` set supplies the concrete ids / project names / locators the +/// tactic must **not** encode. +#[must_use] +pub fn review_proposal(proposal: &TacticProposal, targets: &TargetSet) -> ReviewedProposal { + let tactic_lc = proposal.tactic.to_ascii_lowercase(); + + // 1. Explicit memorisation language. + const OVERFIT_PHRASES: &[&str] = &[ + "memorize", + "memorise", + "hardcode", + "hard-code", + "hard code", + "specific input", + "exact bytes", + "exact input", + "this input", + "known input", + "cached input", + ]; + if let Some(phrase) = OVERFIT_PHRASES.iter().find(|p| tactic_lc.contains(**p)) { + return reject( + proposal, + format!("tactic encodes a memorised input (\"{phrase}\")"), + ); + } + + // 2. Keys off a specific target id or locator. + for target in targets.pinned.iter().chain(targets.held_out_fresh.iter()) { + if tactic_lc.contains(&target.id.to_ascii_lowercase()) { + return reject( + proposal, + format!("tactic names a specific target id '{}'", target.id), + ); + } + let locator = target.locator().to_ascii_lowercase(); + if tactic_lc.contains(&locator) { + return reject( + proposal, + format!( + "tactic names a specific target locator '{}'", + target.locator() + ), + ); + } + // Keying off a concrete project name is target-specific. + let project = target.project.to_ascii_lowercase(); + if project.len() >= 4 && tactic_lc.contains(&project) { + return reject( + proposal, + format!("tactic keys off a specific project '{}'", target.project), + ); + } + } + + ReviewedProposal { + proposal: proposal.clone(), + verdict: ReviewVerdict::Accept, + reason: "tactic generalises across projects/harnesses; no memorised input or \ + target-specific key detected" + .to_string(), + } +} + +fn reject(proposal: &TacticProposal, reason: String) -> ReviewedProposal { + ReviewedProposal { + proposal: proposal.clone(), + verdict: ReviewVerdict::Reject, + reason, + } +} + +/// Run one **offline** analysis cycle: failure-analysis → overfitting-review. +/// This does NOT apply, verify, or roll back tactics — that live loop needs real +/// grading on held-out fresh targets and is Phase 5 (issue #2713). +#[must_use] +pub fn analyze_and_review(report: &RunReport, targets: &TargetSet) -> ImproveReport { + let reviewed: Vec = analyze_failures(report, targets) + .iter() + .map(|p| review_proposal(p, targets)) + .collect(); + let accepted = reviewed + .iter() + .filter(|r| r.verdict == ReviewVerdict::Accept) + .count(); + let rejected = reviewed.len() - accepted; + ImproveReport { + run_id: report.run_id.clone(), + analyzed: reviewed.len(), + proposals: reviewed, + accepted, + rejected, + note: "offline analysis only — applying an accepted tactic, re-running on held-out \ + fresh targets, and keeping it iff reach improves without a precision regression \ + (else roll back) requires live `coin evaluate` grading and is Phase 5 (issue #2713)" + .to_string(), + } +} diff --git a/src/coin_gym/leaderboard.rs b/src/coin_gym/leaderboard.rs new file mode 100644 index 00000000..f9836180 --- /dev/null +++ b/src/coin_gym/leaderboard.rs @@ -0,0 +1,213 @@ +//! Leaderboard comparator (research doc Part 3.3, component 5). +//! +//! Diffs a local run's reach/precision against COIN's **published** targeted-track +//! numbers for the same model. Because grading is execution-based and +//! reproducible from a pinned snapshot, a correct local run should land within +//! variance of the published figures; a **material deviation** is a signal of a +//! harness/config bug, not a capability result. +//! +//! The published table is transcribed from the research doc Part 1.5 +//! (`assets/app.js` / `export/T2_main_targeted.csv`). It is a fixed reference +//! set; refresh it when COIN republishes. + +use serde::Serialize; + +use super::scorer::Score; + +/// Percentage-point gap beyond which a local run is flagged as a material +/// deviation from the published leaderboard (⇒ suspect the harness/config). +pub const MATERIAL_DEVIATION_PCT: f64 = 10.0; + +/// One published leaderboard row (targeted-reachability track). +#[derive(Clone, Copy, Debug, PartialEq, Serialize)] +pub struct LeaderboardEntry { + /// Published rank (1-based). + pub rank: u32, + /// Model name as published. + pub model: &'static str, + /// Agent scaffold used in the published run. + pub scaffold: &'static str, + /// Published reach percentage. + pub reach_pct: f64, + /// Published precision percentage. + pub precision_pct: f64, + /// Targets reached out of 70. + pub reached: u32, + /// Frontier targets reached out of 35. + pub frontier_reached: u32, +} + +/// The published COIN targeted-track leaderboard (8 agents × 70 targets). +#[must_use] +pub fn published_leaderboard() -> &'static [LeaderboardEntry] { + const ENTRIES: &[LeaderboardEntry] = &[ + LeaderboardEntry { + rank: 1, + model: "Claude Opus 4.6", + scaffold: "Claude Code", + reach_pct: 30.0, + precision_pct: 52.5, + reached: 21, + frontier_reached: 1, + }, + LeaderboardEntry { + rank: 2, + model: "Claude Sonnet 4.6", + scaffold: "Claude Code", + reach_pct: 25.7, + precision_pct: 45.0, + reached: 18, + frontier_reached: 0, + }, + LeaderboardEntry { + rank: 3, + model: "Gemini 3.1 Pro", + scaffold: "Gemini CLI", + reach_pct: 24.3, + precision_pct: 51.5, + reached: 17, + frontier_reached: 1, + }, + LeaderboardEntry { + rank: 4, + model: "GPT-5.4", + scaffold: "Codex", + reach_pct: 22.9, + precision_pct: 41.0, + reached: 16, + frontier_reached: 0, + }, + LeaderboardEntry { + rank: 5, + model: "GPT-5.4-mini", + scaffold: "Codex", + reach_pct: 18.6, + precision_pct: 31.0, + reached: 13, + frontier_reached: 0, + }, + LeaderboardEntry { + rank: 6, + model: "GLM-5", + scaffold: "Claude Code", + reach_pct: 14.3, + precision_pct: 27.0, + reached: 10, + frontier_reached: 0, + }, + LeaderboardEntry { + rank: 7, + model: "Gemini 3 Flash", + scaffold: "Gemini CLI", + reach_pct: 12.9, + precision_pct: 15.3, + reached: 9, + frontier_reached: 0, + }, + LeaderboardEntry { + rank: 8, + model: "DeepSeek-V3.2", + scaffold: "Claude Code", + reach_pct: 7.1, + precision_pct: 12.8, + reached: 5, + frontier_reached: 0, + }, + ]; + ENTRIES +} + +/// Result of diffing a local run against the published leaderboard. +#[derive(Clone, Debug, PartialEq, Serialize)] +pub struct LeaderboardComparison { + /// The local model name. + pub model: String, + /// The matched published model name. + pub published_model: String, + /// Local reach percentage. + pub local_reach_pct: f64, + /// Published reach percentage. + pub published_reach_pct: f64, + /// local − published reach (percentage points). + pub reach_delta_pct: f64, + /// Local precision percentage. + pub local_precision_pct: f64, + /// Published precision percentage. + pub published_precision_pct: f64, + /// local − published precision (percentage points). + pub precision_delta_pct: f64, + /// `true` when either delta exceeds [`MATERIAL_DEVIATION_PCT`]. + pub material_deviation: bool, + /// Human-readable interpretation. + pub note: String, +} + +/// Find the published entry whose model matches `model` (normalised). +/// +/// Prefers an **exact** normalised match so a prefix like `gpt-5.4` never +/// shadows `gpt-5.4-mini`; falls back to the *longest* substring match. +#[must_use] +pub fn find_published(model: &str) -> Option<&'static LeaderboardEntry> { + let needle = normalize(model); + if needle.is_empty() { + return None; + } + let board = published_leaderboard(); + if let Some(entry) = board.iter().find(|e| normalize(e.model) == needle) { + return Some(entry); + } + board + .iter() + .filter(|e| { + let n = normalize(e.model); + !n.is_empty() && (n.contains(&needle) || needle.contains(&n)) + }) + .max_by_key(|e| normalize(e.model).len()) +} + +/// Compare a local [`Score`] to the published leaderboard entry for its model. +/// +/// Returns `None` when the model is not on the published leaderboard (nothing to +/// compare against). +#[must_use] +pub fn compare_to_leaderboard(score: &Score) -> Option { + let entry = find_published(&score.model)?; + let local_reach = score.overall.reach_pct(); + let local_precision = score.overall.precision_pct(); + let reach_delta = local_reach - entry.reach_pct; + let precision_delta = local_precision - entry.precision_pct; + let material_deviation = reach_delta.abs() > MATERIAL_DEVIATION_PCT + || precision_delta.abs() > MATERIAL_DEVIATION_PCT; + let note = if score.offline_scaffold { + "offline scaffold run (mock oracle) — deltas are illustrative only; real \ + comparison requires a `coin evaluate` grade on a pinned snapshot (Phase 3)" + .to_string() + } else if material_deviation { + format!( + "material deviation (>{MATERIAL_DEVIATION_PCT:.0} pts) from published — \ + suspect a harness/config bug, not a capability result" + ) + } else { + "within variance of the published leaderboard".to_string() + }; + Some(LeaderboardComparison { + model: score.model.clone(), + published_model: entry.model.to_string(), + local_reach_pct: local_reach, + published_reach_pct: entry.reach_pct, + reach_delta_pct: reach_delta, + local_precision_pct: local_precision, + published_precision_pct: entry.precision_pct, + precision_delta_pct: precision_delta, + material_deviation, + note, + }) +} + +fn normalize(model: &str) -> String { + model + .chars() + .filter(|c| c.is_ascii_alphanumeric()) + .map(|c| c.to_ascii_lowercase()) + .collect() +} diff --git a/src/coin_gym/mod.rs b/src/coin_gym/mod.rs new file mode 100644 index 00000000..6d3fa56e --- /dev/null +++ b/src/coin_gym/mod.rs @@ -0,0 +1,403 @@ +//! LOCAL COIN Gym harness (Phase 4 of issue #2713). +//! +//! A local harness that runs the COIN benchmark shape, scores locally against +//! the published leaderboard, and measures a single-model **baseline** vs. a +//! multi-agent **team**, mirroring skwaq's failure-analysis + +//! overfitting-reviewer gating. See +//! `docs/research/coin-benchmark-and-skwaq-study.md` (Part 3) for the design and +//! `docs/howto/run-the-coin-gym-harness.md` for usage. +//! +//! ## Scope (Phase 4) +//! This module is the **local scaffold**: a target loader, an agent runner +//! (baseline and team strategies behind one interface), a **mockable** harness +//! executor that delegates to `coin evaluate` (real Docker wiring is Phase 3), a +//! scorer, a leaderboard comparator, an offline failure-analyst plus +//! overfitting-reviewer gate, profiles, and the +//! `coin-gym run|score|compare|improve|profiles` CLI. The whole pipeline runs +//! offline against a mock oracle so it is exercised without a VM. Live grading +//! (Phase 3 VM) and the live self-improvement loop with verify/rollback +//! (Phase 5) remain follow-ups on issue #2713. + +pub mod agent_runner; +pub mod executor; +pub mod improve; +pub mod leaderboard; +pub mod profiles; +pub mod scorer; +pub mod target_loader; +pub mod types; + +#[cfg(test)] +mod tests_agent_runner; +#[cfg(test)] +mod tests_cli; +#[cfg(test)] +mod tests_executor; +#[cfg(test)] +mod tests_improve; +#[cfg(test)] +mod tests_leaderboard; +#[cfg(test)] +mod tests_profiles; +#[cfg(test)] +mod tests_scorer; +#[cfg(test)] +mod tests_target_loader; +#[cfg(test)] +mod tests_types; + +use std::collections::BTreeMap; +use std::path::Path; + +use agent_runner::{AgentRunner, BaselineStrategy, FixtureReasoner, TeamStrategy}; +use executor::MockHarnessExecutor; +use improve::analyze_and_review; +use leaderboard::compare_to_leaderboard; +use profiles::{PersistedRun, default_home, ensure_profile, list_profiles, load_run, save_run}; +use scorer::{Score, score_run}; +use target_loader::DemoScenario; +use types::{CoinGymError, CoinGymResult, RunReport, Strategy}; + +/// CLI usage string. +#[must_use] +pub fn coin_gym_usage() -> &'static str { + "usage: coin-gym \n\ + \n\ + commands:\n\ + \x20 run [--strategy baseline|team] [--profile ] [--targets ]\n\ + \x20 score [--profile ]\n\ + \x20 compare [--profile ]\n\ + \x20 improve [--profile ]\n\ + \x20 profiles\n\ + \n\ + Offline scaffold (Phase 4): runs grade against a mock oracle. Live grading\n\ + needs `coin evaluate` on a Docker host (Phase 3); the self-improvement\n\ + verify/rollback loop is Phase 5. See docs/howto/run-the-coin-gym-harness.md." +} + +/// Dispatch the `coin-gym` CLI over an argument iterator (argv minus the +/// program name). +/// +/// # Errors +/// Returns an error for unknown commands, missing/invalid arguments, or any +/// underlying I/O / parse failure. +pub fn dispatch_coin_gym_cli(args: I) -> Result<(), Box> +where + I: IntoIterator, +{ + let home = default_home(); + dispatch_with_home(&home, args)?; + Ok(()) +} + +/// Dispatch against an explicit home directory (used by tests). +pub(crate) fn dispatch_with_home(home: &Path, args: I) -> CoinGymResult<()> +where + I: IntoIterator, +{ + let argv: Vec = args.into_iter().collect(); + let command = argv + .first() + .ok_or_else(|| CoinGymError::Usage(coin_gym_usage().to_string()))?; + let rest = &argv[1..]; + match command.as_str() { + "run" => cmd_run(home, rest), + "score" => cmd_score(home, rest), + "compare" => cmd_compare(home, rest), + "improve" => cmd_improve(home, rest), + "profiles" => cmd_profiles(home, rest), + other => Err(CoinGymError::Usage(format!( + "unknown command '{other}'\n{}", + coin_gym_usage() + ))), + } +} + +// ── Argument parsing ───────────────────────────────────────────────────────── + +struct ParsedArgs { + positionals: Vec, + flags: BTreeMap, +} + +fn parse_args(rest: &[String], allowed_flags: &[&str]) -> CoinGymResult { + let mut positionals = Vec::new(); + let mut flags = BTreeMap::new(); + let mut iter = rest.iter(); + while let Some(arg) = iter.next() { + if let Some(name) = arg.strip_prefix("--") { + if !allowed_flags.contains(&name) { + return Err(CoinGymError::Usage(format!("unknown flag '--{name}'"))); + } + let value = iter + .next() + .ok_or_else(|| CoinGymError::Usage(format!("flag '--{name}' expects a value")))?; + flags.insert(name.to_string(), value.clone()); + } else { + positionals.push(arg.clone()); + } + } + Ok(ParsedArgs { positionals, flags }) +} + +// ── run ────────────────────────────────────────────────────────────────────── + +fn cmd_run(home: &Path, rest: &[String]) -> CoinGymResult<()> { + let parsed = parse_args(rest, &["strategy", "profile", "targets"])?; + let model = parsed + .positionals + .first() + .ok_or_else(|| CoinGymError::Usage("run: expected ".to_string()))? + .clone(); + let strategy = match parsed.flags.get("strategy") { + Some(v) => Strategy::parse(v).map_err(CoinGymError::Usage)?, + None => Strategy::Baseline, + }; + let scenario = match parsed.flags.get("targets") { + Some(path) => DemoScenario::from_path(Path::new(path))?, + None => DemoScenario::sample()?, + }; + validate_offline_scenario(&scenario)?; + let profile_name = parsed.flags.get("profile").map_or_else( + || profiles::sanitize_name(&model), + |p| profiles::sanitize_name(p), + ); + + let report = execute_run(&model, strategy, &scenario)?; + ensure_profile(home, &profile_name, &model)?; + let persisted = PersistedRun { + report: report.clone(), + targets: scenario.targets.clone(), + }; + let path = save_run(home, &profile_name, &persisted)?; + let score = score_run(&report); + + println!("run-id: {}", report.run_id); + println!("model: {}", report.model); + println!("strategy:{}", report.strategy); + println!("snapshot:{}", report.snapshot); + println!("profile: {profile_name}"); + println!("saved: {}", path.display()); + print_offline_note(report.offline_scaffold); + print_score(&score); + Ok(()) +} + +/// Guard against **hollow** offline runs: an offline scaffold run grades against +/// the manifest's mock `oracle` and draws candidates from its `script`. Without +/// pinned targets, oracle coverage, and at least one candidate, a run would +/// "succeed" while producing only meaningless `N`/`W` outcomes. Refuse instead. +fn validate_offline_scenario(scenario: &DemoScenario) -> CoinGymResult<()> { + if scenario.targets.pinned.is_empty() { + return Err(CoinGymError::Usage( + "snapshot has no pinned targets to evaluate".to_string(), + )); + } + let missing_oracle: Vec<&str> = scenario + .targets + .pinned + .iter() + .filter(|t| !scenario.oracle.contains_key(&t.id)) + .map(|t| t.id.as_str()) + .collect(); + if !missing_oracle.is_empty() { + return Err(CoinGymError::Usage(format!( + "offline run requires an `oracle` (mock reaching input) for every pinned target; \ + missing: {}", + missing_oracle.join(", ") + ))); + } + if scenario.script.is_empty() { + return Err(CoinGymError::Usage( + "offline run requires a `script` of agent candidates; the manifest provided none" + .to_string(), + )); + } + if !scenario + .targets + .pinned + .iter() + .any(|t| scenario.script.contains_key(&t.id)) + { + return Err(CoinGymError::Usage( + "offline run requires at least one `script` candidate keyed to a pinned target; \ + the manifest's script keys do not match any pinned target id" + .to_string(), + )); + } + Ok(()) +} + +/// Build the reasoner/executor/strategy and run. Split out so tests can drive it +/// directly without touching disk. +pub(crate) fn execute_run( + model: &str, + strategy: Strategy, + scenario: &DemoScenario, +) -> CoinGymResult { + let reasoner = FixtureReasoner::new(scenario.script.clone()); + let executor = MockHarnessExecutor::from_oracle(scenario.oracle.clone()); + let snapshot = scenario.targets.snapshot.clone(); + match strategy { + Strategy::Baseline => { + let s = BaselineStrategy::new(reasoner); + AgentRunner::new(&s, &executor, model, snapshot).run(&scenario.targets.pinned) + } + Strategy::Team => { + let s = TeamStrategy::new(reasoner); + AgentRunner::new(&s, &executor, model, snapshot).run(&scenario.targets.pinned) + } + } +} + +// ── score ──────────────────────────────────────────────────────────────────── + +fn cmd_score(home: &Path, rest: &[String]) -> CoinGymResult<()> { + let parsed = parse_args(rest, &["profile"])?; + let run_id = require_run_id(&parsed, "score")?; + let profile = sanitized_profile(&parsed); + let persisted = load_run(home, profile.as_deref(), &run_id)?; + let score = score_run(&persisted.report); + println!("run-id: {run_id}"); + print_offline_note(persisted.report.offline_scaffold); + print_score(&score); + Ok(()) +} + +// ── compare ────────────────────────────────────────────────────────────────── + +fn cmd_compare(home: &Path, rest: &[String]) -> CoinGymResult<()> { + let parsed = parse_args(rest, &["profile"])?; + let run_id = require_run_id(&parsed, "compare")?; + let profile = sanitized_profile(&parsed); + let persisted = load_run(home, profile.as_deref(), &run_id)?; + let score = score_run(&persisted.report); + println!("run-id: {run_id}"); + println!("model: {}", score.model); + match compare_to_leaderboard(&score) { + Some(cmp) => { + println!( + "reach: local {:.1}% vs published {:.1}% (Δ {:+.1} pts)", + cmp.local_reach_pct, cmp.published_reach_pct, cmp.reach_delta_pct + ); + println!( + "precision: local {:.1}% vs published {:.1}% (Δ {:+.1} pts)", + cmp.local_precision_pct, cmp.published_precision_pct, cmp.precision_delta_pct + ); + println!("published: {}", cmp.published_model); + println!( + "material-deviation: {}", + if cmp.material_deviation { "YES" } else { "no" } + ); + println!("note: {}", cmp.note); + } + None => { + print_offline_note(persisted.report.offline_scaffold); + println!( + "note: model '{}' is not on the published COIN leaderboard; nothing to compare", + score.model + ); + } + } + Ok(()) +} + +// ── improve ────────────────────────────────────────────────────────────────── + +fn cmd_improve(home: &Path, rest: &[String]) -> CoinGymResult<()> { + let parsed = parse_args(rest, &["profile"])?; + let run_id = require_run_id(&parsed, "improve")?; + let profile = sanitized_profile(&parsed); + let persisted = load_run(home, profile.as_deref(), &run_id)?; + let report = analyze_and_review(&persisted.report, &persisted.targets); + println!("run-id: {run_id}"); + println!("analyzed: {} unreached target(s)", report.analyzed); + println!( + "accepted: {} rejected: {}", + report.accepted, report.rejected + ); + for reviewed in &report.proposals { + println!( + " [{}] {} — {}", + reviewed.verdict_label(), + reviewed.proposal.target_id, + reviewed.reason + ); + println!(" tactic: {}", reviewed.proposal.tactic); + } + println!("note: {}", report.note); + Ok(()) +} + +// ── profiles ───────────────────────────────────────────────────────────────── + +fn cmd_profiles(home: &Path, rest: &[String]) -> CoinGymResult<()> { + let _parsed = parse_args(rest, &[])?; + let profiles = list_profiles(home)?; + if profiles.is_empty() { + println!("no profiles under {}", home.display()); + return Ok(()); + } + println!("profiles under {}:", home.display()); + for p in profiles { + println!("- {} (model={})", p.name, p.model); + } + Ok(()) +} + +// ── helpers ────────────────────────────────────────────────────────────────── + +fn require_run_id(parsed: &ParsedArgs, cmd: &str) -> CoinGymResult { + let run_id = parsed + .positionals + .first() + .cloned() + .ok_or_else(|| CoinGymError::Usage(format!("{cmd}: expected ")))?; + // A run-id becomes a filename (`.json`); reject path separators and + // parent traversal so it can never escape the profile's runs directory. + if run_id.contains('/') || run_id.contains('\\') || run_id.contains("..") { + return Err(CoinGymError::Usage(format!( + "invalid run-id '{run_id}': must not contain path separators or '..'" + ))); + } + Ok(run_id) +} + +/// Sanitised `--profile` value (directory-safe), or `None` when the flag is +/// absent (⇒ search all profiles). +fn sanitized_profile(parsed: &ParsedArgs) -> Option { + parsed + .flags + .get("profile") + .map(|p| profiles::sanitize_name(p)) +} + +fn print_offline_note(offline_scaffold: bool) { + if offline_scaffold { + println!( + "note: OFFLINE SCAFFOLD (mock oracle) — not a real coin evaluate grade (Phase 3)" + ); + } +} + +fn print_score(score: &Score) { + println!( + "reach: {:.1}% ({}/{}) precision: {:.1}% ({}/{})", + score.overall.reach_pct(), + score.overall.reached, + score.overall.total, + score.overall.precision_pct(), + score.overall.reached, + score.overall.submitted, + ); + for fs in &score.by_family { + println!( + " {:<22} reach {:.1}% precision {:.1}% (n={})", + fs.family.label(), + fs.score.reach_pct(), + fs.score.precision_pct(), + fs.score.total, + ); + } + println!(" histogram: {}", score.histogram.render()); +} diff --git a/src/coin_gym/profiles.rs b/src/coin_gym/profiles.rs new file mode 100644 index 00000000..1efbf096 --- /dev/null +++ b/src/coin_gym/profiles.rs @@ -0,0 +1,215 @@ +//! Profiles + run persistence (research doc Part 3.3, component 7). +//! +//! A **profile** is isolated per-model run state so baseline-vs-team and +//! model-vs-model comparisons are reproducible and never cross-contaminate. On +//! disk: +//! +//! ```text +//! /profiles//profile.json # metadata +//! /profiles//runs/.json # persisted runs (report + targets) +//! ``` +//! +//! `` defaults to `target/coin-gym` (relative, like the existing gym) and +//! can be overridden with the `COIN_GYM_HOME` env var. A run is stored together +//! with the target set it was evaluated against so `score`/`compare`/`improve` +//! can reload full context offline. + +use std::path::{Path, PathBuf}; +use std::time::{SystemTime, UNIX_EPOCH}; + +use serde::{Deserialize, Serialize}; + +use super::target_loader::TargetSet; +use super::types::{CoinGymError, CoinGymResult, RunReport}; + +/// Default (relative) home when `COIN_GYM_HOME` is unset. +pub const DEFAULT_HOME: &str = "target/coin-gym"; + +/// Resolve the COIN Gym home directory (`COIN_GYM_HOME` or the default). +#[must_use] +pub fn default_home() -> PathBuf { + std::env::var_os("COIN_GYM_HOME").map_or_else(|| PathBuf::from(DEFAULT_HOME), PathBuf::from) +} + +/// On-disk profile metadata. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct Profile { + /// Profile name (directory-safe). + pub name: String, + /// Model this profile isolates. + pub model: String, + /// Creation time (unix epoch milliseconds). + pub created_at_unix_ms: u128, +} + +/// A run persisted with the target set it was evaluated against. +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +pub struct PersistedRun { + /// The run report. + pub report: RunReport, + /// The target set the run was evaluated against. + pub targets: TargetSet, +} + +/// Directory for a named profile under `home`. +#[must_use] +pub fn profile_dir(home: &Path, name: &str) -> PathBuf { + home.join("profiles").join(name) +} + +/// Directory holding a profile's persisted runs. +#[must_use] +pub fn runs_dir(home: &Path, name: &str) -> PathBuf { + profile_dir(home, name).join("runs") +} + +/// Sanitise an arbitrary string into a directory-safe profile name. +#[must_use] +pub fn sanitize_name(name: &str) -> String { + let cleaned: String = name + .chars() + .map(|c| { + if c.is_ascii_alphanumeric() || c == '-' || c == '_' { + c + } else { + '-' + } + }) + .collect(); + if cleaned.is_empty() { + "default".to_string() + } else { + cleaned + } +} + +/// Create (or load) a profile for `model` under `home`, persisting its metadata. +/// +/// Enforces per-model isolation: reusing an existing profile name with a +/// *different* model is rejected so baseline-vs-team and model-vs-model runs +/// never cross-contaminate a profile. +/// +/// # Errors +/// Returns [`CoinGymError::Usage`] if the profile already exists for a different +/// model, or [`CoinGymError::Io`] on directory-creation or write failure. +pub fn ensure_profile(home: &Path, name: &str, model: &str) -> CoinGymResult { + let dir = profile_dir(home, name); + std::fs::create_dir_all(runs_dir(home, name)) + .map_err(|e| CoinGymError::Io(format!("create {}: {e}", dir.display())))?; + let meta_path = dir.join("profile.json"); + if let Ok(existing) = load_profile_meta(&meta_path) { + if existing.model != model { + return Err(CoinGymError::Usage(format!( + "profile '{name}' is bound to model '{}'; refusing to reuse it for '{model}' \ + (use a distinct --profile per model)", + existing.model + ))); + } + return Ok(existing); + } + let profile = Profile { + name: name.to_string(), + model: model.to_string(), + created_at_unix_ms: now_unix_ms(), + }; + write_json(&meta_path, &profile)?; + Ok(profile) +} + +fn load_profile_meta(path: &Path) -> CoinGymResult { + let raw = std::fs::read_to_string(path) + .map_err(|e| CoinGymError::Io(format!("read {}: {e}", path.display())))?; + serde_json::from_str(&raw).map_err(|e| CoinGymError::Parse(format!("profile.json: {e}"))) +} + +/// List all profiles under `home` (sorted by name). Missing home ⇒ empty list. +/// +/// # Errors +/// Returns [`CoinGymError::Io`] if the profiles directory exists but cannot be +/// read. +pub fn list_profiles(home: &Path) -> CoinGymResult> { + let profiles_root = home.join("profiles"); + if !profiles_root.exists() { + return Ok(Vec::new()); + } + let entries = std::fs::read_dir(&profiles_root) + .map_err(|e| CoinGymError::Io(format!("read {}: {e}", profiles_root.display())))?; + let mut profiles = Vec::new(); + for entry in entries { + let entry = entry.map_err(|e| CoinGymError::Io(format!("dir entry: {e}")))?; + let meta = entry.path().join("profile.json"); + if let Ok(profile) = load_profile_meta(&meta) { + profiles.push(profile); + } + } + profiles.sort_by(|a, b| a.name.cmp(&b.name)); + Ok(profiles) +} + +/// Persist a run (report + targets) under a profile. Returns the file path. +/// +/// Refuses to overwrite an existing run file so a run-id collision surfaces +/// loudly instead of silently destroying a prior run. +/// +/// # Errors +/// Returns [`CoinGymError::Io`] on write failure or if the run file already +/// exists. +pub fn save_run(home: &Path, profile: &str, run: &PersistedRun) -> CoinGymResult { + let dir = runs_dir(home, profile); + std::fs::create_dir_all(&dir) + .map_err(|e| CoinGymError::Io(format!("create {}: {e}", dir.display())))?; + let path = dir.join(format!("{}.json", run.report.run_id)); + if path.exists() { + return Err(CoinGymError::Io(format!( + "run '{}' already exists at {}", + run.report.run_id, + path.display() + ))); + } + write_json(&path, run)?; + Ok(path) +} + +/// Load a persisted run by id. If `profile` is `Some`, look only there; if +/// `None`, search every profile's runs directory. +/// +/// # Errors +/// Returns [`CoinGymError::NotFound`] if no matching run exists, or +/// [`CoinGymError`] on read/parse failure. +pub fn load_run(home: &Path, profile: Option<&str>, run_id: &str) -> CoinGymResult { + if let Some(name) = profile { + let path = runs_dir(home, name).join(format!("{run_id}.json")); + return read_persisted_run(&path); + } + // Search all profiles. + for p in list_profiles(home)? { + let path = runs_dir(home, &p.name).join(format!("{run_id}.json")); + if path.is_file() { + return read_persisted_run(&path); + } + } + Err(CoinGymError::NotFound(format!( + "run '{run_id}' not found under {}", + home.display() + ))) +} + +fn read_persisted_run(path: &Path) -> CoinGymResult { + let raw = std::fs::read_to_string(path) + .map_err(|_| CoinGymError::NotFound(format!("run file {}", path.display())))?; + serde_json::from_str(&raw).map_err(|e| CoinGymError::Parse(format!("persisted run: {e}"))) +} + +fn write_json(path: &Path, value: &T) -> CoinGymResult<()> { + let body = serde_json::to_string_pretty(value) + .map_err(|e| CoinGymError::Parse(format!("serialize {}: {e}", path.display())))?; + std::fs::write(path, body) + .map_err(|e| CoinGymError::Io(format!("write {}: {e}", path.display()))) +} + +fn now_unix_ms() -> u128 { + SystemTime::now() + .duration_since(UNIX_EPOCH) + .map(|d| d.as_millis()) + .unwrap_or(0) +} diff --git a/src/coin_gym/scorer.rs b/src/coin_gym/scorer.rs new file mode 100644 index 00000000..9ac85e17 --- /dev/null +++ b/src/coin_gym/scorer.rs @@ -0,0 +1,193 @@ +//! Scorer (research doc Part 3.3, component 4). +//! +//! Computes COIN's two headline metrics — **reach rate** and **precision** — +//! overall and split by family (frontier vs. non-trivial reachable), plus the +//! `R/W/A/T/N/E` outcome histogram. +//! +//! - **reach rate** = reached / total targets. +//! - **precision** = reached / *submitted* inputs (abstain and no-submission do +//! not count toward the denominator). This exposes over-claiming. + +use serde::Serialize; + +use super::types::{Outcome, OutcomeCode, RunReport, TargetFamily}; + +/// Count of each `R/W/A/T/N/E` outcome. +#[derive(Clone, Copy, Debug, Default, Eq, PartialEq, Serialize)] +pub struct OutcomeHistogram { + /// `R` — reached. + pub reached: usize, + /// `W` — wrong input submitted. + pub wrong_input: usize, + /// `A` — abstained. + pub abstained: usize, + /// `T` — timed out. + pub timed_out: usize, + /// `N` — no submission. + pub no_submission: usize, + /// `E` — error. + pub error: usize, +} + +impl OutcomeHistogram { + /// Tally outcomes into a histogram. + #[must_use] + pub fn tally<'a>(outcomes: impl IntoIterator) -> Self { + let mut h = Self::default(); + for o in outcomes { + match o.code { + OutcomeCode::Reached => h.reached += 1, + OutcomeCode::WrongInput => h.wrong_input += 1, + OutcomeCode::Abstained => h.abstained += 1, + OutcomeCode::TimedOut => h.timed_out += 1, + OutcomeCode::NoSubmission => h.no_submission += 1, + OutcomeCode::Error => h.error += 1, + } + } + h + } + + /// Total across all outcome codes. + #[must_use] + pub fn total(&self) -> usize { + self.reached + + self.wrong_input + + self.abstained + + self.timed_out + + self.no_submission + + self.error + } + + /// Compact `R:_/W:_/A:_/T:_/N:_/E:_` rendering. + #[must_use] + pub fn render(&self) -> String { + format!( + "R:{}/W:{}/A:{}/T:{}/N:{}/E:{}", + self.reached, + self.wrong_input, + self.abstained, + self.timed_out, + self.no_submission, + self.error + ) + } +} + +/// Reach/precision for one family (or overall). +#[derive(Clone, Copy, Debug, PartialEq, Serialize)] +pub struct ReachPrecision { + /// Targets reached. + pub reached: usize, + /// Inputs submitted (precision denominator). + pub submitted: usize, + /// Total targets in scope. + pub total: usize, + /// reached / total (0.0 when total == 0). + pub reach_rate: f64, + /// reached / submitted (0.0 when submitted == 0). + pub precision: f64, +} + +impl ReachPrecision { + /// Compute reach/precision over a set of outcomes. + #[must_use] + pub fn compute<'a>(outcomes: impl IntoIterator) -> Self { + let mut reached = 0usize; + let mut submitted = 0usize; + let mut total = 0usize; + for o in outcomes { + total += 1; + if o.reached() { + reached += 1; + } + if o.submitted() { + submitted += 1; + } + } + Self { + reached, + submitted, + total, + reach_rate: ratio(reached, total), + precision: ratio(reached, submitted), + } + } + + /// reach/precision as percentages, for leaderboard comparison. + #[must_use] + pub fn reach_pct(&self) -> f64 { + self.reach_rate * 100.0 + } + + /// Precision as a percentage. + #[must_use] + pub fn precision_pct(&self) -> f64 { + self.precision * 100.0 + } +} + +/// Reach/precision for a single family. +#[derive(Clone, Copy, Debug, PartialEq, Serialize)] +pub struct FamilyScore { + /// The family scored. + pub family: TargetFamily, + /// Reach/precision within the family. + pub score: ReachPrecision, +} + +/// The full score for a run. +#[derive(Clone, Debug, PartialEq, Serialize)] +pub struct Score { + /// The run this score is for. + pub run_id: String, + /// The model under test. + pub model: String, + /// Overall reach/precision. + pub overall: ReachPrecision, + /// Per-family reach/precision (frontier first, then non-trivial reachable). + pub by_family: Vec, + /// `R/W/A/T/N/E` histogram. + pub histogram: OutcomeHistogram, + /// Whether the run used an offline mock oracle. + pub offline_scaffold: bool, +} + +/// Score a run: overall + per-family reach/precision + outcome histogram. +#[must_use] +pub fn score_run(report: &RunReport) -> Score { + let overall = ReachPrecision::compute(&report.outcomes); + let by_family = [TargetFamily::Frontier, TargetFamily::NonTrivialReachable] + .into_iter() + .filter_map(|family| { + let members: Vec<&Outcome> = report + .outcomes + .iter() + .filter(|o| o.family == family) + .collect(); + if members.is_empty() { + None + } else { + Some(FamilyScore { + family, + score: ReachPrecision::compute(members), + }) + } + }) + .collect(); + Score { + run_id: report.run_id.clone(), + model: report.model.clone(), + overall, + by_family, + histogram: OutcomeHistogram::tally(&report.outcomes), + offline_scaffold: report.offline_scaffold, + } +} + +fn ratio(numerator: usize, denominator: usize) -> f64 { + if denominator == 0 { + 0.0 + } else { + numerator as f64 / denominator as f64 + } +} diff --git a/src/coin_gym/target_loader.rs b/src/coin_gym/target_loader.rs new file mode 100644 index 00000000..10e95c0f --- /dev/null +++ b/src/coin_gym/target_loader.rs @@ -0,0 +1,210 @@ +//! Target loader (research doc Part 3.3, component 1). +//! +//! Pulls a COIN snapshot's targets and reserves a **held-out fresh** slice for +//! the anti-overfit verification gate. Real snapshots are published Docker/HF +//! artifacts pulled on a VM (Phase 3); this loader deliberately targets a JSON +//! manifest so the whole pipeline runs offline in tests and CI. A bundled sample +//! manifest ships with the crate. + +use std::collections::HashMap; +use std::path::{Path, PathBuf}; + +use serde::{Deserialize, Serialize}; + +use super::agent_runner::Candidate; +use super::types::{CoinGymError, CoinGymResult, Target}; + +/// The bundled sample snapshot manifest, embedded at compile time so tests and +/// the offline demo `run` never depend on a checked-out path. +pub const SAMPLE_SNAPSHOT_JSON: &str = include_str!("fixtures/sample_snapshot.json"); + +/// A loaded target set: the pinned evaluation slice plus a reserved held-out +/// fresh slice used only for verification (research doc Part 3.6). +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct TargetSet { + /// Snapshot identifier the targets came from. + pub snapshot: String, + /// The pinned evaluation targets. + pub pinned: Vec, + /// Held-out fresh targets reserved for the overfitting-verification gate. + pub held_out_fresh: Vec, +} + +impl TargetSet { + /// Total number of targets across both slices. + #[must_use] + pub fn total(&self) -> usize { + self.pinned.len() + self.held_out_fresh.len() + } +} + +/// A source of COIN targets. Implementors resolve a snapshot into a +/// [`TargetSet`]. The trait keeps the real (VM-backed) snapshot puller and the +/// offline fixture loader interchangeable. +pub trait TargetSource { + /// Load the target set. + /// + /// # Errors + /// Returns [`CoinGymError`] if the underlying manifest cannot be read or + /// parsed. + fn load(&self) -> CoinGymResult; +} + +// ── Fixture-backed source ───────────────────────────────────────────────────── + +#[derive(Debug, Deserialize)] +struct SnapshotManifest { + snapshot: String, + targets: ManifestTargets, + #[serde(default)] + oracle: HashMap, + #[serde(default)] + script: HashMap, +} + +#[derive(Debug, Deserialize)] +struct ManifestTargets { + #[serde(default)] + pinned: Vec, + #[serde(default)] + held_out_fresh: Vec, +} + +#[derive(Debug, Deserialize)] +struct ScriptEntry { + input: String, + confidence: f64, + #[serde(default)] + rationale: String, +} + +fn parse_manifest(raw: &str) -> CoinGymResult { + serde_json::from_str(raw).map_err(|e| CoinGymError::Parse(format!("snapshot manifest: {e}"))) +} + +/// Loads targets from a JSON snapshot manifest on disk. +#[derive(Clone, Debug)] +pub struct FixtureTargetSource { + path: PathBuf, +} + +impl FixtureTargetSource { + /// Create a source that reads the manifest at `path`. + #[must_use] + pub fn new(path: impl Into) -> Self { + Self { path: path.into() } + } +} + +impl TargetSource for FixtureTargetSource { + fn load(&self) -> CoinGymResult { + let raw = std::fs::read_to_string(&self.path) + .map_err(|e| CoinGymError::Io(format!("read {}: {e}", self.path.display())))?; + let manifest = parse_manifest(&raw)?; + Ok(TargetSet { + snapshot: manifest.snapshot, + pinned: manifest.targets.pinned, + held_out_fresh: manifest.targets.held_out_fresh, + }) + } +} + +/// An in-memory source over a raw JSON manifest string. Used by the embedded +/// sample and by tests. +#[derive(Clone, Debug)] +pub struct InMemoryTargetSource { + raw: String, +} + +impl InMemoryTargetSource { + /// Create a source over an owned JSON manifest string. + #[must_use] + pub fn new(raw: impl Into) -> Self { + Self { raw: raw.into() } + } + + /// A source over the bundled sample snapshot. + #[must_use] + pub fn sample() -> Self { + Self::new(SAMPLE_SNAPSHOT_JSON) + } +} + +impl TargetSource for InMemoryTargetSource { + fn load(&self) -> CoinGymResult { + let manifest = parse_manifest(&self.raw)?; + Ok(TargetSet { + snapshot: manifest.snapshot, + pinned: manifest.targets.pinned, + held_out_fresh: manifest.targets.held_out_fresh, + }) + } +} + +/// A fully-parsed demo scenario: the target set plus the offline `oracle` +/// (ground-truth reaching inputs, consumed by the mock executor) and the +/// `script` (the scaffold agent's proposed candidates). These auxiliary maps +/// exist ONLY to exercise the pipeline offline — a real run gets its oracle from +/// `coin evaluate` and its candidates from a live model. +#[derive(Clone, Debug)] +pub struct DemoScenario { + /// The target set. + pub targets: TargetSet, + /// Ground-truth reaching input per target id (mock-oracle truth). + pub oracle: HashMap, + /// Scripted agent candidate per target id. + pub script: HashMap, +} + +impl DemoScenario { + /// Parse a full demo scenario (targets + oracle + script) from a JSON + /// manifest string. + /// + /// # Errors + /// Returns [`CoinGymError::Parse`] on malformed JSON. + pub fn from_manifest(raw: &str) -> CoinGymResult { + let manifest = parse_manifest(raw)?; + let script = manifest + .script + .into_iter() + .map(|(id, entry)| { + ( + id, + Candidate { + input: entry.input, + confidence: entry.confidence, + rationale: entry.rationale, + }, + ) + }) + .collect(); + Ok(Self { + targets: TargetSet { + snapshot: manifest.snapshot, + pinned: manifest.targets.pinned, + held_out_fresh: manifest.targets.held_out_fresh, + }, + oracle: manifest.oracle, + script, + }) + } + + /// The bundled sample demo scenario. + /// + /// # Errors + /// Returns [`CoinGymError::Parse`] if the embedded fixture is malformed + /// (which would be a build-time bug caught by tests). + pub fn sample() -> CoinGymResult { + Self::from_manifest(SAMPLE_SNAPSHOT_JSON) + } + + /// Load a demo scenario from a JSON manifest file. + /// + /// # Errors + /// Returns [`CoinGymError`] on read or parse failure. + pub fn from_path(path: &Path) -> CoinGymResult { + let raw = std::fs::read_to_string(path) + .map_err(|e| CoinGymError::Io(format!("read {}: {e}", path.display())))?; + Self::from_manifest(&raw) + } +} diff --git a/src/coin_gym/tests_agent_runner.rs b/src/coin_gym/tests_agent_runner.rs new file mode 100644 index 00000000..9fd20674 --- /dev/null +++ b/src/coin_gym/tests_agent_runner.rs @@ -0,0 +1,184 @@ +use std::collections::HashMap; + +use super::agent_runner::{ + AgentRunner, AgentStrategy, BaselineStrategy, Candidate, DEFAULT_THRESHOLD_HINT, + FixtureReasoner, Reasoner, SubmissionDecision, TeamStrategy, +}; +use super::executor::MockHarnessExecutor; +use super::scorer::score_run; +use super::types::{OutcomeCode, Strategy, Target, TargetFamily}; + +fn t(id: &str, family: TargetFamily) -> Target { + Target { + id: id.to_string(), + project: "proj".to_string(), + commit: "c".to_string(), + harness: "h".to_string(), + file: "src/x.c".to_string(), + line: 10, + family, + } +} + +fn cand(input: &str, confidence: f64) -> Candidate { + Candidate { + input: input.to_string(), + confidence, + rationale: "because".to_string(), + } +} + +/// Four targets that make the baseline-vs-team precision trade-off explicit: +/// two high-confidence correct (both submit → R), two low-confidence wrong +/// (baseline submits → W; team abstains → A). +fn ab_scenario() -> (Vec, FixtureReasoner, MockHarnessExecutor) { + let targets = vec![ + t("hit-a", TargetFamily::Frontier), + t("wrong-lowconf-a", TargetFamily::Frontier), + t("wrong-lowconf-b", TargetFamily::NonTrivialReachable), + t("hit-b", TargetFamily::NonTrivialReachable), + ]; + let mut script = HashMap::new(); + script.insert("hit-a".to_string(), cand("correct-a", 0.9)); + script.insert("wrong-lowconf-a".to_string(), cand("guess-a", 0.3)); + script.insert("wrong-lowconf-b".to_string(), cand("guess-b", 0.4)); + script.insert("hit-b".to_string(), cand("correct-b", 0.8)); + let reasoner = FixtureReasoner::new(script); + + let oracle: HashMap = [ + ("hit-a", "correct-a"), + ("wrong-lowconf-a", "actual-a"), + ("wrong-lowconf-b", "actual-b"), + ("hit-b", "correct-b"), + ] + .iter() + .map(|(k, v)| ((*k).to_string(), (*v).to_string())) + .collect(); + let executor = MockHarnessExecutor::from_oracle(oracle); + (targets, reasoner, executor) +} + +fn approx(a: f64, b: f64) -> bool { + (a - b).abs() < 1e-9 +} + +#[test] +fn baseline_submits_everything() { + let (targets, reasoner, executor) = ab_scenario(); + let strategy = BaselineStrategy::new(reasoner); + let report = AgentRunner::new(&strategy, &executor, "m", "snap") + .run(&targets) + .unwrap(); + assert_eq!(report.strategy, Strategy::Baseline); + let codes: Vec = report.outcomes.iter().map(|o| o.code).collect(); + assert_eq!( + codes, + vec![ + OutcomeCode::Reached, + OutcomeCode::WrongInput, + OutcomeCode::WrongInput, + OutcomeCode::Reached, + ] + ); + let score = score_run(&report); + assert!(approx(score.overall.reach_rate, 0.5)); + assert!(approx(score.overall.precision, 0.5)); // 2 reached / 4 submitted +} + +#[test] +fn team_abstains_on_low_confidence_and_lifts_precision() { + let (targets, reasoner, executor) = ab_scenario(); + let strategy = TeamStrategy::new(reasoner); + assert!(approx(strategy.threshold_hint(), DEFAULT_THRESHOLD_HINT)); + let report = AgentRunner::new(&strategy, &executor, "m", "snap") + .run(&targets) + .unwrap(); + assert_eq!(report.strategy, Strategy::Team); + let codes: Vec = report.outcomes.iter().map(|o| o.code).collect(); + assert_eq!( + codes, + vec![ + OutcomeCode::Reached, + OutcomeCode::Abstained, + OutcomeCode::Abstained, + OutcomeCode::Reached, + ] + ); + let score = score_run(&report); + // Same reach as baseline, but precision rises to 1.0 (no wrong submissions). + assert!(approx(score.overall.reach_rate, 0.5)); + assert!(approx(score.overall.precision, 1.0)); +} + +#[test] +fn run_ids_are_unique_across_repeated_runs() { + let (targets, reasoner, executor) = ab_scenario(); + let strategy = BaselineStrategy::new(reasoner); + let r1 = AgentRunner::new(&strategy, &executor, "m", "snap") + .run(&targets) + .unwrap(); + let r2 = AgentRunner::new(&strategy, &executor, "m", "snap") + .run(&targets) + .unwrap(); + assert_ne!(r1.run_id, r2.run_id, "run ids must not collide"); +} + +#[test] +fn no_candidate_yields_no_submission() { + let targets = vec![t("orphan", TargetFamily::Frontier)]; + let reasoner = FixtureReasoner::default(); // empty script + let executor = MockHarnessExecutor::new(); + let strategy = BaselineStrategy::new(reasoner); + let report = AgentRunner::new(&strategy, &executor, "m", "snap") + .run(&targets) + .unwrap(); + assert_eq!(report.outcomes[0].code, OutcomeCode::NoSubmission); +} + +#[test] +fn team_threshold_is_clamped() { + let reasoner = FixtureReasoner::default(); + let hi = TeamStrategy::with_threshold(reasoner.clone(), 5.0); + assert!(approx(hi.threshold_hint(), 1.0)); + let lo = TeamStrategy::with_threshold(reasoner, -1.0); + assert!(approx(lo.threshold_hint(), 0.0)); +} + +#[test] +fn team_evaluate_produces_expected_decisions() { + let mut script = HashMap::new(); + script.insert("go".to_string(), cand("x", 0.75)); + script.insert("stop".to_string(), cand("y", 0.2)); + let reasoner = FixtureReasoner::new(script); + let strategy = TeamStrategy::new(reasoner); + + let submit = strategy.evaluate(&t("go", TargetFamily::Frontier)); + assert!(matches!(submit.decision, SubmissionDecision::Submit { .. })); + + let abstain = strategy.evaluate(&t("stop", TargetFamily::Frontier)); + assert!(matches!( + abstain.decision, + SubmissionDecision::Abstain { .. } + )); + + let none = strategy.evaluate(&t("missing", TargetFamily::Frontier)); + assert!(matches!(none.decision, SubmissionDecision::NoSubmission)); +} + +#[test] +fn custom_reasoner_assess_can_override_confidence() { + // A skeptic that always distrusts: assess returns 0.0 → team always abstains. + #[derive(Clone)] + struct Skeptic; + impl Reasoner for Skeptic { + fn propose(&self, _target: &Target) -> Option { + Some(cand("anything", 0.99)) + } + fn assess(&self, _target: &Target, _candidate: &Candidate) -> f64 { + 0.0 + } + } + let strategy = TeamStrategy::new(Skeptic); + let out = strategy.evaluate(&t("x", TargetFamily::Frontier)); + assert!(matches!(out.decision, SubmissionDecision::Abstain { .. })); +} diff --git a/src/coin_gym/tests_cli.rs b/src/coin_gym/tests_cli.rs new file mode 100644 index 00000000..c6d97e75 --- /dev/null +++ b/src/coin_gym/tests_cli.rs @@ -0,0 +1,249 @@ +use super::profiles::{PersistedRun, ensure_profile, runs_dir, save_run}; +use super::target_loader::DemoScenario; +use super::types::Strategy; +use super::{coin_gym_usage, dispatch_with_home, execute_run}; + +fn args(list: &[&str]) -> Vec { + list.iter().map(|s| (*s).to_string()).collect() +} + +fn approx(a: f64, b: f64) -> bool { + (a - b).abs() < 1e-9 +} + +#[test] +fn execute_run_baseline_vs_team_shows_precision_tradeoff() { + let scenario = DemoScenario::sample().unwrap(); + let baseline = execute_run("claude-opus-4.6", Strategy::Baseline, &scenario).unwrap(); + let team = execute_run("claude-opus-4.6", Strategy::Team, &scenario).unwrap(); + + let bscore = super::scorer::score_run(&baseline); + let tscore = super::scorer::score_run(&team); + + // Same reach on identical targets, but the team's abstention gate lifts + // precision by declining low-confidence (wrong) submissions. + assert!(approx(bscore.overall.reach_rate, 0.6)); + assert!(approx(bscore.overall.precision, 0.6)); + assert!(approx(tscore.overall.reach_rate, 0.6)); + assert!(approx(tscore.overall.precision, 1.0)); + assert!(tscore.overall.precision > bscore.overall.precision); + + assert!(baseline.offline_scaffold); +} + +#[test] +fn run_command_creates_profile_and_run_file() { + let dir = tempfile::tempdir().unwrap(); + let home = dir.path(); + dispatch_with_home( + home, + args(&[ + "run", + "claude-opus-4.6", + "--strategy", + "team", + "--profile", + "opus", + ]), + ) + .unwrap(); + + let runs = runs_dir(home, "opus"); + let entries: Vec<_> = std::fs::read_dir(&runs) + .unwrap() + .filter_map(Result::ok) + .filter(|e| e.path().extension().is_some_and(|x| x == "json")) + .collect(); + assert_eq!(entries.len(), 1, "exactly one run file should be written"); +} + +#[test] +fn score_compare_improve_load_a_saved_run() { + let dir = tempfile::tempdir().unwrap(); + let home = dir.path(); + let scenario = DemoScenario::sample().unwrap(); + let report = execute_run("claude-opus-4.6", Strategy::Baseline, &scenario).unwrap(); + ensure_profile(home, "opus", "claude-opus-4.6").unwrap(); + save_run( + home, + "opus", + &PersistedRun { + report: report.clone(), + targets: scenario.targets.clone(), + }, + ) + .unwrap(); + + dispatch_with_home(home, args(&["score", &report.run_id, "--profile", "opus"])).unwrap(); + dispatch_with_home( + home, + args(&["compare", &report.run_id, "--profile", "opus"]), + ) + .unwrap(); + dispatch_with_home( + home, + args(&["improve", &report.run_id, "--profile", "opus"]), + ) + .unwrap(); +} + +#[test] +fn profiles_command_runs_on_empty_and_populated_home() { + let dir = tempfile::tempdir().unwrap(); + let home = dir.path(); + // empty + dispatch_with_home(home, args(&["profiles"])).unwrap(); + // populated + ensure_profile(home, "opus", "claude-opus-4.6").unwrap(); + dispatch_with_home(home, args(&["profiles"])).unwrap(); +} + +#[test] +fn unknown_command_errors_with_usage() { + let dir = tempfile::tempdir().unwrap(); + let err = dispatch_with_home(dir.path(), args(&["frobnicate"])).unwrap_err(); + assert!(err.to_string().contains("unknown command")); +} + +#[test] +fn empty_args_error() { + let dir = tempfile::tempdir().unwrap(); + let err = dispatch_with_home(dir.path(), Vec::::new()).unwrap_err(); + assert!(err.to_string().contains("usage")); +} + +#[test] +fn run_requires_model() { + let dir = tempfile::tempdir().unwrap(); + let err = dispatch_with_home(dir.path(), args(&["run"])).unwrap_err(); + assert!(err.to_string().contains("expected ")); +} + +#[test] +fn unknown_flag_is_rejected() { + let dir = tempfile::tempdir().unwrap(); + let err = dispatch_with_home(dir.path(), args(&["run", "m", "--bogus", "x"])).unwrap_err(); + assert!(err.to_string().contains("unknown flag")); +} + +#[test] +fn invalid_strategy_is_rejected() { + let dir = tempfile::tempdir().unwrap(); + let err = + dispatch_with_home(dir.path(), args(&["run", "m", "--strategy", "solo"])).unwrap_err(); + assert!(err.to_string().contains("unknown strategy")); +} + +#[test] +fn score_requires_run_id() { + let dir = tempfile::tempdir().unwrap(); + let err = dispatch_with_home(dir.path(), args(&["score"])).unwrap_err(); + assert!(err.to_string().contains("expected ")); +} + +#[test] +fn run_id_with_path_traversal_is_rejected() { + let dir = tempfile::tempdir().unwrap(); + for bad in ["../secret", "a/b", "..\\win", ".."] { + let err = dispatch_with_home(dir.path(), args(&["score", bad])).unwrap_err(); + assert!( + err.to_string().contains("invalid run-id"), + "should reject traversal run-id: {bad}" + ); + } +} + +#[test] +fn explicit_profile_name_is_sanitised() { + // A traversal-y --profile must be confined under /profiles, not escape it. + let dir = tempfile::tempdir().unwrap(); + let home = dir.path(); + dispatch_with_home( + home, + args(&[ + "run", + "m", + "--strategy", + "baseline", + "--profile", + "../escape", + ]), + ) + .unwrap(); + // No directory was created outside the home tree. + assert!(!home.parent().unwrap().join("escape").exists()); + // The sanitised profile lives under home/profiles. + let safe = super::profiles::sanitize_name("../escape"); + assert!(!safe.contains('/') && !safe.contains("..")); + assert!(home.join("profiles").join(&safe).exists()); +} + +#[test] +fn usage_lists_all_subcommands() { + let usage = coin_gym_usage(); + for cmd in ["run", "score", "compare", "improve", "profiles"] { + assert!(usage.contains(cmd), "usage should mention {cmd}"); + } +} + +fn write_manifest(dir: &std::path::Path, name: &str, body: &str) -> String { + let path = dir.join(name); + std::fs::write(&path, body).unwrap(); + path.to_str().unwrap().to_string() +} + +#[test] +fn run_rejects_empty_pinned_manifest() { + let dir = tempfile::tempdir().unwrap(); + let manifest = write_manifest( + dir.path(), + "empty.json", + r#"{"snapshot":"s","targets":{"pinned":[],"held_out_fresh":[]}}"#, + ); + let err = + dispatch_with_home(dir.path(), args(&["run", "m", "--targets", &manifest])).unwrap_err(); + assert!(err.to_string().contains("no pinned targets")); +} + +#[test] +fn run_rejects_manifest_missing_oracle() { + let dir = tempfile::tempdir().unwrap(); + // Pinned target + script, but no oracle → grading would be undefined. + let manifest = write_manifest( + dir.path(), + "no_oracle.json", + r#"{"snapshot":"s","targets":{"pinned":[{"id":"t","project":"p","commit":"c","harness":"h","file":"f","line":1,"family":"frontier"}]},"script":{"t":{"input":"x","confidence":0.9}}}"#, + ); + let err = + dispatch_with_home(dir.path(), args(&["run", "m", "--targets", &manifest])).unwrap_err(); + assert!(err.to_string().contains("oracle")); +} + +#[test] +fn run_rejects_manifest_without_script() { + let dir = tempfile::tempdir().unwrap(); + // Pinned target + oracle, but no candidate script → all-N hollow run. + let manifest = write_manifest( + dir.path(), + "no_script.json", + r#"{"snapshot":"s","targets":{"pinned":[{"id":"t","project":"p","commit":"c","harness":"h","file":"f","line":1,"family":"frontier"}]},"oracle":{"t":"x"}}"#, + ); + let err = + dispatch_with_home(dir.path(), args(&["run", "m", "--targets", &manifest])).unwrap_err(); + assert!(err.to_string().contains("script")); +} + +#[test] +fn run_rejects_manifest_with_disjoint_script_keys() { + let dir = tempfile::tempdir().unwrap(); + // Script exists but is keyed to a non-pinned id → every pinned target would + // yield NoSubmission (a hollow all-N run). Must be rejected. + let manifest = write_manifest( + dir.path(), + "disjoint.json", + r#"{"snapshot":"s","targets":{"pinned":[{"id":"t1","project":"p","commit":"c","harness":"h","file":"f","line":1,"family":"frontier"}]},"oracle":{"t1":"x"},"script":{"not-t1":{"input":"y","confidence":0.9}}}"#, + ); + let err = + dispatch_with_home(dir.path(), args(&["run", "m", "--targets", &manifest])).unwrap_err(); + assert!(err.to_string().contains("pinned target")); +} diff --git a/src/coin_gym/tests_executor.rs b/src/coin_gym/tests_executor.rs new file mode 100644 index 00000000..6fbe50b3 --- /dev/null +++ b/src/coin_gym/tests_executor.rs @@ -0,0 +1,107 @@ +use super::executor::{ + CoinEvaluateConfig, CoinEvaluateExecutor, GradeResult, HarnessExecutor, MockHarnessExecutor, +}; +use super::types::{OutcomeCode, Target, TargetFamily}; + +fn target(id: &str) -> Target { + Target { + id: id.to_string(), + project: "libraw".to_string(), + commit: "abc".to_string(), + harness: "libraw_raf_fuzzer".to_string(), + file: "src/metadata/fuji.cpp".to_string(), + line: 480, + family: TargetFamily::Frontier, + } +} + +#[test] +fn grade_result_maps_to_outcome_code() { + assert_eq!(GradeResult::Reached.to_outcome_code(), OutcomeCode::Reached); + assert_eq!( + GradeResult::WrongInput.to_outcome_code(), + OutcomeCode::WrongInput + ); + assert_eq!( + GradeResult::TimedOut.to_outcome_code(), + OutcomeCode::TimedOut + ); + assert_eq!(GradeResult::Error.to_outcome_code(), OutcomeCode::Error); +} + +#[test] +fn mock_reaches_only_on_matching_input() { + let exec = MockHarnessExecutor::new().with_reaching_input("t1", "magic-bytes"); + let t = target("t1"); + assert_eq!(exec.grade(&t, "magic-bytes").unwrap(), GradeResult::Reached); + assert_eq!(exec.grade(&t, "wrong").unwrap(), GradeResult::WrongInput); +} + +#[test] +fn mock_unknown_target_is_wrong_input() { + let exec = MockHarnessExecutor::new(); + assert_eq!( + exec.grade(&target("unknown"), "anything").unwrap(), + GradeResult::WrongInput + ); +} + +#[test] +fn mock_injected_timeout_and_error_take_precedence() { + let exec = MockHarnessExecutor::new() + .with_reaching_input("t1", "ok") + .with_timeout("t1") + .with_error("t2"); + // error dominates for t2 + assert_eq!(exec.grade(&target("t2"), "x").unwrap(), GradeResult::Error); + // timeout dominates even the correct input for t1 + assert_eq!( + exec.grade(&target("t1"), "ok").unwrap(), + GradeResult::TimedOut + ); +} + +#[test] +fn mock_is_flagged_offline_scaffold() { + assert!(MockHarnessExecutor::new().is_offline_scaffold()); +} + +#[test] +fn coin_evaluate_delegate_is_phase3_gated() { + let exec = CoinEvaluateExecutor::new(CoinEvaluateConfig::new("you/coin@v1")); + assert!(!exec.is_offline_scaffold()); + let err = exec.grade(&target("t1"), "bytes").unwrap_err(); + let msg = err.to_string(); + assert!(msg.contains("Docker")); + assert!(msg.contains("Phase 3")); +} + +#[test] +fn coin_evaluate_builds_delegation_argv() { + let exec = CoinEvaluateExecutor::new(CoinEvaluateConfig::new("you/coin@v1")); + let argv = exec.build_argv(&target("libraw-fuji-480"), "/tmp/input.bin"); + assert_eq!( + argv, + vec![ + "coin".to_string(), + "evaluate".to_string(), + "--dataset".to_string(), + "you/coin@v1".to_string(), + "--target".to_string(), + "libraw-fuji-480".to_string(), + "--input".to_string(), + "/tmp/input.bin".to_string(), + ] + ); +} + +#[test] +fn from_oracle_seeds_reaching_inputs() { + let mut oracle = std::collections::HashMap::new(); + oracle.insert("t1".to_string(), "seed".to_string()); + let exec = MockHarnessExecutor::from_oracle(oracle); + assert_eq!( + exec.grade(&target("t1"), "seed").unwrap(), + GradeResult::Reached + ); +} diff --git a/src/coin_gym/tests_improve.rs b/src/coin_gym/tests_improve.rs new file mode 100644 index 00000000..90c109a3 --- /dev/null +++ b/src/coin_gym/tests_improve.rs @@ -0,0 +1,176 @@ +use super::improve::{ + ReviewVerdict, TacticProposal, analyze_and_review, analyze_failures, review_proposal, +}; +use super::target_loader::TargetSet; +use super::types::{Outcome, OutcomeCode, RunReport, Strategy, Target, TargetFamily}; + +fn target(id: &str, project: &str, harness: &str, file: &str) -> Target { + Target { + id: id.to_string(), + project: project.to_string(), + commit: "c".to_string(), + harness: harness.to_string(), + file: file.to_string(), + line: 100, + family: TargetFamily::Frontier, + } +} + +fn target_set() -> TargetSet { + TargetSet { + snapshot: "snap".to_string(), + pinned: vec![ + target( + "libraw-fuji-480", + "libraw", + "libraw_raf_fuzzer", + "src/metadata/fuji.cpp", + ), + target("liboqs-kem-88", "liboqs", "kem_fuzzer", "src/kem/kem.c"), + target("widget-logic-12", "widget", "logic_fuzzer", "src/logic.c"), + target("reached-one", "libraw", "libraw_raf_fuzzer", "src/x.cpp"), + ], + held_out_fresh: Vec::new(), + } +} + +fn outcome(id: &str, code: OutcomeCode) -> Outcome { + Outcome { + target_id: id.to_string(), + family: TargetFamily::Frontier, + code, + cost_usd: 0.0, + } +} + +fn report_with(outcomes: Vec) -> RunReport { + RunReport { + run_id: "r1".to_string(), + model: "m".to_string(), + strategy: Strategy::Baseline, + snapshot: "snap".to_string(), + started_at_unix_ms: 0, + outcomes, + offline_scaffold: true, + } +} + +#[test] +fn analyze_only_proposes_for_unreached_failures() { + let report = report_with(vec![ + outcome("libraw-fuji-480", OutcomeCode::WrongInput), + outcome("liboqs-kem-88", OutcomeCode::TimedOut), + outcome("widget-logic-12", OutcomeCode::NoSubmission), + outcome("reached-one", OutcomeCode::Reached), // no proposal + ]); + let proposals = analyze_failures(&report, &target_set()); + assert_eq!(proposals.len(), 3); + let ids: Vec<&str> = proposals.iter().map(|p| p.target_id.as_str()).collect(); + assert!(ids.contains(&"libraw-fuji-480")); + assert!(!ids.contains(&"reached-one")); +} + +#[test] +fn abstain_is_not_treated_as_failure() { + let report = report_with(vec![outcome("libraw-fuji-480", OutcomeCode::Abstained)]); + assert!(analyze_failures(&report, &target_set()).is_empty()); +} + +#[test] +fn analyst_tactics_are_general_and_accepted_by_gate() { + let report = report_with(vec![ + outcome("libraw-fuji-480", OutcomeCode::WrongInput), + outcome("liboqs-kem-88", OutcomeCode::WrongInput), + outcome("widget-logic-12", OutcomeCode::WrongInput), + ]); + let set = target_set(); + let proposals = analyze_failures(&report, &set); + for p in &proposals { + let reviewed = review_proposal(p, &set); + assert_eq!( + reviewed.verdict, + ReviewVerdict::Accept, + "analyst tactic should generalise: {}", + p.tactic + ); + } + // Category-specific wording. + let decoder = proposals + .iter() + .find(|p| p.target_id == "libraw-fuji-480") + .unwrap(); + assert!(decoder.tactic.contains("format-gated decoders")); + let crypto = proposals + .iter() + .find(|p| p.target_id == "liboqs-kem-88") + .unwrap(); + assert!(crypto.tactic.contains("cryptographic state machines")); + let generic = proposals + .iter() + .find(|p| p.target_id == "widget-logic-12") + .unwrap(); + assert!(generic.tactic.contains("guarding predicate")); +} + +fn proposal(tactic: &str) -> TacticProposal { + TacticProposal { + id: "p".to_string(), + target_id: "libraw-fuji-480".to_string(), + tactic: tactic.to_string(), + evidence: "ev".to_string(), + } +} + +#[test] +fn gate_rejects_memorised_input_language() { + let set = target_set(); + for bad in [ + "Just hardcode the exact bytes that worked before.", + "Memorize this input and replay it.", + "Reuse the known input for the target.", + ] { + let reviewed = review_proposal(&proposal(bad), &set); + assert_eq!( + reviewed.verdict, + ReviewVerdict::Reject, + "should reject: {bad}" + ); + } +} + +#[test] +fn gate_rejects_target_specific_keys() { + let set = target_set(); + // Names a specific target id. + let by_id = proposal("For libraw-fuji-480 specifically, flip the branch."); + assert_eq!(review_proposal(&by_id, &set).verdict, ReviewVerdict::Reject); + + // Keys off a specific project. + let by_project = proposal("When fuzzing libraw, prefer RAF headers."); + assert_eq!( + review_proposal(&by_project, &set).verdict, + ReviewVerdict::Reject + ); + + // Names a specific locator. + let by_locator = proposal("Drive to libraw:src/metadata/fuji.cpp:100 directly."); + assert_eq!( + review_proposal(&by_locator, &set).verdict, + ReviewVerdict::Reject + ); +} + +#[test] +fn analyze_and_review_summarises_counts_and_phase5_note() { + let report = report_with(vec![ + outcome("libraw-fuji-480", OutcomeCode::WrongInput), + outcome("liboqs-kem-88", OutcomeCode::TimedOut), + ]); + let out = analyze_and_review(&report, &target_set()); + assert_eq!(out.analyzed, 2); + assert_eq!(out.accepted, 2); + assert_eq!(out.rejected, 0); + assert_eq!(out.proposals.len(), 2); + assert!(out.note.contains("Phase 5")); + assert_eq!(out.proposals[0].verdict_label(), "ACCEPT"); +} diff --git a/src/coin_gym/tests_leaderboard.rs b/src/coin_gym/tests_leaderboard.rs new file mode 100644 index 00000000..9476da3c --- /dev/null +++ b/src/coin_gym/tests_leaderboard.rs @@ -0,0 +1,86 @@ +use super::leaderboard::{ + MATERIAL_DEVIATION_PCT, compare_to_leaderboard, find_published, published_leaderboard, +}; +use super::scorer::{OutcomeHistogram, ReachPrecision, Score}; + +fn make_score(model: &str, reach_rate: f64, precision: f64, offline: bool) -> Score { + Score { + run_id: "r".to_string(), + model: model.to_string(), + overall: ReachPrecision { + reached: 0, + submitted: 0, + total: 0, + reach_rate, + precision, + }, + by_family: Vec::new(), + histogram: OutcomeHistogram::default(), + offline_scaffold: offline, + } +} + +#[test] +fn published_table_has_eight_ranked_entries() { + let board = published_leaderboard(); + assert_eq!(board.len(), 8); + for (i, entry) in board.iter().enumerate() { + assert_eq!(entry.rank as usize, i + 1); + } + let top = &board[0]; + assert_eq!(top.model, "Claude Opus 4.6"); + assert!((top.reach_pct - 30.0).abs() < 1e-9); + assert!((top.precision_pct - 52.5).abs() < 1e-9); +} + +#[test] +fn find_published_normalises_model_names() { + let hit = find_published("claude-opus-4.6").unwrap(); + assert_eq!(hit.model, "Claude Opus 4.6"); + let hit2 = find_published("GPT-5.4-mini").unwrap(); + assert_eq!(hit2.model, "GPT-5.4-mini"); + assert!(find_published("nonexistent-model").is_none()); +} + +#[test] +fn find_published_prefers_exact_over_prefix() { + // Regression: "gpt-5.4-mini" must not be shadowed by the "GPT-5.4" prefix. + assert_eq!( + find_published("gpt-5.4-mini").unwrap().model, + "GPT-5.4-mini" + ); + assert_eq!(find_published("gpt-5.4").unwrap().model, "GPT-5.4"); + assert!(find_published("").is_none()); +} + +#[test] +fn compare_flags_within_variance() { + let score = make_score("Claude Opus 4.6", 0.30, 0.525, false); + let cmp = compare_to_leaderboard(&score).unwrap(); + assert!((cmp.reach_delta_pct).abs() < 1e-9); + assert!(!cmp.material_deviation); + assert!(cmp.note.contains("within variance")); +} + +#[test] +fn compare_flags_material_deviation() { + let score = make_score("Claude Opus 4.6", 0.05, 0.10, false); + let cmp = compare_to_leaderboard(&score).unwrap(); + assert!(cmp.reach_delta_pct.abs() > MATERIAL_DEVIATION_PCT); + assert!(cmp.material_deviation); + assert!(cmp.note.contains("material deviation")); +} + +#[test] +fn offline_scaffold_note_is_explicit() { + let score = make_score("Claude Opus 4.6", 0.05, 0.10, true); + let cmp = compare_to_leaderboard(&score).unwrap(); + // Even with a large delta, an offline run is labelled illustrative, not a bug. + assert!(cmp.note.contains("offline scaffold")); +} + +#[test] +fn unknown_model_has_no_comparison() { + let score = make_score("my-local-frankenmodel", 0.5, 0.5, false); + assert!(compare_to_leaderboard(&score).is_none()); +} diff --git a/src/coin_gym/tests_profiles.rs b/src/coin_gym/tests_profiles.rs new file mode 100644 index 00000000..61260d84 --- /dev/null +++ b/src/coin_gym/tests_profiles.rs @@ -0,0 +1,117 @@ +use super::profiles::{ + DEFAULT_HOME, PersistedRun, default_home, ensure_profile, list_profiles, load_run, + sanitize_name, save_run, +}; +use super::target_loader::TargetSet; +use super::types::{RunReport, Strategy}; + +fn persisted(run_id: &str) -> PersistedRun { + PersistedRun { + report: RunReport { + run_id: run_id.to_string(), + model: "claude-opus-4.6".to_string(), + strategy: Strategy::Baseline, + snapshot: "snap".to_string(), + started_at_unix_ms: 1, + outcomes: Vec::new(), + offline_scaffold: true, + }, + targets: TargetSet { + snapshot: "snap".to_string(), + pinned: Vec::new(), + held_out_fresh: Vec::new(), + }, + } +} + +#[test] +fn sanitize_name_is_directory_safe() { + assert_eq!(sanitize_name("claude/opus:4.6"), "claude-opus-4-6"); + assert_eq!(sanitize_name(""), "default"); + assert_eq!(sanitize_name("ok_name-1"), "ok_name-1"); +} + +#[test] +fn default_home_honours_env_override() { + // Default (relative) when unset is a constant. + assert_eq!(DEFAULT_HOME, "target/coin-gym"); + // Note: we do not mutate process env here to avoid cross-test races; the + // override path is covered by exercising the explicit-home APIs below. + let _ = default_home(); +} + +#[test] +fn ensure_profile_persists_metadata_and_is_idempotent() { + let dir = tempfile::tempdir().unwrap(); + let home = dir.path(); + let p1 = ensure_profile(home, "opus", "claude-opus-4.6").unwrap(); + assert_eq!(p1.name, "opus"); + assert_eq!(p1.model, "claude-opus-4.6"); + // Second call with the SAME model loads existing metadata (idempotent). + let p2 = ensure_profile(home, "opus", "claude-opus-4.6").unwrap(); + assert_eq!(p2.created_at_unix_ms, p1.created_at_unix_ms); + assert_eq!(p2.model, "claude-opus-4.6"); +} + +#[test] +fn ensure_profile_rejects_model_mismatch() { + let dir = tempfile::tempdir().unwrap(); + let home = dir.path(); + ensure_profile(home, "opus", "claude-opus-4.6").unwrap(); + // Reusing the profile for a different model must be rejected (isolation). + let err = ensure_profile(home, "opus", "gpt-5.4").unwrap_err(); + assert!(err.to_string().contains("bound to model")); +} + +#[test] +fn save_run_refuses_to_overwrite() { + let dir = tempfile::tempdir().unwrap(); + let home = dir.path(); + ensure_profile(home, "opus", "claude-opus-4.6").unwrap(); + let run = persisted("dup-run-1"); + save_run(home, "opus", &run).unwrap(); + // Second save with the same run-id must fail loudly, not clobber. + let err = save_run(home, "opus", &run).unwrap_err(); + assert!(err.to_string().contains("already exists")); +} + +#[test] +fn save_and_load_run_round_trips() { + let dir = tempfile::tempdir().unwrap(); + let home = dir.path(); + ensure_profile(home, "opus", "claude-opus-4.6").unwrap(); + let run = persisted("claude-opus-4-6-baseline-1"); + let path = save_run(home, "opus", &run).unwrap(); + assert!(path.exists()); + + // Explicit profile. + let loaded = load_run(home, Some("opus"), "claude-opus-4-6-baseline-1").unwrap(); + assert_eq!(loaded.report.run_id, run.report.run_id); + + // Search across all profiles when no profile is given. + let found = load_run(home, None, "claude-opus-4-6-baseline-1").unwrap(); + assert_eq!(found.report.model, "claude-opus-4.6"); +} + +#[test] +fn load_missing_run_is_not_found() { + let dir = tempfile::tempdir().unwrap(); + let err = load_run(dir.path(), None, "nope").unwrap_err(); + assert!(err.to_string().contains("not found")); +} + +#[test] +fn list_profiles_is_empty_then_sorted() { + let dir = tempfile::tempdir().unwrap(); + let home = dir.path(); + assert!(list_profiles(home).unwrap().is_empty()); + + ensure_profile(home, "zeta", "m1").unwrap(); + ensure_profile(home, "alpha", "m2").unwrap(); + let names: Vec = list_profiles(home) + .unwrap() + .into_iter() + .map(|p| p.name) + .collect(); + assert_eq!(names, vec!["alpha".to_string(), "zeta".to_string()]); +} diff --git a/src/coin_gym/tests_scorer.rs b/src/coin_gym/tests_scorer.rs new file mode 100644 index 00000000..d1e43d1c --- /dev/null +++ b/src/coin_gym/tests_scorer.rs @@ -0,0 +1,118 @@ +use super::scorer::{OutcomeHistogram, ReachPrecision, score_run}; +use super::types::{Outcome, OutcomeCode, RunReport, Strategy, TargetFamily}; + +fn outcome(id: &str, family: TargetFamily, code: OutcomeCode) -> Outcome { + Outcome { + target_id: id.to_string(), + family, + code, + cost_usd: 0.0, + } +} + +fn report(outcomes: Vec) -> RunReport { + RunReport { + run_id: "r1".to_string(), + model: "m".to_string(), + strategy: Strategy::Baseline, + snapshot: "snap".to_string(), + started_at_unix_ms: 0, + outcomes, + offline_scaffold: true, + } +} + +fn approx(a: f64, b: f64) -> bool { + (a - b).abs() < 1e-9 +} + +#[test] +fn histogram_tallies_every_code() { + let outcomes = vec![ + outcome("a", TargetFamily::Frontier, OutcomeCode::Reached), + outcome("b", TargetFamily::Frontier, OutcomeCode::WrongInput), + outcome("c", TargetFamily::Frontier, OutcomeCode::Abstained), + outcome("d", TargetFamily::Frontier, OutcomeCode::TimedOut), + outcome("e", TargetFamily::Frontier, OutcomeCode::NoSubmission), + outcome("f", TargetFamily::Frontier, OutcomeCode::Error), + ]; + let h = OutcomeHistogram::tally(&outcomes); + assert_eq!(h.reached, 1); + assert_eq!(h.wrong_input, 1); + assert_eq!(h.abstained, 1); + assert_eq!(h.timed_out, 1); + assert_eq!(h.no_submission, 1); + assert_eq!(h.error, 1); + assert_eq!(h.total(), 6); + assert_eq!(h.render(), "R:1/W:1/A:1/T:1/N:1/E:1"); +} + +#[test] +fn reach_and_precision_use_correct_denominators() { + // 1 reached, 1 wrong (submitted), 1 abstained (not submitted). + let outcomes = vec![ + outcome("a", TargetFamily::Frontier, OutcomeCode::Reached), + outcome("b", TargetFamily::Frontier, OutcomeCode::WrongInput), + outcome("c", TargetFamily::Frontier, OutcomeCode::Abstained), + ]; + let rp = ReachPrecision::compute(&outcomes); + assert_eq!(rp.reached, 1); + assert_eq!(rp.submitted, 2); + assert_eq!(rp.total, 3); + assert!(approx(rp.reach_rate, 1.0 / 3.0)); + assert!(approx(rp.precision, 0.5)); + assert!(approx(rp.reach_pct(), 100.0 / 3.0)); + assert!(approx(rp.precision_pct(), 50.0)); +} + +#[test] +fn precision_is_zero_when_nothing_submitted() { + let outcomes = vec![ + outcome("a", TargetFamily::Frontier, OutcomeCode::Abstained), + outcome("b", TargetFamily::Frontier, OutcomeCode::NoSubmission), + ]; + let rp = ReachPrecision::compute(&outcomes); + assert!(approx(rp.precision, 0.0)); + assert!(approx(rp.reach_rate, 0.0)); +} + +#[test] +fn score_run_splits_by_family_and_skips_empty() { + let outcomes = vec![ + outcome("f1", TargetFamily::Frontier, OutcomeCode::Reached), + outcome("f2", TargetFamily::Frontier, OutcomeCode::WrongInput), + outcome( + "n1", + TargetFamily::NonTrivialReachable, + OutcomeCode::Reached, + ), + ]; + let score = score_run(&report(outcomes)); + assert_eq!(score.by_family.len(), 2); + + let frontier = score + .by_family + .iter() + .find(|f| f.family == TargetFamily::Frontier) + .unwrap(); + assert!(approx(frontier.score.reach_rate, 0.5)); + assert!(approx(frontier.score.precision, 0.5)); + + let ntr = score + .by_family + .iter() + .find(|f| f.family == TargetFamily::NonTrivialReachable) + .unwrap(); + assert!(approx(ntr.score.reach_rate, 1.0)); + + assert!(score.offline_scaffold); + assert_eq!(score.model, "m"); +} + +#[test] +fn single_family_run_yields_one_family_entry() { + let outcomes = vec![outcome("f1", TargetFamily::Frontier, OutcomeCode::Reached)]; + let score = score_run(&report(outcomes)); + assert_eq!(score.by_family.len(), 1); + assert_eq!(score.by_family[0].family, TargetFamily::Frontier); +} diff --git a/src/coin_gym/tests_target_loader.rs b/src/coin_gym/tests_target_loader.rs new file mode 100644 index 00000000..80ee3089 --- /dev/null +++ b/src/coin_gym/tests_target_loader.rs @@ -0,0 +1,102 @@ +use std::io::Write; + +use super::target_loader::{ + DemoScenario, FixtureTargetSource, InMemoryTargetSource, SAMPLE_SNAPSHOT_JSON, TargetSet, + TargetSource, +}; +use super::types::TargetFamily; + +#[test] +fn sample_snapshot_parses_and_has_both_slices() { + let set = InMemoryTargetSource::sample().load().unwrap(); + assert_eq!(set.snapshot, "you/coin@v1-sample"); + assert!(!set.pinned.is_empty(), "sample must have pinned targets"); + assert!( + !set.held_out_fresh.is_empty(), + "sample must reserve a held-out fresh slice" + ); + assert_eq!(set.total(), set.pinned.len() + set.held_out_fresh.len()); +} + +#[test] +fn sample_has_both_families_in_pinned() { + let set = InMemoryTargetSource::sample().load().unwrap(); + assert!( + set.pinned + .iter() + .any(|t| t.family == TargetFamily::Frontier) + ); + assert!( + set.pinned + .iter() + .any(|t| t.family == TargetFamily::NonTrivialReachable) + ); +} + +#[test] +fn demo_scenario_sample_has_oracle_and_script() { + let scenario = DemoScenario::sample().unwrap(); + // Every scripted target must exist in the pinned set. + for id in scenario.script.keys() { + assert!( + scenario.targets.pinned.iter().any(|t| &t.id == id), + "scripted id {id} missing from pinned targets" + ); + } + // Oracle covers every pinned target so offline grading is well-defined. + for t in &scenario.targets.pinned { + assert!( + scenario.oracle.contains_key(&t.id), + "oracle missing reaching input for {}", + t.id + ); + } +} + +#[test] +fn malformed_manifest_is_a_parse_error() { + let src = InMemoryTargetSource::new("{ not json "); + let err = src.load().unwrap_err(); + assert!(err.to_string().contains("parse error")); +} + +#[test] +fn fixture_source_reads_from_disk() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("snap.json"); + let mut f = std::fs::File::create(&path).unwrap(); + f.write_all(SAMPLE_SNAPSHOT_JSON.as_bytes()).unwrap(); + drop(f); + + let set = FixtureTargetSource::new(&path).load().unwrap(); + assert_eq!(set.snapshot, "you/coin@v1-sample"); + + let scenario = DemoScenario::from_path(&path).unwrap(); + assert_eq!(scenario.targets.snapshot, "you/coin@v1-sample"); +} + +#[test] +fn missing_file_is_io_error() { + let src = FixtureTargetSource::new("/no/such/coin/snapshot.json"); + let err = src.load().unwrap_err(); + assert!(err.to_string().contains("io error")); +} + +#[test] +fn target_set_total_counts_both_slices() { + let raw = r#"{ + "snapshot": "s", + "targets": { + "pinned": [ + {"id":"a","project":"p","commit":"c","harness":"h","file":"f","line":1,"family":"frontier"} + ], + "held_out_fresh": [ + {"id":"b","project":"p","commit":"c","harness":"h","file":"f","line":2,"family":"non-trivial-reachable"} + ] + } + }"#; + let set: TargetSet = InMemoryTargetSource::new(raw).load().unwrap(); + assert_eq!(set.pinned.len(), 1); + assert_eq!(set.held_out_fresh.len(), 1); + assert_eq!(set.total(), 2); +} diff --git a/src/coin_gym/tests_types.rs b/src/coin_gym/tests_types.rs new file mode 100644 index 00000000..84075d07 --- /dev/null +++ b/src/coin_gym/tests_types.rs @@ -0,0 +1,111 @@ +use super::types::{CoinGymError, Outcome, OutcomeCode, RunReport, Strategy, Target, TargetFamily}; + +fn target(id: &str, family: TargetFamily) -> Target { + Target { + id: id.to_string(), + project: "proj".to_string(), + commit: "abc".to_string(), + harness: "h".to_string(), + file: "src/x.c".to_string(), + line: 42, + family, + } +} + +#[test] +fn target_family_labels_are_stable() { + assert_eq!(TargetFamily::Frontier.label(), "frontier"); + assert_eq!( + TargetFamily::NonTrivialReachable.label(), + "non-trivial-reachable" + ); + assert_eq!(TargetFamily::Frontier.to_string(), "frontier"); +} + +#[test] +fn target_family_serde_round_trip_kebab() { + let json = serde_json::to_string(&TargetFamily::NonTrivialReachable).unwrap(); + assert_eq!(json, "\"non-trivial-reachable\""); + let back: TargetFamily = serde_json::from_str(&json).unwrap(); + assert_eq!(back, TargetFamily::NonTrivialReachable); +} + +#[test] +fn target_locator_is_project_file_line() { + let t = target("t1", TargetFamily::Frontier); + assert_eq!(t.locator(), "proj:src/x.c:42"); +} + +#[test] +fn strategy_parse_accepts_known_and_rejects_unknown() { + assert_eq!(Strategy::parse("baseline").unwrap(), Strategy::Baseline); + assert_eq!(Strategy::parse("team").unwrap(), Strategy::Team); + assert!(Strategy::parse("solo").is_err()); + assert_eq!(Strategy::Team.label(), "team"); + assert_eq!(Strategy::Baseline.to_string(), "baseline"); +} + +#[test] +fn outcome_code_letters_and_predicates() { + assert_eq!(OutcomeCode::Reached.letter(), 'R'); + assert_eq!(OutcomeCode::WrongInput.letter(), 'W'); + assert_eq!(OutcomeCode::Abstained.letter(), 'A'); + assert_eq!(OutcomeCode::TimedOut.letter(), 'T'); + assert_eq!(OutcomeCode::NoSubmission.letter(), 'N'); + assert_eq!(OutcomeCode::Error.letter(), 'E'); + + assert!(OutcomeCode::Reached.reached()); + assert!(!OutcomeCode::WrongInput.reached()); + + // Only submitted inputs count toward precision's denominator. + assert!(OutcomeCode::Reached.submitted()); + assert!(OutcomeCode::WrongInput.submitted()); + assert!(OutcomeCode::TimedOut.submitted()); + assert!(!OutcomeCode::Abstained.submitted()); + assert!(!OutcomeCode::NoSubmission.submitted()); + assert!(!OutcomeCode::Error.submitted()); +} + +#[test] +fn outcome_helpers_delegate_to_code() { + let o = Outcome { + target_id: "t".to_string(), + family: TargetFamily::Frontier, + code: OutcomeCode::Reached, + cost_usd: 1.5, + }; + assert!(o.reached()); + assert!(o.submitted()); +} + +#[test] +fn run_report_serde_round_trip() { + let report = RunReport { + run_id: "m-baseline-1".to_string(), + model: "m".to_string(), + strategy: Strategy::Baseline, + snapshot: "you/coin@v1".to_string(), + started_at_unix_ms: 123, + outcomes: vec![Outcome { + target_id: "t".to_string(), + family: TargetFamily::Frontier, + code: OutcomeCode::Reached, + cost_usd: 0.0, + }], + offline_scaffold: true, + }; + let json = serde_json::to_string(&report).unwrap(); + let back: RunReport = serde_json::from_str(&json).unwrap(); + assert_eq!(back, report); + assert_eq!(back.target_count(), 1); +} + +#[test] +fn error_display_is_readable() { + assert!( + CoinGymError::NotFound("run x".to_string()) + .to_string() + .contains("not found") + ); + assert_eq!(CoinGymError::Usage("boom".to_string()).to_string(), "boom"); +} diff --git a/src/coin_gym/types.rs b/src/coin_gym/types.rs new file mode 100644 index 00000000..9ac0e5b5 --- /dev/null +++ b/src/coin_gym/types.rs @@ -0,0 +1,259 @@ +//! Core data model for the LOCAL COIN Gym harness (Phase 4). +//! +//! COIN (COde → INput) grades *reachability by execution*: an agent produces a +//! concrete input that must drive a maintainer harness to a chosen target line, +//! verified by re-running the harness on a coverage-instrumented build. See +//! `docs/research/coin-benchmark-and-skwaq-study.md` for the full design this +//! module scaffolds. +//! +//! These types are deliberately serde-friendly, self-contained, and free of any +//! Docker/VM/LLM dependency so the whole pipeline can be exercised offline +//! against a mock oracle (see [`crate::coin_gym::executor::MockHarnessExecutor`]). + +use std::fmt::{self, Display, Formatter}; + +use serde::{Deserialize, Serialize}; + +/// The two COIN target families sampled from the signal algebra (research doc +/// Part 1.3). +#[derive(Clone, Copy, Debug, Eq, PartialEq, Hash, Serialize, Deserialize)] +#[serde(rename_all = "kebab-case")] +pub enum TargetFamily { + /// `(B ∪ C) \ (G ∪ F ∪ L)` — provably reachable by static analysis but never + /// reached by any fuzzer or cheap baseline. Hardest, most + /// contamination-resistant class. + Frontier, + /// `G \ (F ∪ L)` — reached by long-running fuzzing but missed by fresh + /// fuzzers and goal-blind LLM seeds. + NonTrivialReachable, +} + +impl TargetFamily { + /// Stable kebab-case label used in reports and CLI output. + #[must_use] + pub fn label(self) -> &'static str { + match self { + Self::Frontier => "frontier", + Self::NonTrivialReachable => "non-trivial-reachable", + } + } +} + +impl Display for TargetFamily { + fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { + f.write_str(self.label()) + } +} + +/// A single COIN task: a target line `ℓ` in a real project at a pinned commit, +/// reached through one maintainer-written harness (research doc Part 1.2). +#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)] +pub struct Target { + /// Stable identifier for the target within a snapshot. + pub id: String, + /// Upstream project name (e.g. `libraw`). + pub project: String, + /// Pinned commit the target line refers to. + pub commit: String, + /// Maintainer-written harness the agent must drive. + pub harness: String, + /// Source file containing the target line. + pub file: String, + /// 1-based target line number `ℓ`. + pub line: u32, + /// Target family (frontier vs. non-trivial reachable). + pub family: TargetFamily, +} + +impl Target { + /// `project:file:line` — a human-readable locator for logs and reports. + #[must_use] + pub fn locator(&self) -> String { + format!("{}:{}:{}", self.project, self.file, self.line) + } +} + +/// The strategy under test — a single-model baseline vs. a multi-agent team +/// (research doc Part 3.3, component 2). +#[derive(Clone, Copy, Debug, Eq, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "kebab-case")] +pub enum Strategy { + /// One model reads the harness + source and emits candidate input bytes. + Baseline, + /// A skwaq-style debate: a *reacher* proposes an input, a *skeptic* challenges + /// over-claims, and a *synthesizer* submits-or-abstains via a + /// `threshold_hint`-style gate. + Team, +} + +impl Strategy { + /// Stable kebab-case label. + #[must_use] + pub fn label(self) -> &'static str { + match self { + Self::Baseline => "baseline", + Self::Team => "team", + } + } + + /// Parse a CLI `--strategy` value. + /// + /// # Errors + /// Returns an error string for unrecognised values. + pub fn parse(value: &str) -> Result { + match value { + "baseline" => Ok(Self::Baseline), + "team" => Ok(Self::Team), + other => Err(format!( + "unknown strategy '{other}' (expected 'baseline' or 'team')" + )), + } + } +} + +impl Display for Strategy { + fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { + f.write_str(self.label()) + } +} + +/// Per-target outcome code from the results matrix (research doc Part 1.4). +#[derive(Clone, Copy, Debug, Eq, PartialEq, Hash, Serialize, Deserialize)] +pub enum OutcomeCode { + /// `R` — Reached ✓ (verified by harness replay on the instrumented build). + Reached, + /// `W` — Submitted a wrong input (did not reach the line). + WrongInput, + /// `A` — Abstained (deliberately declined to submit). + Abstained, + /// `T` — Timed out. + TimedOut, + /// `N` — No submission (the agent produced nothing). + NoSubmission, + /// `E` — Error during evaluation. + Error, +} + +impl OutcomeCode { + /// Single-letter code (`R`/`W`/`A`/`T`/`N`/`E`). + #[must_use] + pub fn letter(self) -> char { + match self { + Self::Reached => 'R', + Self::WrongInput => 'W', + Self::Abstained => 'A', + Self::TimedOut => 'T', + Self::NoSubmission => 'N', + Self::Error => 'E', + } + } + + /// Whether this outcome reached the target line. + #[must_use] + pub fn reached(self) -> bool { + matches!(self, Self::Reached) + } + + /// Whether the agent *submitted* an input (counts toward precision's + /// denominator). Abstain / no-submission do not. + #[must_use] + pub fn submitted(self) -> bool { + matches!(self, Self::Reached | Self::WrongInput | Self::TimedOut) + } +} + +impl Display for OutcomeCode { + fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { + write!(f, "{}", self.letter()) + } +} + +/// The result of grading one target: the objective oracle's verdict for a single +/// (target, submission) pair. +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +pub struct Outcome { + /// The target this outcome is for. + pub target_id: String, + /// Family of the target (denormalised so scoring needs only outcomes). + pub family: TargetFamily, + /// The outcome code. + pub code: OutcomeCode, + /// Estimated USD cost of producing/grading this target (0.0 when unknown). + pub cost_usd: f64, +} + +impl Outcome { + /// Whether the target line was reached. + #[must_use] + pub fn reached(&self) -> bool { + self.code.reached() + } + + /// Whether an input was submitted (precision denominator membership). + #[must_use] + pub fn submitted(&self) -> bool { + self.code.submitted() + } +} + +/// A complete evaluation pass over a target set for one (model, strategy). +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +pub struct RunReport { + /// Unique run identifier (`--`). + pub run_id: String, + /// Model under test (LiteLLM model id, e.g. `claude-opus-4.6`). + pub model: String, + /// Strategy under test. + pub strategy: Strategy, + /// Snapshot the targets were drawn from (e.g. `you/coin@v1`). + pub snapshot: String, + /// Wall-clock start time (unix epoch milliseconds). + pub started_at_unix_ms: u128, + /// Per-target outcomes, one per evaluated target. + pub outcomes: Vec, + /// `true` when the outcomes came from a mock oracle (offline scaffold run) + /// rather than a real `coin evaluate` grade. Real grading is gated behind + /// Phase 3 (VM + Docker). + pub offline_scaffold: bool, +} + +impl RunReport { + /// Number of targets in the run. + #[must_use] + pub fn target_count(&self) -> usize { + self.outcomes.len() + } +} + +/// Errors surfaced by the COIN Gym harness. Kept small and `Display`-able so it +/// composes cleanly into the CLI's `Box`. +#[derive(Clone, Debug, Eq, PartialEq)] +pub enum CoinGymError { + /// A snapshot / report / fixture failed to parse. + Parse(String), + /// A filesystem operation failed. + Io(String), + /// A referenced entity (run, profile, target) was not found. + NotFound(String), + /// The harness executor (or its gated `coin evaluate` delegate) failed. + Executor(String), + /// A CLI argument was invalid. + Usage(String), +} + +impl Display for CoinGymError { + fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { + match self { + Self::Parse(m) => write!(f, "parse error: {m}"), + Self::Io(m) => write!(f, "io error: {m}"), + Self::NotFound(m) => write!(f, "not found: {m}"), + Self::Executor(m) => write!(f, "executor error: {m}"), + Self::Usage(m) => write!(f, "{m}"), + } + } +} + +impl std::error::Error for CoinGymError {} + +/// Convenience result alias for the module. +pub type CoinGymResult = Result; diff --git a/src/lib.rs b/src/lib.rs index 80911d4b..348ab99c 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -65,6 +65,15 @@ pub mod goal_board_store; pub mod goal_curation; pub mod goals; pub mod greeting_banner; +// Phase 4 of issue #2713: the LOCAL "COIN Gym" harness — runs the COIN +// benchmark shape locally, scores vs. the published leaderboard, and A/Bs a +// single-model baseline against a multi-agent team, mirroring skwaq's +// failure-analysis + overfitting-reviewer gating. The harness executor delegates +// to `coin evaluate` (Docker + instrumented replay is Phase 3/VM); a mock oracle +// exercises the whole pipeline offline. See +// `docs/research/coin-benchmark-and-skwaq-study.md` (design) and +// `docs/howto/run-the-coin-gym-harness.md` (usage). +pub mod coin_gym; pub mod gym; pub mod gym_client; pub mod gym_history; @@ -341,6 +350,7 @@ pub use rpc_circuit_breaker::{CircuitBreakerConfig, CircuitBreakerTransport, Cir pub use rpc_transport::{InMemoryRpcTransport, SubprocessRpcTransport}; pub use test_support::TestAdapter; +pub use coin_gym::{coin_gym_usage, dispatch_coin_gym_cli}; pub use engineer_handoff::{ ENGINEER_HANDOFF_FILE_NAME, ENGINEER_MODE_BOUNDARY, EngineerHandoffContext, SHARED_DEFAULT_STATE_ROOT_SOURCE, SHARED_EXPLICIT_STATE_ROOT_SOURCE, diff --git a/tests/gadugi/coin-gym-harness.sh b/tests/gadugi/coin-gym-harness.sh new file mode 100755 index 00000000..2b940e79 --- /dev/null +++ b/tests/gadugi/coin-gym-harness.sh @@ -0,0 +1,87 @@ +#!/usr/bin/env bash +# Outside-in gadugi scenario for the LOCAL COIN Gym harness CLI (`coin-gym`), +# Phase 4 of issue #2713. +# +# GUARANTEES UNDER TEST (all offline; no VM, no Docker, no network): +# 1. `coin-gym run` drives the full pipeline (target-load → agent-run → grade +# → score) against the bundled sample snapshot and a mock oracle, and +# labels the run OFFLINE SCAFFOLD. +# 2. The multi-agent TEAM strategy lifts precision over the single-model +# BASELINE at equal reach — the central baseline-vs-team design property +# (baseline 3R/2W = 60% precision; team 3R/2A = 100% precision). +# 3. `score`, `compare`, and `improve` reload a saved run and report +# reach/precision, leaderboard deltas, and gated tactic proposals. +# 4. The overfitting-reviewer gate ACCEPTs the analyst's general tactics, and +# `improve` flags the live verify/rollback loop as Phase 5. +# 5. `profiles` lists isolated per-model run state. +# 6. An unknown command exits non-zero with usage. +# +# Hermetic: COIN_GYM_HOME is a throwaway temp dir; nothing touches the real +# ~/.simard state or the network. +set -euo pipefail + +ROOT="$(cd "$(dirname "$0")/../.." && pwd)" +cd "$ROOT" + +HOME_DIR="$(mktemp -d)" +cleanup() { rm -rf "$HOME_DIR"; } +trap cleanup EXIT +export COIN_GYM_HOME="$HOME_DIR" + +run_gym() { cargo run --quiet --bin coin-gym -- "$@"; } + +# --- 1. baseline run: full offline pipeline ----------------------------------- +BASELINE="$(run_gym run claude-opus-4.6 --strategy baseline --profile opus)" +printf '%s\n' "$BASELINE" +printf '%s\n' "$BASELINE" | grep -F "OFFLINE SCAFFOLD" >/dev/null +printf '%s\n' "$BASELINE" | grep -F "reach: 60.0%" >/dev/null +printf '%s\n' "$BASELINE" | grep -F "precision: 60.0%" >/dev/null +printf '%s\n' "$BASELINE" | grep -F "R:3/W:2/A:0/T:0/N:0/E:0" >/dev/null + +# --- 2. team run: abstention gate lifts precision at equal reach --------------- +TEAM="$(run_gym run claude-opus-4.6 --strategy team --profile opus-team)" +printf '%s\n' "$TEAM" +printf '%s\n' "$TEAM" | grep -F "reach: 60.0%" >/dev/null +printf '%s\n' "$TEAM" | grep -F "precision: 100.0%" >/dev/null +printf '%s\n' "$TEAM" | grep -F "R:3/W:0/A:2/T:0/N:0/E:0" >/dev/null + +# --- 3. score / compare / improve reload the saved baseline run --------------- +RUN_ID="$(printf '%s\n' "$BASELINE" | awk '/^run-id:/ { print $2; exit }')" +test -n "$RUN_ID" + +SCORE="$(run_gym score "$RUN_ID")" +printf '%s\n' "$SCORE" +printf '%s\n' "$SCORE" | grep -F "reach: 60.0%" >/dev/null +printf '%s\n' "$SCORE" | grep -F "frontier" >/dev/null +printf '%s\n' "$SCORE" | grep -F "non-trivial-reachable" >/dev/null +# Reloaded mock-oracle runs must still carry the OFFLINE SCAFFOLD warning. +printf '%s\n' "$SCORE" | grep -F "OFFLINE SCAFFOLD" >/dev/null + +COMPARE="$(run_gym compare "$RUN_ID")" +printf '%s\n' "$COMPARE" +printf '%s\n' "$COMPARE" | grep -F "published: Claude Opus 4.6" >/dev/null +printf '%s\n' "$COMPARE" | grep -F "material-deviation:" >/dev/null +printf '%s\n' "$COMPARE" | grep -F "offline scaffold" >/dev/null + +# --- 4. improve: general tactics ACCEPTed; live loop flagged Phase 5 ----------- +IMPROVE="$(run_gym improve "$RUN_ID")" +printf '%s\n' "$IMPROVE" +printf '%s\n' "$IMPROVE" | grep -F "[ACCEPT]" >/dev/null +printf '%s\n' "$IMPROVE" | grep -F "Phase 5" >/dev/null +# The gate must not have rejected the analyst's own general tactics. +printf '%s\n' "$IMPROVE" | grep -F "rejected: 0" >/dev/null + +# --- 5. profiles: isolated per-model run state -------------------------------- +PROFILES="$(run_gym profiles)" +printf '%s\n' "$PROFILES" +printf '%s\n' "$PROFILES" | grep -F "opus" >/dev/null + +# --- 6. unknown command exits non-zero with usage ----------------------------- +if run_gym frobnicate >/tmp/coin-gym-bogus.out 2>&1; then + echo "ERROR: unknown command should have failed" >&2 + exit 1 +fi +grep -F "unknown command" /tmp/coin-gym-bogus.out >/dev/null +rm -f /tmp/coin-gym-bogus.out + +echo "coin-gym-harness gadugi scenario: PASS" diff --git a/tests/gadugi/coin-gym-harness.yaml b/tests/gadugi/coin-gym-harness.yaml new file mode 100644 index 00000000..9382054d --- /dev/null +++ b/tests/gadugi/coin-gym-harness.yaml @@ -0,0 +1,55 @@ +name: "LOCAL COIN Gym harness CLI (#2713 Phase 4)" +description: "Outside-in coverage for the coin-gym CLI: the full offline pipeline (target-load → agent-run → mock-grade → score), the baseline-vs-team precision trade-off at equal reach, score/compare/improve reloading a saved run, the overfitting-reviewer gate accepting general tactics while flagging the live loop as Phase 5, isolated per-model profiles, and usage-on-error. Hermetic and network-free; grades against a mock oracle (real coin evaluate is Phase 3)." +version: "1.0.0" + +config: + timeout: 300000 + retries: 1 + parallel: false + +environment: + requires: [] + +agents: + - name: "cli-agent" + type: "cli" + config: + workingDirectory: "." + defaultTimeout: 300000 + executionMode: "spawn" + captureOutput: true + ioConfig: + encoding: "utf8" + handleInteractivePrompts: false + logConfig: + logCommands: true + logOutput: true + logLevel: "info" + +steps: + - name: "COIN Gym harness end-to-end pipeline" + agent: "cli-agent" + action: "execute_command" + params: + command: "tests/gadugi/coin-gym-harness.sh" + timeout: 300000 + +assertions: [] + +cleanup: + - name: "CLI agent cleanup" + agent: "cli-agent" + action: "cleanup" + +metadata: + tags: + - "cli" + - "behavior" + - "simard" + - "coin-gym" + - "benchmark" + - "baseline-vs-team" + - "overfitting-gate" + priority: "high" + author: "copilot" + test_type: "outside-in"