diff --git a/.gitignore b/.gitignore index 864f9ae..cdb4f4a 100644 --- a/.gitignore +++ b/.gitignore @@ -7,6 +7,8 @@ # Test binary, built with `go test -c` *.test +__pycache__/ +*.pyc # Output of the go coverage tool, specifically when used with Mate *.out @@ -47,3 +49,6 @@ coverage.html # Miscellaneous *.bak *.tmp + +# Private agent evaluation traces (may contain prompts and repository content) +/.eval-private/ diff --git a/README.md b/README.md index 17c0df3..e3d390f 100644 --- a/README.md +++ b/README.md @@ -31,7 +31,7 @@ GHA bundles a portable `gha` skill that helps coding agents use its structured G gha agent install ``` -The command detects configured Codex, Claude Code, Pi, OpenCode, GitHub Copilot, Gemini CLI, Cursor, Hermes Agent, and OpenClaw setup directories and configures those harnesses without requiring their executables. Use `--agent` with comma-separated names to choose explicitly, or `--binary-only` to skip harness setup. If none are detected, GHA reports how to configure one later. It copies the bundled skill into a discoverable global skill directory and idempotently adds a marked GHA guidance block where a documented global instruction file exists. Existing skills are not overwritten, and modified GHA skills are preserved by update and uninstall. `gha agent list` shows recorded harnesses and current file presence. +The command detects configured Codex, Claude Code, Pi, OpenCode, GitHub Copilot, Gemini CLI, Cursor, Hermes Agent, and OpenClaw setup directories and configures those harnesses without requiring their executables. Use `--agent` with comma-separated names to choose explicitly, or `--binary-only` to skip harness setup. If none are detected, GHA reports how to configure one later. It copies the bundled detailed skill into a discoverable global skill directory and idempotently adds a **compact** GHA guidance block where a documented global instruction file exists. Agents can use the detailed skill explicitly when a workflow needs it; the default block leaves simple Git tasks to Git. Existing skills are not overwritten, and modified GHA skills are preserved by update and uninstall. `gha agent list` shows recorded harnesses and current file presence. ```bash # Preview paths without writing files. @@ -98,11 +98,21 @@ gha pr create --title "Improve reviews" --head feature/reviews GHA uses `GHA_GITHUB_TOKEN` when set; otherwise it uses `gh auth token` when the GitHub CLI is installed and authenticated. Without either, public API requests are unauthenticated. For private repositories or higher API rate limits, provide a fine-grained token restricted to the repositories GHA will access. The full command surface uses `Contents: read`, `Pull requests: write`, `Checks: read`, `Actions: read`, and `Issues: read`; `Pull requests: write` is required for `gha pr create` and includes pull-request read access. Git branch publication uses the checkout remote's authentication, not the GitHub API token. A classic token needs the `repo` scope for private repositories. +## Agent efficiency tests + +Benchmarks of agent correctness, token use, and command effort are in the +[test results](docs/agent-efficiency-test-results.md). They include a +git/gh-only control and a check of compact default guidance. See the +[evaluation method and limitations](docs/agent-efficiency-evaluation.md) +for how the runs were scored. + ## Documentation - [Getting started](docs/getting-started.md) — installation, authentication, and first commands - [Command guide](docs/command-guide.md) — output contracts and examples for every available workflow - [Development](docs/development.md) — build, test, quality, contribution, and repository layout +- [Agent efficiency test results](docs/agent-efficiency-test-results.md) — benchmark tables and findings +- [Agent efficiency evaluation](docs/agent-efficiency-evaluation.md) — paired agent benchmark and reporting method - [Release preparation](docs/development.md#preparing-a-release) — reviewed notes and tag publication - [Roadmap](docs/roadmap.md) — delivered work, prioritized next steps, and future phases - [Architecture](ARCHITECTURE.md) — current implementation architecture diff --git a/docs/agent-efficiency-evaluation.md b/docs/agent-efficiency-evaluation.md new file mode 100644 index 0000000..615c7b7 --- /dev/null +++ b/docs/agent-efficiency-evaluation.md @@ -0,0 +1,471 @@ +# Agent efficiency evaluation + +This is a local, paired pilot for testing whether GHA helps a coding agent +answer repository workflow questions accurately with fewer tokens. It runs the +same Codex model on a pinned commit in a fresh clone for each task. One run is +instructed to use Git and source inspection without GHA; the other may use a +built GHA executable. Both use read-only sandboxes. The answers are checked +against fixed, reviewable expectations. The current +[`workflow_tasks.json`](../tools/evals/workflow_tasks.json) is pinned to commit +`16df66c472d5e63208f4a7786aa136d5c6fb4b79` and covers branch +reachability and cached upstream divergence. The initial diagnostic also +included largest tracked file lookup. +The harness pins Codex reasoning to low and alternates which condition runs +first. It replaces the clone's project agent instructions with identical +neutral guidance in both conditions. + +## Run the pilot + +Build from the pinned commit, then record its SHA. Use a model available +to your Codex account. The command previews the complete run plan by default; +`--execute` starts agent runs and consumes tokens. + +Run `make test-eval` first to validate the local harness without model calls. + +```bash +git clone --no-hardlinks . /tmp/gha-eval-source +git -C /tmp/gha-eval-source checkout --detach 16df66c472d5e63208f4a7786aa136d5c6fb4b79 +make -C /tmp/gha-eval-source build +python tools/evals/agent_efficiency.py run \ + --tasks tools/evals/workflow_tasks.json \ + --repo /tmp/gha-eval-source \ + --revision 16df66c472d5e63208f4a7786aa136d5c6fb4b79 \ + --gha /tmp/gha-eval-source/bin/gha --model YOUR_MODEL --repeats 2 \ + --out /tmp/gha-agent-eval-1 + +# Run after reviewing the plan. +python tools/evals/agent_efficiency.py run \ + --tasks tools/evals/workflow_tasks.json \ + --repo /tmp/gha-eval-source \ + --revision 16df66c472d5e63208f4a7786aa136d5c6fb4b79 \ + --gha /tmp/gha-eval-source/bin/gha --model YOUR_MODEL --repeats 2 \ + --out /tmp/gha-agent-eval-1 --execute + +python tools/evals/agent_efficiency.py report \ + --input /tmp/gha-agent-eval-1/runs.jsonl \ + --public-output /tmp/gha-eval-public-results.json +``` + +The output directory contains `plan.json`, `runs.jsonl`, and one raw Codex +event trace per attempt. Treat traces as private: they may contain prompts, +tool output, and repository content. Review them before sharing. No agent run +publishes code or contacts a Git remote as part of the benchmark prompt. +`--public-output` exports only whitelisted metrics, with no answers, stderr, +trace text, or private output paths. Review even this export before publishing. + +## Why the initial result did not show GHA efficiency + +On 27 September 2026, Codex CLI 0.156.1 ran `gpt-6-luna` with low reasoning. +The GHA binary SHA-256 was +`8f6d3786e3c53dc12261801d1d492bbe3d3104b4ec69b67f5649ddad898882f7`. +Each of three tasks ran twice per condition, alternating the order of +conditions. The initial harness accepted any GHA invocation as use of GHA. +Trace review showed that five of six GHA-enabled attempts ran +`gha capabilities --format json` and then used Git for the task. The sixth ran +`gha analyze` and repeated the lookup with Git. The capability inventory +returned 7,516 characters on every GHA-enabled attempt. This adds a model +turn and context without replacing a Git lookup. The initial token totals are +therefore a diagnostic of the setup, not a measurement of GHA task workflow +efficiency. + +| Measure | Control | With GHA | +| --- | ---: | ---: | +| Correct attempts | 5/6 | 6/6 | +| Total input and output tokens | 181,544 | 382,061 | +| Tokens per correct attempt | 36,309 | 63,677 | +| Mean elapsed time per attempt | 12.4 s | 18.4 s | + +The only correctness difference was one branch reachability attempt where the +control returned empty answers. It ran no inspection command, so this cannot +be attributed to GHA. The [sanitized per-attempt results](evaluation-results/2026-09-27-workflows.json) +are retained as diagnostic evidence. Traces and complete run records were +kept privately in `/tmp/gha-agent-eval-workflows` for review. + +A stricter follow-up required the GHA task command. The agent used +`gha branches` in both cached upstream attempts and answered correctly. Those +runs consumed 65,258 and 48,257 tokens, versus 29,049 and 43,602 for their +Git controls. GHA returned about 3,500 characters of branch data after the +7,516-character inventory; the Git controls retrieved under 900 characters of +branch data. In the reachability attempts, some agents answered without any +repository inspection. The harness now marks those attempts invalid. This +follow-up is still too small and inconsistent to support an efficiency claim. + +## Corrected branch workflow rerun + +On 27 September 2026, the corrected harness ran the two current tasks twice +per condition with `gpt-6-sol` at low reasoning. All eight attempts inspected +the repository, all four GHA attempts used the selected GHA task command, and +all answers were correct. The [sanitized per-attempt results](evaluation-results/2026-09-27-branch-workflows-sol.json) +show: + +| Measure | Git control | With GHA | +| --- | ---: | ---: | +| Correct attempts | 4/4 | 4/4 | +| Total input and output tokens | 146,454 | 227,309 | +| Tokens per correct attempt | 36,614 | 56,827 | +| Mean elapsed time per attempt | 18.9 s | 18.4 s | + +In both branch reachability runs, the GHA agent read the 7,516-character +capability inventory, read 729 characters of command help, then received a +1,542-character cleanup result. The Git control used one shell command with +under 300 characters of output. In the upstream runs, the GHA agent read the +inventory and a roughly 3,500-character branch listing, while the Git control +used short focused queries. These output sizes and additional model turns +explain the higher observed token use; they do not prove how much each step +cost in isolation. Both conditions were equally accurate, and the sample is +too small to infer a general effect on quality or time. + +## Developer terminal effort + +The agent trace also provides a command-length proxy. After replacing the +absolute GHA binary path with `gha` and the temporary checkout path with `.`, +the four Git-control attempts issued six shell commands totalling 1,011 +characters. The four GHA attempts issued ten commands totalling 346 characters, +including capability discovery and help. Their four task commands alone +totalled 172 characters. This is an accounting of agent-generated command +strings; it is not a human typing study. + +A developer who knows the commands can use shorter Git paths than the agent +did. For the fixture, `gha branches cleanup --base eval/base` is 37 characters +and one command, while `git branch --merged eval/base` plus +`git branch --no-merged eval/base` total 61 characters and two commands. For +upstream inspection, `gha branches` is 12 characters and `git branch -vv` is +14. This suggests a terminal convenience for cleanup, but little typing +advantage for a simple branch listing. A human study would also measure time +to identify the right branch, use of shell history or completion, and mistakes +or rework after reading the output. + +## What the numbers mean + +`correct` requires every expected answer field to match. `valid_runs` excludes +failed Codex invocations, missing answers, runs without a successful inspection +command, control runs that invoked GHA, and GHA runs that skipped the selected +task workflow. +Success rate counts every scheduled run, including invalid and timed-out runs. +`total_tokens` sums Codex-reported input and output tokens over completed turns, +including retries. Cached input tokens are shown separately and are already +part of input tokens. `tokens_per_correct` divides all measured tokens from +scheduled runs, including wrong answers, by their correct answer count. If any +run lacks usage data, the metric is unavailable. Wall time includes the +entire agent invocation. + +The current two tasks test offline repository lookup using a generated +branch-history fixture. They are an initial measurement pilot, not +sufficient evidence of a general quality or token advantage. Expand the +held-out set to realistic PR review, +branch safety, and release preparation tasks before making a product claim. +Score code or judgment tasks with predeclared checks and blinded human review; +inspect failures where the checks and a valid solution disagree. Publish the +task list, pinned model and revision, repeated paired results, missing usage, +and limitations with any README summary. + +## Skill guidance ablation + +The [three-arm task set](../tools/evals/skill_ablation_tasks.json) tests how +much guidance the agent needs to discover GHA. Every arm has the same GHA +binary on `PATH` and the same pinned repository fixture. The `binary_only` +arm has neutral read-only instructions; `short_guide` adds a brief GHA usage +hint to `AGENTS.md`; `full_skill` installs the repository's complete +`skills/gha/SKILL.md` in the clone's discoverable `.agents/skills/gha` +directory and adds its bundled `AGENT-GUIDANCE.md` block to `AGENTS.md`. +The task prompt itself does not name GHA or forbid direct Git. +Codex runs with an isolated home and read-only sandbox. Each task runs twice +per arm, with arm order rotated on the second pass. + +The set covers combined branch reachability and cached upstream analysis, +branch creation and deletion previews, and a tag publication preview. The +write-intent tasks instruct the agent to use only dry-run commands. The +harness also rejects a GHA write command without `--dry-run` and checks that +Git refs are unchanged after every attempt. It records correctness, actual +GHA use, observed skill-file reads, tokens, elapsed time, and a proxy for +terminal effort: command count and normalized command-string characters. +These command strings are agent generated, not human keystrokes. + +Preview the plan, then add `--execute` to run model calls: + +```bash +python tools/evals/skill_ablation.py run \ + --tasks tools/evals/skill_ablation_tasks.json \ + --revision 16df66c472d5e63208f4a7786aa136d5c6fb4b79 \ + --gha bin/gha --skill skills/gha/SKILL.md \ + --model gpt-6-sol --repeats 2 \ + --out /tmp/gha-skill-ablation-1 + +python tools/evals/skill_ablation.py report \ + --input /tmp/gha-skill-ablation-1/runs.jsonl \ + --tasks tools/evals/skill_ablation_tasks.json \ + --public-output /tmp/gha-skill-ablation-public.json +``` + +The report export contains only whitelisted metrics. Keep the raw traces in +`/tmp` private; they can include repository contents and model prompts. +Interpret skill-loading counts as observed reads in the trace, since Codex +may expose a skill through discovery before a visible file-read command. + +### 27 September 2026 run + +The expanded run used `gpt-6-sol` at low reasoning, pinned commit +`16df66c472d5e63208f4a7786aa136d5c6fb4b79`, two repetitions per task +and arm, and GHA binary SHA-256 +`8f6d3786e3c53dc12261801d1d492bbe3d3104b4ec69b67f5649ddad898882f7`. +All 18 attempts were valid, reported token usage, and left local refs +unchanged. No trace contained a GHA write command without `--dry-run`. +The [sanitized run record](evaluation-results/2026-09-27-skill-ablation-sol.json) +contains per-attempt metrics and no answers or raw trace text. + +| Measure | Binary only | Short guide | Full skill | +| --- | ---: | ---: | ---: | +| Correct | 5/6 | 6/6 | 6/6 | +| Used GHA | 3/6 | 4/6 | 6/6 | +| Observed skill-file read | 0/6 | 0/6 | 6/6 | +| Input and output tokens | 424,554 | 325,434 | 379,920 | +| Tokens per correct answer | 84,911 | 54,239 | 63,320 | +| Mean seconds | 28.1 | 23.9 | 28.5 | +| Agent shell commands | 23 | 21 | 21 | +| Normalized command characters | 2,946 | 1,458 | 1,276 | + +The full skill reliably triggered GHA, but this did not yield the lowest +token use. It ran GHA for both branch-analysis attempts, where focused Git +queries were shorter: that task consumed 123,530 tokens with the full skill, +versus 57,375 with the short guide. All four write-preview attempts in both +guided arms used GHA and were correct. On branch create/delete previews, +the full skill consumed 136,812 tokens versus 143,379 with the short guide. +On tag previews it consumed 119,578 versus 124,680. These task totals include +all turns and discovery overhead. + +The initial exact-string grader marked one short-guide branch-analysis answer +wrong because it included a space after a comma in an otherwise correct +branch list. The grader now ignores spacing around comma separators, and the +saved answers were regraded without rerunning the model. The remaining wrong +binary-only answer came from a `git push --dry-run` tag preview that reported +the local effect as `unchanged` and origin as `create`; it did not provide +GHA's requested preflight effects. Both attempts remain in token and time +totals. Command characters strip Codex's shell-launch wrapper and count the +agent-generated shell body; they do not measure a person's typing time, +completion use, or error recovery. + +Six attempts per arm cannot establish a general quality or token advantage. +The task set uses a local Git fixture, so it does not test live PR review, +CI, provider availability, release preflights, or human terminal work. The +observed pattern supports trying compact install guidance for common tasks +while retaining the full skill for discoverability and workflows where its +extra instructions improve correctness or safety. A larger held-out set and +human terminal study are needed before changing the default installer. + +## All implemented capabilities: 27 September 2026 + +The follow-up used GHA's own `capabilities --format json` inventory as the +scope: 22 available entries, comprising 21 operational commands and the +inventory command itself. `dashboard` was reported unavailable and excluded. +The [direct capability probe](../tools/evals/capability_probe.py) exercised +all 22 against disposable local fixtures and public Raithlin/gha data. Every +write-capable probe used `--dry-run`. Twenty-one exited successfully. The +`update --dry-run` probe failed closed because the currently fetched published +skill does not declare `gha-required-capabilities`; it changed no state. All +22 probes left checkout refs, files, temporary home, and GHA configuration +unchanged. See the [sanitized direct results](evaluation-results/2026-09-27-capability-probe.json). + +The [eight agent tasks](../tools/evals/all_capability_tasks.json) group the +same 22 entries into agent setup, repository diagnosis, branch plans, tag +publication, PR inspection, PR creation preflight, release research, and +release publication preflight. This is command coverage by the direct probe +and task coverage by the agent comparison: a correct agent may use a different +command than the one mapped to its task. Each task ran twice per setup with +`gpt-6-sol` at low reasoning on pinned commit +`16df66c472d5e63208f4a7786aa136d5c6fb4b79`. The three setups were +binary only, short guide, and full installed skill plus guidance. Provider +tasks allowed unauthenticated reads of public GitHub data; agents had isolated +homes and disposable checkouts. All 48 attempts completed, used no +write-capable command without a dry run, and left refs and files unchanged. +The [public PR and release facts](evaluation-results/2026-09-27-provider-snapshot.json) +were checked again after the run. + +| Measure | Binary only | Short guide | Full skill | +| --- | ---: | ---: | ---: | +| Strict correct answers | 13/16 | 14/16 | 15/16 | +| Correct after count ambiguity review | 13/16 | 16/16 | 16/16 | +| Attempts using GHA | 13/16 | 16/16 | 16/16 | +| Total input and output tokens | 1,861,023 | 1,111,159 | 1,476,567 | +| Mean elapsed seconds | 36.5 | 26.6 | 29.6 | +| Agent shell commands | 116 | 99 | 111 | +| Normalized command characters | 16,059 | 4,723 | 5,175 | + +The strict grader expected `22` for available capabilities. Three setup +answers instead gave `21`, counting operational commands but excluding the +inventory command. Their other fields were correct. The public run file +retains the strict score; the reviewed row above accepts either interpretation +of that ambiguous prompt. The binary-only arm's three remaining wrong answers +were substantive: it marked a bounded PR list truncated in both repetitions +when it was complete, and marked a bounded release-notes inventory complete +when GHA reported truncation. The +[sanitized per-attempt results](evaluation-results/2026-09-27-all-capabilities-sol.json) +retain every token, timing, validity, and safety result. Two full-skill PR +preflight answers were initially flagged by a trace parser that treated help +text and a multiline `--body` as writes; the corrected parser rechecked the +saved traces and found both dry-run only. No model attempt was rerun for these +grading corrections. + +| Scenario, two attempts per setup | Binary tokens | Short-guide tokens | Full-skill tokens | +| --- | ---: | ---: | ---: | +| Agent setup and update preview | 167,413 | **113,191** | 150,488 | +| Repository diagnosis | 159,268 | **105,607** | 174,354 | +| Branch change plans | 197,190 | **129,072** | 138,311 | +| Tag publication plan | 219,003 | **92,169** | 135,986 | +| PR inspection | 342,657 | **188,792** | 225,322 | +| PR creation plan | 193,334 | **151,267** | 243,166 | +| Release research | 248,106 | **134,323** | 190,717 | +| Release publication plan | 334,052 | **196,738** | 218,223 | + +The clearest **quality** wins were bounded provider listings: explicit +`truncated` fields prevented the binary-only agent's three wrong completeness +claims. The largest **token** difference was the tag preview, where all arms +were correct but the short guide used 58% fewer tokens than binary only. The +short guide used the fewest tokens in every scenario and 40% fewer overall +than binary only, while matching the full skill's reviewed correctness. It +also used 25% fewer tokens than the full skill. On these complex tasks the +short guide already triggered GHA in all 16 attempts, so the full skill's +reliable invocation did not add a correctness advantage in this sample. The +binary-only arm used GHA in 13 attempts; this is a guidance comparison, not +an enforced GHA-versus-no-GHA comparison. + +The sample has one repository, two repetitions per scenario, task prompts +derived from GHA's command surface, and live public provider reads. Results +do not establish general quality, token, or developer keystroke savings. +Command characters are agent-generated shell text with Codex's launcher +removed; no human typing or completion behavior was measured. The direct +probe and [resumable agent harness](../tools/evals/skill_ablation.py) preserve +the exact task and binary hashes for later replication; private traces are +kept under the ignored `.eval-private/` directory. + +To repeat the sweep, first run `make test-eval` and build `bin/gha` from the +same source revision. Preview the agent plan by omitting `--execute`; adding +it starts model calls. If interrupted, run the same command with `--resume`. + +```bash +python tools/evals/capability_probe.py \ + --revision 16df66c472d5e63208f4a7786aa136d5c6fb4b79 \ + --gha bin/gha --out docs/evaluation-results/capability-probe-new.json + +python tools/evals/skill_ablation.py run \ + --tasks tools/evals/all_capability_tasks.json \ + --revision 16df66c472d5e63208f4a7786aa136d5c6fb4b79 \ + --gha bin/gha --skill skills/gha/SKILL.md --model gpt-6-sol \ + --repeats 2 --out .eval-private/all-capabilities-new --execute + +python tools/evals/skill_ablation.py report \ + --input .eval-private/all-capabilities-new/runs.jsonl \ + --tasks tools/evals/all_capability_tasks.json \ + --public-output docs/evaluation-results/all-capabilities-new.json +``` + +## Git/gh-only control: 28 September 2026 + +The earlier `binary_only` arm was not a raw control: GHA remained on `PATH` +and was used in 13 of 16 attempts. This follow-up compared the saved +`short_guide` GHA attempts with a separate control that had no GHA guidance, +skill, or executable on `PATH`. The control invoked no GHA command. It used +Git and source inspection for local tasks, and authenticated `gh` read calls +for public provider tasks. A preliminary unauthenticated run fell back to +direct public HTTP and is retained as [diagnostic data](evaluation-results/2026-09-28-raw-control-sol.json), +not counted in the git/gh-only provider comparison. + +The seven control tasks are exact copies of the corresponding questions and +expected answers from the [eight-scenario task set](../tools/evals/all_capability_tasks.json). +The GHA-only agent setup scenario was excluded: `capabilities`, `version`, +agent installation, and guidance update have no git/gh counterpart. Both arms +used `gpt-6-sol` at low reasoning, commit +`16df66c472d5e63208f4a7786aa136d5c6fb4b79`, and two repetitions per +scenario. The control ran on 28 September; the reused GHA arm ran on 27 +September. All 28 selected attempts were valid, answered correctly, left refs +and files unchanged, and had complete token usage. + +| Scenario, two attempts per arm | Git/gh tokens | Short-guide GHA tokens | GHA token change | +| --- | ---: | ---: | ---: | +| Repository diagnosis | 164,501 | 105,607 | −36% | +| Branch change plans | 257,572 | 129,072 | −50% | +| Tag publication plan | 241,314 | 92,169 | −62% | +| PR inspection | **151,130** | 188,792 | +25% | +| PR creation plan | 179,515 | 151,267 | −16% | +| Release research | 202,392 | 134,323 | −34% | +| Release publication plan | 238,274 | 196,738 | −17% | +| **Seven-scenario total** | 1,434,698 | **997,968** | **−30%** | + +Both arms scored 14/14. The GHA arm averaged 27.5 seconds per attempt versus +38.7 for git/gh. Agent-generated command strings totaled 4,250 characters +with GHA versus 13,802 for git/gh; these are not measured human keystrokes. +For the three questions less tied to GHA's output vocabulary—repository +diagnosis, PR inspection, and release research—the totals were 428,722 GHA +tokens and 518,023 git/gh tokens (17% fewer with GHA), with both arms 6/6 +correct. PR inspection favored `gh`, while the large tag-plan difference +favored GHA. The tag question asked for `planned` effect words from GHA's +dry-run contract, so that difference is not evidence of the same advantage on +an independently designed tag task. + +This is an exploratory comparison. The questions were designed around GHA's +implemented commands and several answer fields mirror its structured output. +The GHA attempts were reused rather than run concurrently with the control; +public GitHub state, service conditions, and model behavior could have changed. +The control had a token for authenticated `gh` reads while the earlier GHA +arm used unauthenticated public reads. The sample has one repository and two +repetitions per scenario. Equal correctness here does not establish a quality +benefit over git/gh, and the token difference does not establish a general +benefit across repositories or independently written tasks. + +The [sanitized combined comparison](evaluation-results/2026-09-28-git-gh-comparison.json) +contains per-scenario tokens, correctness, time, command count, and command +characters. Its sources are the [saved GHA attempts](evaluation-results/2026-09-27-all-capabilities-sol.json), +[local control attempts](evaluation-results/2026-09-28-raw-control-sol.json), +and [strict gh provider attempts](evaluation-results/2026-09-28-strict-gh-control-sol.json). +The [comparison script](../tools/evals/compare_raw_control.py) checks matching +task definitions, model, revision, repetitions, and control safety signals. +Raw Codex traces remain in the ignored `.eval-private/` directory. The strict +provider traces were checked for actual `gh` calls, direct HTTP clients, +GHA invocation, and accidental token disclosure. + +## Compact guidance check: 28 September 2026 + +Before changing the default installer guidance, a separate small Git +repository was generated from a pinned, single-commit source. Its +[source generator](../tools/evals/make_held_out_source.py) recreates commit +`e801cb80ec060f64fe899f970e921446ef37e438`; the harness adds a new local +branch graph in each disposable clone. The two retained +[held-out questions](../tools/evals/held_out_complex_tasks.json) ask which +topic tips are included in `stable` and whether a tracked topic is actually +in sync with its local origin-tracking ref. Neither prompt names GHA or its +commands. Every setup ran each task twice with `gpt-6-sol` at low reasoning. + +| Setup | Correct | Used GHA | Total tokens | +| --- | ---: | ---: | ---: | +| Short guide only | 4/4 | 0/4 | 142,254 | +| Former long guidance + detailed skill | 4/4 | 4/4 | 293,992 | +| Compact guidance + detailed skill | 4/4 | 0/4 | 142,581 | + +The third setup is the proposed default: it keeps the detailed skill +installed but uses the compact guide in the always-present instruction file. +It behaved like the short-guide-only arm on the local branch questions. These +questions were answerable with focused Git commands, so GHA non-use is not a +failure. A separate [PR and release check](../tools/evals/hybrid_provider_tasks.json) +with the third setup used GHA in all 4/4 attempts and answered all four +correctly, consuming 395,986 tokens. Those provider prompts were taken from +the earlier GHA-shaped sweep and test discovery, not independent quality. + +The original [three-task held-out file](../tools/evals/held_out_guidance_tasks.json) +also asked whether the working tree was clean. The harness itself adds an +untracked `AGENTS.md`, making its predeclared `yes` answer wrong. All four +agents answered `no`; the public file retains those strict failures. The +simple task is excluded from the table above because its expected answer was +incorrect. This correction applies equally to both setups and was made after +the run; no model answers were changed or rerun. The comparison remains a +small local pilot, with two branch tasks and two repetitions, rather than +proof that compact guidance will invoke GHA appropriately across every +repository and harness. + +The [short/full run data](evaluation-results/2026-09-28-held-out-guidance-sol.json), +[compact-plus-skill data](evaluation-results/2026-09-28-held-out-hybrid-sol.json), +and [provider check data](evaluation-results/2026-09-28-hybrid-provider-sol.json) +contain sanitized per-attempt tokens and safety metrics. The source generator +and task definitions allow replication; raw traces remain private in +`.eval-private/`. Based on this check, the installer still copies the detailed +skill for explicit use, while its bundled `AGENT-GUIDANCE.md` becomes a short +default instruction block. Existing user-modified skill files retain the +installer's preservation rules. diff --git a/docs/agent-efficiency-test-results.md b/docs/agent-efficiency-test-results.md new file mode 100644 index 0000000..905eff2 --- /dev/null +++ b/docs/agent-efficiency-test-results.md @@ -0,0 +1,128 @@ +# Agent efficiency test results + +A paired Codex evaluation ran two offline branch tasks twice each with +`gpt-6-sol` at low reasoning. Every attempt inspected the repository, and each +GHA attempt used its selected GHA workflow. + +| Measure | Git control | With GHA | +| --- | ---: | ---: | +| Correct attempts | 4/4 | 4/4 | +| Total input and output tokens | 146,454 | 227,309 | +| Mean elapsed time per attempt | 18.9 s | 18.4 s | + +GHA did not reduce tokens on these short tasks. Each fresh GHA session read the +capability inventory before its task command, adding context and a model turn. +This small test says nothing conclusive about larger workflows where GHA may +replace several Git, GitHub, and CI lookups. See the [method and trace diagnosis](agent-efficiency-evaluation.md) +and [per-attempt results](evaluation-results/2026-09-27-branch-workflows-sol.json). + +For terminal use, the goal is different. On the branch fixture, +`gha branches cleanup --base eval/base` is one 37-character command; the +corresponding `git branch --merged eval/base` and +`git branch --no-merged eval/base` total 61 characters across two commands. +For upstream inspection, `gha branches` (12 characters) and `git branch -vv` +(14 characters) are similar. These are command-length examples, not measured +human keystrokes or time savings. + +## Skill guidance experiment + +An expanded offline evaluation used the same GHA binary and pinned repository +in three setups: binary only, a short GHA guide, and the installed GHA skill +with its guidance block. Six attempts per setup covered combined branch +analysis, branch create/delete previews, and a tag publication preview. +All write-intent GHA commands used `--dry-run`; no Git refs changed. + +| Measure | Binary only | Short guide | Full skill | +| --- | ---: | ---: | ---: | +| Correct attempts | 5/6 | 6/6 | 6/6 | +| Attempts using GHA | 3/6 | 4/6 | 6/6 | +| Total input and output tokens | 424,554 | 325,434 | 379,920 | +| Mean elapsed time per attempt | 28.1 s | 23.9 s | 28.5 s | +| Agent command characters | 2,946 | 1,458 | 1,276 | + +The full skill reliably led the agent to use GHA, while the short guide had +the lowest token use with the same correctness in this small sample. The +short guide used GHA on all four write-preview attempts and used Git for the +two branch-analysis attempts. The full skill saved another 182 agent command +characters but consumed 54,486 more tokens than the short guide. The one +binary-only error came from a tag preview attempted with `git push --dry-run` +that did not answer the requested preflight fields. Command characters are a +proxy from agent traces, not human keystrokes. See the [method and task-level +analysis](agent-efficiency-evaluation.md) and [sanitized run +results](evaluation-results/2026-09-27-skill-ablation-sol.json). + +## All implemented capabilities + +A follow-up covered all 22 entries reported available by `gha capabilities` +(21 operational commands plus the inventory command) in eight workflow +scenarios. The unavailable `dashboard` was excluded. A direct probe exercised +each command; write-capable commands used `--dry-run`. Twenty-one probes +succeeded. `gha update --dry-run` refused a published skill that lacked its +required capability declaration. No probe changed checkout or temporary +agent state. + +The same eight scenarios ran twice with each guidance setup using `gpt-6-sol` +at low reasoning. Public PR and release tasks used unauthenticated GitHub +reads; agents worked in disposable checkouts. Every attempt completed without +changing refs or files. + +| Measure, 16 attempts per setup | Binary only | Short guide | Full skill | +| --- | ---: | ---: | ---: | +| Strict correct answers | 13 | 14 | 15 | +| Correct after resolving a count ambiguity | 13 | 16 | 16 | +| Attempts using GHA | 13 | 16 | 16 | +| Total input and output tokens | 1,861,023 | **1,111,159** | 1,476,567 | +| Agent command characters | 16,059 | **4,723** | 5,175 | + +Three guided setup answers counted 21 operational commands, while the strict +expected count was 22 including `capabilities`. The binary-only arm made +three substantive errors: it misread bounded-result truncation twice for PRs +and once for release notes. The short guide used 40% fewer tokens than binary +only and 25% fewer than the full skill, with the same reviewed correctness as +the full skill. Its clearest quality benefit appeared in PR and release +listings, where GHA explicitly reports whether results were truncated. Agent +command characters are a proxy, not measured developer keystrokes. + +See the [eight-scenario analysis](agent-efficiency-evaluation.md#all-implemented-capabilities-27-september-2026), +[per-attempt results](evaluation-results/2026-09-27-all-capabilities-sol.json), +and [direct command probe](evaluation-results/2026-09-27-capability-probe.json). + +## Git/gh-only control + +A follow-up ran the seven scenarios with a git/gh-only control and compared +them with the saved short-guide GHA attempts. GHA was absent from the control's +`PATH` and never invoked. Agent setup was excluded because GHA's inventory, +installer, and updater have no git/gh equivalent. Provider tasks used +authenticated, read-only `gh` calls; both arms used the same task questions, +model, pinned checkout, and two repetitions. + +| Measure, 14 attempts per arm | Git/gh only | Short guide + GHA | +| --- | ---: | ---: | +| Correct answers | 14 | 14 | +| Total input and output tokens | 1,434,698 | **997,968** | +| Mean elapsed seconds | 38.7 | 27.5 | +| Agent command characters | 13,802 | 4,250 | + +The GHA arm used **30% fewer tokens** with equal correctness in this sample. +For the three less GHA-specific questions (repository diagnosis, PR inspection, +and release research), it used **17% fewer**. Git/gh used **25% fewer tokens on +PR inspection**, while GHA's largest win was the tag preview, whose requested +effect words mirror GHA's output. The tasks were derived from GHA's command +surface, and the saved GHA arm ran a day earlier with unauthenticated public +reads; these results are exploratory, not a general efficiency claim. Command +characters are agent-generated text, not developer keystrokes. See the +[per-scenario comparison](agent-efficiency-evaluation.md#gitgh-only-control-28-september-2026) +and [sanitized result](evaluation-results/2026-09-28-git-gh-comparison.json). + +## Compact guidance check + +A held-out local repository tested two independently written branch questions +twice per setup. The short guide alone, detailed skill with its former long +guidance, and compact guidance with the detailed skill installed each answered +4/4 correctly. The first and third setups used Git for these simple local +checks; the former full setup used GHA on all four attempts. Their token totals +were 142,254, 293,992, and 142,581 respectively. In a separate PR and release +check, compact guidance with the skill installed used GHA on all four attempts +and answered 4/4 correctly. This supports keeping the detailed skill +available while making the compact block the installer default. See the +[held-out method and limitations](agent-efficiency-evaluation.md#compact-guidance-check-28-september-2026). diff --git a/docs/development.md b/docs/development.md index 847681e..bb9f758 100644 --- a/docs/development.md +++ b/docs/development.md @@ -5,6 +5,7 @@ - Go 1.26.5 or later - Make (optional, for using `makefile`) - Git +- Python 3 for the optional agent efficiency evaluation harness - [golangci-lint](https://golangci-lint.run/docs/welcome/install/) for local linting ## Commands @@ -19,6 +20,7 @@ go build ./... # Alternative build command make test # Run all tests go test ./... # Alternative test command make check # Mirror CI: dependencies, tidy check, lint, race tests, and coverage +make test-eval # Test the local evaluation harness without model calls # Code quality make fmt # Format code and tidy dependencies diff --git a/docs/evaluation-results/2026-09-27-all-capabilities-sol.json b/docs/evaluation-results/2026-09-27-all-capabilities-sol.json new file mode 100644 index 0000000..953ac9d --- /dev/null +++ b/docs/evaluation-results/2026-09-27-all-capabilities-sol.json @@ -0,0 +1,1088 @@ +{ + "schema_version": "v1", + "plan": { + "tasks": [ + "agent-setup", + "repository-diagnosis", + "branch-change-plans", + "tag-publication-plan", + "pull-request-inspection", + "pull-request-creation-plan", + "release-research", + "release-publication-plan" + ], + "tasks_sha256": "24b835c61ac28d7e5ab0daf80a7d999ad8cdab50fe960df89c35ab2bb75c37b2", + "revision": "16df66c472d5e63208f4a7786aa136d5c6fb4b79", + "model": "gpt-6-sol", + "repeats": 2, + "gha_sha256": "8f6d3786e3c53dc12261801d1d492bbe3d3104b4ec69b67f5649ddad898882f7", + "skill_sha256": "5a2d5fcc81aee0ff2e8a0f3078fec1761188893fe195d8129cb52496fb6347a2", + "guidance_sha256": "f7c968286d895b36eefc9b04d1d4eed25ac8294189da1947e2d445cf3947a7c1" + }, + "summary": { + "binary_only": { + "runs": 16, + "valid_runs": 16, + "correct": 13, + "success_rate": 0.8125, + "gha_used_runs": 13, + "skill_loaded_runs": 0, + "unsafe_gha_write_runs": 0, + "unsafe_write_attempt_runs": 0, + "refs_changed_runs": 0, + "tree_changed_runs": 0, + "usage_coverage": 16, + "total_tokens": 1861023, + "tokens_per_correct": 143155.61538461538, + "mean_seconds": 36.5138125, + "total_command_count": 116, + "total_command_chars": 16059 + }, + "short_guide": { + "runs": 16, + "valid_runs": 16, + "correct": 14, + "success_rate": 0.875, + "gha_used_runs": 16, + "skill_loaded_runs": 0, + "unsafe_gha_write_runs": 0, + "unsafe_write_attempt_runs": 0, + "refs_changed_runs": 0, + "tree_changed_runs": 0, + "usage_coverage": 16, + "total_tokens": 1111159, + "tokens_per_correct": 79368.5, + "mean_seconds": 26.554125, + "total_command_count": 99, + "total_command_chars": 4723 + }, + "full_skill": { + "runs": 16, + "valid_runs": 16, + "correct": 15, + "success_rate": 0.9375, + "gha_used_runs": 16, + "skill_loaded_runs": 16, + "unsafe_gha_write_runs": 0, + "unsafe_write_attempt_runs": 0, + "refs_changed_runs": 0, + "tree_changed_runs": 0, + "usage_coverage": 16, + "total_tokens": 1476567, + "tokens_per_correct": 98437.8, + "mean_seconds": 29.581625, + "total_command_count": 111, + "total_command_chars": 5175 + } + }, + "runs": [ + { + "task_id": "agent-setup", + "repetition": 0, + "condition": "binary_only", + "valid": true, + "correct": true, + "gha_used": true, + "skill_loaded": false, + "unsafe_gha_write": false, + "unsafe_write_attempt": false, + "refs_unchanged": true, + "tree_unchanged": true, + "inspected_repository": true, + "command_count": 17, + "command_chars": 647, + "seconds": 33.484, + "input_tokens": 83706, + "cached_input_tokens": 63232, + "output_tokens": 728, + "total_tokens": 84434 + }, + { + "task_id": "agent-setup", + "repetition": 0, + "condition": "short_guide", + "valid": true, + "correct": false, + "gha_used": true, + "skill_loaded": false, + "unsafe_gha_write": false, + "unsafe_write_attempt": false, + "refs_unchanged": true, + "tree_unchanged": true, + "inspected_repository": true, + "command_count": 9, + "command_chars": 270, + "seconds": 21.142, + "input_tokens": 65087, + "cached_input_tokens": 46848, + "output_tokens": 331, + "total_tokens": 65418 + }, + { + "task_id": "agent-setup", + "repetition": 0, + "condition": "full_skill", + "valid": true, + "correct": false, + "gha_used": true, + "skill_loaded": true, + "unsafe_gha_write": false, + "unsafe_write_attempt": false, + "refs_unchanged": true, + "tree_unchanged": true, + "inspected_repository": true, + "command_count": 9, + "command_chars": 348, + "seconds": 27.403, + "input_tokens": 74736, + "cached_input_tokens": 45184, + "output_tokens": 478, + "total_tokens": 75214 + }, + { + "task_id": "repository-diagnosis", + "repetition": 0, + "condition": "binary_only", + "valid": true, + "correct": true, + "gha_used": false, + "skill_loaded": false, + "unsafe_gha_write": false, + "unsafe_write_attempt": false, + "refs_unchanged": true, + "tree_unchanged": true, + "inspected_repository": true, + "command_count": 3, + "command_chars": 790, + "seconds": 24.345, + "input_tokens": 71039, + "cached_input_tokens": 48768, + "output_tokens": 466, + "total_tokens": 71505 + }, + { + "task_id": "repository-diagnosis", + "repetition": 0, + "condition": "short_guide", + "valid": true, + "correct": true, + "gha_used": true, + "skill_loaded": false, + "unsafe_gha_write": false, + "unsafe_write_attempt": false, + "refs_unchanged": true, + "tree_unchanged": true, + "inspected_repository": true, + "command_count": 6, + "command_chars": 194, + "seconds": 18.259, + "input_tokens": 52461, + "cached_input_tokens": 29824, + "output_tokens": 251, + "total_tokens": 52712 + }, + { + "task_id": "repository-diagnosis", + "repetition": 0, + "condition": "full_skill", + "valid": true, + "correct": true, + "gha_used": true, + "skill_loaded": true, + "unsafe_gha_write": false, + "unsafe_write_attempt": false, + "refs_unchanged": true, + "tree_unchanged": true, + "inspected_repository": true, + "command_count": 7, + "command_chars": 225, + "seconds": 33.872, + "input_tokens": 74299, + "cached_input_tokens": 49024, + "output_tokens": 302, + "total_tokens": 74601 + }, + { + "task_id": "branch-change-plans", + "repetition": 0, + "condition": "binary_only", + "valid": true, + "correct": true, + "gha_used": true, + "skill_loaded": false, + "unsafe_gha_write": false, + "unsafe_write_attempt": false, + "refs_unchanged": true, + "tree_unchanged": true, + "inspected_repository": true, + "command_count": 7, + "command_chars": 751, + "seconds": 31.969, + "input_tokens": 82676, + "cached_input_tokens": 63360, + "output_tokens": 459, + "total_tokens": 83135 + }, + { + "task_id": "branch-change-plans", + "repetition": 0, + "condition": "short_guide", + "valid": true, + "correct": true, + "gha_used": true, + "skill_loaded": false, + "unsafe_gha_write": false, + "unsafe_write_attempt": false, + "refs_unchanged": true, + "tree_unchanged": true, + "inspected_repository": true, + "command_count": 9, + "command_chars": 396, + "seconds": 22.772, + "input_tokens": 64710, + "cached_input_tokens": 58624, + "output_tokens": 335, + "total_tokens": 65045 + }, + { + "task_id": "branch-change-plans", + "repetition": 0, + "condition": "full_skill", + "valid": true, + "correct": true, + "gha_used": true, + "skill_loaded": true, + "unsafe_gha_write": false, + "unsafe_write_attempt": false, + "refs_unchanged": true, + "tree_unchanged": true, + "inspected_repository": true, + "command_count": 6, + "command_chars": 330, + "seconds": 22.558, + "input_tokens": 68543, + "cached_input_tokens": 48640, + "output_tokens": 263, + "total_tokens": 68806 + }, + { + "task_id": "tag-publication-plan", + "repetition": 0, + "condition": "binary_only", + "valid": true, + "correct": true, + "gha_used": true, + "skill_loaded": false, + "unsafe_gha_write": false, + "unsafe_write_attempt": false, + "refs_unchanged": true, + "tree_unchanged": true, + "inspected_repository": true, + "command_count": 5, + "command_chars": 555, + "seconds": 28.332, + "input_tokens": 118383, + "cached_input_tokens": 73216, + "output_tokens": 505, + "total_tokens": 118888 + }, + { + "task_id": "tag-publication-plan", + "repetition": 0, + "condition": "short_guide", + "valid": true, + "correct": true, + "gha_used": true, + "skill_loaded": false, + "unsafe_gha_write": false, + "unsafe_write_attempt": false, + "refs_unchanged": true, + "tree_unchanged": true, + "inspected_repository": true, + "command_count": 2, + "command_chars": 79, + "seconds": 18.084, + "input_tokens": 45856, + "cached_input_tokens": 29568, + "output_tokens": 134, + "total_tokens": 45990 + }, + { + "task_id": "tag-publication-plan", + "repetition": 0, + "condition": "full_skill", + "valid": true, + "correct": true, + "gha_used": true, + "skill_loaded": true, + "unsafe_gha_write": false, + "unsafe_write_attempt": false, + "refs_unchanged": true, + "tree_unchanged": true, + "inspected_repository": true, + "command_count": 3, + "command_chars": 124, + "seconds": 18.787, + "input_tokens": 67474, + "cached_input_tokens": 48256, + "output_tokens": 210, + "total_tokens": 67684 + }, + { + "task_id": "pull-request-inspection", + "repetition": 0, + "condition": "binary_only", + "valid": true, + "correct": false, + "gha_used": true, + "skill_loaded": false, + "unsafe_gha_write": false, + "unsafe_write_attempt": false, + "refs_unchanged": true, + "tree_unchanged": true, + "inspected_repository": true, + "command_count": 6, + "command_chars": 2809, + "seconds": 51.738, + "input_tokens": 117601, + "cached_input_tokens": 80768, + "output_tokens": 1316, + "total_tokens": 118917 + }, + { + "task_id": "pull-request-inspection", + "repetition": 0, + "condition": "short_guide", + "valid": true, + "correct": true, + "gha_used": true, + "skill_loaded": false, + "unsafe_gha_write": false, + "unsafe_write_attempt": false, + "refs_unchanged": true, + "tree_unchanged": true, + "inspected_repository": true, + "command_count": 6, + "command_chars": 265, + "seconds": 31.328, + "input_tokens": 93293, + "cached_input_tokens": 64128, + "output_tokens": 520, + "total_tokens": 93813 + }, + { + "task_id": "pull-request-inspection", + "repetition": 0, + "condition": "full_skill", + "valid": true, + "correct": true, + "gha_used": true, + "skill_loaded": true, + "unsafe_gha_write": false, + "unsafe_write_attempt": false, + "refs_unchanged": true, + "tree_unchanged": true, + "inspected_repository": true, + "command_count": 6, + "command_chars": 283, + "seconds": 26.508, + "input_tokens": 105683, + "cached_input_tokens": 73472, + "output_tokens": 433, + "total_tokens": 106116 + }, + { + "task_id": "pull-request-creation-plan", + "repetition": 0, + "condition": "binary_only", + "valid": true, + "correct": true, + "gha_used": true, + "skill_loaded": false, + "unsafe_gha_write": false, + "unsafe_write_attempt": false, + "refs_unchanged": true, + "tree_unchanged": true, + "inspected_repository": true, + "command_count": 7, + "command_chars": 918, + "seconds": 37.032, + "input_tokens": 75877, + "cached_input_tokens": 59264, + "output_tokens": 595, + "total_tokens": 76472 + }, + { + "task_id": "pull-request-creation-plan", + "repetition": 0, + "condition": "short_guide", + "valid": true, + "correct": true, + "gha_used": true, + "skill_loaded": false, + "unsafe_gha_write": false, + "unsafe_write_attempt": false, + "refs_unchanged": true, + "tree_unchanged": true, + "inspected_repository": true, + "command_count": 7, + "command_chars": 491, + "seconds": 31.43, + "input_tokens": 85535, + "cached_input_tokens": 66048, + "output_tokens": 485, + "total_tokens": 86020 + }, + { + "task_id": "pull-request-creation-plan", + "repetition": 0, + "condition": "full_skill", + "valid": true, + "correct": true, + "gha_used": true, + "skill_loaded": true, + "unsafe_gha_write": false, + "unsafe_write_attempt": false, + "refs_unchanged": true, + "tree_unchanged": true, + "inspected_repository": true, + "command_count": 8, + "command_chars": 447, + "seconds": 60.716, + "input_tokens": 130092, + "cached_input_tokens": 108288, + "output_tokens": 443, + "total_tokens": 130535 + }, + { + "task_id": "release-research", + "repetition": 0, + "condition": "binary_only", + "valid": true, + "correct": true, + "gha_used": true, + "skill_loaded": false, + "unsafe_gha_write": false, + "unsafe_write_attempt": false, + "refs_unchanged": true, + "tree_unchanged": true, + "inspected_repository": true, + "command_count": 8, + "command_chars": 1395, + "seconds": 42.714, + "input_tokens": 201094, + "cached_input_tokens": 172928, + "output_tokens": 883, + "total_tokens": 201977 + }, + { + "task_id": "release-research", + "repetition": 0, + "condition": "short_guide", + "valid": true, + "correct": true, + "gha_used": true, + "skill_loaded": false, + "unsafe_gha_write": false, + "unsafe_write_attempt": false, + "refs_unchanged": true, + "tree_unchanged": true, + "inspected_repository": true, + "command_count": 7, + "command_chars": 264, + "seconds": 24.603, + "input_tokens": 66917, + "cached_input_tokens": 46848, + "output_tokens": 331, + "total_tokens": 67248 + }, + { + "task_id": "release-research", + "repetition": 0, + "condition": "full_skill", + "valid": true, + "correct": true, + "gha_used": true, + "skill_loaded": true, + "unsafe_gha_write": false, + "unsafe_write_attempt": false, + "refs_unchanged": true, + "tree_unchanged": true, + "inspected_repository": true, + "command_count": 7, + "command_chars": 277, + "seconds": 26.567, + "input_tokens": 76828, + "cached_input_tokens": 53504, + "output_tokens": 503, + "total_tokens": 77331 + }, + { + "task_id": "release-publication-plan", + "repetition": 0, + "condition": "binary_only", + "valid": true, + "correct": true, + "gha_used": true, + "skill_loaded": false, + "unsafe_gha_write": false, + "unsafe_write_attempt": false, + "refs_unchanged": true, + "tree_unchanged": true, + "inspected_repository": true, + "command_count": 8, + "command_chars": 1500, + "seconds": 58.446, + "input_tokens": 195243, + "cached_input_tokens": 169600, + "output_tokens": 1196, + "total_tokens": 196439 + }, + { + "task_id": "release-publication-plan", + "repetition": 0, + "condition": "short_guide", + "valid": true, + "correct": true, + "gha_used": true, + "skill_loaded": false, + "unsafe_gha_write": false, + "unsafe_write_attempt": false, + "refs_unchanged": true, + "tree_unchanged": true, + "inspected_repository": true, + "command_count": 5, + "command_chars": 534, + "seconds": 32.371, + "input_tokens": 97898, + "cached_input_tokens": 62976, + "output_tokens": 534, + "total_tokens": 98432 + }, + { + "task_id": "release-publication-plan", + "repetition": 0, + "condition": "full_skill", + "valid": true, + "correct": true, + "gha_used": true, + "skill_loaded": true, + "unsafe_gha_write": false, + "unsafe_write_attempt": false, + "refs_unchanged": true, + "tree_unchanged": true, + "inspected_repository": true, + "command_count": 9, + "command_chars": 392, + "seconds": 32.351, + "input_tokens": 107734, + "cached_input_tokens": 91136, + "output_tokens": 602, + "total_tokens": 108336 + }, + { + "task_id": "agent-setup", + "repetition": 1, + "condition": "short_guide", + "valid": true, + "correct": false, + "gha_used": true, + "skill_loaded": false, + "unsafe_gha_write": false, + "unsafe_write_attempt": false, + "refs_unchanged": true, + "tree_unchanged": true, + "inspected_repository": true, + "command_count": 6, + "command_chars": 203, + "seconds": 18.118, + "input_tokens": 47557, + "cached_input_tokens": 30080, + "output_tokens": 216, + "total_tokens": 47773 + }, + { + "task_id": "agent-setup", + "repetition": 1, + "condition": "full_skill", + "valid": true, + "correct": true, + "gha_used": true, + "skill_loaded": true, + "unsafe_gha_write": false, + "unsafe_write_attempt": false, + "refs_unchanged": true, + "tree_unchanged": true, + "inspected_repository": true, + "command_count": 10, + "command_chars": 441, + "seconds": 23.508, + "input_tokens": 74827, + "cached_input_tokens": 53504, + "output_tokens": 447, + "total_tokens": 75274 + }, + { + "task_id": "agent-setup", + "repetition": 1, + "condition": "binary_only", + "valid": true, + "correct": true, + "gha_used": true, + "skill_loaded": false, + "unsafe_gha_write": false, + "unsafe_write_attempt": false, + "refs_unchanged": true, + "tree_unchanged": true, + "inspected_repository": true, + "command_count": 10, + "command_chars": 377, + "seconds": 32.079, + "input_tokens": 82360, + "cached_input_tokens": 48000, + "output_tokens": 619, + "total_tokens": 82979 + }, + { + "task_id": "repository-diagnosis", + "repetition": 1, + "condition": "short_guide", + "valid": true, + "correct": true, + "gha_used": true, + "skill_loaded": false, + "unsafe_gha_write": false, + "unsafe_write_attempt": false, + "refs_unchanged": true, + "tree_unchanged": true, + "inspected_repository": true, + "command_count": 5, + "command_chars": 272, + "seconds": 20.023, + "input_tokens": 52606, + "cached_input_tokens": 29824, + "output_tokens": 289, + "total_tokens": 52895 + }, + { + "task_id": "repository-diagnosis", + "repetition": 1, + "condition": "full_skill", + "valid": true, + "correct": true, + "gha_used": true, + "skill_loaded": true, + "unsafe_gha_write": false, + "unsafe_write_attempt": false, + "refs_unchanged": true, + "tree_unchanged": true, + "inspected_repository": true, + "command_count": 8, + "command_chars": 283, + "seconds": 27.957, + "input_tokens": 99286, + "cached_input_tokens": 73600, + "output_tokens": 467, + "total_tokens": 99753 + }, + { + "task_id": "repository-diagnosis", + "repetition": 1, + "condition": "binary_only", + "valid": true, + "correct": true, + "gha_used": false, + "skill_loaded": false, + "unsafe_gha_write": false, + "unsafe_write_attempt": false, + "refs_unchanged": true, + "tree_unchanged": true, + "inspected_repository": true, + "command_count": 4, + "command_chars": 861, + "seconds": 26.296, + "input_tokens": 87250, + "cached_input_tokens": 65408, + "output_tokens": 513, + "total_tokens": 87763 + }, + { + "task_id": "branch-change-plans", + "repetition": 1, + "condition": "short_guide", + "valid": true, + "correct": true, + "gha_used": true, + "skill_loaded": false, + "unsafe_gha_write": false, + "unsafe_write_attempt": false, + "refs_unchanged": true, + "tree_unchanged": true, + "inspected_repository": true, + "command_count": 6, + "command_chars": 323, + "seconds": 21.049, + "input_tokens": 63769, + "cached_input_tokens": 30080, + "output_tokens": 258, + "total_tokens": 64027 + }, + { + "task_id": "branch-change-plans", + "repetition": 1, + "condition": "full_skill", + "valid": true, + "correct": true, + "gha_used": true, + "skill_loaded": true, + "unsafe_gha_write": false, + "unsafe_write_attempt": false, + "refs_unchanged": true, + "tree_unchanged": true, + "inspected_repository": true, + "command_count": 6, + "command_chars": 330, + "seconds": 21.488, + "input_tokens": 69259, + "cached_input_tokens": 49408, + "output_tokens": 246, + "total_tokens": 69505 + }, + { + "task_id": "branch-change-plans", + "repetition": 1, + "condition": "binary_only", + "valid": true, + "correct": true, + "gha_used": true, + "skill_loaded": false, + "unsafe_gha_write": false, + "unsafe_write_attempt": false, + "refs_unchanged": true, + "tree_unchanged": true, + "inspected_repository": true, + "command_count": 8, + "command_chars": 736, + "seconds": 38.991, + "input_tokens": 113471, + "cached_input_tokens": 72576, + "output_tokens": 584, + "total_tokens": 114055 + }, + { + "task_id": "tag-publication-plan", + "repetition": 1, + "condition": "short_guide", + "valid": true, + "correct": true, + "gha_used": true, + "skill_loaded": false, + "unsafe_gha_write": false, + "unsafe_write_attempt": false, + "refs_unchanged": true, + "tree_unchanged": true, + "inspected_repository": true, + "command_count": 2, + "command_chars": 79, + "seconds": 62.398, + "input_tokens": 46033, + "cached_input_tokens": 29568, + "output_tokens": 146, + "total_tokens": 46179 + }, + { + "task_id": "tag-publication-plan", + "repetition": 1, + "condition": "full_skill", + "valid": true, + "correct": true, + "gha_used": true, + "skill_loaded": true, + "unsafe_gha_write": false, + "unsafe_write_attempt": false, + "refs_unchanged": true, + "tree_unchanged": true, + "inspected_repository": true, + "command_count": 3, + "command_chars": 124, + "seconds": 17.833, + "input_tokens": 68086, + "cached_input_tokens": 48896, + "output_tokens": 216, + "total_tokens": 68302 + }, + { + "task_id": "tag-publication-plan", + "repetition": 1, + "condition": "binary_only", + "valid": true, + "correct": true, + "gha_used": true, + "skill_loaded": false, + "unsafe_gha_write": false, + "unsafe_write_attempt": false, + "refs_unchanged": true, + "tree_unchanged": true, + "inspected_repository": true, + "command_count": 5, + "command_chars": 648, + "seconds": 32.254, + "input_tokens": 99534, + "cached_input_tokens": 91264, + "output_tokens": 581, + "total_tokens": 100115 + }, + { + "task_id": "pull-request-inspection", + "repetition": 1, + "condition": "short_guide", + "valid": true, + "correct": true, + "gha_used": true, + "skill_loaded": false, + "unsafe_gha_write": false, + "unsafe_write_attempt": false, + "refs_unchanged": true, + "tree_unchanged": true, + "inspected_repository": true, + "command_count": 6, + "command_chars": 294, + "seconds": 26.125, + "input_tokens": 94610, + "cached_input_tokens": 65152, + "output_tokens": 369, + "total_tokens": 94979 + }, + { + "task_id": "pull-request-inspection", + "repetition": 1, + "condition": "full_skill", + "valid": true, + "correct": true, + "gha_used": true, + "skill_loaded": true, + "unsafe_gha_write": false, + "unsafe_write_attempt": false, + "refs_unchanged": true, + "tree_unchanged": true, + "inspected_repository": true, + "command_count": 6, + "command_chars": 283, + "seconds": 31.736, + "input_tokens": 118720, + "cached_input_tokens": 88576, + "output_tokens": 486, + "total_tokens": 119206 + }, + { + "task_id": "pull-request-inspection", + "repetition": 1, + "condition": "binary_only", + "valid": true, + "correct": false, + "gha_used": true, + "skill_loaded": false, + "unsafe_gha_write": false, + "unsafe_write_attempt": false, + "refs_unchanged": true, + "tree_unchanged": true, + "inspected_repository": true, + "command_count": 12, + "command_chars": 1424, + "seconds": 49.249, + "input_tokens": 222641, + "cached_input_tokens": 195712, + "output_tokens": 1099, + "total_tokens": 223740 + }, + { + "task_id": "pull-request-creation-plan", + "repetition": 1, + "condition": "short_guide", + "valid": true, + "correct": true, + "gha_used": true, + "skill_loaded": false, + "unsafe_gha_write": false, + "unsafe_write_attempt": false, + "refs_unchanged": true, + "tree_unchanged": true, + "inspected_repository": true, + "command_count": 9, + "command_chars": 374, + "seconds": 21.33, + "input_tokens": 64912, + "cached_input_tokens": 46592, + "output_tokens": 335, + "total_tokens": 65247 + }, + { + "task_id": "pull-request-creation-plan", + "repetition": 1, + "condition": "full_skill", + "valid": true, + "correct": true, + "gha_used": true, + "skill_loaded": true, + "unsafe_gha_write": false, + "unsafe_write_attempt": false, + "refs_unchanged": true, + "tree_unchanged": true, + "inspected_repository": true, + "command_count": 8, + "command_chars": 546, + "seconds": 34.371, + "input_tokens": 111989, + "cached_input_tokens": 90752, + "output_tokens": 642, + "total_tokens": 112631 + }, + { + "task_id": "pull-request-creation-plan", + "repetition": 1, + "condition": "binary_only", + "valid": true, + "correct": true, + "gha_used": true, + "skill_loaded": false, + "unsafe_gha_write": false, + "unsafe_write_attempt": false, + "refs_unchanged": true, + "tree_unchanged": true, + "inspected_repository": true, + "command_count": 6, + "command_chars": 1323, + "seconds": 40.94, + "input_tokens": 116110, + "cached_input_tokens": 77568, + "output_tokens": 752, + "total_tokens": 116862 + }, + { + "task_id": "release-research", + "repetition": 1, + "condition": "short_guide", + "valid": true, + "correct": true, + "gha_used": true, + "skill_loaded": false, + "unsafe_gha_write": false, + "unsafe_write_attempt": false, + "refs_unchanged": true, + "tree_unchanged": true, + "inspected_repository": true, + "command_count": 6, + "command_chars": 246, + "seconds": 24.695, + "input_tokens": 66674, + "cached_input_tokens": 58496, + "output_tokens": 401, + "total_tokens": 67075 + }, + { + "task_id": "release-research", + "repetition": 1, + "condition": "full_skill", + "valid": true, + "correct": true, + "gha_used": true, + "skill_loaded": true, + "unsafe_gha_write": false, + "unsafe_write_attempt": false, + "refs_unchanged": true, + "tree_unchanged": true, + "inspected_repository": true, + "command_count": 8, + "command_chars": 377, + "seconds": 32.262, + "input_tokens": 112802, + "cached_input_tokens": 89088, + "output_tokens": 584, + "total_tokens": 113386 + }, + { + "task_id": "release-research", + "repetition": 1, + "condition": "binary_only", + "valid": true, + "correct": false, + "gha_used": false, + "skill_loaded": false, + "unsafe_gha_write": false, + "unsafe_write_attempt": false, + "refs_unchanged": true, + "tree_unchanged": true, + "inspected_repository": true, + "command_count": 5, + "command_chars": 523, + "seconds": 25.318, + "input_tokens": 45554, + "cached_input_tokens": 28288, + "output_tokens": 575, + "total_tokens": 46129 + }, + { + "task_id": "release-publication-plan", + "repetition": 1, + "condition": "short_guide", + "valid": true, + "correct": true, + "gha_used": true, + "skill_loaded": false, + "unsafe_gha_write": false, + "unsafe_write_attempt": false, + "refs_unchanged": true, + "tree_unchanged": true, + "inspected_repository": true, + "command_count": 8, + "command_chars": 439, + "seconds": 31.139, + "input_tokens": 97804, + "cached_input_tokens": 79360, + "output_tokens": 502, + "total_tokens": 98306 + }, + { + "task_id": "release-publication-plan", + "repetition": 1, + "condition": "full_skill", + "valid": true, + "correct": true, + "gha_used": true, + "skill_loaded": true, + "unsafe_gha_write": false, + "unsafe_write_attempt": false, + "refs_unchanged": true, + "tree_unchanged": true, + "inspected_repository": true, + "command_count": 7, + "command_chars": 365, + "seconds": 35.389, + "input_tokens": 109146, + "cached_input_tokens": 88192, + "output_tokens": 741, + "total_tokens": 109887 + }, + { + "task_id": "release-publication-plan", + "repetition": 1, + "condition": "binary_only", + "valid": true, + "correct": true, + "gha_used": true, + "skill_loaded": false, + "unsafe_gha_write": false, + "unsafe_write_attempt": false, + "refs_unchanged": true, + "tree_unchanged": true, + "inspected_repository": true, + "command_count": 5, + "command_chars": 802, + "seconds": 31.034, + "input_tokens": 137010, + "cached_input_tokens": 105984, + "output_tokens": 603, + "total_tokens": 137613 + } + ] +} diff --git a/docs/evaluation-results/2026-09-27-branch-workflows-sol.json b/docs/evaluation-results/2026-09-27-branch-workflows-sol.json new file mode 100644 index 0000000..90b2423 --- /dev/null +++ b/docs/evaluation-results/2026-09-27-branch-workflows-sol.json @@ -0,0 +1,168 @@ +{ + "schema_version": "v1", + "plan": { + "tasks": [ + "branch-reachability", + "cached-upstream" + ], + "revision": "16df66c472d5e63208f4a7786aa136d5c6fb4b79", + "model": "gpt-6-sol", + "repeats": 2, + "gha_sha256": "8f6d3786e3c53dc12261801d1d492bbe3d3104b4ec69b67f5649ddad898882f7" + }, + "summary": { + "control": { + "runs": 4, + "valid_runs": 4, + "correct": 4, + "success_rate": 1.0, + "gha_used_runs": 0, + "gha_workflow_used_runs": 0, + "usage_coverage": 4, + "total_tokens": 146454, + "tokens_per_correct": 36613.5, + "mean_seconds": 18.94275 + }, + "gha": { + "runs": 4, + "valid_runs": 4, + "correct": 4, + "success_rate": 1.0, + "gha_used_runs": 4, + "gha_workflow_used_runs": 4, + "usage_coverage": 4, + "total_tokens": 227309, + "tokens_per_correct": 56827.25, + "mean_seconds": 18.426750000000002 + }, + "paired": { + "both_correct": 4, + "median_tokens_when_both_correct": { + "control": 36574.0, + "gha": 56865.0 + } + } + }, + "runs": [ + { + "task_id": "branch-reachability", + "repetition": 0, + "condition": "control", + "valid": true, + "correct": true, + "gha_used": false, + "gha_workflow_used": false, + "inspected_repository": true, + "seconds": 13.351, + "input_tokens": 29126, + "cached_input_tokens": 14208, + "output_tokens": 169, + "total_tokens": 29295 + }, + { + "task_id": "branch-reachability", + "repetition": 0, + "condition": "gha", + "valid": true, + "correct": true, + "gha_used": true, + "gha_workflow_used": true, + "inspected_repository": true, + "seconds": 19.739, + "input_tokens": 64750, + "cached_input_tokens": 30592, + "output_tokens": 248, + "total_tokens": 64998 + }, + { + "task_id": "cached-upstream", + "repetition": 0, + "condition": "control", + "valid": true, + "correct": true, + "gha_used": false, + "gha_workflow_used": false, + "inspected_repository": true, + "seconds": 19.356, + "input_tokens": 43678, + "cached_input_tokens": 28800, + "output_tokens": 175, + "total_tokens": 43853 + }, + { + "task_id": "cached-upstream", + "repetition": 0, + "condition": "gha", + "valid": true, + "correct": true, + "gha_used": true, + "gha_workflow_used": true, + "inspected_repository": true, + "seconds": 15.185, + "input_tokens": 48609, + "cached_input_tokens": 14336, + "output_tokens": 147, + "total_tokens": 48756 + }, + { + "task_id": "branch-reachability", + "repetition": 1, + "condition": "gha", + "valid": true, + "correct": true, + "gha_used": true, + "gha_workflow_used": true, + "inspected_repository": true, + "seconds": 18.573, + "input_tokens": 64740, + "cached_input_tokens": 47104, + "output_tokens": 234, + "total_tokens": 64974 + }, + { + "task_id": "branch-reachability", + "repetition": 1, + "condition": "control", + "valid": true, + "correct": true, + "gha_used": false, + "gha_workflow_used": false, + "inspected_repository": true, + "seconds": 14.481, + "input_tokens": 29073, + "cached_input_tokens": 25984, + "output_tokens": 165, + "total_tokens": 29238 + }, + { + "task_id": "cached-upstream", + "repetition": 1, + "condition": "gha", + "valid": true, + "correct": true, + "gha_used": true, + "gha_workflow_used": true, + "inspected_repository": true, + "seconds": 20.21, + "input_tokens": 48436, + "cached_input_tokens": 42112, + "output_tokens": 145, + "total_tokens": 48581 + }, + { + "task_id": "cached-upstream", + "repetition": 1, + "condition": "control", + "valid": true, + "correct": true, + "gha_used": false, + "gha_workflow_used": false, + "inspected_repository": true, + "seconds": 28.583, + "input_tokens": 43825, + "cached_input_tokens": 40576, + "output_tokens": 243, + "total_tokens": 44068 + } + ] +} diff --git a/docs/evaluation-results/2026-09-27-capability-probe.json b/docs/evaluation-results/2026-09-27-capability-probe.json new file mode 100644 index 0000000..1002cef --- /dev/null +++ b/docs/evaluation-results/2026-09-27-capability-probe.json @@ -0,0 +1,296 @@ +{ + "schema_version": "v1", + "captured_at_utc": "2026-09-27T21:19:56.155535+00:00", + "revision": "16df66c472d5e63208f4a7786aa136d5c6fb4b79", + "gha_sha256": "8f6d3786e3c53dc12261801d1d492bbe3d3104b4ec69b67f5649ddad898882f7", + "available_count": 22, + "runs": [ + { + "capability": "capabilities", + "fixture": "local", + "write_intent": false, + "dry_run": false, + "exit_code": 0, + "json_output": true, + "error_code": null, + "refs_unchanged": true, + "tree_unchanged": true, + "home_unchanged": true, + "config_unchanged": true + }, + { + "capability": "version", + "fixture": "local", + "write_intent": false, + "dry_run": false, + "exit_code": 0, + "json_output": true, + "error_code": null, + "refs_unchanged": true, + "tree_unchanged": true, + "home_unchanged": true, + "config_unchanged": true + }, + { + "capability": "agent list", + "fixture": "local", + "write_intent": false, + "dry_run": false, + "exit_code": 0, + "json_output": true, + "error_code": null, + "refs_unchanged": true, + "tree_unchanged": true, + "home_unchanged": true, + "config_unchanged": true + }, + { + "capability": "agent install", + "fixture": "local", + "write_intent": true, + "dry_run": true, + "exit_code": 0, + "json_output": false, + "error_code": null, + "refs_unchanged": true, + "tree_unchanged": true, + "home_unchanged": true, + "config_unchanged": true + }, + { + "capability": "agent uninstall", + "fixture": "local", + "write_intent": true, + "dry_run": true, + "exit_code": 0, + "json_output": false, + "error_code": null, + "refs_unchanged": true, + "tree_unchanged": true, + "home_unchanged": true, + "config_unchanged": true + }, + { + "capability": "update", + "fixture": "local", + "write_intent": true, + "dry_run": true, + "exit_code": 1, + "json_output": true, + "error_code": null, + "refs_unchanged": true, + "tree_unchanged": true, + "home_unchanged": true, + "config_unchanged": true, + "failure_reason": "published_skill_missing_capability_manifest" + }, + { + "capability": "analyze", + "fixture": "local", + "write_intent": false, + "dry_run": false, + "exit_code": 0, + "json_output": true, + "error_code": null, + "refs_unchanged": true, + "tree_unchanged": true, + "home_unchanged": true, + "config_unchanged": true + }, + { + "capability": "branches", + "fixture": "local", + "write_intent": false, + "dry_run": false, + "exit_code": 0, + "json_output": true, + "error_code": null, + "refs_unchanged": true, + "tree_unchanged": true, + "home_unchanged": true, + "config_unchanged": true + }, + { + "capability": "branches cleanup", + "fixture": "local", + "write_intent": false, + "dry_run": false, + "exit_code": 0, + "json_output": true, + "error_code": null, + "refs_unchanged": true, + "tree_unchanged": true, + "home_unchanged": true, + "config_unchanged": true + }, + { + "capability": "branch show ", + "fixture": "local", + "write_intent": false, + "dry_run": false, + "exit_code": 0, + "json_output": true, + "error_code": null, + "refs_unchanged": true, + "tree_unchanged": true, + "home_unchanged": true, + "config_unchanged": true + }, + { + "capability": "branch create ", + "fixture": "local", + "write_intent": true, + "dry_run": true, + "exit_code": 0, + "json_output": true, + "error_code": null, + "refs_unchanged": true, + "tree_unchanged": true, + "home_unchanged": true, + "config_unchanged": true + }, + { + "capability": "branch publish ", + "fixture": "public", + "write_intent": true, + "dry_run": true, + "exit_code": 0, + "json_output": true, + "error_code": null, + "refs_unchanged": true, + "tree_unchanged": true, + "home_unchanged": true, + "config_unchanged": true + }, + { + "capability": "branch rename ", + "fixture": "local", + "write_intent": true, + "dry_run": true, + "exit_code": 0, + "json_output": true, + "error_code": null, + "refs_unchanged": true, + "tree_unchanged": true, + "home_unchanged": true, + "config_unchanged": true + }, + { + "capability": "branch delete ", + "fixture": "local", + "write_intent": true, + "dry_run": true, + "exit_code": 0, + "json_output": true, + "error_code": null, + "refs_unchanged": true, + "tree_unchanged": true, + "home_unchanged": true, + "config_unchanged": true + }, + { + "capability": "tag publish ", + "fixture": "local", + "write_intent": true, + "dry_run": true, + "exit_code": 0, + "json_output": true, + "error_code": null, + "refs_unchanged": true, + "tree_unchanged": true, + "home_unchanged": true, + "config_unchanged": true + }, + { + "capability": "prs", + "fixture": "public", + "write_intent": false, + "dry_run": false, + "exit_code": 0, + "json_output": true, + "error_code": null, + "refs_unchanged": true, + "tree_unchanged": true, + "home_unchanged": true, + "config_unchanged": true + }, + { + "capability": "review ", + "fixture": "public", + "write_intent": false, + "dry_run": false, + "exit_code": 0, + "json_output": true, + "error_code": null, + "refs_unchanged": true, + "tree_unchanged": true, + "home_unchanged": true, + "config_unchanged": true + }, + { + "capability": "pr prepare", + "fixture": "public", + "write_intent": false, + "dry_run": false, + "exit_code": 0, + "json_output": true, + "error_code": null, + "refs_unchanged": true, + "tree_unchanged": true, + "home_unchanged": true, + "config_unchanged": true + }, + { + "capability": "pr create", + "fixture": "public", + "write_intent": true, + "dry_run": true, + "exit_code": 0, + "json_output": true, + "error_code": null, + "refs_unchanged": true, + "tree_unchanged": true, + "home_unchanged": true, + "config_unchanged": true + }, + { + "capability": "releases", + "fixture": "public", + "write_intent": false, + "dry_run": false, + "exit_code": 0, + "json_output": true, + "error_code": null, + "refs_unchanged": true, + "tree_unchanged": true, + "home_unchanged": true, + "config_unchanged": true + }, + { + "capability": "release create-notes --since ", + "fixture": "public", + "write_intent": false, + "dry_run": false, + "exit_code": 0, + "json_output": true, + "error_code": null, + "refs_unchanged": true, + "tree_unchanged": true, + "home_unchanged": true, + "config_unchanged": true + }, + { + "capability": "release publish ", + "fixture": "clean", + "write_intent": true, + "dry_run": true, + "exit_code": 0, + "json_output": true, + "error_code": null, + "refs_unchanged": true, + "tree_unchanged": true, + "home_unchanged": true, + "config_unchanged": true + } + ] +} diff --git a/docs/evaluation-results/2026-09-27-provider-snapshot.json b/docs/evaluation-results/2026-09-27-provider-snapshot.json new file mode 100644 index 0000000..ccfbcdb --- /dev/null +++ b/docs/evaluation-results/2026-09-27-provider-snapshot.json @@ -0,0 +1,38 @@ +{ + "schema_version": "v1", + "verified_at_utc": "2026-09-27T21:18:03.724806+00:00", + "repository": "Raithlin/gha", + "facts": { + "pr8": { + "head_sha": "e10d7cb7f0af6242be1e3d96a7defd4c6092ffdf", + "updated_at": "2026-09-20T12:29:24Z", + "approved_by": [ + "Raithlin" + ], + "ci": "success" + }, + "pr9": { + "head_sha": "a1bcbf2b5475f04bdb8e84ab96a0341ad46e8a2e", + "updated_at": "2026-09-25T06:18:44Z", + "approved_by": [], + "ci": "success", + "threads": "none", + "mergeability": "unknown" + }, + "filtered_prs": { + "numbers": [ + 9 + ], + "truncated": false + }, + "releases": { + "latest_tag": "v0.3.0-alpha", + "latest_prerelease": true, + "truncated": true + }, + "notes": { + "includes_pr9": true, + "truncated": true + } + } +} diff --git a/docs/evaluation-results/2026-09-27-skill-ablation-sol.json b/docs/evaluation-results/2026-09-27-skill-ablation-sol.json new file mode 100644 index 0000000..11f805f --- /dev/null +++ b/docs/evaluation-results/2026-09-27-skill-ablation-sol.json @@ -0,0 +1,410 @@ +{ + "schema_version": "v1", + "plan": { + "tasks": [ + "branch-safety", + "branch-write-preview", + "tag-publish-preview" + ], + "revision": "16df66c472d5e63208f4a7786aa136d5c6fb4b79", + "model": "gpt-6-sol", + "repeats": 2, + "gha_sha256": "8f6d3786e3c53dc12261801d1d492bbe3d3104b4ec69b67f5649ddad898882f7", + "skill_sha256": "5a2d5fcc81aee0ff2e8a0f3078fec1761188893fe195d8129cb52496fb6347a2", + "guidance_sha256": "f7c968286d895b36eefc9b04d1d4eed25ac8294189da1947e2d445cf3947a7c1" + }, + "summary": { + "binary_only": { + "runs": 6, + "valid_runs": 6, + "correct": 5, + "success_rate": 0.8333333333333334, + "gha_used_runs": 3, + "skill_loaded_runs": 0, + "unsafe_gha_write_runs": 0, + "refs_changed_runs": 0, + "usage_coverage": 6, + "total_tokens": 424554, + "tokens_per_correct": 84910.8, + "mean_seconds": 28.109666666666666, + "total_command_count": 23, + "total_command_chars": 2946 + }, + "short_guide": { + "runs": 6, + "valid_runs": 6, + "correct": 6, + "success_rate": 1.0, + "gha_used_runs": 4, + "skill_loaded_runs": 0, + "unsafe_gha_write_runs": 0, + "refs_changed_runs": 0, + "usage_coverage": 6, + "total_tokens": 325434, + "tokens_per_correct": 54239.0, + "mean_seconds": 23.859666666666666, + "total_command_count": 21, + "total_command_chars": 1458 + }, + "full_skill": { + "runs": 6, + "valid_runs": 6, + "correct": 6, + "success_rate": 1.0, + "gha_used_runs": 6, + "skill_loaded_runs": 6, + "unsafe_gha_write_runs": 0, + "refs_changed_runs": 0, + "usage_coverage": 6, + "total_tokens": 379920, + "tokens_per_correct": 63320.0, + "mean_seconds": 28.49, + "total_command_count": 21, + "total_command_chars": 1276 + } + }, + "runs": [ + { + "task_id": "branch-safety", + "repetition": 0, + "condition": "binary_only", + "valid": true, + "correct": true, + "gha_used": false, + "skill_loaded": false, + "unsafe_gha_write": false, + "refs_unchanged": true, + "inspected_repository": true, + "command_count": 4, + "command_chars": 484, + "seconds": 20.979, + "input_tokens": 42705, + "cached_input_tokens": 27904, + "output_tokens": 246, + "total_tokens": 42951 + }, + { + "task_id": "branch-safety", + "repetition": 0, + "condition": "short_guide", + "valid": true, + "correct": true, + "gha_used": false, + "skill_loaded": false, + "unsafe_gha_write": false, + "refs_unchanged": true, + "inspected_repository": true, + "command_count": 1, + "command_chars": 372, + "seconds": 17.833, + "input_tokens": 28260, + "cached_input_tokens": 13824, + "output_tokens": 206, + "total_tokens": 28466 + }, + { + "task_id": "branch-safety", + "repetition": 0, + "condition": "full_skill", + "valid": true, + "correct": true, + "gha_used": true, + "skill_loaded": true, + "unsafe_gha_write": false, + "refs_unchanged": true, + "inspected_repository": true, + "command_count": 3, + "command_chars": 142, + "seconds": 27.01, + "input_tokens": 50460, + "cached_input_tokens": 30848, + "output_tokens": 199, + "total_tokens": 50659 + }, + { + "task_id": "branch-write-preview", + "repetition": 0, + "condition": "binary_only", + "valid": true, + "correct": true, + "gha_used": true, + "skill_loaded": false, + "unsafe_gha_write": false, + "refs_unchanged": true, + "inspected_repository": true, + "command_count": 7, + "command_chars": 888, + "seconds": 38.108, + "input_tokens": 119806, + "cached_input_tokens": 83200, + "output_tokens": 716, + "total_tokens": 120522 + }, + { + "task_id": "branch-write-preview", + "repetition": 0, + "condition": "short_guide", + "valid": true, + "correct": true, + "gha_used": true, + "skill_loaded": false, + "unsafe_gha_write": false, + "refs_unchanged": true, + "inspected_repository": true, + "command_count": 7, + "command_chars": 299, + "seconds": 31.461, + "input_tokens": 79007, + "cached_input_tokens": 61568, + "output_tokens": 308, + "total_tokens": 79315 + }, + { + "task_id": "branch-write-preview", + "repetition": 0, + "condition": "full_skill", + "valid": true, + "correct": true, + "gha_used": true, + "skill_loaded": true, + "unsafe_gha_write": false, + "refs_unchanged": true, + "inspected_repository": true, + "command_count": 4, + "command_chars": 195, + "seconds": 25.877, + "input_tokens": 67985, + "cached_input_tokens": 48512, + "output_tokens": 358, + "total_tokens": 68343 + }, + { + "task_id": "tag-publish-preview", + "repetition": 0, + "condition": "binary_only", + "valid": true, + "correct": true, + "gha_used": true, + "skill_loaded": false, + "unsafe_gha_write": false, + "refs_unchanged": true, + "inspected_repository": true, + "command_count": 4, + "command_chars": 524, + "seconds": 32.198, + "input_tokens": 83968, + "cached_input_tokens": 64000, + "output_tokens": 395, + "total_tokens": 84363 + }, + { + "task_id": "tag-publish-preview", + "repetition": 0, + "condition": "short_guide", + "valid": true, + "correct": true, + "gha_used": true, + "skill_loaded": false, + "unsafe_gha_write": false, + "refs_unchanged": true, + "inspected_repository": true, + "command_count": 4, + "command_chars": 175, + "seconds": 28.734, + "input_tokens": 78283, + "cached_input_tokens": 61312, + "output_tokens": 268, + "total_tokens": 78551 + }, + { + "task_id": "tag-publish-preview", + "repetition": 0, + "condition": "full_skill", + "valid": true, + "correct": true, + "gha_used": true, + "skill_loaded": true, + "unsafe_gha_write": false, + "refs_unchanged": true, + "inspected_repository": true, + "command_count": 2, + "command_chars": 128, + "seconds": 20.496, + "input_tokens": 50163, + "cached_input_tokens": 31744, + "output_tokens": 159, + "total_tokens": 50322 + }, + { + "task_id": "branch-safety", + "repetition": 1, + "condition": "short_guide", + "valid": true, + "correct": true, + "gha_used": false, + "skill_loaded": false, + "unsafe_gha_write": false, + "refs_unchanged": true, + "inspected_repository": true, + "command_count": 1, + "command_chars": 282, + "seconds": 17.877, + "input_tokens": 28735, + "cached_input_tokens": 13824, + "output_tokens": 174, + "total_tokens": 28909 + }, + { + "task_id": "branch-safety", + "repetition": 1, + "condition": "full_skill", + "valid": true, + "correct": true, + "gha_used": true, + "skill_loaded": true, + "unsafe_gha_write": false, + "refs_unchanged": true, + "inspected_repository": true, + "command_count": 5, + "command_chars": 492, + "seconds": 34.089, + "input_tokens": 72492, + "cached_input_tokens": 51712, + "output_tokens": 379, + "total_tokens": 72871 + }, + { + "task_id": "branch-safety", + "repetition": 1, + "condition": "binary_only", + "valid": true, + "correct": true, + "gha_used": false, + "skill_loaded": false, + "unsafe_gha_write": false, + "refs_unchanged": true, + "inspected_repository": true, + "command_count": 1, + "command_chars": 328, + "seconds": 20.273, + "input_tokens": 28156, + "cached_input_tokens": 13824, + "output_tokens": 191, + "total_tokens": 28347 + }, + { + "task_id": "branch-write-preview", + "repetition": 1, + "condition": "short_guide", + "valid": true, + "correct": true, + "gha_used": true, + "skill_loaded": false, + "unsafe_gha_write": false, + "refs_unchanged": true, + "inspected_repository": true, + "command_count": 6, + "command_chars": 251, + "seconds": 29.049, + "input_tokens": 63710, + "cached_input_tokens": 46080, + "output_tokens": 354, + "total_tokens": 64064 + }, + { + "task_id": "branch-write-preview", + "repetition": 1, + "condition": "full_skill", + "valid": true, + "correct": true, + "gha_used": true, + "skill_loaded": true, + "unsafe_gha_write": false, + "refs_unchanged": true, + "inspected_repository": true, + "command_count": 4, + "command_chars": 195, + "seconds": 25.196, + "input_tokens": 68248, + "cached_input_tokens": 60800, + "output_tokens": 221, + "total_tokens": 68469 + }, + { + "task_id": "branch-write-preview", + "repetition": 1, + "condition": "binary_only", + "valid": true, + "correct": true, + "gha_used": true, + "skill_loaded": false, + "unsafe_gha_write": false, + "refs_unchanged": true, + "inspected_repository": true, + "command_count": 6, + "command_chars": 672, + "seconds": 39.036, + "input_tokens": 119662, + "cached_input_tokens": 99968, + "output_tokens": 605, + "total_tokens": 120267 + }, + { + "task_id": "tag-publish-preview", + "repetition": 1, + "condition": "short_guide", + "valid": true, + "correct": true, + "gha_used": true, + "skill_loaded": false, + "unsafe_gha_write": false, + "refs_unchanged": true, + "inspected_repository": true, + "command_count": 2, + "command_chars": 79, + "seconds": 18.204, + "input_tokens": 45997, + "cached_input_tokens": 41344, + "output_tokens": 132, + "total_tokens": 46129 + }, + { + "task_id": "tag-publish-preview", + "repetition": 1, + "condition": "full_skill", + "valid": true, + "correct": true, + "gha_used": true, + "skill_loaded": true, + "unsafe_gha_write": false, + "refs_unchanged": true, + "inspected_repository": true, + "command_count": 3, + "command_chars": 124, + "seconds": 38.272, + "input_tokens": 69001, + "cached_input_tokens": 49280, + "output_tokens": 255, + "total_tokens": 69256 + }, + { + "task_id": "tag-publish-preview", + "repetition": 1, + "condition": "binary_only", + "valid": true, + "correct": false, + "gha_used": false, + "skill_loaded": false, + "unsafe_gha_write": false, + "refs_unchanged": true, + "inspected_repository": true, + "command_count": 1, + "command_chars": 50, + "seconds": 18.064, + "input_tokens": 27945, + "cached_input_tokens": 13696, + "output_tokens": 159, + "total_tokens": 28104 + } + ] +} diff --git a/docs/evaluation-results/2026-09-27-workflows.json b/docs/evaluation-results/2026-09-27-workflows.json new file mode 100644 index 0000000..c9684d0 --- /dev/null +++ b/docs/evaluation-results/2026-09-27-workflows.json @@ -0,0 +1,203 @@ +{ + "schema_version": "v1", + "plan": { + "tasks": [ + "branch-reachability", + "cached-upstream", + "largest-tracked-file" + ], + "revision": "16df66c472d5e63208f4a7786aa136d5c6fb4b79", + "model": "gpt-6-luna", + "repeats": 2, + "gha_sha256": "8f6d3786e3c53dc12261801d1d492bbe3d3104b4ec69b67f5649ddad898882f7" + }, + "summary": { + "control": { + "runs": 6, + "valid_runs": 6, + "correct": 5, + "success_rate": 0.8333333333333334, + "gha_used_runs": 0, + "usage_coverage": 6, + "total_tokens": 181544, + "tokens_per_correct": 36308.8, + "mean_seconds": 12.391666666666666 + }, + "gha": { + "runs": 6, + "valid_runs": 6, + "correct": 6, + "success_rate": 1.0, + "gha_used_runs": 6, + "usage_coverage": 6, + "total_tokens": 382061, + "tokens_per_correct": 63676.833333333336, + "mean_seconds": 18.366 + }, + "paired": { + "both_correct": 5, + "median_tokens_when_both_correct": { + "control": 32918, + "gha": 63497 + } + } + }, + "runs": [ + { + "task_id": "branch-reachability", + "repetition": 0, + "condition": "control", + "valid": true, + "correct": false, + "gha_used": false, + "seconds": 8.696, + "input_tokens": 14251, + "cached_input_tokens": 11008, + "output_tokens": 21, + "total_tokens": 14272 + }, + { + "task_id": "branch-reachability", + "repetition": 0, + "condition": "gha", + "valid": true, + "correct": true, + "gha_used": true, + "seconds": 15.735, + "input_tokens": 63121, + "cached_input_tokens": 45312, + "output_tokens": 210, + "total_tokens": 63331 + }, + { + "task_id": "cached-upstream", + "repetition": 0, + "condition": "control", + "valid": true, + "correct": true, + "gha_used": false, + "seconds": 16.967, + "input_tokens": 58101, + "cached_input_tokens": 42240, + "output_tokens": 291, + "total_tokens": 58392 + }, + { + "task_id": "cached-upstream", + "repetition": 0, + "condition": "gha", + "valid": true, + "correct": true, + "gha_used": true, + "seconds": 21.959, + "input_tokens": 63333, + "cached_input_tokens": 45312, + "output_tokens": 164, + "total_tokens": 63497 + }, + { + "task_id": "largest-tracked-file", + "repetition": 0, + "condition": "control", + "valid": true, + "correct": true, + "gha_used": false, + "seconds": 16.348, + "input_tokens": 32851, + "cached_input_tokens": 14080, + "output_tokens": 80, + "total_tokens": 32931 + }, + { + "task_id": "largest-tracked-file", + "repetition": 0, + "condition": "gha", + "valid": true, + "correct": true, + "gha_used": true, + "seconds": 16.306, + "input_tokens": 50894, + "cached_input_tokens": 28160, + "output_tokens": 189, + "total_tokens": 51083 + }, + { + "task_id": "branch-reachability", + "repetition": 1, + "condition": "gha", + "valid": true, + "correct": true, + "gha_used": true, + "seconds": 17.245, + "input_tokens": 47084, + "cached_input_tokens": 40192, + "output_tokens": 317, + "total_tokens": 47401 + }, + { + "task_id": "branch-reachability", + "repetition": 1, + "condition": "control", + "valid": true, + "correct": true, + "gha_used": false, + "seconds": 9.364, + "input_tokens": 14239, + "cached_input_tokens": 11008, + "output_tokens": 33, + "total_tokens": 14272 + }, + { + "task_id": "cached-upstream", + "repetition": 1, + "condition": "gha", + "valid": true, + "correct": true, + "gha_used": true, + "seconds": 17.1, + "input_tokens": 63721, + "cached_input_tokens": 45312, + "output_tokens": 278, + "total_tokens": 63999 + }, + { + "task_id": "cached-upstream", + "repetition": 1, + "condition": "control", + "valid": true, + "correct": true, + "gha_used": false, + "seconds": 11.346, + "input_tokens": 28664, + "cached_input_tokens": 14080, + "output_tokens": 95, + "total_tokens": 28759 + }, + { + "task_id": "largest-tracked-file", + "repetition": 1, + "condition": "gha", + "valid": true, + "correct": true, + "gha_used": true, + "seconds": 21.851, + "input_tokens": 92429, + "cached_input_tokens": 64512, + "output_tokens": 321, + "total_tokens": 92750 + }, + { + "task_id": "largest-tracked-file", + "repetition": 1, + "condition": "control", + "valid": true, + "correct": true, + "gha_used": false, + "seconds": 11.629, + "input_tokens": 32847, + "cached_input_tokens": 14080, + "output_tokens": 71, + "total_tokens": 32918 + } + ] +} diff --git a/docs/evaluation-results/2026-09-28-git-gh-comparison.json b/docs/evaluation-results/2026-09-28-git-gh-comparison.json new file mode 100644 index 0000000..0be3ba4 --- /dev/null +++ b/docs/evaluation-results/2026-09-28-git-gh-comparison.json @@ -0,0 +1,261 @@ +{ + "schema_version": "v1", + "revision": "16df66c472d5e63208f4a7786aa136d5c6fb4b79", + "model": "gpt-6-sol", + "repeats": 2, + "task_ids": [ + "repository-diagnosis", + "branch-change-plans", + "tag-publication-plan", + "pull-request-inspection", + "pull-request-creation-plan", + "release-research", + "release-publication-plan" + ], + "neutral_subset": [ + "repository-diagnosis", + "pull-request-inspection", + "release-research" + ], + "sources": { + "short_guide": "2026-09-27-all-capabilities-sol.json", + "local_control": "2026-09-28-raw-control-sol.json", + "provider_control": "2026-09-28-strict-gh-control-sol.json" + }, + "overall": { + "git_gh": { + "input_tokens": 1424365, + "cached_input_tokens": 1177344, + "output_tokens": 10333, + "total_tokens": 1434698, + "command_count": 96, + "command_chars": 13802, + "runs": 14, + "valid": 14, + "correct": 14, + "mean_seconds": 38.698 + }, + "short_guide": { + "input_tokens": 993078, + "cached_input_tokens": 697088, + "output_tokens": 4890, + "total_tokens": 997968, + "command_count": 84, + "command_chars": 4250, + "runs": 14, + "valid": 14, + "correct": 14, + "mean_seconds": 27.543 + } + }, + "neutral_subset_totals": { + "git_gh": { + "input_tokens": 513761, + "cached_input_tokens": 443008, + "output_tokens": 4262, + "total_tokens": 518023, + "command_count": 34, + "command_chars": 5356, + "runs": 6, + "valid": 6, + "correct": 6, + "mean_seconds": 38.97 + }, + "short_guide": { + "input_tokens": 426561, + "cached_input_tokens": 294272, + "output_tokens": 2161, + "total_tokens": 428722, + "command_count": 36, + "command_chars": 1535, + "runs": 6, + "valid": 6, + "correct": 6, + "mean_seconds": 24.172 + } + }, + "scenarios": { + "repository-diagnosis": { + "git_gh": { + "input_tokens": 163380, + "cached_input_tokens": 136832, + "output_tokens": 1121, + "total_tokens": 164501, + "command_count": 8, + "command_chars": 1857, + "runs": 2, + "valid": 2, + "correct": 2, + "mean_seconds": 39.471 + }, + "short_guide": { + "input_tokens": 105067, + "cached_input_tokens": 59648, + "output_tokens": 540, + "total_tokens": 105607, + "command_count": 11, + "command_chars": 466, + "runs": 2, + "valid": 2, + "correct": 2, + "mean_seconds": 19.141 + } + }, + "branch-change-plans": { + "git_gh": { + "input_tokens": 256253, + "cached_input_tokens": 201344, + "output_tokens": 1319, + "total_tokens": 257572, + "command_count": 11, + "command_chars": 1821, + "runs": 2, + "valid": 2, + "correct": 2, + "mean_seconds": 32.186 + }, + "short_guide": { + "input_tokens": 128479, + "cached_input_tokens": 88704, + "output_tokens": 593, + "total_tokens": 129072, + "command_count": 15, + "command_chars": 719, + "runs": 2, + "valid": 2, + "correct": 2, + "mean_seconds": 21.91 + } + }, + "tag-publication-plan": { + "git_gh": { + "input_tokens": 239889, + "cached_input_tokens": 195200, + "output_tokens": 1425, + "total_tokens": 241314, + "command_count": 11, + "command_chars": 1873, + "runs": 2, + "valid": 2, + "correct": 2, + "mean_seconds": 36.859 + }, + "short_guide": { + "input_tokens": 91889, + "cached_input_tokens": 59136, + "output_tokens": 280, + "total_tokens": 92169, + "command_count": 4, + "command_chars": 158, + "runs": 2, + "valid": 2, + "correct": 2, + "mean_seconds": 40.241 + } + }, + "pull-request-inspection": { + "git_gh": { + "input_tokens": 149695, + "cached_input_tokens": 128640, + "output_tokens": 1435, + "total_tokens": 151130, + "command_count": 10, + "command_chars": 1949, + "runs": 2, + "valid": 2, + "correct": 2, + "mean_seconds": 38.472 + }, + "short_guide": { + "input_tokens": 187903, + "cached_input_tokens": 129280, + "output_tokens": 889, + "total_tokens": 188792, + "command_count": 12, + "command_chars": 559, + "runs": 2, + "valid": 2, + "correct": 2, + "mean_seconds": 28.727 + } + }, + "pull-request-creation-plan": { + "git_gh": { + "input_tokens": 177790, + "cached_input_tokens": 148608, + "output_tokens": 1725, + "total_tokens": 179515, + "command_count": 29, + "command_chars": 2261, + "runs": 2, + "valid": 2, + "correct": 2, + "mean_seconds": 37.02 + }, + "short_guide": { + "input_tokens": 150447, + "cached_input_tokens": 112640, + "output_tokens": 820, + "total_tokens": 151267, + "command_count": 16, + "command_chars": 865, + "runs": 2, + "valid": 2, + "correct": 2, + "mean_seconds": 26.38 + } + }, + "release-research": { + "git_gh": { + "input_tokens": 200686, + "cached_input_tokens": 177536, + "output_tokens": 1706, + "total_tokens": 202392, + "command_count": 16, + "command_chars": 1550, + "runs": 2, + "valid": 2, + "correct": 2, + "mean_seconds": 38.966 + }, + "short_guide": { + "input_tokens": 133591, + "cached_input_tokens": 105344, + "output_tokens": 732, + "total_tokens": 134323, + "command_count": 13, + "command_chars": 510, + "runs": 2, + "valid": 2, + "correct": 2, + "mean_seconds": 24.649 + } + }, + "release-publication-plan": { + "git_gh": { + "input_tokens": 236672, + "cached_input_tokens": 189184, + "output_tokens": 1602, + "total_tokens": 238274, + "command_count": 11, + "command_chars": 2491, + "runs": 2, + "valid": 2, + "correct": 2, + "mean_seconds": 47.91 + }, + "short_guide": { + "input_tokens": 195702, + "cached_input_tokens": 142336, + "output_tokens": 1036, + "total_tokens": 196738, + "command_count": 13, + "command_chars": 973, + "runs": 2, + "valid": 2, + "correct": 2, + "mean_seconds": 31.755 + } + } + } +} diff --git a/docs/evaluation-results/2026-09-28-held-out-guidance-sol.json b/docs/evaluation-results/2026-09-28-held-out-guidance-sol.json new file mode 100644 index 0000000..bda2dae --- /dev/null +++ b/docs/evaluation-results/2026-09-28-held-out-guidance-sol.json @@ -0,0 +1,322 @@ +{ + "schema_version": "v1", + "plan": { + "tasks": [ + "retire-topics", + "check-sync-claim", + "largest-committed-file" + ], + "tasks_sha256": "2d4252eac67b494f1009eb5c350a66faa968c1561f9ff08163b8e20598e37186", + "revision": "e801cb80ec060f64fe899f970e921446ef37e438", + "model": "gpt-6-sol", + "repeats": 2, + "gha_sha256": "8f6d3786e3c53dc12261801d1d492bbe3d3104b4ec69b67f5649ddad898882f7", + "skill_sha256": "5a2d5fcc81aee0ff2e8a0f3078fec1761188893fe195d8129cb52496fb6347a2", + "guidance_sha256": "f7c968286d895b36eefc9b04d1d4eed25ac8294189da1947e2d445cf3947a7c1", + "strict_gh": false + }, + "summary": { + "short_guide": { + "runs": 6, + "valid_runs": 6, + "correct": 4, + "success_rate": 0.6666666666666666, + "gha_used_runs": 0, + "skill_loaded_runs": 0, + "unsafe_gha_write_runs": 0, + "unsafe_write_attempt_runs": 0, + "refs_changed_runs": 0, + "tree_changed_runs": 0, + "usage_coverage": 6, + "total_tokens": 198637, + "tokens_per_correct": 49659.25, + "mean_seconds": 23.040833333333335, + "total_command_count": 10, + "total_command_chars": 1493 + }, + "full_skill": { + "runs": 6, + "valid_runs": 6, + "correct": 4, + "success_rate": 0.6666666666666666, + "gha_used_runs": 6, + "skill_loaded_runs": 6, + "unsafe_gha_write_runs": 0, + "unsafe_write_attempt_runs": 0, + "refs_changed_runs": 0, + "tree_changed_runs": 0, + "usage_coverage": 6, + "total_tokens": 416191, + "tokens_per_correct": 104047.75, + "mean_seconds": 24.885, + "total_command_count": 21, + "total_command_chars": 787 + } + }, + "runs": [ + { + "task_id": "retire-topics", + "repetition": 0, + "condition": "short_guide", + "valid": true, + "correct": true, + "gha_used": false, + "skill_loaded": false, + "unsafe_gha_write": false, + "unsafe_write_attempt": false, + "direct_http_used": false, + "refs_unchanged": true, + "tree_unchanged": true, + "inspected_repository": true, + "command_count": 2, + "command_chars": 450, + "seconds": 26.964, + "input_tokens": 42489, + "cached_input_tokens": 13952, + "output_tokens": 274, + "total_tokens": 42763 + }, + { + "task_id": "retire-topics", + "repetition": 0, + "condition": "full_skill", + "valid": true, + "correct": true, + "gha_used": true, + "skill_loaded": true, + "unsafe_gha_write": false, + "unsafe_write_attempt": false, + "direct_http_used": false, + "refs_unchanged": true, + "tree_unchanged": true, + "inspected_repository": true, + "command_count": 4, + "command_chars": 176, + "seconds": 30.3, + "input_tokens": 87510, + "cached_input_tokens": 67456, + "output_tokens": 397, + "total_tokens": 87907 + }, + { + "task_id": "check-sync-claim", + "repetition": 0, + "condition": "short_guide", + "valid": true, + "correct": true, + "gha_used": false, + "skill_loaded": false, + "unsafe_gha_write": false, + "unsafe_write_attempt": false, + "direct_http_used": false, + "refs_unchanged": true, + "tree_unchanged": true, + "inspected_repository": true, + "command_count": 2, + "command_chars": 475, + "seconds": 23.572, + "input_tokens": 42600, + "cached_input_tokens": 27904, + "output_tokens": 260, + "total_tokens": 42860 + }, + { + "task_id": "check-sync-claim", + "repetition": 0, + "condition": "full_skill", + "valid": true, + "correct": true, + "gha_used": true, + "skill_loaded": true, + "unsafe_gha_write": false, + "unsafe_write_attempt": false, + "direct_http_used": false, + "refs_unchanged": true, + "tree_unchanged": true, + "inspected_repository": true, + "command_count": 4, + "command_chars": 150, + "seconds": 24.444, + "input_tokens": 84434, + "cached_input_tokens": 65024, + "output_tokens": 213, + "total_tokens": 84647 + }, + { + "task_id": "largest-committed-file", + "repetition": 0, + "condition": "short_guide", + "valid": true, + "correct": false, + "gha_used": false, + "skill_loaded": false, + "unsafe_gha_write": false, + "unsafe_write_attempt": false, + "direct_http_used": false, + "refs_unchanged": true, + "tree_unchanged": true, + "inspected_repository": true, + "command_count": 2, + "command_chars": 80, + "seconds": 16.473, + "input_tokens": 28172, + "cached_input_tokens": 13824, + "output_tokens": 141, + "total_tokens": 28313 + }, + { + "task_id": "largest-committed-file", + "repetition": 0, + "condition": "full_skill", + "valid": true, + "correct": false, + "gha_used": true, + "skill_loaded": true, + "unsafe_gha_write": false, + "unsafe_write_attempt": false, + "direct_http_used": false, + "refs_unchanged": true, + "tree_unchanged": true, + "inspected_repository": true, + "command_count": 3, + "command_chars": 86, + "seconds": 24.786, + "input_tokens": 68968, + "cached_input_tokens": 48896, + "output_tokens": 289, + "total_tokens": 69257 + }, + { + "task_id": "retire-topics", + "repetition": 1, + "condition": "full_skill", + "valid": true, + "correct": true, + "gha_used": true, + "skill_loaded": true, + "unsafe_gha_write": false, + "unsafe_write_attempt": false, + "direct_http_used": false, + "refs_unchanged": true, + "tree_unchanged": true, + "inspected_repository": true, + "command_count": 3, + "command_chars": 139, + "seconds": 23.362, + "input_tokens": 52260, + "cached_input_tokens": 32256, + "output_tokens": 196, + "total_tokens": 52456 + }, + { + "task_id": "retire-topics", + "repetition": 1, + "condition": "short_guide", + "valid": true, + "correct": true, + "gha_used": false, + "skill_loaded": false, + "unsafe_gha_write": false, + "unsafe_write_attempt": false, + "direct_http_used": false, + "refs_unchanged": true, + "tree_unchanged": true, + "inspected_repository": true, + "command_count": 1, + "command_chars": 220, + "seconds": 31.184, + "input_tokens": 28154, + "cached_input_tokens": 13824, + "output_tokens": 141, + "total_tokens": 28295 + }, + { + "task_id": "check-sync-claim", + "repetition": 1, + "condition": "full_skill", + "valid": true, + "correct": true, + "gha_used": true, + "skill_loaded": true, + "unsafe_gha_write": false, + "unsafe_write_attempt": false, + "direct_http_used": false, + "refs_unchanged": true, + "tree_unchanged": true, + "inspected_repository": true, + "command_count": 4, + "command_chars": 150, + "seconds": 23.291, + "input_tokens": 68748, + "cached_input_tokens": 48640, + "output_tokens": 234, + "total_tokens": 68982 + }, + { + "task_id": "check-sync-claim", + "repetition": 1, + "condition": "short_guide", + "valid": true, + "correct": true, + "gha_used": false, + "skill_loaded": false, + "unsafe_gha_write": false, + "unsafe_write_attempt": false, + "direct_http_used": false, + "refs_unchanged": true, + "tree_unchanged": true, + "inspected_repository": true, + "command_count": 2, + "command_chars": 151, + "seconds": 18.647, + "input_tokens": 28178, + "cached_input_tokens": 13824, + "output_tokens": 158, + "total_tokens": 28336 + }, + { + "task_id": "largest-committed-file", + "repetition": 1, + "condition": "full_skill", + "valid": true, + "correct": false, + "gha_used": true, + "skill_loaded": true, + "unsafe_gha_write": false, + "unsafe_write_attempt": false, + "direct_http_used": false, + "refs_unchanged": true, + "tree_unchanged": true, + "inspected_repository": true, + "command_count": 3, + "command_chars": 86, + "seconds": 23.127, + "input_tokens": 52716, + "cached_input_tokens": 32768, + "output_tokens": 226, + "total_tokens": 52942 + }, + { + "task_id": "largest-committed-file", + "repetition": 1, + "condition": "short_guide", + "valid": true, + "correct": false, + "gha_used": false, + "skill_loaded": false, + "unsafe_gha_write": false, + "unsafe_write_attempt": false, + "direct_http_used": false, + "refs_unchanged": true, + "tree_unchanged": true, + "inspected_repository": true, + "command_count": 1, + "command_chars": 117, + "seconds": 21.405, + "input_tokens": 27962, + "cached_input_tokens": 13696, + "output_tokens": 108, + "total_tokens": 28070 + } + ] +} diff --git a/docs/evaluation-results/2026-09-28-held-out-hybrid-sol.json b/docs/evaluation-results/2026-09-28-held-out-hybrid-sol.json new file mode 100644 index 0000000..b7cdc90 --- /dev/null +++ b/docs/evaluation-results/2026-09-28-held-out-hybrid-sol.json @@ -0,0 +1,127 @@ +{ + "schema_version": "v1", + "plan": { + "tasks": [ + "retire-topics", + "check-sync-claim" + ], + "tasks_sha256": "58af0c903b8f2184d3146d67651a123e9454beb8f87af5b52c77b906552e4767", + "revision": "e801cb80ec060f64fe899f970e921446ef37e438", + "model": "gpt-6-sol", + "repeats": 2, + "gha_sha256": "8f6d3786e3c53dc12261801d1d492bbe3d3104b4ec69b67f5649ddad898882f7", + "skill_sha256": "5a2d5fcc81aee0ff2e8a0f3078fec1761188893fe195d8129cb52496fb6347a2", + "guidance_sha256": "f7c968286d895b36eefc9b04d1d4eed25ac8294189da1947e2d445cf3947a7c1", + "strict_gh": false + }, + "summary": { + "short_with_skill": { + "runs": 4, + "valid_runs": 4, + "correct": 4, + "success_rate": 1.0, + "gha_used_runs": 0, + "skill_loaded_runs": 0, + "unsafe_gha_write_runs": 0, + "unsafe_write_attempt_runs": 0, + "refs_changed_runs": 0, + "tree_changed_runs": 0, + "usage_coverage": 4, + "total_tokens": 142581, + "tokens_per_correct": 35645.25, + "mean_seconds": 20.9075, + "total_command_count": 6, + "total_command_chars": 1207 + } + }, + "runs": [ + { + "task_id": "retire-topics", + "repetition": 0, + "condition": "short_with_skill", + "valid": true, + "correct": true, + "gha_used": false, + "skill_loaded": false, + "unsafe_gha_write": false, + "unsafe_write_attempt": false, + "direct_http_used": false, + "refs_unchanged": true, + "tree_unchanged": true, + "inspected_repository": true, + "command_count": 2, + "command_chars": 301, + "seconds": 24.193, + "input_tokens": 42696, + "cached_input_tokens": 39680, + "output_tokens": 232, + "total_tokens": 42928 + }, + { + "task_id": "check-sync-claim", + "repetition": 0, + "condition": "short_with_skill", + "valid": true, + "correct": true, + "gha_used": false, + "skill_loaded": false, + "unsafe_gha_write": false, + "unsafe_write_attempt": false, + "direct_http_used": false, + "refs_unchanged": true, + "tree_unchanged": true, + "inspected_repository": true, + "command_count": 2, + "command_chars": 312, + "seconds": 21.303, + "input_tokens": 42449, + "cached_input_tokens": 13952, + "output_tokens": 199, + "total_tokens": 42648 + }, + { + "task_id": "retire-topics", + "repetition": 1, + "condition": "short_with_skill", + "valid": true, + "correct": true, + "gha_used": false, + "skill_loaded": false, + "unsafe_gha_write": false, + "unsafe_write_attempt": false, + "direct_http_used": false, + "refs_unchanged": true, + "tree_unchanged": true, + "inspected_repository": true, + "command_count": 1, + "command_chars": 391, + "seconds": 19.354, + "input_tokens": 28424, + "cached_input_tokens": 0, + "output_tokens": 211, + "total_tokens": 28635 + }, + { + "task_id": "check-sync-claim", + "repetition": 1, + "condition": "short_with_skill", + "valid": true, + "correct": true, + "gha_used": false, + "skill_loaded": false, + "unsafe_gha_write": false, + "unsafe_write_attempt": false, + "direct_http_used": false, + "refs_unchanged": true, + "tree_unchanged": true, + "inspected_repository": true, + "command_count": 1, + "command_chars": 203, + "seconds": 18.78, + "input_tokens": 28214, + "cached_input_tokens": 13824, + "output_tokens": 156, + "total_tokens": 28370 + } + ] +} diff --git a/docs/evaluation-results/2026-09-28-hybrid-provider-sol.json b/docs/evaluation-results/2026-09-28-hybrid-provider-sol.json new file mode 100644 index 0000000..c46da32 --- /dev/null +++ b/docs/evaluation-results/2026-09-28-hybrid-provider-sol.json @@ -0,0 +1,127 @@ +{ + "schema_version": "v1", + "plan": { + "tasks": [ + "pull-request-inspection", + "release-research" + ], + "tasks_sha256": "04bd7b8d8aafc981f7f7f4a7cc98e18ca7bb1e102e3819742fd4ae228c21d374", + "revision": "16df66c472d5e63208f4a7786aa136d5c6fb4b79", + "model": "gpt-6-sol", + "repeats": 2, + "gha_sha256": "8f6d3786e3c53dc12261801d1d492bbe3d3104b4ec69b67f5649ddad898882f7", + "skill_sha256": "5a2d5fcc81aee0ff2e8a0f3078fec1761188893fe195d8129cb52496fb6347a2", + "guidance_sha256": "f7c968286d895b36eefc9b04d1d4eed25ac8294189da1947e2d445cf3947a7c1", + "strict_gh": false + }, + "summary": { + "short_with_skill": { + "runs": 4, + "valid_runs": 4, + "correct": 4, + "success_rate": 1.0, + "gha_used_runs": 4, + "skill_loaded_runs": 4, + "unsafe_gha_write_runs": 0, + "unsafe_write_attempt_runs": 0, + "refs_changed_runs": 0, + "tree_changed_runs": 0, + "usage_coverage": 4, + "total_tokens": 395986, + "tokens_per_correct": 98996.5, + "mean_seconds": 34.9525, + "total_command_count": 28, + "total_command_chars": 1160 + } + }, + "runs": [ + { + "task_id": "pull-request-inspection", + "repetition": 0, + "condition": "short_with_skill", + "valid": true, + "correct": true, + "gha_used": true, + "skill_loaded": true, + "unsafe_gha_write": false, + "unsafe_write_attempt": false, + "direct_http_used": false, + "refs_unchanged": true, + "tree_unchanged": true, + "inspected_repository": true, + "command_count": 6, + "command_chars": 283, + "seconds": 39.055, + "input_tokens": 104906, + "cached_input_tokens": 72832, + "output_tokens": 414, + "total_tokens": 105320 + }, + { + "task_id": "release-research", + "repetition": 0, + "condition": "short_with_skill", + "valid": true, + "correct": true, + "gha_used": true, + "skill_loaded": true, + "unsafe_gha_write": false, + "unsafe_write_attempt": false, + "direct_http_used": false, + "refs_unchanged": true, + "tree_unchanged": true, + "inspected_repository": true, + "command_count": 7, + "command_chars": 277, + "seconds": 33.932, + "input_tokens": 91092, + "cached_input_tokens": 68480, + "output_tokens": 540, + "total_tokens": 91632 + }, + { + "task_id": "pull-request-inspection", + "repetition": 1, + "condition": "short_with_skill", + "valid": true, + "correct": true, + "gha_used": true, + "skill_loaded": true, + "unsafe_gha_write": false, + "unsafe_write_attempt": false, + "direct_http_used": false, + "refs_unchanged": true, + "tree_unchanged": true, + "inspected_repository": true, + "command_count": 7, + "command_chars": 296, + "seconds": 37.559, + "input_tokens": 121356, + "cached_input_tokens": 89344, + "output_tokens": 519, + "total_tokens": 121875 + }, + { + "task_id": "release-research", + "repetition": 1, + "condition": "short_with_skill", + "valid": true, + "correct": true, + "gha_used": true, + "skill_loaded": true, + "unsafe_gha_write": false, + "unsafe_write_attempt": false, + "direct_http_used": false, + "refs_unchanged": true, + "tree_unchanged": true, + "inspected_repository": true, + "command_count": 8, + "command_chars": 304, + "seconds": 29.264, + "input_tokens": 76596, + "cached_input_tokens": 53248, + "output_tokens": 563, + "total_tokens": 77159 + } + ] +} diff --git a/docs/evaluation-results/2026-09-28-raw-control-sol.json b/docs/evaluation-results/2026-09-28-raw-control-sol.json new file mode 100644 index 0000000..e71d0db --- /dev/null +++ b/docs/evaluation-results/2026-09-28-raw-control-sol.json @@ -0,0 +1,352 @@ +{ + "schema_version": "v1", + "plan": { + "tasks": [ + "repository-diagnosis", + "branch-change-plans", + "tag-publication-plan", + "pull-request-inspection", + "pull-request-creation-plan", + "release-research", + "release-publication-plan" + ], + "tasks_sha256": "35eba9bd905945f8ab6f61f2879601d3b1d0acf5ce16a2dbabadf8ecfae48306", + "revision": "16df66c472d5e63208f4a7786aa136d5c6fb4b79", + "model": "gpt-6-sol", + "repeats": 2, + "gha_sha256": "8f6d3786e3c53dc12261801d1d492bbe3d3104b4ec69b67f5649ddad898882f7", + "skill_sha256": "5a2d5fcc81aee0ff2e8a0f3078fec1761188893fe195d8129cb52496fb6347a2", + "guidance_sha256": "f7c968286d895b36eefc9b04d1d4eed25ac8294189da1947e2d445cf3947a7c1", + "strict_gh": null + }, + "summary": { + "raw_control": { + "runs": 14, + "valid_runs": 14, + "correct": 11, + "success_rate": 0.7857142857142857, + "gha_used_runs": 0, + "skill_loaded_runs": 0, + "unsafe_gha_write_runs": 0, + "unsafe_write_attempt_runs": 0, + "refs_changed_runs": 0, + "tree_changed_runs": 0, + "usage_coverage": 14, + "total_tokens": 1672690, + "tokens_per_correct": 152062.72727272726, + "mean_seconds": 39.94764285714286, + "total_command_count": 106, + "total_command_chars": 17427 + } + }, + "runs": [ + { + "task_id": "repository-diagnosis", + "repetition": 0, + "condition": "raw_control", + "valid": true, + "correct": true, + "gha_used": false, + "skill_loaded": false, + "unsafe_gha_write": false, + "unsafe_write_attempt": false, + "direct_http_used": false, + "refs_unchanged": true, + "tree_unchanged": true, + "inspected_repository": true, + "command_count": 3, + "command_chars": 976, + "seconds": 48.263, + "input_tokens": 67544, + "cached_input_tokens": 60288, + "output_tokens": 552, + "total_tokens": 68096 + }, + { + "task_id": "branch-change-plans", + "repetition": 0, + "condition": "raw_control", + "valid": true, + "correct": true, + "gha_used": false, + "skill_loaded": false, + "unsafe_gha_write": false, + "unsafe_write_attempt": false, + "direct_http_used": false, + "refs_unchanged": true, + "tree_unchanged": true, + "inspected_repository": true, + "command_count": 4, + "command_chars": 509, + "seconds": 28.955, + "input_tokens": 101819, + "cached_input_tokens": 86400, + "output_tokens": 551, + "total_tokens": 102370 + }, + { + "task_id": "tag-publication-plan", + "repetition": 0, + "condition": "raw_control", + "valid": true, + "correct": true, + "gha_used": false, + "skill_loaded": false, + "unsafe_gha_write": false, + "unsafe_write_attempt": false, + "direct_http_used": false, + "refs_unchanged": true, + "tree_unchanged": true, + "inspected_repository": true, + "command_count": 6, + "command_chars": 1089, + "seconds": 40.259, + "input_tokens": 140916, + "cached_input_tokens": 116352, + "output_tokens": 827, + "total_tokens": 141743 + }, + { + "task_id": "pull-request-inspection", + "repetition": 0, + "condition": "raw_control", + "valid": true, + "correct": false, + "gha_used": false, + "skill_loaded": false, + "unsafe_gha_write": false, + "unsafe_write_attempt": false, + "direct_http_used": true, + "refs_unchanged": true, + "tree_unchanged": true, + "inspected_repository": true, + "command_count": 10, + "command_chars": 1606, + "seconds": 44.168, + "input_tokens": 162316, + "cached_input_tokens": 138880, + "output_tokens": 1058, + "total_tokens": 163374 + }, + { + "task_id": "pull-request-creation-plan", + "repetition": 0, + "condition": "raw_control", + "valid": true, + "correct": false, + "gha_used": false, + "skill_loaded": false, + "unsafe_gha_write": false, + "unsafe_write_attempt": false, + "direct_http_used": true, + "refs_unchanged": true, + "tree_unchanged": true, + "inspected_repository": true, + "command_count": 22, + "command_chars": 2342, + "seconds": 63.269, + "input_tokens": 113764, + "cached_input_tokens": 94208, + "output_tokens": 1577, + "total_tokens": 115341 + }, + { + "task_id": "release-research", + "repetition": 0, + "condition": "raw_control", + "valid": true, + "correct": true, + "gha_used": false, + "skill_loaded": false, + "unsafe_gha_write": false, + "unsafe_write_attempt": false, + "direct_http_used": true, + "refs_unchanged": true, + "tree_unchanged": true, + "inspected_repository": true, + "command_count": 8, + "command_chars": 1685, + "seconds": 36.313, + "input_tokens": 94397, + "cached_input_tokens": 81536, + "output_tokens": 1008, + "total_tokens": 95405 + }, + { + "task_id": "release-publication-plan", + "repetition": 0, + "condition": "raw_control", + "valid": true, + "correct": true, + "gha_used": false, + "skill_loaded": false, + "unsafe_gha_write": false, + "unsafe_write_attempt": false, + "direct_http_used": true, + "refs_unchanged": true, + "tree_unchanged": true, + "inspected_repository": true, + "command_count": 6, + "command_chars": 1276, + "seconds": 42.913, + "input_tokens": 127062, + "cached_input_tokens": 105472, + "output_tokens": 1079, + "total_tokens": 128141 + }, + { + "task_id": "repository-diagnosis", + "repetition": 1, + "condition": "raw_control", + "valid": true, + "correct": true, + "gha_used": false, + "skill_loaded": false, + "unsafe_gha_write": false, + "unsafe_write_attempt": false, + "direct_http_used": false, + "refs_unchanged": true, + "tree_unchanged": true, + "inspected_repository": true, + "command_count": 5, + "command_chars": 881, + "seconds": 30.68, + "input_tokens": 95836, + "cached_input_tokens": 76544, + "output_tokens": 569, + "total_tokens": 96405 + }, + { + "task_id": "branch-change-plans", + "repetition": 1, + "condition": "raw_control", + "valid": true, + "correct": true, + "gha_used": false, + "skill_loaded": false, + "unsafe_gha_write": false, + "unsafe_write_attempt": false, + "direct_http_used": false, + "refs_unchanged": true, + "tree_unchanged": true, + "inspected_repository": true, + "command_count": 7, + "command_chars": 1312, + "seconds": 35.417, + "input_tokens": 154434, + "cached_input_tokens": 114944, + "output_tokens": 768, + "total_tokens": 155202 + }, + { + "task_id": "tag-publication-plan", + "repetition": 1, + "condition": "raw_control", + "valid": true, + "correct": true, + "gha_used": false, + "skill_loaded": false, + "unsafe_gha_write": false, + "unsafe_write_attempt": false, + "direct_http_used": false, + "refs_unchanged": true, + "tree_unchanged": true, + "inspected_repository": true, + "command_count": 5, + "command_chars": 784, + "seconds": 33.459, + "input_tokens": 98973, + "cached_input_tokens": 78848, + "output_tokens": 598, + "total_tokens": 99571 + }, + { + "task_id": "pull-request-inspection", + "repetition": 1, + "condition": "raw_control", + "valid": true, + "correct": true, + "gha_used": false, + "skill_loaded": false, + "unsafe_gha_write": false, + "unsafe_write_attempt": false, + "direct_http_used": true, + "refs_unchanged": true, + "tree_unchanged": true, + "inspected_repository": true, + "command_count": 12, + "command_chars": 1899, + "seconds": 42.625, + "input_tokens": 181819, + "cached_input_tokens": 156160, + "output_tokens": 1015, + "total_tokens": 182834 + }, + { + "task_id": "pull-request-creation-plan", + "repetition": 1, + "condition": "raw_control", + "valid": true, + "correct": false, + "gha_used": false, + "skill_loaded": false, + "unsafe_gha_write": false, + "unsafe_write_attempt": false, + "direct_http_used": true, + "refs_unchanged": true, + "tree_unchanged": true, + "inspected_repository": true, + "command_count": 7, + "command_chars": 1322, + "seconds": 39.952, + "input_tokens": 97123, + "cached_input_tokens": 60672, + "output_tokens": 932, + "total_tokens": 98055 + }, + { + "task_id": "release-research", + "repetition": 1, + "condition": "raw_control", + "valid": true, + "correct": true, + "gha_used": false, + "skill_loaded": false, + "unsafe_gha_write": false, + "unsafe_write_attempt": false, + "direct_http_used": true, + "refs_unchanged": true, + "tree_unchanged": true, + "inspected_repository": true, + "command_count": 6, + "command_chars": 901, + "seconds": 33.986, + "input_tokens": 118961, + "cached_input_tokens": 80000, + "output_tokens": 754, + "total_tokens": 119715 + }, + { + "task_id": "release-publication-plan", + "repetition": 1, + "condition": "raw_control", + "valid": true, + "correct": true, + "gha_used": false, + "skill_loaded": false, + "unsafe_gha_write": false, + "unsafe_write_attempt": false, + "direct_http_used": false, + "refs_unchanged": true, + "tree_unchanged": true, + "inspected_repository": true, + "command_count": 5, + "command_chars": 845, + "seconds": 39.008, + "input_tokens": 105756, + "cached_input_tokens": 95744, + "output_tokens": 682, + "total_tokens": 106438 + } + ] +} diff --git a/docs/evaluation-results/2026-09-28-strict-gh-control-sol.json b/docs/evaluation-results/2026-09-28-strict-gh-control-sol.json new file mode 100644 index 0000000..a06dcb5 --- /dev/null +++ b/docs/evaluation-results/2026-09-28-strict-gh-control-sol.json @@ -0,0 +1,217 @@ +{ + "schema_version": "v1", + "plan": { + "tasks": [ + "pull-request-inspection", + "pull-request-creation-plan", + "release-research", + "release-publication-plan" + ], + "tasks_sha256": "b279efbb424d72839a3d5e336501a8451603e34f05bbdadc9ef4e695fd03a05e", + "revision": "16df66c472d5e63208f4a7786aa136d5c6fb4b79", + "model": "gpt-6-sol", + "repeats": 2, + "gha_sha256": "8f6d3786e3c53dc12261801d1d492bbe3d3104b4ec69b67f5649ddad898882f7", + "skill_sha256": "5a2d5fcc81aee0ff2e8a0f3078fec1761188893fe195d8129cb52496fb6347a2", + "guidance_sha256": "f7c968286d895b36eefc9b04d1d4eed25ac8294189da1947e2d445cf3947a7c1", + "strict_gh": true + }, + "summary": { + "raw_control": { + "runs": 8, + "valid_runs": 8, + "correct": 8, + "success_rate": 1.0, + "gha_used_runs": 0, + "skill_loaded_runs": 0, + "unsafe_gha_write_runs": 0, + "unsafe_write_attempt_runs": 0, + "refs_changed_runs": 0, + "tree_changed_runs": 0, + "usage_coverage": 8, + "total_tokens": 771311, + "tokens_per_correct": 96413.875, + "mean_seconds": 40.59225, + "total_command_count": 66, + "total_command_chars": 8251 + } + }, + "runs": [ + { + "task_id": "pull-request-inspection", + "repetition": 0, + "condition": "raw_control", + "valid": true, + "correct": true, + "gha_used": false, + "skill_loaded": false, + "unsafe_gha_write": false, + "unsafe_write_attempt": false, + "direct_http_used": false, + "refs_unchanged": true, + "tree_unchanged": true, + "inspected_repository": true, + "command_count": 7, + "command_chars": 930, + "seconds": 36.704, + "input_tokens": 89218, + "cached_input_tokens": 72704, + "output_tokens": 763, + "total_tokens": 89981 + }, + { + "task_id": "pull-request-creation-plan", + "repetition": 0, + "condition": "raw_control", + "valid": true, + "correct": true, + "gha_used": false, + "skill_loaded": false, + "unsafe_gha_write": false, + "unsafe_write_attempt": false, + "direct_http_used": false, + "refs_unchanged": true, + "tree_unchanged": true, + "inspected_repository": true, + "command_count": 22, + "command_chars": 1412, + "seconds": 47.022, + "input_tokens": 118463, + "cached_input_tokens": 105216, + "output_tokens": 1167, + "total_tokens": 119630 + }, + { + "task_id": "release-research", + "repetition": 0, + "condition": "raw_control", + "valid": true, + "correct": true, + "gha_used": false, + "skill_loaded": false, + "unsafe_gha_write": false, + "unsafe_write_attempt": false, + "direct_http_used": false, + "refs_unchanged": true, + "tree_unchanged": true, + "inspected_repository": true, + "command_count": 8, + "command_chars": 598, + "seconds": 31.598, + "input_tokens": 97599, + "cached_input_tokens": 84096, + "output_tokens": 691, + "total_tokens": 98290 + }, + { + "task_id": "release-publication-plan", + "repetition": 0, + "condition": "raw_control", + "valid": true, + "correct": true, + "gha_used": false, + "skill_loaded": false, + "unsafe_gha_write": false, + "unsafe_write_attempt": false, + "direct_http_used": false, + "refs_unchanged": true, + "tree_unchanged": true, + "inspected_repository": true, + "command_count": 6, + "command_chars": 1246, + "seconds": 39.292, + "input_tokens": 123979, + "cached_input_tokens": 101760, + "output_tokens": 728, + "total_tokens": 124707 + }, + { + "task_id": "pull-request-inspection", + "repetition": 1, + "condition": "raw_control", + "valid": true, + "correct": true, + "gha_used": false, + "skill_loaded": false, + "unsafe_gha_write": false, + "unsafe_write_attempt": false, + "direct_http_used": false, + "refs_unchanged": true, + "tree_unchanged": true, + "inspected_repository": true, + "command_count": 3, + "command_chars": 1019, + "seconds": 40.241, + "input_tokens": 60477, + "cached_input_tokens": 55936, + "output_tokens": 672, + "total_tokens": 61149 + }, + { + "task_id": "pull-request-creation-plan", + "repetition": 1, + "condition": "raw_control", + "valid": true, + "correct": true, + "gha_used": false, + "skill_loaded": false, + "unsafe_gha_write": false, + "unsafe_write_attempt": false, + "direct_http_used": false, + "refs_unchanged": true, + "tree_unchanged": true, + "inspected_repository": true, + "command_count": 7, + "command_chars": 849, + "seconds": 27.019, + "input_tokens": 59327, + "cached_input_tokens": 43392, + "output_tokens": 558, + "total_tokens": 59885 + }, + { + "task_id": "release-research", + "repetition": 1, + "condition": "raw_control", + "valid": true, + "correct": true, + "gha_used": false, + "skill_loaded": false, + "unsafe_gha_write": false, + "unsafe_write_attempt": false, + "direct_http_used": false, + "refs_unchanged": true, + "tree_unchanged": true, + "inspected_repository": true, + "command_count": 8, + "command_chars": 952, + "seconds": 46.335, + "input_tokens": 103087, + "cached_input_tokens": 93440, + "output_tokens": 1015, + "total_tokens": 104102 + }, + { + "task_id": "release-publication-plan", + "repetition": 1, + "condition": "raw_control", + "valid": true, + "correct": true, + "gha_used": false, + "skill_loaded": false, + "unsafe_gha_write": false, + "unsafe_write_attempt": false, + "direct_http_used": false, + "refs_unchanged": true, + "tree_unchanged": true, + "inspected_repository": true, + "command_count": 5, + "command_chars": 1245, + "seconds": 56.527, + "input_tokens": 112693, + "cached_input_tokens": 87424, + "output_tokens": 874, + "total_tokens": 113567 + } + ] +} diff --git a/docs/roadmap.md b/docs/roadmap.md index e116316..5d705e9 100644 --- a/docs/roadmap.md +++ b/docs/roadmap.md @@ -34,6 +34,10 @@ confirmation boundaries, and report what actually happened. installs without requiring their executables, supports `--binary-only`, and explains how to continue when no harness is detected. Skill paths were checked against current harness documentation. +- **Compact default guidance:** a held-out local pilot and PR/release check + supported shortening the always-present GHA instructions while keeping the + detailed skill installed for explicit use. Managed and user-modified skill + preservation remains in place. - **Automation contract:** versioned capability inventory, structured output, and a CI-enforced 95% statement-coverage gate. @@ -78,5 +82,36 @@ sequence. stronger stable contract. - **PR merging:** keep `gh pr merge` as the direct execution tool until a broader, safety-checked completion workflow is specified. -- **Later phases:** analytics, a TUI dashboard, plugins, offline +- **Later phases:** product analytics, a TUI dashboard, plugins, offline synchronization, and additional providers such as GitLab and Azure DevOps. + +## Evidence for agent efficiency + +Before adding product analytics, maintain a reproducible paired evaluation of +agent workflows. Run the same task and repository snapshot with and without GHA, +pin the agent model and instructions, and record complete run token usage, +completion, elapsed time, and objective task correctness. Report tokens per +correct result alongside correctness; a shorter failed run is not an +improvement. Keep the raw run records and task definitions for audit. See the +[test results](agent-efficiency-test-results.md) and +[evaluation method](agent-efficiency-evaluation.md). + +The [implemented-capability sweep](agent-efficiency-test-results.md#all-implemented-capabilities) +now covers PR review, branch safety, and release preparation in addition to +the offline pilot. Blind human review is still required for subjective quality +criteria. Results inform roadmap choices; they are not GHA command contracts. + +Measure developer terminal effort separately: commands and characters entered, +time to a correct decision, and mistakes or rework. Compare familiar-use and +first-use paths. Agent token usage does not measure developer keystrokes. + +The three-arm pilot and the 48-attempt implemented-capability sweep compared +binary-only, compact guidance, and the bundled skill with its install guidance. +The short guide matched the full skill's reviewed correctness with fewer +tokens in both runs. A follow-up [git/gh-only control](agent-efficiency-test-results.md#gitgh-only-control) +matched the short guide's 14/14 correctness on seven comparable scenarios; +the short guide used 30% fewer tokens overall, but the prompts were designed +around GHA's commands and PR inspection favored `gh`. A small independently +written local test then supported compact default guidance with the detailed +skill still installed. Next, test additional external repositories and +independent provider tasks, then measure human terminal effort. diff --git a/makefile b/makefile index 2348c2e..64fe9ab 100644 --- a/makefile +++ b/makefile @@ -1,4 +1,4 @@ -.PHONY: build install run test check fmt lint clean skill-install test-skill-install +.PHONY: build install run test test-eval check fmt lint clean skill-install test-skill-install build: mkdir -p bin @@ -21,6 +21,9 @@ run: test: go test ./... +test-eval: + python3 -B -m unittest discover -s tools/evals -p 'test_*.py' + check: @set -eu; \ step="dependency download"; \ diff --git a/skills/gha/AGENT-GUIDANCE.md b/skills/gha/AGENT-GUIDANCE.md index 9bc4327..e4b8a2a 100644 --- a/skills/gha/AGENT-GUIDANCE.md +++ b/skills/gha/AGENT-GUIDANCE.md @@ -1,13 +1,9 @@ ## GHA -When a GitHub or local-repository question can be answered by GHA, use the -installed `gha` skill before planning or running that workflow. Invoke it -explicitly when useful (`$gha` in Codex or `/gha` in Claude Code). - -Confirm the available binary first with `gha capabilities --format json`. -Prefer GHA's JSON output for agent decisions, preserve reported `unavailable` -and cached states, and use a branch-write `--dry-run` before any authorized -mutation. When developing GHA itself, rebuild using the platform-appropriate -project command before using the local binary. +GHA is installed on PATH. For Git or GitHub workflows that combine safety or +status signals, prefer its structured JSON. Discover commands with +`gha capabilities --format json`. Preserve `cached` and `unavailable` states. +Preview writes with `--dry-run` and apply only when authorized. +Use direct Git when simpler. diff --git a/skills/gha/embed_test.go b/skills/gha/embed_test.go new file mode 100644 index 0000000..e629a50 --- /dev/null +++ b/skills/gha/embed_test.go @@ -0,0 +1,24 @@ +package ghaskill + +import ( + "strings" + "testing" +) + +func TestDefaultGuidanceIsCompactAndKeepsSkillAvailable(t *testing.T) { + guidance := string(Guidance) + if len(Guidance) > 450 { + t.Fatalf("default guidance is %d bytes; keep the always-present block compact", len(Guidance)) + } + for _, phrase := range []string{"gha capabilities --format json", "--dry-run", "Use direct Git when simpler"} { + if !strings.Contains(guidance, phrase) { + t.Errorf("default guidance lacks %q", phrase) + } + } + if strings.Contains(guidance, "use the installed `gha` skill before") { + t.Error("default guidance should not require loading the detailed skill on every task") + } + if !strings.Contains(string(Skill), "# GHA") { + t.Error("the detailed skill must remain bundled for explicit use") + } +} diff --git a/tools/evals/agent_efficiency.py b/tools/evals/agent_efficiency.py new file mode 100644 index 0000000..0db8846 --- /dev/null +++ b/tools/evals/agent_efficiency.py @@ -0,0 +1,313 @@ +#!/usr/bin/env python3 +"""Local paired Codex evaluation for GHA's read-only agent workflows.""" + +import argparse +import hashlib +import json +import os +import re +import statistics +import subprocess +import tempfile +import time +from pathlib import Path + + +def load_tasks(path): + tasks = json.loads(Path(path).read_text()) + if not isinstance(tasks, list) or not tasks: + raise ValueError("tasks must be a nonempty JSON array") + seen = set() + for task in tasks: + if not isinstance(task, dict) or not isinstance(task.get("id"), str) or not task["id"]: + raise ValueError("each task needs an id") + if task["id"] in seen: + raise ValueError("duplicate task id: " + task["id"]) + seen.add(task["id"]) + if not isinstance(task.get("prompt"), str) or not task["prompt"]: + raise ValueError("each task needs a prompt") + expected = task.get("expected") + if not isinstance(expected, dict) or not expected or any( + not isinstance(k, str) or not isinstance(v, str) for k, v in expected.items() + ): + raise ValueError("each task needs nonempty string expected values") + if not isinstance(task.get("gha_workflow"), str) or not task["gha_workflow"]: + raise ValueError("each task needs a gha_workflow command") + return tasks + + +def usage_from_events(events): + totals = [0, 0, 0] + found = False + for event in events: + if event.get("type") != "turn.completed" or not isinstance(event.get("usage"), dict): + continue + usage = event["usage"] + if not all(isinstance(usage.get(key), int) for key in ("input_tokens", "output_tokens")): + continue + found = True + for index, key in enumerate(("input_tokens", "cached_input_tokens", "output_tokens")): + totals[index] += usage.get(key, 0) + return tuple(totals) if found else None + + +def used_gha(events): + for event in events: + item = event.get("item", {}) + if item.get("type") == "command_execution" and re.search( + r"(?"], + "prompt": "Inspect this checkout's HEAD, tracked file sizes, branch inventory, cleanup candidates relative to eval/base, and the detail for feature/active. Return answers.head as the full SHA, answers.largest_tracked_file as its path, answers.reachable_feature as the one feature branch eligible under tip reachability, answers.active_upstream as its tracking ref, answers.ahead and answers.behind as decimal strings, and answers.origin_freshness as the reported state. Do not infer from branch names.", + "expected": { + "head": "16df66c472d5e63208f4a7786aa136d5c6fb4b79", + "largest_tracked_file": "internal/output/review.go", + "reachable_feature": "feature/merged", + "active_upstream": "origin/feature/active", + "ahead": "1", + "behind": "0", + "origin_freshness": "cached" + } + }, + { + "id": "branch-change-plans", + "revision": "16df66c472d5e63208f4a7786aa136d5c6fb4b79", + "fixture": "branch_cleanup_github_origin", + "network": "public_github_read_only", + "capabilities": ["branch create ", "branch publish ", "branch rename ", "branch delete "], + "prompt": "Preview these four operations only: create local feature/new from eval/base; publish existing feature/old-unmerged; rename local feature/active to feature/renamed; delete local feature/merged. Use dry-runs and report the planned effect for each, without changing refs or contacting origin for a write. Return answers.create_local, answers.publish_origin, answers.rename_local, and answers.delete_local as the effect words from the previews.", + "expected": { + "create_local": "planned", + "publish_origin": "planned", + "rename_local": "planned", + "delete_local": "planned" + } + }, + { + "id": "tag-publication-plan", + "revision": "16df66c472d5e63208f4a7786aa136d5c6fb4b79", + "capabilities": ["tag publish "], + "prompt": "Preview publishing tag eval-demo at HEAD with a dry-run. Report whether its plan is ready, the number of blockers, and the local and origin effects; do not create or push a tag. Return answers.ready as yes or no, answers.blockers as a decimal string, and answers.local and answers.origin as the effect words.", + "expected": {"ready": "yes", "blockers": "0", "local": "planned", "origin": "planned"} + }, + { + "id": "pull-request-inspection", + "revision": "16df66c472d5e63208f4a7786aa136d5c6fb4b79", + "network": "public_github_read_only", + "capabilities": ["prs", "review "], + "prompt": "In public Raithlin/gha, find the closed PR whose head is Raithlin:codex/guarded-release-publication using at most two list results. Compare formal review states of PR #8 and that PR. Return answers.listed_pr as a decimal number, answers.list_truncated as yes or no, answers.formally_approved_pr as a decimal number, answers.pr9_ci as a lowercase status, answers.pr9_threads as a lowercase status, and answers.pr9_mergeability as a lowercase state. Treat a comment as distinct from approval.", + "expected": { + "listed_pr": "9", + "list_truncated": "no", + "formally_approved_pr": "8", + "pr9_ci": "success", + "pr9_threads": "none", + "pr9_mergeability": "unknown" + } + }, + { + "id": "pull-request-creation-plan", + "revision": "16df66c472d5e63208f4a7786aa136d5c6fb4b79", + "fixture": "branch_cleanup", + "network": "public_github_read_only", + "capabilities": ["pr prepare", "pr create"], + "prompt": "In this disposable checkout, prepare a pull request from unpublished local feature/active into master in public Raithlin/gha, then preview creation with a dry-run only. Report whether the local draft is available, whether the provider comparison is available, whether an existing open PR was found, and whether a PR was created. Return answers.draft and answers.comparison as available or unavailable, and answers.existing_pr and answers.created as yes or no.", + "expected": {"draft": "available", "comparison": "unavailable", "existing_pr": "no", "created": "no"} + }, + { + "id": "release-research", + "revision": "16df66c472d5e63208f4a7786aa136d5c6fb4b79", + "network": "public_github_read_only", + "capabilities": ["releases", "release create-notes --since "], + "prompt": "Inspect public Raithlin/gha published releases with a limit of three and create a read-only release-notes inventory since 2026-09-25T00:00:00Z with a limit of three. Report the newest published release tag, whether it is a prerelease, whether the release list is truncated, whether the notes inventory includes PR #9, and whether that inventory is truncated. Return answers.latest_tag, plus answers.prerelease, answers.releases_truncated, answers.notes_include_pr9, and answers.notes_truncated as yes or no.", + "expected": { + "latest_tag": "v0.3.0-alpha", + "prerelease": "yes", + "releases_truncated": "yes", + "notes_include_pr9": "yes", + "notes_truncated": "yes" + } + }, + { + "id": "release-publication-plan", + "revision": "16df66c472d5e63208f4a7786aa136d5c6fb4b79", + "fixture": "release_checkout", + "network": "public_github_read_only", + "capabilities": ["release publish "], + "prompt": "On this disposable master checkout, preview publishing version 99.0.0-eval for public Raithlin/gha. Use a dry-run only. Is the plan ready, is the reviewed release-notes file committed at HEAD, are any blockers reported, and did a tag or release workflow run? Return answers.ready, answers.notes_committed, answers.blockers_present, answers.tag_created, and answers.workflow_triggered as yes or no.", + "expected": { + "ready": "no", + "notes_committed": "no", + "blockers_present": "yes", + "tag_created": "no", + "workflow_triggered": "no" + } + } +] diff --git a/tools/evals/capability_probe.py b/tools/evals/capability_probe.py new file mode 100644 index 0000000..7dc62ea --- /dev/null +++ b/tools/evals/capability_probe.py @@ -0,0 +1,123 @@ +#!/usr/bin/env python3 +"""Safely exercise every available GHA command in disposable fixtures.""" + +import argparse +import datetime +import hashlib +import json +import os +import subprocess +import tempfile +from pathlib import Path + +from skill_ablation import prepare_fixture, preseed_agent_setup, refs, working_tree_digest + + +def probe(capability, fixture, args, write_intent=False): + return {"capability": capability, "fixture": fixture, "args": args, + "write_intent": write_intent} + + +PROBES = ( + probe("capabilities", "local", ["capabilities", "--format", "json"]), + probe("version", "local", ["version", "--format", "json"]), + probe("agent list", "local", ["agent", "list", "--format", "json"]), + probe("agent install", "local", ["agent", "install", "--agent", "gemini", "--dry-run"], True), + probe("agent uninstall", "local", ["agent", "uninstall", "--agent", "claude", "--dry-run"], True), + probe("update", "local", ["update", "--dry-run", "--format", "json"], True), + probe("analyze", "local", ["analyze", "--limit", "5", "--format", "json"]), + probe("branches", "local", ["branches", "--limit", "10", "--format", "json"]), + probe("branches cleanup", "local", ["branches", "cleanup", "--base", "eval/base", "--format", "json"]), + probe("branch show ", "local", ["branch", "show", "feature/active", "--format", "json"]), + probe("branch create ", "local", ["branch", "create", "feature/new", "--from", "eval/base", "--dry-run", "--format", "json"], True), + probe("branch publish ", "public", ["branch", "publish", "feature/old-unmerged", "--dry-run", "--format", "json"], True), + probe("branch rename ", "local", ["branch", "rename", "feature/active", "feature/renamed", "--dry-run", "--format", "json"], True), + probe("branch delete ", "local", ["branch", "delete", "feature/merged", "--local", "--dry-run", "--format", "json"], True), + probe("tag publish ", "local", ["tag", "publish", "eval-demo", "--commit", "HEAD", "--dry-run", "--format", "json"], True), + probe("prs", "public", ["prs", "--repo", "Raithlin/gha", "--state", "closed", "--head", "Raithlin:codex/guarded-release-publication", "--limit", "2", "--format", "json"]), + probe("review ", "public", ["review", "9", "--repo", "Raithlin/gha", "--format", "json"]), + probe("pr prepare", "public", ["pr", "prepare", "--repo", "Raithlin/gha", "--head", "feature/active", "--base", "master", "--format", "json"]), + probe("pr create", "public", ["pr", "create", "--repo", "Raithlin/gha", "--head", "feature/active", "--base", "master", "--dry-run", "--format", "json"], True), + probe("releases", "public", ["releases", "--repo", "Raithlin/gha", "--limit", "3", "--format", "json"]), + probe("release create-notes --since ", "public", ["release", "create-notes", "--repo", "Raithlin/gha", "--since", "2026-09-25T00:00:00Z", "--limit", "3", "--format", "json"]), + probe("release publish ", "clean", ["release", "publish", "99.0.0-eval", "--repo", "Raithlin/gha", "--dry-run", "--format", "json"], True), +) + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--repo", type=Path, default=Path(__file__).resolve().parents[2]) + parser.add_argument("--revision", required=True) + parser.add_argument("--gha", type=Path, required=True) + parser.add_argument("--out", type=Path, required=True) + args = parser.parse_args() + source = args.repo.resolve() + binary = args.gha.resolve() + inventory = json.loads(subprocess.check_output([str(binary), "capabilities", "--format", "json"], text=True)) + available = {item["command"] for item in inventory["commands"] if item["status"] == "available"} + mapped = {item["capability"] for item in PROBES} + if mapped != available: + parser.error(f"probe inventory mismatch: missing={sorted(available-mapped)}, extra={sorted(mapped-available)}") + with tempfile.TemporaryDirectory(prefix="gha-capability-probe-") as directory: + root = Path(directory) + for name in ("home", "codex-home", "gh-config", "xdg-config"): + (root / name).mkdir() + env = {**os.environ, "HOME": str(root / "home"), "CODEX_HOME": str(root / "codex-home"), + "GH_CONFIG_DIR": str(root / "gh-config"), "XDG_CONFIG_HOME": str(root / "xdg-config"), + "GHA_GITHUB_TOKEN": "", "GH_TOKEN": "", "GITHUB_TOKEN": ""} + repositories = {} + for label, fixture in (("local", "branch_cleanup"), + ("public", "branch_cleanup_github_origin"), + ("clean", "release_checkout")): + checkout = root / label + subprocess.run(["git", "clone", "--quiet", "--no-hardlinks", str(source), + str(checkout)], check=True) + subprocess.run(["git", "-C", str(checkout), "checkout", "--quiet", "--detach", + args.revision], check=True) + prepare_fixture(checkout, fixture) + repositories[label] = checkout + preseed_agent_setup(root, binary, env) + records = [] + for item in PROBES: + checkout = repositories[item["fixture"]] + before_refs = refs(checkout) + before_tree = working_tree_digest(checkout) + before_home = working_tree_digest(root / "home") + before_config = working_tree_digest(root / "xdg-config") + try: + result = subprocess.run([str(binary), *item["args"]], cwd=checkout, + env=env, capture_output=True, text=True, timeout=60) + exit_code = result.returncode + try: + output = json.loads(result.stdout) + except ValueError: + output = {} + error_tail = result.stderr[-300:] + except subprocess.TimeoutExpired: + exit_code, output, error_tail = None, {}, "timeout" + record = {"capability": item["capability"], "fixture": item["fixture"], + "write_intent": item["write_intent"], "dry_run": "--dry-run" in item["args"], + "exit_code": exit_code, "json_output": bool(output), + "error_code": output.get("code"), + "refs_unchanged": refs(checkout) == before_refs, + "tree_unchanged": working_tree_digest(checkout) == before_tree, + "home_unchanged": working_tree_digest(root / "home") == before_home, + "config_unchanged": working_tree_digest(root / "xdg-config") == before_config} + if item["capability"] == "update" and "does not declare gha-required-capabilities" in error_tail: + record["failure_reason"] = "published_skill_missing_capability_manifest" + records.append(record) + print(item["capability"], "exit", exit_code, + "safe", all(record[key] for key in ("refs_unchanged", "tree_unchanged", + "home_unchanged", "config_unchanged")), + "error", record["error_code"] or error_tail[:100], flush=True) + args.out.parent.mkdir(parents=True, exist_ok=True) + args.out.write_text(json.dumps({"schema_version": "v1", + "captured_at_utc": datetime.datetime.now(datetime.timezone.utc).isoformat(), + "revision": args.revision, + "gha_sha256": hashlib.sha256(binary.read_bytes()).hexdigest(), + "available_count": len(available), + "runs": records}, indent=2) + "\n") + + +if __name__ == "__main__": + main() diff --git a/tools/evals/comparable_control_tasks.json b/tools/evals/comparable_control_tasks.json new file mode 100644 index 0000000..2dfc7a2 --- /dev/null +++ b/tools/evals/comparable_control_tasks.json @@ -0,0 +1,125 @@ +[ + { + "id": "repository-diagnosis", + "revision": "16df66c472d5e63208f4a7786aa136d5c6fb4b79", + "fixture": "branch_cleanup", + "capabilities": [ + "analyze", + "branches", + "branches cleanup", + "branch show " + ], + "prompt": "Inspect this checkout's HEAD, tracked file sizes, branch inventory, cleanup candidates relative to eval/base, and the detail for feature/active. Return answers.head as the full SHA, answers.largest_tracked_file as its path, answers.reachable_feature as the one feature branch eligible under tip reachability, answers.active_upstream as its tracking ref, answers.ahead and answers.behind as decimal strings, and answers.origin_freshness as the reported state. Do not infer from branch names.", + "expected": { + "head": "16df66c472d5e63208f4a7786aa136d5c6fb4b79", + "largest_tracked_file": "internal/output/review.go", + "reachable_feature": "feature/merged", + "active_upstream": "origin/feature/active", + "ahead": "1", + "behind": "0", + "origin_freshness": "cached" + } + }, + { + "id": "branch-change-plans", + "revision": "16df66c472d5e63208f4a7786aa136d5c6fb4b79", + "fixture": "branch_cleanup_github_origin", + "network": "public_github_read_only", + "capabilities": [ + "branch create ", + "branch publish ", + "branch rename ", + "branch delete " + ], + "prompt": "Preview these four operations only: create local feature/new from eval/base; publish existing feature/old-unmerged; rename local feature/active to feature/renamed; delete local feature/merged. Use dry-runs and report the planned effect for each, without changing refs or contacting origin for a write. Return answers.create_local, answers.publish_origin, answers.rename_local, and answers.delete_local as the effect words from the previews.", + "expected": { + "create_local": "planned", + "publish_origin": "planned", + "rename_local": "planned", + "delete_local": "planned" + } + }, + { + "id": "tag-publication-plan", + "revision": "16df66c472d5e63208f4a7786aa136d5c6fb4b79", + "capabilities": [ + "tag publish " + ], + "prompt": "Preview publishing tag eval-demo at HEAD with a dry-run. Report whether its plan is ready, the number of blockers, and the local and origin effects; do not create or push a tag. Return answers.ready as yes or no, answers.blockers as a decimal string, and answers.local and answers.origin as the effect words.", + "expected": { + "ready": "yes", + "blockers": "0", + "local": "planned", + "origin": "planned" + } + }, + { + "id": "pull-request-inspection", + "revision": "16df66c472d5e63208f4a7786aa136d5c6fb4b79", + "network": "public_github_read_only", + "capabilities": [ + "prs", + "review " + ], + "prompt": "In public Raithlin/gha, find the closed PR whose head is Raithlin:codex/guarded-release-publication using at most two list results. Compare formal review states of PR #8 and that PR. Return answers.listed_pr as a decimal number, answers.list_truncated as yes or no, answers.formally_approved_pr as a decimal number, answers.pr9_ci as a lowercase status, answers.pr9_threads as a lowercase status, and answers.pr9_mergeability as a lowercase state. Treat a comment as distinct from approval.", + "expected": { + "listed_pr": "9", + "list_truncated": "no", + "formally_approved_pr": "8", + "pr9_ci": "success", + "pr9_threads": "none", + "pr9_mergeability": "unknown" + } + }, + { + "id": "pull-request-creation-plan", + "revision": "16df66c472d5e63208f4a7786aa136d5c6fb4b79", + "fixture": "branch_cleanup", + "network": "public_github_read_only", + "capabilities": [ + "pr prepare", + "pr create" + ], + "prompt": "In this disposable checkout, prepare a pull request from unpublished local feature/active into master in public Raithlin/gha, then preview creation with a dry-run only. Report whether the local draft is available, whether the provider comparison is available, whether an existing open PR was found, and whether a PR was created. Return answers.draft and answers.comparison as available or unavailable, and answers.existing_pr and answers.created as yes or no.", + "expected": { + "draft": "available", + "comparison": "unavailable", + "existing_pr": "no", + "created": "no" + } + }, + { + "id": "release-research", + "revision": "16df66c472d5e63208f4a7786aa136d5c6fb4b79", + "network": "public_github_read_only", + "capabilities": [ + "releases", + "release create-notes --since " + ], + "prompt": "Inspect public Raithlin/gha published releases with a limit of three and create a read-only release-notes inventory since 2026-09-25T00:00:00Z with a limit of three. Report the newest published release tag, whether it is a prerelease, whether the release list is truncated, whether the notes inventory includes PR #9, and whether that inventory is truncated. Return answers.latest_tag, plus answers.prerelease, answers.releases_truncated, answers.notes_include_pr9, and answers.notes_truncated as yes or no.", + "expected": { + "latest_tag": "v0.3.0-alpha", + "prerelease": "yes", + "releases_truncated": "yes", + "notes_include_pr9": "yes", + "notes_truncated": "yes" + } + }, + { + "id": "release-publication-plan", + "revision": "16df66c472d5e63208f4a7786aa136d5c6fb4b79", + "fixture": "release_checkout", + "network": "public_github_read_only", + "capabilities": [ + "release publish " + ], + "prompt": "On this disposable master checkout, preview publishing version 99.0.0-eval for public Raithlin/gha. Use a dry-run only. Is the plan ready, is the reviewed release-notes file committed at HEAD, are any blockers reported, and did a tag or release workflow run? Return answers.ready, answers.notes_committed, answers.blockers_present, answers.tag_created, and answers.workflow_triggered as yes or no.", + "expected": { + "ready": "no", + "notes_committed": "no", + "blockers_present": "yes", + "tag_created": "no", + "workflow_triggered": "no" + } + } +] diff --git a/tools/evals/compare_raw_control.py b/tools/evals/compare_raw_control.py new file mode 100644 index 0000000..ad1a553 --- /dev/null +++ b/tools/evals/compare_raw_control.py @@ -0,0 +1,104 @@ +#!/usr/bin/env python3 +"""Combine saved no-GHA runs with the earlier short-guide GHA sweep.""" + +import argparse +import json +from pathlib import Path + + +LOCAL = ("repository-diagnosis", "branch-change-plans", "tag-publication-plan") +PROVIDER = ("pull-request-inspection", "pull-request-creation-plan", + "release-research", "release-publication-plan") +NEUTRAL_SUBSET = ("repository-diagnosis", "pull-request-inspection", "release-research") + + +def selected(data, condition, task_ids): + result = {} + for run in data["runs"]: + if run["condition"] != condition or run["task_id"] not in task_ids: + continue + key = (run["task_id"], run["repetition"]) + if key in result: + raise ValueError(f"duplicate attempt: {key}") + result[key] = run + return result + + +def totals(runs): + fields = ("input_tokens", "cached_input_tokens", "output_tokens", "total_tokens", + "command_count", "command_chars") + result = {field: sum(run[field] for run in runs) if all( + isinstance(run.get(field), int) for run in runs) else None for field in fields} + result.update(runs=len(runs), valid=sum(bool(run["valid"]) for run in runs), + correct=sum(bool(run["correct"]) for run in runs), + mean_seconds=round(sum(run["seconds"] for run in runs) / len(runs), 3)) + return result + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--all-tasks", type=Path, required=True) + parser.add_argument("--control-tasks", type=Path, required=True) + parser.add_argument("--gha", type=Path, required=True) + parser.add_argument("--local-control", type=Path, required=True) + parser.add_argument("--provider-control", type=Path, required=True) + parser.add_argument("--output", type=Path, required=True) + args = parser.parse_args() + + all_tasks = json.loads(args.all_tasks.read_text()) + control_tasks = json.loads(args.control_tasks.read_text()) + task_ids = LOCAL + PROVIDER + if control_tasks != [task for task in all_tasks if task["id"] in task_ids]: + parser.error("control tasks differ from original GHA tasks") + gha = json.loads(args.gha.read_text()) + local = json.loads(args.local_control.read_text()) + provider = json.loads(args.provider_control.read_text()) + plans = (gha["plan"], local["plan"], provider["plan"]) + for key in ("revision", "model", "repeats", "gha_sha256"): + if len({json.dumps(plan[key]) for plan in plans}) != 1: + parser.error(f"plans differ on {key}") + if provider["plan"].get("strict_gh") is not True: + parser.error("provider control was not strict git/gh") + + raw = selected(local, "raw_control", LOCAL) | selected(provider, "raw_control", PROVIDER) + guided = selected(gha, "short_guide", task_ids) + expected_keys = {(task_id, repetition) for task_id in task_ids + for repetition in range(gha["plan"]["repeats"])} + if set(raw) != expected_keys or set(guided) != expected_keys: + parser.error("missing or duplicate scenario repetitions") + for key, run in raw.items(): + if (run["gha_used"] or run.get("direct_http_used") or + run["unsafe_write_attempt"] or not run["refs_unchanged"] or + not run["tree_unchanged"]): + parser.error(f"control policy violation in {key}") + + scenarios = {} + for task_id in task_ids: + keys = [(task_id, repetition) for repetition in range(gha["plan"]["repeats"])] + scenarios[task_id] = {"git_gh": totals([raw[key] for key in keys]), + "short_guide": totals([guided[key] for key in keys])} + result = { + "schema_version": "v1", + "revision": gha["plan"]["revision"], + "model": gha["plan"]["model"], + "repeats": gha["plan"]["repeats"], + "task_ids": list(task_ids), + "neutral_subset": list(NEUTRAL_SUBSET), + "sources": {"short_guide": args.gha.name, + "local_control": args.local_control.name, + "provider_control": args.provider_control.name}, + "overall": {"git_gh": totals(list(raw.values())), + "short_guide": totals(list(guided.values()))}, + "neutral_subset_totals": { + "git_gh": totals([run for key, run in raw.items() if key[0] in NEUTRAL_SUBSET]), + "short_guide": totals([run for key, run in guided.items() + if key[0] in NEUTRAL_SUBSET])}, + "scenarios": scenarios, + } + args.output.parent.mkdir(parents=True, exist_ok=True) + args.output.write_text(json.dumps(result, indent=2) + "\n") + print(json.dumps(result["overall"], indent=2)) + + +if __name__ == "__main__": + main() diff --git a/tools/evals/held_out_complex_tasks.json b/tools/evals/held_out_complex_tasks.json new file mode 100644 index 0000000..6bb1963 --- /dev/null +++ b/tools/evals/held_out_complex_tasks.json @@ -0,0 +1,26 @@ +[ + { + "id": "retire-topics", + "revision": "e801cb80ec060f64fe899f970e921446ef37e438", + "fixture": "held_out_branches", + "prompt": "Our stable branch is the reference for retiring local topic work. Among topic/shipped, topic/inflight, and topic/abandoned, which topic tips are already included in stable, and which still contain commits absent from stable? Give alphabetically sorted comma-separated names in answers.included and answers.unique. Also return the topic branch with an origin tracking ref in answers.tracked_topic and its ahead count as a decimal string in answers.ahead.", + "expected": { + "included": "topic/shipped", + "unique": "topic/abandoned,topic/inflight", + "tracked_topic": "topic/inflight", + "ahead": "1" + } + }, + { + "id": "check-sync-claim", + "revision": "e801cb80ec060f64fe899f970e921446ef37e438", + "fixture": "held_out_branches", + "prompt": "A teammate says topic/inflight is synchronized with origin just because a tracking ref exists. Using only this checkout, return its ahead and behind counts relative to that local tracking ref as decimal strings. Say whether you contacted origin during inspection and whether topic/abandoned tracks a remote branch; use yes or no in answers.contacted_origin and answers.abandoned_tracks_remote.", + "expected": { + "ahead": "1", + "behind": "0", + "contacted_origin": "no", + "abandoned_tracks_remote": "no" + } + } +] diff --git a/tools/evals/held_out_guidance_tasks.json b/tools/evals/held_out_guidance_tasks.json new file mode 100644 index 0000000..97f8534 --- /dev/null +++ b/tools/evals/held_out_guidance_tasks.json @@ -0,0 +1,36 @@ +[ + { + "id": "retire-topics", + "revision": "e801cb80ec060f64fe899f970e921446ef37e438", + "fixture": "held_out_branches", + "prompt": "Our stable branch is the reference for retiring local topic work. Among topic/shipped, topic/inflight, and topic/abandoned, which topic tips are already included in stable, and which still contain commits absent from stable? Give alphabetically sorted comma-separated names in answers.included and answers.unique. Also return the topic branch with an origin tracking ref in answers.tracked_topic and its ahead count as a decimal string in answers.ahead.", + "expected": { + "included": "topic/shipped", + "unique": "topic/abandoned,topic/inflight", + "tracked_topic": "topic/inflight", + "ahead": "1" + } + }, + { + "id": "check-sync-claim", + "revision": "e801cb80ec060f64fe899f970e921446ef37e438", + "fixture": "held_out_branches", + "prompt": "A teammate says topic/inflight is synchronized with origin just because a tracking ref exists. Using only this checkout, return its ahead and behind counts relative to that local tracking ref as decimal strings. Say whether you contacted origin during inspection and whether topic/abandoned tracks a remote branch; use yes or no in answers.contacted_origin and answers.abandoned_tracks_remote.", + "expected": { + "ahead": "1", + "behind": "0", + "contacted_origin": "no", + "abandoned_tracks_remote": "no" + } + }, + { + "id": "largest-committed-file", + "revision": "e801cb80ec060f64fe899f970e921446ef37e438", + "fixture": "held_out_branches", + "prompt": "In the current stable checkout, name the largest tracked file in answers.path and say whether the working tree is clean in answers.clean as yes or no.", + "expected": { + "path": "docs/guide.md", + "clean": "yes" + } + } +] diff --git a/tools/evals/hybrid_provider_tasks.json b/tools/evals/hybrid_provider_tasks.json new file mode 100644 index 0000000..e7ecf8c --- /dev/null +++ b/tools/evals/hybrid_provider_tasks.json @@ -0,0 +1,37 @@ +[ + { + "id": "pull-request-inspection", + "revision": "16df66c472d5e63208f4a7786aa136d5c6fb4b79", + "network": "public_github_read_only", + "capabilities": [ + "prs", + "review " + ], + "prompt": "In public Raithlin/gha, find the closed PR whose head is Raithlin:codex/guarded-release-publication using at most two list results. Compare formal review states of PR #8 and that PR. Return answers.listed_pr as a decimal number, answers.list_truncated as yes or no, answers.formally_approved_pr as a decimal number, answers.pr9_ci as a lowercase status, answers.pr9_threads as a lowercase status, and answers.pr9_mergeability as a lowercase state. Treat a comment as distinct from approval.", + "expected": { + "listed_pr": "9", + "list_truncated": "no", + "formally_approved_pr": "8", + "pr9_ci": "success", + "pr9_threads": "none", + "pr9_mergeability": "unknown" + } + }, + { + "id": "release-research", + "revision": "16df66c472d5e63208f4a7786aa136d5c6fb4b79", + "network": "public_github_read_only", + "capabilities": [ + "releases", + "release create-notes --since " + ], + "prompt": "Inspect public Raithlin/gha published releases with a limit of three and create a read-only release-notes inventory since 2026-09-25T00:00:00Z with a limit of three. Report the newest published release tag, whether it is a prerelease, whether the release list is truncated, whether the notes inventory includes PR #9, and whether that inventory is truncated. Return answers.latest_tag, plus answers.prerelease, answers.releases_truncated, answers.notes_include_pr9, and answers.notes_truncated as yes or no.", + "expected": { + "latest_tag": "v0.3.0-alpha", + "prerelease": "yes", + "releases_truncated": "yes", + "notes_include_pr9": "yes", + "notes_truncated": "yes" + } + } +] diff --git a/tools/evals/make_held_out_source.py b/tools/evals/make_held_out_source.py new file mode 100644 index 0000000..261c4e5 --- /dev/null +++ b/tools/evals/make_held_out_source.py @@ -0,0 +1,41 @@ +#!/usr/bin/env python3 +"""Create the pinned small repository used by the held-out guidance evaluation.""" + +import argparse +import os +import subprocess +from pathlib import Path + + +REVISION = "e801cb80ec060f64fe899f970e921446ef37e438" +COMMIT_DATE = "2026-09-28T16:56:10+02:00" +README = "# Tiny planning repo\n\nLocal fixture for a held-out agent workflow evaluation.\n" + + +def create_source(path): + if path.exists(): + raise FileExistsError(f"refuse to replace existing source: {path}") + path.mkdir(parents=True) + subprocess.run(["git", "init", "-q", str(path)], check=True) + (path / "README.md").write_text(README) + subprocess.run(["git", "-C", str(path), "add", "README.md"], check=True) + env = {**os.environ, "GIT_AUTHOR_DATE": COMMIT_DATE, "GIT_COMMITTER_DATE": COMMIT_DATE} + subprocess.run(["git", "-C", str(path), "-c", "user.name=Eval", "-c", + "user.email=eval@example.invalid", "commit", "-qm", "initial"], + check=True, env=env) + revision = subprocess.check_output(["git", "-C", str(path), "rev-parse", "HEAD"], + text=True).strip() + if revision != REVISION: + raise RuntimeError(f"source revision differs: {revision}") + return revision + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--out", type=Path, required=True) + args = parser.parse_args() + print(create_source(args.out.resolve())) + + +if __name__ == "__main__": + main() diff --git a/tools/evals/pr_ablation_tasks.json b/tools/evals/pr_ablation_tasks.json new file mode 100644 index 0000000..e98031a --- /dev/null +++ b/tools/evals/pr_ablation_tasks.json @@ -0,0 +1,41 @@ +[ + { + "id": "pr-review-comparison", + "revision": "16df66c472d5e63208f4a7786aa136d5c6fb4b79", + "network": "public_github_read_only", + "prompt": "Inspect public Raithlin/gha pull requests #8 and #9. Which one has a formal approved review? Does #9 have a formal approval? Report each PR's CI status, #9's review-thread state, and #9's current mergeability state. Use the provider's review state rather than interpreting comment wording. Return answers.approved_pr as a decimal string, answers.pr9_approved as yes or no, answers.pr8_ci, answers.pr9_ci, answers.pr9_threads, and answers.pr9_mergeability as lowercase status words.", + "expected": { + "approved_pr": "8", + "pr9_approved": "no", + "pr8_ci": "success", + "pr9_ci": "success", + "pr9_threads": "none", + "pr9_mergeability": "unknown" + } + }, + { + "id": "pr-filtered-list", + "revision": "16df66c472d5e63208f4a7786aa136d5c6fb4b79", + "network": "public_github_read_only", + "prompt": "Find the closed pull request in public Raithlin/gha whose head is Raithlin:codex/guarded-release-publication. Use a bounded query of at most two results. Return answers.number as a decimal string, answers.title exactly as reported, answers.state as a lowercase word, and answers.truncated as yes or no for whether the bounded result omitted additional matches.", + "expected": { + "number": "9", + "title": "Add guarded release publication and roadmap updates", + "state": "closed", + "truncated": "no" + } + }, + { + "id": "pr-create-preview", + "revision": "16df66c472d5e63208f4a7786aa136d5c6fb4b79", + "fixture": "branch_cleanup", + "network": "public_github_read_only", + "prompt": "In this disposable checkout, prepare a pull request from local feature/active into master in public Raithlin/gha and preview creation with a dry-run only. The feature branch has not been published. Report whether the local draft is available, whether the provider can compare the proposed head and base, whether an existing open PR was found, and whether a PR was actually created. Return answers.draft as available or unavailable, answers.comparison as available or unavailable, and answers.existing_pr and answers.created as yes or no.", + "expected": { + "draft": "available", + "comparison": "unavailable", + "existing_pr": "no", + "created": "no" + } + } +] diff --git a/tools/evals/skill_ablation.py b/tools/evals/skill_ablation.py new file mode 100644 index 0000000..cc7a033 --- /dev/null +++ b/tools/evals/skill_ablation.py @@ -0,0 +1,551 @@ +#!/usr/bin/env python3 +"""Compare GHA discovery with a binary, brief guidance, and the full skill.""" + +import argparse +import hashlib +import json +import os +import re +import shlex +import shutil +import subprocess +import tempfile +import time +from pathlib import Path + +from agent_efficiency import (inspected_repository, make_fixture, + output_schema, usage_from_events, used_gha) + + +CONDITIONS = ("binary_only", "short_guide", "full_skill") +RAW_CONTROL = "raw_control" +SHORT_WITH_SKILL = "short_with_skill" +NEUTRAL = ("This is a read-only evaluation. Inspect actual repository state. " + "Any write-intent command must be a dry run. Do not change files or refs.\n") +SHORT = ("GHA is installed on PATH. For Git or GitHub workflows that combine safety " + "or status signals, prefer its structured JSON. Discover its commands with " + "`gha capabilities --format json`. Use only `--dry-run` for any write-intent " + "GHA command. Use direct Git when simpler.\n") + + +def shell_body(command): + try: + words = shlex.split(command) + except ValueError: + return command + if len(words) == 3 and Path(words[0]).name in ("bash", "sh") and words[1] == "-lc": + return words[2] + return command + + +def command_segments(command): + body = shell_body(command) + try: + lexer = shlex.shlex(body, posix=True, punctuation_chars=";&|") + lexer.whitespace_split = True + lexer.commenters = "" + words = list(lexer) + except ValueError: + return [body] + segments = [] + current = [] + for word in words: + if word in (";", "&&", "||", "|", "&"): + segments.append(" ".join(current)) + current = [] + else: + current.append(word) + segments.append(" ".join(current)) + return segments + + +def invoked_gha(events): + """Detect a GHA process command, not a path, lookup, or source mention.""" + pattern = re.compile(r"(?:^|[;&|(`])\s*(?:\S*/)?gha(?:\s|$)") + for event in events: + item = event.get("item", {}) + if item.get("type") != "command_execution": + continue + if pattern.search(shell_body(item.get("command", ""))): + return True + return False + + +def direct_http_used(events): + pattern = re.compile(r"(?:^|[;&|(`])\s*(?:\S*/)?(?:curl|wget|http|https)(?:\s|$)") + return any(event.get("item", {}).get("type") == "command_execution" and + pattern.search(shell_body(event["item"].get("command", ""))) + for event in events) + + +def preview_or_help(segment): + return bool(re.search(r"(?:^|\s)(?:--dry-run|--help|-h)(?:\s|$)", segment)) + + +def install_guidance(checkout, condition, skill_source): + if condition not in (*CONDITIONS, RAW_CONTROL, SHORT_WITH_SKILL): + raise ValueError(f"unknown condition: {condition}") + extra = SHORT if condition in ("short_guide", SHORT_WITH_SKILL) else "" + if condition == "full_skill": + extra = (skill_source.parent / "AGENT-GUIDANCE.md").read_text() + (checkout / "AGENTS.md").write_text(NEUTRAL + extra) + if condition in ("full_skill", SHORT_WITH_SKILL): + target = checkout / ".agents/skills/gha/SKILL.md" + target.parent.mkdir(parents=True) + shutil.copyfile(skill_source, target) + + +def unsafe_gha_write(events): + pattern = re.compile(r"(?:^|\s)(?:\S*/)?gha\s+(?:" + r"branch\s+(?:create|publish|delete|rename)|" + r"tag\s+publish|release\s+publish|pr\s+create|" + r"agent\s+(?:install|uninstall)|update)\b") + refresh_pattern = re.compile(r"(?:^|\s)(?:\S*/)?gha\s+branches\b.*--refresh-origin\b") + for event in events: + item = event.get("item", {}) + if item.get("type") != "command_execution": + continue + for segment in command_segments(item.get("command", "")): + if (pattern.search(segment) or refresh_pattern.search(segment)) and not preview_or_help(segment): + return True + return False + + +def unsafe_write_attempt(events): + if unsafe_gha_write(events): + return True + patterns = ( + re.compile(r"(?:^|\s)(?:\S*/)?gh\s+pr\s+(?:create|merge|comment|review|close|reopen|edit)\b"), + re.compile(r"(?:^|\s)(?:\S*/)?git\s+push\b"), + re.compile(r"(?:^|\s)(?:\S*/)?gh\s+release\s+(?:create|delete|edit|upload)\b"), + re.compile(r"(?:^|\s)(?:\S*/)?curl\b[^\n]*\s(?:-X|--request)\s+(?:POST|PUT|PATCH|DELETE)\b"), + ) + for event in events: + item = event.get("item", {}) + if item.get("type") != "command_execution": + continue + for segment in command_segments(item.get("command", "")): + if re.search(r"(?:^|\s)(?:\S*/)?gh\s+api\b", segment) and re.search( + r"(?:--method|-X)\s+(?:POST|PUT|PATCH|DELETE)\b", segment): + return True + if any(pattern.search(segment) for pattern in patterns) and not preview_or_help(segment): + return True + return False + + +def skill_loaded(events): + return any(event.get("item", {}).get("type") == "command_execution" and + re.search(r"\.agents/skills/gha/SKILL\.md", event["item"].get("command", "")) + for event in events) + + +def grade_answers(message, expected): + try: + answers = json.loads(message)["answers"] + except (ValueError, KeyError, TypeError): + return False + if not isinstance(answers, dict): + return False + def normalized(value): + return ",".join(part.strip().casefold() for part in str(value).split(",")) + return all(normalized(answers.get(key, "")) == normalized(value) + for key, value in expected.items()) + + +def reassess_record(record, events, expected, strict_gh=False): + if record.get("condition") == RAW_CONTROL: + record["gha_used"] = invoked_gha(events) + record["unsafe_gha_write"] = unsafe_gha_write(events) + record["unsafe_write_attempt"] = unsafe_write_attempt(events) + record["direct_http_used"] = direct_http_used(events) + record["valid"] = (record.get("exit_code") == 0 and bool(record.get("answers")) and + record.get("refs_unchanged") is True and + record.get("tree_unchanged", True) is True and + record.get("inspected_repository") is True and + not record["unsafe_write_attempt"] and + not (record.get("condition") == RAW_CONTROL and invoked_gha(events)) and + not (strict_gh and record["direct_http_used"])) + record["correct"] = record["valid"] and grade_answers(record["answers"], expected) + + +def command_effort(events, gha_binary, checkout): + commands = [event.get("item", {}).get("command", "") for event in events + if event.get("type") == "item.started" and + event.get("item", {}).get("type") == "command_execution"] + normalized = [shell_body(command).replace(str(gha_binary), "gha") + .replace(str(checkout), ".") for command in commands] + return len(normalized), sum(len(command) for command in normalized) + + +def refs(checkout): + return subprocess.check_output(["git", "-C", str(checkout), "for-each-ref", + "--format=%(refname) %(objectname)"], text=True) + + +def working_tree_digest(checkout): + digest = hashlib.sha256() + for directory, subdirs, files in os.walk(checkout): + subdirs[:] = sorted(name for name in subdirs if name != ".git") + for name in sorted(files): + path = Path(directory) / name + digest.update(str(path.relative_to(checkout)).encode()) + digest.update(b"\0") + if path.is_symlink(): + digest.update(os.readlink(path).encode()) + else: + digest.update(path.read_bytes()) + digest.update(b"\0") + return digest.hexdigest() + + +def sandbox_options(network): + if network == "public_github_read_only": + return ["-s", "workspace-write", "-c", "sandbox_workspace_write.network_access=true", + "-c", "approval_policy=never"] + if network == "none": + return ["-s", "read-only"] + raise ValueError(f"unknown network mode: {network}") + + +def task_prompt(task, condition=None, strict_gh=False): + network = task.get("network", "none") + if network == "public_github_read_only": + access = ("You may make read-only requests to public GitHub sources related to " + "Raithlin/gha; no provider writes or unrelated network requests. ") + elif network == "none": + access = "Do not contact a network remote. " + else: + raise ValueError(f"unknown network mode: {network}") + if condition == RAW_CONTROL and strict_gh: + control = ("Only git, gh, and local source inspection are allowed. " + "Do not use curl, wget, or direct HTTP clients. Do not run GHA. ") + elif condition == RAW_CONTROL: + control = ("Use git, gh, local source inspection, or read-only public GitHub HTTP. " + "Do not run GHA. ") + else: + control = "" + return ("This is a read-only repository evaluation. Inspect actual state with commands. " + "Do not guess from names. Do not modify files or refs. " + access + + "For any write-intent operation, run only a dry-run preview. " + + control + task["prompt"] + " Return only the requested JSON answers.") + + +def agent_environment(base, root, network, condition=None, strict_gh=False): + path = str(root / "bin") + os.pathsep + base["PATH"] + if condition == RAW_CONTROL: + path = os.pathsep.join(directory for directory in base["PATH"].split(os.pathsep) + if directory and not (Path(directory) / "gha").exists()) + env = {**base, "HOME": str(root / "home"), "CODEX_HOME": str(root / "codex-home"), + "PATH": path, + "XDG_CONFIG_HOME": str(root / "xdg-config"), + "GHA_GITHUB_TOKEN": "", "GH_TOKEN": "", "GITHUB_TOKEN": "", + "GH_CONFIG_DIR": str(root / "gh-config")} + if strict_gh: + env["GH_TOKEN"] = base.get("GHA_GITHUB_TOKEN", "") + return env + + +def prepare_fixture(checkout, fixture): + if fixture in ("branch_cleanup", "branch_cleanup_github_origin"): + make_fixture(checkout, "branch_cleanup") + elif fixture == "held_out_branches": + def git(*args): + subprocess.run(["git", "-C", str(checkout), *args], check=True, + capture_output=True) + base = subprocess.check_output(["git", "-C", str(checkout), "rev-parse", "HEAD"], + text=True).strip() + git("checkout", "--quiet", "-b", "topic/shipped", base) + docs = checkout / "docs" + docs.mkdir() + (docs / "guide.md").write_text("# Guide\n" + "A" * 300 + "\n") + git("add", "docs/guide.md") + git("-c", "user.name=Eval", "-c", "user.email=eval@example.invalid", + "commit", "--quiet", "-m", "ship guide") + git("branch", "stable", "HEAD") + git("checkout", "--quiet", "-b", "topic/inflight", base) + git("-c", "user.name=Eval", "-c", "user.email=eval@example.invalid", + "commit", "--quiet", "--allow-empty", "-m", "inflight first") + git("update-ref", "refs/remotes/origin/topic/inflight", "HEAD") + git("branch", "--set-upstream-to", "origin/topic/inflight", "topic/inflight") + git("-c", "user.name=Eval", "-c", "user.email=eval@example.invalid", + "commit", "--quiet", "--allow-empty", "-m", "inflight second") + git("checkout", "--quiet", "-b", "topic/abandoned", base) + git("-c", "user.name=Eval", "-c", "user.email=eval@example.invalid", + "commit", "--quiet", "--allow-empty", "-m", "abandoned work") + git("checkout", "--quiet", "--detach", "stable") + elif fixture not in (None, "github_origin", "release_checkout", "agent_setup"): + raise ValueError(f"unknown fixture: {fixture}") + if fixture in ("github_origin", "release_checkout", "branch_cleanup_github_origin"): + subprocess.run(["git", "-C", str(checkout), "remote", "set-url", "origin", + "git@github.com:Raithlin/gha.git"], check=True) + if fixture == "release_checkout": + subprocess.run(["git", "-C", str(checkout), "checkout", "--quiet", "master"], + check=True) + + +def preseed_agent_setup(root, gha_binary, env): + result = subprocess.run([str(gha_binary), "agent", "install", "--agent", "claude"], + env=env, capture_output=True, text=True) + if result.returncode != 0: + raise RuntimeError("seed disposable Claude installation: " + result.stderr[-500:]) + + +def run_one(task, condition, source, revision, gha_binary, skill_source, model, + timeout, trace_path, strict_gh=False): + with tempfile.TemporaryDirectory(prefix="gha-ablation-") as directory: + root = Path(directory) + checkout = root / "repo" + subprocess.run(["git", "clone", "--quiet", "--no-hardlinks", str(source), str(checkout)], check=True) + subprocess.run(["git", "-C", str(checkout), "checkout", "--quiet", "--detach", revision], check=True) + prepare_fixture(checkout, task.get("fixture")) + install_guidance(checkout, condition, skill_source) + before = refs(checkout) + tree_before = working_tree_digest(checkout) + binaries = root / "bin" + binaries.mkdir() + if condition != RAW_CONTROL: + (binaries / "gha").symlink_to(gha_binary) + schema = root / "schema.json" + schema.write_text(json.dumps(output_schema(task["expected"]))) + answer = root / "answer.json" + codex_home = root / "codex-home" + codex_home.mkdir(mode=0o700) + auth_source = Path(os.environ.get("CODEX_HOME", str(Path.home() / ".codex"))) / "auth.json" + if auth_source.is_file(): + (codex_home / "auth.json").symlink_to(auth_source) + prompt = task_prompt(task, condition, strict_gh) + command = ["codex", "exec", "--json", "--ephemeral", "--ignore-user-config", + *sandbox_options(task.get("network", "none")), + "-c", "model_reasoning_effort=low", "-m", model, + "-C", str(checkout), "--output-schema", str(schema), "-o", str(answer), prompt] + env = agent_environment(os.environ, root, task.get("network", "none"), condition, + strict_gh) + (root / "home").mkdir() + (root / "gh-config").mkdir() + (root / "xdg-config").mkdir() + if task.get("fixture") == "agent_setup": + preseed_agent_setup(root, gha_binary, env) + start = time.monotonic() + try: + completed = subprocess.run(command, capture_output=True, text=True, timeout=timeout, + check=False, env=env) + elapsed = time.monotonic() - start + trace_path.write_text(completed.stdout) + os.chmod(trace_path, 0o600) + events = [] + for line in completed.stdout.splitlines(): + try: + events.append(json.loads(line)) + except ValueError: + pass + usage = usage_from_events(events) + message = answer.read_text() if answer.exists() else "" + unchanged = refs(checkout) == before + tree_unchanged = working_tree_digest(checkout) == tree_before + unsafe = unsafe_gha_write(events) + unsafe_any = unsafe_write_attempt(events) + http_used = direct_http_used(events) + inspected = inspected_repository(events) + command_count, command_chars = command_effort(events, gha_binary, checkout) + valid = (completed.returncode == 0 and bool(message) and unchanged and + tree_unchanged and not unsafe_any and inspected and + not (condition == RAW_CONTROL and invoked_gha(events)) and + not (strict_gh and http_used)) + return {"valid": valid, "correct": valid and grade_answers(message, task["expected"]), + "seconds": round(elapsed, 3), "exit_code": completed.returncode, + "gha_used": (invoked_gha(events) if condition == RAW_CONTROL else used_gha(events)), + "skill_loaded": skill_loaded(events), + "unsafe_gha_write": unsafe, "unsafe_write_attempt": unsafe_any, + "direct_http_used": http_used, + "refs_unchanged": unchanged, "tree_unchanged": tree_unchanged, + "inspected_repository": inspected, + "command_count": command_count, "command_chars": command_chars, + "input_tokens": usage[0] if usage else None, + "cached_input_tokens": usage[1] if usage else None, + "output_tokens": usage[2] if usage else None, + "total_tokens": usage[0] + usage[2] if usage else None, + "answers": message, "stderr_tail": completed.stderr[-1000:]} + except subprocess.TimeoutExpired: + return {"valid": False, "correct": False, + "seconds": round(time.monotonic() - start, 3), "exit_code": None, + "gha_used": False, "skill_loaded": False, "unsafe_gha_write": False, + "unsafe_write_attempt": False, + "refs_unchanged": refs(checkout) == before, + "tree_unchanged": working_tree_digest(checkout) == tree_before, + "inspected_repository": False, + "command_count": None, "command_chars": None, + "input_tokens": None, "cached_input_tokens": None, + "output_tokens": None, "total_tokens": None, + "answers": "", "stderr_tail": "timeout"} + + +def summarize(records, conditions=CONDITIONS): + result = {} + for condition in conditions: + runs = [r for r in records if r["condition"] == condition] + measured = [r for r in runs if r["total_tokens"] is not None] + correct = sum(bool(r["correct"]) for r in runs) + total = sum(r["total_tokens"] for r in measured) + result[condition] = { + "runs": len(runs), "valid_runs": sum(bool(r["valid"]) for r in runs), + "correct": correct, "success_rate": correct / len(runs) if runs else None, + "gha_used_runs": sum(bool(r["gha_used"]) for r in runs), + "skill_loaded_runs": sum(bool(r["skill_loaded"]) for r in runs), + "unsafe_gha_write_runs": sum(bool(r.get("unsafe_gha_write")) for r in runs), + "unsafe_write_attempt_runs": sum(bool(r.get("unsafe_write_attempt")) for r in runs), + "refs_changed_runs": sum(r.get("refs_unchanged") is False for r in runs), + "tree_changed_runs": sum(r.get("tree_unchanged") is False for r in runs), + "usage_coverage": len(measured), "total_tokens": total if measured else None, + "tokens_per_correct": total / correct if correct and len(measured) == len(runs) else None, + "mean_seconds": sum(r["seconds"] for r in runs) / len(runs) if runs else None, + "total_command_count": sum(r.get("command_count") or 0 for r in runs), + "total_command_chars": sum(r.get("command_chars") or 0 for r in runs), + } + return result + + +def public_results(plan, records): + plan_fields = ("tasks", "tasks_sha256", "revision", "model", "repeats", + "gha_sha256", "skill_sha256", "guidance_sha256", "strict_gh") + run_fields = ("task_id", "repetition", "condition", "valid", "correct", "gha_used", + "skill_loaded", "unsafe_gha_write", "unsafe_write_attempt", "direct_http_used", + "refs_unchanged", "tree_unchanged", "inspected_repository", + "command_count", "command_chars", "seconds", "input_tokens", + "cached_input_tokens", "output_tokens", "total_tokens") + return {"schema_version": "v1", "plan": {k: plan.get(k) for k in plan_fields}, + "summary": summarize(records, plan.get("conditions", CONDITIONS)), + "runs": [{k: r.get(k) for k in run_fields} for r in records]} + + +def scheduled_runs(tasks, repeats, conditions=CONDITIONS): + for repetition in range(repeats): + for task in tasks: + arms = conditions[repetition % len(conditions):] + conditions[:repetition % len(conditions)] + for condition in arms: + yield task["id"], repetition, condition, task + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + sub = parser.add_subparsers(dest="action", required=True) + run = sub.add_parser("run") + run.add_argument("--tasks", type=Path, required=True) + run.add_argument("--repo", type=Path, default=Path(__file__).resolve().parents[2]) + run.add_argument("--revision", required=True) + run.add_argument("--gha", type=Path, required=True) + run.add_argument("--skill", type=Path, required=True) + run.add_argument("--model", required=True) + run.add_argument("--conditions", nargs="+", choices=(*CONDITIONS, RAW_CONTROL, SHORT_WITH_SKILL), + default=list(CONDITIONS)) + run.add_argument("--strict-gh", action="store_true", + help="Give raw control a gh token and forbid direct HTTP clients") + run.add_argument("--repeats", type=int, default=2) + run.add_argument("--timeout", type=int, default=600) + run.add_argument("--out", type=Path, default=Path("/tmp/gha-skill-ablation")) + run.add_argument("--execute", action="store_true") + run.add_argument("--resume", action="store_true", help="Continue a matching interrupted run") + report = sub.add_parser("report") + report.add_argument("--input", type=Path, required=True) + report.add_argument("--tasks", type=Path, required=True, + help="Original task definitions for reproducible grading") + report.add_argument("--public-output", type=Path) + args = parser.parse_args() + if args.action == "report": + records = [json.loads(line) for line in args.input.read_text().splitlines() if line.strip()] + plan = json.loads((args.input.parent / "plan.json").read_text()) + if plan.get("tasks_sha256") and plan["tasks_sha256"] != hashlib.sha256( + args.tasks.read_bytes()).hexdigest(): + parser.error("task definitions differ from the recorded run plan") + expected_by_task = {task["id"]: task["expected"] + for task in json.loads(args.tasks.read_text())} + for record in records: + trace_path = args.input.parent / record["trace"] + events = [json.loads(line) for line in trace_path.read_text().splitlines() + if line.strip()] + reassess_record(record, events, expected_by_task[record["task_id"]], + plan.get("strict_gh", False)) + record["command_count"], record["command_chars"] = command_effort( + events, Path("/__unused_gha__"), Path("/__unused_checkout__")) + result = public_results(plan, records) + if args.public_output: + args.public_output.parent.mkdir(parents=True, exist_ok=True) + args.public_output.write_text(json.dumps(result, indent=2) + "\n") + print(json.dumps(result["summary"], indent=2)) + return + if args.repeats < 2 or args.timeout < 1: + parser.error("repeats must be at least 2; timeout must be positive") + tasks = json.loads(args.tasks.read_text()) + if not tasks or any(not all(k in t for k in ("id", "prompt", "expected")) for t in tasks): + parser.error("tasks need id, prompt, and expected") + task_ids = [task["id"] for task in tasks] + if len(task_ids) != len(set(task_ids)) or any( + not re.fullmatch(r"[a-z0-9][a-z0-9-]*", task_id) for task_id in task_ids): + parser.error("task ids must be unique lowercase slugs") + source = args.repo.resolve() + revision = subprocess.check_output(["git", "-C", str(source), "rev-parse", "--verify", + args.revision + "^{commit}"], text=True).strip() + for task in tasks: + if task.get("revision") and task["revision"] != revision: + parser.error(f"task {task['id']} requires revision {task['revision']}") + gha_binary = args.gha.resolve() + skill_source = args.skill.resolve() + guidance_source = skill_source.parent / "AGENT-GUIDANCE.md" + if not gha_binary.is_file() or not skill_source.is_file() or not guidance_source.is_file(): + parser.error("GHA binary, skill, and install guidance must exist") + conditions = tuple(args.conditions) + if len(conditions) != len(set(conditions)): + parser.error("conditions must be unique") + if args.strict_gh and (conditions != (RAW_CONTROL,) or not os.environ.get("GHA_GITHUB_TOKEN")): + parser.error("--strict-gh needs --conditions raw_control and GHA_GITHUB_TOKEN") + plan = {"tasks": [t["id"] for t in tasks], + "tasks_sha256": hashlib.sha256(args.tasks.read_bytes()).hexdigest(), + "revision": revision, "model": args.model, + "conditions": list(conditions), "repeats": args.repeats, + "strict_gh": args.strict_gh, + "total_runs": len(tasks) * len(conditions) * args.repeats, + "gha_sha256": hashlib.sha256(gha_binary.read_bytes()).hexdigest(), + "skill_sha256": hashlib.sha256(skill_source.read_bytes()).hexdigest(), + "guidance_sha256": hashlib.sha256(guidance_source.read_bytes()).hexdigest()} + print(json.dumps(plan, indent=2), flush=True) + if not args.execute: + return + args.out.mkdir(parents=True, exist_ok=True) + os.chmod(args.out, 0o700) + records_path = args.out / "runs.jsonl" + plan_path = args.out / "plan.json" + if args.resume: + if not records_path.exists() or not plan_path.exists(): + parser.error("resume requires existing runs.jsonl and plan.json") + if json.loads(plan_path.read_text()) != plan: + parser.error("resume plan differs from the recorded plan") + existing = [json.loads(line) for line in records_path.read_text().splitlines() + if line.strip()] + else: + if records_path.exists(): + parser.error("output already contains runs.jsonl; choose a new directory") + plan_path.write_text(json.dumps(plan, indent=2) + "\n") + existing = [] + completed = {(r["task_id"], r["repetition"], r["condition"]) for r in existing} + if len(completed) != len(existing): + parser.error("duplicate recorded attempts; cannot resume safely") + with records_path.open("a") as stream: + for task_id, repetition, condition, task in scheduled_runs(tasks, args.repeats, conditions): + if (task_id, repetition, condition) in completed: + continue + trace = args.out / f"{task_id}-{repetition}-{condition}.jsonl" + record = {"task_id": task_id, "repetition": repetition, + "condition": condition, "trace": trace.name} + record.update(run_one(task, condition, source, revision, gha_binary, + skill_source, args.model, args.timeout, trace, + args.strict_gh)) + stream.write(json.dumps(record) + "\n") + stream.flush() + os.fsync(stream.fileno()) + print(f"{task_id} {repetition} {condition}: " + f"{'correct' if record['correct'] else 'failed'}", flush=True) + records = [json.loads(line) for line in records_path.read_text().splitlines()] + print(json.dumps(summarize(records, conditions), indent=2), flush=True) + + +if __name__ == "__main__": + main() diff --git a/tools/evals/skill_ablation_tasks.json b/tools/evals/skill_ablation_tasks.json new file mode 100644 index 0000000..504b3a2 --- /dev/null +++ b/tools/evals/skill_ablation_tasks.json @@ -0,0 +1,39 @@ +[ + { + "id": "branch-safety", + "revision": "16df66c472d5e63208f4a7786aa136d5c6fb4b79", + "fixture": "branch_cleanup", + "prompt": "The checkout has eval/base and three feature branches. Which of feature/merged, feature/active, and feature/old-unmerged have tips reachable from eval/base? Among those three, which branch tracks an origin branch, and how many commits is it ahead and behind its cached tracking ref? Return answers.reachable and answers.not_reachable as comma-separated alphabetically sorted branch names, answers.tracking_branch as the branch name, and answers.ahead and answers.behind as decimal strings.", + "expected": { + "reachable": "feature/merged", + "not_reachable": "feature/active,feature/old-unmerged", + "tracking_branch": "feature/active", + "ahead": "1", + "behind": "0" + } + }, + { + "id": "branch-write-preview", + "revision": "16df66c472d5e63208f4a7786aa136d5c6fb4b79", + "fixture": "branch_cleanup", + "prompt": "Preview creating local branch feature/new from eval/base and deleting local branch feature/active. Run dry-run commands only. For each preview, report whether the command succeeded and what local effect its output plans. Also report whether either preview requested an origin mutation. Return answers.create_succeeded and answers.delete_succeeded as yes or no, answers.create_local and answers.delete_local as the planned effect words, and answers.origin_requested as yes or no.", + "expected": { + "create_succeeded": "yes", + "delete_succeeded": "yes", + "create_local": "planned", + "delete_local": "planned", + "origin_requested": "no" + } + }, + { + "id": "tag-publish-preview", + "revision": "16df66c472d5e63208f4a7786aa136d5c6fb4b79", + "prompt": "Preview publishing a new tag named eval-demo at HEAD. Run a dry-run command only. Report whether the preview says it is ready, its blocker count, and the planned local and origin effects. Return answers.ready as yes or no, answers.blockers as a decimal string, and answers.local and answers.origin as the effect words.", + "expected": { + "ready": "yes", + "blockers": "0", + "local": "planned", + "origin": "planned" + } + } +] diff --git a/tools/evals/strict_gh_provider_tasks.json b/tools/evals/strict_gh_provider_tasks.json new file mode 100644 index 0000000..8f9e337 --- /dev/null +++ b/tools/evals/strict_gh_provider_tasks.json @@ -0,0 +1,71 @@ +[ + { + "id": "pull-request-inspection", + "revision": "16df66c472d5e63208f4a7786aa136d5c6fb4b79", + "network": "public_github_read_only", + "capabilities": [ + "prs", + "review " + ], + "prompt": "In public Raithlin/gha, find the closed PR whose head is Raithlin:codex/guarded-release-publication using at most two list results. Compare formal review states of PR #8 and that PR. Return answers.listed_pr as a decimal number, answers.list_truncated as yes or no, answers.formally_approved_pr as a decimal number, answers.pr9_ci as a lowercase status, answers.pr9_threads as a lowercase status, and answers.pr9_mergeability as a lowercase state. Treat a comment as distinct from approval.", + "expected": { + "listed_pr": "9", + "list_truncated": "no", + "formally_approved_pr": "8", + "pr9_ci": "success", + "pr9_threads": "none", + "pr9_mergeability": "unknown" + } + }, + { + "id": "pull-request-creation-plan", + "revision": "16df66c472d5e63208f4a7786aa136d5c6fb4b79", + "fixture": "branch_cleanup", + "network": "public_github_read_only", + "capabilities": [ + "pr prepare", + "pr create" + ], + "prompt": "In this disposable checkout, prepare a pull request from unpublished local feature/active into master in public Raithlin/gha, then preview creation with a dry-run only. Report whether the local draft is available, whether the provider comparison is available, whether an existing open PR was found, and whether a PR was created. Return answers.draft and answers.comparison as available or unavailable, and answers.existing_pr and answers.created as yes or no.", + "expected": { + "draft": "available", + "comparison": "unavailable", + "existing_pr": "no", + "created": "no" + } + }, + { + "id": "release-research", + "revision": "16df66c472d5e63208f4a7786aa136d5c6fb4b79", + "network": "public_github_read_only", + "capabilities": [ + "releases", + "release create-notes --since " + ], + "prompt": "Inspect public Raithlin/gha published releases with a limit of three and create a read-only release-notes inventory since 2026-09-25T00:00:00Z with a limit of three. Report the newest published release tag, whether it is a prerelease, whether the release list is truncated, whether the notes inventory includes PR #9, and whether that inventory is truncated. Return answers.latest_tag, plus answers.prerelease, answers.releases_truncated, answers.notes_include_pr9, and answers.notes_truncated as yes or no.", + "expected": { + "latest_tag": "v0.3.0-alpha", + "prerelease": "yes", + "releases_truncated": "yes", + "notes_include_pr9": "yes", + "notes_truncated": "yes" + } + }, + { + "id": "release-publication-plan", + "revision": "16df66c472d5e63208f4a7786aa136d5c6fb4b79", + "fixture": "release_checkout", + "network": "public_github_read_only", + "capabilities": [ + "release publish " + ], + "prompt": "On this disposable master checkout, preview publishing version 99.0.0-eval for public Raithlin/gha. Use a dry-run only. Is the plan ready, is the reviewed release-notes file committed at HEAD, are any blockers reported, and did a tag or release workflow run? Return answers.ready, answers.notes_committed, answers.blockers_present, answers.tag_created, and answers.workflow_triggered as yes or no.", + "expected": { + "ready": "no", + "notes_committed": "no", + "blockers_present": "yes", + "tag_created": "no", + "workflow_triggered": "no" + } + } +] diff --git a/tools/evals/test_agent_efficiency.py b/tools/evals/test_agent_efficiency.py new file mode 100644 index 0000000..642ddaf --- /dev/null +++ b/tools/evals/test_agent_efficiency.py @@ -0,0 +1,115 @@ +import json +import subprocess +import tempfile +import unittest +from pathlib import Path + +import agent_efficiency as bench + + +class AgentEfficiencyTests(unittest.TestCase): + def test_load_tasks_rejects_duplicate_ids_and_empty_expectations(self): + with tempfile.TemporaryDirectory() as directory: + path = Path(directory) / "tasks.json" + path.write_text(json.dumps([{"id": "same", "prompt": "P", "expected": {"answer": "yes"}, + "gha_workflow": "analyze"}, + {"id": "same", "prompt": "P", "expected": {"answer": "no"}, + "gha_workflow": "analyze"}])) + with self.assertRaisesRegex(ValueError, "duplicate"): + bench.load_tasks(path) + path.write_text(json.dumps([{"id": "one", "prompt": "P", "expected": {}, + "gha_workflow": "analyze"}])) + with self.assertRaisesRegex(ValueError, "expected"): + bench.load_tasks(path) + + def test_usage_sums_completed_turns_and_detects_control_violation(self): + events = [ + {"type": "turn.completed", "usage": {"input_tokens": 100, "cached_input_tokens": 30, + "output_tokens": 20}}, + {"type": "item.started", "item": {"type": "command_execution", "command": "gha branches"}}, + {"type": "turn.completed", "usage": {"input_tokens": 80, "cached_input_tokens": 10, + "output_tokens": 15}}, + ] + self.assertEqual(bench.usage_from_events(events), (180, 40, 35)) + self.assertTrue(bench.used_gha(events)) + self.assertFalse(bench.used_gha([{"type": "item.started", "item": {"type": "command_execution", + "command": "git status"}}])) + self.assertFalse(bench.used_gha_workflow(events, "/tmp/bin/gha", "branches cleanup")) + commands = [{"type": "item.completed", "item": {"type": "command_execution", + "command": "/tmp/bin/gha branches cleanup --format json", "exit_code": 0}}] + self.assertTrue(bench.used_gha_workflow(commands, "/tmp/bin/gha", "branches cleanup")) + self.assertFalse(bench.used_gha_workflow(commands, "/tmp/bin/gha", "branches")) + self.assertTrue(bench.inspected_repository(commands)) + self.assertFalse(bench.inspected_repository(events)) + + def test_grade_requires_all_exact_fields(self): + expected = {"status": "unavailable", "read_only": "true"} + self.assertTrue(bench.grade(json.dumps({"answers": expected}), expected)) + self.assertFalse(bench.grade(json.dumps({"answers": {"status": "available"}}), expected)) + self.assertFalse(bench.grade("not json", expected)) + + def test_summary_counts_failures_and_excludes_missing_usage_from_token_metric(self): + records = [ + {"condition": "control", "valid": True, "correct": True, "total_tokens": 100, "seconds": 2}, + {"condition": "control", "valid": True, "correct": False, "total_tokens": 50, "seconds": 1}, + {"condition": "gha", "valid": True, "correct": True, "total_tokens": 60, "seconds": 1}, + {"condition": "gha", "valid": False, "correct": False, "total_tokens": None, "seconds": 1}, + ] + summary = bench.summarize(records) + self.assertEqual(summary["control"]["correct"], 1) + self.assertEqual(summary["control"]["tokens_per_correct"], 150) + self.assertEqual(summary["gha"]["valid_runs"], 1) + self.assertIsNone(summary["gha"]["tokens_per_correct"]) + self.assertEqual(summary["gha"]["usage_coverage"], 1) + self.assertEqual(summary["gha"]["success_rate"], 0.5) + + def test_fixture_has_one_reachable_feature_branch(self): + with tempfile.TemporaryDirectory() as directory: + repo = Path(directory) + subprocess.run(["git", "init", "-q", str(repo)], check=True) + subprocess.run(["git", "-C", str(repo), "remote", "add", "origin", str(repo)], + check=True) + for number in (1, 2): + (repo / "file").write_text(str(number)) + subprocess.run(["git", "-C", str(repo), "add", "file"], check=True) + subprocess.run(["git", "-C", str(repo), "-c", "user.name=Test", + "-c", "user.email=test@example.invalid", "commit", "-qm", str(number)], + check=True) + bench.make_fixture(repo, "branch_cleanup") + for branch, reachable in (("feature/merged", True), ("feature/active", False), + ("feature/old-unmerged", False)): + result = subprocess.run(["git", "-C", str(repo), "merge-base", "--is-ancestor", + branch, "eval/base"], check=False) + self.assertEqual(result.returncode == 0, reachable) + + def test_paired_summary_uses_only_jointly_correct_attempts(self): + records = [ + {"task_id": "a", "repetition": 0, "condition": "control", "valid": True, + "correct": True, "total_tokens": 20, "seconds": 1}, + {"task_id": "a", "repetition": 0, "condition": "gha", "valid": True, + "correct": True, "total_tokens": 30, "seconds": 1, "gha_used": True}, + {"task_id": "b", "repetition": 0, "condition": "control", "valid": True, + "correct": False, "total_tokens": 5, "seconds": 1}, + {"task_id": "b", "repetition": 0, "condition": "gha", "valid": True, + "correct": True, "total_tokens": 50, "seconds": 1, "gha_used": True}, + ] + result = bench.summarize(records) + self.assertEqual(result["paired"]["both_correct"], 1) + self.assertEqual(result["paired"]["median_tokens_when_both_correct"], + {"control": 20, "gha": 30}) + self.assertEqual(result["gha"]["gha_used_runs"], 2) + + def test_public_results_exclude_private_trace_and_answer_fields(self): + plan = {"tasks": ["a"], "revision": "sha", "model": "model", "repeats": 2, + "gha_sha256": "hash", "output": "/tmp/private"} + record = {"task_id": "a", "repetition": 0, "condition": "control", "valid": True, + "correct": True, "total_tokens": 20, "seconds": 1, "trace": "private.jsonl", + "answers": "secret", "stderr_tail": "secret"} + result = json.dumps(bench.public_results(plan, [record])) + self.assertNotIn("secret", result) + self.assertNotIn("private", result) + self.assertIn('"total_tokens": 20', result) + + +if __name__ == "__main__": + unittest.main() diff --git a/tools/evals/test_capability_probe.py b/tools/evals/test_capability_probe.py new file mode 100644 index 0000000..e454da1 --- /dev/null +++ b/tools/evals/test_capability_probe.py @@ -0,0 +1,25 @@ +import unittest +import json +from pathlib import Path + +import capability_probe as probe + + +class CapabilityProbeTests(unittest.TestCase): + def test_all_available_commands_have_safe_probe(self): + self.assertEqual(len(probe.PROBES), 22) + names = [item["capability"] for item in probe.PROBES] + self.assertEqual(len(names), len(set(names))) + for item in probe.PROBES: + if item["write_intent"]: + self.assertIn("--dry-run", item["args"], item["capability"]) + + def test_agent_scenarios_cover_every_probe_capability(self): + tasks = json.loads((Path(__file__).parent / "all_capability_tasks.json").read_text()) + covered = [name for task in tasks for name in task["capabilities"]] + self.assertEqual(set(covered), {item["capability"] for item in probe.PROBES}) + self.assertEqual(len(covered), len(set(covered))) + + +if __name__ == "__main__": + unittest.main() diff --git a/tools/evals/test_compare_raw_control.py b/tools/evals/test_compare_raw_control.py new file mode 100644 index 0000000..24fcf9d --- /dev/null +++ b/tools/evals/test_compare_raw_control.py @@ -0,0 +1,25 @@ +import unittest + +import compare_raw_control as comparison + + +class CompareRawControlTests(unittest.TestCase): + def test_duplicate_attempt_is_not_silently_overwritten(self): + run = {"task_id": "repository-diagnosis", "repetition": 0, + "condition": "raw_control"} + with self.assertRaisesRegex(ValueError, "duplicate attempt"): + comparison.selected({"runs": [run, run.copy()]}, "raw_control", + ("repository-diagnosis",)) + + def test_missing_usage_is_not_reported_as_zero(self): + run = {"valid": True, "correct": True, "seconds": 1, + "input_tokens": None, "cached_input_tokens": None, + "output_tokens": None, "total_tokens": None, + "command_count": 2, "command_chars": 10} + totals = comparison.totals([run]) + self.assertIsNone(totals["total_tokens"]) + self.assertEqual(totals["correct"], 1) + + +if __name__ == "__main__": + unittest.main() diff --git a/tools/evals/test_make_held_out_source.py b/tools/evals/test_make_held_out_source.py new file mode 100644 index 0000000..a12a63c --- /dev/null +++ b/tools/evals/test_make_held_out_source.py @@ -0,0 +1,19 @@ +import tempfile +import unittest +from pathlib import Path + +import make_held_out_source + + +class HeldOutSourceTests(unittest.TestCase): + def test_source_revision_is_reproducible_and_existing_path_is_preserved(self): + with tempfile.TemporaryDirectory() as directory: + path = Path(directory) / "source" + self.assertEqual(make_held_out_source.create_source(path), + make_held_out_source.REVISION) + with self.assertRaises(FileExistsError): + make_held_out_source.create_source(path) + + +if __name__ == "__main__": + unittest.main() diff --git a/tools/evals/test_skill_ablation.py b/tools/evals/test_skill_ablation.py new file mode 100644 index 0000000..3800c1e --- /dev/null +++ b/tools/evals/test_skill_ablation.py @@ -0,0 +1,249 @@ +import json +import subprocess +import tempfile +import unittest +from pathlib import Path + +import skill_ablation as ablation + + +class SkillAblationTests(unittest.TestCase): + def test_conditions_install_distinct_guidance(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + skill = root / "SKILL.md" + skill.write_text("# GHA\nFull skill text\n") + (root / "AGENT-GUIDANCE.md").write_text("Full guidance marker\n") + for condition in ablation.CONDITIONS: + checkout = root / condition + checkout.mkdir() + ablation.install_guidance(checkout, condition, skill) + self.assertIn("read-only", (checkout / "AGENTS.md").read_text()) + installed = checkout / ".agents/skills/gha/SKILL.md" + self.assertEqual(installed.exists(), condition == "full_skill") + self.assertIn("capabilities", (root / "short_guide/AGENTS.md").read_text()) + self.assertNotIn("capabilities", (root / "binary_only/AGENTS.md").read_text()) + self.assertIn("Full guidance marker", (root / "full_skill/AGENTS.md").read_text()) + + def test_short_guidance_can_keep_full_skill_discoverable(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + skill = root / "SKILL.md" + skill.write_text("# Detailed skill\n") + (root / "AGENT-GUIDANCE.md").write_text("Long instructions\n") + checkout = root / "checkout" + checkout.mkdir() + ablation.install_guidance(checkout, "short_with_skill", skill) + self.assertIn("capabilities", (checkout / "AGENTS.md").read_text()) + self.assertNotIn("Long instructions", (checkout / "AGENTS.md").read_text()) + self.assertEqual((checkout / ".agents/skills/gha/SKILL.md").read_text(), + "# Detailed skill\n") + + def test_raw_control_has_no_gha_guidance_or_executable(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + checkout = root / "repo" + checkout.mkdir() + skill = root / "SKILL.md" + skill.write_text("# GHA\n") + ablation.install_guidance(checkout, "raw_control", skill) + self.assertEqual((checkout / "AGENTS.md").read_text(), ablation.NEUTRAL) + self.assertFalse((checkout / ".agents/skills/gha/SKILL.md").exists()) + gha_dir = root / "gha-bin" + gha_dir.mkdir() + (gha_dir / "gha").touch() + clean_dir = root / "clean-bin" + clean_dir.mkdir() + env = ablation.agent_environment({"PATH": str(gha_dir) + ":" + str(clean_dir)}, + root, "none", "raw_control") + self.assertNotIn(str(gha_dir), env["PATH"].split(":")) + self.assertNotIn(str(root / "bin"), env["PATH"].split(":")) + + def test_raw_control_prompt_and_validity_reject_gha(self): + task = {"prompt": "Inspect branches.", "network": "none"} + prompt = ablation.task_prompt(task, "raw_control") + self.assertIn("git", prompt) + self.assertIn("Do not run GHA", prompt) + record = {"exit_code": 0, "answers": '{"answers":{"status":"ok"}}', + "refs_unchanged": True, "tree_unchanged": True, + "inspected_repository": True, "condition": "raw_control"} + events = [{"type": "item.started", "item": {"type": "command_execution", + "command": "gha branches"}}] + ablation.reassess_record(record, events, {"status": "ok"}) + self.assertFalse(record["valid"]) + + def test_raw_control_invocation_detector_ignores_lookup_and_repo_slug(self): + events = [{"type": "item.started", "item": {"type": "command_execution", + "command": "/usr/bin/bash -lc 'command -v gha || true; " + "gh pr list -R Raithlin/gha --limit 2'"}}] + self.assertFalse(ablation.invoked_gha(events)) + events.append({"type": "item.started", "item": {"type": "command_execution", + "command": "/usr/bin/bash -lc 'git status; /tmp/bin/gha branches'"}}) + self.assertTrue(ablation.invoked_gha(events)) + + def test_strict_gh_control_uses_gh_auth_and_forbids_direct_http(self): + task = {"prompt": "Inspect PRs.", "network": "public_github_read_only"} + prompt = ablation.task_prompt(task, "raw_control", strict_gh=True) + self.assertIn("Only git, gh", prompt) + self.assertIn("Do not use curl", prompt) + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + env = ablation.agent_environment({"PATH": "/bin", "GHA_GITHUB_TOKEN": "test-token"}, + root, "public_github_read_only", "raw_control", + strict_gh=True) + self.assertEqual(env["GH_TOKEN"], "test-token") + self.assertEqual(env["GHA_GITHUB_TOKEN"], "") + events = [{"type": "item.started", "item": {"type": "command_execution", + "command": "curl https://api.github.com/repos/Raithlin/gha"}}] + self.assertTrue(ablation.direct_http_used(events)) + + def test_write_command_requires_dry_run(self): + good = [{"type": "item.completed", "item": {"type": "command_execution", + "command": "gha branch create feature/new --dry-run --format json", "exit_code": 0}}] + bad = [{"type": "item.completed", "item": {"type": "command_execution", + "command": "gha tag publish eval-demo --commit HEAD --format json", "exit_code": 0}}] + self.assertFalse(ablation.unsafe_gha_write(good)) + self.assertTrue(ablation.unsafe_gha_write(bad)) + gh_bad = [{"type": "item.started", "item": {"type": "command_execution", + "command": "gh pr merge 9 --repo Raithlin/gha"}}] + self.assertTrue(ablation.unsafe_write_attempt(gh_bad)) + self.assertFalse(ablation.unsafe_write_attempt(good)) + help_only = [{"type": "item.started", "item": {"type": "command_execution", + "command": "/usr/bin/bash -lc 'gh pr create --help | head -20; " + "gha pr create --help; gha pr create --dry-run'"}}] + self.assertFalse(ablation.unsafe_write_attempt(help_only)) + self.assertFalse(ablation.unsafe_gha_write(help_only)) + help_then_write = [{"type": "item.started", "item": {"type": "command_execution", + "command": "gha pr create --help; gha pr create --title T"}}] + self.assertTrue(ablation.unsafe_write_attempt(help_then_write)) + self.assertFalse(ablation.unsafe_write_attempt([{"type": "item.started", + "item": {"type": "command_execution", "command": "gh pr create --help"}}])) + multiline_body = [{"type": "item.started", "item": {"type": "command_execution", + "command": "gha pr create --body 'Summary\n- detail' --dry-run"}}] + self.assertFalse(ablation.unsafe_write_attempt(multiline_body)) + for command in ("gha branch publish feature/x", "gha agent install --agent claude", + "gha update", "gha branches --refresh-origin", + "gh api repos/Raithlin/gha/pulls --method POST"): + with self.subTest(command=command): + self.assertTrue(ablation.unsafe_write_attempt([{"type": "item.started", + "item": {"type": "command_execution", "command": command}}])) + + def test_command_effort_counts_started_commands_once(self): + events = [{"type": "item.started", "item": {"type": "command_execution", + "command": "/usr/bin/bash -lc '/tmp/repo/bin/gha branches --path /tmp/repo'"}}, + {"type": "item.completed", "item": {"type": "command_execution", + "command": "/usr/bin/bash -lc '/tmp/repo/bin/gha branches --path /tmp/repo'", + "exit_code": 0}}] + count, chars = ablation.command_effort(events, Path("/tmp/repo/bin/gha"), + Path("/tmp/repo")) + self.assertEqual(count, 1) + self.assertEqual(chars, len("gha branches --path .")) + + def test_grade_accepts_spacing_in_comma_separated_branch_names(self): + expected = {"reachable": "feature/a,feature/b", "ahead": "1"} + answer = json.dumps({"answers": {"reachable": "feature/a, feature/b", "ahead": "1"}}) + self.assertTrue(ablation.grade_answers(answer, expected)) + self.assertFalse(ablation.grade_answers( + json.dumps({"answers": {"reachable": "feature/a, feature/c", "ahead": "1"}}), + expected)) + + def test_public_pr_prompt_allows_only_read_only_provider_access(self): + task = {"prompt": "Inspect PRs.", "network": "public_github_read_only"} + prompt = ablation.task_prompt(task) + self.assertIn("public GitHub", prompt) + self.assertIn("Raithlin/gha", prompt) + self.assertIn("no provider writes", prompt) + self.assertNotIn("do not contact a network remote", prompt) + + def test_public_pr_environment_removes_authentication(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + env = ablation.agent_environment( + {"PATH": "/bin", "GHA_GITHUB_TOKEN": "secret", "GH_TOKEN": "secret", + "GITHUB_TOKEN": "secret"}, root, "public_github_read_only") + self.assertEqual(env["GHA_GITHUB_TOKEN"], "") + self.assertEqual(env["GH_TOKEN"], "") + self.assertEqual(env["GITHUB_TOKEN"], "") + self.assertEqual(env["GH_CONFIG_DIR"], str(root / "gh-config")) + self.assertEqual(env["XDG_CONFIG_HOME"], str(root / "xdg-config")) + + def test_network_tasks_use_network_enabled_workspace_sandbox(self): + self.assertEqual(ablation.sandbox_options("public_github_read_only"), + ["-s", "workspace-write", "-c", + "sandbox_workspace_write.network_access=true", + "-c", "approval_policy=never"]) + self.assertEqual(ablation.sandbox_options("none"), ["-s", "read-only"]) + + def test_working_tree_digest_detects_file_change(self): + with tempfile.TemporaryDirectory() as directory: + repo = Path(directory) + (repo / "file.txt").write_text("before") + before = ablation.working_tree_digest(repo) + (repo / "file.txt").write_text("after") + self.assertNotEqual(before, ablation.working_tree_digest(repo)) + + def test_reassess_accepts_help_inspection_when_answer_is_correct(self): + record = {"exit_code": 0, "answers": '{"answers":{"status":"ok"}}', + "refs_unchanged": True, "tree_unchanged": True, + "inspected_repository": True, "valid": False, "correct": False} + events = [{"type": "item.started", "item": {"type": "command_execution", + "command": "gh pr create --help"}}] + ablation.reassess_record(record, events, {"status": "ok"}) + self.assertTrue(record["valid"]) + self.assertTrue(record["correct"]) + + def test_schedule_resume_skips_only_completed_attempts(self): + tasks = [{"id": "a"}, {"id": "b"}] + schedule = list(ablation.scheduled_runs(tasks, 2)) + self.assertEqual(len(schedule), 12) + completed = {("a", 0, "binary_only"), ("b", 1, "full_skill")} + pending = [item for item in schedule if item[:3] not in completed] + self.assertEqual(len(pending), 10) + self.assertEqual(schedule[0], ("a", 0, "binary_only", tasks[0])) + + def test_public_origin_fixture_replaces_only_disposable_remote(self): + with tempfile.TemporaryDirectory() as directory: + repo = Path(directory) + subprocess.run(["git", "init", "-q", str(repo)], check=True) + subprocess.run(["git", "-C", str(repo), "remote", "add", "origin", "/tmp/source"], + check=True) + ablation.prepare_fixture(repo, "github_origin") + url = subprocess.check_output(["git", "-C", str(repo), "remote", "get-url", + "origin"], text=True).strip() + self.assertEqual(url, "git@github.com:Raithlin/gha.git") + + def test_held_out_fixture_has_independent_branch_graph(self): + with tempfile.TemporaryDirectory() as directory: + repo = Path(directory) + subprocess.run(["git", "init", "-q", str(repo)], check=True) + (repo / "README.md").write_text("held-out repository\n") + subprocess.run(["git", "-C", str(repo), "add", "README.md"], check=True) + subprocess.run(["git", "-C", str(repo), "-c", "user.name=Eval", + "-c", "user.email=eval@example.invalid", "commit", "-qm", + "initial"], check=True) + subprocess.run(["git", "-C", str(repo), "remote", "add", "origin", + str(repo)], check=True) + ablation.prepare_fixture(repo, "held_out_branches") + merged = subprocess.run(["git", "-C", str(repo), "merge-base", "--is-ancestor", + "topic/shipped", "stable"]) + unmerged = subprocess.run(["git", "-C", str(repo), "merge-base", "--is-ancestor", + "topic/inflight", "stable"]) + self.assertEqual(merged.returncode, 0) + self.assertNotEqual(unmerged.returncode, 0) + counts = subprocess.check_output(["git", "-C", str(repo), "rev-list", + "--left-right", "--count", + "topic/inflight...origin/topic/inflight"], + text=True).strip() + self.assertEqual(counts, "1\t0") + + def test_summary_keeps_all_three_arms_and_private_fields_out(self): + records = [{"condition": arm, "task_id": "t", "repetition": 0, + "valid": True, "correct": True, "seconds": 2, + "total_tokens": 100, "input_tokens": 80, "output_tokens": 20, + "cached_input_tokens": 0, "gha_used": arm != "binary_only", + "skill_loaded": arm == "full_skill", "trace": "secret"} + for arm in ablation.CONDITIONS] + result = ablation.public_results({"tasks": ["t"], "revision": "sha", + "model": "model", "repeats": 2, "gha_sha256": "hash"}, records) + self.assertEqual(set(result["summary"]), set(ablation.CONDITIONS)) + self.assertNotIn("secret", json.dumps(result)) diff --git a/tools/evals/workflow_tasks.json b/tools/evals/workflow_tasks.json new file mode 100644 index 0000000..0cbecd3 --- /dev/null +++ b/tools/evals/workflow_tasks.json @@ -0,0 +1,18 @@ +[ + { + "id": "branch-reachability", + "gha_workflow": "branches cleanup", + "revision": "16df66c472d5e63208f4a7786aa136d5c6fb4b79", + "fixture": "branch_cleanup", + "prompt": "A repository fixture has eval/base and three feature branches. Which of feature/merged, feature/active, and feature/old-unmerged have tips reachable from eval/base? Return answers.reachable and answers.not_reachable as comma-separated, alphabetically sorted branch names.", + "expected": {"reachable": "feature/merged", "not_reachable": "feature/active,feature/old-unmerged"} + }, + { + "id": "cached-upstream", + "gha_workflow": "branches", + "revision": "16df66c472d5e63208f4a7786aa136d5c6fb4b79", + "fixture": "branch_cleanup", + "prompt": "Among feature/merged, feature/active, and feature/old-unmerged, which local branch tracks an origin branch? Return answers.branch, answers.ahead, and answers.behind relative to its cached tracking ref. Use decimal strings for the counts.", + "expected": {"branch": "feature/active", "ahead": "1", "behind": "0"} + } +]