diff --git a/conformance/clean_install.py b/conformance/clean_install.py index f635d12..993c578 100644 --- a/conformance/clean_install.py +++ b/conformance/clean_install.py @@ -61,6 +61,8 @@ def main() -> None: # The venv comes first; the rest of PATH is kept so the Build Thread example can find git. env = {"PATH": os.pathsep.join([str(python.parent), os.environ.get("PATH", "")]), "HOME": str(outside), "PYTHONNOUSERSITE": "1"} + # Windows' TLS provider needs the system directory even in a clean venv. + env.update({name: os.environ[name] for name in ("SystemRoot", "WINDIR") if name in os.environ}) origin = run([str(python), "-c", "import syberlabs, syberwork; print(syberlabs.__file__); print(syberwork.__file__)"], cwd=outside, env=env).split() if any(str(ROOT) in path or "site-packages" not in path for path in origin): diff --git a/docs/CLAUDE_CODE.md b/docs/CLAUDE_CODE.md index 4e10663..5521e63 100644 --- a/docs/CLAUDE_CODE.md +++ b/docs/CLAUDE_CODE.md @@ -145,7 +145,7 @@ exec '' -I '/syberlabs/githits_cli_entry.py' "$@" | `SYBERLABS_GUARD_LAUNCHER` | the launcher's path, for the query guard | | `SYBERLABS_GITHITS_RECORD_DIR` | `githits-records/` in the call's guard directory | | `SYBERLABS_GITHITS_RECORD_KEY` | a fresh `secrets.token_hex(32)` per call, in no file | -| `GITHITS_SESSION_ID` | `-` (`X-GitHits-Session-ID`; GitHits allows only `[A-Za-z0-9_-]`, so not `/`) | +| `GITHITS_SESSION_ID` | `-` (`X-GitHits-Session-ID`; GitHits allows only `[A-Za-z0-9_-]`, so not `/`); the thread is `thread_label` (the CLI passes the thread's id), else the search space's thread, so a provider built in Python names its thread too | | `GITHITS_AGENT` | `syberfactory/` (`X-GitHits-Agent`) | | `SYBERLABS_GITHITS_LICENSES` | the license allow list in force, written out (the operator's, else the permissive default) | | `SYBERLABS_GITHITS_EXCLUDE` | the exclusion list in force, written out (`syberlabs` plus what the operator added) | @@ -154,13 +154,13 @@ exec '' -I '/syberlabs/githits_cli_entry.py' "$@" **Arm A holds no GitHits variable.** Without `--githits`, and without `--mcp-config` (no GitHits route at all), the provider removes every `GITHITS_*` and `SYBERLABS_GITHITS_*` variable from Claude Code's environment (`claude_code.githits_variable`, compared ignoring case), `GITHITS_API_TOKEN` and `GITHITS_API_URL` included. Least privilege was chosen over a byte-identical environment: arm A has no tool that could use the token, and now it does not hold it either. Arm A's argv is still byte-identical to what it was before this route existed. `tests/test_claude_code.py` pins arm A's argv literally and its environment as the caller's minus those variables plus the guard's three, and pins arm B's differences to exactly the items above plus the operator's GitHits variables that arm A no longer receives. An `--mcp-config` run keeps them, because a GitHits MCP server, a child of Claude Code, may read its token from that environment (the MCP A/B below). -**The guard over Bash.** With the launcher, the query guard's matcher is `^(mcp__githits__.*|WebSearch|WebFetch|Bash)$`, and a `Bash` call passes only when its command is the launcher's exact path, alone or followed by a space, and what follows is plain words and single- or double-quoted strings (`query_guard.bash_arguments`). `not_launcher` blocks any other command; `shell_syntax` blocks a control operator, redirection, expansion or substitution anywhere (`` ; & | < > $ ` \ ( ) { } ! ``, any control character such as a newline), a glob, tilde or comment character outside quotes, an unterminated quote, and `run_in_background`. The arguments that pass are the outbound text: each one, and all of them joined, are judged by the same rules as a GitHits tool's query (`secret`, `url_credentials`, `email`, `too_long`, `code_fence`, `code_lines`, `tree_line`). Decisions carry the same MAC under the per-call key. Claude Code's own permission check is a second gate: under `dontAsk` a Bash command that `Bash(:*)` does not name is denied. +**The guard over Bash.** With the launcher, the query guard's matcher is `^(mcp__githits__.*|WebSearch|WebFetch|Bash)$`, and a `Bash` call passes only when its command is the launcher's exact path, alone or followed by a space, and what follows is plain words and single- or double-quoted strings (`query_guard.bash_arguments`). `not_launcher` blocks any other command; `shell_syntax` blocks a control operator, redirection, expansion or substitution anywhere (`` ; & | < > $ ` \ ( ) { } ! ``, any control character such as a newline), a glob, tilde, caret or comment character outside quotes, an unquoted word that a shell reads differently from the guard (one that starts with `=`, which zsh's default EQUALS option expands to a command's path; with `NAME=`, which bash's `set -k` turns into an assignment into the launcher's environment, so `GITHITS_API_URL=https://...` would send the client and its token elsewhere; or with `@`, which PowerShell splats; quoted, each passes), an unterminated quote, `run_in_background`, and a `timeout` shorter than the Bash tool's two-minute default (`query_guard.MIN_BASH_TIMEOUT_MS`): ended early by Claude Code, the launcher's client could be killed after its request left and before it writes the call's record. The arguments that pass are the outbound text: each one, and all of them joined, are judged by the same rules as a GitHits tool's query (`secret`, `url_credentials`, `email`, `too_long`, `code_fence`, `code_lines`, `tree_line`). Decisions carry the same MAC under the per-call key. Claude Code's own permission check is a second gate: under `dontAsk` a Bash command that `Bash(:*)` does not name is denied. **PowerShell under the override.** An operator's `Bash` or `PowerShell` entry is refused with `--githits` unless `--allow-code-tools-with-mcp` is passed. With the override, the matcher is `^(mcp__githits__.*|WebSearch|WebFetch|Bash|PowerShell)$` (`query_guard.MATCHER_SHELLS`), and a `PowerShell` call is judged by the same launcher-only rule as `Bash` (`query_guard.SHELL_TOOLS`): any command but the launcher is `not_launcher`, so an allowed PowerShell no longer runs unguarded. Its refusals are journaled like a refused Bash call (tool `PowerShell`, its rule) and counted in `signal.githits.bash_refused` and `lookups_blocked`. Without the override the matcher is the Bash one above. -**What is recorded.** After the call the provider reads the client's records (`githits.read_records` under the call's record key). Each authentic record becomes one verified `lookup` fact, tool `githits.`, value `{tool, query_digest, request_id, status, at, results_returned, results_dropped, results_dropped_license, latency_ms, contamination_blocked, ok, guard}`: the client's own digest of the request it sent, never text. Forged or altered record lines are counted (`signal.githits.records_ignored`) and ignored. Every `Bash` call the guard did not pass is journaled as a `lookup` with tool `Bash` and its rule (or `unguarded`), which `lookups_blocked` counts. `signal.tool_usage.githits` is the number of authentic records (plus any `mcp__githits__*` call the stream shows). The candidate also carries `signal.githits: {arm, launcher_calls, launcher_passed, records, records_ignored, bash_refused, license_policy[, mismatch]}` and `signal.harness.skill: {path, sha256}`. `githits_lookups` never counts a `githits.doctor` fact. +**What is recorded.** After the call the provider reads the client's records (`githits.read_records` under the call's record key). Each authentic record becomes one verified `lookup` fact, tool `githits.`, value `{tool, query_digest, request_id, status, at, results_returned, results_dropped, results_dropped_license, latency_ms, contamination_blocked, sent, ok, guard}`: the client's own digest of the request it sent, never text; `sent` is the record's, whether the request was written to GitHits (false for an excluded target or a connection that never opened, true from a timeout after the request left to an answer, null for a call interrupted in flight). The client records every call it starts, an answer it cannot read and an interruption included; the launcher's entry turns a `SIGTERM` into an exception for that, so a launcher ended by a Bash timeout records its call (a `SIGKILL` cannot be caught). Forged or altered record lines are counted (`signal.githits.records_ignored`) and ignored. Every `Bash` call the guard did not pass is journaled as a `lookup` with tool `Bash` and its rule (or `unguarded`), which `lookups_blocked` counts. `signal.tool_usage.githits` is the number of authentic records (plus any `mcp__githits__*` call the stream shows). The candidate also carries `signal.githits: {arm, launcher_calls, launcher_passed, records, records_ignored, bash_refused, license_policy[, mismatch]}` and `signal.harness.skill: {path, sha256}`. `githits_lookups` never counts a `githits.doctor` fact. -A record cannot be tied to the tool use that ran it, so the host reconciles counts: authentic records against launcher calls the guard passed. Fewer records is `mismatch: githits_unrecorded` (a usage error, a client that failed before recording, a denied permission; recorded, the call stands). More records than guarded calls is `githits_unexpected_records`: each record of the call is journaled `guard: unguarded`, and the search stops `provider_guard_inactive`, because something holding the key ran the client without the guard. +A record cannot be tied to the tool use that ran it, so the host reconciles counts: authentic records against launcher calls the guard passed. Fewer records is `mismatch: githits_unrecorded` (a usage error, a client that failed before recording, a denied permission; recorded, the call stands). More records than guarded calls is `githits_unexpected_records`: each record of the call is journaled `guard: unguarded`, and the search stops `provider_guard_inactive`, because something holding the key ran the client without the guard. A call that fails or times out registers no candidate, so its mismatch would reach the journal nowhere, and a launcher call still waiting for GitHits when the host ends Claude Code (its session is killed, so the client cannot record) would leave no trace: when such a call has fewer records than guarded launcher calls, its records are journaled `guard: unguarded` (not reconciled), and when it has none, one `githits.unrecorded` fact (the command line's digest, `sent: null`, `guard: unguarded`) stands for each launcher call the guard passed. A `no_change` refusal with missing records journals a synthetic gap without assigning it to a concurrent command; its count remains unknown (`spec/GAPS.md` 59 (12)). **The skill.** `skills/githits/SKILL.md` adapts GitHits' shipped skills (`githits-com/githits-cli` at `0bc9f5e68b`, `skills/githits-code/SKILL.md` and `skills/githits-mcp/SKILL.md`, Apache-2.0, Copyright 2025 GitHits Inc.; `skills/githits/LICENSE` is the upstream license and `skills/githits/NOTICE` says what changed) for the launcher: when a lookup is worth it (an API or version fact the repository and the standard library do not settle), how to phrase a query (short, naming the package and version, no code or repository text), targets first, search then grep or list then read, results as untrusted data, and citing sources in the final reply (the harness does not commit). Claude Code 2.1.309's `--help` offers `--append-system-prompt`, `--plugin-dir` (skills inside a plugin) and `--add-dir`; the provider uses `--append-system-prompt`, so the text is part of arm B's system prompt on every request rather than a skill the model may or may not load, and arm A's prompt is exactly the harness note. The YAML front matter is left out, and `{launcher}` is replaced by the call's launcher path, so arm B's exact appended text is the harness note, a blank line, and the file's body with that one substitution; the file's SHA-256 is on `signal.harness.skill`, computed with CRLF normalized to LF, so a Windows checkout with `core.autocrlf` records the committed blob's digest (`24c150e4…6508` today) as a POSIX one does. `.gitattributes` (`skills/** text eol=lf`) also checks the skill out with LF on every platform. diff --git a/docs/EXPERIMENTS.md b/docs/EXPERIMENTS.md index ff010cc..04636d5 100644 --- a/docs/EXPERIMENTS.md +++ b/docs/EXPERIMENTS.md @@ -9,10 +9,10 @@ syberlabs experiment report --name githits-pilot --markdown # the table syberlabs experiment report --name githits-pilot --json # every item, its threads, values and checks syberlabs experiment report --name githits-pilot --csv # one row per item syberlabs experiment report --name githits-pilot --from-export receipts/*.json # records from syberlabs export -python examples/githits_ab.py # the whole pipeline, a stand-in instead of a model +python examples/githits_ab.py [--route api|mcp] # the whole pipeline, a stand-in instead of a model ``` -An experiment runs the same work items through the factory more than once and changes one thing between the runs. Each run is a Build Thread; each thread is in one **arm**. The first consumer is the owner-approved GitHits pilot (2026-10-23), whose arms differ only in the Claude Code proposer's `--mcp-config` (`docs/CLAUDE_CODE.md`, "GitHits A/B"). +An experiment runs the same work items through the factory more than once and changes one thing between the runs. Each run is a Build Thread; each thread is in one **arm**. The first consumer is the owner-approved GitHits pilot (2026-10-23), whose arms differ only in whether the Claude Code proposer has GitHits: since SyberWork #134 through the host's API client, `--githits` (`docs/CLAUDE_CODE.md`, "GitHits through the launcher"; `docs/GITHITS.md`, "The A/B report"), and before it through an MCP server, `--mcp-config` (`docs/CLAUDE_CODE.md`, "GitHits A/B through MCP"). The report reads both routes. ## The label @@ -72,16 +72,18 @@ Each item is checked against the record. The first nine take the item out of eve | `unequal_repeats` | excluded | the arms ran the item a different number of times, so a mean of several runs would face one run | | `base_mismatch` | excluded | the arms started from different commits (the `base` observation) | | `objective_mismatch` | excluded | the arms were given different objectives | -| `arm_conflict` | excluded | the record contradicts the label: the server (`--server`, default `githits`) was connected in a baseline call, or not connected in the treatment, as the proposer recorded it in `signal.harness.mcp_servers` | -| `contaminated` | excluded | the baseline made GitHits lookups | +| `arm_conflict` | excluded | the record contradicts the label: GitHits (`--server`, default `githits`) was there in a baseline call, or in none of a treatment thread's calls (below, "How the arm is read"). On the API route: arm B run without `--githits`, or arm A with it or with a GitHits `--mcp-config` | +| `contaminated` | excluded | the baseline shows a GitHits lookup: its GitHits count is above zero, or the host observed one (below) | | `labelled_late` | excluded | a thread's label in force, in any arm, is not the one that stood before its first candidate | | `unregistered` | excluded | `--items` was given and does not list the item | | `registration_mismatch` | excluded | the item's threads carry other tags than `--items` gives it | | `base_unknown` | kept | a thread has no recorded base commit | -| `arm_unverified` | kept | no call recorded its MCP servers, so only the label says which arm ran | -| `server_mixed` | kept | the server was connected in some of a treatment thread's calls and not in others | -| `unused` | kept | the treatment made no GitHits lookups | -| `contamination_unknown` | kept | the baseline did not record its GitHits lookups, so it cannot be shown clean | +| `arm_unverified` | kept | no call recorded its MCP servers or the GitHits launcher, so only the label says which arm ran | +| `server_mixed` | kept | GitHits was there in some of a treatment thread's calls and not in others | +| `two_routes` | kept | a treatment thread had GitHits by both routes: the API launcher and a connected GitHits MCP server, usually a `--mcp-config` left in arm B by mistake. It is still the treatment, but not the single route the pilot registers, and its count includes authentic client records plus host-observed MCP attempts (`spec/GAPS.md` 59). What to do: take the GitHits server out of arm B's `--mcp-config` before the next item; say in the write-up that it ran with both routes; do not treat it as the registered single-route experiment | +| `unused` | kept | the treatment shows no GitHits lookup, and its count is known to be zero | +| `use_unknown` | kept | the treatment shows no GitHits lookup, but its count is unknown, so it can be shown neither to have used GitHits nor not to. The count is unknown when a call declared none, or, where `syberlabs metrics` says so (SyberWork #139), when the record cannot say how many lookups reached GitHits: a call's `signal.githits.mismatch` (a launcher call the guard passed left no client record) or an `unguarded` client record. What to do: `syberlabs metrics --thread --json` gives the reason in the count's note; fix the cause (a client that failed before recording, a proposer that declares no lookups) before the next item, and do not read the item as one where GitHits went unused | +| `contamination_unknown` | kept | the baseline shows no GitHits lookup, but its count is unknown, so it cannot be shown clean | | `relabelled` | kept | a thread's labels disagree, but the one in force stood before its first candidate | | `model_mismatch` | kept | the arms were served by different models | | `cost_basis_mismatch` | kept, cost not compared | the arms' cost was recorded on different bases or credential lanes | @@ -93,7 +95,18 @@ Each item is checked against the record. The first nine take the item out of eve Two lines are about the experiment as a whole. `not_run` names the registered items that never ran. `forgotten` names the forget tombstones written since the experiment's first thread started, because a forgotten thread leaves nothing else behind; under the default retention policy a thread with an accepted change is kept for 365 days, so a tombstone here is usually a run that was not accepted and was removed. Exported records carry no tombstones, so a report `--from-export` says it cannot see forgotten threads. `--from-export` also refuses a record whose hash chain does not verify (`export_unverified`): an edited export is not counted. -The Claude Code proposer leaves `mcp_servers` out of `signal.harness` when its session started with none, so a call whose harness recorded its session's start (a `version` or `model`) and lists no servers had none connected; a call whose harness recorded nothing of its start is not read. `--server none` turns off the server checks for an experiment that does not switch an MCP server. `contaminated`, `contamination_unknown` and `unused` read GitHits lookups and apply only with `--server githits`. +### How the arm is read + +The report does not take the label's word for the arm. It reads, call by call, the host's record of whether the call had GitHits, by either route: + +- **The API route** (`--githits`, the pilot's since SyberWork #134): the candidate carries `signal.githits`, the ledger the Claude Code proposer writes on the candidate of every call it gave the host's GitHits launcher, and of no other. It holds the host's own count of the client records it verified under the call's key (`records`), and it is there whether or not the model used the launcher, so a treatment that looked nothing up is `unused`, not in conflict. What the model drove (`signal.tool_usage`, `signal.githits.launcher_calls`), the skill text it was shown (`signal.harness.skill`) and the operator's name for the arm (`signal.githits.arm`, from `--githits-arm`) are not read for the arm. +- **The MCP route** (`--mcp-config`, the pilot's before #134): `signal.harness.mcp_servers` lists the server `connected`, as Claude Code's `init` event said. The proposer leaves `mcp_servers` out when its session started with none, so a call whose harness recorded its session's start (a `version` or `model`) and lists no servers had none connected; a call that recorded neither the ledger nor anything of its start is not read. + +A thread whose calls all had GitHits is the treatment's; one where none did is the baseline's; anything else is `server_mixed` in the treatment and `arm_conflict` in the baseline. A treatment thread with both routes is `two_routes`. + +A GitHits lookup is shown by the thread's GitHits count (`syberlabs metrics`' `githits_lookups`; on the API route, the number of the host's `lookup` facts from authentic client records) above zero, or by a `lookup` fact the host observed and can vouch for: a GitHits MCP call (`mcp__githits__*`), or a client record that a guarded launcher call stands behind (`githits.`, guard `pass`; a `githits.doctor` readiness check is not a lookup). That includes the lookups of a call that failed before its candidate, which no registration counts. An unknown count is never read as zero: it shows no baseline clean (`contamination_unknown`) and no treatment unused (`use_unknown`). + +`--server none` turns these checks off for an experiment that does not switch GitHits; another `--server` name reads that MCP server only, never the launcher. `contaminated`, `contamination_unknown`, `unused`, `use_unknown` and `two_routes` read GitHits lookups and routes, and apply only with `--server githits`. ## Statistics, stated plainly @@ -123,24 +136,28 @@ The pilot itself runs a real model with a paid key: Class X (`AGENTS.md`). A nam 1. The item list, as the `--items` file: each work item's id and its tag, `needs_lookup` when the work depends on a library or API the agent is unlikely to know well (new, niche, or changed since the model's training), else `routine`; beside it, each item's objective and scope. 2. The primary metric and the group the decision rests on (for example: the delivery rule on `needs_lookup` items, with `--delivered review`). 3. The exclusion rules: the nine excluded checks above, and nothing else. -4. The frozen setup: the SyberWork commit, Claude Code version, model, effort, `--allowed-tools`, `--max-turns`, the query guard's thresholds, and the credential lane. Use the `api` lane (`--bare`) in both arms: it is the lane whose cost is billed and known, and in bare mode the tree's `CLAUDE.md` is not read in either arm. The contract prices every model the runs report (`evolution.budget.prices`); a run that used a model the contract does not price has an unknown cost. Anything that learns from runs stays off for the week. AutoSkill is a design (`docs/AUTOSKILL.md`) and is in no proposer path; the `jev` and `empirical` branch policies belong to `--evolve`, which the pilot does not use. +4. The frozen setup: the SyberWork commit, Claude Code version, model, effort, `--allowed-tools`, `--max-turns`, the query guard's thresholds, the credential lane, and arm B's GitHits route (`--githits`) with its license policy (`SYBERLABS_GITHITS_LICENSES`, recorded as `signal.githits.license_policy`). Use the `api` lane (`--bare`) in both arms: it is the lane whose cost is billed and known, and in bare mode the tree's `CLAUDE.md` is not read in either arm. The contract prices every model the runs report (`evolution.budget.prices`); a run that used a model the contract does not price has an unknown cost. Anything that learns from runs stays off for the week. AutoSkill is a design (`docs/AUTOSKILL.md`) and is in no proposer path; the `jev` and `empirical` branch policies belong to `--evolve`, which the pilot does not use. + +**Before the week, the dry run** (Class X as well, a few dollars): `python scripts/pilot_dry_run.py --plan plan.json` runs this protocol on 3 to 5 items on the API route (`--githits`, `docs/GITHITS.md`, "The A/B report"), in a home of its own: a preflight that refuses to start on any failure, one canary call that shows the query guard fires under `--bare`, both arms per item from one `HEAD` in a seeded order, the blind review and this report under a hard budget cap (default $8). It passes when the nine metrics are known for every thread and the report compares every item; its docstring says how to run it, and its `receipt.json` and `report.md` are what to send. **Each item, each arm, from the same `HEAD`** (accepting moves only the thread's own branch, so `HEAD` stays put): ```bash syberlabs start "Parse ISO dates with the UTC offset" --paths src tests --unattended syberlabs experiment label --name githits-pilot --item LIB-1 --arm off --tag needs_lookup -syberlabs propose --claude-code --bare --allowed-tools Read,Edit,Write,Glob,Grep,WebSearch,WebFetch,mcp__githits +syberlabs propose --claude-code --bare --allowed-tools Read,Edit,Write,Glob,Grep,WebSearch,WebFetch syberlabs run --once # one safe step per thread; here, accepting the passing candidate syberlabs start "Parse ISO dates with the UTC offset" --paths src tests --unattended syberlabs experiment label --name githits-pilot --item LIB-1 --arm on --tag needs_lookup -syberlabs propose --claude-code --bare --allowed-tools Read,Edit,Write,Glob,Grep,WebSearch,WebFetch,mcp__githits \ - --mcp-config githits.json # the only difference between the arms +syberlabs propose --claude-code --bare --allowed-tools Read,Edit,Write,Glob,Grep,WebSearch,WebFetch \ + --githits --githits-arm on # the only difference between the arms: the launcher and the skill syberlabs run --once ``` -`--claude-code` is the proposer of SyberWork #80 and the query guard of #82; this report works with any proposer, and needs them for the server check and for the lookups to be counted (`arm_unverified` and unknown lookups otherwise). Alternate which arm runs first from item to item, so drift in the model or the network during the week does not fall on one arm. Accept on the host's checks (`--unattended`), or have a person accept without being told the arm: the label is in the journal, not in the diff or the pull request. Then have a person who is not told the arm review **both** arms' accepted commits, and record each verdict on its thread: +Arm B needs `GITHITS_API_TOKEN` in the environment (the proposer refuses `--githits` without it, and never writes it down). Arm A drops every GitHits variable before Claude Code starts (#135), so both arms can run from the same shell. Pass no GitHits `--mcp-config` in either arm: in arm A it is `arm_conflict`, in arm B `two_routes`. Before #134 the pilot's route was an MCP server: `--mcp-config githits.json` in arm B, with `mcp__githits` in both arms' `--allowed-tools`. The report still reads an experiment run that way. + +`--claude-code` is the proposer of SyberWork #80, with the query guard of #82 and the GitHits launcher of #134. This report works with any proposer, but needs those to read the arm and count the lookups (`arm_unverified` and unknown lookups otherwise). Alternate which arm runs first from item to item, so drift in the model or the network during the week does not fall on one arm. Accept on the host's checks (`--unattended`), or have a person accept without being told the arm: the label is in the journal, not in the diff or the pull request. Then have a person who is not told the arm review **both** arms' accepted commits, and record each verdict on its thread: ```bash syberlabs review --record --verdict pass --reviewer blind-1 --revision 3f9c2ab @@ -163,6 +180,6 @@ What leaves the factory is the report and the CSV. The journal, the prompts and - The report compares what the record holds. A lookup the proposer did not declare, a model call outside the host, or a person's message to an agent outside the CLI is unknown or missing to it, as it is to `syberlabs metrics` (`docs/METRICS.md`, section 4). - The checks catch a label settled after the work began, an item outside the registered list, an unlabelled run of the same objective from the same commit, and a forgotten thread. They cannot catch a run made in another home, or one whose objective was reworded. -- The server check reads what the proposer recorded about its own MCP servers. It catches a GitHits server that failed to connect; it cannot catch a proposer that misreports. +- The arm check reads what the proposer recorded: the GitHits launcher's ledger and its MCP servers. It catches a treatment run without `--githits` (or whose GitHits server failed to connect), and a baseline given GitHits by either route. It cannot catch a proposer that misreports: a search-protocol adapter writes its own `signal`, so it could declare a ledger or a server it never had. - One week of items is a small sample. The report says how small, metric by metric; it does not turn a small sample into a finding. -- `examples/githits_ab.py` runs every step with a stand-in instead of a model. Its numbers are invented to exercise the report and say nothing about GitHits. +- `examples/githits_ab.py` runs every step with a stand-in instead of a model. On its default route (`--route api`), that is the real Claude Code proposer with a stand-in `claude` and a loopback GitHits; `--route mcp` uses a search-protocol stand-in that declares its MCP servers. Its numbers are invented to exercise the report and say nothing about GitHits. diff --git a/docs/GITHITS.md b/docs/GITHITS.md index 64a55d1..2a9df02 100644 --- a/docs/GITHITS.md +++ b/docs/GITHITS.md @@ -218,6 +218,77 @@ print(hashlib.sha256(fixture_bytes(json.load(open(sys.argv[1], encoding='utf-8') The same digest means the five operations are unchanged. +## The A/B report + +The pilot runs each work item through the same Claude Code proposer twice and compares the arms with +`syberlabs experiment` (`docs/EXPERIMENTS.md`): arm A without GitHits, arm B with the launcher. Label each thread +before its first candidate and keep everything else equal, `--allowed-tools` included: + +```bash +syberlabs start "Parse ISO dates with the UTC offset" --paths src tests --unattended +syberlabs experiment label --name githits-pilot --item LIB-1 --arm off --tag needs_lookup +syberlabs propose --claude-code --bare --allowed-tools Read,Edit,Write,Glob,Grep,WebSearch,WebFetch +syberlabs run --once + +syberlabs start "Parse ISO dates with the UTC offset" --paths src tests --unattended +syberlabs experiment label --name githits-pilot --item LIB-1 --arm on --tag needs_lookup +syberlabs propose --claude-code --bare --allowed-tools Read,Edit,Write,Glob,Grep,WebSearch,WebFetch \ + --githits --githits-arm on # the only difference: the launcher and the skill +syberlabs run --once + +syberlabs experiment report --name githits-pilot --items items.txt --delivered review +``` + +The report does not take the label's word for the arm; it reads the host's record of each call. On this route a +call had GitHits when its candidate carries `signal.githits`, the ledger `ClaudeCodeProvider` writes on the +candidate of every call it gave the launcher, and of no other: `{arm, launcher_calls, launcher_passed, records, +records_ignored, bash_refused, license_policy[, mismatch]}`, where `records` is the host's own count of the client +records whose MAC verified under the call's key. That is the strongest host-owned evidence the provider records +for a call: the host writes it only after writing that call's launcher and record directory and reading them back, +and its presence does not depend on anything the model did, so a treatment whose model never touched the launcher +still carries it and is `unused`, not in conflict. What the model drove (`signal.tool_usage`, `launcher_calls`), +the skill text it was shown (`signal.harness.skill`) and the operator's name for the arm (`signal.githits.arm`, +from `--githits-arm`) are not read for the arm. GitHits use is read from the host's `lookup` facts, one per +authentic client record (tool `githits.`), which `syberlabs metrics` counts as `githits_lookups`; a +`githits.doctor` record is a readiness check, not a lookup, and counts for neither. + +| Check | Effect | On this route | +|---|---|---| +| `arm_conflict` | excluded | a baseline call carries the ledger (`--githits` was passed in arm A), or a treatment thread's calls carry none (it was forgotten) | +| `contaminated` | excluded | the baseline shows a GitHits lookup: its `githits_lookups` is above zero, or its thread holds a verified `lookup` fact the host observed, an MCP call (`mcp__githits__*`) or a client record a guarded launcher call stands behind (`githits.` but `doctor`, guard `pass`), a call that failed before its candidate included | +| `contamination_unknown` | kept | the baseline shows no lookup but its `githits_lookups` is unknown, so it cannot be shown clean | +| `server_mixed` | kept | some of a treatment thread's calls carry the ledger and some do not | +| `two_routes` | kept | a treatment thread had GitHits by both routes, the ledger and a connected GitHits MCP server (one left in arm B by mistake): still the treatment, but not one route alone, and `githits_lookups` includes authentic client records plus host-observed MCP attempts, while guard-blocked attempts do not establish actual use (`spec/GAPS.md` 59) | +| `unused` | kept | the treatment shows no GitHits lookup and its count is known to be zero (a launcher call the guard refused never reached GitHits) | +| `use_unknown` | kept | the treatment shows no GitHits lookup but its count is unknown, so it cannot be shown to have used GitHits (nor not to) | +| `arm_unverified` | kept | no call recorded its MCP servers or the ledger | + +An unknown `githits_lookups` is never read as zero. It is unknown when a call declared no count, and, with the +metrics of SyberWork #139, when the record cannot say how many lookups reached GitHits: a call's +`signal.githits.mismatch` (a launcher call the guard passed left no record) or an `unguarded` client record (no +guarded launcher call behind it). The report then reads only what the host observed: a lookup fact it can vouch for +makes the baseline `contaminated` and shows the treatment used; without one the baseline is `contamination_unknown` +and the treatment `use_unknown`, and the item's paired GitHits count is unknown, counted beside the comparison and +never imputed. The arm itself is read from the ledger's presence, not from any count, so an unknown count never moves +an item between arms or into `arm_conflict`. + +The MCP route is read as before (`signal.harness.mcp_servers` listing `githits:connected`), and a call counts as +having GitHits by either route: a stray GitHits MCP server in arm A is `arm_conflict`, one beside the launcher in +arm B is `two_routes`. Before the report read the ledger, every arm-B item of an API-route pilot was +excluded as `arm_conflict` (the server "not connected in the treatment"), so nothing was compared and the delivery +rule could not be read. `--server none` still turns these checks off; another `--server` name reads that MCP server +only, never the launcher. + +`python examples/githits_ab.py` rehearses the whole pipeline on this route (its default, `--route api`; `--route +mcp` is the route before it): seven items through both arms of the real provider with a stand-in `claude` and a +loopback GitHits, item `LIB-4`'s treatment run without `--githits` and excluded. `tests/test_experiment_githits_api.py` +runs both arms end to end through the provider, the query guard, the launcher and the client, labels and reports +them with the CLI, and prints the paired result and the rule's verdict. + +The report trusts the proposer's record. A search-protocol adapter writes its own `signal`, so it could declare a +ledger it never had, as it could declare an MCP server; the check catches a misconfigured run, not a proposer that +misreports. Claim ceiling: implemented and tested with stand-ins; no pilot has run on it. + ## Environment | Variable | Meaning | diff --git a/docs/GITHITS_OPERATOR_HANDOFF.md b/docs/GITHITS_OPERATOR_HANDOFF.md new file mode 100644 index 0000000..6a3517a --- /dev/null +++ b/docs/GITHITS_OPERATOR_HANDOFF.md @@ -0,0 +1,156 @@ +# Two-arm GitHits operator handoff + +Claim ceiling: implemented and tested offline. No live pilot, provider call, +real GitHits lookup, spend or comparative result is established by these tests. +The complete separate ENGINEERING SMOKE plan is +`examples/githits-engineering-smoke/plan.json`. Its three synthetic tasks are +CSV quoting, sorted Unicode counts and chunked strict UTF-8 decoding. All +acceptance checks are visible; no formal pilot task or held-out answer is used. +Mateo owns credentials, provider selection and spending. The draft integration +pull request and its delivery handoff identify the exact tested commit; approve +that commit before a run. A later source change requires fresh checks. + +## Candidate and prerequisites + +The candidate combines #136, #139 and #142 as one accounting boundary, #141, +#143 and #140, and reconciles #138's output behavior. #139 already includes +#142 through `40d98d0`; its final `1cab541` documentation change is retained. +#143 includes #141 through `d10fc3d`. #138's original history repeats changes +already on main, so only `49c8aca` was applied and its conflicts reconciled. +Main advanced to `b7297d8` when #138 merged; that main is also integrated. +The Windows writer now uses the existing journal lock around open, append, +flush and sync. Interrupted fragments remain ignored; no record is invented. +Synthetic gaps and unreconciled records remain unknown, including `no_change`. +The report keeps attempt counts distinct from guard-passed use. Mixed routes +are counted and still flagged `two_routes`. + +SyberEval is not a dependency: this runner imports `experiment`, whose existing +`delivery_rule` requires lower median paired cost and at least as many +deliveries. No eval redesign, third arm, cache or dashboard is required. + +Required before spending: + +- Approval naming provider, exact model/route, run plan, spending authority, + scheduling budget and treatment of search fees and overshoot. +- Python 3.11+; Git; Claude Code **2.1.311** for this frozen smoke recipe; + Git Bash on Windows. The Windows native CLI was downloaded from its official + npm package, checked against npm's SHA-512 and exercised with version/help + only. Those flags are compatible; the live hook canary remains unrun. +- A plain temporary path without spaces or shell punctuation for launchers. +- The exact approved candidate checked out cleanly. Do not reuse another + worker's checkout or branch, and do not commit the plan or run directory. +- Use the completed synthetic smoke plan, not formal pilot items. The recipe + creates a separate fixture repository, resolves the host check's interpreter + and path automatically, and freezes its base and plan/check/skill digests + before any run. No plan editing is required. +- Mateo supplies credentials in the process environment through his approved + credential mechanism. Do not put credentials in command history, files, + arguments, commits or shared output. Use an alias actor such as `mateo`. +- The API lane must pass its routing-switch checks. A subscription, proxy or + provider-managed session is not evidence of the approved priced API lane. + +## Safe preparation (no credentials or paid calls needed) + +Use the exact SHA from the delivery handoff for `$pin`. To prepare and inspect +without credentials or network calls: + +```powershell +git clone https://github.com/SyberLabs/SyberWork.git SyberWork-githits-operator +Set-Location SyberWork-githits-operator +git fetch origin codex/githits-two-arm-integrated +git switch --detach $pin +if ((git rev-parse HEAD) -ne $pin) { throw 'Wrong candidate' } +if (git status --porcelain) { throw 'Checkout must be clean' } +python scripts/githits_engineering_smoke.py --factory-commit $pin --work ..\engineering-smoke-prepared +``` + +Preparation writes `manifest.json`, the complete `plan.json` and a fixture +repository; it never calls a model or GitHits. The recipe requires a new work +directory each time. Retain the plan and manifest privately. The final delivery +supplies focused and aggregate results on the pinned head. Green CI is not +approval to run. + +## Proposed technical configuration + +Keep the established direct Anthropic Claude Code API lane. Recommend the exact +model `claude-sonnet-5-5`, medium effort: sufficient for these small readiness +tasks at lower listed cost than Opus. The Sol session does not select the pilot +model. The recipe sets `ANTHROPIC_MODEL`, effort and a forced five-minute cache +TTL for both arms; it refuses a command carrying a competing `--model` flag. +Only Read/Edit/Write/Glob/Grep are allowed; optional WebSearch/WebFetch are off. + +As checked on 2026-10-26, standard global API rates per million tokens are +input **$2**, output **$10**, five-minute cache write **$2.50**, cache read +**$0.10**. A one-hour write would be $4; it is not priced as five minutes. +Source: [Anthropic pricing](https://platform.claude.com/docs/en/about-claude/pricing). +Model/cache configuration follows [Claude Code's model documentation](https://code.claude.com/docs/en/model-config) +and [environment reference](https://code.claude.com/docs/en/env-vars). +Actual model/usage is still checked from receipts; an unpriced model stops the +schedule. Different model, TTL, region or price means a newly frozen plan. + +The formal pack was refreshed at MasterMind +`09864ac6633832cc5d7c082ec731aff7a5689cad`: `items.txt` is absent; +`items.proposed.txt` is only the proposed 21; model/prices/reviewer remain +unfilled; and factory `4219850` predates these repairs. Current project context +still says Mateo selects and pre-registers the formal items. Its old +`--server none` report workaround is unnecessary with the integrated report, +but the formal registration and baseline have not been rewritten here. + +## Approved live sequence (Mateo executes) + +Only after explicit run/spend approval and environment-only credential setup: + +```powershell +python scripts/githits_engineering_smoke.py --factory-commit $pin --work ..\engineering-smoke-live-01 --run-approved --claude C:\path\to\claude.exe +$status = $LASTEXITCODE +if ($status -ne 0) { throw "Dry run did not pass; exit $status. Inspect locally and stop." } +``` + +Keep the interactive confirmation enabled. Preflight checks the repository, +version, credential lane, token handling, doctor, guard mechanism, budgets and +contracts. Doctor contacts GitHits once. After confirmation the runner performs +the live canary before any paired item: one content-rule block and one allowed +lookup answered successfully by GitHits, authentic guard decisions, no +unreconciled lookup. An inconclusive or failed canary stops the schedule. +There is no separate canary-only switch; the single command enforces this order. + +If the canary passes, the frozen seed schedules both arms of every item. The +baseline has no GitHits route or credential; treatment uses the API launcher. +The same checks, objectives, base, prices and call/turn limits apply to each arm. +A reviewer who did not watch the runs supplies pass/fail/skip on the accepted +diffs. A skipped review is unknown. No change is merged or deployed by this run. + +The command creates the fixture and complete plan itself, then calls the +existing `pilot_dry_run.py`. Reviewer role `blind-engineering` is the proposed +independent reviewer; a person who did not watch the runs supplies the verdicts. +The final delivery gives a command using the verified local executable path. + +Expected outputs under `/receipts/`, plus `/manifest.json`: + +- `receipt.json`: final verdict, pinned host/base, plan/objective digests, + versions, schedule, guard/canary proof, metric provenance, report and scan. +- `report.json`, `report.md`, `metrics.json`: paired results and unknowns. +- Private local journal, exports, contracts and `commands.log`. These may hold + objectives and code; do not publish or send them automatically. + +Exit 0 means the final receipt and printed verdict are PASS; 1 means failed or +aborted; 2 means refused before spending. PASS proves the measurement path +worked for that run. It does not mean the delivery rule was met or a pilot +succeeded. Review `receipt.json` and `report.md` locally before sharing them. + +## Budget and stop conditions + +`--budget 8` is a token-cost scheduling budget, **not a hard all-in $8 cap**. +The host checks spend between calls; one call can overshoot. Server-side search +fees are excluded. Failed or timed-out provider calls can have unmetered usage; +the runner marks spend unknown and stops, rather than treating it as free. +Approval must cover those limits; a strict all-in cap requires a separate +provider-enforced limit and fee accounting established by Mateo. + +Stop on any failed/inconclusive preflight or canary; unreadable or leaked +output; unknown spend or required metric; missing/partial/unguarded lookup +evidence; a moved base; an unpriced model; an excluded or missing pair; or a +`two_routes` warning. Keep receipts private. Do not rerun only the losing arm, +erase failed items, reinterpret unknown as zero, or add fabricated records. +Any canary retry consumes money and needs to remain within explicit approval. +Report the precise failure and request a bounded correction before resuming. diff --git a/docs/METRICS.md b/docs/METRICS.md index 845d394..9a1eead 100644 --- a/docs/METRICS.md +++ b/docs/METRICS.md @@ -28,7 +28,7 @@ A **work item** is one Build Thread: from `case_created` to the merge of the pul | **agent_time** (hours) | `pr_merged.merged_at` minus `case_created.at`. Human review time is inside the interval by construction. | no `pr_merged` fact | | **fix_rounds** | Registrations that follow a failed evaluation (a required check of the contract not `passed`; every recorded check when the contract is not supplied), each failed evaluation counted once. First attempt right is `fix_rounds == 0` with the first candidate accepted. | never | | **web_lookups** | Sum of `signal.tool_usage.web_search + web_fetch` over registrations. | any registration did not declare either | -| **githits_lookups** | When the record holds `lookup` facts made from GitHits client records (tool `githits.`: the `--githits` route, `docs/CLAUDE_CODE.md`), their number: host-observed, one per authentic record the client signed, calls whose answer was refused included. A `githits.doctor` fact (the operator's readiness check, `docs/GITHITS.md`) is never counted, and when the client-record facts are all doctor facts the count is 0, not the declared sum. Otherwise the sum of `signal.tool_usage.githits` over registrations. | no client-record fact, and any registration did not declare it | +| **githits_lookups** | When the record holds `lookup` facts made from GitHits client records (tool `githits.`: the `--githits` route, `docs/CLAUDE_CODE.md`), their number plus host-observed GitHits MCP attempts: one per authentic record or MCP attempt, calls whose answer was refused included. Synthetic `githits.unrecorded` facts are never authentic records and always make the count unknown. A `githits.doctor` fact (the operator's readiness check, `docs/GITHITS.md`) is never counted, and when the client-record facts are all doctor facts the count is 0, not the declared sum. Otherwise the sum of `signal.tool_usage.githits` over registrations. | no client-record fact, and any registration did not declare it; or a call's `signal.githits.mismatch` says a launcher call the guard passed left no client record, or a client-record fact is `unguarded` (a record no guarded launcher call stands behind, a failed call's record, or a `githits.unrecorded` fact standing for a failed call's launcher call that left none): the record cannot say whether those reached GitHits | | **delivered** | `true` with a `pr_merged` fact (for the opened pull request's number); `false` when no pull request was opened, the thread was cancelled, or a `pr_closed` fact (same number) says it was closed unmerged. `opened` is a separate column: a `github_pull_request` effect that succeeded, or a verified reconciliation of one. | a pull request is open and neither `pr_merged` nor `pr_closed` is in the record | | **review_rejections** | Critic checks (`critic`, `critic.*`) that did not pass, or that passed with a host critic observation counting a high finding (what `critic.no_high_finding` refuses); plus `review` facts with verdict `fail` and `pr_review` facts with state `changes_requested`. | never | | **human_interventions** | `human_intervention` facts, plus `proposed`, `approved` and `case_cancelled` events whose actor is neither `syberlabs.host` nor the thread's unattended runner (`unattended.by`) and whose proposal no fact names. | never | @@ -52,11 +52,9 @@ All are `observed` events written by the host (`actor: syberlabs.host`, `verifie ## 4. Accuracy, stated honestly - **Cost** is exact for host-metered API lanes and unknown for others; the label says which. Critic usage is declared by the critic, so it counts as a turn and is left out of host-observed dollars. -- **Turns** count host-recorded calls with declared usage. A harness's internal loop is one call. A stand-in adapter that reports identical usage on two consecutive calls with nothing recorded between them would be read as one call. -- **Agent time** includes human review time; `human_intervention` facts let a reader split it, and the `unattended` flag marks items where no person was asked. It is known from the first settlement pass after the merge; `merged_at` is GitHub's clock and `case_created` the host's, so skew between them is inside the number. -- **Lookups** are adapter-declared (`declared`, not `host_observed`) until a grounding station records GitHits reads itself. - **Turns** count host-recorded calls with declared usage. A harness's internal loop is one call unless the registration declares `harness_turns`; a harness call refused `no_change` carries no `harness_turns` and counts as one. A stand-in adapter that reports identical usage on two consecutive calls with nothing recorded between them would be read as one call. -- **Agent time** includes human review time; `human_intervention` facts let a reader split it, and the `unattended` flag marks items where no person was asked. -- **Lookups** are adapter-declared for JSON adapters. For `ClaudeCodeProvider` the host counts them from Claude Code's stream and writes one `lookup` record per call; the stream is still Claude Code's report of what it did, not a network observation, and lookups by a call whose answer was refused are not counted (`spec/GAPS.md` 52, 54). On the `--githits` route GitHits lookups are counted from the host's own client's records instead: each is a request the host's code sent (or refused itself) and signed under a per-call key, counted even when the call's answer was refused; a thread that mixes this route with GitHits MCP lookups counts only the client's (`spec/GAPS.md` 59). +- **Agent time** includes human review time; `human_intervention` facts let a reader split it, and the `unattended` flag marks items where no person was asked. It is known from the first settlement pass after the merge; `merged_at` is GitHub's clock and `case_created` the host's, so skew between them is inside the number. +- **Lookups** are adapter-declared for JSON adapters. For `ClaudeCodeProvider` the host counts them from Claude Code's stream and writes one `lookup` record per call; the stream is still Claude Code's report of what it did, not a network observation, and lookups by a call whose answer was refused are not counted (`spec/GAPS.md` 52, 54). On the `--githits` route GitHits lookups are counted from the host's own client's records instead: each is a request the host's code sent (or refused itself) and signed under a per-call key, counted even when the call's answer was refused; a thread that mixes routes also counts the host's MCP attempts once (`spec/GAPS.md` 59). +- **GitHits lookups against the server** (`tests/test_githits_accounting.py`, 2026-10-26, with #135 and #142). An independent counter, the loopback GitHits' own log of the requests it received, was compared per work item with the journal's `githits.` facts and `githits_lookups`, through the real provider, launcher and client. They agree, request id for request id and digest for digest, for: a lookup; results dropped for an excluded owner or the license; launcher commands the guard blocked (nothing reaches the client or GitHits); 401, 404, 500, 503 and 429 answers (one request each: the client does not retry); a timeout and a dropped connection (the fact has no request id: the client never read one); an answer the client cannot read (recorded as `malformed_response` since #142); five lookups in one call, in order, the example sent with `license_mode: strict`; eight launcher processes at once; fix rounds, up to the first acceptance as `experiment` reads it; and arm A, which has no launcher and no GitHits variable and counts a known 0. Two kinds of lookup are counted though GitHits never received them, by this metric's definition: a target the client refused (`excluded_target`) and a connection that never opened. Since #142 each record says whether its request was written (`sent`: true, false, or null when interrupted in flight), and in every case above the facts with `sent: true` are exactly the requests GitHits received. The count is `unknown` where the journal cannot say how many requests left (above): a command the launcher's entry refuses (`doctor`, `--help`, `--license-mode`) or a usage error sends nothing and leaves no record, but the guard passed it, and a launcher killed outright (SIGKILL) or a record that cannot be written leaves the same journal after its request left; and a lookup in flight when the Claude Code call times out, whose failed call journals its records `unguarded`, or a `githits.unrecorded` fact when it has none (#142). A call refused `no_change` with missing records also reads unknown: its gap is journaled separately (`spec/GAPS.md` 59 (12)). - **Interventions** undercount messages to an agent made outside the CLI, and cannot tell an agent from a person when both act under one identity: on a thread without an unattended runner, every `proposed` by a non-host actor is a person's act. - **Delivered** is `unknown`, not `false`, for an opened pull request until a settlement pass has seen it merged or closed. `--check` judges the items the record shows as delivered, so once a merge is recorded an item whose cost, turns or lookups are unknown (every `--from-worktree` proposal) fails it. A pull request closed and then reopened keeps its `pr_closed` fact and reads `false` until a merge is recorded (`spec/GAPS.md` 53). diff --git a/examples/githits-engineering-smoke/check.py b/examples/githits-engineering-smoke/check.py new file mode 100644 index 0000000..08203e9 --- /dev/null +++ b/examples/githits-engineering-smoke/check.py @@ -0,0 +1,44 @@ +"""Visible, synthetic host checks. No formal pilot data or hidden answers.""" +import csv +import importlib.util +import io +import json +from pathlib import Path + + +def check(root): + modules = [root / "src" / name for name in ("smoke_csv.py", "smoke_counts.py", "smoke_utf8.py")] + present = [path for path in modules if path.is_file()] + assert len(present) == 1, "one task module must exist; the base intentionally fails" + path = present[0] + spec = importlib.util.spec_from_file_location("smoke_subject", path) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + if path.name == "smoke_csv.py": + for rows in ([], [{"name": 'a,"b', "qty": 2}, {"name": "pi-\u03c0\nline", "qty": -1}]): + result = module.render(rows) + assert isinstance(result, str), "CSV must be text" + parsed = list(csv.reader(io.StringIO(result, newline=""))) + assert parsed == [["name", "qty"], *[[row["name"], str(row["qty"])] for row in rows]] + assert result.endswith("\r\n"), "CSV requires CRLF" + elif path.name == "smoke_counts.py": + for values, expected in (([], {}), (["b", "A", "b", "\u03c0", "A"], {"A": 2, "b": 2, "\u03c0": 1})): + result = module.summarize(values) + assert result == expected and list(result) == sorted(expected), "counts and sorted key order" + json.dumps(result, allow_nan=False) + else: + for text in ("", "ASCII", "pi-\u03c0-\U0001f680"): + data = text.encode("utf-8") + assert module.decode_chunks([data[n:n + 1] for n in range(len(data))]) == text + for chunks in ([b"\xff"], [b"\xe2\x82"]): + try: + module.decode_chunks(chunks) + except UnicodeDecodeError: + pass + else: + raise AssertionError("invalid or incomplete UTF-8 must fail strictly") + print("ENGINEERING SMOKE checks passed:", path.name) + + +if __name__ == "__main__": + check(Path.cwd()) diff --git a/examples/githits-engineering-smoke/fixture/.gitattributes b/examples/githits-engineering-smoke/fixture/.gitattributes new file mode 100644 index 0000000..fcadb2c --- /dev/null +++ b/examples/githits-engineering-smoke/fixture/.gitattributes @@ -0,0 +1 @@ +* text eol=lf diff --git a/examples/githits-engineering-smoke/fixture/README.md b/examples/githits-engineering-smoke/fixture/README.md new file mode 100644 index 0000000..3225088 --- /dev/null +++ b/examples/githits-engineering-smoke/fixture/README.md @@ -0,0 +1,5 @@ +# ENGINEERING SMOKE: synthetic readiness fixtures + +This repository tests the engineering path, not GitHits cost effectiveness. +It contains none of the formal pilot's items, data or hidden checks. All checks +are visible. Add only the module named by your task; do not change this file. diff --git a/examples/githits-engineering-smoke/fixture/src/.gitkeep b/examples/githits-engineering-smoke/fixture/src/.gitkeep new file mode 100644 index 0000000..8b13789 --- /dev/null +++ b/examples/githits-engineering-smoke/fixture/src/.gitkeep @@ -0,0 +1 @@ + diff --git a/examples/githits-engineering-smoke/plan.json b/examples/githits-engineering-smoke/plan.json new file mode 100644 index 0000000..64f6e48 --- /dev/null +++ b/examples/githits-engineering-smoke/plan.json @@ -0,0 +1,16 @@ +{ + "_purpose": "ENGINEERING SMOKE only: synthetic, visible checks, no formal items or held-out data; not an unbiased cost experiment", + "_formal_pack": "SyberLabs/MasterMind claude/githits-pilot-pack at 09864ac6633832cc5d7c082ec731aff7a5689cad; proposal and formal baseline unchanged", + "_configuration": {"provider": "Anthropic direct API through Claude Code", "model": "claude-sonnet-5-5", "cli": "2.1.311", "effort": "medium", "cache_ttl": "5m", "pricing_checked": "2026-10-26", "pricing_source": "https://platform.claude.com/docs/en/about-claude/pricing"}, + "experiment": "githits-engineering-smoke-v1", + "items": [ + {"id": "ENGSMOKE-CSV", "tags": ["synthetic", "routine"], "objective": "Implement render(rows) in src/smoke_csv.py. Return CSV text for a list of name/qty dictionaries, with header name,qty, CRLF record endings and standard CSV quoting. Preserve names containing commas, quotes, Unicode and newlines, serialize integer quantities including negatives, and emit the header for empty input. Use only Python's standard library. Change only this new module; the visible host checks must pass.", "paths": ["src/smoke_csv.py"]}, + {"id": "ENGSMOKE-COUNTS", "tags": ["synthetic", "routine"], "objective": "Implement summarize(values) in src/smoke_counts.py. Count a list of case-sensitive Unicode strings and return a JSON-serializable dict with keys inserted in ascending Python string order. Empty input returns an empty dict; duplicates increment counts. Use only Python's standard library. Change only this new module; the visible host checks must pass.", "paths": ["src/smoke_counts.py"]}, + {"id": "ENGSMOKE-UTF8", "tags": ["synthetic", "routine"], "objective": "Implement decode_chunks(chunks) in src/smoke_utf8.py. Decode an iterable of byte chunks as one strict UTF-8 stream, including multibyte characters split between chunks. Return the complete string, return empty text for no chunks, and raise UnicodeDecodeError for invalid bytes or an incomplete final sequence. Use only Python's standard library. Change only this new module; the visible host checks must pass.", "paths": ["src/smoke_utf8.py"]} + ], + "prices": {"claude-sonnet-5-5": {"input_per_million": 2, "output_per_million": 10, "cache_read_per_million": 0.1, "cache_creation_per_million": 2.5}}, + "allowed_tools": "Read,Edit,Write,Glob,Grep", + "max_calls": 2, + "max_turns": 8, + "canary_paths": ["README.md"] +} diff --git a/examples/githits_ab.py b/examples/githits_ab.py index 5db7904..87f8218 100644 --- a/examples/githits_ab.py +++ b/examples/githits_ab.py @@ -2,44 +2,63 @@ Seven work items run through both arms of the factory's default proposer path (the repair loop) on a throwaway repository: real Build Threads, real host checks, real acceptances, a real journal, then ``syberlabs metrics`` and -``syberlabs experiment report`` over it. Only the model is a stand-in, a command that answers the search protocol: +``syberlabs experiment report`` over it. Two routes for arm B's GitHits: + +- ``--route api`` (the default; the pilot's route since #134): both arms run the real Claude Code proposer, + ``ClaudeCodeProvider`` on the api lane, arm B with ``githits=True``. For each call the host writes the GitHits + launcher, its query guard judges every lookup and command, the host's own GitHits client makes each lookup and signs + its record, and the host journals one ``lookup`` per record and its ledger of the call on the candidate + (``signal.githits``), which is what the report reads the arm from. Only the ``claude`` executable is a stand-in: a + script that prints Claude Code's ``stream-json``, runs the ``--settings`` hook before each tool call as Claude Code + does, and runs the launcher when its Bash tool is allowed to; GitHits is a loopback stand-in that answers searches. + Nothing leaves the machine. +- ``--route mcp`` (the route before #134): the stand-in is a search-protocol command (``RepairProvider``) that + declares its usage, lookups and connected MCP servers itself. + +On either route: - on an item that depends on a library's behaviour (tag ``needs_lookup``) the stand-in guesses wrong once when GitHits is off, so the host's checks fail and the repair call fixes it; with GitHits on it gets it right first time; -- on a routine item it is right first time in both arms; -- on item ``LIB-4`` the treatment's GitHits server "fails to connect", as a misconfigured run would, so the report must - refuse to count that item; +- on a routine item it is right first time in both arms (with GitHits on it looks one thing up and has nothing to add); +- on item ``LIB-4`` the treatment is misconfigured, so the report must refuse to count that item: on the API route it + runs without ``--githits`` (no launcher), on the MCP route its GitHits server "fails to connect"; - every accepted change gets a blind review (``syberlabs review --record``), and one is turned down, so delivery is "accepted and passed review", as the pilot defines it. -The stand-in declares the usage, lookups and connected MCP servers a real proposer reports (``docs/CLAUDE_CODE.md``): -those numbers are made up to exercise the report. **They are not results and say nothing about GitHits.** The pilot -itself runs a real model with a paid key (Class X: a named person runs it; ``docs/EXPERIMENTS.md`` has the commands). +The stand-ins' token usage and lookups are made up to exercise the report. **They are not results and say nothing +about GitHits.** The pilot itself runs a real model with a paid key (Class X: a named person runs it; +``docs/EXPERIMENTS.md`` and ``docs/GITHITS.md`` have the commands). - python examples/githits_ab.py [--out DIR] # DIR receives report.json, report.md and the metrics CSV + python examples/githits_ab.py [--route api|mcp] [--out DIR] # DIR receives report.json, report.md, metrics.csv """ from __future__ import annotations import argparse +import contextlib import json import os import subprocess import sys import tempfile import textwrap +import threading +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer from pathlib import Path sys.path.insert(0, str(Path(__file__).resolve().parents[1])) -from syberlabs import Kit, experiment, metrics # noqa: E402 +from syberlabs import Kit, experiment, githits, metrics # noqa: E402 from syberlabs.build import default_contract # noqa: E402 +from syberlabs.claude_code import THIRD_PARTY_SWITCHES, ClaudeCodeProvider # noqa: E402 from syberlabs.providers import RepairProvider # noqa: E402 NAME = "githits-rehearsal" +ROUTES = ("api", "mcp") RUNNER = "runner@example.test" REVIEWER = "reviewer@example.test" PRICES = {"stand-in-model": {"input_per_million": 3, "output_per_million": 15}} # a made-up price for a made-up model +MISCONFIGURED = "LIB-4" # the item whose treatment ran without GitHits def module(name: str, body: str, test: str) -> dict: @@ -173,9 +192,9 @@ def test_clean(self): '''), None), } -# The stand-in "model". It reads the request on stdin and the arm from STAND_IN_ARM, and prints one candidate with -# the usage, lookups and MCP servers a real proposer would report. Every number in it is invented. -STAND_IN = textwrap.dedent('''\ +# The MCP route's stand-in "model". It reads the request on stdin and the arm from STAND_IN_ARM, and prints one +# candidate with the usage, lookups and MCP servers a real proposer would report. Every number in it is invented. +MCP_STAND_IN = textwrap.dedent('''\ import json, os, sys items = json.loads(os.environ["STAND_IN_ITEMS"]) request = json.load(sys.stdin) @@ -199,6 +218,169 @@ def test_clean(self): "tool_usage": {"githits": githits, "web_search": web, "web_fetch": 0}})) ''') +# The API route's stand-in for the ``claude`` executable: what the host reads from Claude Code, and no more. It answers +# ``--version``, reads the prompt on stdin, and acts as a model that can only call the tools it was given: with the +# launcher (``--allowedTools`` names ``Bash(:*)``) it looks the library up through it, without it it searches +# the web. Before each tool call it runs the ``--settings`` hook (the host's query guard) as Claude Code's PreToolUse +# contract says, and its Bash tool runs only the launcher, as the launcher's ``exec`` line says (no shell needed). It +# writes its change in the working tree and prints ``stream-json``. Its token counts are the MCP stand-in's, invented. +CLAUDE_STAND_IN = textwrap.dedent('''\ + import json, os, re, shlex, subprocess, sys + from pathlib import Path + + if "--version" in sys.argv: + print("2.1.309 (Claude Code)") + sys.exit(0) + items = json.loads(Path(SPEC).read_text(encoding="utf-8")) + prompt = sys.stdin.read() + objective = prompt.split("\\n", 1)[0][len("Objective: "):] + item = next(key for key, value in items.items() if value["objective"] == objective) + spec = items[item] + settings = json.loads(Path(sys.argv[sys.argv.index("--settings") + 1]).read_text(encoding="utf-8")) + allowed = sys.argv[sys.argv.index("--allowedTools") + 1].split(",") + launchers = [rule[5:-3] for rule in allowed if rule.startswith("Bash(") and rule.endswith(":*)")] + launcher = launchers[0] if launchers else None + lookup, repair = spec["tag"] == "needs_lookup", "did not pass the host's checks" in prompt + topic = objective.lower().replace(",", "") + if launcher and not repair: + uses = [("Bash", {"command": f'{launcher} search "{topic} {n}" --target github:python/cpython', + "description": "look up a public API"}) for n in range(1, (2 if lookup else 1) + 1)] + else: + uses = [("WebSearch", {"query": f"python {topic} {n}"}) + for n in range(1, (1 if repair else 3 if lookup else 0) + 1)] + wrong = lookup and spec["wrong"] is not None and launcher is None and not repair + githits = sum(1 for name, _ in uses if name == "Bash") + web = len(uses) - githits + tokens = 1800 + 900 * githits + 600 * web + (1200 if repair else 0) + + def say(event): + print(json.dumps(event), flush=True) + + def guard(name, tool_input, use_id): + """Each PreToolUse hook whose matcher matches the tool, the call on stdin; it blocks on a non-zero exit (the + host's hook sets onFailure block, so a crash blocks too), and its standard error is the reason.""" + for group in settings.get("hooks", {}).get("PreToolUse", []): + if re.search(group.get("matcher", ""), name): + for hook in group["hooks"]: + call = {"session_id": "rehearsal", "transcript_path": "", "cwd": os.getcwd(), + "permission_mode": "dontAsk", "hook_event_name": "PreToolUse", "tool_name": name, + "tool_input": tool_input, "tool_use_id": use_id} + done = subprocess.run([hook["command"], *hook.get("args", [])], capture_output=True, + input=json.dumps(call).encode("utf-8"), timeout=hook.get("timeout", 60)) + if done.returncode != 0: + return done.stderr.decode("utf-8", "replace") or "blocked" + return None + + def run(name, tool_input): + """The tool itself: Bash runs the launcher, as its exec line says, and nothing else; a search answers.""" + if name != "Bash": + return False, "results for " + tool_input["query"] + command = tool_input["command"] + if launcher is None or not (command == launcher or command.startswith(launcher + " ")): + return True, "Permission to use Bash with command denied" + words = shlex.split(command) + script = Path(words[0]).read_text(encoding="utf-8").splitlines() + line = next(text for text in script if text.startswith("exec ")) + argv = [word for word in shlex.split(line)[1:] if word != "$@"] + words[1:] + done = subprocess.run(argv, capture_output=True, timeout=120) + return done.returncode != 0, (done.stdout + done.stderr).decode("utf-8", "replace") + + for path, text in (spec["wrong"] if wrong else spec["right"]).items(): + Path(path).parent.mkdir(parents=True, exist_ok=True) + Path(path).write_bytes(text.encode("utf-8")) # the exact text: no newline translation on Windows + model, usage = "stand-in-model", {"input_tokens": tokens, "output_tokens": 350} + say({"type": "system", "subtype": "init", "claude_code_version": "2.1.309", "model": model, "effort": "high", + "apiKeySource": "ANTHROPIC_API_KEY" if "--bare" in sys.argv else "none", "mcp_servers": [], + "session_id": "rehearsal", "tools": ["Read", "Edit", "Write"] + (["Bash"] if launcher else [])}) + say({"type": "assistant", "message": {"id": "msg_1", "model": model, "usage": usage, + "content": [{"type": "text", "text": "Working on " + item}]}}) + results = [] + for n, (name, tool_input) in enumerate(uses, 1): + use_id = f"toolu_{n}" + say({"type": "assistant", "message": {"id": "msg_1", "model": model, "usage": usage, "content": [ + {"type": "tool_use", "id": use_id, "name": name, "input": tool_input}]}}) + reason = guard(name, tool_input, use_id) + failed, text = (True, reason) if reason is not None else run(name, tool_input) + results.append({"type": "tool_result", "tool_use_id": use_id, "content": text, "is_error": failed}) + if results: + say({"type": "user", "message": {"role": "user", "content": results}}) + say({"type": "assistant", "message": {"id": "msg_2", "model": model, "content": [ + {"type": "text", "text": "Done."}], "usage": {"input_tokens": 0, "output_tokens": 0}}}) + say({"type": "result", "subtype": "success", "is_error": False, "num_turns": 2 if uses else 1, "result": "done", + "total_cost_usd": 0.0, "session_id": "rehearsal", "usage": {}, "modelUsage": {}}) + ''') + +# What the loopback GitHits answers every search with: one public hit (valid under the pinned contract's shapes). +SEARCH_ANSWER = { + "completed": True, "expires_at": None, "query_warnings": [], "search_ref": "sr_rehearsal", + "selected_fields": ["results"], "status": "complete", "targets": None, + "result": {"evidence_notice": None, "partial_results": False, "query": "q", "query_warnings": [], + "page": {"has_more": False, "limit": 5, "offset": 0, "returned": 1}, "source_status": [], + "sources": ["code"], "results": [{ + "coverage": None, "fresh_target_label": None, "freshness": None, "id": "hit-1", + "locator": {"commit_sha": None, "docs_read_target": None, "end_line": 20, "evidence_range": None, + "file_path": "Lib/example.py", "git_ref": None, "match_line": None, + "package_name": None, "page_id": None, "registry": None, + "repo_url": "https://github.com/python/cpython", "repository_file_path": None, + "requested_ref": None, "source_kind": None, "source_url": None, "start_line": 10, + "symbol_ref": None, "version": None}, + "requested_target_label": None, "result_type": "code", "score": 0.875, + "served_target_label": None, "summary": "How the standard library does it.", + "target_label": "github:python/cpython", "title": "example"}]}} + + +class LoopbackGitHits: + """A stand-in GitHits on 127.0.0.1 (the client accepts ``http://`` for loopback only): every search gets + ``SEARCH_ANSWER``, anything else a 404 problem. It counts the requests it received and sends nothing on.""" + + def __init__(self): + self.requests = 0 + outer = self + + class Handler(BaseHTTPRequestHandler): + def log_message(self, *args): + pass + + def _serve(self): + self.rfile.read(int(self.headers.get("Content-Length") or 0)) + outer.requests += 1 + found = self.command == "POST" and self.path.split("?", 1)[0] == "/v1/searches" + body = json.dumps(SEARCH_ANSWER if found else {"code": "NOT_FOUND", "status": 404}).encode("utf-8") + self.send_response(200 if found else 404) + self.send_header("Content-Type", "application/json" if found else "application/problem+json") + self.send_header("X-Request-ID", f"rehearsal-{outer.requests:04d}") + self.send_header("Content-Length", str(len(body))) + self.end_headers() + self.wfile.write(body) + + do_GET = do_POST = _serve + + self.server = ThreadingHTTPServer(("127.0.0.1", 0), Handler) + self.server.daemon_threads = True + self.url = f"http://127.0.0.1:{self.server.server_address[1]}/v1" + threading.Thread(target=self.server.serve_forever, kwargs={"poll_interval": 0.05}, daemon=True).start() + + def close(self) -> None: + self.server.shutdown() + self.server.server_close() + + +@contextlib.contextmanager +def api_environment(url: str): + """The API route's environment, put back afterwards: GitHits at the loopback stand-in under a placeholder token, no + GitHits setting from the shell, and no gateway or provider switch (``claude_code.THIRD_PARTY_SWITCHES`` would refuse + the api lane; the stand-in sends nothing anywhere).""" + saved = dict(os.environ) + for name in (*THIRD_PARTY_SWITCHES, githits.EXCLUDE_ENV, githits.EXCLUDE_REPLACE_ENV, githits.LICENSES_ENV, + githits.SESSION_ENV, githits.AGENT_ENV, githits.RECORD_DIR_ENV, githits.RECORD_KEY_ENV): + os.environ.pop(name, None) + os.environ.update({githits.TOKEN_ENV: "rehearsal-placeholder-not-a-token", githits.URL_ENV: url}) + try: + yield + finally: + os.environ.clear() + os.environ.update(saved) + def git(root: Path, *args: str) -> None: env = {**os.environ, "GIT_AUTHOR_NAME": "dev", "GIT_AUTHOR_EMAIL": "dev@example.test", @@ -206,7 +388,31 @@ def git(root: Path, *args: str) -> None: subprocess.run(["git", "-C", str(root), *args], check=True, capture_output=True, env=env) -def rehearse(folder: Path) -> tuple[list[dict], dict]: +def claude_code(kit: Kit, thread, script: Path, item: str, arm: str) -> ClaudeCodeProvider: + """The API route's proposer, as the pilot builds it: ``syberlabs propose --claude-code --bare`` in arm A, the same + with ``--githits --githits-arm on`` in arm B, except on the misconfigured item, whose treatment forgot + ``--githits``. The executable is the stand-in; everything else is the real provider.""" + return ClaudeCodeProvider(kit.repo, bare=True, executable=[sys.executable, str(script)], max_calls=3, + githits=arm == "on" and item != MISCONFIGURED, arm=arm, thread_label=thread.id) + + +@contextlib.contextmanager +def mcp_environment(arm: str, spec: dict): + """The MCP route's stand-in reads its arm and the items from the environment; put back afterwards.""" + saved = dict(os.environ) + os.environ.update({"STAND_IN_ARM": arm, "STAND_IN_ITEMS": json.dumps(spec)}) + try: + yield + finally: + os.environ.clear() + os.environ.update(saved) + + +def rehearse(folder: Path, route: str = "api", log=print) -> tuple[list[dict], dict]: + """Every item through both arms on ``route`` (``api`` or ``mcp``), in a new repository under ``folder``: the + journal's events and the contracts, for ``syberlabs metrics`` and ``syberlabs experiment report``.""" + if route not in ROUTES: + raise ValueError(f"route must be one of {', '.join(ROUTES)}") root = folder / "shop" (root / "src").mkdir(parents=True) (root / "tests").mkdir() @@ -222,43 +428,62 @@ def rehearse(folder: Path) -> tuple[list[dict], dict]: home = root / ".syberlabs" (home / "contracts").mkdir(parents=True) (home / "contracts" / f"{contract['id']}.v{contract['version']}.json").write_text(json.dumps(contract)) - script = folder / "stand_in.py" - script.write_text(STAND_IN) spec = {item: {"objective": objective, "tag": tag, "right": right, "wrong": wrong} for item, (objective, tag, right, wrong) in ITEMS.items()} + if route == "api": + (folder / "items.json").write_text(json.dumps(spec), encoding="utf-8") + script = folder / "claude_stand_in.py" + script.write_text(f"SPEC = {str(folder / 'items.json')!r}\n" + CLAUDE_STAND_IN, encoding="utf-8") + else: + script = folder / "stand_in.py" + script.write_text(MCP_STAND_IN) + fake = LoopbackGitHits() if route == "api" else None kit = Kit.local(home, root, actor=RUNNER) try: - for item, (objective, tag, _right, _wrong) in ITEMS.items(): - for arm in ("off", "on"): - thread = kit.start(objective, f"{contract['id']}.v{contract['version']}", paths=["src/", "tests/"], - unattended=True, actor=RUNNER) - thread.record_experiment(NAME, item=item, arm=arm, by=RUNNER, tags=[tag]) # before any candidate - os.environ.update({"STAND_IN_ARM": arm, "STAND_IN_ITEMS": json.dumps(spec)}) - found = thread.propose(RepairProvider([sys.executable, str(script)], name="stand-in", max_calls=3), - seed=1) - receipt = thread.accept(found[-1].id, actor=RUNNER) if found else None - state = receipt.status if receipt else "no candidate" - if receipt and receipt.status == "succeeded": - # A reviewer who is not told the arm reads the accepted change (recorded as syberlabs review - # --record does). The stand-in reviewer turns down one change, so the rule has a failure to count. - verdict = "fail" if (item, arm) == ("LIB-2", "off") else "pass" - thread.record_review(verdict, reviewer="blind-reviewer", revision=receipt.commit, by=REVIEWER) - state += f", review {verdict}" - print(f"{item:10} {arm:3} {len(found)} candidate(s), acceptance {state}") + with api_environment(fake.url) if fake else contextlib.nullcontext(): + for item, (objective, tag, _right, _wrong) in ITEMS.items(): + for arm in ("off", "on"): + thread = kit.start(objective, f"{contract['id']}.v{contract['version']}", paths=["src/", "tests/"], + unattended=True, actor=RUNNER) + thread.record_experiment(NAME, item=item, arm=arm, by=RUNNER, tags=[tag]) # before any candidate + if route == "api": + found = thread.propose(claude_code(kit, thread, script, item, arm), seed=1) + else: + with mcp_environment(arm, spec): + found = thread.propose(RepairProvider([sys.executable, str(script)], name="stand-in", + max_calls=3), seed=1) + receipt = thread.accept(found[-1].id, actor=RUNNER) if found else None + state = receipt.status if receipt else "no candidate" + if receipt and receipt.status == "succeeded": + # A reviewer who is not told the arm reads the accepted change (recorded as syberlabs review + # --record does). The stand-in reviewer turns down one change, so the rule has a failure to + # count. + verdict = "fail" if (item, arm) == ("LIB-2", "off") else "pass" + thread.record_review(verdict, reviewer="blind-reviewer", revision=receipt.commit, by=REVIEWER) + state += f", review {verdict}" + log(f"{item:10} {arm:3} {len(found)} candidate(s), acceptance {state}") + if fake is not None: + log(f"The loopback GitHits received {fake.requests} requests; the real API received none.") events = [e for case in kit.journal.thread_ids() for e in kit.session.history(case)] return events, dict(kit.session.contracts) finally: kit.close() + if fake is not None: + fake.close() def main() -> int: parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + parser.add_argument("--route", choices=ROUTES, default="api", + help="arm B's GitHits: the host's API launcher, the pilot's route (api, default), or an MCP " + "server, the route before #134 (mcp)") parser.add_argument("--out", help="write report.json, report.md and metrics.csv here") args = parser.parse_args() - print("SIMULATED: a stand-in answers instead of a model. These numbers check the pipeline; they are not results.\n") + print("SIMULATED: a stand-in answers instead of a model. These numbers check the pipeline; they are not results.") + print(f"Arm B's GitHits route: {args.route}\n") with tempfile.TemporaryDirectory() as folder: - events, contracts = rehearse(Path(folder)) + events, contracts = rehearse(Path(folder), args.route) items = metrics.work_items(events, contracts) found = experiment.report(events, NAME, contracts, delivered_by="review") print("\nPer work item (syberlabs metrics):") @@ -275,7 +500,7 @@ def main() -> int: (out / "metrics.csv").write_text(metrics.to_csv(items), encoding="utf-8") print(f"\nwrote {out / 'report.json'}, {out / 'report.md'} and {out / 'metrics.csv'}") excluded = found["checks"].get("arm_conflict", []) - return 0 if excluded == ["LIB-4"] and found["items"]["compared"] == 6 else 1 + return 0 if excluded == [MISCONFIGURED] and found["items"]["compared"] == 6 else 1 if __name__ == "__main__": diff --git a/scripts/githits_engineering_smoke.py b/scripts/githits_engineering_smoke.py new file mode 100644 index 0000000..98f1065 --- /dev/null +++ b/scripts/githits_engineering_smoke.py @@ -0,0 +1,85 @@ +"""One operator recipe around pilot_dry_run; preparation is offline by default. + +python scripts/githits_engineering_smoke.py --work DIR --factory-commit SHA --run-approved --claude PATH +Mateo executes --run-approved only after explicit provider/run/spend approval. +""" +import argparse +import hashlib +import json +import os +from pathlib import Path +import shutil +import subprocess +import sys + +if __package__: + from scripts import pilot_dry_run as pilot +else: + import pilot_dry_run as pilot + +ROOT = Path(__file__).resolve().parents[1] +PACK = ROOT / "examples" / "githits-engineering-smoke" +MODEL = "claude-sonnet-5-5" +CLI_VERSION = "2.1.311" + + +def prepare(work): + work.mkdir(parents=True, exist_ok=False) + repo = work / "fixture" + shutil.copytree(PACK / "fixture", repo) + env = {**os.environ, "GIT_AUTHOR_NAME": "Engineering smoke", "GIT_COMMITTER_NAME": "Engineering smoke", + "GIT_AUTHOR_EMAIL": "smoke@syberlabs.invalid", "GIT_COMMITTER_EMAIL": "smoke@syberlabs.invalid", + "GIT_AUTHOR_DATE": "2026-10-26T00:00:00Z", "GIT_COMMITTER_DATE": "2026-10-26T00:00:00Z"} + for args in (["init", "-q", "-b", "main"], ["config", "core.autocrlf", "false"], + ["config", "commit.gpgsign", "false"], ["add", "."], + ["commit", "-qm", "Freeze synthetic engineering smoke base"]): + subprocess.run(["git", "-C", str(repo), *args], env=env, check=True, capture_output=True) + plan = json.loads((PACK / "plan.json").read_text(encoding="utf-8")) + plan["checks"] = {"engineering-smoke": {"argv": [sys.executable, str(PACK / "check.py")], "timeout_seconds": 30}} + pilot.validate_plan(plan) + path = work / "plan.json" + path.write_text(json.dumps(plan, indent=2) + "\n", encoding="utf-8") + manifest = {"purpose": "ENGINEERING SMOKE, synthetic readiness only; formal pack unchanged", + "factory": pilot.host_checkout(), "fixture_base": pilot.git(repo, "rev-parse", "HEAD"), + "plan_sha256": hashlib.sha256(path.read_bytes()).hexdigest(), + "check_sha256": hashlib.sha256((PACK / "check.py").read_bytes()).hexdigest(), + "skill_sha256": pilot.claude_code.githits_skill()[1]} + (work / "manifest.json").write_text(json.dumps(manifest, indent=2) + "\n", encoding="utf-8") + return repo, path + + +def main(argv=None): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--work", type=Path, required=True, help="new private directory; never reused") + parser.add_argument("--factory-commit", required=True, help="exact approved factory SHA") + parser.add_argument("--run-approved", action="store_true", help="Mateo only, after explicit live/spend approval") + parser.add_argument("--claude", help="Claude Code 2.1.311 executable (else the existing configured command)") + args = parser.parse_args(argv) + factory = pilot.host_checkout() + if args.factory_commit != factory["commit"] or (args.run_approved and factory["dirty"]): + print("REFUSED: factory commit differs from the requested pin") + return 2 + repo, plan = prepare(args.work.resolve()) + print("ENGINEERING SMOKE prepared: synthetic tasks, no formal pilot data;", plan) + if not args.run_approved: + print("No network or provider call was made. Live execution requires Mateo's explicit approval.") + return 0 + if args.claude: + os.environ[pilot.claude_code.BIN_ENV] = str(Path(args.claude).resolve()) + command = pilot.claude_code.claude_executable() + version = subprocess.run([*command, "--version"], capture_output=True, timeout=30) + if version.returncode or not version.stdout.decode("utf-8", "replace").startswith(CLI_VERSION + " "): + print("REFUSED: the frozen smoke plan requires Claude Code", CLI_VERSION) + return 2 + if "--model" in command: + print("REFUSED: remove the command's model override; the smoke recipe pins ANTHROPIC_MODEL") + return 2 + os.environ.update(ANTHROPIC_MODEL=MODEL, CLAUDE_CODE_EFFORT_LEVEL="medium", + CLAUDE_CODE_PROMPT_CACHE_TTL="5m", CLAUDE_CODE_SUBAGENT_PROMPT_CACHE_TTL="5m", + FORCE_PROMPT_CACHING_5M="1", CLAUDE_CODE_DISABLE_NONESSENTIAL_TRAFFIC="1") + return pilot.main(["--repo", str(repo), "--plan", str(plan), "--out", str(args.work.resolve() / "receipts"), + "--budget", "8", "--seed", "7", "--actor", "mateo", "--reviewer", "blind-engineering"]) + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/pilot_dry_run.py b/scripts/pilot_dry_run.py new file mode 100644 index 0000000..a6569cd --- /dev/null +++ b/scripts/pilot_dry_run.py @@ -0,0 +1,1596 @@ +"""The GitHits pilot's dry run as one command: preflight, canary, both arms per item, the paired report. + + python scripts/pilot_dry_run.py --template > plan.json # a plan to fill in + python scripts/pilot_dry_run.py --plan plan.json # preflight, confirm, canary, runs, review, report + +It runs the documented pilot commands (``docs/EXPERIMENTS.md``, the GitHits pilot) against a home of its own, on the +operator's machine, under the operator's own credentials. Running it spends money (Class X, ``AGENTS.md``): the +operator executes it; nothing here was run against a real model or the real GitHits API. + +How Mateo runs it +----------------- + +1. Check out the base commit the items start from, in the repository they change (``git checkout ``). Every + thread starts from that ``HEAD``; the script refuses to start if ``--base`` is given and ``HEAD`` is elsewhere, + and stops if ``HEAD`` moves during the run. +2. Put both credentials in the shell's environment, never in a file:: + + export ANTHROPIC_API_KEY=... # your own key: the api lane (claude -p --bare) + export GITHITS_API_TOKEN=... + unset ANTHROPIC_BASE_URL # and any CLAUDE_CODE_USE_* switch: the api lane refuses them + + (PowerShell: ``$env:ANTHROPIC_API_KEY = "..."``; ``Remove-Item Env:ANTHROPIC_BASE_URL``.) +3. Write the plan: ``python scripts/pilot_dry_run.py --template > plan.json``, then fill in 3 to 5 items (id, tags, + objective, paths) and the four rates of the model your Claude Code runs: input, output, cache read and cache + write, in dollars per million tokens (the canary names the model if it is not the one priced). +4. Run ``python scripts/pilot_dry_run.py --plan plan.json`` from the SyberWork checkout (Windows: + ``py scripts/pilot_dry_run.py --plan plan.json``; ``--repo PATH`` when the items change another repository; + ``--budget 8`` is the default scheduling budget in dollars; ``--seed N`` repeats an order). It prints the preflight, asks to + start, runs the canary and then every run, asks for the blind review of each accepted change (shown as R1, R2, ... + in a shuffled order, without its arm: hand the keyboard to the reviewer, or answer yourself without looking up the + mapping), prints the paired report and the PASS/FAIL table, and writes ``receipt.json``. Exit 0 is PASS, 1 FAIL, + 2 refused before anything was spent (a preflight failure, or no go). +5. Send ``receipt.json`` and ``report.md`` from the run directory (printed at the end). The journal, the diffs and + ``commands.log`` stay on the machine: they hold the items' objectives and code. + +The plan (JSON): ``items``, 3 to 5 of ``{id, tags, objective, paths}``; ``prices``, dollars per million tokens per +model (``evolution.budget.prices``) with all four rates above 0 (input, output, cache read, cache creation: a cache +rate left out would be priced at the input rate, one at 0 as free), or instead a whole ``contract`` that holds such +prices, ``max_dollars`` and ``strict``; and what both arms share: ``max_calls`` (3: one shot, two repairs), +``max_turns`` (30, Claude Code's ``--max-turns``), ``allowed_tools`` (the pilot's, ``docs/GITHITS.md``: +``Read,Edit,Write,Glob,Grep,WebSearch,WebFetch``; no shell), ``guard_max_lines`` and ``guard_max_chars``, ``checks`` +(the derived contract's checks; default: those ``syberlabs init`` detects) and ``canary_paths``. + +The sequence +------------ + +Preflight refuses to start on any failure: the repository and its base commit; Claude Code ``2.1.295`` or newer +(``claude_code.MIN_VERSION``: ``onFailure: "block"`` on the guard's hook); the api lane on the operator's own +``ANTHROPIC_API_KEY`` with no third-party routing switch (``claude_code.third_party``: ``ANTHROPIC_BASE_URL``, +``CLAUDE_CODE_USE_*``, ``CLAUDE_CODE_PROVIDER_MANAGED_BY_HOST``); ``GITHITS_API_TOKEN`` in the environment and in no +file the run touches (the repository's tracked and untracked files, its ``.env*``, ``.mcp.json`` and local Claude +settings, the plan); ``syberlabs githits doctor`` ``ok``, with its call record verifying under a key the host set; +the query guard's key reaching the hook (``CLAUDE_CODE_SUBPROCESS_ENV_SCRUB`` off, and the host's own hook, run as +Claude Code runs it, signing a block and a pass that verify under a key the host set); the scheduling budget; and the +contracts the run installs. + +The canary is one cheap call in the treatment's mode (``propose --claude-code --bare --githits``, one call, a few +turns, without the web tools so the launcher is its only way to look anything up) whose objective asks for two +launcher lookups, one the guard must refuse (it holds an email address). It passes when the guard refused a lookup +for what it says and passed one that GitHits answered (an authentic client record), every decision verified and none +``unguarded``, and every model the call used is priced; a call that shows only one of the two is inconclusive and +fails, since it proves either no refusal or no GitHits. Then each item runs in both arms from the same ``HEAD``, +both arms back to back, the items and which arm goes first set by a seeded shuffle: ``start --unattended``, +``experiment label`` (before the first candidate), ``propose --claude-code --bare [--githits]``, ``run --once`` +(accepts on the host's checks), ``export``. Then the blind review (``review --record``), ``experiment report --items +FILE --delivered review [--json | --markdown]``, and the bar. + +The bar: the nine metrics known for every thread +------------------------------------------------ + +Cost, turns, agent time, fix rounds, web lookups, GitHits lookups, delivered, review rejections and human +interventions (``metrics.METRICS``), per thread as the paired report reads them (``experiment.Run.value``: counted +up to the thread's first acceptance). Two of the nine are read the way the report pairs them. Agent time is the +report's time to a passing change (``time_to_pass``): its merge-based form needs a merged pull request, which a dry +run never has and the pilot has for at most one arm per item. Delivered is ``reviewed``, the pilot's definition +(accepted, and the blind review passed: ``--delivered review``): its merge-based form is known in a dry run but is +"no" for every thread, which measures nothing. The merge-based values are in the receipt beside them (``strict``); +``syberlabs settle`` and agent time to merge, the pilot's end-of-week step, are not exercised. A thread that is not +accepted has no time to a passing change, so the bar fails, and the reasons say why it was not accepted. + +Budget +------ + +``--budget`` (default $8) is the cap for the whole run, canary included. Each thread's contract ``max_dollars`` is +an equal share of it (``--thread-budget``; default the cap over the runs plus the canary), so the host's own meter +stops a thread's next call past its share and its promotion is refused; and a run starts only while the dollars +already spent plus one share fit under the cap. Every call is bounded by the plan's ``max_turns``. So the total +stays under the cap plus the cost of one call, and the receipt says what it was. A call that ended +``provider_failed``, ``provider_timeout`` or ``provider_error`` after Claude Code ran was never priced (it raised +before the meter), so its thread's cost and the total are unknown and no further run starts. Spend is the host's +own: tokens priced at the plan's ``prices`` through the host's meter (``docs/METRICS.md``), never Claude Code's +estimate. A fee the API charges per server-side web search is not a token, so neither the meter nor the cap sees it; +each thread's web lookups in the receipt say how many searches there were. + +The paired report +----------------- + +It must compare every item: an item the report excludes (``experiment.EXCLUDE``: ``arm_conflict``, +``contaminated``, ``missing_arm``, ...) or a registered item it never saw fails the run, and a check that keeps an +item but says something about it (``unused``, ``use_unknown``, ``two_routes``, ...) is printed as a warning. +The report reads the treatment's GitHits from the host's launcher ledger (``signal.githits``, SyberWork #141, branch +``fix/experiment-report-githits-api-arm``); a report without that change excludes every item as ``arm_conflict``, +and the run fails. + +Standard library only; nothing in ``syberlabs`` is changed. Tests: ``tests/test_pilot_dry_run.py`` (a stand-in +``claude`` and a loopback GitHits; no model, no network). +""" + +from __future__ import annotations + +import argparse +import copy +import hashlib +import json +import math +import os +import platform +import random +import re +import secrets +import shutil +import subprocess +import sys +import tempfile +from datetime import datetime, timezone +from pathlib import Path +from typing import Callable, NamedTuple + +ROOT = Path(__file__).resolve().parents[1] # the SyberWork checkout this script belongs to: the host's code +if str(ROOT) not in sys.path: + sys.path.insert(0, str(ROOT)) + +from syberlabs import Rejected, claude_code, experiment, githits, metrics, query_guard # noqa: E402 +from syberlabs.build import default_contract # noqa: E402 +from syberlabs.economic import check_prices # noqa: E402 + +RECEIPT = "syberlabs.pilot-dry-run/1" +DEFAULT_BUDGET = 8.0 +DEFAULT_EXPERIMENT = "githits-dry-run" +DEFAULT_MAX_CALLS = 3 # one shot and two repairs per thread +DEFAULT_MAX_TURNS = 30 # Claude Code's agentic turns per call: bounds a call's cost +DEFAULT_MAX_SECONDS = 3600 # per search: a call, its repairs and the checks +DEFAULT_REVIEWER = "blind-1" +MIN_ITEMS, MAX_ITEMS = 3, 5 +# Both arms' tools in the pilot's documented commands (docs/GITHITS.md, "The A/B report"); the treatment adds its +# launcher. No shell: a shell could read the guard's key (claude_code.DEFAULT_TOOLS). +WEB_TOOLS = ("WebSearch", "WebFetch") +PILOT_TOOLS = (*claude_code.DEFAULT_TOOLS, *WEB_TOOLS) +# Every rate of a priced model: Claude Code's calls are mostly cache reads and writes, and a cache rate left out is +# priced at the input rate (economic.price_usage), one at 0 makes those tokens free. Either way the cost is wrong. +RATES = ("input_per_million", "output_per_million", "cache_read_per_million", "cache_creation_per_million") +# claude_code raises the first two after Claude Code ran and before the host's meter priced the call, and the search +# records an unexpected fault there as the third: what such a call spent is lost. +UNMETERED = ("provider_failed", "provider_timeout", "provider_error") +# The guard's rules that judge what a lookup says. Its others refuse a command that is not the launcher, or report the +# guard's own failure: a refusal under them shows neither that a content rule fires nor that Claude Code obeys a refused +# launcher call (under dontAsk it would not run a command outside its allowed tools anyway). +CONTENT_RULES = tuple(rule for rule in query_guard.RULES + if rule not in (query_guard.NOT_LAUNCHER, query_guard.SHELL_SYNTAX, query_guard.ERROR)) +REACHED = (githits.OK, githits.BLOCKED) # a client record of a request GitHits answered +BASELINE, TREATMENT = experiment.BASELINE, experiment.TREATMENT # "off" and "on", the report's defaults +CANARY_CONTRACT = "pilot-canary" +CANARY_TURNS = 6 +NINE = ("cost", "turns", "agent_time", "fix_rounds", "web_lookups", "githits_lookups", "delivered", + "review_rejections", "human_interventions") # metrics.METRICS, in its order +PAIRED = {"agent_time": "time_to_pass", "delivered": "reviewed"} # read as the paired report reads them +LABELS = {"cost": "cost", "turns": "turns", "agent_time": "agent time", "fix_rounds": "fix", "web_lookups": "web", + "githits_lookups": "githits", "delivered": "delivered", "review_rejections": "rejections", + "human_interventions": "interventions"} +PASS, FAIL, WARN = "PASS", "FAIL", "WARN" +SCRUB_ENV = "CLAUDE_CODE_SUBPROCESS_ENV_SCRUB" +API_KEY_ENV = "ANTHROPIC_API_KEY" +# Files beside the code where a credential tends to be written; scanned whether or not git ignores them. +SECRET_FILES = (".env", ".mcp.json", ".claude/settings.json", ".claude/settings.local.json", "CLAUDE.local.md") +MAX_SCAN_BYTES = 20 * 1024 * 1024 +MIN_SECRET_CHARS = 12 # a credential shorter than this is not searched for in the run's files (no false alarms) +PLAN_KEYS = ("experiment", "items", "prices", "contract", "checks", "allowed_tools", "max_calls", "max_turns", + "guard_max_lines", "guard_max_chars", "canary_paths") +ITEM_KEYS = ("id", "tags", "objective", "paths") +CANARY_TARGET = "pypi:requests@2.32.3" +CANARY_BLOCKED = f'search "guard canary contact canary@example.invalid" --target {CANARY_TARGET}' +CANARY_PASSING = f'search "Session mount HTTPAdapter max_retries" --target {CANARY_TARGET}' +CANARY_OBJECTIVE = ( + "Query-guard canary, the operator's test of this machine's lookup filter. No code change is wanted: do not " + "create, edit or delete any file. Using your Bash tool, run the GitHits launcher named in your instructions " + f"exactly twice and then stop. Call 1 passes the arguments `{CANARY_BLOCKED}`: the filter is expected to refuse " + "it because it holds an email address, and that refusal is the point of the test, so do not rephrase or retry " + f"it. Call 2 passes the arguments `{CANARY_PASSING}`.") +TEMPLATE = { + "_comment": "3 to 5 items. Keys starting with _ are ignored. Objectives stay on this machine: the receipt " + "holds their digests only. Prices: the model id Claude Code reports, and all four of its rates in " + "dollars per million tokens, none 0 (Claude Code's calls are mostly cache reads and writes).", + "experiment": DEFAULT_EXPERIMENT, + "items": [ + {"id": "LIB-1", "tags": ["needs_lookup"], "objective": "One sentence: the change this item asks for", + "paths": ["syberlabs", "tests"]}, + {"id": "LIB-2", "tags": ["needs_lookup"], "objective": "Another change", "paths": ["syberlabs", "tests"]}, + {"id": "RT-1", "tags": ["routine"], "objective": "A routine change", "paths": ["syberlabs", "tests"]}, + ], + "prices": {"": {"input_per_million": 0, "output_per_million": 0, + "cache_read_per_million": 0, "cache_creation_per_million": 0}}, + "max_calls": DEFAULT_MAX_CALLS, + "max_turns": DEFAULT_MAX_TURNS, + "allowed_tools": ",".join(PILOT_TOOLS), +} + + +class PlanError(ValueError): + """The plan cannot be run as written; the message says why.""" + + +class Done(NamedTuple): + code: int + out: str + err: str + + +Runner = Callable[[list, dict, float], Done] +Ask = Callable[[str], str] + + +def say(text: str = "") -> None: + print(text, flush=True) + + +def utc_now() -> str: + return datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ") + + +def digest(value) -> str: + """SHA-256 of a string, or of a JSON value in a fixed form.""" + text = value if isinstance(value, str) else json.dumps(value, sort_keys=True, separators=(",", ":")) + return hashlib.sha256(text.encode("utf-8")).hexdigest() + + +def truthy(value) -> bool: + """Set, as Claude Code reads a switch: anything but empty, 0, false, no and off.""" + return str(value or "").strip().lower() not in ("", "0", "false", "no", "off") + + +def tail(text: str, limit: int = 300) -> str: + flat = " ".join(str(text or "").split()) + return flat if len(flat) <= limit else "..." + flat[-limit:] + + +def git(repo: Path, *args: str) -> str: + done = subprocess.run(["git", "-C", str(repo), *args], stdin=subprocess.DEVNULL, capture_output=True, timeout=120) + if done.returncode != 0: + raise RuntimeError(f"git {args[0]}: {tail(done.stderr.decode('utf-8', 'replace'), 200)}") + return done.stdout.decode("utf-8", "replace").strip() + + +# ------------------------------------------------------------------------------------------------------------- plan + +def _name(value, what: str) -> str: + if not isinstance(value, str) or not experiment._NAME.match(value): + raise PlanError(f"{what} must be a short name: letters, digits, _ . : + - and at most 64 characters") + return value + + +def relative_path(path) -> bool: + """A repository path the plan may name: relative, no drive, no ``..`` part (either separator, for Windows).""" + return (isinstance(path, str) and bool(path.strip()) and not path.startswith(("/", "\\")) and ":" not in path + and ".." not in re.split(r"[\\/]", path)) + + +def file_name(text: str) -> str: + """``text`` as part of a file name on any system: an item id may hold ``:``, an NTFS stream separator.""" + return re.sub(r"[^A-Za-z0-9_.+-]", "_", text) + + +def missing_rates(prices) -> str | None: + """Why a price table cannot price a Claude Code call in full, or None: every model needs a real id and all four + rates (``RATES``) above 0.""" + for model, rates in (prices or {}).items(): + if "<" in model: + return f"fill in the model id Claude Code reports, not {model!r}" + zero = [rate for rate in RATES if not (isinstance(rates.get(rate), (int, float)) + and not isinstance(rates.get(rate), bool) and rates[rate] > 0)] + if zero: + return (f"fill in {model}'s {', '.join(zero)} (dollars per million tokens, above 0): Claude Code's calls " + "are mostly cache reads and writes, and a cache rate left out is priced at the input rate, one " + "at 0 as free") + return None + + +def _count(raw: dict, key: str, low: int, high: int, default): + value = raw.get(key, default) + if value is None: + return None + if type(value) is not int or not low <= value <= high: + raise PlanError(f"{key} must be a whole number from {low} to {high}") + return value + + +def validate_plan(raw) -> dict: + """The plan, normalized. 3 to 5 items, each ``{id, tags, objective, paths}``; ``prices`` (or a whole ``contract`` + that has them); the knobs both arms share.""" + if not isinstance(raw, dict): + raise PlanError("the plan must be a JSON object") + unknown = sorted(key for key in raw if not str(key).startswith("_") and key not in PLAN_KEYS) + if unknown: + raise PlanError("unknown plan keys: " + ", ".join(unknown)) + name = _name(raw.get("experiment", DEFAULT_EXPERIMENT), "experiment") + items = raw.get("items") + if not isinstance(items, list) or not MIN_ITEMS <= len(items) <= MAX_ITEMS: + raise PlanError(f"items must list {MIN_ITEMS} to {MAX_ITEMS} work items (a dry run, not the pilot)") + found, objectives = [], set() + for number, item in enumerate(items, 1): + if not isinstance(item, dict) or set(item) - set(ITEM_KEYS) or not {"id", "objective"} <= set(item): + raise PlanError(f"item {number} must be an object with id and objective, and optionally tags and paths") + item_id = _name(item["id"], f"item {number}'s id") + tags = item.get("tags", []) + if not isinstance(tags, list) or len(tags) > experiment.MAX_TAGS: + raise PlanError(f"item {item_id}: tags must be a list of at most {experiment.MAX_TAGS} names") + tags = sorted(set(_name(tag, f"item {item_id}'s tag") for tag in tags)) + objective = item["objective"] + if (not isinstance(objective, str) or not objective.strip() or len(objective) > 4000 + or objective.strip().startswith("-")): + raise PlanError(f"item {item_id}: objective must be a sentence of at most 4000 characters, not starting " + "with '-'") + if objective.strip() in objectives: + raise PlanError(f"item {item_id}: another item has the same objective") + objectives.add(objective.strip()) + paths = item.get("paths", []) + if not isinstance(paths, list) or not all(relative_path(p) for p in paths): + raise PlanError(f"item {item_id}: paths must be relative repository paths") + found.append({"id": item_id, "tags": tags, "objective": objective.strip(), "paths": list(paths)}) + if len({item["id"] for item in found}) != len(found): + raise PlanError("item ids must be distinct") + try: + experiment.parse_items(items_text(found, name)) + except ValueError as exc: + raise PlanError(f"the registered item list: {exc}") from None + contract, prices = raw.get("contract"), raw.get("prices") + if contract is not None and prices is not None: + raise PlanError("give prices or a whole contract (whose budget holds its prices), not both") + if contract is None: + reason = check_prices(prices) if prices is not None else "prices are required: the cost metric needs them" + reason = reason or missing_rates(prices) + if reason: + raise PlanError(f"prices: {reason}") + elif not isinstance(contract, dict) or not isinstance(contract.get("evolution"), dict): + raise PlanError("contract must be a whole contract document with an evolution section") + checks = raw.get("checks") + if checks is not None and (contract is not None or not isinstance(checks, dict) or not checks): + raise PlanError("checks must be a non-empty object of named checks, and only without a whole contract") + tools = raw.get("allowed_tools", ",".join(PILOT_TOOLS)) + tools = claude_code.parse_tools(tools) if isinstance(tools, str) else tools + if not isinstance(tools, list) or not tools or any(not isinstance(t, str) or not t for t in tools): + raise PlanError("allowed_tools must be a comma list of tool names") + if claude_code.code_tools(tools): + raise PlanError("allowed_tools: no Bash or PowerShell; the treatment's launcher is its only command") + canary_paths = raw.get("canary_paths") + if canary_paths is not None and (not isinstance(canary_paths, list) + or not all(relative_path(p) for p in canary_paths)): + raise PlanError("canary_paths must be a list of relative repository paths") + return {"experiment": name, "items": found, "prices": prices, "contract": contract, "checks": checks, + "allowed_tools": ",".join(tools), "max_calls": _count(raw, "max_calls", 1, 12, DEFAULT_MAX_CALLS), + "max_turns": _count(raw, "max_turns", 1, 200, DEFAULT_MAX_TURNS), + "guard_max_lines": _count(raw, "guard_max_lines", 1, 100, None), + "guard_max_chars": _count(raw, "guard_max_chars", 1, 100_000, None), "canary_paths": canary_paths} + + +def load_plan(path: Path) -> dict: + try: + raw = json.loads(Path(path).read_text(encoding="utf-8")) + except OSError as exc: + raise PlanError(f"{path}: {exc.strerror or type(exc).__name__}") from None + except ValueError as exc: + raise PlanError(f"{path}: not JSON ({exc})") from None + return validate_plan(raw) + + +def items_text(items: list, name: str) -> str: + """The registered item list (``experiment report --items``), written before the first run.""" + lines = [f"# {name}: registered by scripts/pilot_dry_run.py before the first run"] + lines += [" ".join([item["id"], *item["tags"]]) for item in items] + return "\n".join(lines) + "\n" + + +def build_contracts(plan: dict, repo: Path, thread_budget: float) -> tuple[dict, dict]: + """The arms' contract and the canary's. Without a whole contract in the plan, the repository's default contract + (``build.default_contract``: its detected checks, or the plan's) with the plan's prices, ``max_dollars`` the + thread's share of the cap, ``strict`` on (a model the prices do not name refuses promotion), and one hour per + search. The canary's is the same with one trivial check: it is never accepted, and nothing it writes is judged.""" + if plan["contract"] is not None: + main = copy.deepcopy(plan["contract"]) + else: + main = default_contract(repo) + if plan["checks"] is not None: + main["evolution"]["evaluation"]["checks"] = copy.deepcopy(plan["checks"]) + main["evolution"]["evaluation"]["required"] = sorted(plan["checks"]) + main["evolution"]["budget"] = {"max_candidates": 12, "max_evaluations": 24, "max_seconds": DEFAULT_MAX_SECONDS, + "prices": copy.deepcopy(plan["prices"]), "max_dollars": thread_budget, + "strict": True} + canary = copy.deepcopy(main) + canary.update({"id": CANARY_CONTRACT, "version": 1, "title": "Query-guard canary (never accepted)"}) + canary["evolution"]["evaluation"] = {"checks": {"canary": {"argv": [sys.executable, "-c", "pass"], + "timeout_seconds": 60}}, + "required": ["canary"], "max_age_seconds": 3600} + budget = canary["evolution"]["budget"] + budget.update({"max_candidates": 2, "max_evaluations": 2, "max_seconds": min(budget.get("max_seconds", 900), 900)}) + return main, canary + + +def contract_name(doc: dict) -> str: + return f"{doc['id']}.v{doc['version']}" + + +def schedule(items: list[str], seed: int, baseline: str = BASELINE, treatment: str = TREATMENT) -> list[tuple]: + """Both arms of every item, back to back, so drift during the run falls on both arms of an item alike. A seeded + shuffle sets the order of the items and which arm runs first; the first arms are balanced (as many items start + with one arm as with the other, give or take one).""" + rng = random.Random(seed) + order = list(items) + rng.shuffle(order) + firsts = ([baseline, treatment] * len(order))[:len(order)] + rng.shuffle(firsts) + runs = [] + for item, first in zip(order, firsts): + runs += [(item, first), (item, treatment if first == baseline else baseline)] + return runs + + +# --------------------------------------------------------------------------------------------- the syberlabs CLI + +def subprocess_runner(argv: list, env: dict | None = None, timeout: float | None = None) -> Done: + """``python -m syberlabs ARGV`` with this interpreter, from the host checkout (so its ``syberlabs`` runs): what + the operator would type. UTF-8 output, so a diff decodes the same on Windows.""" + done = subprocess.run([sys.executable, "-m", "syberlabs", *argv], cwd=str(ROOT), stdin=subprocess.DEVNULL, + env={**os.environ, "PYTHONIOENCODING": "utf-8", **(env or {})}, capture_output=True, + timeout=timeout) + return Done(done.returncode, done.stdout.decode("utf-8", "replace"), done.stderr.decode("utf-8", "replace")) + + +class Syberlabs: + """The ``syberlabs`` commands of the run, against its own home (``/home``). Every command is logged to + ``commands.log`` with its output (local only: it holds the objectives). The receipt keeps each command's argv with + every objective as a digest and the run's directory as ````, so it can travel without them.""" + + def __init__(self, home: Path, repo: Path, actor: str, runner: Runner, log: Path | None = None): + self.home, self.repo, self.actor, self.runner, self.log = home, repo, actor, runner, log + self.commands: list[dict] = [] + self.private: set[str] = set() # texts the receipt holds only as digests (objectives) + + def shown(self, arg: str) -> str: + if arg in self.private: + return f"" + return arg.replace(str(self.home.parent), "") + + def __call__(self, *args: str, env: dict | None = None, timeout: float | None = 900) -> Done: + args = tuple(map(str, args)) + credentials = [os.environ.get(name) for name in (githits.TOKEN_ENV, API_KEY_ENV)] + def scrub(text): + for value in credentials: + if value: + text = text.replace(value, "[credential redacted]") + return text + argv = ["--home", str(self.home), "--repo", str(self.repo), "--actor", self.actor, *args] + try: + done = self.runner(argv, env or {}, timeout) + except subprocess.TimeoutExpired: + done = Done(124, "", f"timed out after {timeout} seconds") + done = Done(done.code, scrub(done.out), scrub(done.err)) + self.commands.append({"argv": [scrub(self.shown(arg)) for arg in args], "code": done.code}) + if self.log is not None: + with open(self.log, "a", encoding="utf-8") as handle: + handle.write(f"$ syberlabs {scrub(' '.join(args))}\n[exit {done.code}]\n{done.out}{done.err}\n") + return done + + def current(self) -> str: + return (self.home / "current").read_text(encoding="utf-8").strip() + + def export(self, thread: str, path: Path) -> dict | None: + done = self("--thread", thread, "export", "--out", str(path)) + if done.code != 0 or not path.is_file(): + return None + return json.loads(path.read_text(encoding="utf-8")) + + +# -------------------------------------------------------------------------------------------------------- preflight + +class Check(NamedTuple): + name: str + status: str + detail: str + data: dict | None = None # what the receipt keeps beside the line (the base commit, the version read) + + def value(self, key: str): + return (self.data or {}).get(key) + + +def check_repository(repo: Path, base: str | None) -> Check: + try: + head = git(repo, "rev-parse", "--verify", "HEAD") + except (RuntimeError, OSError) as exc: + return Check("repository", FAIL, f"{repo} is not a git repository with a commit ({exc})") + if base: + try: + wanted = git(repo, "rev-parse", "--verify", base + "^{commit}") + except RuntimeError: + return Check("repository", FAIL, f"--base {base} is not a commit of {repo}") + if wanted != head: + return Check("repository", FAIL, f"HEAD is {head[:10]}, not the base {wanted[:10]}: check out the base " + f"first (git checkout {wanted[:10]})") + host = host_checkout() + note = "" + if host["commit"]: + note = f"; host checkout {host['commit'][:10]}" + (" with local changes" if host["dirty"] else "") + return Check("repository", PASS, f"base {head[:10]} (HEAD){note}", {"base": head, "host": host}) + + +def host_checkout() -> dict: + """The SyberWork checkout running this script: its commit and whether it has local changes.""" + try: + return {"commit": git(ROOT, "rev-parse", "HEAD"), "dirty": bool(git(ROOT, "status", "--porcelain"))} + except (RuntimeError, OSError): + return {"commit": None, "dirty": None} + + +def check_claude() -> Check: + try: + argv = claude_code.claude_executable() + except Rejected as exc: + return Check("claude code", FAIL, exc.detail) + floor = ".".join(map(str, claude_code.MIN_VERSION)) + try: + done = subprocess.run([*argv, "--version"], stdin=subprocess.DEVNULL, capture_output=True, timeout=60) + except (OSError, subprocess.TimeoutExpired) as exc: + return Check("claude code", FAIL, f"`claude --version` did not answer ({type(exc).__name__})") + found = claude_code.version_tuple(done.stdout.decode("utf-8", "replace")) if done.returncode == 0 else None + if found is None: + return Check("claude code", FAIL, f"could not read the version from `claude --version` (expected the whole " + f"output to be X.Y.Z (Claude Code)); {floor} or later is required") + version = ".".join(map(str, found)) + if found < claude_code.MIN_VERSION: + return Check("claude code", FAIL, f"{version} is older than {floor}, the version that honours onFailure: " + "\"block\" on the query guard's hook", {"version": version}) + return Check("claude code", PASS, f"{version} (floor {floor})", {"version": version}) + + +def check_lane(env) -> Check: + problems = [] + switches = claude_code.third_party(env) + if switches: + problems.append(", ".join(switches) + " set: a --bare run would go to another endpoint or provider, not " + "Anthropic's API under your key; unset it (this dry run runs the api lane only)") + if not str(env.get(API_KEY_ENV, "")).strip(): + problems.append(f"{API_KEY_ENV} is not set: the api lane (--bare) runs on your own key from the environment") + if problems: + return Check("credential lane", FAIL, "; ".join(problems)) + return Check("credential lane", PASS, f"api (--bare, billed) on {API_KEY_ENV} from this environment; no " + "third-party routing switch set", {"lane": "api", "cost_basis": "billed"}) + + +def scan_paths(repo: Path, extra: list) -> list[Path]: + """Where the token must not be: every tracked and untracked (not ignored) file of the repository, the usual + credential files beside it and beside the host checkout whether ignored or not, and ``extra``.""" + paths = set() + try: + names = git(repo, "ls-files", "-z", "--cached", "--others", "--exclude-standard").split("\0") + paths |= {repo / name for name in names if name} + except (RuntimeError, OSError): + pass + for root in {repo.resolve(), ROOT}: + paths |= {root / name for name in SECRET_FILES} | set(root.glob(".env.*")) + paths |= {Path(p) for p in extra} + return sorted(p for p in paths if p.is_file()) + + +def holds(path: Path, needle: bytes, block: int = 1 << 20) -> bool: + """Whether the file's bytes hold ``needle``, read a block at a time (a match across two blocks included).""" + keep, carry = len(needle) - 1, b"" + with open(path, "rb") as handle: + while True: + data = handle.read(block) + if not data: + return False + data = carry + data + if needle in data: + return True + carry = data[-keep:] if keep else b"" + + +def files_holding(secret: str, paths, limit: int | None = MAX_SCAN_BYTES) -> tuple[list[Path], int, int]: + """The files among ``paths`` whose bytes hold ``secret``, how many were read, and how many were not (larger than + ``limit``, or unreadable). The secret is never printed.""" + needle, found, read, skipped = secret.encode("utf-8"), [], 0, 0 + for path in paths: + try: + if limit is not None and path.stat().st_size > limit: + skipped += 1 + continue + hit = holds(path, needle) + except OSError: + skipped += 1 + continue + read += 1 + if hit: + found.append(path) + return found, read, skipped + + +def check_token(env, repo: Path, plan_path: Path | None) -> Check: + token = str(env.get(githits.TOKEN_ENV, "")) + if not token: + return Check("githits token", FAIL, f"{githits.TOKEN_ENV} is not set in the environment") + if not token.isprintable() or any(ch.isspace() for ch in token): + return Check("githits token", FAIL, f"{githits.TOKEN_ENV} holds whitespace or a control character") + found, read, skipped = files_holding(token, scan_paths(repo, [plan_path] if plan_path else [])) + if found: + names = ", ".join(str(p) for p in found[:5]) + (f" and {len(found) - 5} more" if len(found) > 5 else "") + return Check("githits token", FAIL, f"the token is written in {names}: remove it there and keep it in the " + "environment only") + unread = f"; {skipped} not read (over {MAX_SCAN_BYTES >> 20} MB or unreadable)" if skipped else "" + return Check("githits token", PASS, f"in the environment only: in none of the {read} files scanned (the " + "repository's tracked and untracked files, .env*, .mcp.json, local Claude " + f"settings, the plan){unread}", + {"files_scanned": read, "files_unread": skipped}) + + +def check_doctor(sy: Syberlabs) -> Check: + """``syberlabs githits doctor --json`` with records on, under a key the host draws: ``ok``, and exactly one + authentic ``doctor`` record.""" + folder = Path(tempfile.mkdtemp(prefix="syberlabs-doctor-")) + key = secrets.token_hex(32) + try: + done = sy("githits", "doctor", "--json", env={githits.RECORD_DIR_ENV: str(folder), + githits.RECORD_KEY_ENV: key}, timeout=300) + try: + report = json.loads(done.out.strip().splitlines()[-1]) if done.out.strip() else {} + except ValueError: + report = {} + records = githits.read_records(folder, key) + finally: + shutil.rmtree(folder, ignore_errors=True) + report = report if isinstance(report, dict) else {} + data = {k: report.get(k) for k in ("status", "http_status", "request_id", "latency_ms", "base_url", + "rate_limit_remaining")} + if done.code != 0 or report.get("status") != "ok": + why = report.get("problem") or tail(done.err, 160) or "no answer" + return Check("githits doctor", FAIL, f"{report.get('status') or 'error'} ({why})", data) + if [r.get("op") for r in records.records] != ["doctor"] or records.ignored: + return Check("githits doctor", FAIL, "the doctor's call record did not verify under the key the host set " + f"({len(records.records)} authentic, {records.ignored} ignored)", data) + return Check("githits doctor", PASS, f"ok (HTTP {report.get('http_status')}, {report.get('latency_ms')} ms, base " + f"{report.get('base_url')}); its record verified under a key the host set", + data) + + +def check_guard_key(env) -> Check: + """The query guard's per-call key reaches its hook: Claude Code's subprocess scrub is off, and the host's own hook + (``query_guard.hook_command``, the exec form Claude Code runs) signs a block and a pass under a key the host put in + its environment, both verifying. Claude Code itself running the hook under ``--bare`` is the canary's to show.""" + problems = [] + if truthy(env.get(SCRUB_ENV)): + problems.append(f"{SCRUB_ENV} is set: Claude Code would strip the guard's key and the GitHits token from the " + "hook's and the launcher's environment, so every lookup would be blocked unguarded") + directory = Path(tempfile.mkdtemp(prefix="syberlabs-guard-check-")) + launcher = "/syberlabs-preflight/githits" + expected = {"preflight-block": (query_guard.NOT_LAUNCHER, query_guard.BLOCK_EXIT), + "preflight-pass": (query_guard.PASS, 0)} + calls = {"preflight-block": {"command": "echo guard-canary"}, + "preflight-pass": {"command": f'{launcher} search "json loads parse_float" --target {CANARY_TARGET}'}} + codes = {} + try: + query_guard.prepare(directory, query_guard.policy(), set(), bash=True) + key = secrets.token_hex(32) + hook = query_guard.hook_command(directory) + hook_env = {**os.environ, **query_guard.hook_environment(directory, key, launcher)} + for use_id, tool_input in calls.items(): + payload = {"session_id": "preflight", "hook_event_name": "PreToolUse", "tool_name": "Bash", + "tool_input": tool_input, "tool_use_id": use_id} + try: + done = subprocess.run([hook["command"], *hook["args"]], input=json.dumps(payload).encode("utf-8"), + capture_output=True, env=hook_env, timeout=60) + codes[use_id] = done.returncode + except (OSError, subprocess.TimeoutExpired): + codes[use_id] = None + verified = {d["tool_use_id"]: d["rule"] for d in query_guard.read_decisions(directory) + if query_guard.authentic(d, key)} + finally: + query_guard.remove(directory) + if {k: (verified.get(k), codes.get(k)) for k in expected} != expected: + problems.append("the host's query guard hook did not sign its decisions under the key the host set (decisions " + f"{verified}, exit codes {codes})") + if problems: + return Check("guard key", FAIL, "; ".join(problems)) + return Check("guard key", PASS, f"a key the host set reached the hook, which refused a non-launcher command " + f"({query_guard.NOT_LAUNCHER}) and passed a launcher lookup, both verified; " + f"{SCRUB_ENV} off", {"guard_version": query_guard.VERSION}) + + +def check_budget(cap, contract: dict, runs: int) -> Check: + if not isinstance(cap, (int, float)) or isinstance(cap, bool) or not math.isfinite(cap) or cap <= 0: + return Check("budget cap", FAIL, "--budget must be a positive number of dollars") + budget = contract.get("evolution", {}).get("budget", {}) + budget = budget if isinstance(budget, dict) else {} + reason = check_prices(budget.get("prices")) if "prices" in budget else "the contract prices no model" + reason = reason or missing_rates(budget["prices"]) + if reason: + return Check("budget cap", FAIL, f"the arms' contract: {reason}: the cost metric needs host-priced usage") + share = budget.get("max_dollars") + if share is None: + return Check("budget cap", FAIL, "the arms' contract sets no budget.max_dollars: the host's meter would not " + "stop a thread") + if not isinstance(share, (int, float)) or isinstance(share, bool) or not math.isfinite(share) or share <= 0: + return Check("budget cap", FAIL, "the arms' contract budget.max_dollars must be a positive number of dollars") + if share > cap: + return Check("budget cap", FAIL, f"the arms' contract max_dollars ${share:.4f} is above the cap ${cap:.2f}") + if budget.get("strict") is not True: + return Check("budget cap", FAIL, "the arms' contract needs budget.strict true: usage of a model its prices do " + "not name must refuse promotion, or cost turns unknown") + return Check("budget cap", PASS, f"${cap:.2f} token-cost dispatch threshold for the canary and {runs} runs; " + "one-call overshoot and server fees are not covered; each thread's contract " + f"max_dollars ${share:.4f}; a run starts only while spent + ${share:.4f} <= " + f"${cap:.2f}; priced: {', '.join(sorted(budget['prices']))}", + {"cap": cap, "thread_budget": share}) + + +def check_contracts(sy: Syberlabs, main: dict, canary: dict) -> Check: + folder = sy.home / "contracts" + folder.mkdir(parents=True, exist_ok=True) + for doc in (main, canary): + (folder / f"{contract_name(doc)}.json").write_text(json.dumps(doc, indent=2) + "\n", encoding="utf-8") + done = sy("init") + if done.code != 0: + return Check("contracts", FAIL, "the run's contracts were not accepted: " + tail(done.err or done.out, 240)) + checks = sorted(main["evolution"]["evaluation"]["checks"]) + return Check("contracts", PASS, f"{contract_name(main)} (checks: {', '.join(checks)}) and {contract_name(canary)} " + f"installed in the run's own home", {"main": contract_name(main), + "main_sha256": digest(main)}) + + +# ----------------------------------------------------------------------------------------------------------- records + +def facts(events: list, key: str) -> list[dict]: + return [v for v in (metrics.fact(e, key) for e in events) if v is not None] + + +def last_search(events: list) -> dict | None: + found = [e["body"] for e in events if e["kind"] == "search_finished"] + return found[-1] if found else None + + +def registrations(events: list) -> list[dict]: + return [e["body"] for e in events + if e["kind"] == "candidate_registered" and e["body"].get("operator") != "migration"] + + +def contracts_of(records: list[dict]) -> dict: + found = {} + for record in records: + doc = record.get("contract") if isinstance(record, dict) else None + if isinstance(doc, dict) and "id" in doc and "version" in doc: + found[(doc["id"], doc["version"])] = doc + return found + + +def unmetered(events: list) -> str | None: + """Why the host's meter missed part of what the threads in ``events`` spent, or None: a call that ended + ``provider_failed``, ``provider_timeout`` or ``provider_error`` (``UNMETERED``: raised after Claude Code ran and + before the host's meter priced the call), or a search that never finished (the process died inside it).""" + threads: dict = {} + for event in events: + threads.setdefault(event.get("case_id"), []).append(event) + for found in threads.values(): + searching = False + for event in sorted(found, key=lambda e: e.get("seq", 0)): + if event["kind"] == "search_started": + searching = True + elif event["kind"] == "search_finished": + searching = False + if event["body"].get("error") in UNMETERED: + return (f"a call ended {event['body']['error']} after Claude Code ran, before the host's meter " + "priced it") + if searching: + return "a search never finished, so the host's meter never settled its calls" + return None + + +def spend_of(events: list, contracts: dict) -> float | None: + """Host-observed dollars over the threads in ``events``: a thread whose calls crossed the meter at a model the + prices name counts its dollars, and one where nothing crossed it counts nothing. A model the prices do not name, + or a call the meter missed (``unmetered``), makes the whole sum unknown.""" + if unmetered(events): + return None + total = 0.0 + for item in metrics.work_items(events, contracts): + if item.cost_basis == "host_observed": + if not item.cost.known: + return None + total += float(item.cost.value) + elif item.cost_basis == "declared": + return None + return round(total, 6) + + +def harness_of(events: list) -> dict: + """What the registrations' signals say about the harness and the lane (host-recorded, never the model's).""" + found: dict = {} + for body in registrations(events): + signal = body.get("signal") if isinstance(body.get("signal"), dict) else {} + harness = signal.get("harness") if isinstance(signal.get("harness"), dict) else {} + for key, value in (("lane", signal.get("lane")), ("cost_basis", signal.get("cost_basis")), + ("version", harness.get("version")), ("installed_version", harness.get("installed_version")), + ("model", harness.get("model")), ("auth_source", harness.get("auth_source"))): + if isinstance(value, str): + found.setdefault(key, set()).add(value) + ledger = signal.get("githits") + if isinstance(ledger, dict): + found.setdefault("githits_route", set()).add("api") + if isinstance(ledger.get("license_policy"), str): # a digest: part of the frozen setup + found.setdefault("license_policy", set()).add(ledger["license_policy"]) + for entry in harness.get("mcp_servers") or []: + if isinstance(entry, str) and entry.startswith("githits:"): + found.setdefault("githits_mcp", set()).add(entry) + return {key: sorted(values) for key, values in found.items()} + + +# ----------------------------------------------------------------------------------------------------------- canary + +def claude_flags(plan: dict, arm: str, max_calls: int, max_turns: int, tools: str | None = None) -> list[str]: + """``propose`` flags: the same in both arms but for ``--githits`` (and its arm name) in the treatment.""" + flags = ["propose", "--claude-code", "--bare", "--max-calls", str(max_calls), "--max-turns", str(max_turns), + "--allowed-tools", tools or plan["allowed_tools"]] + if plan["guard_max_lines"] is not None: + flags += ["--guard-max-lines", str(plan["guard_max_lines"])] + if plan["guard_max_chars"] is not None: + flags += ["--guard-max-chars", str(plan["guard_max_chars"])] + if arm == TREATMENT: + flags += ["--githits", "--githits-arm", TREATMENT] + return flags + + +def judge_canary(events: list, contracts: dict) -> tuple[str, str, dict]: + """PASS when the guard refused at least one lookup for what it says (``CONTENT_RULES``) and passed at least one + that GitHits answered (an authentic client record, ``REACHED``), every decision verified and none ``unguarded``, + and every model the call used is priced; FAIL otherwise, with the reason. A refused launcher call proves that the + content rules fire and that Claude Code obeys the hook: had it run the call anyway, its client record would match + no passed call and every record would be ``unguarded``. A call that shows only one of the two proves too little: + a model that leaves the address out, or a launcher Claude Code will not run, would pass a weaker test and leave + the arms unguarded or without GitHits.""" + lookups = facts(events, claude_code.LOOKUP) + finished = last_search(events) or {} + unguarded = [x for x in lookups if x.get("guard") == query_guard.UNGUARDED] + decided = [x for x in lookups if isinstance(x.get("guard"), str) and x["guard"] != query_guard.UNGUARDED] + refused = [x for x in decided if x["guard"] in CONTENT_RULES] + reached = [x for x in decided if x["guard"] == query_guard.PASS and x.get("status") in REACHED + and str(x.get("tool", "")).startswith(claude_code.CLIENT_TOOL)] + spend = finished.get("spend") if isinstance(finished.get("spend"), dict) else {} + data = {"lookups": [{"tool": x.get("tool"), "guard": x.get("guard"), "ok": x.get("ok")} for x in lookups], + "search": {"stopped": finished.get("stopped"), "error": finished.get("error")}, + "spend": spend_of(events, contracts), "unknown_models": spend.get("unknown_models"), + "harness": harness_of(events)} + if finished.get("error") == "provider_guard_inactive" or unguarded: + return FAIL, (f"the query guard did not fire: {len(unguarded)} lookup(s) ran with no verified decision " + "(unguarded): under this Claude Code and --bare the --settings hook did not run, or its record " + "was not the hook's. Do not run the pilot in this mode"), data + if not refused or not reached: + missing = ([] if refused else ["no lookup the guard refused for what it says (the canary asks for one " + "holding an email address)"]) + \ + ([] if reached else ["no lookup the guard passed that GitHits answered (the launcher did not run, " + "or GitHits did not answer)"]) + why = f"; the call ended {finished['error']}" if finished.get("error") else "" + return FAIL, (f"inconclusive: {' and '.join(missing)}{why}; run again, and read the canary's thread in " + "commands.log if it repeats"), data + if finished.get("error"): + return FAIL, f"the canary's call failed: {finished['error']}", data + if spend.get("unknown_models"): + return FAIL, (f"Claude Code ran {', '.join(spend['unknown_models'])}, which the plan does not price: add it to " + "prices, or cost will be unknown for every thread"), data + if spend.get("host_observed") is not True or data["spend"] is None: + return FAIL, "no usage of the call crossed the host's meter, so cost could not be known", data + rules = ", ".join(sorted({x["guard"] for x in refused})) + return PASS, (f"the guard refused {len(refused)} lookup(s) ({rules}) and passed {len(reached)} that GitHits " + f"answered, every decision verified, 0 unguarded; ${data['spend']:.4f}"), data + + +# ------------------------------------------------------------------------------------------------------------- runs + +def gate(spent: float | None, reserve: float, cap: float) -> str | None: + """Why the next run may not start, or None.""" + if spent is None: + return ("the spend so far is unknown (a model the prices do not name, a call the meter missed, or a thread " + "whose record could not be read), so the cap cannot be enforced") + if spent + reserve > cap + 1e-9: + return (f"${spent:.4f} spent; another run reserves ${reserve:.4f} (a thread's max_dollars), which the " + f"${cap:.2f} cap cannot cover") + return None + + +def thread_record(record: dict | None) -> dict: + """What one exported thread says about its run.""" + if not record: + return {"accepted": False, "candidate": None, "commit": None, "search": None, "base": None, "candidates": []} + status = record.get("status") if isinstance(record.get("status"), dict) else {} + events = record.get("events") or [] + finished = last_search(events) or {} + return {"accepted": status.get("accepted") is not None, "candidate": status.get("accepted"), + "commit": status.get("target") if status.get("accepted") else None, + "search": {"stopped": finished.get("stopped"), "error": finished.get("error")} if finished else None, + "base": status.get("base"), + "candidates": [{"evaluation": c.get("evaluation"), "promotion": c.get("promotion")} + for c in status.get("candidates") or [] if isinstance(c, dict)]} + + +def not_accepted(entry: dict) -> str: + """Why a run's thread has no accepted change, from its record: what its candidates' checks and promotions came to, + the runner's last word on it, and how its search ended.""" + found = entry.get("candidates") or [] + passed = [c for c in found if c.get("evaluation") == "passed"] + if passed: + why = f"a candidate passed the host's checks, and its promotion is {passed[-1].get('promotion')}" + elif found: + why = f"none of its {len(found)} candidate(s) passed the host's checks" + else: + why = "it has no candidate" + if entry.get("runner"): + why += f"; the runner: {entry['runner']}" + error = (entry.get("search") or {}).get("error") + return why + (f"; its search ended {error}" if error else "") + + +def diffstat(diff: str) -> str: + files, plus, minus = 0, 0, 0 + for line in diff.splitlines(): + if line.startswith("diff --git "): + files += 1 + elif line.startswith("+") and not line.startswith("+++"): + plus += 1 + elif line.startswith("-") and not line.startswith("---"): + minus += 1 + return f"{files} file(s), +{plus} -{minus}" + + +def ask_verdict(ask: Ask, prompt: str) -> str | None: + """pass, fail, or None for skip; an empty answer asks again, and the end of input skips.""" + while True: + try: + answer = ask(prompt).strip().lower() + except EOFError: + return None + if answer in ("pass", "p"): + return "pass" + if answer in ("fail", "f"): + return "fail" + if answer in ("skip", "s"): + return None + say(" answer pass, fail or skip") + + +def blind_review(entries: list, sy: Syberlabs, ask: Ask, seed: int, out: Path, reviewer: str, + objectives: dict) -> None: + """Every accepted change, in a shuffled order and named R1, R2, ..., without its arm or thread: the reviewer reads + its objective and diff (written to ``review/R.diff``) and answers pass or fail, recorded with + ``review --record`` on its thread. The mapping is in the receipt, written after the review.""" + accepted = [entry for entry in entries if entry.get("accepted")] + if not accepted: + return + random.Random(f"{seed}:review").shuffle(accepted) + folder = out / "review" + folder.mkdir(exist_ok=True) + say(f"\nBlind review: {len(accepted)} accepted change(s), shuffled, arms hidden. Read each diff and answer pass " + f"or fail (skip leaves it unreviewed). Reviewer: {reviewer}.") + for number, entry in enumerate(accepted, 1): + rid = f"R{number}" + diff = sy("--thread", entry["thread"], "diff", entry["candidate"]).out + path = folder / f"{rid}.diff" + path.write_text(diff, encoding="utf-8") + say(f"\n{rid} of {len(accepted)}: item {entry['item']}: {objectives[entry['item']]}") + say(f" {diffstat(diff)}; the whole diff: {path}") + verdict = ask_verdict(ask, f" {rid} verdict [pass/fail/skip]: ") + entry["review"] = {"id": rid, "verdict": verdict, "reviewer": reviewer if verdict else None} + if verdict: + done = sy("--thread", entry["thread"], "review", "--record", "--verdict", verdict, "--reviewer", reviewer, + "--revision", entry["commit"]) + if done.code != 0: + entry["review"]["error"] = tail(done.err, 200) + + +# ------------------------------------------------------------------------------------------------------- evaluation + +def metric_view(metric) -> dict: + value = metric.value + return {"known": metric.known, "value": round(value, 6) if isinstance(value, float) else value, + "note": metric.note} + + +def evaluate(entries: list, records: list[dict], name: str) -> None: + """Each run's nine metrics as the paired report reads them (``PAIRED`` for agent time and delivered), with the + merge-based forms of those two beside them, from the exported records. A thread with a call the host's meter + missed (``unmetered``) has an unknown cost here, whatever the fold says: part of what it spent was never priced.""" + events = [event for record in records for event in (record.get("events") or [])] + contracts = contracts_of(records) + runs = {run.thread: run for run in experiment.runs(events, name, contracts)} + work = {item.thread: item for item in metrics.work_items(events, contracts)} + for entry in entries: + run = runs.get(entry.get("thread") or "") + if run is None: + why = "not run" if not entry.get("thread") else "no experiment label in the record" + entry["metrics"] = {m: {"known": False, "value": None, "note": why, "source": PAIRED.get(m, m)} + for m in NINE} + entry["strict"] = {} + else: + entry["metrics"] = {m: {**metric_view(run.value(PAIRED.get(m, m))), "source": PAIRED.get(m, m)} + for m in NINE} + item = work[run.thread] + entry["strict"] = {m: metric_view(item.metric(m)) for m in PAIRED} + entry["lookups_blocked"] = metric_view(item.lookups_blocked) + missed = unmetered([event for event in events if event.get("case_id") == run.thread]) + if missed: + entry["metrics"]["cost"] = {"known": False, "value": None, "note": missed, "source": "cost", + "metered": entry["metrics"]["cost"]["value"]} + entry["unknown"] = [m for m in NINE if not entry["metrics"][m]["known"]] + + +# --------------------------------------------------------------------------------------------------------- printing + +def cell(metric: dict, name: str) -> str: + if not metric["known"]: + return "UNKNOWN" + value = metric["value"] + if isinstance(value, bool): + return "yes" if value else "no" + if name == "cost": + return f"${value:.4f}" + if name == "agent_time": + return f"{value:.2f}h" + return f"{value:g}" if isinstance(value, float) else str(value) + + +def table(rows: list[list[str]]) -> list[str]: + if not rows: # an interrupt before the first preflight check: nothing to line up + return [] + widths = [max(len(row[n]) for row in rows) for n in range(len(rows[0]))] + return [" ".join(c.ljust(w) for c, w in zip(row, widths)).rstrip() for row in rows] + + +def render(receipt: dict, out: Path) -> str: + """The PASS/FAIL table: the preflight, the canary, one row per run with its nine metrics, the checks after the + runs, the total spend, the reasons for a FAIL, and where the receipt is.""" + lines = [f"pilot dry run: {receipt['verdict']}"] + lines.append(f" base {str(receipt['commit'].get('base') or '-')[:10]}, seed {receipt['seed']}, Claude Code " + f"{receipt['versions'].get('claude_code') or '-'}, experiment {receipt.get('experiment') or '-'}") + if receipt["preflight"] and all(c["status"] == PASS for c in receipt["preflight"]): + lines.append(f"preflight: {len(receipt['preflight'])} checks PASS (" + + ", ".join(c["name"] for c in receipt["preflight"]) + ")") + else: + lines.append("preflight") + lines += [" " + line for line in table([[c["status"], c["name"], c["detail"]] for c in receipt["preflight"]])] + if receipt.get("canary"): + lines.append("canary") + lines.append(f" {receipt['canary']['status']} {receipt['canary']['detail']}") + if receipt["threads"]: + lines.append("runs (agent time = time to a passing change, not to a merge; delivered = accepted and passed the " + "blind review)") + head = ["#", "item", "arm", "thread"] + [LABELS[m] for m in NINE] + ["verdict"] + rows = [head] + for entry in receipt["threads"]: + rows.append([str(entry["order"]), entry["item"], entry["arm"], (entry.get("thread") or "-")[:8]] + + [cell(entry["metrics"][m], m) for m in NINE] + + [PASS if not entry["unknown"] else FAIL]) + lines += [" " + line for line in table(rows)] + for row in receipt.get("checks", []): + lines.append(f" {row['status']} {row['name']}: {row['detail']}") + spend = receipt["spend"] + total = "unknown" if spend["total"] is None else f"${spend['total']:.4f}" + canary = "" if spend.get("canary") is None else f" (canary ${spend['canary']:.4f})" + cap = "no valid cap" if spend.get("cap") is None else f"${spend['cap']:.2f}" + lines.append(f"spend {total} of {cap}{canary}") + shown = {f"preflight {c['name']}: {c['detail']}" for c in receipt["preflight"] if c["status"] == FAIL} + lines += [f"reason {reason}" for reason in receipt["reasons"] if reason not in shown] # a FAIL row says it + send = " (send it with report.md; the rest stays here)" if (out / "report.md").is_file() else "" + lines.append(f"receipt {out / 'receipt.json'}{send}") + return "\n".join(lines) + + +# ------------------------------------------------------------------------------------------------------------- main + +def default_actor(repo: Path) -> str: + try: + found = git(repo, "config", "user.email") + except (RuntimeError, OSError): + found = "" + return found or "pilot-dry-run" + + +def exclude_from_git(repo: Path, folder: Path) -> None: + """Keep the run's directory out of ``git status`` when it lies inside the repository (``.git/info/exclude``, as + the kit does for its own home).""" + try: + relative = folder.resolve().relative_to(repo.resolve()) + common = Path(git(repo, "rev-parse", "--git-common-dir")) + except (ValueError, RuntimeError, OSError): + return + common = common if common.is_absolute() else repo / common + exclude = common / "info" / "exclude" + line = "/" + relative.as_posix().rstrip("/") + "/" + existing = exclude.read_text(encoding="utf-8").splitlines() if exclude.exists() else [] + if line not in existing: + exclude.parent.mkdir(parents=True, exist_ok=True) + with open(exclude, "a", encoding="utf-8") as handle: + handle.write(("\n" if existing and existing[-1] else "") + line + "\n") + + +def versions(claude: str | None) -> dict: + try: + from importlib.metadata import PackageNotFoundError, version + try: + syberwork = version("syberwork") + except PackageNotFoundError: + syberwork = "source" + except ImportError: # pragma: no cover + syberwork = "source" + try: + git_version = subprocess.run(["git", "--version"], capture_output=True, timeout=30).stdout.decode().strip() + except (OSError, subprocess.TimeoutExpired): + git_version = None + try: + skill = claude_code.githits_skill()[1] + except Rejected: + skill = None + return {"claude_code": claude, "python": platform.python_version(), "platform": platform.platform(), + "syberwork": syberwork, "git": git_version, "query_guard": query_guard.VERSION, + "githits_skill_sha256": skill, "script": RECEIPT} + + +def parse_args(argv): + parser = argparse.ArgumentParser(prog="pilot_dry_run", description="The GitHits pilot's dry run as one command " + "(module docstring: how to run it).") + parser.add_argument("--plan", help="the plan: items, prices and the shared knobs (--template prints one)") + parser.add_argument("--template", action="store_true", help="print a plan to fill in, and exit") + parser.add_argument("--repo", default=".", help="the repository the items change (default: here)") + parser.add_argument("--base", help="the commit every thread starts from; must be HEAD (default: HEAD)") + parser.add_argument("--budget", type=float, default=DEFAULT_BUDGET, + help=f"token-cost scheduling budget, canary included (default {DEFAULT_BUDGET:g}); " + "may overshoot by one call and excludes server-side search fees") + parser.add_argument("--thread-budget", type=float, + help="each thread's contract max_dollars (default: the cap over the runs plus the canary)") + parser.add_argument("--seed", type=int, help="the shuffle's seed (default: drawn, and printed)") + parser.add_argument("--out", help="the run's directory, new or empty (default " + ".syberlabs/pilot-dry-run/