diff --git a/docs/EXPERIMENTS.md b/docs/EXPERIMENTS.md index ff010cc..38f85f7 100644 --- a/docs/EXPERIMENTS.md +++ b/docs/EXPERIMENTS.md @@ -9,10 +9,10 @@ syberlabs experiment report --name githits-pilot --markdown # the table syberlabs experiment report --name githits-pilot --json # every item, its threads, values and checks syberlabs experiment report --name githits-pilot --csv # one row per item syberlabs experiment report --name githits-pilot --from-export receipts/*.json # records from syberlabs export -python examples/githits_ab.py # the whole pipeline, a stand-in instead of a model +python examples/githits_ab.py [--route api|mcp] # the whole pipeline, a stand-in instead of a model ``` -An experiment runs the same work items through the factory more than once and changes one thing between the runs. Each run is a Build Thread; each thread is in one **arm**. The first consumer is the owner-approved GitHits pilot (2026-10-23), whose arms differ only in the Claude Code proposer's `--mcp-config` (`docs/CLAUDE_CODE.md`, "GitHits A/B"). +An experiment runs the same work items through the factory more than once and changes one thing between the runs. Each run is a Build Thread; each thread is in one **arm**. The first consumer is the owner-approved GitHits pilot (2026-10-23), whose arms differ only in whether the Claude Code proposer has GitHits: since SyberWork #134 through the host's API client, `--githits` (`docs/CLAUDE_CODE.md`, "GitHits through the launcher"; `docs/GITHITS.md`, "The A/B report"), and before it through an MCP server, `--mcp-config` (`docs/CLAUDE_CODE.md`, "GitHits A/B through MCP"). The report reads both routes. ## The label @@ -72,16 +72,18 @@ Each item is checked against the record. The first nine take the item out of eve | `unequal_repeats` | excluded | the arms ran the item a different number of times, so a mean of several runs would face one run | | `base_mismatch` | excluded | the arms started from different commits (the `base` observation) | | `objective_mismatch` | excluded | the arms were given different objectives | -| `arm_conflict` | excluded | the record contradicts the label: the server (`--server`, default `githits`) was connected in a baseline call, or not connected in the treatment, as the proposer recorded it in `signal.harness.mcp_servers` | -| `contaminated` | excluded | the baseline made GitHits lookups | +| `arm_conflict` | excluded | the record contradicts the label: GitHits (`--server`, default `githits`) was there in a baseline call, or in none of a treatment thread's calls (below, "How the arm is read"). On the API route: arm B run without `--githits`, or arm A with it or with a GitHits `--mcp-config` | +| `contaminated` | excluded | the baseline shows a GitHits lookup: its GitHits count is above zero, or the host observed one (below) | | `labelled_late` | excluded | a thread's label in force, in any arm, is not the one that stood before its first candidate | | `unregistered` | excluded | `--items` was given and does not list the item | | `registration_mismatch` | excluded | the item's threads carry other tags than `--items` gives it | | `base_unknown` | kept | a thread has no recorded base commit | -| `arm_unverified` | kept | no call recorded its MCP servers, so only the label says which arm ran | -| `server_mixed` | kept | the server was connected in some of a treatment thread's calls and not in others | -| `unused` | kept | the treatment made no GitHits lookups | -| `contamination_unknown` | kept | the baseline did not record its GitHits lookups, so it cannot be shown clean | +| `arm_unverified` | kept | no call recorded its MCP servers or the GitHits launcher, so only the label says which arm ran | +| `server_mixed` | kept | GitHits was there in some of a treatment thread's calls and not in others | +| `two_routes` | kept | a treatment thread had GitHits by both routes: the API launcher and a connected GitHits MCP server, usually a `--mcp-config` left in arm B by mistake. It is still the treatment, but not the single route the pilot registers, and where the thread has client records its GitHits count is those records only, without the MCP server's lookups (`spec/GAPS.md` 59). What to do: take the GitHits server out of arm B's `--mcp-config` before the next item; read this item's GitHits count as a floor, and say in the write-up that it ran with both routes | +| `unused` | kept | the treatment shows no GitHits lookup, and its count is known to be zero | +| `use_unknown` | kept | the treatment shows no GitHits lookup, but its count is unknown, so it can be shown neither to have used GitHits nor not to. The count is unknown when a call declared none, or, where `syberlabs metrics` says so (SyberWork #139), when the record cannot say how many lookups reached GitHits: a call's `signal.githits.mismatch` (a launcher call the guard passed left no client record) or an `unguarded` client record. What to do: `syberlabs metrics --thread --json` gives the reason in the count's note; fix the cause (a client that failed before recording, a proposer that declares no lookups) before the next item, and do not read the item as one where GitHits went unused | +| `contamination_unknown` | kept | the baseline shows no GitHits lookup, but its count is unknown, so it cannot be shown clean | | `relabelled` | kept | a thread's labels disagree, but the one in force stood before its first candidate | | `model_mismatch` | kept | the arms were served by different models | | `cost_basis_mismatch` | kept, cost not compared | the arms' cost was recorded on different bases or credential lanes | @@ -93,7 +95,18 @@ Each item is checked against the record. The first nine take the item out of eve Two lines are about the experiment as a whole. `not_run` names the registered items that never ran. `forgotten` names the forget tombstones written since the experiment's first thread started, because a forgotten thread leaves nothing else behind; under the default retention policy a thread with an accepted change is kept for 365 days, so a tombstone here is usually a run that was not accepted and was removed. Exported records carry no tombstones, so a report `--from-export` says it cannot see forgotten threads. `--from-export` also refuses a record whose hash chain does not verify (`export_unverified`): an edited export is not counted. -The Claude Code proposer leaves `mcp_servers` out of `signal.harness` when its session started with none, so a call whose harness recorded its session's start (a `version` or `model`) and lists no servers had none connected; a call whose harness recorded nothing of its start is not read. `--server none` turns off the server checks for an experiment that does not switch an MCP server. `contaminated`, `contamination_unknown` and `unused` read GitHits lookups and apply only with `--server githits`. +### How the arm is read + +The report does not take the label's word for the arm. It reads, call by call, the host's record of whether the call had GitHits, by either route: + +- **The API route** (`--githits`, the pilot's since SyberWork #134): the candidate carries `signal.githits`, the ledger the Claude Code proposer writes on the candidate of every call it gave the host's GitHits launcher, and of no other. It holds the host's own count of the client records it verified under the call's key (`records`), and it is there whether or not the model used the launcher, so a treatment that looked nothing up is `unused`, not in conflict. What the model drove (`signal.tool_usage`, `signal.githits.launcher_calls`), the skill text it was shown (`signal.harness.skill`) and the operator's name for the arm (`signal.githits.arm`, from `--githits-arm`) are not read for the arm. +- **The MCP route** (`--mcp-config`, the pilot's before #134): `signal.harness.mcp_servers` lists the server `connected`, as Claude Code's `init` event said. The proposer leaves `mcp_servers` out when its session started with none, so a call whose harness recorded its session's start (a `version` or `model`) and lists no servers had none connected; a call that recorded neither the ledger nor anything of its start is not read. + +A thread whose calls all had GitHits is the treatment's; one where none did is the baseline's; anything else is `server_mixed` in the treatment and `arm_conflict` in the baseline. A treatment thread with both routes is `two_routes`. + +A GitHits lookup is shown by the thread's GitHits count (`syberlabs metrics`' `githits_lookups`; on the API route, the number of the host's `lookup` facts from authentic client records) above zero, or by a `lookup` fact the host observed and can vouch for: a GitHits MCP call (`mcp__githits__*`), or a client record that a guarded launcher call stands behind (`githits.`, guard `pass`; a `githits.doctor` readiness check is not a lookup). That includes the lookups of a call that failed before its candidate, which no registration counts. An unknown count is never read as zero: it shows no baseline clean (`contamination_unknown`) and no treatment unused (`use_unknown`). + +`--server none` turns these checks off for an experiment that does not switch GitHits; another `--server` name reads that MCP server only, never the launcher. `contaminated`, `contamination_unknown`, `unused`, `use_unknown` and `two_routes` read GitHits lookups and routes, and apply only with `--server githits`. ## Statistics, stated plainly @@ -123,24 +136,26 @@ The pilot itself runs a real model with a paid key: Class X (`AGENTS.md`). A nam 1. The item list, as the `--items` file: each work item's id and its tag, `needs_lookup` when the work depends on a library or API the agent is unlikely to know well (new, niche, or changed since the model's training), else `routine`; beside it, each item's objective and scope. 2. The primary metric and the group the decision rests on (for example: the delivery rule on `needs_lookup` items, with `--delivered review`). 3. The exclusion rules: the nine excluded checks above, and nothing else. -4. The frozen setup: the SyberWork commit, Claude Code version, model, effort, `--allowed-tools`, `--max-turns`, the query guard's thresholds, and the credential lane. Use the `api` lane (`--bare`) in both arms: it is the lane whose cost is billed and known, and in bare mode the tree's `CLAUDE.md` is not read in either arm. The contract prices every model the runs report (`evolution.budget.prices`); a run that used a model the contract does not price has an unknown cost. Anything that learns from runs stays off for the week. AutoSkill is a design (`docs/AUTOSKILL.md`) and is in no proposer path; the `jev` and `empirical` branch policies belong to `--evolve`, which the pilot does not use. +4. The frozen setup: the SyberWork commit, Claude Code version, model, effort, `--allowed-tools`, `--max-turns`, the query guard's thresholds, the credential lane, and arm B's GitHits route (`--githits`) with its license policy (`SYBERLABS_GITHITS_LICENSES`, recorded as `signal.githits.license_policy`). Use the `api` lane (`--bare`) in both arms: it is the lane whose cost is billed and known, and in bare mode the tree's `CLAUDE.md` is not read in either arm. The contract prices every model the runs report (`evolution.budget.prices`); a run that used a model the contract does not price has an unknown cost. Anything that learns from runs stays off for the week. AutoSkill is a design (`docs/AUTOSKILL.md`) and is in no proposer path; the `jev` and `empirical` branch policies belong to `--evolve`, which the pilot does not use. **Each item, each arm, from the same `HEAD`** (accepting moves only the thread's own branch, so `HEAD` stays put): ```bash syberlabs start "Parse ISO dates with the UTC offset" --paths src tests --unattended syberlabs experiment label --name githits-pilot --item LIB-1 --arm off --tag needs_lookup -syberlabs propose --claude-code --bare --allowed-tools Read,Edit,Write,Glob,Grep,WebSearch,WebFetch,mcp__githits +syberlabs propose --claude-code --bare --allowed-tools Read,Edit,Write,Glob,Grep,WebSearch,WebFetch syberlabs run --once # one safe step per thread; here, accepting the passing candidate syberlabs start "Parse ISO dates with the UTC offset" --paths src tests --unattended syberlabs experiment label --name githits-pilot --item LIB-1 --arm on --tag needs_lookup -syberlabs propose --claude-code --bare --allowed-tools Read,Edit,Write,Glob,Grep,WebSearch,WebFetch,mcp__githits \ - --mcp-config githits.json # the only difference between the arms +syberlabs propose --claude-code --bare --allowed-tools Read,Edit,Write,Glob,Grep,WebSearch,WebFetch \ + --githits --githits-arm on # the only difference between the arms: the launcher and the skill syberlabs run --once ``` -`--claude-code` is the proposer of SyberWork #80 and the query guard of #82; this report works with any proposer, and needs them for the server check and for the lookups to be counted (`arm_unverified` and unknown lookups otherwise). Alternate which arm runs first from item to item, so drift in the model or the network during the week does not fall on one arm. Accept on the host's checks (`--unattended`), or have a person accept without being told the arm: the label is in the journal, not in the diff or the pull request. Then have a person who is not told the arm review **both** arms' accepted commits, and record each verdict on its thread: +Arm B needs `GITHITS_API_TOKEN` in the environment (the proposer refuses `--githits` without it, and never writes it down). Arm A drops every GitHits variable before Claude Code starts (#135), so both arms can run from the same shell. Pass no GitHits `--mcp-config` in either arm: in arm A it is `arm_conflict`, in arm B `two_routes`. Before #134 the pilot's route was an MCP server: `--mcp-config githits.json` in arm B, with `mcp__githits` in both arms' `--allowed-tools`. The report still reads an experiment run that way. + +`--claude-code` is the proposer of SyberWork #80, with the query guard of #82 and the GitHits launcher of #134. This report works with any proposer, but needs those to read the arm and count the lookups (`arm_unverified` and unknown lookups otherwise). Alternate which arm runs first from item to item, so drift in the model or the network during the week does not fall on one arm. Accept on the host's checks (`--unattended`), or have a person accept without being told the arm: the label is in the journal, not in the diff or the pull request. Then have a person who is not told the arm review **both** arms' accepted commits, and record each verdict on its thread: ```bash syberlabs review --record --verdict pass --reviewer blind-1 --revision 3f9c2ab @@ -163,6 +178,6 @@ What leaves the factory is the report and the CSV. The journal, the prompts and - The report compares what the record holds. A lookup the proposer did not declare, a model call outside the host, or a person's message to an agent outside the CLI is unknown or missing to it, as it is to `syberlabs metrics` (`docs/METRICS.md`, section 4). - The checks catch a label settled after the work began, an item outside the registered list, an unlabelled run of the same objective from the same commit, and a forgotten thread. They cannot catch a run made in another home, or one whose objective was reworded. -- The server check reads what the proposer recorded about its own MCP servers. It catches a GitHits server that failed to connect; it cannot catch a proposer that misreports. +- The arm check reads what the proposer recorded: the GitHits launcher's ledger and its MCP servers. It catches a treatment run without `--githits` (or whose GitHits server failed to connect), and a baseline given GitHits by either route. It cannot catch a proposer that misreports: a search-protocol adapter writes its own `signal`, so it could declare a ledger or a server it never had. - One week of items is a small sample. The report says how small, metric by metric; it does not turn a small sample into a finding. -- `examples/githits_ab.py` runs every step with a stand-in instead of a model. Its numbers are invented to exercise the report and say nothing about GitHits. +- `examples/githits_ab.py` runs every step with a stand-in instead of a model. On its default route (`--route api`), that is the real Claude Code proposer with a stand-in `claude` and a loopback GitHits; `--route mcp` uses a search-protocol stand-in that declares its MCP servers. Its numbers are invented to exercise the report and say nothing about GitHits. diff --git a/docs/GITHITS.md b/docs/GITHITS.md index 01a839f..a90f862 100644 --- a/docs/GITHITS.md +++ b/docs/GITHITS.md @@ -217,6 +217,77 @@ print(hashlib.sha256(fixture_bytes(json.load(open(sys.argv[1], encoding='utf-8') The same digest means the five operations are unchanged. +## The A/B report + +The pilot runs each work item through the same Claude Code proposer twice and compares the arms with +`syberlabs experiment` (`docs/EXPERIMENTS.md`): arm A without GitHits, arm B with the launcher. Label each thread +before its first candidate and keep everything else equal, `--allowed-tools` included: + +```bash +syberlabs start "Parse ISO dates with the UTC offset" --paths src tests --unattended +syberlabs experiment label --name githits-pilot --item LIB-1 --arm off --tag needs_lookup +syberlabs propose --claude-code --bare --allowed-tools Read,Edit,Write,Glob,Grep,WebSearch,WebFetch +syberlabs run --once + +syberlabs start "Parse ISO dates with the UTC offset" --paths src tests --unattended +syberlabs experiment label --name githits-pilot --item LIB-1 --arm on --tag needs_lookup +syberlabs propose --claude-code --bare --allowed-tools Read,Edit,Write,Glob,Grep,WebSearch,WebFetch \ + --githits --githits-arm on # the only difference: the launcher and the skill +syberlabs run --once + +syberlabs experiment report --name githits-pilot --items items.txt --delivered review +``` + +The report does not take the label's word for the arm; it reads the host's record of each call. On this route a +call had GitHits when its candidate carries `signal.githits`, the ledger `ClaudeCodeProvider` writes on the +candidate of every call it gave the launcher, and of no other: `{arm, launcher_calls, launcher_passed, records, +records_ignored, bash_refused, license_policy[, mismatch]}`, where `records` is the host's own count of the client +records whose MAC verified under the call's key. That is the strongest host-owned evidence the provider records +for a call: the host writes it only after writing that call's launcher and record directory and reading them back, +and its presence does not depend on anything the model did, so a treatment whose model never touched the launcher +still carries it and is `unused`, not in conflict. What the model drove (`signal.tool_usage`, `launcher_calls`), +the skill text it was shown (`signal.harness.skill`) and the operator's name for the arm (`signal.githits.arm`, +from `--githits-arm`) are not read for the arm. GitHits use is read from the host's `lookup` facts, one per +authentic client record (tool `githits.`), which `syberlabs metrics` counts as `githits_lookups`; a +`githits.doctor` record is a readiness check, not a lookup, and counts for neither. + +| Check | Effect | On this route | +|---|---|---| +| `arm_conflict` | excluded | a baseline call carries the ledger (`--githits` was passed in arm A), or a treatment thread's calls carry none (it was forgotten) | +| `contaminated` | excluded | the baseline shows a GitHits lookup: its `githits_lookups` is above zero, or its thread holds a verified `lookup` fact the host observed, an MCP call (`mcp__githits__*`) or a client record a guarded launcher call stands behind (`githits.` but `doctor`, guard `pass`), a call that failed before its candidate included | +| `contamination_unknown` | kept | the baseline shows no lookup but its `githits_lookups` is unknown, so it cannot be shown clean | +| `server_mixed` | kept | some of a treatment thread's calls carry the ledger and some do not | +| `two_routes` | kept | a treatment thread had GitHits by both routes, the ledger and a connected GitHits MCP server (one left in arm B by mistake): still the treatment, but not one route alone, and where it has client records `githits_lookups` counts only those, leaving the MCP lookups out (`spec/GAPS.md` 59) | +| `unused` | kept | the treatment shows no GitHits lookup and its count is known to be zero (a launcher call the guard refused never reached GitHits) | +| `use_unknown` | kept | the treatment shows no GitHits lookup but its count is unknown, so it cannot be shown to have used GitHits (nor not to) | +| `arm_unverified` | kept | no call recorded its MCP servers or the ledger | + +An unknown `githits_lookups` is never read as zero. It is unknown when a call declared no count, and, with the +metrics of SyberWork #139, when the record cannot say how many lookups reached GitHits: a call's +`signal.githits.mismatch` (a launcher call the guard passed left no record) or an `unguarded` client record (no +guarded launcher call behind it). The report then reads only what the host observed: a lookup fact it can vouch for +makes the baseline `contaminated` and shows the treatment used; without one the baseline is `contamination_unknown` +and the treatment `use_unknown`, and the item's paired GitHits count is unknown, counted beside the comparison and +never imputed. The arm itself is read from the ledger's presence, not from any count, so an unknown count never moves +an item between arms or into `arm_conflict`. + +The MCP route is read as before (`signal.harness.mcp_servers` listing `githits:connected`), and a call counts as +having GitHits by either route: a stray GitHits MCP server in arm A is `arm_conflict`, one beside the launcher in +arm B is `two_routes`. Before the report read the ledger, every arm-B item of an API-route pilot was +excluded as `arm_conflict` (the server "not connected in the treatment"), so nothing was compared and the delivery +rule could not be read. `--server none` still turns these checks off; another `--server` name reads that MCP server +only, never the launcher. + +`python examples/githits_ab.py` rehearses the whole pipeline on this route (its default, `--route api`; `--route +mcp` is the route before it): seven items through both arms of the real provider with a stand-in `claude` and a +loopback GitHits, item `LIB-4`'s treatment run without `--githits` and excluded. `tests/test_experiment_githits_api.py` +runs both arms end to end through the provider, the query guard, the launcher and the client, labels and reports +them with the CLI, and prints the paired result and the rule's verdict. + +The report trusts the proposer's record. A search-protocol adapter writes its own `signal`, so it could declare a +ledger it never had, as it could declare an MCP server; the check catches a misconfigured run, not a proposer that +misreports. Claim ceiling: implemented and tested with stand-ins; no pilot has run on it. + ## Environment | Variable | Meaning | diff --git a/examples/githits_ab.py b/examples/githits_ab.py index 5db7904..87f8218 100644 --- a/examples/githits_ab.py +++ b/examples/githits_ab.py @@ -2,44 +2,63 @@ Seven work items run through both arms of the factory's default proposer path (the repair loop) on a throwaway repository: real Build Threads, real host checks, real acceptances, a real journal, then ``syberlabs metrics`` and -``syberlabs experiment report`` over it. Only the model is a stand-in, a command that answers the search protocol: +``syberlabs experiment report`` over it. Two routes for arm B's GitHits: + +- ``--route api`` (the default; the pilot's route since #134): both arms run the real Claude Code proposer, + ``ClaudeCodeProvider`` on the api lane, arm B with ``githits=True``. For each call the host writes the GitHits + launcher, its query guard judges every lookup and command, the host's own GitHits client makes each lookup and signs + its record, and the host journals one ``lookup`` per record and its ledger of the call on the candidate + (``signal.githits``), which is what the report reads the arm from. Only the ``claude`` executable is a stand-in: a + script that prints Claude Code's ``stream-json``, runs the ``--settings`` hook before each tool call as Claude Code + does, and runs the launcher when its Bash tool is allowed to; GitHits is a loopback stand-in that answers searches. + Nothing leaves the machine. +- ``--route mcp`` (the route before #134): the stand-in is a search-protocol command (``RepairProvider``) that + declares its usage, lookups and connected MCP servers itself. + +On either route: - on an item that depends on a library's behaviour (tag ``needs_lookup``) the stand-in guesses wrong once when GitHits is off, so the host's checks fail and the repair call fixes it; with GitHits on it gets it right first time; -- on a routine item it is right first time in both arms; -- on item ``LIB-4`` the treatment's GitHits server "fails to connect", as a misconfigured run would, so the report must - refuse to count that item; +- on a routine item it is right first time in both arms (with GitHits on it looks one thing up and has nothing to add); +- on item ``LIB-4`` the treatment is misconfigured, so the report must refuse to count that item: on the API route it + runs without ``--githits`` (no launcher), on the MCP route its GitHits server "fails to connect"; - every accepted change gets a blind review (``syberlabs review --record``), and one is turned down, so delivery is "accepted and passed review", as the pilot defines it. -The stand-in declares the usage, lookups and connected MCP servers a real proposer reports (``docs/CLAUDE_CODE.md``): -those numbers are made up to exercise the report. **They are not results and say nothing about GitHits.** The pilot -itself runs a real model with a paid key (Class X: a named person runs it; ``docs/EXPERIMENTS.md`` has the commands). +The stand-ins' token usage and lookups are made up to exercise the report. **They are not results and say nothing +about GitHits.** The pilot itself runs a real model with a paid key (Class X: a named person runs it; +``docs/EXPERIMENTS.md`` and ``docs/GITHITS.md`` have the commands). - python examples/githits_ab.py [--out DIR] # DIR receives report.json, report.md and the metrics CSV + python examples/githits_ab.py [--route api|mcp] [--out DIR] # DIR receives report.json, report.md, metrics.csv """ from __future__ import annotations import argparse +import contextlib import json import os import subprocess import sys import tempfile import textwrap +import threading +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer from pathlib import Path sys.path.insert(0, str(Path(__file__).resolve().parents[1])) -from syberlabs import Kit, experiment, metrics # noqa: E402 +from syberlabs import Kit, experiment, githits, metrics # noqa: E402 from syberlabs.build import default_contract # noqa: E402 +from syberlabs.claude_code import THIRD_PARTY_SWITCHES, ClaudeCodeProvider # noqa: E402 from syberlabs.providers import RepairProvider # noqa: E402 NAME = "githits-rehearsal" +ROUTES = ("api", "mcp") RUNNER = "runner@example.test" REVIEWER = "reviewer@example.test" PRICES = {"stand-in-model": {"input_per_million": 3, "output_per_million": 15}} # a made-up price for a made-up model +MISCONFIGURED = "LIB-4" # the item whose treatment ran without GitHits def module(name: str, body: str, test: str) -> dict: @@ -173,9 +192,9 @@ def test_clean(self): '''), None), } -# The stand-in "model". It reads the request on stdin and the arm from STAND_IN_ARM, and prints one candidate with -# the usage, lookups and MCP servers a real proposer would report. Every number in it is invented. -STAND_IN = textwrap.dedent('''\ +# The MCP route's stand-in "model". It reads the request on stdin and the arm from STAND_IN_ARM, and prints one +# candidate with the usage, lookups and MCP servers a real proposer would report. Every number in it is invented. +MCP_STAND_IN = textwrap.dedent('''\ import json, os, sys items = json.loads(os.environ["STAND_IN_ITEMS"]) request = json.load(sys.stdin) @@ -199,6 +218,169 @@ def test_clean(self): "tool_usage": {"githits": githits, "web_search": web, "web_fetch": 0}})) ''') +# The API route's stand-in for the ``claude`` executable: what the host reads from Claude Code, and no more. It answers +# ``--version``, reads the prompt on stdin, and acts as a model that can only call the tools it was given: with the +# launcher (``--allowedTools`` names ``Bash(:*)``) it looks the library up through it, without it it searches +# the web. Before each tool call it runs the ``--settings`` hook (the host's query guard) as Claude Code's PreToolUse +# contract says, and its Bash tool runs only the launcher, as the launcher's ``exec`` line says (no shell needed). It +# writes its change in the working tree and prints ``stream-json``. Its token counts are the MCP stand-in's, invented. +CLAUDE_STAND_IN = textwrap.dedent('''\ + import json, os, re, shlex, subprocess, sys + from pathlib import Path + + if "--version" in sys.argv: + print("2.1.309 (Claude Code)") + sys.exit(0) + items = json.loads(Path(SPEC).read_text(encoding="utf-8")) + prompt = sys.stdin.read() + objective = prompt.split("\\n", 1)[0][len("Objective: "):] + item = next(key for key, value in items.items() if value["objective"] == objective) + spec = items[item] + settings = json.loads(Path(sys.argv[sys.argv.index("--settings") + 1]).read_text(encoding="utf-8")) + allowed = sys.argv[sys.argv.index("--allowedTools") + 1].split(",") + launchers = [rule[5:-3] for rule in allowed if rule.startswith("Bash(") and rule.endswith(":*)")] + launcher = launchers[0] if launchers else None + lookup, repair = spec["tag"] == "needs_lookup", "did not pass the host's checks" in prompt + topic = objective.lower().replace(",", "") + if launcher and not repair: + uses = [("Bash", {"command": f'{launcher} search "{topic} {n}" --target github:python/cpython', + "description": "look up a public API"}) for n in range(1, (2 if lookup else 1) + 1)] + else: + uses = [("WebSearch", {"query": f"python {topic} {n}"}) + for n in range(1, (1 if repair else 3 if lookup else 0) + 1)] + wrong = lookup and spec["wrong"] is not None and launcher is None and not repair + githits = sum(1 for name, _ in uses if name == "Bash") + web = len(uses) - githits + tokens = 1800 + 900 * githits + 600 * web + (1200 if repair else 0) + + def say(event): + print(json.dumps(event), flush=True) + + def guard(name, tool_input, use_id): + """Each PreToolUse hook whose matcher matches the tool, the call on stdin; it blocks on a non-zero exit (the + host's hook sets onFailure block, so a crash blocks too), and its standard error is the reason.""" + for group in settings.get("hooks", {}).get("PreToolUse", []): + if re.search(group.get("matcher", ""), name): + for hook in group["hooks"]: + call = {"session_id": "rehearsal", "transcript_path": "", "cwd": os.getcwd(), + "permission_mode": "dontAsk", "hook_event_name": "PreToolUse", "tool_name": name, + "tool_input": tool_input, "tool_use_id": use_id} + done = subprocess.run([hook["command"], *hook.get("args", [])], capture_output=True, + input=json.dumps(call).encode("utf-8"), timeout=hook.get("timeout", 60)) + if done.returncode != 0: + return done.stderr.decode("utf-8", "replace") or "blocked" + return None + + def run(name, tool_input): + """The tool itself: Bash runs the launcher, as its exec line says, and nothing else; a search answers.""" + if name != "Bash": + return False, "results for " + tool_input["query"] + command = tool_input["command"] + if launcher is None or not (command == launcher or command.startswith(launcher + " ")): + return True, "Permission to use Bash with command denied" + words = shlex.split(command) + script = Path(words[0]).read_text(encoding="utf-8").splitlines() + line = next(text for text in script if text.startswith("exec ")) + argv = [word for word in shlex.split(line)[1:] if word != "$@"] + words[1:] + done = subprocess.run(argv, capture_output=True, timeout=120) + return done.returncode != 0, (done.stdout + done.stderr).decode("utf-8", "replace") + + for path, text in (spec["wrong"] if wrong else spec["right"]).items(): + Path(path).parent.mkdir(parents=True, exist_ok=True) + Path(path).write_bytes(text.encode("utf-8")) # the exact text: no newline translation on Windows + model, usage = "stand-in-model", {"input_tokens": tokens, "output_tokens": 350} + say({"type": "system", "subtype": "init", "claude_code_version": "2.1.309", "model": model, "effort": "high", + "apiKeySource": "ANTHROPIC_API_KEY" if "--bare" in sys.argv else "none", "mcp_servers": [], + "session_id": "rehearsal", "tools": ["Read", "Edit", "Write"] + (["Bash"] if launcher else [])}) + say({"type": "assistant", "message": {"id": "msg_1", "model": model, "usage": usage, + "content": [{"type": "text", "text": "Working on " + item}]}}) + results = [] + for n, (name, tool_input) in enumerate(uses, 1): + use_id = f"toolu_{n}" + say({"type": "assistant", "message": {"id": "msg_1", "model": model, "usage": usage, "content": [ + {"type": "tool_use", "id": use_id, "name": name, "input": tool_input}]}}) + reason = guard(name, tool_input, use_id) + failed, text = (True, reason) if reason is not None else run(name, tool_input) + results.append({"type": "tool_result", "tool_use_id": use_id, "content": text, "is_error": failed}) + if results: + say({"type": "user", "message": {"role": "user", "content": results}}) + say({"type": "assistant", "message": {"id": "msg_2", "model": model, "content": [ + {"type": "text", "text": "Done."}], "usage": {"input_tokens": 0, "output_tokens": 0}}}) + say({"type": "result", "subtype": "success", "is_error": False, "num_turns": 2 if uses else 1, "result": "done", + "total_cost_usd": 0.0, "session_id": "rehearsal", "usage": {}, "modelUsage": {}}) + ''') + +# What the loopback GitHits answers every search with: one public hit (valid under the pinned contract's shapes). +SEARCH_ANSWER = { + "completed": True, "expires_at": None, "query_warnings": [], "search_ref": "sr_rehearsal", + "selected_fields": ["results"], "status": "complete", "targets": None, + "result": {"evidence_notice": None, "partial_results": False, "query": "q", "query_warnings": [], + "page": {"has_more": False, "limit": 5, "offset": 0, "returned": 1}, "source_status": [], + "sources": ["code"], "results": [{ + "coverage": None, "fresh_target_label": None, "freshness": None, "id": "hit-1", + "locator": {"commit_sha": None, "docs_read_target": None, "end_line": 20, "evidence_range": None, + "file_path": "Lib/example.py", "git_ref": None, "match_line": None, + "package_name": None, "page_id": None, "registry": None, + "repo_url": "https://github.com/python/cpython", "repository_file_path": None, + "requested_ref": None, "source_kind": None, "source_url": None, "start_line": 10, + "symbol_ref": None, "version": None}, + "requested_target_label": None, "result_type": "code", "score": 0.875, + "served_target_label": None, "summary": "How the standard library does it.", + "target_label": "github:python/cpython", "title": "example"}]}} + + +class LoopbackGitHits: + """A stand-in GitHits on 127.0.0.1 (the client accepts ``http://`` for loopback only): every search gets + ``SEARCH_ANSWER``, anything else a 404 problem. It counts the requests it received and sends nothing on.""" + + def __init__(self): + self.requests = 0 + outer = self + + class Handler(BaseHTTPRequestHandler): + def log_message(self, *args): + pass + + def _serve(self): + self.rfile.read(int(self.headers.get("Content-Length") or 0)) + outer.requests += 1 + found = self.command == "POST" and self.path.split("?", 1)[0] == "/v1/searches" + body = json.dumps(SEARCH_ANSWER if found else {"code": "NOT_FOUND", "status": 404}).encode("utf-8") + self.send_response(200 if found else 404) + self.send_header("Content-Type", "application/json" if found else "application/problem+json") + self.send_header("X-Request-ID", f"rehearsal-{outer.requests:04d}") + self.send_header("Content-Length", str(len(body))) + self.end_headers() + self.wfile.write(body) + + do_GET = do_POST = _serve + + self.server = ThreadingHTTPServer(("127.0.0.1", 0), Handler) + self.server.daemon_threads = True + self.url = f"http://127.0.0.1:{self.server.server_address[1]}/v1" + threading.Thread(target=self.server.serve_forever, kwargs={"poll_interval": 0.05}, daemon=True).start() + + def close(self) -> None: + self.server.shutdown() + self.server.server_close() + + +@contextlib.contextmanager +def api_environment(url: str): + """The API route's environment, put back afterwards: GitHits at the loopback stand-in under a placeholder token, no + GitHits setting from the shell, and no gateway or provider switch (``claude_code.THIRD_PARTY_SWITCHES`` would refuse + the api lane; the stand-in sends nothing anywhere).""" + saved = dict(os.environ) + for name in (*THIRD_PARTY_SWITCHES, githits.EXCLUDE_ENV, githits.EXCLUDE_REPLACE_ENV, githits.LICENSES_ENV, + githits.SESSION_ENV, githits.AGENT_ENV, githits.RECORD_DIR_ENV, githits.RECORD_KEY_ENV): + os.environ.pop(name, None) + os.environ.update({githits.TOKEN_ENV: "rehearsal-placeholder-not-a-token", githits.URL_ENV: url}) + try: + yield + finally: + os.environ.clear() + os.environ.update(saved) + def git(root: Path, *args: str) -> None: env = {**os.environ, "GIT_AUTHOR_NAME": "dev", "GIT_AUTHOR_EMAIL": "dev@example.test", @@ -206,7 +388,31 @@ def git(root: Path, *args: str) -> None: subprocess.run(["git", "-C", str(root), *args], check=True, capture_output=True, env=env) -def rehearse(folder: Path) -> tuple[list[dict], dict]: +def claude_code(kit: Kit, thread, script: Path, item: str, arm: str) -> ClaudeCodeProvider: + """The API route's proposer, as the pilot builds it: ``syberlabs propose --claude-code --bare`` in arm A, the same + with ``--githits --githits-arm on`` in arm B, except on the misconfigured item, whose treatment forgot + ``--githits``. The executable is the stand-in; everything else is the real provider.""" + return ClaudeCodeProvider(kit.repo, bare=True, executable=[sys.executable, str(script)], max_calls=3, + githits=arm == "on" and item != MISCONFIGURED, arm=arm, thread_label=thread.id) + + +@contextlib.contextmanager +def mcp_environment(arm: str, spec: dict): + """The MCP route's stand-in reads its arm and the items from the environment; put back afterwards.""" + saved = dict(os.environ) + os.environ.update({"STAND_IN_ARM": arm, "STAND_IN_ITEMS": json.dumps(spec)}) + try: + yield + finally: + os.environ.clear() + os.environ.update(saved) + + +def rehearse(folder: Path, route: str = "api", log=print) -> tuple[list[dict], dict]: + """Every item through both arms on ``route`` (``api`` or ``mcp``), in a new repository under ``folder``: the + journal's events and the contracts, for ``syberlabs metrics`` and ``syberlabs experiment report``.""" + if route not in ROUTES: + raise ValueError(f"route must be one of {', '.join(ROUTES)}") root = folder / "shop" (root / "src").mkdir(parents=True) (root / "tests").mkdir() @@ -222,43 +428,62 @@ def rehearse(folder: Path) -> tuple[list[dict], dict]: home = root / ".syberlabs" (home / "contracts").mkdir(parents=True) (home / "contracts" / f"{contract['id']}.v{contract['version']}.json").write_text(json.dumps(contract)) - script = folder / "stand_in.py" - script.write_text(STAND_IN) spec = {item: {"objective": objective, "tag": tag, "right": right, "wrong": wrong} for item, (objective, tag, right, wrong) in ITEMS.items()} + if route == "api": + (folder / "items.json").write_text(json.dumps(spec), encoding="utf-8") + script = folder / "claude_stand_in.py" + script.write_text(f"SPEC = {str(folder / 'items.json')!r}\n" + CLAUDE_STAND_IN, encoding="utf-8") + else: + script = folder / "stand_in.py" + script.write_text(MCP_STAND_IN) + fake = LoopbackGitHits() if route == "api" else None kit = Kit.local(home, root, actor=RUNNER) try: - for item, (objective, tag, _right, _wrong) in ITEMS.items(): - for arm in ("off", "on"): - thread = kit.start(objective, f"{contract['id']}.v{contract['version']}", paths=["src/", "tests/"], - unattended=True, actor=RUNNER) - thread.record_experiment(NAME, item=item, arm=arm, by=RUNNER, tags=[tag]) # before any candidate - os.environ.update({"STAND_IN_ARM": arm, "STAND_IN_ITEMS": json.dumps(spec)}) - found = thread.propose(RepairProvider([sys.executable, str(script)], name="stand-in", max_calls=3), - seed=1) - receipt = thread.accept(found[-1].id, actor=RUNNER) if found else None - state = receipt.status if receipt else "no candidate" - if receipt and receipt.status == "succeeded": - # A reviewer who is not told the arm reads the accepted change (recorded as syberlabs review - # --record does). The stand-in reviewer turns down one change, so the rule has a failure to count. - verdict = "fail" if (item, arm) == ("LIB-2", "off") else "pass" - thread.record_review(verdict, reviewer="blind-reviewer", revision=receipt.commit, by=REVIEWER) - state += f", review {verdict}" - print(f"{item:10} {arm:3} {len(found)} candidate(s), acceptance {state}") + with api_environment(fake.url) if fake else contextlib.nullcontext(): + for item, (objective, tag, _right, _wrong) in ITEMS.items(): + for arm in ("off", "on"): + thread = kit.start(objective, f"{contract['id']}.v{contract['version']}", paths=["src/", "tests/"], + unattended=True, actor=RUNNER) + thread.record_experiment(NAME, item=item, arm=arm, by=RUNNER, tags=[tag]) # before any candidate + if route == "api": + found = thread.propose(claude_code(kit, thread, script, item, arm), seed=1) + else: + with mcp_environment(arm, spec): + found = thread.propose(RepairProvider([sys.executable, str(script)], name="stand-in", + max_calls=3), seed=1) + receipt = thread.accept(found[-1].id, actor=RUNNER) if found else None + state = receipt.status if receipt else "no candidate" + if receipt and receipt.status == "succeeded": + # A reviewer who is not told the arm reads the accepted change (recorded as syberlabs review + # --record does). The stand-in reviewer turns down one change, so the rule has a failure to + # count. + verdict = "fail" if (item, arm) == ("LIB-2", "off") else "pass" + thread.record_review(verdict, reviewer="blind-reviewer", revision=receipt.commit, by=REVIEWER) + state += f", review {verdict}" + log(f"{item:10} {arm:3} {len(found)} candidate(s), acceptance {state}") + if fake is not None: + log(f"The loopback GitHits received {fake.requests} requests; the real API received none.") events = [e for case in kit.journal.thread_ids() for e in kit.session.history(case)] return events, dict(kit.session.contracts) finally: kit.close() + if fake is not None: + fake.close() def main() -> int: parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + parser.add_argument("--route", choices=ROUTES, default="api", + help="arm B's GitHits: the host's API launcher, the pilot's route (api, default), or an MCP " + "server, the route before #134 (mcp)") parser.add_argument("--out", help="write report.json, report.md and metrics.csv here") args = parser.parse_args() - print("SIMULATED: a stand-in answers instead of a model. These numbers check the pipeline; they are not results.\n") + print("SIMULATED: a stand-in answers instead of a model. These numbers check the pipeline; they are not results.") + print(f"Arm B's GitHits route: {args.route}\n") with tempfile.TemporaryDirectory() as folder: - events, contracts = rehearse(Path(folder)) + events, contracts = rehearse(Path(folder), args.route) items = metrics.work_items(events, contracts) found = experiment.report(events, NAME, contracts, delivered_by="review") print("\nPer work item (syberlabs metrics):") @@ -275,7 +500,7 @@ def main() -> int: (out / "metrics.csv").write_text(metrics.to_csv(items), encoding="utf-8") print(f"\nwrote {out / 'report.json'}, {out / 'report.md'} and {out / 'metrics.csv'}") excluded = found["checks"].get("arm_conflict", []) - return 0 if excluded == ["LIB-4"] and found["items"]["compared"] == 6 else 1 + return 0 if excluded == [MISCONFIGURED] and found["items"]["compared"] == 6 else 1 if __name__ == "__main__": diff --git a/syberlabs/cli.py b/syberlabs/cli.py index 2a025c3..e366770 100644 --- a/syberlabs/cli.py +++ b/syberlabs/cli.py @@ -288,7 +288,9 @@ def main(argv: list[str] | None = None) -> int: comparing.add_argument("--baseline", default="off") comparing.add_argument("--treatment", default="on") comparing.add_argument("--server", default="githits", - help="the MCP server whose recorded connection must match each arm; none to skip that check") + help="what the record must show in the treatment's calls and in no baseline call: this MCP " + "server connected, or for githits (the default) also the host's GitHits API launcher " + "(signal.githits, --githits); none to skip the arm, contamination and use checks") comparing.add_argument("--items", metavar="FILE", help="the item list registered before the first run, one 'ITEM-ID [tag ...]' per line: items " "outside it, or with other tags, are excluded, and listed items that never ran are named") diff --git a/syberlabs/experiment.py b/syberlabs/experiment.py index 3395020..f4d7539 100644 --- a/syberlabs/experiment.py +++ b/syberlabs/experiment.py @@ -4,8 +4,9 @@ syberlabs experiment report --name githits-pilot [--delivered review] [--items FILE] [--json | --csv | --markdown] An experiment runs the same work items through the factory more than once and changes one thing between the runs, its -arms (the GitHits pilot: the proposer's MCP configuration, ``docs/CLAUDE_CODE.md``). A thread joins an experiment through -an ``experiment`` fact, a verified host observation written by ``syberlabs experiment label`` +arms (the GitHits pilot: whether the Claude Code proposer had GitHits, through the host's API launcher, +``--githits``, or before that through an MCP server, ``--mcp-config``; ``docs/CLAUDE_CODE.md``). A thread joins an +experiment through an ``experiment`` fact, a verified host observation written by ``syberlabs experiment label`` (``Thread.record_experiment``): ``{name, item, arm, tags, by, at}``, versioned by the experiment's name. ``report(events, name)`` folds a journal into one row per item: each arm's threads, their numbers from @@ -28,6 +29,30 @@ - the checks that make a pair comparable, or say why it is not (``CHECKS``), and, when a registered item list is given, the registered items that never ran. +Which arm ran is read from the record, call by call, and held to the label (``arm_conflict``, ``server_mixed``). For +the GitHits pilot a call had GitHits when the host's record of that call says so, by either route: + +- the API route (``ClaudeCodeProvider(githits=True)``, ``docs/GITHITS.md``): the candidate's ``signal.githits``, the + ledger the host writes for a call it gave the GitHits launcher, with its own count of the client records it verified + under the call's record key (``records``). The host writes it only when it wrote the launcher for that call, and + whether or not the model then used it, so a treatment that looked nothing up is ``unused``, not in conflict. What the + model drove (``signal.tool_usage``, ``signal.githits.launcher_calls``), the skill text it was shown + (``signal.harness.skill``) and the operator's name for the arm (``signal.githits.arm``) are not read for the arm; +- the MCP route: ``signal.harness.mcp_servers`` lists the server ``connected``, as Claude Code's ``init`` event said. + +A call with GitHits by either route counts; a treatment thread that had both (an MCP server left in arm B beside the +launcher) is still the treatment, and is said: ``two_routes``, kept, since where client records exist the metrics +station counts only those (``spec/GAPS.md`` 59). In the baseline either route is ``arm_conflict``. + +A GitHits lookup is shown by the metrics station's ``githits_lookups`` above zero (on the API route, its count of the +host's ``lookup`` facts from authentic client records), or by a verified ``lookup`` fact the host observed: an MCP +server call (``mcp__githits__*``), or a client record a guarded launcher call stands behind (``githits.``, guard +``pass``; ``githits.doctor``, a readiness check, is no lookup), including those of a call that failed before its +candidate. A baseline that shows one is ``contaminated``; a treatment that shows none is ``unused`` when its count is +known to be zero. An unknown count (a call that declared none, or, where ``metrics`` says so, a launcher call or +client record the record cannot vouch for: ``signal.githits.mismatch``, an ``unguarded`` record) is never read as +zero: a baseline that shows no lookup is then ``contamination_unknown``, a treatment ``use_unknown``. + Statistics, stated plainly. The item is the unit. Finishing (``accepted``, ``reviewed``) is compared over every valid item; efficiency (cost, turns, time, fix rounds, lookups, rejections, interventions) over the valid items accepted in both arms, so a cheap run that never finished is not counted as a win. Within that, a metric is compared over the items @@ -57,7 +82,11 @@ KEY = "experiment" PROMOTE = "accept_change" # build.PROMOTE, the default contract's promotion action; experiment may not import build BASELINE, TREATMENT = "off", "on" -SERVER = "githits" # the MCP server whose connection tells the arms apart in the GitHits pilot +SERVER = "githits" # tells the GitHits pilot's arms apart: its MCP server, or the host's API launcher +LAUNCHER = "githits" # claude_code: ``signal.githits``, the host's ledger of the API launcher of one call +LOOKUP = "lookup" # claude_code.LOOKUP, metrics.LOOKUP: one host-observed lookup, ``{tool, guard}`` +CLIENT_TOOL = "githits." # claude_code.CLIENT_TOOL: a lookup fact made from an authentic GitHits client record +DOCTOR = CLIENT_TOOL + "doctor" # metrics.CLIENT_DOCTOR: a readiness check's record, never a lookup MAX_TAGS = 8 PLACES = 9 _NAME = re.compile(r"^[A-Za-z0-9][A-Za-z0-9_.:+-]{0,63}$") @@ -83,16 +112,21 @@ "unequal_repeats": "the arms ran the item a different number of times, so a mean of several runs would face one run", "base_mismatch": "the arms started from different commits", "objective_mismatch": "the arms were given different objectives", - "arm_conflict": "the record contradicts the label: the server was connected in the baseline, or not in the treatment", - "contaminated": "the baseline made lookups on the treatment's server", + "arm_conflict": "the record contradicts the label: the server was there in the baseline (connected, or for GitHits " + "the host's API launcher), or not in the treatment", + "contaminated": "the baseline made lookups on the treatment's server, declared or host-observed", "labelled_late": "a thread's label was written or changed after its first candidate, so it was not fixed before the " "work ran", "unregistered": "the item is not in the registered item list", "registration_mismatch": "the item's tags differ from the registered item list", "base_unknown": "a thread has no recorded base commit, so the start could not be compared", - "arm_unverified": "no call recorded its MCP servers, so only the label says which arm ran", - "server_mixed": "the server was connected in some of a thread's calls and not in others", + "arm_unverified": "no call recorded its MCP servers or a GitHits launcher, so only the label says which arm ran", + "server_mixed": "the server was there in some of a thread's calls and not in others", + "two_routes": "the treatment had GitHits by two routes, the host's API launcher and its MCP server, not one; where " + "it has client records, its GitHits count (metrics) leaves the MCP server's lookups out", "unused": "the treatment made no lookups on its server", + "use_unknown": "the record cannot say how many lookups the treatment made on its server, and shows none, so it " + "cannot be shown to have used it", "contamination_unknown": "the baseline did not record its lookups on the server, so it cannot be shown clean", "relabelled": "a thread's labels disagree, but the one in force is the one that stood when its work began", "model_mismatch": "the arms were served by different models", @@ -156,13 +190,15 @@ class Run: relabelled: bool base: str | None objective: str | None - server: bool | str | None # connected in every call that recorded its servers, in none, "mixed", or None + server: bool | str | None # there (connected, or the launcher) in every call that said, in none, "mixed", None models: tuple[str, ...] accepted: metrics.Metric time_to_pass: metrics.Metric reviewed: metrics.Metric # accepted, and a review of the accepted commit passed (unknown while unreviewed) work: metrics.WorkItemMetrics upto: metrics.WorkItemMetrics | None = None # the same fold over the events up to acceptance + observed: int = 0 # lookup facts showing a lookup on the server, in the whole thread (``_observed``) + routes: tuple[str, ...] = () # how its calls had GitHits: ``launcher``, ``mcp`` or both (``_routes``) def value(self, name: str) -> metrics.Metric: """A compared number. An accepted thread's work metrics are read up to its first acceptance, so what happened @@ -182,37 +218,93 @@ def _promotion(contracts, contract) -> str: return PROMOTE +def _launcher(signal) -> bool: + """Did the host give this call the GitHits API launcher? ``signal.githits``, the ledger ``claude_code`` writes on + the candidate of every call it ran with ``githits=True``, and only then: an object holding the host's own count of + the authentic client records it read back for the call (``records``, a count, zero when the model looked nothing + up). A bare count or flag under that name (an adapter's ``signal`` is its own) is not that ledger.""" + found = signal.get(LAUNCHER) if isinstance(signal, dict) else None + return isinstance(found, dict) and type(found.get("records")) is int and found["records"] >= 0 + + def _servers(registrations: list[dict], server: str | None) -> bool | str | None: - """Was ``server`` connected? Read from the ``signal.harness.mcp_servers`` a harness recorded (``name:status``). + """Was ``server`` there? Per call, read from the record the host keeps of it: for GitHits, the API launcher it + provisioned (``_launcher``); for any server, the ``signal.harness.mcp_servers`` a harness recorded + (``name:status``), connected. The Claude Code provider leaves ``mcp_servers`` out when its session started with none, so a harness that recorded - its session's start (a ``version`` or ``model``) and lists no servers had none connected. A harness that recorded - neither says nothing about its servers, and that call is not read.""" + its session's start (a ``version`` or ``model``) and lists no servers had none connected. A call with no launcher + whose harness recorded neither says nothing about its servers, and is not read.""" if server is None: return None seen = [] for event in registrations: signal = event["body"].get("signal") harness = signal.get("harness") if isinstance(signal, dict) else None + launcher = server == SERVER and _launcher(signal) if not isinstance(harness, dict): + if launcher: + seen.append(True) continue - if "mcp_servers" not in harness and not any(isinstance(harness.get(k), str) for k in ("version", "model")): + if (not launcher and "mcp_servers" not in harness + and not any(isinstance(harness.get(k), str) for k in ("version", "model"))): continue - connected = False - for entry in harness.get("mcp_servers") or []: - if isinstance(entry, str): - name, _, status = entry.partition(":") - elif isinstance(entry, dict): - name, status = entry.get("name"), entry.get("status") - else: - continue - connected = connected or (name == server and status == "connected") - seen.append(connected) + seen.append(launcher or _connected(harness, server)) if not seen: return None return True if all(seen) else False if not any(seen) else "mixed" +def _connected(harness: dict, server: str) -> bool: + """The ``signal.harness.mcp_servers`` a harness recorded (``name:status``) list ``server`` connected.""" + for entry in harness.get("mcp_servers") or []: + if isinstance(entry, str): + name, _, status = entry.partition(":") + elif isinstance(entry, dict): + name, status = entry.get("name"), entry.get("status") + else: + continue + if name == server and status == "connected": + return True + return False + + +def _routes(registrations: list[dict], server: str | None) -> tuple[str, ...]: + """The routes by which a thread's calls had GitHits: ``launcher`` (the host's API launcher, ``_launcher``) and + ``mcp`` (its MCP server connected). Read for ``githits`` only: the launcher is GitHits'.""" + found = set() + for event in registrations if server == SERVER else (): + signal = event["body"].get("signal") + harness = signal.get("harness") if isinstance(signal, dict) else None + if _launcher(signal): + found.add("launcher") + if isinstance(harness, dict) and _connected(harness, server): + found.add("mcp") + return tuple(sorted(found)) + + +def _observed(events: list[dict], server: str | None) -> int: + """The thread's verified ``lookup`` facts that show a lookup on the server: every ``mcp____*`` call the + stream showed, and for GitHits each client record a guarded launcher call stands behind (``githits.``, guard + ``pass``; not ``githits.doctor``, a readiness check). A call journals its lookups before its candidate, and before it + raises when it fails, so this sees a lookup that no registration counts. A client record no guarded launcher call + stands behind (``unguarded``) is left to ``githits_lookups``: the record cannot say whether it reached GitHits, and + ``metrics`` may count it or call the count unknown.""" + if server is None: + return 0 + found = 0 + for event in events: + value = metrics.fact(event, LOOKUP) or {} + tool = value.get("tool") + if not isinstance(tool, str): + continue + if tool.startswith(f"mcp__{server}__"): + found += 1 + elif server == SERVER and tool.startswith(CLIENT_TOOL) and tool != DOCTOR and value.get("guard") == "pass": + found += 1 + return found + + def _models(registrations: list[dict]) -> tuple[str, ...]: found = set() for event in registrations: @@ -294,7 +386,8 @@ def _run(case_id: str, events: list[dict], name: str, work: metrics.WorkItemMetr label=event["hash"], labelled_late=late, relabelled=len({_identity(v) for _e, v in labels}) > 1, base=_base(events), objective=_objective(events), server=_servers(registrations, server), models=_models(registrations), accepted=accepted, time_to_pass=time_to_pass, - reviewed=_reviewed(events, accept, commit), work=work, upto=upto) + reviewed=_reviewed(events, accept, commit), work=work, upto=upto, observed=_observed(events, server), + routes=_routes(registrations, server)) def _threads(events: Iterable[dict]) -> dict[str, list[dict]]: @@ -324,6 +417,12 @@ def _mean(runs_: Sequence[Run], name: str) -> float | None: return round(statistics.fmean(float(v.value) for v in values), PLACES) +def _looked_up(run: Run) -> bool: + """The record shows the run made a GitHits lookup: ``githits_lookups`` known and above zero, or a lookup fact the + host observed (``_observed``).""" + return (run.work.githits_lookups.value or 0) > 0 or run.observed > 0 + + def _flags(item_runs: dict[str, list[Run]], baseline: str, treatment: str, server: str | None) -> list[str]: base_runs, treat_runs = item_runs.get(baseline, []), item_runs.get(treatment, []) everyone = base_runs + treat_runs @@ -346,13 +445,17 @@ def _flags(item_runs: dict[str, list[Run]], baseline: str, treatment: str, serve flags.append("arm_unverified") if any(run.server == "mixed" for run in treat_runs): flags.append("server_mixed") + if any(set(run.routes) >= {"launcher", "mcp"} for run in treat_runs): + flags.append("two_routes") if server == SERVER: - if any((run.work.githits_lookups.value or 0) > 0 for run in base_runs): + # An unknown count is never read as zero: it shows no baseline clean and no treatment unused. + if any(_looked_up(run) for run in base_runs): flags.append("contaminated") elif any(not run.work.githits_lookups.known for run in base_runs): flags.append("contamination_unknown") - if treat_runs and all(run.work.githits_lookups.value == 0 for run in treat_runs): - flags.append("unused") + if treat_runs and not any(_looked_up(run) for run in treat_runs): + flags.append("unused" if all(run.work.githits_lookups.value == 0 for run in treat_runs) + else "use_unknown") every_arm = [run for found in item_runs.values() for run in found] # a run moved into another arm still counts if any(run.labelled_late for run in every_arm): flags.append("labelled_late") @@ -514,8 +617,9 @@ def report(events: Iterable[dict], name: str, contracts: Mapping | None = None, registered: Mapping[str, Sequence[str]] | None = None, forgotten: Sequence[Mapping] | None = None) -> dict: """The experiment's items, their checks, and the treatment compared with the baseline. Pure. - ``server`` names the MCP server whose recorded connection must match the arm (``githits`` for the GitHits pilot); - None skips the arm, contamination and use checks for an experiment that does not turn a server on and off. + ``server`` names what the record must show in the treatment and not in the baseline: an MCP server, connected, and + for ``githits`` (the GitHits pilot, the default) also the host's GitHits API launcher; None skips the arm, + contamination and use checks for an experiment that does not turn a server on and off. ``delivered_by`` (``accept`` or ``review``) is what counts as delivered in ``delivery_rule``. ``registered`` is the item list written down before the first run (``parse_items``): an item it does not list, or lists with other tags, is excluded, and a listed item that never ran is named. ``forgotten`` is the home's forget tombstones diff --git a/tests/test_experiment_githits_api.py b/tests/test_experiment_githits_api.py new file mode 100644 index 0000000..5de564d --- /dev/null +++ b/tests/test_experiment_githits_api.py @@ -0,0 +1,431 @@ +"""The A/B report on the GitHits pilot's API route: arm B is ``ClaudeCodeProvider(githits=True)`` (#129, #134), and +the host's record that a call had GitHits is the launcher ledger on its candidate, ``signal.githits``, not an MCP +server. + +``tests/test_experiment.py`` covers the report on the MCP route; it is unchanged. Here the same checks are read off the +API route's record: first synthetic journals, ``tests.test_experiment.run`` threads given the signal and facts +``claude_code`` writes on that route, so each check can be read off the events that make it; then both arms of the pilot +end to end through the real ``ClaudeCodeProvider`` (the stand-in ``claude`` of ``tests/test_claude_code.py``, whose Bash +tool runs the real launcher behind the real query guard, and the host's own GitHits client against a loopback fake), +labelled with ``syberlabs experiment label`` and compared with ``syberlabs experiment report``; and last the rehearsal, +``examples/githits_ab.py``, on that route. No model is called and nothing reaches the real GitHits API. +""" + +import contextlib +import copy +import importlib.util +import io +import json +import os +import secrets +import tempfile +import unittest +from pathlib import Path +from unittest import mock + +from spec.validate import validate_events +from syberlabs import experiment, githits +from syberlabs.claude_code import DEFAULT_TOOLS, THIRD_PARTY_SWITCHES, ClaudeCodeProvider +from syberlabs.cli import main as cli +from tests.test_build_thread import GOOD, REPORT +from tests.test_claude_code import COST, PRICES, ClaudeCodeCase +from tests.test_experiment import NAME, run +from tests.test_githits import FakeGitHits +from tests.test_metrics import HOST, h + +RUNNER, REVIEWER = "runner@example.test", "reviewer@example.test" +BROKEN = {**GOOD, "src/report.py": "def rows():\n return []\n"} # the host's tests fail: the report rows are gone +SKILL = {"path": "skills/githits/SKILL.md", "sha256": "2" * 64} +GITHITS_SETTINGS = (githits.EXCLUDE_ENV, githits.EXCLUDE_REPLACE_ENV, githits.LICENSES_ENV, githits.SESSION_ENV, + githits.AGENT_ENV, githits.RECORD_DIR_ENV, githits.RECORD_KEY_ENV) + + +def ledger(records=0, arm="b"): + """``signal.githits`` as ``claude_code`` writes it on the candidate of a call it gave the launcher.""" + return {"arm": arm, "launcher_calls": records, "launcher_passed": records, "records": records, + "records_ignored": 0, "bash_refused": 0, "license_policy": "1" * 64} + + +def api(events, case, launcher=(True,)): + """``run()``'s thread ``case`` as the API route records it. Call n had the launcher when ``launcher[n]`` (the last + entry repeated) is true: its candidate carries the ledger, whose ``records`` is the call's GitHits count, and the + skill's digest. No call lists an MCP server.""" + out = copy.deepcopy(events) + calls = [e for e in out if e["case_id"] == case and e["kind"] == "candidate_registered"] + for n, event in enumerate(calls): + signal = event["body"]["signal"] + # Each call its own harness (``run()`` shares one object between a thread's calls). + signal["harness"] = {k: v for k, v in signal["harness"].items() if k != "mcp_servers"} + if launcher[min(n, len(launcher) - 1)]: + signal["githits"] = ledger((signal.get("tool_usage") or {}).get("githits", 0)) + signal["harness"]["skill"] = dict(SKILL) + return out + + +def ledger_says(events, case, **fields): + """``events`` with ``fields`` set on the ledger of every call of thread ``case``.""" + for event in events: + if event["case_id"] == case and event["kind"] == "candidate_registered": + event["body"]["signal"]["githits"].update(fields) + return events + + +def lookups_of(events, case): + """The metrics station's ``githits_lookups`` for thread ``case``, as the report reads it.""" + return next(r for r in experiment.runs(events, NAME) if r.thread == case).work.githits_lookups + + +def looked_up(events, case, *tools, guard="pass"): + """``events`` with one verified ``lookup`` fact per tool appended to thread ``case``, as ``space.observe`` writes + them (a call that fails journals its lookups and registers nothing).""" + out = list(events) + last = max((e for e in out if e["case_id"] == case), key=lambda e: e["seq"]) + for n, tool in enumerate(tools, 1): + seq = last["seq"] + n + out.append({"case_id": case, "seq": seq, "kind": "observed", "at": last["at"] + n, "previous": "0" * 64, + "hash": h(case, seq), "body": { + "key": "lookup", "source": HOST, "version": f"s9.1.{n}", "actor": HOST, "verified": True, + "value": {"tool": tool, "query_digest": "3" * 64, "ok": guard == "pass", "guard": guard}}}) + return out + + +class ApiRouteChecks(unittest.TestCase): + """The checks of ``tests/test_experiment.py``'s ``Checks``, with the treatment's GitHits on the API route.""" + + def pair(self, off=None, on=None, *, off_launcher=(False,), on_launcher=(True,)): + events = run("x-off", "X", "off", **(off or {}))[0] + run("x-on", "X", "on", **{"githits": 1, **(on or {})})[0] + return api(api(events, "x-off", off_launcher), "x-on", on_launcher) + + def row(self, events, **report): + return experiment.report(events, NAME, **report)["rows"][0] + + def test_the_launcher_ledger_is_the_treatments_record_of_githits(self): + events = self.pair() + self.assertEqual(self.row(events)["flags"], [], "a clean pair: no MCP server anywhere, and no conflict") + self.assertEqual({r.thread: r.server for r in experiment.runs(events, NAME)}, {"x-off": False, "x-on": True}) + bare = run("x-off", "X", "off")[0] + run("x-on", "X", "on", githits=1)[0] + self.assertEqual(self.row(bare)["excluded"], ["arm_conflict"], + "the same pair without the ledger shows no GitHits in the treatment; before the report read " + "the ledger, every API-route treatment read like this") + + def test_arm_conflict_when_the_baseline_had_the_launcher_or_the_treatment_did_not(self): + row = self.row(self.pair(on={"githits": 0}, on_launcher=(False,))) + self.assertEqual((row["excluded"], row["flags"]), (["arm_conflict"], ["arm_conflict", "unused"]), + "a treatment run without --githits had no GitHits, and used none") + self.assertEqual(self.row(self.pair(off_launcher=(True,)))["flags"], ["arm_conflict"], + "a baseline given the launcher is in conflict even if it looked nothing up") + self.assertEqual(self.row(self.pair(off={"githits": 2}, off_launcher=(True,)))["excluded"], + ["arm_conflict", "contaminated"]) + text = experiment.table(experiment.report(self.pair(on_launcher=(False,)), NAME)) + self.assertIn("check arm_conflict (excluded): X: the record contradicts the label: the server was there in the " + "baseline (connected, or for GitHits the host's API launcher), or not in the treatment", text) + + def test_contaminated_counts_every_host_observed_githits_lookup_of_the_baseline(self): + for tool in ("githits.search", "githits.example", "mcp__githits__search"): + with self.subTest(tool=tool): + events = looked_up(self.pair(), "x-off", tool) + self.assertEqual(self.row(events)["excluded"], ["contaminated"], + "no candidate of the baseline counted it, but the host journaled it") + for tool, guard in (("WebSearch", "pass"), ("Bash", "not_launcher"), ("githits.doctor", "pass")): + with self.subTest(tool=tool): + self.assertEqual(self.row(looked_up(self.pair(), "x-off", tool, guard=guard))["flags"], [], + "a web lookup, a shell command the guard refused, or a readiness check is not a " + "GitHits lookup") + + def test_unused_when_the_treatment_made_no_githits_lookup(self): + idle = self.pair(on={"githits": 0}) + self.assertEqual(self.row(idle)["flags"], ["unused"], + "the launcher was there and the model never used it: kept, and said, not a conflict") + self.assertEqual(self.row(looked_up(idle, "x-on", "Bash", guard="shell_syntax"))["flags"], ["unused"], + "a launcher call the guard refused never reached GitHits") + self.assertEqual(self.row(looked_up(idle, "x-on", "githits.doctor"))["flags"], ["unused"], + "a readiness check is not a lookup (metrics leaves it out of githits_lookups too)") + self.assertEqual(self.row(looked_up(idle, "x-on", "mcp__githits__search"))["flags"], [], + "a lookup the host observed in a call that registered nothing is still a use") + + def test_an_unknown_lookup_count_is_never_read_as_zero(self): + """``githits_lookups`` is unknown when a call declared no count and, where ``metrics`` reads it so (#139), when + a launcher call or a client record cannot be vouched for (``signal.githits.mismatch``, ``unguarded``). The + report never reads it as zero: a baseline that shows no lookup is ``contamination_unknown``, a treatment + ``use_unknown``, never ``unused``; a lookup the host observed shows use whatever the count says. These hold on + either reading of the count, so they assume neither.""" + silent = self.pair(on={"tools": False}) # unknown on every reading: no call declared a count + row = self.row(silent) + self.assertEqual(row["flags"], ["use_unknown"], + "the treatment can be shown neither to have used GitHits nor not") + self.assertIsNone(row["values"]["on"]["githits_lookups"], "and its paired count is unknown, never 0") + self.assertEqual(self.row(self.pair(off={"tools": False}))["flags"], ["contamination_unknown"]) + self.assertEqual(self.row(looked_up(silent, "x-on", "mcp__githits__search"))["flags"], [], + "a lookup the host observed shows use whatever the count says") + self.assertEqual(self.row(looked_up(silent, "x-on", "githits.search"))["flags"], [], + "so does a client record a guarded launcher call stands behind") + # A launcher call the guard passed that left no record: the count is 0 or unknown, as metrics reads it. + gap = ledger_says(self.pair(on={"githits": 0}), "x-on", mismatch="githits_unrecorded", launcher_calls=1, + launcher_passed=1) + self.assertEqual(self.row(gap)["flags"], ["unused"] if lookups_of(gap, "x-on").known else ["use_unknown"]) + self.assertEqual(self.row(looked_up(gap, "x-on", "githits.search"))["flags"], []) + # A client record no guarded launcher call stands behind, in the baseline: counted, or unknown. + doubtful = looked_up(self.pair(), "x-off", "githits.search", guard="unguarded") + known = lookups_of(doubtful, "x-off").known + self.assertEqual(self.row(doubtful)["flags"], ["contaminated"] if known else ["contamination_unknown"], + "never clean: contaminated if the count includes it, contamination_unknown if it cannot") + + def test_a_thread_with_the_launcher_in_some_calls_only_is_mixed(self): + events = self.pair(on={"calls": 2}, on_launcher=(True, False)) + self.assertEqual({r.thread: r.server for r in experiment.runs(events, NAME)}["x-on"], "mixed") + self.assertEqual(self.row(events)["flags"], ["server_mixed"], "a treatment call without it: kept, and said") + baseline = self.pair(off={"calls": 2}, off_launcher=(False, True)) + self.assertEqual(self.row(baseline)["excluded"], ["arm_conflict"], + "a baseline with the launcher in one call of two") + + def test_only_the_hosts_ledger_says_the_launcher_was_there(self): + for bad in (2, True, "githits", [], {"arm": "on"}, {"records": -1}, {"records": "1"}, {"records": True}, + {"records": 1.0}): + with self.subTest(githits=bad): + events = self.pair(on_launcher=(False,)) + for event in events: + if event["case_id"] == "x-on" and event["kind"] == "candidate_registered": + event["body"]["signal"]["githits"] = bad + self.assertEqual(self.row(events)["excluded"], ["arm_conflict"], + "a count, a flag or an object without the host's record count is not the ledger") + events = self.pair(on={"githits": 3}, on_launcher=(False,)) + for event in events: + if event["case_id"] == "x-on" and event["kind"] == "candidate_registered": + event["body"]["signal"]["harness"]["skill"] = dict(SKILL) + self.assertEqual(self.row(events)["excluded"], ["arm_conflict"], + "lookup counts the model drove, and the skill text it was shown, do not say the launcher was " + "there") + events = self.pair() + for event in events: + if event["kind"] == "candidate_registered" and event["case_id"] == "x-on": + event["body"]["signal"]["githits"].update(arm="off", launcher_calls=0) + event["body"]["signal"].pop("harness") + self.assertEqual(self.row(events)["flags"], [], + "the operator's name for the arm and the model's calls are not read; the ledger alone " + "stands for the call when no harness was recorded") + + def test_either_route_counts_for_githits_and_only_for_githits(self): + events = run("x-off", "X", "off")[0] + run("x-on", "X", "on", calls=2, githits=1, server="githits:connected")[0] + events = api(events, "x-on", launcher=(False, True)) + for event in events: # call 1 through the MCP server, call 2 through the launcher + if event["case_id"] == "x-on" and event["body"].get("id") == "c1": + event["body"]["signal"]["harness"]["mcp_servers"] = ["githits:connected"] + self.assertEqual({r.thread: (r.server, r.routes) for r in experiment.runs(events, NAME)}, + {"x-off": (False, ()), "x-on": (True, ("launcher", "mcp"))}) + self.assertEqual(self.row(events)["flags"], ["two_routes"], + "GitHits in every call, by one route or the other: the treatment, said to have had two") + self.assertEqual(self.row(self.pair(), server="docs")["excluded"], ["arm_conflict"], + "the launcher is GitHits'; another server named on the command line is not it") + self.assertEqual(self.row(self.pair(on_launcher=(False,)), server=None)["flags"], [], + "server None: an experiment that switches nothing") + + +class ApiArmEndToEnd(ClaudeCodeCase): + """Both arms of the pilot as #134 runs them, modelled on ``tests/test_experiment.py``'s ``WithClaudeCode`` (#83): + arm A is ``ClaudeCodeProvider`` as before, arm B the same with ``githits=True``. The stand-in ``claude`` lists no + MCP server; arm B's lookups go through the launcher, the real query guard and the host's GitHits client to a + loopback fake. Each thread is labelled with ``syberlabs experiment label`` before its first candidate, accepted by + the unattended runner, and reviewed blind; the report has to read the arm, the lookups and the price off that + record, with no help.""" + + def setUp(self): + super().setUp() + self.fake = FakeGitHits() + self.addCleanup(self.fake.close) + # The api lane as an operator's machine has it: a key, no gateway or provider switch inherited from the + # environment the suite runs in, and GitHits settings from nowhere but this test. + env = {k: v for k, v in os.environ.items() if k not in THIRD_PARTY_SWITCHES and k not in GITHITS_SETTINGS} + env.update({"ANTHROPIC_API_KEY": "test-placeholder", githits.TOKEN_ENV: "ghtest_" + secrets.token_hex(16), + githits.URL_ENV: self.fake.url}) + patcher = mock.patch.dict(os.environ, env, clear=True) + patcher.start() + self.addCleanup(patcher.stop) + budget = {"max_candidates": 6, "max_evaluations": 6, "max_seconds": 600, "prices": PRICES, "max_dollars": 1.0} + self.kit_ = self.kit(self.contract(**{"evolution.budget": budget})) + + def cli(self, *args): + out = io.StringIO() + with contextlib.redirect_stdout(out), contextlib.redirect_stderr(out): + code = cli(["--home", str(self.root / ".syberlabs"), "--repo", str(self.root), "--actor", + "owner@example.test", *args]) + return code, out.getvalue() + + @staticmethod + def launcher(words: str) -> list: + """A lookup through the launcher: the stand-in puts the path ``--allowedTools`` names in place of + ``{launcher}``; without that rule its Bash tool denies the command, as Claude Code's ``dontAsk`` does.""" + return ["Bash", {"command": "{launcher} " + words, "description": "look up a public API"}] + + def run_arm(self, item, arm, tag, *, githits_on, name, first=None, repair=None, lookups=(), **options): + """One labelled thread through ``ClaudeCodeProvider`` (``options`` passed to it): labelled, proposed, + accepted, reviewed blind.""" + kit = self.kit_ + thread = kit.start(f"Add a CSV export ({item})", paths=["src/", "tests/"], unattended=True, actor=RUNNER) + (kit.home / "current").write_text(thread.id + "\n") + code, text = self.cli("experiment", "label", "--name", NAME, "--item", item, "--arm", arm, "--tag", tag) + self.assertEqual(code, 0, text) + self.assertNotIn("labelled_late", text) + claude = self.stand_in(name, first=first, repair=repair, lookups=[list(entry) for entry in lookups]) + provider = ClaudeCodeProvider(kit.repo, bare=True, executable=claude, max_calls=3, githits=githits_on, + arm=arm, thread_label=thread.id, **options) + found = thread.propose(provider) + receipt = thread.accept(found[-1].id, actor=RUNNER) + self.assertEqual(receipt.status, "succeeded", receipt.reason) + thread.record_review("pass", reviewer="blind", revision=receipt.commit, by=REVIEWER) + self.assertEqual(validate_events(thread.history(), "thread"), []) + return thread, found + + def events(self): + return [e for case in self.kit_.journal.thread_ids() for e in self.kit_.session.history(case)] + + def test_both_arms_through_the_api_launcher_give_a_paired_result_and_a_verdict(self): + web = ["WebSearch", {"query": "zq91 web query: python csv DictWriter header row"}] + search = self.launcher('search "csv DictWriter writeheader zq92" --target pypi:requests@2.32.3') + idle = self.launcher('search "title case names zq93" --target pypi:requests@2.32.3') + # LIB-1 needs a lookup: without GitHits the first try fails the host's tests and a repair fixes it. + _, off = self.run_arm("LIB-1", "off", "needs_lookup", githits_on=False, name="lib-off", first=BROKEN, + repair={"src/report.py": REPORT}, lookups=[web]) + thread, on = self.run_arm("LIB-1", "on", "needs_lookup", githits_on=True, name="lib-on", lookups=[search]) + self.assertEqual(([c.evaluation for c in off], [c.evaluation for c in on]), (["failed", "passed"], ["passed"])) + self.assertEqual(on[0].signal["githits"]["records"], 1, "the host read one authentic client record") + self.assertNotIn("mcp_servers", on[0].signal["harness"], "arm B connected no MCP server") + self.assertEqual([e["body"]["value"]["tool"] for e in thread.history() if e["kind"] == "observed" + and e["body"]["key"] == "lookup"], ["githits.search"]) + # ROUTINE-1 needs none: both arms are right first time; arm B calls GitHits once and has nothing to add. + self.run_arm("ROUTINE-1", "off", "routine", githits_on=False, name="routine-off") + self.run_arm("ROUTINE-1", "on", "routine", githits_on=True, name="routine-on", lookups=[idle]) + self.assertEqual(len(self.fake.requests), 2, "only arm B's launcher calls reached GitHits") + + found = experiment.report(self.events(), NAME, dict(self.kit_.session.contracts), delivered_by="review") + self.assertEqual(found["items"], {"labelled": 2, "compared": 2, "excluded": 0}) + self.assertEqual(found["checks"], {}, "the record agrees with the labels: the launcher in arm B only, GitHits " + "used there only, the same base, objective, model and lane") + self.assertEqual({(r.item, r.arm): r.server for r in experiment.runs(self.events(), NAME)}, + {("LIB-1", "off"): False, ("LIB-1", "on"): True, ("ROUTINE-1", "off"): False, + ("ROUTINE-1", "on"): True}) + rows = {row["item"]: row for row in found["rows"]} + lib, routine = rows["LIB-1"]["values"], rows["ROUTINE-1"]["values"] + self.assertEqual((lib["off"]["cost"], lib["on"]["cost"]), (round(2 * COST, 6), COST), + "host-observed dollars on the api lane: two priced calls off, one on") + self.assertEqual((lib["off"]["fix_rounds"], lib["on"]["fix_rounds"]), (1.0, 0.0)) + self.assertEqual((lib["off"]["turns"], lib["on"]["turns"]), (4.0, 2.0), "Claude Code's num_turns, per call") + self.assertEqual((lib["off"]["web_lookups"], lib["on"]["web_lookups"]), (2.0, 0.0)) + self.assertEqual((lib["off"]["githits_lookups"], lib["on"]["githits_lookups"]), (0.0, 1.0), + "arm B's lookup is the host's fact from the client's record") + self.assertEqual((routine["off"]["cost"], routine["on"]["cost"], routine["on"]["githits_lookups"]), + (COST, COST, 1.0)) + self.assertEqual({(arm, row["values"][arm]["reviewed"]) for row in found["rows"] for arm in ("off", "on")}, + {("off", 1.0), ("on", 1.0)}) + rule = found["rule"] + self.assertEqual((rule["delivered"], rule["pending"], rule["cost_pairs"]), ({"off": 2.0, "on": 2.0}, 0, 2)) + self.assertEqual(rule["median_cost_difference"], round(-COST / 2, 6), "LIB-1 saves a call, ROUTINE-1 ties") + self.assertIs(rule["met"], True) + self.assertEqual(found["per_arm"]["on"]["lookups_blocked"], 0, "every launcher call passed the guard") + + code, text = self.cli("experiment", "report", "--name", NAME, "--delivered", "review") + self.assertEqual(code, 0, text) + self.assertIn("2 items compared, 0 excluded, 4 threads", text) + verdict = next(line for line in text.splitlines() if line.startswith("rule (")) + self.assertTrue(verdict.startswith("rule (accepted and reviewed = delivered): met."), verdict) + lines = ["", "GitHits pilot, API route, both arms through ClaudeCodeProvider (stand-in claude, loopback " + "GitHits):"] + for row in found["rows"]: + for arm in ("off", "on"): + value = row["values"][arm] + lines.append(f" {row['item']:9} {arm:3} cost ${value['cost']:.4f} turns {value['turns']:g} " + f"fix rounds {value['fix_rounds']:g} web {value['web_lookups']:g} " + f"githits {value['githits_lookups']:g} reviewed {value['reviewed']:g}") + print("\n".join(lines + [" " + verdict])) + + def test_a_stray_githits_mcp_server_beside_the_launcher(self): + """Arm B runs the API launcher and, by mistake, a GitHits MCP server as well (``--mcp-config``), and the model + uses both. GitHits was there, so the item is no ``arm_conflict``; the treatment is said to have had two routes + (``two_routes``, kept: the pilot pre-registers its exclusions, and this is not one). Its paired GitHits count is + the metrics station's count of client records, which leaves the MCP lookup out (``spec/GAPS.md`` 59), while + the report sees both. The same stray server in arm A is ``arm_conflict``, and its lookup ``contaminated``.""" + config = self.root.parent / "githits-mcp.json" + config.write_text(json.dumps({"mcpServers": {"githits": {"command": "githits-stand-in"}}}), encoding="utf-8") + mcp = ["mcp__githits__search", {"query": "zq95 githits mcp query: csv writer header"}] + search = self.launcher('search "csv DictWriter writeheader zq96" --target pypi:requests@2.32.3') + tools = [*DEFAULT_TOOLS, "mcp__githits"] + self.run_arm("Z", "off", "needs_lookup", githits_on=False, name="z-off") + thread, on = self.run_arm("Z", "on", "needs_lookup", githits_on=True, name="z-on", lookups=[mcp, search], + mcp_config=config, allowed_tools=tools) + self.assertEqual((on[0].signal["harness"]["mcp_servers"], on[0].signal["githits"]["records"]), + (["githits:connected"], 1), "both routes there, and both used") + self.assertEqual([e["body"]["value"]["tool"] for e in thread.history() if e["kind"] == "observed" + and e["body"]["key"] == "lookup"], ["mcp__githits__search", "githits.search"]) + self.run_arm("W", "off", "needs_lookup", githits_on=False, name="w-off", lookups=[mcp], mcp_config=config, + allowed_tools=tools) + self.run_arm("W", "on", "needs_lookup", githits_on=True, name="w-on", lookups=[search]) + found = experiment.report(self.events(), NAME, dict(self.kit_.session.contracts)) + rows = {row["item"]: row for row in found["rows"]} + self.assertEqual((rows["Z"]["flags"], rows["Z"]["excluded"]), (["two_routes"], [])) + runs = {(r.item, r.arm): r for r in experiment.runs(self.events(), NAME)} + self.assertEqual((runs["Z", "on"].server, runs["Z", "on"].routes, runs["Z", "on"].observed), (True, + ("launcher", "mcp"), 2)) + self.assertEqual(rows["Z"]["values"]["on"]["githits_lookups"], 1.0, + "metrics counts the client's record only; the MCP lookup is the report's to see") + self.assertEqual(rows["W"]["excluded"], ["arm_conflict", "contaminated"]) + self.assertEqual(found["items"], {"labelled": 2, "compared": 1, "excluded": 1}) + code, text = self.cli("experiment", "report", "--name", NAME) + self.assertEqual(code, 0, text) + self.assertIn("check two_routes (kept): Z: the treatment had GitHits by two routes", text) + + def test_an_api_pilot_run_against_its_label_is_excluded(self): + search = self.launcher('search "csv DictWriter writeheader zq94" --target pypi:requests@2.32.3') + # X: the operator passed --githits in the baseline (with the arm name "off"), and the model used it. + self.run_arm("X", "off", "needs_lookup", githits_on=True, name="x-off", lookups=[search]) + self.run_arm("X", "on", "needs_lookup", githits_on=True, name="x-on", lookups=[search]) + # Y: the operator forgot --githits in the treatment, so its Bash call was denied and nothing was looked up. + self.run_arm("Y", "off", "routine", githits_on=False, name="y-off") + self.run_arm("Y", "on", "routine", githits_on=False, name="y-on", lookups=[search]) + found = experiment.report(self.events(), NAME, dict(self.kit_.session.contracts)) + rows = {row["item"]: row for row in found["rows"]} + self.assertEqual(rows["X"]["excluded"], ["arm_conflict", "contaminated"]) + self.assertEqual((rows["Y"]["excluded"], rows["Y"]["flags"]), (["arm_conflict"], ["arm_conflict", "unused"])) + self.assertEqual(found["items"], {"labelled": 2, "compared": 0, "excluded": 2}) + code, text = self.cli("experiment", "report", "--name", NAME) + self.assertEqual(code, 1) + self.assertIn("check arm_conflict (excluded): X, Y", text) + self.assertIn("nothing was compared", text) + + +def load_rehearsal(): + path = Path(__file__).resolve().parents[1] / "examples" / "githits_ab.py" + spec = importlib.util.spec_from_file_location("githits_ab", path) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +class Rehearsal(unittest.TestCase): + """``examples/githits_ab.py`` on its default route, the API launcher: seven items through both arms of the real + ``ClaudeCodeProvider``, with its own stand-in ``claude`` and loopback GitHits.""" + + def test_the_api_route_rehearsal_compares_six_items_and_excludes_the_misconfigured_one(self): + rehearsal, said = load_rehearsal(), [] + with tempfile.TemporaryDirectory() as folder: + before = dict(os.environ) + events, contracts = rehearsal.rehearse(Path(folder), "api", log=said.append) + self.assertEqual(dict(os.environ), before, "the rehearsal puts the environment back") + found = experiment.report(events, rehearsal.NAME, contracts, delivered_by="review") + self.assertEqual(found["items"], {"labelled": 7, "compared": 6, "excluded": 1}) + self.assertEqual(found["checks"], {"arm_conflict": ["LIB-4"], "unused": ["LIB-4"]}, + "the treatment that ran without --githits, and nothing else") + self.assertEqual((found["rule"]["met"], found["by_tag"]["needs_lookup"]["rule"]["met"]), (False, True), + "docs/EXPERIMENTS.md: the rehearsal's rule is not met overall and met on needs_lookup") + self.assertIn("The loopback GitHits received 9 requests; the real API received none.", said) + runs = experiment.runs(events, rehearsal.NAME) + self.assertEqual({(r.arm, r.server) for r in runs if r.item != "LIB-4"}, {("off", False), ("on", True)}) + lookups = {r.item: r.value("githits_lookups") for r in runs if r.arm == "on" and r.item != "LIB-4"} + self.assertEqual({item: metric.value for item, metric in lookups.items()}, + {"LIB-1": 2, "LIB-2": 2, "LIB-3": 2, "ROUTINE-1": 1, "ROUTINE-2": 1, "ROUTINE-3": 1}) + self.assertTrue(all("host-observed" in metric.note for metric in lookups.values()), + "counted from the host's lookup facts, not from what the stand-in declared") + with self.assertRaises(ValueError): + rehearsal.rehearse(Path("unused"), "carrier-pigeon") + + +if __name__ == "__main__": + unittest.main()