From 14c13da728efb91ce8c0eb302abc7a9cae1e4c56 Mon Sep 17 00:00:00 2001 From: SyberLabs candidate Date: Mon, 2 Nov 2026 00:06:14 -0800 Subject: [PATCH] patch: Refuse unscreened --bare lookups unless overridden, stop Git Bash rewriting launcher arguments, mirror lane refusals in eval spec validation, fix docs digest and a flaky pace test SyberLabs-Thread: b4b760bd-a5dc-4ca5-abfc-db9772103579 SyberLabs-Candidate: c1 SyberLabs-Provider: claude@opus-5.5 --- docs/CLAUDE_CODE.md | 24 ++++--- docs/EXPERIMENTS.md | 4 +- spec/GAPS.md | 1 + syberlabs/claude_code.py | 45 ++++++++++-- syberlabs/cli.py | 6 ++ syberlabs/eval/arms.py | 30 +++++++- tests/test_claude_code.py | 143 +++++++++++++++++++++++++++++++++++-- tests/test_eval_core.py | 38 ++++++++++ tests/test_experiment.py | 5 +- tests/test_githits_pace.py | 18 +++-- 10 files changed, 285 insertions(+), 29 deletions(-) diff --git a/docs/CLAUDE_CODE.md b/docs/CLAUDE_CODE.md index d1313c5f..0b82213d 100644 --- a/docs/CLAUDE_CODE.md +++ b/docs/CLAUDE_CODE.md @@ -4,7 +4,7 @@ syberlabs propose --claude-code --trusted-tree # your own Claude Code login (subscription_local) syberlabs propose --claude-code --bare # ANTHROPIC_API_KEY in your environment (api) syberlabs propose --claude-code --bare --max-calls 1 # one shot instead of the repair loop -syberlabs propose --claude-code --bare --mcp-config githits.json --allowed-tools Read,Edit,Write,Glob,Grep,mcp__githits +syberlabs propose --claude-code --lane api-hooked --trusted-tree --mcp-config githits.json --allowed-tools Read,Edit,Write,Glob,Grep,mcp__githits syberlabs propose --claude-code --bare --guard-max-lines 3 --guard-max-chars 400 # the query guard's thresholds (defaults) syberlabs propose --claude-code --lane api-hooked --trusted-tree # ANTHROPIC_API_KEY without --bare: the guard's hook runs syberlabs propose --claude-code --lane api-hooked --trusted-tree --githits # GitHits through the API client (arm B) @@ -13,6 +13,11 @@ syberlabs propose --claude-code --lane api-hooked --trusted-tree --githits # G `--githits` refuses `--bare`: Claude Code 2.1.312 does not run `--settings` hooks under `--bare` (below, the query guard), so the treatment runs only on a lane where its guard runs, `api-hooked` for an API key. +For the same reason `--bare` refuses the other lookups the hook would screen: `WebSearch` or `WebFetch` in +`--allowed-tools`, or an `--mcp-config` that names an MCP server (`invalid_provider`, pointing to `--lane +api-hooked`). Under `--bare` such a lookup would leave first and be flagged `unguarded` only afterwards. +`--allow-unscreened-bare` accepts that, and is recorded as `signal.guard.allow_unscreened_bare`. + `syberlabs.claude_code.ClaudeCodeProvider` is the repair loop (`providers.RepairProvider`, the factory's default proposer path) with one call replaced. Each call: 1. checks out the parent, the thread's base on the first call and the failed candidate on a repair, in a detached temporary worktree (`gitspace.Repo.worktree`; the developer's working tree is never touched); @@ -126,7 +131,7 @@ An allowed query passes unchanged and silently (exit 0, no output). A blocked on **What the operator tunes.** `--guard-max-lines N` and `--guard-max-chars M` on `syberlabs propose --claude-code`, or `SYBERLABS_GUARD_MAX_LINES` and `SYBERLABS_GUARD_MAX_CHARS` in the host's environment (the flags win), or `guard_max_lines` and `guard_max_chars` on `ClaudeCodeProvider`. They are host policy, never contract data. The rules themselves and the 16-character tree-line floor are fixed in `query_guard.py` (`VERSION` 1). -**Limits, stated plainly.** The guard is a filter over what the model typed, not a proof that no code leaves the machine. A model can paraphrase code, split it into several short queries, split one credential across two lookups (each half alone is no credential shape), separate a key's characters with spaces, encode it (base64, base32, hex, URL or percent-encoding, HTML entities, ROT13, Morse, one character per word: none is decoded), use a lookalike the fold does not map (only NFKC, `Cf` removal and a fixed table of Cyrillic and Greek letters are applied; Armenian, Cherokee or other scripts are not), put it inside one line of a longer query (the tree rule compares whole lines), or quote a line it wrote during the run (the digests are taken from the parent's tracked files before the run). The identifier rule the packet sketched (a path or identifier from the tree that is in no public dependency name) is not built. A false positive costs a rephrased query: a query that happens to equal a 16-character line of the tree, an `user@host` that is not an address, a long but public question. **Does a `--settings` hook run under `--bare`? No (Claude Code 2.1.312, live, 2026-10-30).** The headless page said bare mode skips the *auto-discovery* of hooks and lists `--settings` as the way to pass settings, while the `--help` says bare mode skips "hooks (those defined in settings and by installed plugins)". The `--help` is right. A hook diagnosis on a Freestyle Linux VM ran the same `PreToolUse` hook through `--settings` on the same command with and without `--bare` (`claude -p [--bare] --restricted --tools Bash --allowedTools "Bash(echo:*)" --setting-sources user --strict-mcp-config --settings `): the hook fired only without `--bare`, and without `--bare` it worked with `apiKeySource: ANTHROPIC_API_KEY`. The pilot canary then ran `propose --claude-code --bare --githits`: the model ran the launcher with no hook, the client sent the lookup (its record says `sent: true`, `status: ok`), and only afterwards did the host stop the run `provider_guard_inactive` with both lookups `unguarded`. Fail closed after the fact was not enough for a lookup that leaves the machine. Two changes follow: the launcher now refuses before the network without the hook's decision (the pre-send refusal, below), and `--githits` refuses `--bare` and runs on `api-hooked` (above). WebSearch, WebFetch and `--mcp-config` lookups on a `--bare` lane are still caught only after the fact (`spec/GAPS.md` 64 (3)). +**Limits, stated plainly.** The guard is a filter over what the model typed, not a proof that no code leaves the machine. A model can paraphrase code, split it into several short queries, split one credential across two lookups (each half alone is no credential shape), separate a key's characters with spaces, encode it (base64, base32, hex, URL or percent-encoding, HTML entities, ROT13, Morse, one character per word: none is decoded), use a lookalike the fold does not map (only NFKC, `Cf` removal and a fixed table of Cyrillic and Greek letters are applied; Armenian, Cherokee or other scripts are not), put it inside one line of a longer query (the tree rule compares whole lines), or quote a line it wrote during the run (the digests are taken from the parent's tracked files before the run). The identifier rule the packet sketched (a path or identifier from the tree that is in no public dependency name) is not built. A false positive costs a rephrased query: a query that happens to equal a 16-character line of the tree, an `user@host` that is not an address, a long but public question. **Does a `--settings` hook run under `--bare`? No (Claude Code 2.1.312, live, 2026-10-30).** The headless page said bare mode skips the *auto-discovery* of hooks and lists `--settings` as the way to pass settings, while the `--help` says bare mode skips "hooks (those defined in settings and by installed plugins)". The `--help` is right. A hook diagnosis on a Freestyle Linux VM ran the same `PreToolUse` hook through `--settings` on the same command with and without `--bare` (`claude -p [--bare] --restricted --tools Bash --allowedTools "Bash(echo:*)" --setting-sources user --strict-mcp-config --settings `): the hook fired only without `--bare`, and without `--bare` it worked with `apiKeySource: ANTHROPIC_API_KEY`. The pilot canary then ran `propose --claude-code --bare --githits`: the model ran the launcher with no hook, the client sent the lookup (its record says `sent: true`, `status: ok`), and only afterwards did the host stop the run `provider_guard_inactive` with both lookups `unguarded`. Fail closed after the fact was not enough for a lookup that leaves the machine. Two changes follow: the launcher now refuses before the network without the hook's decision (the pre-send refusal, below), and `--githits` refuses `--bare` and runs on `api-hooked` (above). WebSearch, WebFetch and `--mcp-config` lookups on a `--bare` lane are now refused when the provider is built, unless the operator passes `--allow-unscreened-bare` (recorded `signal.guard.allow_unscreened_bare`); with that override they are still caught only after the fact (`spec/GAPS.md` 65). ## GitHits through the launcher (`--githits`, arm B) @@ -143,12 +148,13 @@ Both arms run the same lane, so their argv differ only in the GitHits additions ```sh #!/bin/sh +export MSYS_NO_PATHCONV=1 MSYS2_ARG_CONV_EXCL='*' exec '' -I '/syberlabs/githits_cli_entry.py' "$@" ``` `githits_cli_entry.py` puts the host checkout first on `sys.path`, refuses to run if `syberlabs` came from anywhere else, and runs the host's GitHits client with the arguments. It accepts only the five lookups (`search`, `example`, `read`, `list`, `grep`); `doctor` and anything else exit 2 with a usage message and write no record, and `example` takes no `--license-mode`, so it always sends `license_mode: strict` (`docs/GITHITS.md`, the launcher's entry). `doctor` stays the operator's: `syberlabs githits doctor`. An `example` is held on the host until this host's last one is 40 s old, across processes and arms (`syberlabs/githits_pace.py`). `-I` keeps the worktree's directory and `PYTHONPATH` off the import path, so the candidate's own `syberlabs/` never runs. Claude Code's Bash tool runs bash (Git Bash on Windows), which runs the script. The launcher is named by its absolute path with forward slashes, the exact text the model types; a path holding a space or a quote is refused (`provider_unavailable`). -**The pre-send refusal.** The launcher sends nothing unless the query guard decided on that very call. When the hook passes a launcher command, its decision also carries `arguments_digest`, the SHA-256 of the arguments it read from the command (`query_guard.bash_arguments`), under the same per-call MAC. The launcher's entry, once its lookup is parsed and before it is paced or sent, looks in the guard directory beside itself (`SYBERLABS_GUARD_LAUNCHER`'s parent) for a decision with `rule: pass`, a shell tool, an `arguments_digest` equal to the SHA-256 of its own argv, and a MAC that verifies under `SYBERLABS_GUARD_KEY`, and claims it by creating a file named for its tool use id in `launches/` exclusively, so one decision lets one launch through (`query_guard.claim_launch`). The hook's words and the launcher's argv agree because the hook passes only plain words and quoted strings, which the shell hands over unchanged; a disagreement refuses (fails closed). With no such decision (no hook ran, as under `--bare`; no key or launcher in the environment; a decision forged without the key, for other arguments, or already used) the entry records the call `status: unguarded`, `sent: false` (`githits.Client.refuse`), prints `refused: unguarded: ; no request was sent`, exits 2, and makes no HTTP request. The host then reconciles as before: under `--bare` no launcher call was guarded, so the record is `githits_unexpected_records`, journaled `guard: unguarded`, and the search stops `provider_guard_inactive`, now with the record saying the lookup never left. The entry is the provider's only: the operator's lookups and `doctor` run through `syberlabs githits`, which has no gate (the operator is not a model, and no hook stands in front of them), so a decision is required whenever the launcher runs and never on the operator's path. The limit is the MAC's: a process holding the key (an MCP server, an allowed shell) could still sign a decision (above). +**The pre-send refusal.** The launcher sends nothing unless the query guard decided on that very call. When the hook passes a launcher command, its decision also carries `arguments_digest`, the SHA-256 of the arguments it read from the command (`query_guard.bash_arguments`), under the same per-call MAC. The launcher's entry, once its lookup is parsed and before it is paced or sent, looks in the guard directory beside itself (`SYBERLABS_GUARD_LAUNCHER`'s parent) for a decision with `rule: pass`, a shell tool, an `arguments_digest` equal to the SHA-256 of its own argv, and a MAC that verifies under `SYBERLABS_GUARD_KEY`, and claims it by creating a file named for its tool use id in `launches/` exclusively, so one decision lets one launch through (`query_guard.claim_launch`). The hook's words and the launcher's argv agree because the hook passes only plain words and quoted strings, which the shell hands over unchanged; a disagreement refuses (fails closed). Git Bash on Windows would still rewrite a word that starts with `/` when it starts a Windows program (`/api/v1` arrives as `C:/Program Files/Git/api/v1`), so the launcher exports `MSYS_NO_PATHCONV=1` and `MSYS2_ARG_CONV_EXCL='*'` before its `exec`; other shells ignore both. With no such decision (no hook ran, as under `--bare`; no key or launcher in the environment; a decision forged without the key, for other arguments, or already used) the entry records the call `status: unguarded`, `sent: false` (`githits.Client.refuse`), prints `refused: unguarded: ; no request was sent`, exits 2, and makes no HTTP request. The host then reconciles as before: under `--bare` no launcher call was guarded, so the record is `githits_unexpected_records`, journaled `guard: unguarded`, and the search stops `provider_guard_inactive`, now with the record saying the lookup never left. The entry is the provider's only: the operator's lookups and `doctor` run through `syberlabs githits`, which has no gate (the operator is not a model, and no hook stands in front of them), so a decision is required whenever the launcher runs and never on the operator's path. The limit is the MAC's: a process holding the key (an MCP server, an allowed shell) could still sign a decision (above). **What arm B adds.** Argv: `Bash` in `--tools`, `Bash(:*)` in `--allowedTools`, and the skill after the harness note in `--append-system-prompt`. Environment, for the launcher's client: @@ -175,7 +181,7 @@ exec '' -I '/syberlabs/githits_cli_entry.py' "$@" A record cannot be tied to the tool use that ran it, so the host reconciles counts: authentic records against launcher calls the guard passed. Fewer records is `mismatch: githits_unrecorded` (a usage error, a client that failed before recording, a denied permission; recorded, the call stands). More records than guarded calls is `githits_unexpected_records`: each record of the call is journaled `guard: unguarded`, and the search stops `provider_guard_inactive`, because something holding the key ran the client without the guard. A call that fails or times out registers no candidate, so its mismatch would reach the journal nowhere, and a launcher call still waiting for GitHits when the host ends Claude Code (its session is killed, so the client cannot record) would leave no trace: when such a call has fewer records than guarded launcher calls, its records are journaled `guard: unguarded` (not reconciled), and when it has none, one `githits.unrecorded` fact (the command line's digest, `sent: null`, `guard: unguarded`) stands for each launcher call the guard passed. A `no_change` refusal with missing records journals a synthetic gap without assigning it to a concurrent command; its count remains unknown (`spec/GAPS.md` 59 (12)). -**The skill.** `skills/githits/SKILL.md` adapts GitHits' shipped skills (`githits-com/githits-cli` at `0bc9f5e68b`, `skills/githits-code/SKILL.md` and `skills/githits-mcp/SKILL.md`, Apache-2.0, Copyright 2025 GitHits Inc.; `skills/githits/LICENSE` is the upstream license and `skills/githits/NOTICE` says what changed) for the launcher: when a lookup is worth it (an API or version fact the repository and the standard library do not settle), how to phrase a query (short, naming the package and version, no code or repository text), targets first, search then grep or list then read, results as untrusted data, and citing sources in the final reply (the harness does not commit). Claude Code 2.1.309's `--help` offers `--append-system-prompt`, `--plugin-dir` (skills inside a plugin) and `--add-dir`; the provider uses `--append-system-prompt`, so the text is part of arm B's system prompt on every request rather than a skill the model may or may not load, and arm A's prompt is exactly the harness note. The YAML front matter is left out, and `{launcher}` is replaced by the call's launcher path, so arm B's exact appended text is the harness note, a blank line, and the file's body with that one substitution; the file's SHA-256 is on `signal.harness.skill`, computed with CRLF normalized to LF, so a Windows checkout with `core.autocrlf` records the committed blob's digest (`24c150e4…6508` today) as a POSIX one does. `.gitattributes` (`skills/** text eol=lf`) also checks the skill out with LF on every platform. +**The skill.** `skills/githits/SKILL.md` adapts GitHits' shipped skills (`githits-com/githits-cli` at `0bc9f5e68b`, `skills/githits-code/SKILL.md` and `skills/githits-mcp/SKILL.md`, Apache-2.0, Copyright 2025 GitHits Inc.; `skills/githits/LICENSE` is the upstream license and `skills/githits/NOTICE` says what changed) for the launcher: when a lookup is worth it (an API or version fact the repository and the standard library do not settle), how to phrase a query (short, naming the package and version, no code or repository text), targets first, search then grep or list then read, results as untrusted data, and citing sources in the final reply (the harness does not commit). Claude Code 2.1.309's `--help` offers `--append-system-prompt`, `--plugin-dir` (skills inside a plugin) and `--add-dir`; the provider uses `--append-system-prompt`, so the text is part of arm B's system prompt on every request rather than a skill the model may or may not load, and arm A's prompt is exactly the harness note. The YAML front matter is left out, and `{launcher}` is replaced by the call's launcher path, so arm B's exact appended text is the harness note, a blank line, and the file's body with that one substitution; the file's SHA-256 is on `signal.harness.skill`, computed with CRLF normalized to LF, so a Windows checkout with `core.autocrlf` records the committed blob's digest (`59e25c01…a409` today) as a POSIX one does. `.gitattributes` (`skills/** text eol=lf`) also checks the skill out with LF on every platform. **Not verified here.** No real Claude Code and no real GitHits ran. The tests use a stand-in `claude` whose Bash tool is emulated (a command no `Bash(:*)` rule names is denied; the launcher's `exec` line is run as `sh` would run it) and a loopback fake GitHits; a separate test runs the launcher under a real `sh` when one is on `PATH`. Whether Claude Code 2.1.309 matches `Bash(:*)` against a Windows path with forward slashes, and runs it through Git Bash, is for the first live call: `syberlabs githits doctor`, then one guarded lookup whose record and fact are checked. @@ -184,11 +190,13 @@ A record cannot be tied to the tool use that ran it, so the host reconciles coun Before `--githits`, the pilot's route was an MCP server. With this provider the only difference between those arms is `--mcp-config FILE`: ```bash -syberlabs propose --claude-code --bare --allowed-tools Read,Edit,Write,Glob,Grep,WebSearch,WebFetch,mcp__githits # off -syberlabs propose --claude-code --bare --allowed-tools Read,Edit,Write,Glob,Grep,WebSearch,WebFetch,mcp__githits \ - --mcp-config githits.json # on +syberlabs propose --claude-code --lane api-hooked --trusted-tree --allowed-tools Read,Edit,Write,Glob,Grep,WebSearch,WebFetch,mcp__githits # off +syberlabs propose --claude-code --lane api-hooked --trusted-tree --allowed-tools Read,Edit,Write,Glob,Grep,WebSearch,WebFetch,mcp__githits \ + --mcp-config githits.json # on ``` +These arms used to run on `--bare`, where no hook screens `WebSearch`, `WebFetch` or the server's tools, so `--bare` now refuses both of them unless `--allow-unscreened-bare` is passed (recorded on `signal.guard`); on `api-hooked` the guard's hook runs. + Use the same `--allowed-tools` in both arms; the off arm has no GitHits server to call (`--strict-mcp-config` is always passed), so naming its tools there changes nothing. The two argv vectors differ in the last two items only, apart from the path of each call's own temporary guard settings file; the guard and its policy are the same in both arms (`tests/test_claude_code.py` pins this). The `init` event's MCP servers and their status are recorded in `signal.harness.mcp_servers`, so a reader can see that GitHits connected in the on arm. ## What is recorded, and what is not @@ -200,7 +208,7 @@ Per call, on the candidate's `signal` (all parsed by the host from the stream, b - `tool_usage`: `{web_search, web_fetch, githits, other: {tool: n}}`. GitHits is any `mcp__githits__*` tool; web is `WebSearch`, `WebFetch` and the server tools `web_search` and `web_fetch`. It is bounded by `providers.tool_usage`: at most 16 other names; - `harness`: Claude Code's version, model, effort, the credential source name (`apiKeySource`, never a credential), the MCP servers with their status, and the lane the call ran on (`lane`); - `lane`, `cost_basis`, `estimated_usd`, and `skipped_paths` (how many changed files were binary, not UTF-8 or over 1 MiB, and so left out); -- `guard`: the query guard's policy for the call, `{version, max_lines, max_chars}`; +- `guard`: the query guard's policy for the call, `{version, max_lines, max_chars, policy_digest}`, plus `allow_code_tools_with_mcp` or `allow_unscreened_bare` (`true`) when the operator passed that override; - with `--githits`: `githits` (launcher calls, records, mismatch; above) and `harness.skill`. Per lookup (each GitHits or web tool call), one verified `observed` event, key `lookup`, source and actor `syberlabs.host`, version `..`, value `{tool, query_digest, query_chars, at, ok, guard}`. The provider writes it through the search space (`SearchSpace.observe`; `GitSearchSpace.observe` records it with `Session.observe` and accepts only the key `lookup`, a value of at most 4096 bytes and a short version), so a provider built in Python keeps its lookups in the journal; there is no callback from the CLI any more. `query_digest` is the SHA-256 of the query string: `query` for WebSearch and GitHits search, `url` for WebFetch, otherwise the tool input as canonical JSON. `at` is when the host read the line (the stream carries no time of its own). `ok` is false when the stream holds no successful result for that call, or the guard blocked it. `guard` is `pass`, a rule name or `unguarded` (the query guard, above). A call's lookups are recorded before its candidate. diff --git a/docs/EXPERIMENTS.md b/docs/EXPERIMENTS.md index c8a7da59..a8b584a0 100644 --- a/docs/EXPERIMENTS.md +++ b/docs/EXPERIMENTS.md @@ -145,12 +145,12 @@ The pilot itself runs a real model with a paid key: Class X (`AGENTS.md`). A nam ```bash syberlabs start "Parse ISO dates with the UTC offset" --paths src tests --unattended syberlabs experiment label --name githits-pilot --item LIB-1 --arm off --tag needs_lookup -syberlabs propose --claude-code --bare --allowed-tools Read,Edit,Write,Glob,Grep,WebSearch,WebFetch +syberlabs propose --claude-code --lane api-hooked --trusted-tree --allowed-tools Read,Edit,Write,Glob,Grep,WebSearch,WebFetch syberlabs run --once # one safe step per thread; here, accepting the passing candidate syberlabs start "Parse ISO dates with the UTC offset" --paths src tests --unattended syberlabs experiment label --name githits-pilot --item LIB-1 --arm on --tag needs_lookup -syberlabs propose --claude-code --bare --allowed-tools Read,Edit,Write,Glob,Grep,WebSearch,WebFetch \ +syberlabs propose --claude-code --lane api-hooked --trusted-tree --allowed-tools Read,Edit,Write,Glob,Grep,WebSearch,WebFetch \ --githits --githits-arm on # the only difference between the arms: the launcher and the skill syberlabs run --once ``` diff --git a/spec/GAPS.md b/spec/GAPS.md index 28ce3d0e..c028fcde 100644 --- a/spec/GAPS.md +++ b/spec/GAPS.md @@ -126,3 +126,4 @@ The schemas describe SyberWork as it behaves. They are not enforced at runtime e 63. **The GitHits pilot's example pacing and license reading rest on GitHits' documentation, not on observed traffic.** (1) The host paces `example` calls to one per 40 s per host (`syberlabs/githits_pace.py`, called by the launcher entry); GitHits' limit is per organization, so two hosts, or an operator's own `syberlabs githits example` (not paced), share the organization's budget without seeing each other's slots. (2) A loopback base URL is not paced unless `SYBERLABS_GITHITS_PACE_SECONDS` asks: a loopback proxy that forwards to GitHits would go unpaced. (3) Calls queued behind several others wait one interval each, so a Bash tool timeout shorter than the queue ends a call before it is sent, and the host then flags it `githits_unrecorded`. (4) The time file is plain text a process of the same user can rewrite; it paces cooperating processes and is no security boundary. (5) The license reading admits an example's `References`/`Sources` links by their `(SPDX)` and passes an example with neither code (a fenced, indented or HTML block) nor a source; inline code in prose is not read as code, and the three documented shapes (`complete`, `degraded`, `no_example` in the pinned OpenAPI document) are the only evidence of what real responses look like until the first live example is recorded. (6) The engineering smoke pins Claude Code 2.1.312 exactly because the host and the pilot VM run it; that version's flags were not re-checked here. 64. **The GitHits launcher fails closed before the network, and the treatment runs on a hooked API lane; neither is verified live, and other lookups on a `--bare` lane are still caught only after the fact.** The live canary of 2026-10-30 (Claude Code 2.1.312 on a Freestyle Linux VM) settled 56 (2): a `--settings` `PreToolUse` hook does not run under `--bare`, so `propose --claude-code --bare --githits` let the model run the launcher with no guard, and the client sent the lookup (`sent: true`) before the host stopped the run `provider_guard_inactive`. Now the launcher's entry sends nothing without claiming the hook's MAC-verified decision for its exact argv (`query_guard.claim_launch`; refused calls are recorded `status: unguarded`, `sent: false`), and `--githits` refuses `--bare` and runs on `--lane api-hooked` (the API key without `--bare`, a fresh `CLAUDE_CONFIG_DIR` and home per call, `--disable-slash-commands`; `docs/CLAUDE_CODE.md`). Not solved: (1) a decision is bound to the launcher's arguments (their SHA-256) and claimed once, not to a tool use id, which the launcher cannot see; a process holding the per-call key (a stdio MCP server, a shell under `--allow-code-tools-with-mcp`) can still sign a decision for its own arguments, as it could before; (2) the hook's words and the launcher's argv agree for plain words and quoted strings under a POSIX shell; a shell or platform that hands over different bytes (Git Bash's argument conversion to a Windows process, a non-UTF-8 byte) refuses a legitimate lookup (fail closed), which no live run has exercised; (3) WebSearch, WebFetch and `--mcp-config` (`mcp__githits__*`) lookups on a `--bare` lane have no pre-send gate: the hook does not run, so they still leave and are flagged `unguarded` afterwards; only `--githits` refuses `--bare`, and `api-hooked` or a login lane is where those tools are guarded; (4) `api-hooked` has run only against the stand-in `claude`: whether 2.1.312 with an empty config directory and home starts headless on `ANTHROPIC_API_KEY` (`apiKeySource`) and runs the hook is for its first live call, and the flags were read from 2.1.309's `--help` because 2.1.312's was not on the building machine; managed (policy) settings still load, `--setting-sources` has no value meaning `--settings` alone, `USERPROFILE` moves only on Windows, and the worktree's memory files (`CLAUDE.md`) are read in both arms, so the lane needs `--trusted-tree`; (5) the example license parser now reads any ATX or bold heading that names a source or reference ("## Source code" included) as a sources section and a fenced or indented block inside a blockquote as code, both of which can only drop more examples (fail closed); how often that drops a usable GitHits example is not measured; (6) the default deny-list (`SyberLabs`, `sykosyber`, `pypi:syberwork`, `deepwiki.com`, `deepwiki.org`) detects a fork of a SyberLabs repository only by its name (`githits.FORK_NAMES`) in a result's URL or a target, so a renamed fork, a vendored copy, or SyberLabs code mirrored under another name and site still passes, and no mirror or front-end other than DeepWiki is listed. +65. **`--bare` refuses the lookups no hook would screen there, the launcher keeps Git Bash from rewriting its arguments, and the eval spec refuses what the provider refuses; none of it is verified live.** `guard-lane-2` (2026-11-01) answers #155's verification. Built: a `--bare` provider whose allowed tools include `WebSearch` or `WebFetch` (`claude_code.web_tools`), or whose `--mcp-config` names an MCP server, is refused `invalid_provider` when built and pointed to `--lane api-hooked`, because Claude Code 2.1.312 runs no `--settings` hook under `--bare` and such a lookup would leave before it is flagged `unguarded`; `--allow-unscreened-bare` overrides and is recorded as `signal.guard.allow_unscreened_bare` (64 (3) is closed for every run without the override); the launcher exports `MSYS_NO_PATHCONV=1` and `MSYS2_ARG_CONV_EXCL='*'` before its `exec`, so Git Bash hands a word starting with `/` to the host's Python as typed (64 (2), for that conversion); `eval/arms.check` refuses, at spec validation, `githits` with `bare`, `bare` with `api-hooked`, `api` without `bare`, and `bare` with a web tool or any `mcp_config`; the cross-process pacing test asserts on the slot stamps each request took, not on arrival times. Not solved: (1) the override exists because the historical MCP A/B ran on `--bare`; a run that passes it is caught only after the fact, as before; (2) the eval spec has no override, and it refuses every `mcp_config` on `bare`, also one naming no server, which the provider would build; (3) the two variables are read from Git for Windows' and MSYS2's documentation and exercised against the Git Bash on the building machine and CI's Windows runner, not under Claude Code's own Bash tool on Windows, which is for the first live Windows call; a non-UTF-8 byte or another shell's conversion still refuses (fails closed); (4) a server tool Claude Code calls for itself (`web_search`, `web_fetch`) is refused on `--bare` only when its name is in `--allowed-tools`. diff --git a/syberlabs/claude_code.py b/syberlabs/claude_code.py index ef214f6c..1ae22bb8 100644 --- a/syberlabs/claude_code.py +++ b/syberlabs/claude_code.py @@ -2,8 +2,8 @@ syberlabs propose --claude-code --trusted-tree # the operator's login (subscription_local) syberlabs propose --claude-code --bare # an API key in the environment (api) - syberlabs propose --claude-code --bare --mcp-config githits.json # the same, with one more MCP server syberlabs propose --claude-code --lane api-hooked --trusted-tree # an API key, without --bare: hooks run + syberlabs propose --claude-code --lane api-hooked --trusted-tree --mcp-config githits.json # one more MCP server syberlabs propose --claude-code --lane api-hooked --trusted-tree --githits # the same, with the GitHits launcher ``ClaudeCodeProvider`` is ``providers.RepairProvider`` with one call replaced: instead of a JSON adapter, each call @@ -79,6 +79,14 @@ prove. ``allow_code_tools_with_mcp`` (``--allow-code-tools-with-mcp``) overrides, and is recorded as ``signal.guard.allow_code_tools_with_mcp``. +Unscreened lookups. Under ``--bare`` no ``--settings`` hook runs (``BARE_HOOKS``), so no lookup is screened before +it leaves: a ``WebSearch`` or ``WebFetch`` call, or a call to a server ``--mcp-config`` names (``mcp__githits__*``), +is sent and flagged ``unguarded`` only afterwards. A ``--bare`` provider whose allowed tools include ``WebSearch`` or +``WebFetch`` (``web_tools``), or whose ``--mcp-config`` names an MCP server (``names_mcp_server``), is therefore +refused when it is built (``invalid_provider``), pointing to ``--lane api-hooked``, where the hook runs. +``allow_unscreened_bare`` (``--allow-unscreened-bare``) overrides, and is recorded as +``signal.guard.allow_unscreened_bare``. + Credential lanes (bring your own credential: SyberLabs never stores, pools or forwards a credential; the operator's own Claude Code runs under their own login, or with their API key). ``api`` (``--bare``, the default there) is ``billed``: the tokens are priced under the contract's ``budget.prices`` through the host's meter, as an @@ -271,6 +279,13 @@ def code_tools(tools: Sequence[str]) -> list[str]: return [name for name in built_in_tools(tools) if name.lower() in named] +def web_tools(tools: Sequence[str]) -> list[str]: + """The web lookup tools (``WEB_TOOLS``: ``WebSearch``, ``WebFetch`` and the server tools' names) among + ``--allowedTools`` entries, a specifier dropped, compared ignoring case.""" + named = {name.lower() for name in WEB_TOOLS} + return [name for name in built_in_tools(tools) if name.lower() in named] + + def names_mcp_server(path: Path) -> bool: """False only for a JSON object whose ``mcpServers`` is absent or an empty object; a file that cannot be read or parsed, or any other shape, counts as naming a server (fail closed).""" @@ -311,7 +326,12 @@ def githits_skill() -> tuple[str, str]: def launcher_text(python: str, entry: Path) -> str: """The launcher: a POSIX shell script (Claude Code's Bash tool runs bash, Git Bash on Windows) that replaces - itself with the host's Python running the host checkout's entry under ``-I``, with the call's arguments.""" + itself with the host's Python running the host checkout's entry under ``-I``, with the call's arguments. It + first turns off Git Bash's (MSYS2's) argument conversion: handing a word that starts with ``/`` to a Windows + program, MSYS2 rewrites it as a Windows path (``/api/v1`` becomes ``C:/Program Files/Git/api/v1``), so the + launcher's argv would differ from the words the query guard's hook digested, and an honest lookup would be + refused. ``MSYS_NO_PATHCONV`` (Git for Windows) and ``MSYS2_ARG_CONV_EXCL='*'`` (MSYS2) both mean "convert + nothing"; any other shell ignores them.""" def quoted(path) -> str: text = Path(path).as_posix() if "'" in text or "\n" in text: @@ -319,6 +339,8 @@ def quoted(path) -> str: return "'" + text + "'" return ("#!/bin/sh\n# SyberLabs GitHits launcher for one Claude Code call: written by the host, read-only, removed " "after the call.\n# It runs the host checkout's GitHits client, never the candidate worktree's.\n" + "# Git Bash would rewrite an argument starting with / for a Windows program; pass every word as typed.\n" + "export MSYS_NO_PATHCONV=1 MSYS2_ARG_CONV_EXCL='*'\n" f"exec {quoted(python)} -I {quoted(entry)} \"$@\"\n") @@ -653,7 +675,7 @@ def __init__(self, repo: Repo | str | Path, *, bare: bool = False, trusted_tree: name: str = "claude-code", revision: str = "1", timeout: int = 1800, max_calls: int | None = None, guard_max_lines: int | None = None, guard_max_chars: int | None = None, allow_code_tools_with_mcp: bool = False, githits: bool = False, arm: str = "b", - thread_label: str | None = None): + thread_label: str | None = None, allow_unscreened_bare: bool = False): lane = lane or ("api" if bare else "subscription_local") if lane not in LANES: raise Rejected("invalid_provider", "lane must be one of " + ", ".join(LANES)) @@ -727,6 +749,16 @@ def __init__(self, repo: Repo | str | Path, *, bare: bool = False, trusted_tree: "inherits (the MCP server holds it already), and a code-running tool can read it and " "sign a forged decision. Drop the tool, or pass --allow-code-tools-with-mcp to " "accept that (recorded in signal.guard)") + if bare and not allow_unscreened_bare: + unscreened = web_tools(tools) + (["--mcp-config naming an MCP server"] + if mcp_config is not None and names_mcp_server(mcp_config) else []) + if unscreened: + raise Rejected("invalid_provider", + f"refusing {', '.join(unscreened)} with --bare: {BARE_HOOKS}. Under --bare the query " + "guard screens no lookup before it leaves; it is flagged unguarded only afterwards. " + f"Run these lookups where the guard's hook runs: --lane {HOOKED} --trusted-tree (the " + "API key), or a login lane. Or pass --allow-unscreened-bare to accept unscreened " + "lookups (recorded in signal.guard)") try: guard = query_guard.policy(guard_max_lines, guard_max_chars) except ValueError as exc: @@ -738,6 +770,7 @@ def __init__(self, repo: Repo | str | Path, *, bare: bool = False, trusted_tree: self.mcp_config, self.allowed_tools, self.max_turns = mcp_config, tools, max_turns self.guard = guard # host policy: {version, max_lines, max_chars} self.allow_code_tools_with_mcp = bool(allow_code_tools_with_mcp) # the operator's override, on the signal + self.allow_unscreened_bare = bool(allow_unscreened_bare) # the operator's override, on the signal self.lookups: list[dict] = [] # every lookup of every call, as recorded (digests, never query text) self.version = self._installed_version() @@ -830,7 +863,8 @@ def githits_environment(self, records: Path, record_key: str, session: str) -> d (fresh per call), the host's example pacing directory (``SYBERLABS_GITHITS_PACE_DIR``: the operator's, else the host's temporary-directory default, resolved here, so a launcher whose ``TMPDIR`` or home differs still shares one pacing slot), the session and agent headers, and the license and exclusion policy as - explicit lists (the operator's, else the defaults; ``SYBERLABS_GITHITS_EXCLUDE`` only adds to ``SyberLabs``). + explicit lists (the operator's, else the defaults; ``SYBERLABS_GITHITS_EXCLUDE`` adds to the default + deny-list, ``githits.DEFAULT_EXCLUDE``, and never replaces it here). The token is the operator's own ``GITHITS_API_TOKEN``, inherited, never written.""" licenses = githits_client.LicensePolicy.from_env(os.environ) exclude = githits_client.Exclusions.from_env( @@ -984,7 +1018,8 @@ def _call(self, space: SearchSpace, request: dict, parents: list[str]): signal = {"lane": self.lane, "cost_basis": self.cost_basis, "harness": {**transcript.harness(), "installed_version": self.version, "lane": self.lane}, "guard": {**self.guard, "policy_digest": hook_env[query_guard.POLICY_DIGEST_ENV], - **({"allow_code_tools_with_mcp": True} if self.allow_code_tools_with_mcp else {})}} + **({"allow_code_tools_with_mcp": True} if self.allow_code_tools_with_mcp else {}), + **({"allow_unscreened_bare": True} if self.allow_unscreened_bare else {})}} if githits_signal is not None: signal["harness"]["skill"] = {"path": SKILL_NAME, "sha256": self.skill[1]} signal["githits"] = githits_signal diff --git a/syberlabs/cli.py b/syberlabs/cli.py index 6f78a6de..372f5e1f 100644 --- a/syberlabs/cli.py +++ b/syberlabs/cli.py @@ -176,6 +176,11 @@ def main(argv: list[str] | None = None) -> int: help="for --claude-code: allow Bash or PowerShell in --allowed-tools although --mcp-config " "names an MCP server (refused otherwise: the guard's key is in Claude Code's " "environment); recorded in signal.guard") + propose.add_argument("--allow-unscreened-bare", action="store_true", + help="for --claude-code --bare: allow WebSearch or WebFetch in --allowed-tools, or an " + "--mcp-config naming an MCP server (refused otherwise: no --settings hook runs under " + "--bare, so those lookups leave unscreened; use --lane api-hooked); recorded in " + "signal.guard") propose.add_argument("--githits", action="store_true", help="for --claude-code: GitHits through the API client (arm B of the pilot): a per-call " "launcher as the only Bash command, guarded, its records journaled as lookups, and the " @@ -638,6 +643,7 @@ def _run(kit: Kit, args) -> int: max_calls=args.max_calls, guard_max_lines=args.guard_max_lines, guard_max_chars=args.guard_max_chars, allow_code_tools_with_mcp=args.allow_code_tools_with_mcp, + allow_unscreened_bare=args.allow_unscreened_bare, githits=args.githits, arm=args.githits_arm, thread_label=thread.id if args.githits else None) found = thread.propose(provider, seed=args.seed) diff --git a/syberlabs/eval/arms.py b/syberlabs/eval/arms.py index 57d3c558..45d39000 100644 --- a/syberlabs/eval/arms.py +++ b/syberlabs/eval/arms.py @@ -14,6 +14,11 @@ ``max_turns``, ``mcp_config`` and ``executable`` (a stand-in ``claude`` for tests), and ``githits`` passed through by name to the provider's keyword of that name; a ``githits`` arm's lookups carry the run's thread and the arm's name (``X-GitHits-Session-ID``, ``-Agent``). + ``check`` refuses at spec validation what the provider would refuse when built: ``githits`` with ``bare`` (no + ``--settings`` hook runs under ``--bare``), ``bare`` with lane ``api-hooked`` (that lane is the API key without + ``--bare``), lane ``api`` without ``bare`` (use ``api-hooked``), and, as the spec has no override, ``bare`` with + ``WebSearch`` or ``WebFetch`` in ``tools`` or with any ``mcp_config`` (those lookups would leave unscreened; the + provider builds an ``mcp_config`` naming no server, which an arm has no use for). The runner does not set Claude Code's model: the arm's ``model`` and ``effort`` are the spec's declaration, and the model that served is in the candidate's ``signal.harness``. @@ -70,13 +75,34 @@ def check(arm: dict, budget: dict) -> list[str]: errors += [f"{key}: only for evolve arms" for key in EVOLVE_ONLY if key in arm] if kind == "single" and arm.get("max_calls", 1) != 1: errors.append("max_calls: a single arm makes one call") - if kind == "claude-code" and arm.get("bare") and arm.get("lane") in ("subscription_local", "seat_team_enterprise"): - errors.append("lane: --bare never reads a Claude login") + if kind == "claude-code": + errors += claude_code_lane(arm) if "max_calls" in arm and arm["max_calls"] > budget["calls"]: errors.append(f"max_calls: above the spec's budget.calls ({budget['calls']})") return errors +def claude_code_lane(arm: dict) -> list[str]: + """The provider's lane refusals (``claude_code.ClaudeCodeProvider``) for a claude-code arm, as spec errors.""" + from syberlabs.claude_code import HOOKED, web_tools + bare, lane, errors = arm.get("bare") is True, arm.get("lane"), [] + if bare and lane in ("subscription_local", "seat_team_enterprise"): + errors.append("lane: --bare never reads a Claude login") + if bare and lane == HOOKED: + errors.append(f"lane: {HOOKED} is the API-key lane without --bare, so the query guard's hook runs; drop bare") + if not bare and lane == "api": + errors.append(f"lane: the api lane is --bare; for the API key without --bare use {HOOKED}") + if bare and arm.get("githits") is True: + errors.append(f"githits: refused with bare, where no --settings hook runs; use lane {HOOKED} with trusted_tree") + if bare and web_tools(arm.get("tools") or []): + errors.append(f"tools: {', '.join(web_tools(arm['tools']))} with bare would leave unscreened (no hook runs " + f"under --bare); use lane {HOOKED} with trusted_tree") + if bare and "mcp_config" in arm: + errors.append(f"mcp_config: an MCP server's lookups with bare would leave unscreened (no hook runs under " + f"--bare); use lane {HOOKED} with trusted_tree") + return errors + + def arm_digest(arm: dict) -> str: return digest(arm) diff --git a/tests/test_claude_code.py b/tests/test_claude_code.py index 7e674192..01ea56fd 100644 --- a/tests/test_claude_code.py +++ b/tests/test_claude_code.py @@ -37,7 +37,7 @@ from syberlabs.claude_code import (BIN_ENV, CODE_TOOLS, DEFAULT_TOOLS, GITHITS_ENTRY, HOOKED, LOGIN_LANE_DROPPED_ENV, LOOKUP, SKILL_PATH, THIRD_PARTY_SWITCHES, ClaudeCodeProvider, Transcript, code_tools, githits_skill, instruction_file, parse_tools, query_text, version_tuple, - write_launcher) + web_tools, write_launcher) from syberlabs.cli import main as cli from syberlabs.events import verify_events from syberlabs.gitspace import IDENTITY @@ -839,8 +839,9 @@ def test_two_arms_differ_only_in_the_mcp_config_flags(self): config = self.root.parent / "githits.json" config.write_text(json.dumps({"mcpServers": {"githits": {"command": "githits-stand-in"}}}), encoding="utf-8") claude = self.stand_in("ab") - off = ClaudeCodeProvider(kit.repo, bare=True, executable=claude) - on = ClaudeCodeProvider(kit.repo, bare=True, executable=claude, mcp_config=config) + # guard-lane-2: an MCP server on --bare is unscreened, so this A/B needs the operator's recorded override. + off = ClaudeCodeProvider(kit.repo, bare=True, executable=claude, allow_unscreened_bare=True) + on = ClaudeCodeProvider(kit.repo, bare=True, executable=claude, mcp_config=config, allow_unscreened_bare=True) self.assertEqual(on.command(), off.command() + ["--mcp-config", str(config.resolve())]) command = off.command() for flag in ("-p", "--strict-mcp-config", "--no-session-persistence"): @@ -883,7 +884,7 @@ def test_every_call_is_restricted_and_names_its_built_in_tools(self): self.assertEqual(command[command.index("--tools") + 1], ",".join(DEFAULT_TOOLS)) self.assertNotIn("Bash", DEFAULT_TOOLS, "no shell by default: a shell could read the guard's key") self.assertEqual(command[command.index("--permission-mode") + 1], "dontAsk") - wide = ClaudeCodeProvider(kit.repo, bare=True, executable=VERSION_ONLY, + wide = ClaudeCodeProvider(kit.repo, bare=True, executable=VERSION_ONLY, allow_unscreened_bare=True, # WebFetch allowed_tools=["Read", "Edit", "WebFetch", "Bash(git log *)", "mcp__githits"]).command() self.assertEqual(wide[wide.index("--tools") + 1], "Read,Edit,WebFetch,Bash", "built-ins only (--tools does not affect MCP tools), a rule's specifier dropped") @@ -947,7 +948,8 @@ def test_a_code_running_tool_beside_an_mcp_server_is_refused_and_says_why(self): # No server named, or no code-running tool: built as before. for config, tools in ((None, ["Read", "Bash"]), (self.mcp_file("empty.json", {}), ["Read", "Bash"]), (githits, ["Read", "Edit", "WebFetch", "mcp__githits"])): - ClaudeCodeProvider(kit.repo, bare=True, executable=VERSION_ONLY, mcp_config=config, allowed_tools=tools) + ClaudeCodeProvider(kit.repo, bare=True, executable=VERSION_ONLY, mcp_config=config, allowed_tools=tools, + allow_unscreened_bare=True) # guard-lane-2's refusal is tested in UnscreenedBare def test_the_override_builds_it_and_is_recorded_in_signal_guard(self): kit = self.kit() @@ -955,7 +957,8 @@ def test_the_override_builds_it_and_is_recorded_in_signal_guard(self): claude = self.stand_in("override") thread = kit.start("Add a CSV export", paths=["src/", "tests/"]) provider = self.provider(kit, thread, claude, bare=True, mcp_config=githits, max_calls=1, - allowed_tools=["Read", "Edit", "Bash", "mcp__githits"], allow_code_tools_with_mcp=True) + allowed_tools=["Read", "Edit", "Bash", "mcp__githits"], allow_code_tools_with_mcp=True, + allow_unscreened_bare=True) [candidate] = thread.propose(provider) self.assertIs(candidate.signal["guard"]["allow_code_tools_with_mcp"], True) self.assertEqual(candidate.signal["guard"]["policy_digest"], POLICY_DIGEST, "policy.json itself is unchanged") @@ -966,6 +969,83 @@ def test_the_override_builds_it_and_is_recorded_in_signal_guard(self): self.assertNotIn("allow_code_tools_with_mcp", plain.signal["guard"], "recorded only when the operator overrides") +class UnscreenedBare(ClaudeCodeCase): + """guard-lane-2, M1 of #155's verification: Claude Code 2.1.312 runs no ``--settings`` hook under ``--bare``, so + a ``WebSearch`` or ``WebFetch`` lookup, or a call to a server ``--mcp-config`` names, leaves unscreened and is + flagged ``unguarded`` only afterwards. A ``--bare`` provider allowing any of them is refused when built, unless + the operator passes the recorded override; ``--lane api-hooked`` is where they are guarded. On f466967 each of + these providers was built without a word.""" + + def mcp_file(self, name, servers): + path = self.root.parent / name + path.write_text(json.dumps({"mcpServers": servers}) if servers is not None else "{not json", encoding="utf-8") + return path + + def test_bare_refuses_web_tools_and_an_mcp_server_and_points_to_api_hooked(self): + kit = self.kit() + githits_mcp = self.mcp_file("githits.json", {"githits": {"command": "githits-stand-in"}}) + broken = self.mcp_file("broken.json", None) + for kwargs in ({"allowed_tools": ["Read", "Edit", "WebSearch"]}, {"allowed_tools": ["Read", "WebFetch"]}, + {"allowed_tools": ["Read", "websearch"]}, {"allowed_tools": ["Read", "WebFetch(domain:pypi.org)"]}, + {"mcp_config": githits_mcp}, {"mcp_config": broken}, + {"mcp_config": githits_mcp, "allowed_tools": ["Read", "Edit", "mcp__githits"]}, + {"lane": "provider_3p", "allowed_tools": ["Read", "WebSearch"]}): + with self.subTest(kwargs=kwargs), self.assertRaises(Rejected) as caught: + ClaudeCodeProvider(kit.repo, bare=True, executable=VERSION_ONLY, **kwargs) + self.assertEqual(caught.exception.code, "invalid_provider") + for words in ("with --bare", "does not run --settings PreToolUse hooks under --bare", + "flagged unguarded only afterwards", f"--lane {HOOKED} --trusted-tree", + "--allow-unscreened-bare"): + self.assertIn(words, caught.exception.detail, kwargs) + self.assertEqual(web_tools(["Read", "WebSearch", "webfetch(x)", "WebSearchX", "mcp__githits__search", "Bash"]), + ["WebSearch", "webfetch"]) + # No web tool and no server named: --bare is built as before. On api-hooked the hook screens them. + for kwargs in ({}, {"allowed_tools": ["Read", "Edit", "mcp__githits"]}, {"mcp_config": self.mcp_file("e.json", {})}): + ClaudeCodeProvider(kit.repo, bare=True, executable=VERSION_ONLY, **kwargs) + with mock.patch.dict(os.environ, {"ANTHROPIC_API_KEY": "test-placeholder"}): + for name in THIRD_PARTY_SWITCHES: + os.environ.pop(name, None) + hooked = ClaudeCodeProvider(kit.repo, lane=HOOKED, trusted_tree=True, executable=VERSION_ONLY, + mcp_config=githits_mcp, allowed_tools=["Read", "WebSearch", "WebFetch"]) + self.assertFalse(hooked.bare) + self.assertFalse(hooked.allow_unscreened_bare) + + def test_the_override_builds_it_and_is_recorded_in_signal_guard(self): + kit = self.kit() + githits_mcp = self.mcp_file("githits.json", {"githits": {"command": "githits-stand-in"}}) + claude = self.stand_in("unscreened") + thread = kit.start("Add a CSV export", paths=["src/", "tests/"]) + provider = self.provider(kit, thread, claude, bare=True, mcp_config=githits_mcp, max_calls=1, + allowed_tools=["Read", "Edit", "WebSearch", "mcp__githits"], allow_unscreened_bare=True) + [candidate] = thread.propose(provider) + self.assertIs(candidate.signal["guard"]["allow_unscreened_bare"], True) + [run] = self.runs("unscreened") + self.assertIn("--bare", run["argv"]) + self.assertNotIn("--allow-unscreened-bare", run["argv"], "the host's switch, never Claude Code's") + other = kit.start("Add a CSV export (no override)", paths=["src/", "tests/"]) + [plain] = other.propose(self.provider(kit, other, claude, bare=True, max_calls=1)) + self.assertNotIn("allow_unscreened_bare", plain.signal["guard"], "recorded only when the operator overrides") + + def test_no_documented_bare_invocation_runs_unscreened_lookups_without_the_override(self): + """docs/CLAUDE_CODE.md (:7 and the MCP A/B block) and docs/EXPERIMENTS.md showed ``--bare`` with + ``--mcp-config`` or ``WebSearch,WebFetch`` as plain usage. Every documented ``syberlabs propose`` line (a + continuation joined) that combines ``--bare`` with either must carry the override.""" + root = Path(claude_code.__file__).resolve().parents[1] + sources = {path: path.read_text(encoding="utf-8") for path in sorted((root / "docs").glob("*.md"))} + sources[Path(claude_code.__file__)] = claude_code.__doc__ + seen = 0 + for path, text in sources.items(): + for line in re.sub(r"\\\n\s*", " ", text).splitlines(): + command = line.split(" #", 1)[0].strip() # a trailing shell comment is prose + if not command.startswith("syberlabs propose --claude-code") or not re.search(r"--bare\b", command): + continue + seen += 1 + unscreened = "--mcp-config" in command or re.search(r"\bWeb(Search|Fetch)\b", command) + if unscreened: + self.assertIn("--allow-unscreened-bare", command, f"{path.name}: {command.strip()}") + self.assertGreater(seen, 3, "the docs still show --bare invocations to check") + + def launcher_lookup(command: str) -> list: """A ``Bash`` lookup through the launcher: ``{launcher}`` is replaced by the stand-in with the path the provider allowed.""" @@ -1140,7 +1220,8 @@ def test_an_mcp_config_run_without_githits_keeps_the_operators_githits_variables config.write_text(json.dumps({"mcpServers": {"githits": {"command": "githits-stand-in"}}}), encoding="utf-8") operator = {k: v for k, v in os.environ.items() if claude_code.githits_variable(k)} self.assertEqual(set(operator), {githits.TOKEN_ENV, githits.URL_ENV, githits.LICENSES_ENV, *OPERATOR_GITHITS}) - mcp_only = ClaudeCodeProvider(kit.repo, bare=True, executable=VERSION_ONLY, mcp_config=config) + mcp_only = ClaudeCodeProvider(kit.repo, bare=True, executable=VERSION_ONLY, mcp_config=config, + allow_unscreened_bare=True) self.assertFalse(mcp_only.githits, "no --githits: the MCP route only") kept = {k: v for k, v in mcp_only.environment().items() if claude_code.githits_variable(k)} self.assertEqual(kept, operator, "the operator's GitHits variables, the token included, unchanged") @@ -1343,10 +1424,37 @@ def test_the_launcher_is_read_only_outside_the_worktree_and_runs_the_host_client with tempfile.TemporaryDirectory(prefix="with space ") as folder, self.assertRaises(Rejected): write_launcher(Path(folder)) + def test_the_launcher_hands_words_starting_with_a_slash_over_unchanged(self): + """guard-lane-2, L1 of #155's verification: Git Bash rewrites an argument that starts with ``/`` when it + starts a Windows program (``/api/v1`` became ``C:/Program Files/Git/api/v1``), so the launcher's argv and + the hook's digest differed and an honest lookup was refused. The launcher now exports MSYS_NO_PATHCONV=1 + and MSYS2_ARG_CONV_EXCL='*' before its exec. Run under a real ``sh`` when one is on PATH (Git Bash's on + Windows, where the conversion happens).""" + text = claude_code.launcher_text(sys.executable, GITHITS_ENTRY) + lines = text.splitlines() + export = lines.index("export MSYS_NO_PATHCONV=1 MSYS2_ARG_CONV_EXCL='*'") + self.assertLess(export, next(i for i, line in enumerate(lines) if line.startswith("exec ")), "before the exec") + sh = shutil.which("sh") + if not sh: + self.skipTest("no sh on PATH") + with tempfile.TemporaryDirectory() as folder: + entry = Path(folder) / "argv_entry.py" + entry.write_text("import json, sys\nprint(json.dumps(sys.argv[1:]))\n", encoding="utf-8") + launcher = write_launcher(Path(folder), entry=entry) + words = ["grep", "/api/v1", "--target", "pypi:requests@2.32.3", "/usr/lib"] + env = {k: v for k, v in os.environ.items() if k not in ("MSYS_NO_PATHCONV", "MSYS2_ARG_CONV_EXCL")} + done = subprocess.run([sh, "-c", launcher + " " + " ".join(words)], capture_output=True, timeout=120, + env=env) + os.chmod(launcher, stat.S_IREAD | stat.S_IWRITE) # so the temporary directory can be removed on Windows + self.assertEqual(done.returncode, 0, done.stderr[-400:]) + self.assertEqual(json.loads(done.stdout.decode("utf-8")), words, "the words the hook digested, unchanged") + def test_the_skill_is_the_adapted_apache_text_and_carries_its_license(self): template, digest = githits_skill() self.assertEqual(digest, hashlib.sha256(SKILL_PATH.read_bytes().replace(b"\r\n", b"\n")).hexdigest()) self.assertEqual(digest, SKILL_SHA256, "the committed blob's digest") + docs = (Path(claude_code.__file__).resolve().parents[1] / "docs" / "CLAUDE_CODE.md").read_text(encoding="utf-8") + self.assertIn(f"`{SKILL_SHA256[:8]}…{SKILL_SHA256[-4:]}`", docs, "guard-lane-2: docs quote today's digest") self.assertFalse(template.startswith("---"), "the front matter stays out of the prompt") self.assertIn("{launcher}", template) for words in ("Look up only when a failing check", "Never before a check has failed", "installed version", @@ -1562,6 +1670,27 @@ def test_the_code_tools_refusal_and_its_override_from_the_terminal(self): [registered] = [e["body"] for e in events if e.get("kind") == "candidate_registered"] self.assertIs(registered["signal"]["guard"]["allow_code_tools_with_mcp"], True) + def test_the_unscreened_bare_refusal_and_its_override_from_the_terminal(self): + """guard-lane-2, M1: ``--bare`` with WebSearch is refused before anything runs; ``--allow-unscreened-bare`` + builds it and the candidate records the override.""" + self.assertEqual(self.run_cli("init")[0], 0) + self.assertEqual(self.run_cli("start", "Add a CSV export for report rows", "--paths", "src", "tests")[0], 0) + env = {BIN_ENV: " ".join(self.stand_in("cli-bare-web"))} + flags = ("propose", "--claude-code", "--bare", "--max-calls", "1", "--allowed-tools", "Read,Edit,WebSearch") + code, _, err = self.run_cli(*flags, env=env) + self.assertEqual(code, 2) + self.assertIn("invalid_provider", err) + self.assertIn("--allow-unscreened-bare", err) + self.assertIn(f"--lane {HOOKED}", err) + self.assertEqual(self.runs("cli-bare-web"), [], "refused before anything ran") + code, out, err = self.run_cli(*flags, "--allow-unscreened-bare", env=env) + self.assertEqual(code, 0, out + err) + exported = self.root.parent / "cli-bare-web-export.json" + self.assertEqual(self.run_cli("export", "--out", str(exported))[0], 0) + events = json.loads(exported.read_text(encoding="utf-8"))["events"] + [registered] = [e["body"] for e in events if e.get("kind") == "candidate_registered"] + self.assertIs(registered["signal"]["guard"]["allow_unscreened_bare"], True) + if __name__ == "__main__": unittest.main() diff --git a/tests/test_eval_core.py b/tests/test_eval_core.py index 63d181b5..d1c0a8da 100644 --- a/tests/test_eval_core.py +++ b/tests/test_eval_core.py @@ -374,6 +374,44 @@ def test_claude_code_passes_its_options_and_githits_by_name(self): self.assertTrue(arms.check(dict(arm, adapter=SIM), self.BUDGET)) self.assertTrue(arms.check(dict(arm, bare=True), self.BUDGET), "--bare with a login lane") + def test_claude_code_lane_refusals_are_the_providers_at_spec_validation(self): + """guard-lane-2, L2 of #155's verification: a spec whose claude-code arm the provider would refuse when + built (githits with bare, bare with api-hooked, api without bare; and M1's unscreened lookups on bare) is + refused by ``arms.check``, before any run starts. On f466967 each passed validation.""" + from unittest import mock + + from syberlabs import githits + from syberlabs.claude_code import THIRD_PARTY_SWITCHES + + repo = Path(tempfile.mkdtemp()) + self.addCleanup(shutil.rmtree, repo, True) + git(repo, "init", "-q") + config = repo.parent / f"{repo.name}-githits.json" + config.write_text(json.dumps({"mcpServers": {"githits": {"command": "githits-stand-in"}}}), encoding="utf-8") + self.addCleanup(config.unlink) + base = {"name": "cc", "kind": "claude-code", "model": "claude-sonnet-5-5", "effort": "medium", + "executable": ["{python}", "-c", "print('2.1.309 (Claude Code)')"]} + env = {k: v for k, v in os.environ.items() if k not in THIRD_PARTY_SWITCHES} + env.update({githits.TOKEN_ENV: "test-not-a-token", "ANTHROPIC_API_KEY": "test-placeholder"}) + for extra, key in (({"bare": True, "lane": "api", "githits": True}, "githits"), + ({"bare": True, "lane": "api-hooked"}, "lane"), + ({"lane": "api", "trusted_tree": True}, "lane"), + ({"bare": True, "tools": ["Read", "WebSearch"]}, "tools"), + ({"bare": True, "lane": "api", "tools": ["Read", "WebFetch"]}, "tools"), + ({"bare": True, "mcp_config": str(config)}, "mcp_config")): + arm = {**base, **extra} + with self.subTest(arm=extra): + errors = arms.check(arm, self.BUDGET) + self.assertTrue(errors and all(e.startswith(key + ":") for e in errors), errors) + with mock.patch.dict(os.environ, env, clear=True), self.assertRaises(Rejected) as caught: + arms.build(arm, repo=repo, budget=self.BUDGET, prices=PRICES) + self.assertEqual(caught.exception.code, "invalid_provider", "the provider refuses it too") + for extra in ({"bare": True, "lane": "api", "tools": ["Read", "Edit"]}, + {"lane": "api-hooked", "trusted_tree": True, "githits": True, "tools": ["Read", "WebSearch"]}, + {"lane": "subscription_local", "trusted_tree": True, "mcp_config": str(config)}): + with self.subTest(allowed=extra): + self.assertEqual(arms.check({**base, **extra}, self.BUDGET), []) + def test_the_environment_names_the_model_and_withholds_the_seed_and_tokens(self): arm = {"name": "single", "kind": "single", "model": "simulated", "effort": "low", "adapter": SIM} task = {"source": "fixture:pricing"} diff --git a/tests/test_experiment.py b/tests/test_experiment.py index 0c25a890..95af4f31 100644 --- a/tests/test_experiment.py +++ b/tests/test_experiment.py @@ -702,7 +702,10 @@ def test_the_report_reads_arm_lookups_and_cost_off_the_claude_code_record(self): actor="runner@example.test") thread.record_experiment(NAME, item="LIB-1", arm=arm, by="owner@example.test", tags=["needs_lookup"]) with self.api_lane(): - [candidate] = thread.propose(ClaudeCodeProvider(kit.repo, bare=True, max_calls=1, **kwargs)) + # guard-lane-2: WebSearch and an MCP server on --bare are unscreened; the historical MCP A/B ran + # there, so both arms carry the operator's recorded override. + [candidate] = thread.propose(ClaudeCodeProvider(kit.repo, bare=True, max_calls=1, + allow_unscreened_bare=True, **kwargs)) receipt = thread.accept(candidate.id, actor="runner@example.test") self.assertEqual(receipt.status, "succeeded", receipt.reason) events = [e for case in kit.journal.thread_ids() for e in kit.session.history(case)] diff --git a/tests/test_githits_pace.py b/tests/test_githits_pace.py index 87433375..1b85e557 100644 --- a/tests/test_githits_pace.py +++ b/tests/test_githits_pace.py @@ -130,10 +130,12 @@ def test_two_concurrent_processes_take_slots_at_least_the_interval_apart(self): self.assertGreaterEqual(held_second, 1000, "the second was held, its lock wait included") def test_two_concurrent_launcher_examples_are_spaced_recorded_and_other_ops_unpaced(self): - arrivals = [] + arrivals, slots = [], [] + stamp_file = Path(self.pace.name) / "pace" / githits_pace.STAMP_FILE def example(request): arrivals.append(time.time()) + slots.append(githits_pace._last(stamp_file)) # the slot the arriving request took (the latest stamp) body = {"result": "No suitable example was found.\n", "search_status": "complete", "solution_id": "12345678-1234-4234-8234-123456789abc"} return 200, {"Content-Type": "application/json", "X-Request-ID": f"req-ex{len(arrivals)}"}, \ @@ -153,9 +155,17 @@ def example(request): _, err = process.communicate(timeout=120) self.assertEqual(process.returncode, 0, err.decode("utf-8", "replace")) self.assertEqual(len(arrivals), 2) - # Each slot is stamped just before its request leaves, so arrivals sit a whole interval apart less the first - # request's own few milliseconds of transit. - self.assertGreaterEqual(abs(arrivals[1] - arrivals[0]), self.INTERVAL - 0.25) + # guard-lane-2: what pacing guarantees is the slots, not the arrivals. Each slot is stamped before its request + # leaves, and the client's own setup and transit between stamp and arrival vary per process (a slow runner + # once saw arrivals 1.56 s apart on a 2 s interval), so the arrivals' spacing is not asserted. Each request + # reads, on arrival, the stamp of the slot it took: the second slot cannot be taken until the first is a + # whole interval old, so a request can see the other's slot only if its own transit outlasted the interval. + self.assertTrue(all(slot is not None for slot in slots), "both examples took a slot in the shared directory") + first, second = sorted(slots) + self.assertGreaterEqual(second - first, self.INTERVAL, "the second slot is a whole interval after the first") + for arrived, slot in zip(arrivals, slots): + self.assertGreaterEqual(arrived, slot, "a request leaves only after its slot is stamped") + self.assertEqual(githits_pace._last(stamp_file), second, "the last slot taken is the second") records = self.records().records self.assertEqual([r["op"] for r in records], ["example", "example"]) held = sorted(r["paced_ms"] for r in records)