From d77877ff7ec9b25d4e1e033cb7a5d14c9e28c546 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 22 Sep 2026 14:19:06 +0000 Subject: [PATCH] fix(gate): replay a script gate's evals, and name its script by file not address Two things a policy shipping a `kind: script` gate hit the moment it was adopted from a catalog. `chock check --only evals` replays a staged-files case in a throwaway repository holding only the case's own files. That is whole for a declarative gate, whose JSON is the entire check, and empty-handed for a script gate, whose program lives under the policy's implementations/ and is resolved from the repository root: every case observed "not installed" and refused, so a suite could only pass by expecting block. The runner now copies the policy's implementations/ to where the compiled gate names the script, as `chock sync` would have, before running the case. Copied, never staged: the material a case puts before the gate stays the files it lists. The ambient line rendered the compiled `script` param, the file's path from the repository root, which differs between a catalog tree (base//...) and an adopter (.agents/policies//...). The packaged SKILL.md carries that line, so `chock plugin build --check` could not be clean in both places at once. The bare file name is rendered now, which is what the manifest declared. The stability-script golden moves with it: an emitter change, so the next release is a minor one under the stability rule. Co-Authored-By: Claude Fable 5.1 Signed-off-by: Claude --- CHANGELOG.md | 17 ++++++ spec/gate-dsl.md | 9 ++- src/chock/compile/emitters/advisory.py | 7 ++- src/chock/eval/execute.py | 19 +++++++ .../stability-script/ambient-rule/ambient.md | 2 +- tests/test_gate_script_kind.py | 56 +++++++++++++++++++ 6 files changed, 105 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3aa92c5..3451736 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,22 @@ # Chock changelog +## Unreleased + +- **A script gate's evals replay.** `chock check --only evals` runs a staged-files case against + the compiled gate in a throwaway repository that holds only the case's own files -- whole + for a declarative gate, whose JSON is the entire check, and empty-handed for a `kind: script` + gate, whose program lives under the policy's `implementations/` and is resolved from the + repository root. Every such case observed "not installed" and refused, so a suite could only + pass by expecting `block`. The runner now copies the policy's `implementations/` to where the + compiled gate names the script, as `chock sync` would have, before running the case. +- **A script gate's ambient line names the script, not its address.** The `on(...)` line an + agent reads rendered the compiled `script` param, which is the file's path from the repository + root and so differs between a catalog tree (`base//...`) and an adopter + (`.agents/policies//...`). The packaged `SKILL.md` carries that line, so `chock plugin build + --check` could not be clean in both places at once. The bare file name is rendered now, which is + what the manifest declares; the `stability-script` golden moves with it (an emitter change, so + this is a minor release under the stability rule). + ## 0.9.3 — A `kind: script` gate that runs a policy's own program, and a native Devin plugin format and marketplace tree on agentseam 0.3.2 - **`kind: script` gate**: a `hook.gate` whose check is the policy's own program, for a check no diff --git a/spec/gate-dsl.md b/spec/gate-dsl.md index 8e6ba36..e286d00 100644 --- a/spec/gate-dsl.md +++ b/spec/gate-dsl.md @@ -124,9 +124,12 @@ missing script, a crash, a timeout (30s) -- refuses too, in the runner's words: reaches no decision never reports an allow it never established. `chock compile` rewrites `script` to the file's path from the repository root, which is all -the runner has. `chock check` refuses a name that is not a bare `.py` file name, and a script -the policy does not ship. The script is deterministic code under `implementations/`, so SEC-2 -applies to it as to any guard. +the runner has; the ambient line an agent reads keeps the bare name, so the packaged `SKILL.md` +is the same wherever the policy sits. `chock check` refuses a name that is not a bare `.py` file +name, and a script the policy does not ship. The script is deterministic code under +`implementations/`, so SEC-2 applies to it as to any guard, and `chock check --only evals` +copies that directory into the throwaway repository a staged-files case is replayed in, where +the compiled gate names it. ## Runtime note diff --git a/src/chock/compile/emitters/advisory.py b/src/chock/compile/emitters/advisory.py index 4eb4ea0..121b67f 100644 --- a/src/chock/compile/emitters/advisory.py +++ b/src/chock/compile/emitters/advisory.py @@ -61,7 +61,12 @@ def _render_params(params: dict[str, Any]) -> str: rendered: list[str] = [] used = 0 for key, value in params.items(): - item = f"{key}={_clip(_scalar(value))}" + # A script param's compiled value is the file's path from the repository root, which + # depends on where the policy sits (`.agents/policies/` once adopted, a catalog tree + # before). The line an agent reads, and the packaged SKILL.md that carries it, must not + # change with the address: the bare name is what the manifest declared. + shown = Path(str(value)).name if key == "script" else value + item = f"{key}={_clip(_scalar(shown))}" if rendered and used + len(item) > _PARAMS_CHARS: rendered.append("...") break diff --git a/src/chock/eval/execute.py b/src/chock/eval/execute.py index 86d4f3d..be83cf6 100644 --- a/src/chock/eval/execute.py +++ b/src/chock/eval/execute.py @@ -82,6 +82,24 @@ def _prepare(repo: Path, spec: dict[str, Any]) -> None: _git(repo, "add", *sorted(staged)) +def _install_script(repo: Path, policy_dir: Path, gate_spec: dict[str, Any]) -> None: + """Put a script gate's program where the compiled gate names it, as `chock sync` would have. + + A declarative gate is whole in its JSON, so the throwaway repo needs nothing else. A script + gate names a file under the policy's `implementations/`, and the runner resolves that name + from the repository root -- a root that, here, holds only the case's own files. Copied, not + staged: the material a case puts before the gate is the files it lists, never the gate's + own program. + """ + if gate_spec.get("kind") != "script": + return + named = str((gate_spec.get("params") or {}).get("script", "")) + source = Path(policy_dir) / "implementations" + if not named or not source.is_dir(): + return + shutil.copytree(source, repo / Path(named).parent, dirs_exist_ok=True) + + def _run_gate(repo: Path, gate_spec: dict[str, Any], spec: dict[str, Any]) -> tuple[str, str]: """Return (verdict, detail) by running the compiled gate as a git hook would.""" gate_path = repo / "gate.json" @@ -188,6 +206,7 @@ def run_case(case: Case, policy_dir: Path, repo_root: Path, guards: list[Path]) }[source] return CaseResult(case, "error", detail=reason) _prepare(repo, spec) + _install_script(repo, policy_dir, gate_spec) verdict, detail = _run_gate(repo, gate_spec, spec) if source == "manifest": detail = f"{detail} [gate derived from manifest; policy not compiled]" diff --git a/tests/fixtures/emitter_stability/golden/stability-script/ambient-rule/ambient.md b/tests/fixtures/emitter_stability/golden/stability-script/ambient-rule/ambient.md index bde5f82..ea6025b 100644 --- a/tests/fixtures/emitter_stability/golden/stability-script/ambient-rule/ambient.md +++ b/tests/fixtures/emitter_stability/golden/stability-script/ambient-rule/ambient.md @@ -1,6 +1,6 @@ ``` -on(commit|tool_use): block(script) script=tests/fixtures/emitter_stability/policies/sta... +on(commit|tool_use): block(script) script=stability-script-gate.py Refused by the fixture's own script. ``` diff --git a/tests/test_gate_script_kind.py b/tests/test_gate_script_kind.py index b74f002..d64725a 100644 --- a/tests/test_gate_script_kind.py +++ b/tests/test_gate_script_kind.py @@ -17,7 +17,10 @@ import yaml from conftest import init_repo, stage +from chock.compile.emitters.advisory import advisory_lines from chock.compile.emitters.in_agent import GATE_FILE, STOP_FRAGMENT, WRITE_FRAGMENT, emit_pre_tool_use, emit_stop +from chock.eval.execute import run_case +from chock.eval.model import Case from chock.gate import runner from chock.gate.build import build_gate_json from chock.gate.runner import WRITE_PATH_KINDS, run @@ -222,3 +225,56 @@ def test_a_declared_script_must_be_shipped(tmp_path: Path) -> None: def test_a_shipped_script_validates(tmp_path: Path) -> None: policy, _ = _policy(tmp_path) assert _messages(_spec(SCRIPT), policy) == [] + + +# --- an eval case replays against the script, not against its absence -------------------------- + + +def _case(files: dict[str, str], expect: str) -> Case: + return Case( + id="t", + category="trigger", + prompt="p", + expect="e", + policy_id=POLICY_ID, + execute={"files": files, "event": "commit", "expect": expect}, + ) + + +@pytest.mark.parametrize(("text", "expect"), [(MARKER, "block"), ("clean", "allow")]) +def test_an_eval_case_finds_the_script_in_the_throwaway_repo(tmp_path: Path, text: str, expect: str) -> None: + """The throwaway repo holds only the case's files; the gate's own program has to be put there.""" + init_repo(tmp_path) + policy, _ = _policy(tmp_path) + result = run_case(_case({"App.java": text}, expect), policy, tmp_path, guards=[]) + assert result.outcome == "pass", result.detail + + +def test_an_eval_case_never_stages_the_script_itself(tmp_path: Path) -> None: + """A script that refuses its own text must not refuse every case by being in the writes.""" + init_repo(tmp_path) + policy, _ = _policy(tmp_path) + (policy / "implementations" / SCRIPT).write_text(GATE_SCRIPT.replace('"FORBIDDEN" in t', '"sys.exit" in t')) + result = run_case(_case({"App.java": "clean"}, "allow"), policy, tmp_path, guards=[]) + assert result.outcome == "pass", result.detail + + +# --- the ambient line names the script wherever the policy sits ------------------------------------ + + +def test_the_ambient_line_names_the_script_not_its_address(tmp_path: Path) -> None: + """One packaged SKILL.md must be right in a catalog tree and in .agents/policies alike.""" + adopted, _ = _policy(tmp_path) + catalog_root = tmp_path / "catalog" + (catalog_root / ".chock").mkdir(parents=True) + catalog = catalog_root / "base" / POLICY_ID + (catalog / "implementations").mkdir(parents=True) + (catalog / "manifest.yaml").write_text((adopted / "manifest.yaml").read_text(encoding="utf-8"), encoding="utf-8") + (catalog / "implementations" / SCRIPT).write_text(GATE_SCRIPT, encoding="utf-8") + assert advisory_lines(adopted, yaml.safe_load((adopted / "manifest.yaml").read_text()), tmp_path) == advisory_lines( + catalog, yaml.safe_load((catalog / "manifest.yaml").read_text()), catalog_root + ) + assert ( + f"script={SCRIPT}" + in advisory_lines(adopted, yaml.safe_load((adopted / "manifest.yaml").read_text()), tmp_path)[0] + )