fix(gate): replay a script gate's evals, and name its script by file not address

CHANGELOG.md

@@ -1,5 +1,22 @@
 # Chock changelog
 
+## Unreleased
+
+- **A script gate's evals replay.** `chock check --only evals` runs a staged-files case against
+  the compiled gate in a throwaway repository that holds only the case's own files -- whole
+  for a declarative gate, whose JSON is the entire check, and empty-handed for a `kind: script`
+  gate, whose program lives under the policy's `implementations/` and is resolved from the
+  repository root. Every such case observed "not installed" and refused, so a suite could only
+  pass by expecting `block`. The runner now copies the policy's `implementations/` to where the
+  compiled gate names the script, as `chock sync` would have, before running the case.
+- **A script gate's ambient line names the script, not its address.** The `on(...)` line an
+  agent reads rendered the compiled `script` param, which is the file's path from the repository
+  root and so differs between a catalog tree (`base/<id>/...`) and an adopter
+  (`.agents/policies/<id>/...`). The packaged `SKILL.md` carries that line, so `chock plugin build
+  --check` could not be clean in both places at once. The bare file name is rendered now, which is
+  what the manifest declares; the `stability-script` golden moves with it (an emitter change, so
+  this is a minor release under the stability rule).
+
 ## 0.9.3 — A `kind: script` gate that runs a policy's own program, and a native Devin plugin format and marketplace tree on agentseam 0.3.2
 
 - **`kind: script` gate**: a `hook.gate` whose check is the policy's own program, for a check no

spec/gate-dsl.md

@@ -124,9 +124,12 @@ missing script, a crash, a timeout (30s) -- refuses too, in the runner's words:
 reaches no decision never reports an allow it never established.
 
 `chock compile` rewrites `script` to the file's path from the repository root, which is all
-the runner has. `chock check` refuses a name that is not a bare `.py` file name, and a script
-the policy does not ship. The script is deterministic code under `implementations/`, so SEC-2
-applies to it as to any guard.
+the runner has; the ambient line an agent reads keeps the bare name, so the packaged `SKILL.md`
+is the same wherever the policy sits. `chock check` refuses a name that is not a bare `.py` file
+name, and a script the policy does not ship. The script is deterministic code under
+`implementations/`, so SEC-2 applies to it as to any guard, and `chock check --only evals`
+copies that directory into the throwaway repository a staged-files case is replayed in, where
+the compiled gate names it.
 
 ## Runtime note
 

src/chock/compile/emitters/advisory.py

@@ -61,7 +61,12 @@ def _render_params(params: dict[str, Any]) -> str:
     rendered: list[str] = []
     used = 0
     for key, value in params.items():
-        item = f"{key}={_clip(_scalar(value))}"
+        # A script param's compiled value is the file's path from the repository root, which
+        # depends on where the policy sits (`.agents/policies/<id>` once adopted, a catalog tree
+        # before). The line an agent reads, and the packaged SKILL.md that carries it, must not
+        # change with the address: the bare name is what the manifest declared.
+        shown = Path(str(value)).name if key == "script" else value
+        item = f"{key}={_clip(_scalar(shown))}"
         if rendered and used + len(item) > _PARAMS_CHARS:
             rendered.append("...")
             break

src/chock/eval/execute.py

@@ -82,6 +82,24 @@ def _prepare(repo: Path, spec: dict[str, Any]) -> None:
         _git(repo, "add", *sorted(staged))
 
 
+def _install_script(repo: Path, policy_dir: Path, gate_spec: dict[str, Any]) -> None:
+    """Put a script gate's program where the compiled gate names it, as `chock sync` would have.
+
+    A declarative gate is whole in its JSON, so the throwaway repo needs nothing else. A script
+    gate names a file under the policy's `implementations/`, and the runner resolves that name
+    from the repository root -- a root that, here, holds only the case's own files. Copied, not
+    staged: the material a case puts before the gate is the files it lists, never the gate's
+    own program.
+    """
+    if gate_spec.get("kind") != "script":
+        return
+    named = str((gate_spec.get("params") or {}).get("script", ""))
+    source = Path(policy_dir) / "implementations"
+    if not named or not source.is_dir():
+        return
+    shutil.copytree(source, repo / Path(named).parent, dirs_exist_ok=True)
+
+
 def _run_gate(repo: Path, gate_spec: dict[str, Any], spec: dict[str, Any]) -> tuple[str, str]:
     """Return (verdict, detail) by running the compiled gate as a git hook would."""
     gate_path = repo / "gate.json"
@@ -188,6 +206,7 @@ def run_case(case: Case, policy_dir: Path, repo_root: Path, guards: list[Path])
                     }[source]
                     return CaseResult(case, "error", detail=reason)
                 _prepare(repo, spec)
+                _install_script(repo, policy_dir, gate_spec)
                 verdict, detail = _run_gate(repo, gate_spec, spec)
                 if source == "manifest":
                     detail = f"{detail} [gate derived from manifest; policy not compiled]"

tests/fixtures/emitter_stability/golden/stability-script/ambient-rule/ambient.md

@@ -1,6 +1,6 @@
 <!-- chock:hooks:start (compiled by chock -- edit .agents/policies/stability-script/) -->
 ```
-on(commit|tool_use): block(script) script=tests/fixtures/emitter_stability/policies/sta...
+on(commit|tool_use): block(script) script=stability-script-gate.py
 Refused by the fixture's own script.
 ```
 <!-- chock:hooks:end -->

tests/test_gate_script_kind.py

@@ -17,7 +17,10 @@
 import yaml
 from conftest import init_repo, stage
 
+from chock.compile.emitters.advisory import advisory_lines
 from chock.compile.emitters.in_agent import GATE_FILE, STOP_FRAGMENT, WRITE_FRAGMENT, emit_pre_tool_use, emit_stop
+from chock.eval.execute import run_case
+from chock.eval.model import Case
 from chock.gate import runner
 from chock.gate.build import build_gate_json
 from chock.gate.runner import WRITE_PATH_KINDS, run
@@ -222,3 +225,56 @@ def test_a_declared_script_must_be_shipped(tmp_path: Path) -> None:
 def test_a_shipped_script_validates(tmp_path: Path) -> None:
     policy, _ = _policy(tmp_path)
     assert _messages(_spec(SCRIPT), policy) == []
+
+
+# --- an eval case replays against the script, not against its absence --------------------------
+
+
+def _case(files: dict[str, str], expect: str) -> Case:
+    return Case(
+        id="t",
+        category="trigger",
+        prompt="p",
+        expect="e",
+        policy_id=POLICY_ID,
+        execute={"files": files, "event": "commit", "expect": expect},
+    )
+
+
+@pytest.mark.parametrize(("text", "expect"), [(MARKER, "block"), ("clean", "allow")])
+def test_an_eval_case_finds_the_script_in_the_throwaway_repo(tmp_path: Path, text: str, expect: str) -> None:
+    """The throwaway repo holds only the case's files; the gate's own program has to be put there."""
+    init_repo(tmp_path)
+    policy, _ = _policy(tmp_path)
+    result = run_case(_case({"App.java": text}, expect), policy, tmp_path, guards=[])
+    assert result.outcome == "pass", result.detail
+
+
+def test_an_eval_case_never_stages_the_script_itself(tmp_path: Path) -> None:
+    """A script that refuses its own text must not refuse every case by being in the writes."""
+    init_repo(tmp_path)
+    policy, _ = _policy(tmp_path)
+    (policy / "implementations" / SCRIPT).write_text(GATE_SCRIPT.replace('"FORBIDDEN" in t', '"sys.exit" in t'))
+    result = run_case(_case({"App.java": "clean"}, "allow"), policy, tmp_path, guards=[])
+    assert result.outcome == "pass", result.detail
+
+
+# --- the ambient line names the script wherever the policy sits ------------------------------------
+
+
+def test_the_ambient_line_names_the_script_not_its_address(tmp_path: Path) -> None:
+    """One packaged SKILL.md must be right in a catalog tree and in .agents/policies alike."""
+    adopted, _ = _policy(tmp_path)
+    catalog_root = tmp_path / "catalog"
+    (catalog_root / ".chock").mkdir(parents=True)
+    catalog = catalog_root / "base" / POLICY_ID
+    (catalog / "implementations").mkdir(parents=True)
+    (catalog / "manifest.yaml").write_text((adopted / "manifest.yaml").read_text(encoding="utf-8"), encoding="utf-8")
+    (catalog / "implementations" / SCRIPT).write_text(GATE_SCRIPT, encoding="utf-8")
+    assert advisory_lines(adopted, yaml.safe_load((adopted / "manifest.yaml").read_text()), tmp_path) == advisory_lines(
+        catalog, yaml.safe_load((catalog / "manifest.yaml").read_text()), catalog_root
+    )
+    assert (
+        f"script={SCRIPT}"
+        in advisory_lines(adopted, yaml.safe_load((adopted / "manifest.yaml").read_text()), tmp_path)[0]
+    )