From c9052a52b4884a6174a4d42838bd159b62cbf31d Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 22 Sep 2026 02:14:05 +0000 Subject: [PATCH] feat(gate): add a script kind that runs a policy's own program The declarative gate kinds answer questions a closed table can hold. A check that models flow through a method body cannot be written that way, and until now the only door for a script was the shell guard, which judges a command and never sees the file being written. `kind: script` names a Python file under the policy's implementations/. The runner hands it the same material every declarative kind reads -- the staged blobs at commit, the write at tool use and at the turn's end -- as JSON on stdin, and carries back its verdict: exit 0 allows, exit 1 blocks with the script's own words as the reason. Any other exit, a missing script, or a script that gives no verdict within 30 seconds refuses rather than allowing what it never judged. `chock compile` rewrites the bare name to the policy-relative path the git hook runs from. `chock check` refuses a name that is not a bare .py file and a declared script that is not shipped. The kind joins WRITE_PATH_KINDS, so it rides the existing pre-tool-use and stop fragments; a new emitter-stability fixture pins those bytes. Co-Authored-By: Claude Fable 5.1 Signed-off-by: Claude --- .chock/bin/gate.py | 59 ++++- CHANGELOG.md | 12 + docs/concepts.md | 3 +- spec/gate-dsl.md | 30 ++- src/chock/gate/build.py | 7 + src/chock/gate/runner.py | 59 ++++- src/chock/gate/schema.py | 9 + src/chock/validation/checks_gate_shape.py | 29 ++- .../validation/checks_manifest_schema.py | 4 +- .../validation/schemas/manifest.hook.json | 3 +- .../stability-script/ambient-rule/ambient.md | 6 + .../golden/stability-script/ci-gate/gate.json | 12 + .../golden/stability-script/ci-gate/step.yaml | 12 + .../stability-script/git-hook/gate.json | 12 + .../git-hook/git-pre-commit.sh | 11 + .../managed-setting/managed-settings.json | 4 + .../stability-script/pre-tool-use/gate.json | 12 + .../pre-tool-use/gemini_cli-write-hooks.json | 10 + .../pre-tool-use/pretooluse-write.json | 10 + .../stop/antigravity-hooks.json | 14 ++ .../stop/codex_cli-hooks.json | 15 ++ .../stability-script/stop/devin-hooks.json | 12 + .../golden/stability-script/stop/gate.json | 12 + .../stop/gemini_cli-hooks.json | 14 ++ .../golden/stability-script/stop/stop.json | 9 + .../stability-script/stop/tabnine-hooks.json | 15 ++ .../implementations/stability-script-gate.py | 6 + .../policies/stability-script/manifest.yaml | 48 ++++ tests/test_gate_script_kind.py | 224 ++++++++++++++++++ 29 files changed, 665 insertions(+), 8 deletions(-) create mode 100644 tests/fixtures/emitter_stability/golden/stability-script/ambient-rule/ambient.md create mode 100644 tests/fixtures/emitter_stability/golden/stability-script/ci-gate/gate.json create mode 100644 tests/fixtures/emitter_stability/golden/stability-script/ci-gate/step.yaml create mode 100644 tests/fixtures/emitter_stability/golden/stability-script/git-hook/gate.json create mode 100644 tests/fixtures/emitter_stability/golden/stability-script/git-hook/git-pre-commit.sh create mode 100644 tests/fixtures/emitter_stability/golden/stability-script/managed-setting/managed-settings.json create mode 100644 tests/fixtures/emitter_stability/golden/stability-script/pre-tool-use/gate.json create mode 100644 tests/fixtures/emitter_stability/golden/stability-script/pre-tool-use/gemini_cli-write-hooks.json create mode 100644 tests/fixtures/emitter_stability/golden/stability-script/pre-tool-use/pretooluse-write.json create mode 100644 tests/fixtures/emitter_stability/golden/stability-script/stop/antigravity-hooks.json create mode 100644 tests/fixtures/emitter_stability/golden/stability-script/stop/codex_cli-hooks.json create mode 100644 tests/fixtures/emitter_stability/golden/stability-script/stop/devin-hooks.json create mode 100644 tests/fixtures/emitter_stability/golden/stability-script/stop/gate.json create mode 100644 tests/fixtures/emitter_stability/golden/stability-script/stop/gemini_cli-hooks.json create mode 100644 tests/fixtures/emitter_stability/golden/stability-script/stop/stop.json create mode 100644 tests/fixtures/emitter_stability/golden/stability-script/stop/tabnine-hooks.json create mode 100644 tests/fixtures/emitter_stability/policies/stability-script/implementations/stability-script-gate.py create mode 100644 tests/fixtures/emitter_stability/policies/stability-script/manifest.yaml create mode 100644 tests/test_gate_script_kind.py diff --git a/.chock/bin/gate.py b/.chock/bin/gate.py index 055c4c3..a043d6c 100755 --- a/.chock/bin/gate.py +++ b/.chock/bin/gate.py @@ -160,7 +160,7 @@ def head_blob(self, path: str) -> str: #: Kinds whose question a write can answer. A branch name is not in a tool call, so #: forbidden_ref has nothing to read here; saying so beats passing it empty and calling #: that an allow. -WRITE_PATH_KINDS = frozenset({"content_regex"}) +WRITE_PATH_KINDS = frozenset({"content_regex", "script"}) #: Events at which a line-level waiver is honoured: the ones where a human staged the text. At @@ -339,11 +339,68 @@ def _kind_test_integrity(ctx: GateContext, params: dict, _event: str) -> GateRes return GateResult(allowed=not matches, matches=matches) +#: A script gate's budget to answer. Past it the script has not decided, and an undecided +#: gate refuses. +_SCRIPT_TIMEOUT_SECONDS = 30 + +#: The exit codes a script gate speaks -- this runner's own, so a policy's script reads like +#: the runner that calls it. Anything else is not a verdict. +_SCRIPT_ALLOW, _SCRIPT_BLOCK = 0, 1 + +_UNDECIDED = " -- refusing rather than allowing what it never judged" + + +def _kind_script(ctx: GateContext, params: dict, event: str) -> GateResult: + """Hand the material to the policy's own script and carry back its verdict. + + The script reads `{"event", "repo_root", "writes": {path: text}}` on stdin -- the staged + blobs at commit and push, the write itself at tool use and at the turn's end -- so one + script serves every surface the declarative kinds do, and reads them the same way. It + answers with an exit code: 0 allows; 1 refuses, with its own words on stderr. A missing + script, a crash or a timeout refuses too, in this runner's words: a gate that cannot + reach a decision never reports an allow it never established. + """ + named = str(params.get("script", "")) + script = ctx.repo_root / named + if not script.is_file(): + return GateResult(allowed=False, message=f"script gate: {named!r} is not installed{_UNDECIDED}") + writes = {path: ctx.staged_blob(path) for path in ctx.staged_paths()} + if not writes: + return GateResult(allowed=True) + payload = json.dumps({"event": event, "repo_root": str(ctx.repo_root), "writes": writes}) + try: + proc = subprocess.run( # noqa: S603 -- the script is the policy's own, named in its manifest + [sys.executable, str(script)], + input=payload, + capture_output=True, + text=True, + encoding="utf-8", + errors="replace", + cwd=ctx.repo_root, + timeout=_SCRIPT_TIMEOUT_SECONDS, + check=False, + ) + except subprocess.TimeoutExpired: + budget = f"gave no verdict within {_SCRIPT_TIMEOUT_SECONDS}s" + return GateResult(allowed=False, message=f"script gate: {script.name} {budget}{_UNDECIDED}") + except OSError as exc: + return GateResult(allowed=False, message=f"script gate: {script.name} could not run ({exc}){_UNDECIDED}") + spoken = ((proc.stderr or "") + (proc.stdout or "")).strip() + if proc.returncode == _SCRIPT_ALLOW: + return GateResult(allowed=True) + if proc.returncode == _SCRIPT_BLOCK: + return GateResult(allowed=False, message=spoken or f"blocked by {script.name}") + first = spoken.splitlines()[0] if spoken else "" + detail = f": {first}" if first else "" + return GateResult(allowed=False, message=f"script gate: {script.name} exited {proc.returncode}{detail}{_UNDECIDED}") + + KINDS = { "content_regex": _kind_content_regex, "forbidden_ref": _kind_forbidden_ref, "dependency_allowlist": _kind_dependency_allowlist, "test_integrity": _kind_test_integrity, + "script": _kind_script, } diff --git a/CHANGELOG.md b/CHANGELOG.md index 74ca2bf..66b8c5c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,18 @@ ## Unreleased +- **`kind: script` gate**: a `hook.gate` whose check is the policy's own program, for a check no + declarative kind can hold -- a parser, a flow model over a method body, a rule table larger + than `params`. The runner hands the script the same material every declarative kind reads + (`{"event", "repo_root", "writes"}` on stdin: the staged blobs at commit and push, the write + itself at tool use and at the turn's end) and carries back its exit code -- `0` allows, `1` + refuses in the script's own words. A missing script, a crash or a timeout refuses in the + runner's words, never allows. It is a write-path kind, so the existing emitters wire it to the + write fragment and to `stop` unchanged: a script-backed policy now reaches every vendor a + `content_regex` gate does. Until now a script could only be a shell guard (`--guard`, + argv-shaped, never shown the file being written) or a git-event script (commit and push only); + the gap a script-backed gate left was named in the catalog's own a11y changelog. `chock check` + refuses a script name that is not a bare `.py` file name, and one the policy does not ship. - **`devin` plugin format**: `chock plugin build --format devin` packages a policy as a native Devin plugin (`.devin-plugin/plugin.json` + `skills//SKILL.md` + a root-level `hooks.json`, not the nested `hooks/hooks.json` every other format uses). Same guard, same adapter, diff --git a/docs/concepts.md b/docs/concepts.md index c280d29..907936d 100644 --- a/docs/concepts.md +++ b/docs/concepts.md @@ -43,7 +43,8 @@ fields include: ## Gate A hook's `hook.gate` in `manifest.yaml` selects a deterministic **kind** (`content_regex`, `forbidden_ref`, -`dependency_allowlist`), its `params`, the events it runs on (`on: [commit|push|tool_use]`), and the message +`dependency_allowlist`, `test_integrity`, or `script` for a check the policy's own program makes), its +`params`, the events it runs on (`on: [commit|push|tool_use]`), and the message shown on block. `chock compile` flattens it to `gate.json`, which the vendored `.chock/bin/gate.py` runner enforces at git-hook time. See [Gate DSL](../spec/gate-dsl.md). diff --git a/spec/gate-dsl.md b/spec/gate-dsl.md index 7dd8f20..8e6ba36 100644 --- a/spec/gate-dsl.md +++ b/spec/gate-dsl.md @@ -7,7 +7,7 @@ For `artifact: hook` policies, the gate is declared under `hook.gate` in `manife | field | required | type | notes | |-------|----------|------|-------| -| `kind` | yes | string | `content_regex`, `forbidden_ref`, `dependency_allowlist`, `test_integrity`, or `egress_allowlist` (gateway-only) | +| `kind` | yes | string | `content_regex`, `forbidden_ref`, `dependency_allowlist`, `test_integrity`, `script`, or `egress_allowlist` (gateway-only) | | `on` | yes | list | events: `commit`, `push`, `tool_use`. The key must be quoted `"on"` in YAML. | | `action` | yes | string | `block`, `verify`, or `warn` | | `message` | yes | string | printed to stderr when the gate blocks | @@ -100,6 +100,34 @@ files matching `test_path_regex`), and a vacuous assertion added in place of a r Only the staged diff is read (`removed_lines`/`added_lines`), so a file that already contained fewer assertions before this commit does not block it. +### `kind: script` + +| param | required | type | notes | +|-------|----------|------|-------| +| `script` | yes | string | a file name under the policy's `implementations/` directory: bare (no `/`, so nothing outside that directory) and `.py`, run by the runner's own interpreter | + +For a check no declarative kind can hold -- one that has to parse, follow a value through a +method body, or read a rule table too large for `params`. The runner hands the policy's own +program the same material every kind above reads, as JSON on stdin: + +```json +{"event": "tool_use", "repo_root": "/path/to/repo", "writes": {"src/App.java": ""}} +``` + +`event` is `commit`, `push` or `tool_use`. `writes` is the staged blobs at `commit` and +`push`, and the write itself at `tool_use` -- the file a tool call is about to write, or what +the turn left on disk at its end -- so one script serves every surface, and it reaches the +write path and the turn's end exactly as `content_regex` does. The script answers with its +exit code: `0` allows; `1` refuses, with its own words on stderr, which become the reason +shown (the gate's `message` is not printed for a script that spoke). Any other outcome -- a +missing script, a crash, a timeout (30s) -- refuses too, in the runner's words: a gate that +reaches no decision never reports an allow it never established. + +`chock compile` rewrites `script` to the file's path from the repository root, which is all +the runner has. `chock check` refuses a name that is not a bare `.py` file name, and a script +the policy does not ship. The script is deterministic code under `implementations/`, so SEC-2 +applies to it as to any guard. + ## Runtime note The emitted git-hook shim probes for a working Python 3 interpreter in the order `python3`, `python`, `py` and then calls `.chock/bin/gate.py`. This makes enforcement work on stock Windows as well as POSIX without `pip install`. diff --git a/src/chock/gate/build.py b/src/chock/gate/build.py index 0296298..a036acb 100644 --- a/src/chock/gate/build.py +++ b/src/chock/gate/build.py @@ -7,6 +7,7 @@ from pathlib import Path from typing import Any +from chock.compile.emitters import policy_rel_path from chock.config import load_config from chock.manifest import load_manifest @@ -55,6 +56,12 @@ def build_gate_json(policy_dir: Path, repo_root: Path) -> dict[str, Any] | None: spec["params"]["refs"] = [str(r) for r in resolved] spec["params"].pop("config_key", None) + if spec["kind"] == "script": + # The manifest names a file under the policy's own implementations/. The compiled gate + # carries where that is from the repository root, which is all the runner has to go on. + script = str(spec["params"].get("script", "")) + spec["params"]["script"] = f"{policy_rel_path(policy_dir)}/implementations/{script}" + return spec diff --git a/src/chock/gate/runner.py b/src/chock/gate/runner.py index 055c4c3..a043d6c 100644 --- a/src/chock/gate/runner.py +++ b/src/chock/gate/runner.py @@ -160,7 +160,7 @@ def head_blob(self, path: str) -> str: #: Kinds whose question a write can answer. A branch name is not in a tool call, so #: forbidden_ref has nothing to read here; saying so beats passing it empty and calling #: that an allow. -WRITE_PATH_KINDS = frozenset({"content_regex"}) +WRITE_PATH_KINDS = frozenset({"content_regex", "script"}) #: Events at which a line-level waiver is honoured: the ones where a human staged the text. At @@ -339,11 +339,68 @@ def _kind_test_integrity(ctx: GateContext, params: dict, _event: str) -> GateRes return GateResult(allowed=not matches, matches=matches) +#: A script gate's budget to answer. Past it the script has not decided, and an undecided +#: gate refuses. +_SCRIPT_TIMEOUT_SECONDS = 30 + +#: The exit codes a script gate speaks -- this runner's own, so a policy's script reads like +#: the runner that calls it. Anything else is not a verdict. +_SCRIPT_ALLOW, _SCRIPT_BLOCK = 0, 1 + +_UNDECIDED = " -- refusing rather than allowing what it never judged" + + +def _kind_script(ctx: GateContext, params: dict, event: str) -> GateResult: + """Hand the material to the policy's own script and carry back its verdict. + + The script reads `{"event", "repo_root", "writes": {path: text}}` on stdin -- the staged + blobs at commit and push, the write itself at tool use and at the turn's end -- so one + script serves every surface the declarative kinds do, and reads them the same way. It + answers with an exit code: 0 allows; 1 refuses, with its own words on stderr. A missing + script, a crash or a timeout refuses too, in this runner's words: a gate that cannot + reach a decision never reports an allow it never established. + """ + named = str(params.get("script", "")) + script = ctx.repo_root / named + if not script.is_file(): + return GateResult(allowed=False, message=f"script gate: {named!r} is not installed{_UNDECIDED}") + writes = {path: ctx.staged_blob(path) for path in ctx.staged_paths()} + if not writes: + return GateResult(allowed=True) + payload = json.dumps({"event": event, "repo_root": str(ctx.repo_root), "writes": writes}) + try: + proc = subprocess.run( # noqa: S603 -- the script is the policy's own, named in its manifest + [sys.executable, str(script)], + input=payload, + capture_output=True, + text=True, + encoding="utf-8", + errors="replace", + cwd=ctx.repo_root, + timeout=_SCRIPT_TIMEOUT_SECONDS, + check=False, + ) + except subprocess.TimeoutExpired: + budget = f"gave no verdict within {_SCRIPT_TIMEOUT_SECONDS}s" + return GateResult(allowed=False, message=f"script gate: {script.name} {budget}{_UNDECIDED}") + except OSError as exc: + return GateResult(allowed=False, message=f"script gate: {script.name} could not run ({exc}){_UNDECIDED}") + spoken = ((proc.stderr or "") + (proc.stdout or "")).strip() + if proc.returncode == _SCRIPT_ALLOW: + return GateResult(allowed=True) + if proc.returncode == _SCRIPT_BLOCK: + return GateResult(allowed=False, message=spoken or f"blocked by {script.name}") + first = spoken.splitlines()[0] if spoken else "" + detail = f": {first}" if first else "" + return GateResult(allowed=False, message=f"script gate: {script.name} exited {proc.returncode}{detail}{_UNDECIDED}") + + KINDS = { "content_regex": _kind_content_regex, "forbidden_ref": _kind_forbidden_ref, "dependency_allowlist": _kind_dependency_allowlist, "test_integrity": _kind_test_integrity, + "script": _kind_script, } diff --git a/src/chock/gate/schema.py b/src/chock/gate/schema.py index 5a92eea..9032af5 100644 --- a/src/chock/gate/schema.py +++ b/src/chock/gate/schema.py @@ -55,6 +55,15 @@ "allowlist_pragma": {"type": "string"}, }, }, + "script": { + **_CLOSED_OBJECT, + "required": ["script"], + "properties": { + # A bare file name under the policy's own implementations/: no separator, so no + # way out of that directory, and .py only, which the runner's own interpreter runs. + "script": {"type": "string", "pattern": r"^[A-Za-z0-9._-]+\.py$"}, + }, + }, "egress_allowlist": { **_CLOSED_OBJECT, "required": ["allowed_hosts"], diff --git a/src/chock/validation/checks_gate_shape.py b/src/chock/validation/checks_gate_shape.py index 3779226..3bd2ab6 100644 --- a/src/chock/validation/checks_gate_shape.py +++ b/src/chock/validation/checks_gate_shape.py @@ -3,6 +3,7 @@ from __future__ import annotations import re +from pathlib import Path from typing import Any from chock.gate.runner import KINDS @@ -13,8 +14,15 @@ _CATEGORY = "manifest_gate_params" -def _validate_gate(gate: dict[str, Any], gate_ref: str, report: Report, *, tool_use_allowed: bool = False) -> None: - """Validate a gate spec: kind is known and params match the kind schema.""" +def _validate_gate( + gate: dict[str, Any], + gate_ref: str, + report: Report, + *, + tool_use_allowed: bool = False, + artifact_dir: Path | None = None, +) -> None: + """Validate a gate spec: kind is known, params match the kind schema, a named script is shipped.""" kind = gate.get("kind") if not kind: report.add(Finding(gate_ref, _CATEGORY, "error", "gate is missing 'kind'")) @@ -49,6 +57,7 @@ def _validate_gate(gate: dict[str, Any], gate_ref: str, report: Report, *, tool_ if param_schema is not None: params = gate.get("params") or {} validator = schema_validator(param_schema) + seen = len(report.errors) for exc in sorted(validator.iter_errors(params), key=lambda e: e.path): path_str = "/".join(str(p) for p in exc.path) report.add( @@ -60,6 +69,22 @@ def _validate_gate(gate: dict[str, Any], gate_ref: str, report: Report, *, tool_ ) ) _validate_regex_params(params, gate_ref, report) + if kind == "script" and artifact_dir is not None and len(report.errors) == seen: + _validate_script_shipped(params, artifact_dir, gate_ref, report) + + +def _validate_script_shipped(params: dict[str, Any], artifact_dir: Path, gate_ref: str, report: Report) -> None: + """The runner refuses when a named script is missing; validate says so before the first commit does.""" + name = str(params.get("script", "")) + if not (Path(artifact_dir) / "implementations" / name).is_file(): + report.add( + Finding( + f"{gate_ref} (params)", + _CATEGORY, + "error", + f"script gate names implementations/{name}, which is not there", + ) + ) _REGEX_PARAM_SUFFIXES = ("_pattern", "_regex", "_pragma") diff --git a/src/chock/validation/checks_manifest_schema.py b/src/chock/validation/checks_manifest_schema.py index 60faa9c..f00f8c5 100644 --- a/src/chock/validation/checks_manifest_schema.py +++ b/src/chock/validation/checks_manifest_schema.py @@ -86,7 +86,9 @@ def _check_manifest_block_needs_gate(artifact_dir: Path, manifest: dict[str, Any hook = manifest.get("hook") or {} if hook.get("gate"): - _validate_gate(hook["gate"], str(_manifest_ref(artifact_dir)), report, tool_use_allowed=True) + _validate_gate( + hook["gate"], str(_manifest_ref(artifact_dir)), report, tool_use_allowed=True, artifact_dir=artifact_dir + ) return if hook.get("script"): return # the script refuses at the declared event; checks_script_events pins it to disk diff --git a/src/chock/validation/schemas/manifest.hook.json b/src/chock/validation/schemas/manifest.hook.json index a3547fe..4c82cff 100644 --- a/src/chock/validation/schemas/manifest.hook.json +++ b/src/chock/validation/schemas/manifest.hook.json @@ -23,7 +23,8 @@ "content_regex", "forbidden_ref", "dependency_allowlist", - "test_integrity" + "test_integrity", + "script" ] }, "on": { diff --git a/tests/fixtures/emitter_stability/golden/stability-script/ambient-rule/ambient.md b/tests/fixtures/emitter_stability/golden/stability-script/ambient-rule/ambient.md new file mode 100644 index 0000000..bde5f82 --- /dev/null +++ b/tests/fixtures/emitter_stability/golden/stability-script/ambient-rule/ambient.md @@ -0,0 +1,6 @@ + +``` +on(commit|tool_use): block(script) script=tests/fixtures/emitter_stability/policies/sta... +Refused by the fixture's own script. +``` + diff --git a/tests/fixtures/emitter_stability/golden/stability-script/ci-gate/gate.json b/tests/fixtures/emitter_stability/golden/stability-script/ci-gate/gate.json new file mode 100644 index 0000000..8f73db1 --- /dev/null +++ b/tests/fixtures/emitter_stability/golden/stability-script/ci-gate/gate.json @@ -0,0 +1,12 @@ +{ + "kind": "script", + "on": [ + "commit", + "tool_use" + ], + "action": "block", + "message": "Refused by the fixture's own script.", + "params": { + "script": "tests/fixtures/emitter_stability/policies/stability-script/implementations/stability-script-gate.py" + } +} \ No newline at end of file diff --git a/tests/fixtures/emitter_stability/golden/stability-script/ci-gate/step.yaml b/tests/fixtures/emitter_stability/golden/stability-script/ci-gate/step.yaml new file mode 100644 index 0000000..9a84271 --- /dev/null +++ b/tests/fixtures/emitter_stability/golden/stability-script/ci-gate/step.yaml @@ -0,0 +1,12 @@ +# Auto-generated by chock compile. +# Policy: stability-script +- name: chock-ci-gate (stability-script) + run: | + PY="" + for c in python3 python py; do + if command -v "$c" >/dev/null 2>&1 && "$c" -c 'import tomllib' >/dev/null 2>&1; then PY="$c"; break; fi + done + [ -n "$PY" ] || { echo "gate: no python >= 3.11 (with tomllib) found on PATH" >&2; exit 2; } + base="${GITHUB_BASE_REF:?ci-gate needs GITHUB_BASE_REF -- run this step on the pull_request event}" + "$PY" .chock/bin/gate.py run --gate .chock/compiled/stability-script/ci-gate/gate.json --event ci --base "origin/$base" \ + --head-ref "${GITHUB_HEAD_REF:-}" diff --git a/tests/fixtures/emitter_stability/golden/stability-script/git-hook/gate.json b/tests/fixtures/emitter_stability/golden/stability-script/git-hook/gate.json new file mode 100644 index 0000000..8f73db1 --- /dev/null +++ b/tests/fixtures/emitter_stability/golden/stability-script/git-hook/gate.json @@ -0,0 +1,12 @@ +{ + "kind": "script", + "on": [ + "commit", + "tool_use" + ], + "action": "block", + "message": "Refused by the fixture's own script.", + "params": { + "script": "tests/fixtures/emitter_stability/policies/stability-script/implementations/stability-script-gate.py" + } +} \ No newline at end of file diff --git a/tests/fixtures/emitter_stability/golden/stability-script/git-hook/git-pre-commit.sh b/tests/fixtures/emitter_stability/golden/stability-script/git-hook/git-pre-commit.sh new file mode 100644 index 0000000..bb9ad54 --- /dev/null +++ b/tests/fixtures/emitter_stability/golden/stability-script/git-hook/git-pre-commit.sh @@ -0,0 +1,11 @@ +#!/usr/bin/env bash +# Auto-generated by chock compile. Declarative gate: stability-script +set -eu +repo_root="$(git rev-parse --show-toplevel)" +PY="" +for c in python3 python py; do + if command -v "$c" >/dev/null 2>&1 && "$c" -c 'import tomllib' >/dev/null 2>&1; then PY="$c"; break; fi +done +[ -n "$PY" ] || { echo "gate: no python >= 3.11 (with tomllib) found on PATH" >&2; exit 2; } +exec "$PY" "$repo_root/.chock/bin/gate.py" run \ + --gate "$repo_root/.chock/compiled/stability-script/git-hook/gate.json" --event pre-commit diff --git a/tests/fixtures/emitter_stability/golden/stability-script/managed-setting/managed-settings.json b/tests/fixtures/emitter_stability/golden/stability-script/managed-setting/managed-settings.json new file mode 100644 index 0000000..8efb266 --- /dev/null +++ b/tests/fixtures/emitter_stability/golden/stability-script/managed-setting/managed-settings.json @@ -0,0 +1,4 @@ +{ + "deny": [], + "ask": [] +} \ No newline at end of file diff --git a/tests/fixtures/emitter_stability/golden/stability-script/pre-tool-use/gate.json b/tests/fixtures/emitter_stability/golden/stability-script/pre-tool-use/gate.json new file mode 100644 index 0000000..8f73db1 --- /dev/null +++ b/tests/fixtures/emitter_stability/golden/stability-script/pre-tool-use/gate.json @@ -0,0 +1,12 @@ +{ + "kind": "script", + "on": [ + "commit", + "tool_use" + ], + "action": "block", + "message": "Refused by the fixture's own script.", + "params": { + "script": "tests/fixtures/emitter_stability/policies/stability-script/implementations/stability-script-gate.py" + } +} \ No newline at end of file diff --git a/tests/fixtures/emitter_stability/golden/stability-script/pre-tool-use/gemini_cli-write-hooks.json b/tests/fixtures/emitter_stability/golden/stability-script/pre-tool-use/gemini_cli-write-hooks.json new file mode 100644 index 0000000..746a39d --- /dev/null +++ b/tests/fixtures/emitter_stability/golden/stability-script/pre-tool-use/gemini_cli-write-hooks.json @@ -0,0 +1,10 @@ +{ + "matcher": "write_file|replace", + "hooks": [ + { + "type": "command", + "command": "@CHOCK_PYTHON@ \"${CLAUDE_PROJECT_DIR}/.chock/bin/gemini_cli.py\" --gate \"${CLAUDE_PROJECT_DIR}/.chock/compiled/stability-script/pre-tool-use/gate.json\"", + "timeout": 30 + } + ] +} \ No newline at end of file diff --git a/tests/fixtures/emitter_stability/golden/stability-script/pre-tool-use/pretooluse-write.json b/tests/fixtures/emitter_stability/golden/stability-script/pre-tool-use/pretooluse-write.json new file mode 100644 index 0000000..35134b6 --- /dev/null +++ b/tests/fixtures/emitter_stability/golden/stability-script/pre-tool-use/pretooluse-write.json @@ -0,0 +1,10 @@ +{ + "matcher": "Write|Edit|MultiEdit|NotebookEdit", + "hooks": [ + { + "type": "command", + "command": "@CHOCK_PYTHON@ \"${CLAUDE_PROJECT_DIR}/.chock/bin/claude_code.py\" --gate \"${CLAUDE_PROJECT_DIR}/.chock/compiled/stability-script/pre-tool-use/gate.json\"", + "timeout": 30 + } + ] +} \ No newline at end of file diff --git a/tests/fixtures/emitter_stability/golden/stability-script/stop/antigravity-hooks.json b/tests/fixtures/emitter_stability/golden/stability-script/stop/antigravity-hooks.json new file mode 100644 index 0000000..c2cc02f --- /dev/null +++ b/tests/fixtures/emitter_stability/golden/stability-script/stop/antigravity-hooks.json @@ -0,0 +1,14 @@ +{ + "agentseam": { + "Stop": [ + { + "hooks": [ + { + "type": "command", + "command": "@CHOCK_PYTHON@ \".chock/bin/antigravity.py\" --gate \".chock/compiled/stability-script/stop/gate.json\"" + } + ] + } + ] + } +} \ No newline at end of file diff --git a/tests/fixtures/emitter_stability/golden/stability-script/stop/codex_cli-hooks.json b/tests/fixtures/emitter_stability/golden/stability-script/stop/codex_cli-hooks.json new file mode 100644 index 0000000..02c51fb --- /dev/null +++ b/tests/fixtures/emitter_stability/golden/stability-script/stop/codex_cli-hooks.json @@ -0,0 +1,15 @@ +{ + "hooks": { + "Stop": [ + { + "hooks": [ + { + "type": "command", + "command": "@CHOCK_PYTHON@ \".chock/bin/codex_cli.py\" --gate \".chock/compiled/stability-script/stop/gate.json\"", + "commandWindows": "& @CHOCK_PYTHON@ \".chock/bin/codex_cli.py\" --gate \".chock/compiled/stability-script/stop/gate.json\"" + } + ] + } + ] + } +} \ No newline at end of file diff --git a/tests/fixtures/emitter_stability/golden/stability-script/stop/devin-hooks.json b/tests/fixtures/emitter_stability/golden/stability-script/stop/devin-hooks.json new file mode 100644 index 0000000..2eb7295 --- /dev/null +++ b/tests/fixtures/emitter_stability/golden/stability-script/stop/devin-hooks.json @@ -0,0 +1,12 @@ +{ + "Stop": [ + { + "hooks": [ + { + "type": "command", + "command": "@CHOCK_PYTHON@ \".chock/bin/devin.py\" --gate \".chock/compiled/stability-script/stop/gate.json\"" + } + ] + } + ] +} \ No newline at end of file diff --git a/tests/fixtures/emitter_stability/golden/stability-script/stop/gate.json b/tests/fixtures/emitter_stability/golden/stability-script/stop/gate.json new file mode 100644 index 0000000..8f73db1 --- /dev/null +++ b/tests/fixtures/emitter_stability/golden/stability-script/stop/gate.json @@ -0,0 +1,12 @@ +{ + "kind": "script", + "on": [ + "commit", + "tool_use" + ], + "action": "block", + "message": "Refused by the fixture's own script.", + "params": { + "script": "tests/fixtures/emitter_stability/policies/stability-script/implementations/stability-script-gate.py" + } +} \ No newline at end of file diff --git a/tests/fixtures/emitter_stability/golden/stability-script/stop/gemini_cli-hooks.json b/tests/fixtures/emitter_stability/golden/stability-script/stop/gemini_cli-hooks.json new file mode 100644 index 0000000..453dd4f --- /dev/null +++ b/tests/fixtures/emitter_stability/golden/stability-script/stop/gemini_cli-hooks.json @@ -0,0 +1,14 @@ +{ + "hooks": { + "AfterAgent": [ + { + "hooks": [ + { + "type": "command", + "command": "@CHOCK_PYTHON@ \".chock/bin/gemini_cli.py\" --gate \".chock/compiled/stability-script/stop/gate.json\"" + } + ] + } + ] + } +} \ No newline at end of file diff --git a/tests/fixtures/emitter_stability/golden/stability-script/stop/stop.json b/tests/fixtures/emitter_stability/golden/stability-script/stop/stop.json new file mode 100644 index 0000000..592f036 --- /dev/null +++ b/tests/fixtures/emitter_stability/golden/stability-script/stop/stop.json @@ -0,0 +1,9 @@ +{ + "hooks": [ + { + "type": "command", + "command": "@CHOCK_PYTHON@ \"${CLAUDE_PROJECT_DIR}/.chock/bin/claude_code.py\" --gate \"${CLAUDE_PROJECT_DIR}/.chock/compiled/stability-script/stop/gate.json\"", + "timeout": 30 + } + ] +} \ No newline at end of file diff --git a/tests/fixtures/emitter_stability/golden/stability-script/stop/tabnine-hooks.json b/tests/fixtures/emitter_stability/golden/stability-script/stop/tabnine-hooks.json new file mode 100644 index 0000000..c26c700 --- /dev/null +++ b/tests/fixtures/emitter_stability/golden/stability-script/stop/tabnine-hooks.json @@ -0,0 +1,15 @@ +{ + "hooks": { + "AfterAgent": [ + { + "hooks": [ + { + "type": "command", + "command": "@CHOCK_PYTHON@ \".chock/bin/tabnine.py\" --gate \".chock/compiled/stability-script/stop/gate.json\"", + "name": "agentseam" + } + ] + } + ] + } +} \ No newline at end of file diff --git a/tests/fixtures/emitter_stability/policies/stability-script/implementations/stability-script-gate.py b/tests/fixtures/emitter_stability/policies/stability-script/implementations/stability-script-gate.py new file mode 100644 index 0000000..8bb5dfd --- /dev/null +++ b/tests/fixtures/emitter_stability/policies/stability-script/implementations/stability-script-gate.py @@ -0,0 +1,6 @@ +"""Fixture gate script: allows everything. Its job is to exist, so the emitters have a script to name.""" + +import sys + +sys.stdin.read() +sys.exit(0) diff --git a/tests/fixtures/emitter_stability/policies/stability-script/manifest.yaml b/tests/fixtures/emitter_stability/policies/stability-script/manifest.yaml new file mode 100644 index 0000000..662d651 --- /dev/null +++ b/tests/fixtures/emitter_stability/policies/stability-script/manifest.yaml @@ -0,0 +1,48 @@ +# Frozen fixture: exercises the emitters for a script-backed gate (git-hook, ci-gate, +# pre-tool-use write fragment, stop, ambient-rule). Do not "improve" this policy -- its only +# job is to be compiled identically by every patch release. See test_emitter_stability.py. +id: stability-script +name: "Emitter Stability Script Gate" +version: "0.0.1" +description: > + Fixture policy for the emitter-stability guarantee. A gate whose check is a script the + policy ships, so every surface a write-path gate reaches has output worth comparing + byte-for-byte across releases. +artifact: hook +enforcement: block +effects: +- read_only +approval: + required: false + +hook: + gate: + kind: script + "on": [commit, tool_use] + action: block + message: > + Refused by the fixture's own script. + params: + script: stability-script-gate.py + +provenance: + author: "chock-core" + created_at: "2026-09-22T00:00:00Z" + updated_at: "2026-09-22T00:00:00Z" + source_repo: "https://github.com/open-coder-ai/chock" + license: "Apache-2.0" + trust_tier: "community" + +lifecycle: + status: draft + reviewed_by: + - "chock-core" + +security: + content_instructions: never-obey + +changelog: +- version: 0.0.1 + date: '2026-09-22' + changes: + - "Frozen emitter-stability fixture for a script-backed gate." diff --git a/tests/test_gate_script_kind.py b/tests/test_gate_script_kind.py new file mode 100644 index 0000000..b74f002 --- /dev/null +++ b/tests/test_gate_script_kind.py @@ -0,0 +1,224 @@ +"""A gate whose check is a script the policy ships: the runner's material, the script's verdict. + +The declarative kinds answer questions a closed table can hold. A flow model over a method +body cannot be written that way, and until now the only door for a script was the shell +guard, which judges a command and never sees the file being written. `kind: script` hands +the policy's own program the same material every declarative kind reads -- the staged blobs +at commit, the write at tool use and at the turn's end -- and carries back its verdict. +""" + +from __future__ import annotations + +import json +import textwrap +from pathlib import Path + +import pytest +import yaml +from conftest import init_repo, stage + +from chock.compile.emitters.in_agent import GATE_FILE, STOP_FRAGMENT, WRITE_FRAGMENT, emit_pre_tool_use, emit_stop +from chock.gate import runner +from chock.gate.build import build_gate_json +from chock.gate.runner import WRITE_PATH_KINDS, run +from chock.validation.checks_gate_shape import _validate_gate +from chock.validation.report import Report + +POLICY_ID = "scripted" +SCRIPT = "scripted-gate.py" +MARKER = "FORBIDDEN" + +#: Refuses any write carrying MARKER, in its own words. Two other markers make it misbehave, +#: so the runner's answer to a script that gives no verdict can be pinned without a second file. +GATE_SCRIPT = textwrap.dedent( + """\ + import json, sys, time + payload = json.load(sys.stdin) + texts = payload["writes"] + if any("CRASH" in t for t in texts.values()): + sys.exit(3) + if any("HANG" in t for t in texts.values()): + time.sleep(5) + hits = sorted(p for p, t in texts.items() if "FORBIDDEN" in t) + if hits: + print("scripted: forbidden marker in " + ", ".join(hits), file=sys.stderr) + sys.exit(1) + sys.exit(0) + """ +) + + +def _policy(repo: Path, *, on=("commit", "tool_use"), ship_script: bool = True) -> tuple[Path, dict]: + policy = repo / ".agents" / "policies" / POLICY_ID + (policy / "implementations").mkdir(parents=True, exist_ok=True) + manifest = { + "id": POLICY_ID, + "name": "Scripted", + "version": "0.0.1", + "description": "d", + "artifact": "hook", + "enforcement": "block", + "hook": { + "gate": {"kind": "script", "on": list(on), "action": "block", "message": "m", "params": {"script": SCRIPT}} + }, + "provenance": {"author": "t"}, + "lifecycle": {"status": "draft"}, + } + (policy / "manifest.yaml").write_text(yaml.safe_dump(manifest), encoding="utf-8") + if ship_script: + (policy / "implementations" / SCRIPT).write_text(GATE_SCRIPT, encoding="utf-8") + return policy, manifest + + +def _gate(repo: Path, **kw) -> Path: + """The compiled gate.json for a scripted policy in `repo`, where the git hook would find it.""" + policy, _ = _policy(repo, **kw) + spec = build_gate_json(policy, repo) + assert spec is not None + gate = repo / ".chock" / "compiled" / POLICY_ID / "git-hook" / GATE_FILE + gate.parent.mkdir(parents=True, exist_ok=True) + gate.write_text(json.dumps(spec), encoding="utf-8") + return gate + + +# --- the compiled gate ------------------------------------------------------------------------ + + +def test_the_compiled_gate_names_the_script_from_the_repository_root(tmp_path: Path) -> None: + policy, _ = _policy(tmp_path) + spec = build_gate_json(policy, tmp_path) + assert spec is not None + assert spec["params"]["script"] == f".agents/policies/{POLICY_ID}/implementations/{SCRIPT}" + + +def test_a_script_can_answer_for_a_write() -> None: + assert "script" in WRITE_PATH_KINDS + + +# --- at commit, the staged blobs ------------------------------------------------------------------ + + +def test_a_staged_file_the_script_refuses_blocks_the_commit(tmp_path: Path, capsys: pytest.CaptureFixture) -> None: + init_repo(tmp_path) + gate = _gate(tmp_path) + stage(tmp_path, "App.java", f"x = {MARKER}\n") + assert run(gate, "pre-commit", None, tmp_path) == 1 + assert "scripted: forbidden marker in App.java" in capsys.readouterr().err + + +def test_a_clean_staged_file_passes(tmp_path: Path) -> None: + init_repo(tmp_path) + gate = _gate(tmp_path) + stage(tmp_path, "App.java", "x = 1\n") + assert run(gate, "pre-commit", None, tmp_path) == 0 + + +def test_nothing_staged_is_nothing_to_refuse(tmp_path: Path) -> None: + init_repo(tmp_path) + assert run(_gate(tmp_path), "pre-commit", None, tmp_path) == 0 + + +# --- at tool use and at the turn's end, the write -------------------------------------------------- + + +@pytest.mark.parametrize("event", ["pre-tool-use", "stop"]) +def test_a_write_is_judged_by_the_same_script(tmp_path: Path, event: str) -> None: + init_repo(tmp_path) + gate = _gate(tmp_path) + assert run(gate, event, None, tmp_path, writes={"App.java": MARKER}) == 1 + assert run(gate, event, None, tmp_path, writes={"App.java": "clean"}) == 0 + + +def test_the_scripts_own_words_are_the_reason(tmp_path: Path, capsys: pytest.CaptureFixture) -> None: + """The policy's generic message would say less than the script, which knows which rule and where.""" + init_repo(tmp_path) + gate = _gate(tmp_path) + run(gate, "pre-tool-use", None, tmp_path, writes={"App.java": MARKER}) + assert "scripted: forbidden marker in App.java" in capsys.readouterr().err + + +# --- no verdict is a refusal ---------------------------------------------------------------------- + + +def test_a_script_that_crashes_refuses(tmp_path: Path, capsys: pytest.CaptureFixture) -> None: + init_repo(tmp_path) + gate = _gate(tmp_path) + assert run(gate, "pre-tool-use", None, tmp_path, writes={"App.java": "CRASH"}) == 1 + err = capsys.readouterr().err + assert "exited 3" in err + assert "refusing" in err + + +def test_a_script_that_is_not_installed_refuses(tmp_path: Path, capsys: pytest.CaptureFixture) -> None: + init_repo(tmp_path) + gate = _gate(tmp_path, ship_script=False) + assert run(gate, "pre-tool-use", None, tmp_path, writes={"App.java": "clean"}) == 1 + assert "not installed" in capsys.readouterr().err + + +def test_a_script_that_never_answers_refuses( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture +) -> None: + monkeypatch.setattr(runner, "_SCRIPT_TIMEOUT_SECONDS", 1) + init_repo(tmp_path) + gate = _gate(tmp_path) + assert run(gate, "pre-tool-use", None, tmp_path, writes={"App.java": "HANG"}) == 1 + assert "no verdict" in capsys.readouterr().err + + +# --- the emitters wire it where every write-path gate goes --------------------------------------- + + +def test_a_script_gate_reaches_the_write_path(tmp_path: Path) -> None: + policy, manifest = _policy(tmp_path) + out = tmp_path / ".chock" / "compiled" / POLICY_ID / "pre-tool-use" + out.mkdir(parents=True) + names = {p.name for p in emit_pre_tool_use(policy, out, manifest)} + assert {GATE_FILE, WRITE_FRAGMENT} <= names + spec = json.loads((out / GATE_FILE).read_text(encoding="utf-8")) + assert spec["kind"] == "script" + assert spec["params"]["script"].endswith(f"implementations/{SCRIPT}") + + +def test_a_script_gate_reaches_the_turns_end(tmp_path: Path) -> None: + policy, manifest = _policy(tmp_path) + out = tmp_path / ".chock" / "compiled" / POLICY_ID / "stop" + out.mkdir(parents=True) + names = {p.name for p in emit_stop(policy, out, manifest)} + assert {GATE_FILE, STOP_FRAGMENT} <= names + + +def test_a_commit_only_script_gate_stays_out_of_the_agent(tmp_path: Path) -> None: + policy, manifest = _policy(tmp_path, on=("commit",)) + pre = tmp_path / ".chock" / "compiled" / POLICY_ID / "pre-tool-use" + end = tmp_path / ".chock" / "compiled" / POLICY_ID / "stop" + assert emit_pre_tool_use(policy, pre, manifest) == [] + assert emit_stop(policy, end, manifest) == [] + + +# --- the validator refuses what the runner would ---------------------------------------------------- + + +def _messages(gate: dict, artifact_dir: Path | None = None) -> list[str]: + report = Report() + _validate_gate(gate, "manifest", report, tool_use_allowed=True, artifact_dir=artifact_dir) + return [f.message for f in report.errors] + + +def _spec(script: str) -> dict: + return {"kind": "script", "on": ["commit"], "action": "block", "message": "m", "params": {"script": script}} + + +@pytest.mark.parametrize("name", ["../escape.py", "sub/dir.py", "gate.sh", ""]) +def test_the_script_must_be_a_bare_python_file_name(name: str) -> None: + assert any("script" in m for m in _messages(_spec(name))) + + +def test_a_declared_script_must_be_shipped(tmp_path: Path) -> None: + policy, _ = _policy(tmp_path, ship_script=False) + assert any("not there" in m for m in _messages(_spec(SCRIPT), policy)) + + +def test_a_shipped_script_validates(tmp_path: Path) -> None: + policy, _ = _policy(tmp_path) + assert _messages(_spec(SCRIPT), policy) == []