From 3f109322b34c697255affaab884da06aed8c6a41 Mon Sep 17 00:00:00 2001 From: Nishar Date: Sun, 13 Sep 2026 08:35:25 -0500 Subject: [PATCH 1/2] test(conformance): add offline-verifiable ACTION fixture bundles Serialize a representative set of delegation-linked action evidence cases to disk as self-contained chain.json + dag.json + expected.json bundles under tests/fixtures/action/, re-verified through the shipped `ca2a verify-dag --chain` command against the committed blobs. Until now every ACTION case was built in memory, so the offline-verification claim was never exercised end to end from files. Adds a seeded, byte-reproducible generator (scripts/gen_action_fixtures.py, with a --check staleness mode), a loader test that reads the committed blobs, and lifts the committed-blob helper into tests/committed_blobs.py so the existing example test and the new loader test share it. The bundles cover the offline provenance, authorization-denial, and validity-window axes; they do not exercise holder-proof authorization replay, which is not offline-replayable evidence. Idea credit: @Ahmedibrahim222 (#36). Refs #164. Co-Authored-By: Claude Opus 4.8 (1M context) Signed-off-by: Nishar --- scripts/gen_action_fixtures.py | 205 ++++++++++++++++++ tests/committed_blobs.py | 36 +++ .../test_action_fixture_bundles.py | 101 +++++++++ tests/fixtures/action/README.md | 58 +++++ .../action/action-001-verified/chain.json | 29 +++ .../action/action-001-verified/dag.json | 27 +++ .../action/action-001-verified/expected.json | 10 + .../chain.json | 29 +++ .../action-002-parent-hash-mismatch/dag.json | 27 +++ .../expected.json | 10 + .../action-006-policy-denial/chain.json | 42 ++++ .../action/action-006-policy-denial/dag.json | 55 +++++ .../action-006-policy-denial/expected.json | 10 + .../action-010-scope-escalation/chain.json | 27 +++ .../action-010-scope-escalation/dag.json | 25 +++ .../action-010-scope-escalation/expected.json | 10 + .../action-012-credential-expired/chain.json | 33 +++ .../action-012-credential-expired/dag.json | 27 +++ .../expected.json | 10 + tests/unit/test_committed_examples_verify.py | 19 +- 20 files changed, 772 insertions(+), 18 deletions(-) create mode 100644 scripts/gen_action_fixtures.py create mode 100644 tests/committed_blobs.py create mode 100644 tests/conformance/test_action_fixture_bundles.py create mode 100644 tests/fixtures/action/README.md create mode 100644 tests/fixtures/action/action-001-verified/chain.json create mode 100644 tests/fixtures/action/action-001-verified/dag.json create mode 100644 tests/fixtures/action/action-001-verified/expected.json create mode 100644 tests/fixtures/action/action-002-parent-hash-mismatch/chain.json create mode 100644 tests/fixtures/action/action-002-parent-hash-mismatch/dag.json create mode 100644 tests/fixtures/action/action-002-parent-hash-mismatch/expected.json create mode 100644 tests/fixtures/action/action-006-policy-denial/chain.json create mode 100644 tests/fixtures/action/action-006-policy-denial/dag.json create mode 100644 tests/fixtures/action/action-006-policy-denial/expected.json create mode 100644 tests/fixtures/action/action-010-scope-escalation/chain.json create mode 100644 tests/fixtures/action/action-010-scope-escalation/dag.json create mode 100644 tests/fixtures/action/action-010-scope-escalation/expected.json create mode 100644 tests/fixtures/action/action-012-credential-expired/chain.json create mode 100644 tests/fixtures/action/action-012-credential-expired/dag.json create mode 100644 tests/fixtures/action/action-012-credential-expired/expected.json diff --git a/scripts/gen_action_fixtures.py b/scripts/gen_action_fixtures.py new file mode 100644 index 0000000..9e84e6e --- /dev/null +++ b/scripts/gen_action_fixtures.py @@ -0,0 +1,205 @@ +#!/usr/bin/env python3 +"""Generate the offline-verifiable ACTION fixture bundles under tests/fixtures/action/. + +Each bundle is a chain.json + dag.json an auditor re-checks with +``ca2a verify-dag --chain``, plus an expected.json stating the verdict. Keys are +seeded, so regenerating is byte-stable and does not churn the diff. The loader +test (tests/conformance/test_action_fixture_bundles.py) verifies the committed +blobs, which is what catches a bundle nobody regenerated. + +Idea credit: @Ahmedibrahim222 (agentrust-io/ca2a#36). Tracked in #164. + + python scripts/gen_action_fixtures.py +""" + +# ruff: noqa: T201 +from __future__ import annotations + +import hashlib +import json +import sys +from dataclasses import dataclass, replace +from pathlib import Path + +REPO_ROOT = Path(__file__).resolve().parent.parent +sys.path.insert(0, str(REPO_ROOT / "src")) + +from cryptography.hazmat.primitives.asymmetric.ed25519 import Ed25519PrivateKey # noqa: E402 + +from ca2a_runtime.delegation import DelegationCredential # noqa: E402 +from ca2a_runtime.provenance import DelegationRecord, denial_record_for, record_for # noqa: E402 + +FIXTURE_DIR = REPO_ROOT / "tests" / "fixtures" / "action" +_WRONG_PARENT_HASH = "0" * 64 # well-formed hex that matches no record's hash + +_VERIFIED = { + "provenance_status": "verified", + "authorization_decision": "allowed", + "controller_outcome": "accepted", +} +_INVALID = { + "provenance_status": "invalid", + "authorization_decision": "not_evaluated", + "controller_outcome": "not_evaluated", +} +_DENIED = { + "provenance_status": "verified", + "authorization_decision": "denied", + "controller_outcome": "not_evaluated", +} + + +@dataclass(frozen=True) +class Bundle: + name: str + chain: list[DelegationCredential] + records: list[DelegationRecord] + verdict: dict[str, str] + code: str | None = None + at_time: int | None = None + + +def _keypair(label: str) -> tuple[Ed25519PrivateKey, str]: + """A deterministic Ed25519 keypair (RFC 8032 signing is deterministic).""" + priv = Ed25519PrivateKey.from_private_bytes(hashlib.sha256(label.encode()).digest()) + return priv, priv.public_key().public_bytes_raw().hex() + + +def _chain( + case: str, + scopes: list[frozenset[str]], + *, + not_before: int | None = None, + not_after: int | None = None, +) -> list[DelegationCredential]: + """A signed chain, one hop per scope. sign() does not check attenuation, so a + case may pass a widening scope to exercise the escalation check.""" + chain: list[DelegationCredential] = [] + priv, pub = _keypair(f"{case}::0") + parent_id: str | None = None + for depth, scope in enumerate(scopes): + next_priv, next_pub = _keypair(f"{case}::{depth + 1}") + cred = DelegationCredential( + credential_id=f"cred-{depth}", + issuer=pub, + subject=next_pub, + scope=scope, + depth=depth, + parent_id=parent_id, + not_before=not_before, + not_after=not_after, + ).sign(priv) + chain.append(cred) + parent_id = cred.credential_id + priv, pub = next_priv, next_pub + return chain + + +def _allow_records(chain: list[DelegationCredential]) -> list[DelegationRecord]: + records: list[DelegationRecord] = [] + parent_hash: str | None = None + for depth, cred in enumerate(chain): + rec = record_for(cred, record_id=f"rec-{depth}", parent_record_hash=parent_hash) + records.append(rec) + parent_hash = rec.record_hash() + return records + + +def _bundles() -> list[Bundle]: + # ACTION-001: narrowing chain with a matching linked DAG. + c = _chain( + "action-001", + [ + frozenset({"robot.move", "robot.inspect", "robot.stop"}), + frozenset({"robot.move", "robot.inspect"}), + ], + ) + verified = Bundle("action-001-verified", c, _allow_records(c), _VERIFIED) + + # ACTION-002: child record points at a parent hash that matches nothing. + c = _chain( + "action-002", + [ + frozenset({"robot.move", "robot.inspect", "robot.stop"}), + frozenset({"robot.move", "robot.inspect"}), + ], + ) + recs = _allow_records(c) + recs[1] = replace(recs[1], parent_record_hash=_WRONG_PARENT_HASH) + parent_mismatch = Bundle( + "action-002-parent-hash-mismatch", c, recs, _INVALID, "PROVENANCE_LINK_BROKEN" + ) + + # ACTION-005/006: valid chain whose leaf hop refuses an out-of-scope call, + # recorded as a linked denial rather than dropped. + c = _chain( + "action-006", + [ + frozenset({"task:read", "task:write", "tool:search", "tool:purchase"}), + frozenset({"task:read", "tool:search", "tool:purchase"}), + frozenset({"tool:search"}), + ], + ) + recs = _allow_records(c) + reason = "capability 'tool:purchase' is not in the effective scope" + recs.append( + denial_record_for( + c[-1], + record_id="rec-denied-purchase", + parent_record_hash=recs[-1].record_hash(), + requested_capability="tool:purchase", + effective_scope=c[-1].scope, + reason=reason, + ) + ) + denial = Bundle("action-006-policy-denial", c, recs, _DENIED, reason) + + # ACTION-010: hop 1 widens rather than narrows. + c = _chain("action-010", [frozenset({"robot.move"}), frozenset({"robot.move", "robot.fly"})]) + escalation = Bundle( + "action-010-scope-escalation", c, _allow_records(c), _INVALID, "SCOPE_ESCALATION" + ) + + # ACTION-012: windowed chain replayed after its window has lapsed. + c = _chain( + "action-012", + [ + frozenset({"robot.move", "robot.inspect", "robot.stop"}), + frozenset({"robot.move", "robot.inspect"}), + ], + not_before=1_000, + not_after=2_000, + ) + expired = Bundle( + "action-012-credential-expired", + c, + _allow_records(c), + _INVALID, + "CREDENTIAL_EXPIRED", + at_time=3_000, + ) + + return [verified, parent_mismatch, denial, escalation, expired] + + +def main() -> int: + for b in _bundles(): + d = FIXTURE_DIR / b.name + d.mkdir(parents=True, exist_ok=True) + chain_doc = {"chain": [{**c.body(), "signature": c.signature} for c in b.chain]} + dag_doc = {"records": [r.body() for r in b.records]} + expected = { + "trusted_root_issuer": b.chain[0].issuer, + "at_time": b.at_time, + "verdict": b.verdict, + "code": b.code, + } + (d / "chain.json").write_text(json.dumps(chain_doc, indent=2) + "\n", encoding="utf-8") + (d / "dag.json").write_text(json.dumps(dag_doc, indent=2) + "\n", encoding="utf-8") + (d / "expected.json").write_text(json.dumps(expected, indent=2) + "\n", encoding="utf-8") + print(f"wrote {len(_bundles())} bundles under {FIXTURE_DIR}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/committed_blobs.py b/tests/committed_blobs.py new file mode 100644 index 0000000..ada2376 --- /dev/null +++ b/tests/committed_blobs.py @@ -0,0 +1,36 @@ +"""Read committed blobs out of git rather than the working tree. + +Tests that verify committed artifacts (delegation chains, provenance DAGs, the +ACTION fixture bundles) must read what is *in the repository*, not what a demo +or a generator just wrote beside them. Otherwise a run that regenerates the +artifact makes its own test pass, and a change to a hashed body that forgets to +regenerate the committed copy still goes green. Reading the committed blob means +a stale artifact fails instead. + +Both ``tests/unit/test_committed_examples_verify.py`` and +``tests/conformance/test_action_fixture_bundles.py`` import from here so the one +helper backs both suites. +""" + +from __future__ import annotations + +import subprocess +from pathlib import Path + +# tests/committed_blobs.py -> parents[1] is the repository root. +REPO_ROOT = Path(__file__).resolve().parents[1] + + +def committed(path: str) -> str | None: + """The blob at HEAD for ``path``, or None if git cannot tell us.""" + try: + out = subprocess.run( # noqa: S603 + ["git", "show", f"HEAD:{path}"], # noqa: S607 + capture_output=True, + text=True, + cwd=REPO_ROOT, + check=False, + ) + except OSError: + return None + return out.stdout if out.returncode == 0 else None diff --git a/tests/conformance/test_action_fixture_bundles.py b/tests/conformance/test_action_fixture_bundles.py new file mode 100644 index 0000000..ae121e5 --- /dev/null +++ b/tests/conformance/test_action_fixture_bundles.py @@ -0,0 +1,101 @@ +"""The committed ACTION fixture bundles must verify offline as documented. + +Each bundle under ``tests/fixtures/action//`` is a self-contained +``chain.json`` + ``dag.json`` an auditor re-checks with the shipped +``ca2a verify-dag --chain`` command, plus an ``expected.json`` that states the +verdict (see ``tests/fixtures/action/README.md``). These tests run that command +against the *committed* blobs, so a change to the record body or the chain model +that forgets to regenerate the bundles fails here rather than silently passing +against a bundle the generator just rewrote. Regenerate with +``python scripts/gen_action_fixtures.py``. + +This exercises the offline provenance / authorization-denial / validity-window +axes. It does not exercise holder-proof authorization replay, which needs live +audience/secret/challenge material and is not offline-replayable evidence; see +``tests/conformance/README.md`` on the ACTION helper and holder-proof binding. +""" + +from __future__ import annotations + +import json +from pathlib import Path +from typing import Any + +import pytest + +from ca2a_runtime.cli import main as cli_main +from tests.committed_blobs import REPO_ROOT, committed + +_BUNDLE_ROOT = REPO_ROOT / "tests" / "fixtures" / "action" +BUNDLES = ( + sorted(p.name for p in _BUNDLE_ROOT.iterdir() if p.is_dir()) if _BUNDLE_ROOT.is_dir() else [] +) + + +def _run_verify_dag( + chain: str, + dag: str, + expected: dict[str, Any], + tmp_path: Path, + capsys: pytest.CaptureFixture[str], +) -> tuple[int, dict[str, Any]]: + """Run the shipped CLI on the committed blobs, returning (exit_code, output).""" + chain_path = tmp_path / "chain.json" + dag_path = tmp_path / "dag.json" + chain_path.write_text(chain, encoding="utf-8") + dag_path.write_text(dag, encoding="utf-8") + + argv = [ + "verify-dag", + "--dag", + str(dag_path), + "--chain", + str(chain_path), + "--trusted-root-issuer", + expected["trusted_root_issuer"], + ] + if expected.get("at_time") is not None: + argv += ["--at-time", str(expected["at_time"])] + rc = cli_main(argv) + out = json.loads(capsys.readouterr().out.strip().splitlines()[-1]) + return rc, out + + +@pytest.mark.parametrize("bundle", BUNDLES) +def test_action_bundle_verifies_as_documented( + bundle: str, tmp_path: Path, capsys: pytest.CaptureFixture[str] +) -> None: + base = f"tests/fixtures/action/{bundle}" + chain = committed(f"{base}/chain.json") + dag = committed(f"{base}/dag.json") + expected_blob = committed(f"{base}/expected.json") + if chain is None or dag is None or expected_blob is None: + pytest.skip(f"{bundle} is not committed yet or git is unavailable") + expected = json.loads(expected_blob) + + rc, out = _run_verify_dag(chain, dag, expected, tmp_path, capsys) + + verdict = expected["verdict"] + if verdict["provenance_status"] == "invalid": + # A defect in the chain or its records: the CLI fails closed and names + # the reason. A bundle that agrees on failing but disagrees on why is + # the case worth catching. + assert rc == 1, out + assert out["verified"] is False + assert out["code"] == expected["code"] + elif verdict["authorization_decision"] == "denied": + # Valid provenance plus a recorded authorization denial: the DAG still + # verifies, and the refusal is evidence rather than an absence of it. + assert rc == 0, out + assert out["verified"] is True + assert out.get("outcome") == "denied" + assert out.get("denial_reason") == expected["code"] + else: + assert rc == 0, out + assert out["verified"] is True + assert out.get("outcome") != "denied" + + +def test_bundles_are_present() -> None: + """Guard against an empty parametrization silently passing zero cases.""" + assert BUNDLES, "no ACTION fixture bundles found under tests/fixtures/action/" diff --git a/tests/fixtures/action/README.md b/tests/fixtures/action/README.md new file mode 100644 index 0000000..daf01b1 --- /dev/null +++ b/tests/fixtures/action/README.md @@ -0,0 +1,58 @@ +# ACTION fixture bundles + +Self-contained, offline-verifiable bundles for the delegation-linked action +evidence cases (Group 7, `ACTION-*`) in +[`tests/conformance/README.md`](../../conformance/README.md). + +Every other ACTION case is assembled in memory inside +`tests/conformance/test_profile_conformance.py`. These bundles instead serialize +a case to disk so it is re-verified end-to-end with the **shipped offline +verifier**, the same `ca2a verify-dag --chain` an auditor runs, not through live +Python objects. That is the point: cA2A's offline-verifiable provenance claim, +checked from committed files by a third party with no access to the runtime. + +Idea proposed by @Ahmedibrahim222 in +[#36](https://github.com/agentrust-io/ca2a/issues/36); scoped and tracked in +[#164](https://github.com/agentrust-io/ca2a/issues/164). + +## Layout + +One directory per case, each holding: + +| File | What it is | +|---|---| +| `chain.json` | The signed delegation chain (`{"chain": [...]}`), as `ca2a verify-chain` consumes it. | +| `dag.json` | The provenance records (`{"records": [...]}`), as `ca2a verify-dag --dag` consumes them. | +| `expected.json` | The documented verdict: the ACTION three-axis outcome (`provenance_status` / `authorization_decision` / `controller_outcome`), the `trusted_root_issuer`, an optional `at_time`, and the reason `code` a failure or denial carries. | + +## What these do and do not cover + +The offline path covers provenance (chain signatures, attenuation, and validity +windows), the DAG, the chain↔record cross-check, and any recorded denial +outcome. It does **not** exercise the holder-proof *authorization-replay* axis: +that needs live audience / secret / challenge material and is not +offline-replayable evidence. See the ACTION helper and holder-proof note in +[`tests/conformance/README.md`](../../conformance/README.md) rather than a +restatement here. + +## Cases + +| Bundle | ACTION | Offline verdict | +|---|---|---| +| `action-001-verified` | ACTION-001 | `verify-dag` verifies; provenance verified, authorized, accepted. | +| `action-002-parent-hash-mismatch` | ACTION-002 | Fails closed with `PROVENANCE_LINK_BROKEN`. | +| `action-006-policy-denial` | ACTION-005/006 | Verifies with a recorded authorization denial (`outcome: denied`). | +| `action-010-scope-escalation` | ACTION-010 | Fails closed with `SCOPE_ESCALATION`. | +| `action-012-credential-expired` | ACTION-012 | Fails closed with `CREDENTIAL_EXPIRED` when replayed at `at_time=3000`. | + +## Regenerating + +The bundles are generated with seeded keys, so their bytes are reproducible: + +```bash +python scripts/gen_action_fixtures.py +``` + +`tests/conformance/test_action_fixture_bundles.py` verifies the **committed** +blobs (read out of git, not the working tree), so a change to the record body or +the chain model that forgets to regenerate these fails there. diff --git a/tests/fixtures/action/action-001-verified/chain.json b/tests/fixtures/action/action-001-verified/chain.json new file mode 100644 index 0000000..09fa7b5 --- /dev/null +++ b/tests/fixtures/action/action-001-verified/chain.json @@ -0,0 +1,29 @@ +{ + "chain": [ + { + "credential_id": "cred-0", + "issuer": "eca68016b05d3a93c7c7fdc6a684d5bf809652c1b90802be2e0218bce332aa07", + "subject": "9ba3b932dedd958cd2736281741ca548ee03f0bbcfd955b77f804764b5e496e2", + "scope": [ + "robot.inspect", + "robot.move", + "robot.stop" + ], + "depth": 0, + "parent_id": null, + "signature": "5dc08de6d4e8934e95dfa12153ba8a238cf4daac1b0e9ce9b7d4faf9a5f24b9018b75c54f8034cb26bcd6bd8e1712364e3fa2f377b36d9e5a9ff05e03123a90d" + }, + { + "credential_id": "cred-1", + "issuer": "9ba3b932dedd958cd2736281741ca548ee03f0bbcfd955b77f804764b5e496e2", + "subject": "08efb8d82b9f954b85c2ebb8e736523ce5c715a61ee303f8bb605673a48c5560", + "scope": [ + "robot.inspect", + "robot.move" + ], + "depth": 1, + "parent_id": "cred-0", + "signature": "17a1713893e482f0cbc3e6f358ffe3b7da75b474188265fbfbe581e10538950f7a0a3fe4c3fa92a0b29f5566e1bf323089d0862b5f0051d854198708da856b0b" + } + ] +} diff --git a/tests/fixtures/action/action-001-verified/dag.json b/tests/fixtures/action/action-001-verified/dag.json new file mode 100644 index 0000000..e77c2c4 --- /dev/null +++ b/tests/fixtures/action/action-001-verified/dag.json @@ -0,0 +1,27 @@ +{ + "records": [ + { + "record_id": "rec-0", + "credential_id": "cred-0", + "subject": "9ba3b932dedd958cd2736281741ca548ee03f0bbcfd955b77f804764b5e496e2", + "scope": [ + "robot.inspect", + "robot.move", + "robot.stop" + ], + "parent_record_hash": null, + "caller_attestation": "not_offered" + }, + { + "record_id": "rec-1", + "credential_id": "cred-1", + "subject": "08efb8d82b9f954b85c2ebb8e736523ce5c715a61ee303f8bb605673a48c5560", + "scope": [ + "robot.inspect", + "robot.move" + ], + "parent_record_hash": "f302c4bb389270f77e4419d2d412bd6864e0368c27a6e5df99f480c0c24a92ee", + "caller_attestation": "not_offered" + } + ] +} diff --git a/tests/fixtures/action/action-001-verified/expected.json b/tests/fixtures/action/action-001-verified/expected.json new file mode 100644 index 0000000..130fba4 --- /dev/null +++ b/tests/fixtures/action/action-001-verified/expected.json @@ -0,0 +1,10 @@ +{ + "trusted_root_issuer": "eca68016b05d3a93c7c7fdc6a684d5bf809652c1b90802be2e0218bce332aa07", + "at_time": null, + "verdict": { + "provenance_status": "verified", + "authorization_decision": "allowed", + "controller_outcome": "accepted" + }, + "code": null +} diff --git a/tests/fixtures/action/action-002-parent-hash-mismatch/chain.json b/tests/fixtures/action/action-002-parent-hash-mismatch/chain.json new file mode 100644 index 0000000..b390b26 --- /dev/null +++ b/tests/fixtures/action/action-002-parent-hash-mismatch/chain.json @@ -0,0 +1,29 @@ +{ + "chain": [ + { + "credential_id": "cred-0", + "issuer": "502b5def5d7e08f161a8df11c88b012607d78eebf090d2ee97d2420523ba8c60", + "subject": "42278f8c88f3af6e4e5b85319e077bf1979cefdd498d7dda1f4201d22ffd40d8", + "scope": [ + "robot.inspect", + "robot.move", + "robot.stop" + ], + "depth": 0, + "parent_id": null, + "signature": "3a0878ec9bc2f78dcd7e5f14e95623907021594b94eab91abf0097b667912eb75a958f58fcffd395dcf653de5f33cc618b3a8ed5e425dcc0b7c9456a5259bb09" + }, + { + "credential_id": "cred-1", + "issuer": "42278f8c88f3af6e4e5b85319e077bf1979cefdd498d7dda1f4201d22ffd40d8", + "subject": "3ebb3ddb2865c10726b847ef6b6381516bcce4ab46056c0fd323d5f52e1b52f6", + "scope": [ + "robot.inspect", + "robot.move" + ], + "depth": 1, + "parent_id": "cred-0", + "signature": "a67c32a264e71131923d385080d9d41012768978f2b92c4f4db2b3d09d553d04f70f5c2f47ad6c5f2b49067b97da5b78069c0e19aeb569bfa3d1076306d2390f" + } + ] +} diff --git a/tests/fixtures/action/action-002-parent-hash-mismatch/dag.json b/tests/fixtures/action/action-002-parent-hash-mismatch/dag.json new file mode 100644 index 0000000..e1b928b --- /dev/null +++ b/tests/fixtures/action/action-002-parent-hash-mismatch/dag.json @@ -0,0 +1,27 @@ +{ + "records": [ + { + "record_id": "rec-0", + "credential_id": "cred-0", + "subject": "42278f8c88f3af6e4e5b85319e077bf1979cefdd498d7dda1f4201d22ffd40d8", + "scope": [ + "robot.inspect", + "robot.move", + "robot.stop" + ], + "parent_record_hash": null, + "caller_attestation": "not_offered" + }, + { + "record_id": "rec-1", + "credential_id": "cred-1", + "subject": "3ebb3ddb2865c10726b847ef6b6381516bcce4ab46056c0fd323d5f52e1b52f6", + "scope": [ + "robot.inspect", + "robot.move" + ], + "parent_record_hash": "0000000000000000000000000000000000000000000000000000000000000000", + "caller_attestation": "not_offered" + } + ] +} diff --git a/tests/fixtures/action/action-002-parent-hash-mismatch/expected.json b/tests/fixtures/action/action-002-parent-hash-mismatch/expected.json new file mode 100644 index 0000000..93cce4b --- /dev/null +++ b/tests/fixtures/action/action-002-parent-hash-mismatch/expected.json @@ -0,0 +1,10 @@ +{ + "trusted_root_issuer": "502b5def5d7e08f161a8df11c88b012607d78eebf090d2ee97d2420523ba8c60", + "at_time": null, + "verdict": { + "provenance_status": "invalid", + "authorization_decision": "not_evaluated", + "controller_outcome": "not_evaluated" + }, + "code": "PROVENANCE_LINK_BROKEN" +} diff --git a/tests/fixtures/action/action-006-policy-denial/chain.json b/tests/fixtures/action/action-006-policy-denial/chain.json new file mode 100644 index 0000000..0e4e5c9 --- /dev/null +++ b/tests/fixtures/action/action-006-policy-denial/chain.json @@ -0,0 +1,42 @@ +{ + "chain": [ + { + "credential_id": "cred-0", + "issuer": "dc2871d6a0b4abfdcaeafd45835f98c8749339d276756b5fcfa4f16edcb114b9", + "subject": "145d18ce3912673785911a5877aee902f1510cf92e45019594dfb19f0dc72221", + "scope": [ + "task:read", + "task:write", + "tool:purchase", + "tool:search" + ], + "depth": 0, + "parent_id": null, + "signature": "015047202761cb6f4e69483473aae1d3623daee6dd8fd6d439d21af1eb35cefc489983590d51a32928cfe225fc65f4cd9eb061e9cfcb7a4c8e939efe66bbd209" + }, + { + "credential_id": "cred-1", + "issuer": "145d18ce3912673785911a5877aee902f1510cf92e45019594dfb19f0dc72221", + "subject": "146e2b567ce93bf4a7107344192639924fc9e8e9d7fc4a35916842a708d3f439", + "scope": [ + "task:read", + "tool:purchase", + "tool:search" + ], + "depth": 1, + "parent_id": "cred-0", + "signature": "89f004bf77b0aa802db6477fa94672ba9d6dffb201069b44c094901ae4b11a224ad5492a51bbb6aa227d0edfe05278a237fa2896c8eae501f249b6b887ffc305" + }, + { + "credential_id": "cred-2", + "issuer": "146e2b567ce93bf4a7107344192639924fc9e8e9d7fc4a35916842a708d3f439", + "subject": "46bdafab2912db188ebaf25c714232f63dcfd656103393ba6d8504341723ea59", + "scope": [ + "tool:search" + ], + "depth": 2, + "parent_id": "cred-1", + "signature": "539890d0f1098461513208ead516bd150fcfd96e188873c3703cd35db63b794eb679d138a38b7faba4b0d858642f671b803e58b2d098007ed34e1f8de18f680a" + } + ] +} diff --git a/tests/fixtures/action/action-006-policy-denial/dag.json b/tests/fixtures/action/action-006-policy-denial/dag.json new file mode 100644 index 0000000..ae7d79b --- /dev/null +++ b/tests/fixtures/action/action-006-policy-denial/dag.json @@ -0,0 +1,55 @@ +{ + "records": [ + { + "record_id": "rec-0", + "credential_id": "cred-0", + "subject": "145d18ce3912673785911a5877aee902f1510cf92e45019594dfb19f0dc72221", + "scope": [ + "task:read", + "task:write", + "tool:purchase", + "tool:search" + ], + "parent_record_hash": null, + "caller_attestation": "not_offered" + }, + { + "record_id": "rec-1", + "credential_id": "cred-1", + "subject": "146e2b567ce93bf4a7107344192639924fc9e8e9d7fc4a35916842a708d3f439", + "scope": [ + "task:read", + "tool:purchase", + "tool:search" + ], + "parent_record_hash": "684134dc3af3964e7495b28f01b58f159d8e9ea9226ca3a1bdbab2863c085d3e", + "caller_attestation": "not_offered" + }, + { + "record_id": "rec-2", + "credential_id": "cred-2", + "subject": "46bdafab2912db188ebaf25c714232f63dcfd656103393ba6d8504341723ea59", + "scope": [ + "tool:search" + ], + "parent_record_hash": "a629d8044d2525db9fdff1b2c76cef429eaed4a87414823f152e04d2d70e4264", + "caller_attestation": "not_offered" + }, + { + "record_id": "rec-denied-purchase", + "credential_id": "cred-2", + "subject": "46bdafab2912db188ebaf25c714232f63dcfd656103393ba6d8504341723ea59", + "scope": [ + "tool:search" + ], + "parent_record_hash": "3ab0b6a9f9dca0d2122e8384216edef283489a5e07d050c0e1a482eb660906a3", + "caller_attestation": "not_offered", + "decision": "deny", + "requested_capability": "tool:purchase", + "effective_scope": [ + "tool:search" + ], + "denial_reason": "capability 'tool:purchase' is not in the effective scope" + } + ] +} diff --git a/tests/fixtures/action/action-006-policy-denial/expected.json b/tests/fixtures/action/action-006-policy-denial/expected.json new file mode 100644 index 0000000..ebb716b --- /dev/null +++ b/tests/fixtures/action/action-006-policy-denial/expected.json @@ -0,0 +1,10 @@ +{ + "trusted_root_issuer": "dc2871d6a0b4abfdcaeafd45835f98c8749339d276756b5fcfa4f16edcb114b9", + "at_time": null, + "verdict": { + "provenance_status": "verified", + "authorization_decision": "denied", + "controller_outcome": "not_evaluated" + }, + "code": "capability 'tool:purchase' is not in the effective scope" +} diff --git a/tests/fixtures/action/action-010-scope-escalation/chain.json b/tests/fixtures/action/action-010-scope-escalation/chain.json new file mode 100644 index 0000000..e09ea4b --- /dev/null +++ b/tests/fixtures/action/action-010-scope-escalation/chain.json @@ -0,0 +1,27 @@ +{ + "chain": [ + { + "credential_id": "cred-0", + "issuer": "e27e716965cbce4eef52281613aa99875510fc0f3b4fa4795182fe48f3b421de", + "subject": "332dd879a876ef73fd723f7cb033c233f36889441513db7b7e0a5ea7125d7af9", + "scope": [ + "robot.move" + ], + "depth": 0, + "parent_id": null, + "signature": "f4d8d011f564f29a7a92067eeb37ffc2f08dabfaed348927fea8478058864aeed23babe984a98bf4164b63d1af4fb9e1c5af1ef22168c046725d750c97693707" + }, + { + "credential_id": "cred-1", + "issuer": "332dd879a876ef73fd723f7cb033c233f36889441513db7b7e0a5ea7125d7af9", + "subject": "78101bbe9a5c44a96630b0fbaf8ed1fac4d55a1e750762932f6a21439d4d7742", + "scope": [ + "robot.fly", + "robot.move" + ], + "depth": 1, + "parent_id": "cred-0", + "signature": "0a4cc8256c062a21c6b6cbe2efdd24b6b173980dc9a95102f66b6d762b931126186dcc2ad490c1210f38f76648af1464c25e7f77b92af157e17a07f194cfa301" + } + ] +} diff --git a/tests/fixtures/action/action-010-scope-escalation/dag.json b/tests/fixtures/action/action-010-scope-escalation/dag.json new file mode 100644 index 0000000..4881e55 --- /dev/null +++ b/tests/fixtures/action/action-010-scope-escalation/dag.json @@ -0,0 +1,25 @@ +{ + "records": [ + { + "record_id": "rec-0", + "credential_id": "cred-0", + "subject": "332dd879a876ef73fd723f7cb033c233f36889441513db7b7e0a5ea7125d7af9", + "scope": [ + "robot.move" + ], + "parent_record_hash": null, + "caller_attestation": "not_offered" + }, + { + "record_id": "rec-1", + "credential_id": "cred-1", + "subject": "78101bbe9a5c44a96630b0fbaf8ed1fac4d55a1e750762932f6a21439d4d7742", + "scope": [ + "robot.fly", + "robot.move" + ], + "parent_record_hash": "cfa23fdb38c7637605d9225fc01a801173fcc51fde1479f4480e98caa9db362a", + "caller_attestation": "not_offered" + } + ] +} diff --git a/tests/fixtures/action/action-010-scope-escalation/expected.json b/tests/fixtures/action/action-010-scope-escalation/expected.json new file mode 100644 index 0000000..335adb0 --- /dev/null +++ b/tests/fixtures/action/action-010-scope-escalation/expected.json @@ -0,0 +1,10 @@ +{ + "trusted_root_issuer": "e27e716965cbce4eef52281613aa99875510fc0f3b4fa4795182fe48f3b421de", + "at_time": null, + "verdict": { + "provenance_status": "invalid", + "authorization_decision": "not_evaluated", + "controller_outcome": "not_evaluated" + }, + "code": "SCOPE_ESCALATION" +} diff --git a/tests/fixtures/action/action-012-credential-expired/chain.json b/tests/fixtures/action/action-012-credential-expired/chain.json new file mode 100644 index 0000000..8714f11 --- /dev/null +++ b/tests/fixtures/action/action-012-credential-expired/chain.json @@ -0,0 +1,33 @@ +{ + "chain": [ + { + "credential_id": "cred-0", + "issuer": "fc8a4a1752f57831aeaf8a96cd7b7007dd9b0cbf28db54bffce703713af6bd73", + "subject": "47c7f46429eeeceb9f121244392b8010c1c3ad3e9697b410b8132ebeb721f9ab", + "scope": [ + "robot.inspect", + "robot.move", + "robot.stop" + ], + "depth": 0, + "parent_id": null, + "not_before": 1000, + "not_after": 2000, + "signature": "c12797a405282260a40914c82842dcd602b1cd08aa8a562430ec04014b38abe915b9f6249ce8c8f2e5874f7ae0fbac52d5091b5772be38b03be08eacfb68990d" + }, + { + "credential_id": "cred-1", + "issuer": "47c7f46429eeeceb9f121244392b8010c1c3ad3e9697b410b8132ebeb721f9ab", + "subject": "472c24dda3b18c306f3b544535d2776f98657cf2a884c3565bd4b7b2702d7599", + "scope": [ + "robot.inspect", + "robot.move" + ], + "depth": 1, + "parent_id": "cred-0", + "not_before": 1000, + "not_after": 2000, + "signature": "ca59249454357ff66d2e8a72d97370b7072db8f470a30a252f69c7d74edf89c5393a67abe279346e48e023d46fe05a8880a323a83eff8f7ebbe3b7ad11298e0d" + } + ] +} diff --git a/tests/fixtures/action/action-012-credential-expired/dag.json b/tests/fixtures/action/action-012-credential-expired/dag.json new file mode 100644 index 0000000..7106a33 --- /dev/null +++ b/tests/fixtures/action/action-012-credential-expired/dag.json @@ -0,0 +1,27 @@ +{ + "records": [ + { + "record_id": "rec-0", + "credential_id": "cred-0", + "subject": "47c7f46429eeeceb9f121244392b8010c1c3ad3e9697b410b8132ebeb721f9ab", + "scope": [ + "robot.inspect", + "robot.move", + "robot.stop" + ], + "parent_record_hash": null, + "caller_attestation": "not_offered" + }, + { + "record_id": "rec-1", + "credential_id": "cred-1", + "subject": "472c24dda3b18c306f3b544535d2776f98657cf2a884c3565bd4b7b2702d7599", + "scope": [ + "robot.inspect", + "robot.move" + ], + "parent_record_hash": "3a2dfa9ca8a41ff60c946de48f2f8530d537f6e25d1ef082a574a1f23a2cd068", + "caller_attestation": "not_offered" + } + ] +} diff --git a/tests/fixtures/action/action-012-credential-expired/expected.json b/tests/fixtures/action/action-012-credential-expired/expected.json new file mode 100644 index 0000000..4827cfb --- /dev/null +++ b/tests/fixtures/action/action-012-credential-expired/expected.json @@ -0,0 +1,10 @@ +{ + "trusted_root_issuer": "fc8a4a1752f57831aeaf8a96cd7b7007dd9b0cbf28db54bffce703713af6bd73", + "at_time": 3000, + "verdict": { + "provenance_status": "invalid", + "authorization_decision": "not_evaluated", + "controller_outcome": "not_evaluated" + }, + "code": "CREDENTIAL_EXPIRED" +} diff --git a/tests/unit/test_committed_examples_verify.py b/tests/unit/test_committed_examples_verify.py index bdb0998..254443d 100644 --- a/tests/unit/test_committed_examples_verify.py +++ b/tests/unit/test_committed_examples_verify.py @@ -14,14 +14,12 @@ from __future__ import annotations import json -import subprocess from pathlib import Path import pytest from ca2a_runtime.cli import main as cli_main - -REPO_ROOT = Path(__file__).resolve().parents[2] +from tests.committed_blobs import committed as _committed EXAMPLES = [ "examples/cross-operator-delegation", @@ -29,21 +27,6 @@ ] -def _committed(path: str) -> str | None: - """The blob at HEAD for ``path``, or None if git cannot tell us.""" - try: - out = subprocess.run( # noqa: S603 - ["git", "show", f"HEAD:{path}"], # noqa: S607 - capture_output=True, - text=True, - cwd=REPO_ROOT, - check=False, - ) - except OSError: - return None - return out.stdout if out.returncode == 0 else None - - @pytest.mark.parametrize("example", EXAMPLES) def test_committed_dag_verifies(example: str, tmp_path: Path) -> None: dag = _committed(f"{example}/dag.json") From 0b4ed206cc064ad2b056b5e28a8e4d0c09c1082a Mon Sep 17 00:00:00 2001 From: Nishar Date: Sun, 13 Sep 2026 15:19:38 -0500 Subject: [PATCH 2/2] =?UTF-8?q?test(conformance):=20address=20review=20?= =?UTF-8?q?=E2=80=94=20fail=20on=20missing=20bundle,=20restore=20--check,?= =?UTF-8?q?=20clarify=20verdict=20labels?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Responds to @carloshvp's review of PR #165: - Loader failed open on a missing committed blob. Add tests/committed_blobs.git_source_available() and, when git is available, assert each REQUIRED_BUNDLES blob is present at HEAD (a missing artifact now fails instead of skipping). Skips remain only when git/the source archive is unavailable. Pin the bundle set so dropping or adding a whole directory is a deliberate, test-visible change rather than silent coverage loss. - Restore the advertised --check mode in scripts/gen_action_fixtures.py: it now compares generated bytes to disk without writing and exits nonzero for any stale or missing file, instead of always rewriting and returning 0. - Distinguish scenario labels from observed offline verdicts in the fixtures README and the loader docstring: expected.json's `verdict` is the scenario's ACTION classification, while the offline CLI only observes provenance and, for a denial, the recorded denial outcome. authorization_decision=allowed and controller_outcome are not offline-observable and are not asserted; the controller-input limitation stays explicit. Co-Authored-By: Claude Opus 4.8 (1M context) Signed-off-by: Nishar --- scripts/gen_action_fixtures.py | 42 ++++++++++++--- tests/committed_blobs.py | 20 +++++++ .../test_action_fixture_bundles.py | 54 ++++++++++++++----- tests/fixtures/action/README.md | 28 ++++++++-- 4 files changed, 121 insertions(+), 23 deletions(-) diff --git a/scripts/gen_action_fixtures.py b/scripts/gen_action_fixtures.py index 9e84e6e..28d2160 100644 --- a/scripts/gen_action_fixtures.py +++ b/scripts/gen_action_fixtures.py @@ -9,7 +9,8 @@ Idea credit: @Ahmedibrahim222 (agentrust-io/ca2a#36). Tracked in #164. - python scripts/gen_action_fixtures.py + python scripts/gen_action_fixtures.py # (re)write the bundles + python scripts/gen_action_fixtures.py --check # compare only; nonzero if stale/missing """ # ruff: noqa: T201 @@ -182,10 +183,11 @@ def _bundles() -> list[Bundle]: return [verified, parent_mismatch, denial, escalation, expired] -def main() -> int: +def _bundle_files() -> dict[Path, str]: + """Every bundle file keyed by absolute path, as canonical text.""" + files: dict[Path, str] = {} for b in _bundles(): d = FIXTURE_DIR / b.name - d.mkdir(parents=True, exist_ok=True) chain_doc = {"chain": [{**c.body(), "signature": c.signature} for c in b.chain]} dag_doc = {"records": [r.body() for r in b.records]} expected = { @@ -194,10 +196,36 @@ def main() -> int: "verdict": b.verdict, "code": b.code, } - (d / "chain.json").write_text(json.dumps(chain_doc, indent=2) + "\n", encoding="utf-8") - (d / "dag.json").write_text(json.dumps(dag_doc, indent=2) + "\n", encoding="utf-8") - (d / "expected.json").write_text(json.dumps(expected, indent=2) + "\n", encoding="utf-8") - print(f"wrote {len(_bundles())} bundles under {FIXTURE_DIR}") + files[d / "chain.json"] = json.dumps(chain_doc, indent=2) + "\n" + files[d / "dag.json"] = json.dumps(dag_doc, indent=2) + "\n" + files[d / "expected.json"] = json.dumps(expected, indent=2) + "\n" + return files + + +def main(argv: list[str] | None = None) -> int: + argv = sys.argv[1:] if argv is None else argv + files = _bundle_files() + + if "--check" in argv: + # Compare only; never write. Nonzero if any file is missing or drifted, + # so a stale committed bundle is reported instead of silently rewritten. + stale = [ + path + for path, content in files.items() + if not path.is_file() or path.read_text(encoding="utf-8") != content + ] + if stale: + print("stale or missing ACTION fixture bundles; rerun scripts/gen_action_fixtures.py:") + for path in stale: + print(f" {path.relative_to(REPO_ROOT)}") + return 1 + print(f"ACTION fixture bundles are in sync ({len(files)} files).") + return 0 + + for path, content in files.items(): + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(content, encoding="utf-8") + print(f"wrote {len(files)} files under {FIXTURE_DIR}") return 0 diff --git a/tests/committed_blobs.py b/tests/committed_blobs.py index ada2376..efc0594 100644 --- a/tests/committed_blobs.py +++ b/tests/committed_blobs.py @@ -34,3 +34,23 @@ def committed(path: str) -> str | None: except OSError: return None return out.stdout if out.returncode == 0 else None + + +def git_source_available() -> bool: + """True when this runs inside a git work tree with a resolvable ``HEAD``. + + Lets a test tell two ``committed()`` -> ``None`` cases apart: git or the + source archive being unavailable (a legitimate skip) versus the blob simply + not being committed at ``HEAD`` (a failure the caller should raise on). + """ + try: + out = subprocess.run( # noqa: S603 + ["git", "rev-parse", "--verify", "HEAD"], # noqa: S607 + capture_output=True, + text=True, + cwd=REPO_ROOT, + check=False, + ) + except OSError: + return False + return out.returncode == 0 diff --git a/tests/conformance/test_action_fixture_bundles.py b/tests/conformance/test_action_fixture_bundles.py index ae121e5..ab76d5c 100644 --- a/tests/conformance/test_action_fixture_bundles.py +++ b/tests/conformance/test_action_fixture_bundles.py @@ -9,10 +9,16 @@ against a bundle the generator just rewrote. Regenerate with ``python scripts/gen_action_fixtures.py``. -This exercises the offline provenance / authorization-denial / validity-window -axes. It does not exercise holder-proof authorization replay, which needs live -audience/secret/challenge material and is not offline-replayable evidence; see -``tests/conformance/README.md`` on the ACTION helper and holder-proof binding. +**What is asserted vs. what is a scenario label.** ``expected.json``'s +``verdict`` is the scenario's ACTION three-axis *classification* from +``tests/conformance/README.md``, not the offline verifier's output. The offline +path only observes: ``provenance_status`` (verified vs. a fail-closed code) and, +for a recorded denial, ``authorization_decision == "denied"`` (the CLI's denial +outcome). ``authorization_decision == "allowed"`` and every ``controller_outcome`` +are NOT offline-observable — the CLI never reports an allowed action or an +accepted/rejected controller outcome — so this test does not assert them. That +holder-proof and controller boundary is the one documented in +``tests/conformance/README.md``; these fixtures do not widen it. """ from __future__ import annotations @@ -24,11 +30,19 @@ import pytest from ca2a_runtime.cli import main as cli_main -from tests.committed_blobs import REPO_ROOT, committed +from tests.committed_blobs import REPO_ROOT, committed, git_source_available _BUNDLE_ROOT = REPO_ROOT / "tests" / "fixtures" / "action" -BUNDLES = ( - sorted(p.name for p in _BUNDLE_ROOT.iterdir() if p.is_dir()) if _BUNDLE_ROOT.is_dir() else [] + +# The bundle set is pinned here, not discovered from disk, so deleting a whole +# bundle directory fails a test (its committed blobs go missing below) instead +# of silently shrinking a filesystem-discovered parametrization. +REQUIRED_BUNDLES = ( + "action-001-verified", + "action-002-parent-hash-mismatch", + "action-006-policy-denial", + "action-010-scope-escalation", + "action-012-credential-expired", ) @@ -61,16 +75,22 @@ def _run_verify_dag( return rc, out -@pytest.mark.parametrize("bundle", BUNDLES) +@pytest.mark.parametrize("bundle", REQUIRED_BUNDLES) def test_action_bundle_verifies_as_documented( bundle: str, tmp_path: Path, capsys: pytest.CaptureFixture[str] ) -> None: + if not git_source_available(): + pytest.skip("git or the committed source archive is unavailable; cannot read HEAD blobs") + base = f"tests/fixtures/action/{bundle}" chain = committed(f"{base}/chain.json") dag = committed(f"{base}/dag.json") expected_blob = committed(f"{base}/expected.json") - if chain is None or dag is None or expected_blob is None: - pytest.skip(f"{bundle} is not committed yet or git is unavailable") + # git is available, so None here is an absent HEAD blob (missing evidence), + # not an unavailable source tree: fail rather than skip. + assert chain is not None, f"{base}/chain.json is missing from HEAD" + assert dag is not None, f"{base}/dag.json is missing from HEAD" + assert expected_blob is not None, f"{base}/expected.json is missing from HEAD" expected = json.loads(expected_blob) rc, out = _run_verify_dag(chain, dag, expected, tmp_path, capsys) @@ -96,6 +116,14 @@ def test_action_bundle_verifies_as_documented( assert out.get("outcome") != "denied" -def test_bundles_are_present() -> None: - """Guard against an empty parametrization silently passing zero cases.""" - assert BUNDLES, "no ACTION fixture bundles found under tests/fixtures/action/" +def test_committed_bundle_set_matches_required() -> None: + """Adding or removing a whole bundle directory must be a deliberate change. + + Pins the on-disk set against ``REQUIRED_BUNDLES`` so neither a dropped + directory (which would also fail its verify test) nor an unwired new one + slips through unnoticed. + """ + present = ( + {p.name for p in _BUNDLE_ROOT.iterdir() if p.is_dir()} if _BUNDLE_ROOT.is_dir() else set() + ) + assert present == set(REQUIRED_BUNDLES) diff --git a/tests/fixtures/action/README.md b/tests/fixtures/action/README.md index daf01b1..4824cdc 100644 --- a/tests/fixtures/action/README.md +++ b/tests/fixtures/action/README.md @@ -23,7 +23,25 @@ One directory per case, each holding: |---|---| | `chain.json` | The signed delegation chain (`{"chain": [...]}`), as `ca2a verify-chain` consumes it. | | `dag.json` | The provenance records (`{"records": [...]}`), as `ca2a verify-dag --dag` consumes them. | -| `expected.json` | The documented verdict: the ACTION three-axis outcome (`provenance_status` / `authorization_decision` / `controller_outcome`), the `trusted_root_issuer`, an optional `at_time`, and the reason `code` a failure or denial carries. | +| `expected.json` | The scenario's ACTION three-axis **classification** (`verdict`), the `trusted_root_issuer`, an optional `at_time`, and the reason `code` a failure or denial carries. | + +### Scenario label vs. observed offline verdict + +`verdict` is the scenario's ACTION classification (the label from +`tests/conformance/README.md`), **not** what the offline verifier reports. +`ca2a verify-dag --chain` observes only two of the three axes: + +- `provenance_status` — `verified`, or a fail-closed reason `code`. Always asserted. +- `authorization_decision` — only the `denied` case is offline-observable, as the + CLI's recorded-denial outcome (`code` holds the denial reason). Asserted for + denial bundles. + +The CLI never reports an *allowed* action or an *accepted*/*rejected* +`controller_outcome`; those come from the live authorization and controller +paths, so the loader test does not assert them and they must be read as scenario +labels only. `controller_outcome` in particular remains a claimed test input, +not cryptographically proven evidence — the same limitation stated for the ACTION +helper in `tests/conformance/README.md`. These bundles do not widen that boundary. ## What these do and do not cover @@ -50,9 +68,13 @@ restatement here. The bundles are generated with seeded keys, so their bytes are reproducible: ```bash -python scripts/gen_action_fixtures.py +python scripts/gen_action_fixtures.py # (re)write the bundles +python scripts/gen_action_fixtures.py --check # compare only; exits nonzero if any bundle is stale or missing ``` `tests/conformance/test_action_fixture_bundles.py` verifies the **committed** blobs (read out of git, not the working tree), so a change to the record body or -the chain model that forgets to regenerate these fails there. +the chain model that forgets to regenerate these fails there. It reads each +bundle in `REQUIRED_BUNDLES` from `HEAD`: when git is available a missing blob +fails (rather than skipping), and the on-disk bundle set is pinned so dropping a +whole directory cannot silently reduce coverage.