From 8d4cce8dbab72b769c1bcad263210a65538c4fee Mon Sep 17 00:00:00 2001 From: Alejandro Castellanos <5405628+k2jac9@users.noreply.github.com> Date: Sun, 15 Nov 2026 13:31:02 -0500 Subject: [PATCH 01/51] docs: review evidence reliability and plan bounded P0 work --- .../2026-11-15-evidence-reliability-p0.md | 271 ++++++++++++++++++ .../2026-11-15-evidence-reliability-review.md | 203 +++++++++++++ 2 files changed, 474 insertions(+) create mode 100644 docs/superpowers/plans/2026-11-15-evidence-reliability-p0.md create mode 100644 docs/superpowers/specs/2026-11-15-evidence-reliability-review.md diff --git a/docs/superpowers/plans/2026-11-15-evidence-reliability-p0.md b/docs/superpowers/plans/2026-11-15-evidence-reliability-p0.md new file mode 100644 index 0000000..78a99c4 --- /dev/null +++ b/docs/superpowers/plans/2026-11-15-evidence-reliability-p0.md @@ -0,0 +1,271 @@ +# KairoTensor Eligibility and Atomic Correction Implementation Plan + +> **For agentic workers:** Use `superpowers:executing-plans` or +> `superpowers:subagent-driven-development` to execute task by task. Every checkbox +> below is future work. The owner requested review and planning first; this +> document does not claim implementation or authorize a live model run. + +**Goal:** Explain current eligibility and apply several admission corrections +atomically without weakening read, lineage or accounting guarantees. + +**Architecture:** Extend the existing Rust/SQLite Store. One snapshot evaluator +serves explanation, current read and resume; one Immediate writer transaction +applies a bounded correction and publishes only final changed support verdicts. + +**Tech Stack:** Rust 2021, existing rusqlite `=0.32.1` with bundled SQLite, +serde/serde_json and existing fixture tests. No new service, model, hash library +or persisted schema is needed for this milestone. + +**Spec:** `docs/superpowers/specs/2026-11-15-evidence-reliability-review.md`, +especially sections 2-4, plus the owner's pasted design sections 3.2 and 5. +Baseline: `20fab84a6baa525452191ecf80eeee339f0b5ac7`. + +## Global Constraints + +- Rust and the existing local SQLite database remain authoritative. +- Preserve occurrence identity, ordered inputs, lineage and historical receipts. +- Preserve every incurred charge and unresolved reservation; no new model calls. +- Missing witnesses are not evidence of a proposition's negation. +- Opaque/model heads keep coarse invalidation. Explicit reuse remains separate. +- Authenticate actor/scope and diagnostic access in the embedding runtime. +- No schema migration, evidence payload disclosure or unrelated dependency upgrade. +- Preserve PR #18 and PR #19 and the published-source experiment archives. +- Reconcile the exact base, allowed files and runtime before dispatching a worker. +- Keep tests and contracts unchanged to clear failures; retain failed receipts. +- Use workstation storage admission and bounded Cargo output. During the recorded + exception, use `kairotensor-cargo`; afterward use `workstation-cargo` and obtain + a fresh passing admission. The exception is not a permanent build entitlement. + +## Review Focus + +1. A revoke/read or head-replacement race must return one coherent snapshot; + exercise the actual gate-to-payload path, not only independent SQL queries. +2. A two-record witness replacement must not create transient unsupported heads + or intermediate charges, including consumers shared by several edits. +3. A late failure must roll back memberships, indexes, heads, lineage and charges. +4. Empty/no-op stale requests must conflict; no-op requests must not repair an + already-stale consumer or leak missing-versus-withheld payload information. +5. Historical Accepted and explicit model reuse must never become an implicit + current-read authorization. Resumes must use the same evaluator. + +## Preflight and interfaces + +Recheck remote main and both draft PRs. If source differs, update this plan's +mapping through the technical owner before execution. Preserve other worktrees. +Native worktree tools did not expose the Projects placement required by owner +policy; this review used `.worktrees/reliability-review` with a local Git exclude. + +Task 1 qualifies dependency identity for Tasks 2-3. Task 2 produces the shared +eligibility types and evaluator; Task 3 changes writer helpers without changing +that evaluator's snapshot semantics. Neither task consumes the future semantic +bridge, bundle or policy registry. + +Run commands from the selected worktree. The `cargo` commands below describe the +test invocation; on this workstation prefix them through the admitted launcher, +for example `kairotensor-cargo --budget-gib 2 test --locked --test eligibility`. +Do not start a build following a refused admission. + +### Task 1: Qualify the reproducible baseline + +**Files:** create `Cargo.lock`, `docs/runtime-adoption.md`; modify +`.github/workflows/check.yml` only to enforce the accepted lock in existing Cargo +invocations. No production Rust files or manifest version upgrades. + +**Inputs:** current source, existing manifest constraints, exact Rust executable +and the retained verification lock under the review artifact directory. +**Produces:** a reviewed lockfile, locked CI commands and a source/runtime/lock +verification receipt. This is a reproducibility change, not proof of a complete +runtime compatibility matrix. + +- [ ] Confirm `git ls-files Cargo.lock` and preserve the baseline's missing-lock + failure. On the unchanged baseline, `cargo test --locked --all-targets` must + refuse to create a lock. If a newer source already has a lock, inspect its + identity instead of reproducing an obsolete missing-file failure. +- [ ] Resolve under existing manifest constraints, inspect the full dependency + diff and licenses, record the chosen lock SHA-256 and exact Rust identity. + Verify upstream freshness before any upgrade decision. Preserve historical + pins; a proposed version migration is a separate reviewed change. +- [ ] Run `cargo test --locked --all-targets` and the existing Python reference + and script suites against that resolution. Expected: all tests pass; each + environment-dependent skip is identified. Preserve the exact lock and logs. +- [ ] Add `--locked` to existing CI Cargo build/test/run commands. Record baseline, + latest-stable, upcoming and previous-stable lane status with exact known + identities and explicit unexecuted/unavailable gaps in `docs/runtime-adoption.md`. + Do not claim missing lanes passed or choose a floating preview as a pin. +- [ ] Review and commit `build: freeze the tested dependency resolution`. + +**Done:** the feature implementation has a reproducible dependency baseline; +unresolved runtime-policy adoption remains visible and separately owned. + +### Task 2: Explain the authoritative current-read gate + +**Files:** create `src/eligibility.rs`, `tests/eligibility.rs`, +`docs/current-eligibility.md`; modify `src/attempts.rs`, `src/support.rs`, +`src/lib.rs`, `README.md`. A test-only race seam may live in these modules; +it must not add a public hook or a persisted format. + +**Public interface:** + +```rust +Store::explain_current(&mut self, actor: &str, scope: &str, branch: &str) + -> rusqlite::Result + +pub enum EligibilityState { Empty, Ready, NeedsReview } +pub enum EligibilityProof { CoarseEpoch, SupportV1 } +pub enum EligibilityReason { NoHead, CoarseViewChanged, SupportVerdictChanged } +pub struct CurrentExplanation { + pub state: EligibilityState, + pub head_id: Option, + pub branch_revision: Option, + pub captured_epoch: Option, + pub current_epoch: i64, + pub proof_kind: Option, + pub reason: Option, +} +``` + +**Internal interface:** a payload-free evaluator over `&rusqlite::Connection`, +the optional current `Branch` and captured current epoch; return the same +`CurrentExplanation`. Its caller retains the transaction. Extend support +inspection to distinguish `not a support head`, `same verdict` and `changed +verdict`; retain an existing boolean adapter where needed by unchanged callers. + +| Snapshot | Explanation | +| --- | --- | +| No branch / no head | Empty, NoHead, no proof or captured epoch; branch_revision is None only when no branch exists | +| Captured epoch equals current epoch | Ready, CoarseEpoch, no reason | +| Epoch differs, current support verdict equals published verdict | Ready, SupportV1, no reason | +| Epoch differs, support verdict differs | NeedsReview, SupportV1, SupportVerdictChanged | +| Epoch differs, opaque/model head | NeedsReview, CoarseEpoch, CoarseViewChanged | + +Malformed stored state remains an error, not Ready. Ready means eligible under +the Store contract, not true. No payload, request bytes, occurrence payloads or +free-form validator reasons appear in the explanation or its debug output. + +- [ ] Add `explanation_matches_current_gate`, + `historical_acceptance_does_not_grant_current_access`, + `model_reuse_requires_explicit_resolution`, and + `explanation_contains_no_payload_or_validation_text`. Exercise every table row, + support in both result directions, empty branches and other actor/scope pairs. + Compare separate public calls only while no writer can intervene. +- [ ] Run `cargo test --locked --test eligibility`. Expected initially: failure + because the new interface is missing; preserve the actual error. +- [ ] Extract the evaluator and use it in `read_current`, `explain_current` and + the existing `begin_resume` path. Keep eligibility and accepted-output loading + in the same transaction; do not add a public authorization cache. Explanation + checks accepted-delivery existence without fetching output bytes. +- [ ] Add controlled WAL/barrier tests + `revocation_snapshot_never_exposes_payload` and + `head_replacement_keeps_metadata_and_output_in_one_snapshot` in the actual + read path. Pause after gate evaluation, commit through another connection, + then resume output loading. The pending read may return its older coherent + snapshot; the next read must observe withdrawal/replacement. No sleep-based + race timing. At the payload boundary, after the concurrent writer commits, + the test-only seam must assert `!connection.is_autocommit()` and reread the + scope epoch plus branch head/revision through that same connection. They must + equal the gate's captured values. This directly proves snapshot retention; + immutable historical output alone cannot detect a transaction split. Verify + that a split-transaction mutation fails this snapshot assertion, and that a + head-reselection mutation fails the metadata/output consistency assertion. + Keep mutations in isolated verification copies; restore source afterward. +- [ ] Verify unchanged historical disposition, admission events, branch revisions + and charged/reserved usage after explanations. Verify `begin_resume` refuses + the same stale heads and preserves its existing error behavior. +- [ ] Run `cargo test --locked --lib --test eligibility --test dispatch --test + model_reuse --test selective_revision --test deterministic_verdict`. + Expected: all named tests and the internal race tests pass. +- [ ] Document reason codes and snapshot boundaries, perform focused review, + then commit `feat: explain current eligibility through the shared read gate`. + +**Done:** inspect why a head is unavailable without serving its bytes, changing +admission, granting a new send or rewriting its historical acceptance. + +### Task 3: Correct memberships against an expected epoch + +**Files:** modify `src/lib.rs`, `src/support.rs`, `examples/selective_revision.rs`; +create `tests/atomic_revision_batch.rs`, `docs/atomic-correction.md`. + +**Public interface:** + +```rust +pub struct AdmissionEdit { pub evidence: i64, pub admitted: bool } +pub enum BatchRevision { + Applied { epoch: i64, changes: Vec }, + EpochConflict { current_epoch: i64 }, +} +Store::revise_admissions_if_current( + &mut self, actor: &str, scope: &str, expected_epoch: i64, + edits: &[AdmissionEdit], +) -> rusqlite::Result +``` + +**Request contract:** nonempty actor/scope; expected_epoch >= 0; at most 1,024 +edits; unique positive occurrence IDs. Validate this shape first. Acquire the +Immediate writer transaction and compare epoch before inspecting occurrence +existence. A structurally valid stale request returns EpochConflict, including +empty/no-op or unknown-occurrence requests. At a matching epoch, validate every +occurrence exists before the first mutation. This ordering is an explicit new +API decision; test it rather than leaving error precedence accidental. + +**Writer contract:** use transaction-scoped existing admission/index helpers; +apply effective edits in input order; union only consumers of effective changes; +evaluate final verdicts once and publish changed branches in branch-name order. +Every new attempt captures the final epoch. Reuse existing `SupportRevision`. +An unchanged verdict retains its attempt and charge. Empty/all-no-op requests at +the expected epoch return Applied without events, index writes or charges. + +- [ ] Add `replacement_has_no_intermediate_verdict`, + `second_edit_failure_rolls_back_every_write`, + `concurrent_correction_has_one_winner`, `duplicate_edit_is_rejected`, + `stale_empty_and_noop_batches_conflict`, `noop_leaves_stale_consumers_alone`, + `overlapping_consumers_publish_once_at_final_epoch`, and + `request_shape_and_epoch_precedence_are_stable`. +- [ ] Run `cargo test --locked --test atomic_revision_batch`. + Expected initially: missing interface failure, retained verbatim. +- [ ] Extract transaction-scoped helpers from the existing single-edit path. + Implement one transaction, not a loop of public mutating methods. Do not + change the old method's no-op or duplicate-retry semantics. +- [ ] Extend the rollback test with a trigger that aborts the second effective + admission, and a separate later-publication failure. A pre-validation failure + alone does not prove rollback after writes. Compare complete persistent state + before/after and after reopening, including charges and unresolved holds. +- [ ] Exercise two independent connections with explicit barriers and one + expected epoch: exactly one Applied and one EpochConflict, no duplicate charge. + A retry with the old epoch adds no events even after an intervening reversal. +- [ ] Exhaust all 16 initial and 16 final admission subsets of a four-occurrence + rule fixture. Compare final verdicts to a fresh evaluator/Store; assert + lineage and charges separately because a fresh store has different history. + Add reproducible seeded edit sequences and report their seed on failure. +- [ ] Mutation checks: bypass epoch comparison; publish after each edit; + remove consumer deduplication. Each corresponding invariant test must fail. + Mutations stay in isolated verification copies and never alter the tests or + acceptance contract; retain failure receipts and restore the candidate. +- [ ] Run `cargo test --locked --test atomic_revision_batch --test + atomic_revision --test bulk_setup --test selective_revision --test eligibility`. + Expected: all pass, including unchanged-head and one-charge assertions. +- [ ] Document the 1,024-edit bound, conflict/error ordering, lost-response + reread requirement and final-view semantics; update the example; focused + review and commit `feat: atomically revise admissions at an expected epoch`. + +**Done:** one mixed withdraw/admit correction is atomic, reference-equivalent +and independently useful with the explanation API. No migration is needed. + +## Integration and independent review + +- [ ] Run `cargo test --locked --all-targets`, + `python3 -m unittest discover -s reference -p test_memory.py`, + `python3 reference/belief_oracle.py`, and + `python3 -m unittest discover -s scripts -p 'test_*.py'`. + Report environment-dependent skips separately; execute binary-dependent checks + with the built examples when required by the existing CI workflow. +- [ ] Obtain an independent review of the full source range and the five Review + Focus conditions. Fix important findings with a reproduced failing test and + a passing targeted/integration run. Preserve every finding and disposition. +- [ ] Record exact source, Rust identity, lock digest, commands, test counts, + mutation failures and resource accounting. Do not modify historical benchmark + receipts or claim new model/benchmark performance. +- [ ] Prepare the branch for owner review. Merge, publication, hosted spending + and the separately authorized Adaption smoke are not part of this milestone. + +Next: write the Store-backed incident bridge plan using section 3 of the review. +Bundle replay, policy activation and portability remain later, separate plans. diff --git a/docs/superpowers/specs/2026-11-15-evidence-reliability-review.md b/docs/superpowers/specs/2026-11-15-evidence-reliability-review.md new file mode 100644 index 0000000..0ffe6db --- /dev/null +++ b/docs/superpowers/specs/2026-11-15-evidence-reliability-review.md @@ -0,0 +1,203 @@ +# KairoTensor evidence reliability: verified review and design decisions + +Date: 15 November 2026, America/Toronto. +Scope: **review and planning only**. No proposed production feature is implemented +by this delivery. The owner separately authorized a 48-hour storage exception. + +## 1. Source and recent work + +Remote `main` remains `20fab84a6baa525452191ecf80eeee339f0b5ac7`. +The original checkout is clean on `docs/epistemics-submission-20261115`, +`99b6fe24177e56b450c543c95723961fed440d4d`. Its changes are the submission report, +provenance disclosure, evidence index and reproduction material prepared in this +conversation. Preserve that branch, its PDF and archived experimental evidence. + +- [PR #19](https://github.com/k2jac9/kairotensor/pull/19) is draft, open and unmerged; + both final-head CI checks passed. It changes documentation only. +- [PR #18](https://github.com/k2jac9/kairotensor/pull/18) is draft, open and unmerged + at `a8bcc705e4047765b34f6152aa303a6d40c5c34d`. Its hosted transport and accounting + code has offline/mock verification, not confirmed provider execution. +- The supplied ChatGPT conversation link returned no readable content. The full + plan pasted by the owner is the design input for this review. +- Review receipts are retained outside the checkout under + `/home/neo/Projects/kairotensor-artifacts/reliability-review-20261115/`. + +The planning worktree is based directly on main. Neither draft PR is silently +merged or treated as implemented main-branch functionality. + +## 2. What exists, and what remains proposed + +| Area | Verified source | Current boundary / remaining work | +| --- | --- | --- | +| Current read and resume | `src/attempts.rs:85`, `:316`, `:342` | Both already hold a database transaction around eligibility and accepted output. There is no `explain_current`. Extract and share this gate; do not claim a demonstrated existing revocation race. | +| Support eligibility | `src/support.rs:41-64` | A surviving support/v1 verdict can remain current across an epoch change. Its boolean helper loses the distinction between a non-support head and a changed support verdict. | +| Atomic correction | `src/support.rs:304`; `src/lib.rs:138` | One membership plus affected publications is atomic; bulk grants exist. Mixed withdraw/admit correction with expected-epoch compare-and-swap is absent. | +| Model reuse | `src/model_reuse.rs`; `tests/model_reuse.rs:55` | Explicit closed-computation resolution can reuse one sampled response. Ordinary current reads still use coarse invalidation for model heads. | +| Recovery and timing | `src/recovery.rs:84`; `src/timed_recovery.rs:256`; `src/recovery_runtime.rs:69` | Timed capture is complete relative to its requested list. Claim, post-claim clock check and settlement exist. This does not validate a semantic source mapping or prove that the declared list is complete. | +| Incident workload | `scripts/incident_routing.py:184`; `docs/incident-routing-workload.md` | Frozen rendering and protocol/scoring separation exist. Eligibility flags are declared fixture state; an actual Rust Store-backed incident bridge is absent. | +| General replay / policy registry | Existing recovery audit and Python policy replay | No `DecisionBundleV1`, diagnostic authorizer, general replay API or Store policy registry was found. These remain separate future designs. | +| Dependency identity | `Cargo.toml`; `.github/workflows/check.yml` | No tracked `Cargo.lock` exists. The current CI uses Cargo without `--locked`; a source SHA alone does not identify its resolved dependencies. | + +## 3. Changes to the supplied plan + +### First establish a reproducible baseline + +A fresh `cargo test --locked --offline --all-targets` failed because the baseline +has no lockfile. A subsequent offline resolution produced a retained candidate +lock, then failed because `ahash 0.8.12` was unavailable locally. These failures are +preserved. Any successful later run uses the newly recorded resolution; it is not +an assertion about the dependency identity of historical CI. + +Before feature changes, qualify and commit a lockfile and make CI enforce it. +Record the tested Rust identity. Maintain a separate, explicit runtime-policy +adoption record: latest stable, upcoming preview and previous supported stable +must be freshly identified, pinned and tested before claiming that matrix. +Neither an observed local compiler version nor old passing CI establishes it. +Do not combine unreviewed dependency upgrades with the gate refactor. + +### Explain the gate that actually serves bytes + +The payload-free evaluator must serve `read_current`, `explain_current`, and +`begin_resume`. Include `src/support.rs` in the allowed files so the evaluator can +distinguish non-support heads from changed support verdicts without inventing +another authority. Keep current snapshot semantics: a read established before a +revocation may finish using that earlier coherent snapshot. A snapshot that +already contains the revocation must not return the revoked result. + +Independent public reads can observe different commits. Do not write a race test +that assumes two separate calls are one snapshot. The existing raw-SQL test in +`tests/atomic_revision.rs:220` does not exercise the production read path. Add a +controlled test-only barrier in the real gate-to-payload path and test both +revocation and concurrent head replacement there. + +The final independent review also identified a false-confidence risk in the +proposed mutation test: immutable historical bytes fetched by a saved head ID +can look correct even after splitting transactions. The revised plan separately +asserts transaction liveness and the retained epoch/head snapshot at the payload +boundary, then checks observable metadata/output consistency. + +### Batch only effective edits, and publish the final state + +Do not implement a batch by looping `set_admitted_and_revise`: that would publish +intermediate verdicts and charge for them. Use one Immediate transaction and +evaluate the union of consumers after applying all effective edits. + +An unchanged requested membership must not repair an already-stale consumer as +an incidental side effect. Existing no-op behavior deliberately avoids that +(`tests/atomic_revision.rs:164-175`). A stale expected epoch conflicts even for +empty or all-no-op batches. Duplicate IDs are invalid. This remains optimistic +concurrency, not an idempotent operation-key API. + +Proposed additional bound: at most **1,024 edits per batch**. This is a new API +limit for review, not an existing measured capacity result. Specify malformed +request and epoch-conflict precedence in the execution plan and test it. + +### Make the semantic bridge a separate next milestone + +The following points must be frozen before its implementation WorkOrder: + +1. Map `(workload_version, case_id, external_evidence_id)` to one Store occurrence. + Evidence IDs such as `e01` repeat across cases. Reject omissions, duplicates, + aliases, empty mappings, unknown fields and version mismatches. +2. Capture the question, owner roster, policy/context and report text that can + change the computation, or bind immutable configuration in its versioned + contract. Render from `CapturedInputs`; do not substitute corpus payloads or + caller-declared current flags. +3. Preserve exact Python renderer bytes for the frozen v1 fixture. Pin a reviewed + SHA-256 dependency in `Cargo.toml` and `Cargo.lock` here, before bundle export + needs it. Do not write cryptography by hand. +4. Use strict typed parsing that rejects duplicate JSON keys and unknown fields. + Match the frozen protocol's depth limit of 64. Generic `serde_json::Value` + alone loses duplicate keys. Keep semantic labels out of request construction, + admission, runtime validation and retry feedback; well-formed wrong answers + are not magically detected by protocol validation. +5. Reuse `RecoveryStart`, `RecoveryPreparation`, reservation, timed claim and + runtime settlement. `Existing(id)` is not a new send opportunity. The public + reservation reader exposes reserved request/count/cap, not arbitrary captured + input reconstruction. Initially refuse ambiguous restart states; richer + reconstruction needs its own reviewed interface. +6. Revocation before claim refuses dispatch. Revocation after claim may be too + late to prevent invocation: refuse stale publication and retain incurred + usage/ambiguous holds. A timer cannot revoke a send that already happened. +7. Fixtures may invoke canned adapters. They must issue zero provider/model + calls and must not be reported as model-quality measurements. + +## 4. Prioritized delivery boundaries + +| Boundary | Deliverable | Exit gate | +| --- | --- | --- | +| A0 | Reproducible baseline and visible runtime-policy gaps | Committed tested lock; CI refuses unreviewed resolution; exact source/runtime/lock receipt. | +| A1 | Payload-free current eligibility explanation | Shared read/resume evaluator; parity and controlled race tests; no migration. | +| A2 | Expected-epoch atomic membership correction | Final-state equivalence, rollback, one-winner concurrency, unchanged accounting and no-op behavior; no migration. | +| B | Store-backed incident fixture | Strict case-scoped mappings, captured bytes, byte-identical rendering, label-independent validation and existing timed gates. Separate implementation plan. | +| C | Authorized decision export and deterministic replay | Freeze bundle schema, explicit error type, authorizer contract, redaction boundary and versioned validator registry first; no execution during replay. | +| D | Evaluated policy activation | Freeze counters/denominators, authority, supported artifacts, exact comparison manifest and next free schema version; bootstrap/promote/rollback have separate gates. | +| E | Portable envelope and bounded diagnostics | Only after a named second consumer needs it; Rocket-owned adapter stays a separate task. No live ablation execution implied. | + +Do not put C-E in the same execution batch as A. A policy registry must not +manufacture send authority; a stored evaluation remains an attestation. Preserve +unknown usage and every uncertain hold. Rollback advances policy history and +requires current compatibility/authority, not proof of superiority. A repeatedly +used holdout becomes selection data; a final comparison needs a fresh lockbox. + +The original references to MemoryDelta, RecallTrace, Precedent, Bodega, +NeverTwice and Countersign lack repository URLs, commit identities and license +records in the pasted text. Their implementations were not independently +verified in this review. Obtain those identities before copying code or claiming +a source-pinned comparison. The architectural ideas do not require copying them. + +## 5. Adaption remains a separate experiment + +The owner authorized one synthetic DeepSeek V4 Flash smoke request, at most one +Adaption credit / US$1, with no retry, top-up or subscription change. That +authorization is retained for a future execution pass; this pass is review only. + +Fresh observations: PR #18 is unchanged and unmerged; `ADAPTION_API_KEY` is not +present in this process environment. A bounded filename inventory of the local +repository and its artifact directory found no hosted/Adaption smoke receipts. +That does not prove another runner never sent a request or that no charge exists. +The connected Adaption tools expose training/dataset operations, not proof of +this account's inference model access or all-category request pricing. + +Before any launch, reconcile possible prior claims/requests across runners and +provider records, verify exact model access, an all-in one-request billing bound, +credit eligibility, secure credentials and request identity. Keep the full +monetary hold through unknown remote completion. An absent local receipt or a +timeout is not permission to resend. Hosted timing/accounting is not equivalent +to the local Qwen experiment. + +## 6. Verification and operational constraints + +Fresh review checks passed: **185 Rust tests across 24 test binaries**, with no +failures or ignored tests, plus 18 reference, 8 belief-oracle and 88 incident +Python tests (**114 Python tests**). The Rust command was +`kairotensor-cargo --budget-gib 2 test --locked --all-targets`, using +`rustc 1.99.1 (58cf8f9c3 2026-10-13)` and the generated verification lock with +SHA-256 `0298c2ecbad412b31f56d8c712bbea76c1b4cb0e055fdaee009bf967b170a820`. +The lock, `rust-qualified-baseline.json`, and complete logs are retained in the +review artifact directory. This qualifies that newly recorded resolution; it +does not identify historical CI dependencies, execute the entire CI workflow, +or establish the requested runtime matrix. No model/provider calls were made. + +An independent reviewer inspected the main-branch gates and tests. Its findings +are incorporated above: read/resume sharing, real-path race coverage, final-state +batch publication, effective-change/no-op semantics, case-local evidence IDs, +strict label-independent parsing and narrower restart guarantees. These are plan +corrections and coverage gaps, not claims that the new features already exist. + +Normal storage preflight refused the proposed 2 GiB Rust build. The owner then +authorized a two-day exception. A project-scoped launcher now permits a 20 GiB +floor until **2026-11-17 18:04:38 UTC / 13:04:38 Toronto**. It uses the existing +reservation ledger and ENOSPC checks. `kairotensor-cargo --budget-gib 2 ...` uses +the installed Cargo recipe with a scoped admission entry; `kairotensor-storage` +provides its preflight/status. A persistent user timer expires the policy, and +each launch independently checks the deadline. The root policy stays at 50 GiB. +No existing reservation or user data was deleted. The exception does not +authorize feature implementation or provider spending. + +## 7. Recommendation + +Review and execute boundary A first using the accompanying plan. Treat the +submitted research snapshot as frozen historical evidence. Measure new behavior +on new source-bound fixtures, and amend research claims only when new results +exist. Keep B-E and the one-request hosted smoke as separately reviewed work. From 9141d8f1439c6beca319aa8331c9347cabaf51f3 Mon Sep 17 00:00:00 2001 From: Alejandro Castellanos <5405628+k2jac9@users.noreply.github.com> Date: Sun, 15 Nov 2026 13:54:55 -0500 Subject: [PATCH 02/51] build: freeze the tested dependency resolution --- .github/workflows/check.yml | 26 ++-- Cargo.lock | 277 ++++++++++++++++++++++++++++++++++++ docs/runtime-adoption.md | 138 ++++++++++++++++++ 3 files changed, 428 insertions(+), 13 deletions(-) create mode 100644 Cargo.lock create mode 100644 docs/runtime-adoption.md diff --git a/.github/workflows/check.yml b/.github/workflows/check.yml index 6fff673..802d562 100644 --- a/.github/workflows/check.yml +++ b/.github/workflows/check.yml @@ -15,19 +15,19 @@ jobs: with: persist-credentials: false - name: Test the Rust store, including the withdrawal example - run: cargo test --all-targets + run: cargo test --locked --all-targets - name: Run the revision and model handoff examples run: | - cargo run --example withdrawal - cargo run --example deterministic_verdict - cargo run --example deadline_aware_recovery - cargo run --example runtime_recovery - cargo run --example selective_revision - cargo run --example corrected_handoff - cargo run --example scoped_model_reuse - cargo run --example scoped_model_reuse -- --replay-invalid - cargo run --example token_cap_eval - cargo run --example budgeted_recovery + cargo run --locked --example withdrawal + cargo run --locked --example deterministic_verdict + cargo run --locked --example deadline_aware_recovery + cargo run --locked --example runtime_recovery + cargo run --locked --example selective_revision + cargo run --locked --example corrected_handoff + cargo run --locked --example scoped_model_reuse + cargo run --locked --example scoped_model_reuse -- --replay-invalid + cargo run --locked --example token_cap_eval + cargo run --locked --example budgeted_recovery - name: Check the original reference contracts run: | python3 -m unittest discover -s reference -p test_memory.py @@ -40,7 +40,7 @@ jobs: run: python3 scripts/recovery_policy_eval.py --fixture - name: Collect and score Store-backed synthetic recovery trajectories run: | - cargo run --example recovery_policy_eval -- --trace "$RUNNER_TEMP/recovery-policy-trace.json" + cargo run --locked --example recovery_policy_eval -- --trace "$RUNNER_TEMP/recovery-policy-trace.json" python3 scripts/recovery_policy_eval.py "$RUNNER_TEMP/recovery-policy-trace.json" KAIROTENSOR_RECOVERY_RUNNER="$PWD/target/debug/examples/recovery_policy_eval" python3 -m unittest discover -s scripts -p test_recovery_policy_runner.py KAIROTENSOR_RECOVERY_RUNNER="$PWD/target/debug/examples/recovery_policy_eval" python3 -m unittest discover -s scripts -p test_audit_recovery_bundle.py @@ -48,7 +48,7 @@ jobs: run: python3 scripts/recovery_calibration.py --output "$RUNNER_TEMP/recovery-calibration.json" - name: Exercise offline timed runtime through the fixture Adapter - run: cargo run --example timed_recovery_runner -- --out "$RUNNER_TEMP/timed-recovery-runner" + run: cargo run --locked --example timed_recovery_runner -- --out "$RUNNER_TEMP/timed-recovery-runner" - name: Prepare frozen semantic calibration requests without inference run: python3 scripts/incident_routing.py prepare --out "$RUNNER_TEMP/incident-routing-calibration" diff --git a/Cargo.lock b/Cargo.lock new file mode 100644 index 0000000..ca1d8fe --- /dev/null +++ b/Cargo.lock @@ -0,0 +1,277 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "ahash" +version = "0.8.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a15f179cd60c4584b8a8c596927aadc462e27f2ca70c04e0071964a73ba7a75" +dependencies = [ + "cfg-if", + "once_cell", + "version_check", + "zerocopy", +] + +[[package]] +name = "bitflags" +version = "2.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3ded4057c258ba199e2d26386d3af3780957ecaee6c4ef4041c6b4b8b97c0b06" + +[[package]] +name = "cc" +version = "1.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6651c9ed80effdc7db0ff72512157f901af5e3549e341e24b1dd4887d836d838" +dependencies = [ + "find-msvc-tools", + "shlex", +] + +[[package]] +name = "cfg-if" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4e7648175b45a9a48536d676f68d918270699102aa8dab5496df06904c914600" + +[[package]] +name = "fallible-iterator" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2acce4a10f12dc2fb14a218589d4f1f62ef011b2d0cc4b3cb1bba8e94da14649" + +[[package]] +name = "fallible-streaming-iterator" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7360491ce676a36bf9bb3c56c1aa791658183a54d2744120f27285738d90465a" + +[[package]] +name = "find-msvc-tools" +version = "0.1.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aedcfb3409746eddb02b9e19ebda1c3394f759a152e48ee875a0844d1b955484" + +[[package]] +name = "hashbrown" +version = "0.14.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e5274423e17b7c9fc20b6e7e208532f9b19825d82dfd615708b70edd83df41f1" +dependencies = [ + "ahash", +] + +[[package]] +name = "hashlink" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ba4ff7128dee98c7dc9794b6a411377e1404dba1c97deb8d1a55297bd25d8af" +dependencies = [ + "hashbrown", +] + +[[package]] +name = "itoa" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" + +[[package]] +name = "kairotensor" +version = "0.1.0" +dependencies = [ + "libc", + "rusqlite", + "serde", + "serde_json", +] + +[[package]] +name = "libc" +version = "0.2.177" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2874a2af47a2325c2001a6e6fad9b16a53b802102b528163885171cf92b15976" + +[[package]] +name = "libsqlite3-sys" +version = "0.30.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2e99fb7a497b1e3339bc746195567ed8d3e24945ecd636e3619d20b9de9e9149" +dependencies = [ + "cc", + "pkg-config", + "vcpkg", +] + +[[package]] +name = "memchr" +version = "2.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98" + +[[package]] +name = "once_cell" +version = "1.21.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" + +[[package]] +name = "pkg-config" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f6b464fbc74e149a392436b17d523f769e057cb6877f6a5c4618bc6f11800548" + +[[package]] +name = "proc-macro2" +version = "1.0.107" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "985e7ec9bb745e6ce6535b544d84d6cd6f7ad8bd711c398938ae983b91a766d9" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "quote" +version = "1.0.47" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fbf4db142a473a8d80c26bbf18454ed458bf8d26c8219c331daecfdbd079001" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "rusqlite" +version = "0.32.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7753b721174eb8ff87a9a0e799e2d7bc3749323e773db92e0984debb00019d6e" +dependencies = [ + "bitflags", + "fallible-iterator", + "fallible-streaming-iterator", + "hashlink", + "libsqlite3-sys", + "smallvec", +] + +[[package]] +name = "serde" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4148590afebada386688f18773da617792bf2ef03ffc1e4cbd2b1d45b023e0ba" +dependencies = [ + "serde_core", + "serde_derive", +] + +[[package]] +name = "serde_core" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67dca2c9c51e58a4791a4b1ed58308b39c64224d349a935ab5039aa360942a48" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7a5d71263a5a7d47b41f6b3f06ba276f10cc18b0931f1799f710578e2309348" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.9", +] + +[[package]] +name = "serde_json" +version = "1.0.154" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7e9cc8b1b85264074fbcc02a88680c4096b1e47df8f739dceb03bf482f04bd6" +dependencies = [ + "itoa", + "memchr", + "serde", + "serde_core", + "zmij", +] + +[[package]] +name = "shlex" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8fadd59c855ef2080decdef8ff161eb6661b86933c9d82e5ba29dc602a55aba" + +[[package]] +name = "smallvec" +version = "1.16.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b3dc8af474f516a851ff4bd12db780f948b9250ad37211e4eec0bccea54e01b" + +[[package]] +name = "syn" +version = "2.0.119" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "872831b642d1a07999a962a351ed35b955ea2cfc8f3862091e2a240a84f17297" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "syn" +version = "3.0.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d78c8dee4c7bf0e14673097256fed6142ce9d3b85a408189d07482442145823b" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "unicode-ident" +version = "1.0.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2c754d6c33795a1c324727428e5a7dedb5b06195f9890bdbcba760d3e246563" + +[[package]] +name = "vcpkg" +version = "0.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" + +[[package]] +name = "version_check" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" + +[[package]] +name = "zerocopy" +version = "0.8.62" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "86502bf56ac7c77571a32e2647bb2a15894565e981fb2a48d7bde2d91c965a9d" +dependencies = [ + "zerocopy-derive", +] + +[[package]] +name = "zerocopy-derive" +version = "0.8.62" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5457206954b06561e2608c7e19cf58b1926586d999c246eebe4502f7e2039d1a" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "zmij" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29666d0abbfad1e3dc4dcf6144730dd3a3ab225bbbdac83319345b1b44ccfc1b" diff --git a/docs/runtime-adoption.md b/docs/runtime-adoption.md new file mode 100644 index 0000000..c582f1a --- /dev/null +++ b/docs/runtime-adoption.md @@ -0,0 +1,138 @@ +# Runtime adoption and reproducible baseline + +Checked on 2026-11-15. This change freezes a locally verified dependency +resolution and requires Cargo to honor it in the existing CI commands. Runtime +migration, hosted CI execution, release acceptance and rollback qualification +remain separate gates. + +## Source and dependency identity + +- Repository: `https://github.com/k2jac9/kairotensor.git`; branch: + `feat/evidence-reliability-20261115`. +- Source tested: `8d4cce8dbab72b769c1bcad263210a65538c4fee`, with the added + `Cargo.lock` and locked CI commands. Production Rust, manifests, examples and + tests are unchanged by this task. +- Lock format: 4; SHA-256: + `0298c2ecbad412b31f56d8c712bbea76c1b4cb0e055fdaee009bf967b170a820`. +- The lock is byte-identical to the retained `Cargo.verification.lock` tested on + `20fab84a6baa525452191ecf80eeee339f0b5ac7`. There are no additions, removals or + version changes relative to that resolution. The source repository had no + tracked lock, so the historical CI transitive resolution is not established. +- Existing constraints remain unchanged: `rusqlite =0.32.1` with bundled SQLite, + Linux `libc =0.2.177`, `serde 1` with derive, and `serde_json 1`. The selected + serde and serde_json versions are `1.0.229` and `1.0.154` respectively. Newer + versions of exact-pinned dependencies are outside this reproducibility change. + +The complete 32 external package/version/license inventory is retained in +`dependency-review.json` in the evidence packet below, using locked Cargo +metadata. Declared licenses include MIT, Apache-2.0, BSD-2-Clause, Unlicense and +Unicode-3.0 combinations. Every external package declares a license; the root +unpublished crate has no manifest license declaration. This inspection covers +manifest declarations and the full resolved graph, including optional packages +not compiled on this host; it is not a vulnerability or legal audit. + +## Exact executed environment + +The local host was Linux `x86_64-unknown-linux-gnu`, NixOS `26.5.0`, kernel `7.3.4`. +The storage wrapper's existing directory label `nixos26.05-x86_64` is a cache key, +not an independently measured OS version. + +| Component | Executed identity | Executable SHA-256 | +| --- | --- | --- | +| Rust | `1.99.1`, commit `58cf8f9c35b355199727627d07ae34a4e2846d67`, LLVM `23.1.1` | `c45c3ff06bc89e67f851c575aa2eb8624e71e83bd5a4bba7378147c95c702909` | +| Cargo | `1.99.1`, commit `5f94df4789f005f9a352888e8355ffc645b7ed0e` | `73d1b427364c2cf348812d53860ab93b87831d94831cdfeb7d6a237a9c477330` | +| Python | `3.15.2`, built 2026-10-25, GCC `16.2.0` | `7f7f9b35ee0ca35d3dbd073bd34e37e1364fae5570ab027f20dfbcb5e6787001` | + +Rust and Cargo resolved through rustup to +`/home/neo/.rustup/toolchains/1.99.1-x86_64-unknown-linux-gnu/bin/`. +Python resolved to +`/nix/store/w3ps4vf470m69d9l5bcbbipay9insx32-python3-3.15.2/bin/python3.15`. +These are source-bound local qualification identities, not proof of a durable +project runtime pin or an operational deployment baseline. + +## Fresh upstream selections and execution gaps + +The owner policy requires current stable, upcoming and previous compatibility +lanes. Fresh upstream selections below are research inputs only; no runtime was +installed, upgraded or promoted by this task. + +| Role | Rust selection as of 2026-11-15 | Python selection as of 2026-11-15 | Execution status | +| --- | --- | --- | --- | +| Local qualification baseline | `1.99.1` with exact executable identity above | `3.15.2` with exact executable identity above | Existing local CI workload passed, as detailed below. Accepted live baseline is not established. | +| Latest stable target | `1.100.0 (0ef1bc386 2026-11-09)`, distribution dated `2026-11-12` | `3.15.2` | Rust unexecuted. Python executed locally; hosted runner identity and complete cross-runtime matrix remain unestablished. | +| Latest published upcoming preview | `1.102.0-nightly (5f4effd57 2026-11-13)`, distribution dated `2026-11-15`; beta `1.101.0-beta.2 (63931fb6b 2026-11-12)`, dated `2026-11-14`, also recorded | `3.16.0a2`; artifact digest not established | Available upstream; neither runtime preview lane was executed or promoted. | +| Previous stable compatibility | `1.99.1`, newest published patch of the previous minor line; ongoing upstream support status not established | `3.14.10`, supported bugfix line | Rust local workload passed using the same baseline run. Python lane unexecuted; artifact digest not established. | +| Separately promised older consumers | Not established | Not established | Consumer inventory and any additional required checks remain owned gaps. | +| Exact accepted rollback set | Not established | Not established | Historical receipts retained; no rollback recovery exercise claimed. | + +The retained Rust manifests pin the selected `x86_64-unknown-linux-gnu` full +`.tar.xz` distributions to these SHA-256 values: + +- Stable `1.100.0`: `98ee6bec240e1cfe944f17e7092412dc5ef6e008484bb668c6936d35a7d15acd`. +- Beta `1.101.0-beta.2`: `86eb70ada7f97ce39df670733e92a942047fbbee5fc39b81f6f2fa0643651adc`. +- Nightly `2026-11-15`: `2b7c1b434dc3bed75aa47b1bd05ec2d24edf01bbc1a3ffa2d9a4895962658aef`. + +These are upstream manifest declarations; the toolchain archives were not +downloaded or independently hash-verified. The additional retained `1.98.0` +manifest was an exploratory lookup and is not the previous-stable lane selection. + +Primary sources checked: [Rust release announcements](https://blog.rust-lang.org/releases/), +[Rust stable manifest](https://static.rust-lang.org/dist/channel-rust-stable.toml), +[Rust beta manifest](https://static.rust-lang.org/dist/channel-rust-beta.toml), +[Rust nightly manifest](https://static.rust-lang.org/dist/channel-rust-nightly.toml), +[Python downloads and supported lines](https://www.python.org/downloads/), and +[Python 3.16.0a2](https://www.python.org/downloads/release/python-3160a2/). +Retained manifest bytes, retrieval times and manifest hashes are in +`rust-*.toml` and `upstream-rust.json`. Recheck upstream weekly and before any +migration decision; next adoption review is due 2026-11-22. + +## Verification receipt and CI mapping + +Evidence packet: +`/home/neo/Projects/kairotensor-artifacts/reliability-implementation-20261115/task-a0/`. +The machine-readable summary is `verification-receipt.json`; each command and log +path is retained in `ci-checks.json` and `rust-tests.json`. + +Local `kairotensor-storage preflight --budget-gib 2 --output-dir ` passed. +Cargo commands used `kairotensor-cargo --budget-gib 2` with +`KAIROTENSOR_REQUIRE_LINUX_CLOCK=1`. Output was keyed by checkout, source, runtime, +OS/architecture label and lock hash. The explicitly authorized project storage +exception was active at verification time, retaining a 20 GiB emergency floor +until 2026-11-17 18:04:38 UTC. Historical reservations and receipts were preserved. + +| Existing `contracts` CI workload | Local result | +| --- | --- | +| `cargo test --locked --all-targets` | 185 passed across 24 test binaries; 0 failures, 0 ignored | +| All 12 existing `cargo run --locked` invocations | Passed, including invalid replay, Store-backed trace collection and timed fixture Adapter | +| Python reference contracts | 18 passed; oracle 8 passed | +| Full `scripts/test_*.py` discovery | 203 scheduled, 175 passed, 28 skipped because `KAIROTENSOR_RECOVERY_RUNNER` was absent in that initial CI-equivalent step | +| Compiled-runner discovery of `test_recovery_policy_runner.py` and `test_audit_recovery_bundle.py` | 15 and 19 passed respectively, 0 skipped; these exercise all 28 initially skipped tests plus 6 already runnable checks | +| Archived output caps, recovery policy fixture and collected trace scoring | Passed | +| Calibration, routing preparation, preflight, offline controller and fake Transformers fixtures | Passed; zero new model calls or provider spending | + +All 25 example/Python CI commands following the Rust suite exited zero. The +compiled-runner environment used the wrapper's exact keyed executable path in +place of CI's default `target/debug/examples/recovery_policy_eval`. No validation +was weakened. Existing Rust dead-code warnings and two Python unclosed SQLite +connection `ResourceWarning`s remain visible in the logs. + +The baseline missing-lock refusal and original offline-cache failure remain +unchanged under +`/home/neo/Projects/kairotensor-artifacts/reliability-review-20261115/` as +`rust-baseline.*` and `rust-resolved-baseline.*`; `historical-receipts.json` +records their identities. A new offline full-metadata inspection also exposed +uncached optional `syn 2.0.119`; the retained `cargo-metadata.stderr.log` records +that failure. Locked metadata with network access then succeeded without changing +the resolution. A committed lock enables reproducible resolution, not a promise +that every optional dependency is available offline. + +`.github/workflows/check.yml` retains its existing `contracts` job and gates, +adding only `--locked` to its 13 Cargo invocations (one test and twelve runs). +It still uses `ubuntu-latest` with unpinned runner-provided Rust/Python. This task +performed local verification only; no fresh hosted CI run is claimed. Latest +stable and preview jobs, immutable runner/runtime pins, supported consumers, +service dependencies, accepted rollback artifacts, promotion procedure, isolated +rollback exercise, observation window, retirement date and owner acceptance are +not established. The project technical owner owns these adoption gaps, with +independent review required before promotion. Adding `--locked` does not close +them or retire historical pins or rollback assets. From 0d7f61cd8f757186b72f716052f506d4a49e70f8 Mon Sep 17 00:00:00 2001 From: Alejandro Castellanos <5405628+k2jac9@users.noreply.github.com> Date: Sun, 15 Nov 2026 14:16:13 -0500 Subject: [PATCH 03/51] feat: explain current eligibility through the shared read gate --- README.md | 7 + docs/current-eligibility.md | 51 ++++++ src/attempts.rs | 177 +++++++++++++++++-- src/eligibility.rs | 113 ++++++++++++ src/lib.rs | 2 + src/support.rs | 14 +- tests/eligibility.rs | 337 ++++++++++++++++++++++++++++++++++++ 7 files changed, 683 insertions(+), 18 deletions(-) create mode 100644 docs/current-eligibility.md create mode 100644 src/eligibility.rs create mode 100644 tests/eligibility.rs diff --git a/README.md b/README.md index 1bd444a..22bbcd3 100644 --- a/README.md +++ b/README.md @@ -28,6 +28,13 @@ The Rust store provides three ordered batch operations. Each call commits in one Evidence ingestion and rule publication are separate commit boundaries. A caller can reuse IDs from a successful import if rule registration fails. For an existing catalogue, bulk grants update witness counts immediately. Use `revise_support` for an affected head whose verdict needs a new publication. The selective revision example exercises bulk setup followed by an atomic withdrawal. +## Current eligibility + +`Store::explain_current(actor, scope, branch)` explains why a head is ready, +empty, or needs review without loading its payload or changing historical +receipts and usage. It shares the authoritative gate with current reads and +resume. [Reason codes and snapshot contract](docs/current-eligibility.md) + ## Selective revision The second example asks whether a test run has a reproduction record. An original report **and** its input manifest qualify together. A self-contained record from a second run also qualifies on its own. Withdrawing the original report preserves the accepted reproduction result through the second run. In the same transaction, the separate check that requires the original report publishes `unsupported`. diff --git a/docs/current-eligibility.md b/docs/current-eligibility.md new file mode 100644 index 0000000..a90f647 --- /dev/null +++ b/docs/current-eligibility.md @@ -0,0 +1,51 @@ +# Explain current eligibility + +`Store::explain_current(actor, scope, branch)` returns metadata explaining the +same gate used by `read_current` and `begin_resume`. `Ready` means eligible under +the Store contract; it does not mean the saved statement is true. The embedding +runtime must authenticate the actor/scope and authorize diagnostic access. + +| Snapshot | State | Proof | Reason | +| --- | --- | --- | --- | +| No branch or no head | `Empty` | None | `NoHead` | +| Head's captured epoch equals the current scope epoch | `Ready` | `CoarseEpoch` | None | +| Epoch changed; support/v1 verdict still matches its publication | `Ready` | `SupportV1` | None | +| Epoch changed; support/v1 verdict changed | `NeedsReview` | `SupportV1` | `SupportVerdictChanged` | +| Epoch changed; opaque or model head | `NeedsReview` | `CoarseEpoch` | `CoarseViewChanged` | + +The explanation includes `head_id`, `branch_revision`, `captured_epoch`, and +`current_epoch`. `branch_revision` is absent only when the branch does not exist. +An empty branch has no head, captured epoch, or proof. With an unchanged epoch, +even a support head uses the coarse epoch proof. A changed support verdict is +ineligible in either direction: supported to unsupported or the reverse. +Missing witnesses do not prove the negation of the underlying proposition. + +The evaluator verifies an accepted delivery exists without fetching its output. +A missing or nonaccepted delivery for a stored head is an error, including when +its epoch is stale. Database read errors propagate. Explanations contain no +output bytes, serialized request, occurrence payload, or free-form validator +reason, including in their `Debug` representation. + +Each explanation is evaluated within one SQLite read transaction. Current reads +retain that transaction through loading the accepted output. Resume retains its +existing writer transaction through the shared gate, output loading, and capture +of the new attempt and lineage. A writer committing during a WAL read may leave +that pending read on the older coherent snapshot. The next read observes the new +state. Separate calls to `explain_current` and `read_current` may see different +snapshots; an explanation is never an authorization cache or permission to send. + +Explanations do not change admission events, branch revisions, attempts, +historical dispositions, charges, or unresolved reservations. Historical +`Accepted` receipts remain historical even after current access is withdrawn. +Opaque and model heads retain coarse invalidation. A compatible model result +requires a new opted-in attempt and explicit `resolve_model_attempt`; explaining +it neither performs reuse nor dispatches work. + +The eligibility tests cover every table row, both support-result directions, +actor/scope separation, empty branches, history and usage preservation, malformed +heads, and explicit model reuse. Internal test-only channel barriers pause the +actual read after gate evaluation, permit a second connection to commit, and +check immediately before output loading that the transaction remains active and +the same connection still sees the captured epoch, head, and branch revision. +They then compare the output with the captured head and check the next read. +There is no public race hook or new persisted format. diff --git a/src/attempts.rs b/src/attempts.rs index b429ccd..6b97ad3 100644 --- a/src/attempts.rs +++ b/src/attempts.rs @@ -82,18 +82,16 @@ pub(crate) fn ensure_branch( } // The caller retains one SQLite snapshot throughout the gate and payload read. -fn current_head(db: &Connection, branch: &Branch, epoch: i64) -> Result { - let Some(head) = branch.head else { - return Ok(Current::Empty); - }; - let certified_epoch: i64 = db.query_row( - "SELECT view_epoch FROM attempts WHERE id = ?1", - [head], - |r| r.get(0), - )?; - if certified_epoch != epoch && !crate::support::is_current(db, head)? { - return Ok(Current::NeedsReview); +fn current_head(db: &Connection, branch: Option<&Branch>, epoch: i64) -> Result { + let explanation = crate::eligibility::evaluate(db, branch, epoch)?; + match explanation.state { + crate::EligibilityState::Empty => return Ok(Current::Empty), + crate::EligibilityState::NeedsReview => return Ok(Current::NeedsReview), + crate::EligibilityState::Ready => {} } + let head = explanation.head_id.expect("ready gate has a head"); + #[cfg(test)] + eligibility_snapshot_tests::before_payload(db, &explanation); db.query_row( "SELECT output FROM deliveries WHERE attempt_id = ?1 AND disposition = 'accepted'", [head], @@ -327,7 +325,7 @@ impl Store { let branch = find_branch(&tx, actor, scope, branch)?.ok_or_else(|| invalid("no head to resume"))?; let epoch = view_epoch(&tx, actor, scope)?; - let output = match current_head(&tx, &branch, epoch)? { + let output = match current_head(&tx, Some(&branch), epoch)? { Current::Empty => return Err(invalid("no head to resume")), Current::NeedsReview => return Err(invalid("head must be rebuilt before resume")), Current::Ready(output) => output, @@ -341,10 +339,8 @@ impl Store { /// Return only bytes eligible at this read's snapshot boundary. pub fn read_current(&mut self, actor: &str, scope: &str, branch: &str) -> Result { let tx = self.conn.transaction()?; - let result = match find_branch(&tx, actor, scope, branch)? { - None => Current::Empty, - Some(branch) => current_head(&tx, &branch, view_epoch(&tx, actor, scope)?)?, - }; + let branch = find_branch(&tx, actor, scope, branch)?; + let result = current_head(&tx, branch.as_ref(), view_epoch(&tx, actor, scope)?)?; tx.commit()?; Ok(result) } @@ -620,3 +616,152 @@ mod tests { Ok(()) } } + +#[cfg(test)] +mod eligibility_snapshot_tests { + use super::*; + use crate::{CurrentExplanation, EligibilityState}; + use std::cell::RefCell; + use std::path::PathBuf; + use std::rc::Rc; + use std::sync::{ + atomic::{AtomicU64, Ordering}, + mpsc, + }; + use std::time::Duration; + + type PayloadBoundary = Box; + thread_local! { + static PAYLOAD_BOUNDARY: RefCell> = RefCell::new(None); + } + + pub(super) fn before_payload(db: &Connection, explanation: &CurrentExplanation) { + PAYLOAD_BOUNDARY.with(|hook| { + if let Some(hook) = hook.borrow_mut().take() { + hook(db, explanation); + } + }); + } + + struct Database(PathBuf); + impl Database { + fn new() -> Self { + static NEXT: AtomicU64 = AtomicU64::new(0); + Self(std::env::temp_dir().join(format!( + "kairotensor-snapshot-{}-{}.sqlite", + std::process::id(), + NEXT.fetch_add(1, Ordering::Relaxed) + ))) + } + } + impl Drop for Database { + fn drop(&mut self) { + for suffix in ["", "-wal", "-shm"] { + let _ = std::fs::remove_file(format!("{}{suffix}", self.0.display())); + } + } + } + + // Channel barriers have a timeout only to diagnose missing hooks/writer failures; + // scheduling and the concurrent commit never depend on elapsed time or sleeps. + fn concurrent_read(replace: bool) -> Result<()> { + let file = Database::new(); + let mut reader = Store::open(&file.0)?; + reader.conn.pragma_update(None, "journal_mode", "WAL")?; + let id = reader.observe_and_admit_many("a", "s", &[("source", b"input")])?[0]; + let first = reader.begin_attempt("a", "s", "b", &[id])?; + reader.settle(first, b"old snapshot output", 1)?; + let mut writer = Store::open(&file.0)?; + let (gate_tx, gate_rx) = mpsc::channel(); + let (commit_tx, commit_rx) = mpsc::channel(); + let worker = std::thread::spawn(move || -> Result> { + gate_rx + .recv_timeout(Duration::from_secs(5)) + .expect("read reached gate barrier"); + let replacement = if replace { + let next = writer.begin_attempt("a", "s", "b", &[id])?; + writer.settle(next, b"new snapshot output", 2)?; + Some(next) + } else { + writer.set_admitted("a", "s", id, false)?; + None + }; + commit_tx.send(()).unwrap(); + Ok(replacement) + }); + let captured = Rc::new(RefCell::new(None)); + let captured_at_boundary = captured.clone(); + PAYLOAD_BOUNDARY.with(|hook| { + *hook.borrow_mut() = Some(Box::new(move |db, explanation| { + gate_tx.send(()).unwrap(); + commit_rx + .recv_timeout(Duration::from_secs(5)) + .expect("writer committed at barrier"); + assert!( + !db.is_autocommit(), + "payload boundary must retain an active transaction" + ); + assert_eq!( + view_epoch(db, "a", "s").unwrap(), + explanation.current_epoch, + "payload boundary must retain the gate scope epoch" + ); + let branch = find_branch(db, "a", "s", "b").unwrap().unwrap(); + assert_eq!( + branch.head, explanation.head_id, + "payload boundary must retain the gate head" + ); + assert_eq!( + Some(branch.revision), + explanation.branch_revision, + "payload boundary must retain the gate branch revision" + ); + *captured_at_boundary.borrow_mut() = Some(explanation.clone()); + })) + }); + let result = reader.read_current("a", "s", "b")?; + let replacement = worker.join().unwrap()?; + let explanation = captured + .borrow() + .clone() + .expect("actual read payload boundary was reached"); + assert_eq!(explanation.state, EligibilityState::Ready); + assert_eq!(explanation.head_id, Some(first)); + assert_eq!(explanation.branch_revision, Some(1)); + assert_eq!(explanation.current_epoch, 1); + assert_eq!( + result, + Current::Ready(b"old snapshot output".to_vec()), + "gate metadata and payload must come from the same snapshot" + ); + if let Some(replacement) = replacement { + assert_eq!( + reader.explain_current("a", "s", "b")?.head_id, + Some(replacement) + ); + assert_eq!( + reader.explain_current("a", "s", "b")?.branch_revision, + Some(2) + ); + assert_eq!( + reader.read_current("a", "s", "b")?, + Current::Ready(b"new snapshot output".to_vec()) + ); + } else { + assert_eq!(reader.explain_current("a", "s", "b")?.current_epoch, 2); + assert_eq!(reader.read_current("a", "s", "b")?, Current::NeedsReview); + assert!(reader.begin_resume("a", "s", "b", &[]).is_err()); + } + Ok(()) + } + + #[test] + fn revocation_snapshot_never_exposes_payload() -> Result<()> { + concurrent_read(false) + } + + #[test] + fn head_replacement_keeps_metadata_and_output_in_one_snapshot() -> Result<()> { + concurrent_read(true) + } +} diff --git a/src/eligibility.rs b/src/eligibility.rs new file mode 100644 index 0000000..1714180 --- /dev/null +++ b/src/eligibility.rs @@ -0,0 +1,113 @@ +//! Payload-free diagnostics for the authoritative current-read gate. +use rusqlite::{Connection, Result}; + +use crate::attempts::{find_branch, Branch}; +use crate::support::{inspect_current, SupportStatus}; +use crate::{view_epoch, Store}; + +/// Eligibility under the Store contract, not a claim that an output is true. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum EligibilityState { + Empty, + Ready, + NeedsReview, +} + +/// The proof used at this snapshot; model reuse requires explicit resolution. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum EligibilityProof { + CoarseEpoch, + SupportV1, +} + +/// Fixed reason codes never contain evidence or free-form validation text. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum EligibilityReason { + NoHead, + CoarseViewChanged, + SupportVerdictChanged, +} + +/// Metadata from one read transaction. This is not a reusable authorization. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct CurrentExplanation { + pub state: EligibilityState, + pub head_id: Option, + pub branch_revision: Option, + pub captured_epoch: Option, + pub current_epoch: i64, + pub proof_kind: Option, + pub reason: Option, +} + +// The caller must retain its transaction through any subsequent payload load. +// This evaluator never selects output, request, evidence, or validator text. +pub(crate) fn evaluate( + db: &Connection, + branch: Option<&Branch>, + current_epoch: i64, +) -> Result { + let mut explanation = CurrentExplanation { + state: EligibilityState::Empty, + head_id: branch.and_then(|b| b.head), + branch_revision: branch.map(|b| b.revision), + captured_epoch: None, + current_epoch, + proof_kind: None, + reason: Some(EligibilityReason::NoHead), + }; + let Some((branch, head)) = branch.and_then(|b| b.head.map(|head| (b, head))) else { + return Ok(explanation); + }; + // Missing/nonaccepted delivery or a head belonging to another branch is a + // malformed stored head, even if its epoch would otherwise be stale. + let captured_epoch: i64 = db.query_row( + "SELECT a.view_epoch FROM attempts a + JOIN deliveries d ON d.attempt_id = a.id AND d.disposition = 'accepted' + WHERE a.id = ?1 AND a.branch_id = ?2", + rusqlite::params![head, branch.id], + |row| row.get(0), + )?; + explanation.captured_epoch = Some(captured_epoch); + let (state, proof, reason) = if captured_epoch == current_epoch { + (EligibilityState::Ready, EligibilityProof::CoarseEpoch, None) + } else { + match inspect_current(db, head)? { + SupportStatus::SameVerdict => { + (EligibilityState::Ready, EligibilityProof::SupportV1, None) + } + SupportStatus::ChangedVerdict => ( + EligibilityState::NeedsReview, + EligibilityProof::SupportV1, + Some(EligibilityReason::SupportVerdictChanged), + ), + SupportStatus::NotSupport => ( + EligibilityState::NeedsReview, + EligibilityProof::CoarseEpoch, + Some(EligibilityReason::CoarseViewChanged), + ), + } + }; + explanation.state = state; + explanation.proof_kind = Some(proof); + explanation.reason = reason; + Ok(explanation) +} + +impl Store { + /// Explain the current head without loading its payload or changing history. + /// The embedding runtime authenticates actor/scope and diagnostic access. + /// A later read evaluates its own snapshot; this result grants no access. + pub fn explain_current( + &mut self, + actor: &str, + scope: &str, + branch: &str, + ) -> Result { + let tx = self.conn.transaction()?; + let branch = find_branch(&tx, actor, scope, branch)?; + let result = evaluate(&tx, branch.as_ref(), view_epoch(&tx, actor, scope)?)?; + tx.commit()?; + Ok(result) + } +} diff --git a/src/lib.rs b/src/lib.rs index 0c17391..3ab5f11 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1,6 +1,8 @@ //! Local versioned evidence memory with actor-scoped admission and atomic publication. mod attempts; +mod eligibility; +pub use eligibility::{CurrentExplanation, EligibilityProof, EligibilityReason, EligibilityState}; mod timed_recovery; pub use timed_recovery::{RecoveryReadiness, RecoveryRuntime, RecoveryCalibration, RecoveryTiming, RecoveryClock, RecoveryPreparation, TimingRefusal, TimedRecoveryClaim}; mod report_verdict; diff --git a/src/support.rs b/src/support.rs index 4f71808..dc88c1d 100644 --- a/src/support.rs +++ b/src/support.rs @@ -59,8 +59,18 @@ fn rule_state(db: &Connection, head: i64) -> Result> { .optional() } -pub(crate) fn is_current(db: &Connection, head: i64) -> Result { - Ok(rule_state(db, head)?.is_some_and(|state| state.current == state.published)) +pub(crate) enum SupportStatus { + NotSupport, + SameVerdict, + ChangedVerdict, +} + +pub(crate) fn inspect_current(db: &Connection, head: i64) -> Result { + Ok(match rule_state(db, head)? { + None => SupportStatus::NotSupport, + Some(state) if state.current == state.published => SupportStatus::SameVerdict, + Some(_) => SupportStatus::ChangedVerdict, + }) } // Called exactly once per effective transition, in the admission writer's diff --git a/tests/eligibility.rs b/tests/eligibility.rs new file mode 100644 index 0000000..b376789 --- /dev/null +++ b/tests/eligibility.rs @@ -0,0 +1,337 @@ +use kairotensor::{ + Current, CurrentExplanation, Disposition, EligibilityProof as Proof, + EligibilityReason as Reason, EligibilityState as State, ModelAction, Store, +}; +use rusqlite::{Connection, Result}; +use std::path::PathBuf; +use std::sync::atomic::{AtomicU64, Ordering}; + +struct Database(PathBuf); +impl Database { + fn new() -> Self { + static NEXT: AtomicU64 = AtomicU64::new(0); + Self(std::env::temp_dir().join(format!( + "kairotensor-eligibility-{}-{}.sqlite", + std::process::id(), + NEXT.fetch_add(1, Ordering::Relaxed) + ))) + } +} +impl Drop for Database { + fn drop(&mut self) { + let _ = std::fs::remove_file(&self.0); + } +} + +// No concurrent writer exists during these separate public calls. +fn check( + store: &mut Store, + actor: &str, + scope: &str, + branch: &str, + expected: CurrentExplanation, + current: Current, +) -> Result<()> { + assert_eq!(store.explain_current(actor, scope, branch)?, expected); + assert_eq!(store.read_current(actor, scope, branch)?, current); + Ok(()) +} + +#[test] +fn explanation_matches_current_gate() -> Result<()> { + let mut store = Store::in_memory()?; + let empty = CurrentExplanation { + state: State::Empty, + head_id: None, + branch_revision: None, + captured_epoch: None, + current_epoch: 0, + proof_kind: None, + reason: Some(Reason::NoHead), + }; + check(&mut store, "a", "s", "b", empty.clone(), Current::Empty)?; + store.begin_attempt("a", "s", "b", &[])?; + check( + &mut store, + "a", + "s", + "b", + CurrentExplanation { + branch_revision: Some(0), + ..empty.clone() + }, + Current::Empty, + )?; + assert!(store + .begin_resume("a", "s", "missing", &[]) + .unwrap_err() + .to_string() + .contains("no head to resume")); + assert!(store + .begin_resume("a", "s", "b", &[]) + .unwrap_err() + .to_string() + .contains("no head to resume")); + let id = store.observe("private source", b"private evidence")?; + store.set_admitted("a", "s", id, true)?; + let opaque = store.begin_attempt("a", "s", "b", &[id])?; + store.settle(opaque, b"opaque", 3)?; + let ready = CurrentExplanation { + state: State::Ready, + head_id: Some(opaque), + branch_revision: Some(1), + captured_epoch: Some(1), + current_epoch: 1, + proof_kind: Some(Proof::CoarseEpoch), + reason: None, + }; + check( + &mut store, + "a", + "s", + "b", + ready.clone(), + Current::Ready(b"opaque".to_vec()), + )?; + store.set_admitted("a", "s", id, false)?; + check( + &mut store, + "a", + "s", + "b", + CurrentExplanation { + state: State::NeedsReview, + current_epoch: 2, + reason: Some(Reason::CoarseViewChanged), + ..ready + }, + Current::NeedsReview, + )?; + for (actor, scope) in [("other", "s"), ("a", "other")] { + check(&mut store, actor, scope, "b", empty.clone(), Current::Empty)?; + } + + // Both published verdicts: same-verdict reuse and transitions in both directions. + for initially_admitted in [false, true] { + let mut store = Store::in_memory()?; + let witness = store.observe("witness", b"sensitive")?; + let unrelated = store.observe("unrelated", b"other")?; + store.set_admitted("a", "s", witness, initially_admitted)?; + let head = store.publish_support("a", "s", "support", &[vec![witness]])?; + let epoch = i64::from(initially_admitted); + let initial = CurrentExplanation { + state: State::Ready, + head_id: Some(head), + branch_revision: Some(1), + captured_epoch: Some(epoch), + current_epoch: epoch, + proof_kind: Some(Proof::CoarseEpoch), + reason: None, + }; + let output = if initially_admitted { + b"supported".to_vec() + } else { + b"unsupported".to_vec() + }; + check( + &mut store, + "a", + "s", + "support", + initial.clone(), + Current::Ready(output.clone()), + )?; + store.set_admitted("a", "s", unrelated, true)?; + let selective = CurrentExplanation { + proof_kind: Some(Proof::SupportV1), + current_epoch: epoch + 1, + ..initial + }; + check( + &mut store, + "a", + "s", + "support", + selective.clone(), + Current::Ready(output.clone()), + )?; + assert_eq!(store.begin_resume("a", "s", "support", &[])?.1, output); + store.set_admitted("a", "s", witness, !initially_admitted)?; + check( + &mut store, + "a", + "s", + "support", + CurrentExplanation { + state: State::NeedsReview, + current_epoch: epoch + 2, + reason: Some(Reason::SupportVerdictChanged), + ..selective + }, + Current::NeedsReview, + )?; + assert!(store + .begin_resume("a", "s", "support", &[]) + .unwrap_err() + .to_string() + .contains("head must be rebuilt before resume")); + } + Ok(()) +} + +#[test] +fn historical_acceptance_does_not_grant_current_access() -> Result<()> { + let file = Database::new(); + let mut store = Store::open(&file.0)?; + let id = store.observe_and_admit_many("a", "s", &[("source", b"evidence")])?[0]; + let head = store.begin_attempt("a", "s", "b", &[id])?; + let budget = store.create_token_budget(100)?; + store.reserve_dispatch(budget, head, b"request", 2, 8)?; + assert_eq!(store.settle(head, b"accepted", 7)?, Disposition::Accepted); + let pending = store.begin_attempt("a", "s", "uncertain", &[])?; + store.reserve_dispatch(budget, pending, b"unknown send", 4, 10)?; + store.set_admitted("a", "s", id, false)?; + let before_budget = store.token_budget_state(budget)?; + let audit = Connection::open(&file.0)?; + let before = snapshot(&audit)?; + for _ in 0..3 { + assert_eq!( + store.explain_current("a", "s", "b")?.state, + State::NeedsReview + ); + assert_eq!(store.read_current("a", "s", "b")?, Current::NeedsReview); + assert!(store + .begin_resume("a", "s", "b", &[]) + .unwrap_err() + .to_string() + .contains("head must be rebuilt before resume")); + assert_eq!(store.settle(head, b"accepted", 7)?, Disposition::Accepted); + assert_eq!( + store.explain_current("missing", "s", "b")?.state, + State::Empty + ); + } + assert_eq!(snapshot(&audit)?, before); + assert_eq!(store.token_budget_state(budget)?, before_budget); + assert_eq!(store.charged_usage()?, 7); + assert_eq!(before_budget.spent, 7); + assert_eq!(before_budget.reserved, 14); + Ok(()) +} + +fn snapshot(db: &Connection) -> Result>> { + let mut snapshot = Vec::new(); + for table in [ + "branches", + "attempts", + "attempt_inputs", + "admission_events", + "deliveries", + "dispatches", + "token_budgets", + "token_reservations", + ] { + let mut statement = db.prepare(&format!("SELECT * FROM {table} ORDER BY 1, 2"))?; + let columns = statement.column_count(); + snapshot.extend( + statement + .query_map([], |row| (0..columns).map(|i| row.get(i)).collect())? + .collect::>>()?, + ); + } + Ok(snapshot) +} + +#[test] +fn model_reuse_requires_explicit_resolution() -> Result<()> { + let mut store = Store::in_memory()?; + let id = store.observe_and_admit_many("a", "s", &[("source", b"evidence")])?[0]; + let (origin, _) = store.begin_model_attempt("a", "s", "b", &[id], "closed/v1")?; + assert_eq!( + store.resolve_model_attempt(origin, b"request")?, + ModelAction::Dispatch + ); + store.settle(origin, b"model output", 9)?; + store.observe_and_admit_many("a", "s", &[("irrelevant", b"new")])?; + let stale = store.explain_current("a", "s", "b")?; + assert_eq!(stale.state, State::NeedsReview); + assert_eq!(stale.proof_kind, Some(Proof::CoarseEpoch)); + assert_eq!(stale.reason, Some(Reason::CoarseViewChanged)); + assert!(store.begin_resume("a", "s", "b", &[]).is_err()); + let (next, _) = store.begin_model_attempt("a", "s", "b", &[id], "closed/v1")?; + assert_eq!(store.explain_current("a", "s", "b")?, stale); + assert_eq!( + store.resolve_model_attempt(next, b"request")?, + ModelAction::Reused { + origin_attempt: origin, + output: b"model output".to_vec() + } + ); + let ready = store.explain_current("a", "s", "b")?; + assert_eq!(ready.state, State::Ready); + assert_eq!(ready.head_id, Some(next)); + assert_eq!(ready.branch_revision, Some(2)); + assert_eq!(ready.proof_kind, Some(Proof::CoarseEpoch)); + assert_eq!( + store.read_current("a", "s", "b")?, + Current::Ready(b"model output".to_vec()) + ); + assert_eq!(store.charged_usage()?, 9); + Ok(()) +} + +#[test] +fn explanation_contains_no_payload_or_validation_text() -> Result<()> { + let mut store = Store::in_memory()?; + let evidence = + store.observe_and_admit_many("a", "s", &[("SECRET_SOURCE", b"SECRET_EVIDENCE")])?[0]; + let head = store.begin_attempt("a", "s", "b", &[evidence])?; + store.record_dispatch(head, b"SECRET_REQUEST")?; + store.settle(head, b"SECRET_OUTPUT", 2)?; + let rejected = store.begin_attempt("a", "s", "b", &[evidence])?; + store.record_dispatch(rejected, b"SECRET_REJECTED_REQUEST")?; + store.reject_invalid( + rejected, + b"SECRET_REJECTED", + 4, + "SECRET_VALIDATOR", + "SECRET_REASON", + )?; + let ready = store.explain_current("a", "s", "b")?; + assert_eq!(ready.state, State::Ready); + store.set_admitted("a", "s", evidence, false)?; + let stale = store.explain_current("a", "s", "b")?; + assert_eq!(stale.state, State::NeedsReview); + for explanation in [ready, stale] { + // Fixed typed fields constrain the public surface; Debug must not leak any text. + let CurrentExplanation { + state: _, + head_id: _, + branch_revision: _, + captured_epoch: _, + current_epoch: _, + proof_kind: _, + reason: _, + } = explanation; + assert!(!format!("{explanation:?}").contains("SECRET")); + } + Ok(()) +} + +#[test] +fn malformed_head_without_accepted_delivery_is_an_error() -> Result<()> { + let file = Database::new(); + let mut store = Store::open(&file.0)?; + let pending = store.begin_attempt("a", "s", "b", &[])?; + let db = Connection::open(&file.0)?; + db.execute("UPDATE branches SET head=?1, revision=1", [pending])?; + for changed_epoch in [false, true] { + if changed_epoch { + store.observe_and_admit_many("a", "s", &[("new", b"x")])?; + } + assert!(store.explain_current("a", "s", "b").is_err()); + assert!(store.read_current("a", "s", "b").is_err()); + assert!(store.begin_resume("a", "s", "b", &[]).is_err()); + } + Ok(()) +} From c322e575c565513b157ec88fa734c87343455d57 Mon Sep 17 00:00:00 2001 From: Alejandro Castellanos <5405628+k2jac9@users.noreply.github.com> Date: Sun, 15 Nov 2026 14:19:01 -0500 Subject: [PATCH 04/51] docs: define autonomous reliability milestones and decision boundaries --- .../plans/2026-11-15-autonomous-roadmap.md | 82 ++++++ .../plans/2026-11-15-decision-replay.md | 179 +++++++++++++ .../2026-11-15-evidence-reliability-p0.md | 10 +- .../plans/2026-11-15-incident-store-bridge.md | 173 +++++++++++++ .../plans/2026-11-15-offline-diagnostics.md | 140 ++++++++++ .../plans/2026-11-15-policy-activation.md | 242 ++++++++++++++++++ .../plans/2026-11-15-runtime-qualification.md | 94 +++++++ .../2026-11-15-reliability-continuation.md | 208 +++++++++++++++ 8 files changed, 1124 insertions(+), 4 deletions(-) create mode 100644 docs/superpowers/plans/2026-11-15-autonomous-roadmap.md create mode 100644 docs/superpowers/plans/2026-11-15-decision-replay.md create mode 100644 docs/superpowers/plans/2026-11-15-incident-store-bridge.md create mode 100644 docs/superpowers/plans/2026-11-15-offline-diagnostics.md create mode 100644 docs/superpowers/plans/2026-11-15-policy-activation.md create mode 100644 docs/superpowers/plans/2026-11-15-runtime-qualification.md create mode 100644 docs/superpowers/specs/2026-11-15-reliability-continuation.md diff --git a/docs/superpowers/plans/2026-11-15-autonomous-roadmap.md b/docs/superpowers/plans/2026-11-15-autonomous-roadmap.md new file mode 100644 index 0000000..f15b170 --- /dev/null +++ b/docs/superpowers/plans/2026-11-15-autonomous-roadmap.md @@ -0,0 +1,82 @@ +# KairoTensor autonomous execution map + +Authority: on 15 November 2026 the owner authorized autonomous implementation, +research, documented decisions and continued execution. The owner explicitly +excluded any Rocket connection. Existing available credits may be used for +bounded, useful experiments; this is not authority for top-ups, subscription +changes or spending beyond verified existing balances. + +This map supersedes the earlier review-only execution restriction. Historical +reviews and receipts remain unchanged. Checkboxes in the component plans describe +future work; completion is recorded with commits and evidence in the ledger. + +## Execution order + +| Stage | Executable plan | Dependency / deliverable | +| --- | --- | --- | +| A | [P0](2026-11-15-evidence-reliability-p0.md) | Qualified lock, shared eligibility explanation, atomic batch correction. | +| B | [Incident bridge](2026-11-15-incident-store-bridge.md) | After A; captured context, exact fixture requests, existing timed execution gates. | +| C | [Decision replay](2026-11-15-decision-replay.md) | After B; authorized export, honest missing-data outcomes, pure replay. | +| D | [Policy activation](2026-11-15-policy-activation.md) | After A-C; comparable offline evaluation and transactional activation. | +| E1 | [Offline diagnostics](2026-11-15-offline-diagnostics.md) | After C; bounded plans and recorded-outcome scoring. | +| E2 | Same diagnostics plan, conditional task | Portable interchange only when a real second consumer requires it. No Rocket integration. | +| F | Bounded experiments | After relevant offline gates; separate run manifests and exact credit bounds. | +| G | [Runtime qualification](2026-11-15-runtime-qualification.md) | After integrated core; qualify isolated current/preview/previous lanes and exact project pins. | + +Read the [continuation specification](../specs/2026-11-15-reliability-continuation.md) +with each component plan. It resolves details absent from the original proposal. +Architecture stays Rust/SQLite first; Python remains an offline fixture/scoring +layer. No new service, distributed authority or programmable policy language. + +## Autonomous operating contract + +1. Reconcile source and literal allowed files before each task. Use one code + implementer at a time and an independent reviewer at each completed task. + The coordinator may write planning documents concurrently in disjoint files. +2. Record BASE/HEAD, task identity, reviewed spec and allowed files before work. + Preserve a failing test, implement, run relevant checks, review, fix and commit. + Never alter a worker's grader/permissions to hide a failure. The coordinator + documents any necessary successor scope before dispatching it. +3. Resolve routine ambiguity from source, primary research and this specification. + Record `Ruling: decision — evidence/reason — consequence if wrong` in the + ledger. Continue without routine owner check-ins. +4. Reuse existing storage admission. The project exception expires on + 17 November 2026 at 18:04:38 UTC. Its expiry is not permission to extend it, + discard evidence or release another workload's reservations. Refused heavy + work leaves read-only review and lightweight planning available. +5. At every integration boundary run the locked all-target Rust suite and the + existing Python suites. Run fixture examples with explicit artifact output + paths. Repeat broad checks only after changes or unresolved concerns. +6. Retain failed tests, mutations, evaluations, source/runtime/dependency + identities and cost receipts in `kairotensor-artifacts`. Never overwrite an + archived research result with a newer one. +7. Finish local implementation, commits and review autonomously. No shared-branch + push, PR merge, deployment, publication or physical disk modification is implied. + Defer those external actions while continuing authorized local work. + +## Experiment discipline + +Before spending, inspect supported account/model/balance tools and prior-run +receipts. An absent local receipt does not establish that another runner did +not send. A connected training tool does not prove hosted inference access. +Freeze a hypothesis, source, request set, model/runtime, validators, scorer, +split, deadline, call cap, measured balance and all-category cost ceiling. +Keep reserves for ambiguous sends; do not retry an unknown operation blindly. + +Choose experiments by expected information value, not by the amount of credit +available. Initial priorities: a bounded hosted connectivity/accounting smoke, +then a paired Store-backed semantic fixture using a fresh final holdout if +earlier data influenced design. Do not silently compare hosted timings with +local Qwen hardware or convert fixture byte counts into model-token savings. +If access, complete billing bounds or prior-send state cannot be established, +record the limitation and advance the other implementation tasks. + +## Completion means + +A-E1 have passing source-bound tests, independent review dispositions and +documentation; runtime-policy gaps and experimental unknowns remain explicit. +E2 is deferred until its actual-consumer condition is met. No fake consumer is +invented to make a checklist complete. F reports measured results, including +negative/inconclusive outcomes; available credit is not a scientific result. +G closes runtime gaps that can be qualified locally and retains clear unavailable +or failed lanes; configured hosted CI is not a claim that hosted CI ran. diff --git a/docs/superpowers/plans/2026-11-15-decision-replay.md b/docs/superpowers/plans/2026-11-15-decision-replay.md new file mode 100644 index 0000000..4c578cc --- /dev/null +++ b/docs/superpowers/plans/2026-11-15-decision-replay.md @@ -0,0 +1,179 @@ +# Authorized decision export and pure replay implementation plan + +> **For agentic workers:** Use `superpowers:subagent-driven-development` or +> `superpowers:executing-plans`; execute tasks sequentially with tests and review. + +**Goal:** Export an authorized consistent decision snapshot and reconstruct or +validate what its recorded contracts actually support, without external effects. +**Architecture:** A Store read transaction builds an export; pure bounded parsing +and compiled validators consume it. No new transaction authority or migration. +**Tech Stack:** existing Rust/SQLite/serde and B's SHA-256/protocol primitives. +**Spec:** [continuation specification](../specs/2026-11-15-reliability-continuation.md), +section C. Prerequisites: A and B reviewed. + +## Global Constraints + +- Default current access and diagnostic historical access are distinct. +- Diagnostic approval does not override withdrawn evidence or restore bytes. +- A historical Accepted receipt is preserved, never used as an access grant. +- Authorize the exact dependency selection within the retained snapshot. +- Preserve arbitrary bytes, repeated input positions, lineage and uncertain holds. +- Do not export cross-scope/global budget totals or infer missing validators. +- Pure replay cannot invoke tools, a model, network, Store initialization or + migrations. No Rocket integration; use normal storage admission. + +## Review Focus + +1. Revocation/settlement/head changes must not splice different export snapshots. +2. Denied/redacted output must contain no payload-derived values or existence leak. +3. Unknown validator or missing lineage must not become reproducible success. +4. Changed bytes/accounting and duplicate occurrence loss must be detected. +5. A changed validator cannot rewrite historical disposition or release a hold. + +### Task 1: Bounded bundle schema and pure integrity validation + +**Files:** create `src/decision_bundle.rs`, `src/decision_bundle/types.rs`, +`src/decision_bundle/replay.rs`, `tests/decision_bundle.rs`, +`docs/decision-bundle-v1.md`; modify `src/lib.rs`, `src/report_verdict.rs` +(expose the existing pure derive function crate-internally only). + +**Public interfaces:** + +```rust +pub fn parse_decision_bundle(bytes: &[u8]) -> Result; +pub fn validate_recorded_bundle(bundle: &DecisionBundleV1) -> ReplayReport; +pub fn reevaluate_bundle(bundle: &DecisionBundleV1, validator: &str) -> ReplayReport; +``` + +`BundleError`: InvalidSchema, InvalidEncoding, LimitExceeded, InconsistentLedger, +Store(rusqlite::Error). Strict versioned structs deny unknown/duplicate fields. +Required top-level fields: schema_version=1, root_attempt, actor, scope, +export_purpose, snapshot (root branch/head/revision/current epoch), attempts, +evidence, and integrity manifest. Each byte field is `{hex, byte_len, sha256}`. +Attempt records contain ID, branch, captured epoch/revision, parent/inherited_from, +ordered candidates and inputs, optional dispatch request, explicit delivery state, +optional validator binding/provenance, optional immutable runtime/manifest, +per-attempt accounting, reuse origin, and support/deterministic contract if bound. +Each input entry preserves position/evidence_id and role (Context, Policy, Report, +Evidence or Unknown). Derive roles only from a recognized immutable contract; +unknown legacy roles remain Unknown. A role is descriptive metadata, never an +independent authorization or justification for omitting an input. +Recovery attempts also export recovery ID, ordinal, state and predecessor ID; +the recognized bridge contract maps ordinal0/1 to Initial/ProtocolRetry and binds +the fixed feedback string. Unrecognized phase/version is RendererUnavailable. +Evidence records contain occurrence ID, source bytes, exact payload bytes and +authorized scope reference. Numeric identities are positive checked i64s; repeated +positions are preserved but duplicate keys/record definitions refuse. + +Delivery state is Recorded { raw, disposition, usage, optional rejection metadata } +or Unresolved. Accounting contains unit, optional quote/granted/budget reference, +charge and hold bound; unknown measured usage stays null. Never invent times. +The integrity manifest binds the versioned ordered record serialization and +root selection; include a digest excluding its own digest field. Preserve all +original raw byte envelopes; reserialization of embedded bytes is forbidden. + +`ReplayReport` has independent consistency, reconstruction and validation +results; recorded_disposition, validator_provenance, missing_requirements, +charged/held units and source_bundle_sha256. Result statuses are Matched, +Mismatch, Incomplete or NotReproducible; missing requirements include +ValidatorUnavailable, RendererUnavailable, MissingInput, MissingDelivery, +MissingDependency, UnknownAccountingUnit and HistoricalTimingUnavailable. +Reevaluation additionally names the new validator and never changes the bundle. + +- [ ] Write failing tests for changed bytes/digests, non-UTF8 round-trip, identical + bytes/distinct IDs, repeated input positions, unknown schema, oversized/deep + JSON, inconsistent quote/usage/holds and unresolved delivery with retained hold. +- [ ] Implement strict parsing and consistency checks. A hash match establishes + internal consistency only, not authenticated execution. +- [ ] Add pure allowlist entries for report-verdict/v1 and B's incident protocol. + Replay compares deterministic output or exact reconstructed request as applicable. + Missing legacy validator is NotReproducible. Unknown version never falls back. +- [ ] Test changed validator as a separate report, missing inherited/reuse input, + and original disposition preservation. A panic-on-call fake adapter must remain + unused; the replay API takes no backend capability. +- [ ] Run `cargo test --locked --test decision_bundle` and locked all-targets; + review and commit `feat: define bounded pure decision bundle replay`. + +**Done:** parsing/replay is deterministic, effect-free and explicit about limits. + +### Task 2: Snapshot-bound Store export and diagnostic authority + +**Files:** create `src/decision_bundle/export.rs`; modify +`src/decision_bundle.rs`, `src/decision_bundle/types.rs`, `tests/decision_bundle.rs`, +`docs/decision-bundle-v1.md`. Use A's crate-internal evaluator; if its visibility +needs a change, `src/eligibility.rs` is allowed for that visibility only. + +**Interfaces:** + +```rust +pub struct BundleRequest { + pub actor: String, pub scope: String, pub attempt: i64, + pub purpose: BundlePurpose, // CurrentRead or DiagnosticReplay +} +pub trait DiagnosticAuthorizer { + fn authorize(&self, request: &DiagnosticReadRequest) -> Result; +} +pub enum BundleExport { Complete(DecisionBundleV1), Redacted(RedactedBundleV1) } +Store::export_decision_bundle(&mut self, request: &BundleRequest, + authorizer: &impl DiagnosticAuthorizer) -> Result; +``` + +`DiagnosticReadRequest` includes authenticated actor/scope, root ID/purpose and +the full ordered dependency selection: every output attempt and input occurrence, +their own actor/scope, branch revisions, head IDs and current/captured epochs. +No boolean authorization receipt is persisted or reusable. `DenyDiagnostics` +implements the default false result. Redacted has only the caller-supplied +requested_attempt and generic Unavailable reason. + +Keep one read transaction through metadata enumeration, authority, admission +checks and payload fetch. For CurrentRead apply A's gate to root and every required +output dependency; require every exported input admitted. For DiagnosticReplay, +authorize the exact selection, still require current input admission, then permit +historical output bytes. Omit global balances and unrelated budget members. +Copy bytes only after access checks; commit the read transaction before return. +Metadata/source/validator free text are sensitive too. Physical unavailability +or denied access never causes an archive lookup. SQL/I/O errors are not success. + +- [ ] Add failing tests for default denial, authorized historical output with + admitted inputs, revocation despite diagnostic approval, missing/withheld target + equivalence, source/request/reason canaries and shared budgets across scopes. +- [ ] Implement bounded dependency closure without treating all parents as data + dependencies. Export support formulas and recorded bindings, not a fabricated + historical witness counter. Missing required dependency redacts the result. +- [ ] Add controlled concurrent withdrawal, settlement and head-change tests. + Assert transaction liveness and unchanged selected rows at payload fetch; + output-only comparisons do not prove a retained snapshot. Callback receives + exact dependency IDs and cannot accidentally authorize a broader closure. +- [ ] Verify export leaves admissions, heads, usage and holds unchanged, including + after reopen. Run the target suite and relevant P0/recovery tests; review and + commit `feat: authorize decision exports within one Store snapshot`. + +**Done:** every complete bundle is an authorized export of one coherent snapshot; +redaction is explicit and payload-free. + +### Task 3: Separate-process replay and versioned Python reader + +**Files:** create `examples/decision_replay.rs`, `docs/decision-replay.md`, +`scripts/decision_bundle_v1.py`, `scripts/test_decision_bundle_v1.py`; +modify `scripts/audit_recovery_bundle.py` only for an explicit `--decision-v1 FILE` +branch, preserving the legacy directory mode unchanged; modify +`.github/workflows/check.yml` for offline replay steps. + +Example commands: `decision_replay export-fixture PATH` generates an authorized +fixture using Store; `decision_replay replay PATH` only parses and calls pure +replay, never Store::open. Python reader checks bounded hex bytes/digests and +prints integrity/availability facts; it must not claim to implement an unknown +Rust validator. Unknown modes refuse, no auto-detection that changes legacy mode. + +- [ ] Add failing subprocess tests that compare export/replay results, binary + preservation, redacted refusal, unknown validator and unchanged file hashes. +- [ ] Implement CLI and versioned reader; do not add a live/generate mode. +- [ ] Run `cargo run --locked --example decision_replay -- export-fixture PATH`, + then `cargo run --locked --example decision_replay -- replay PATH` in a second + process; run the new Python suite plus `test_audit_recovery_bundle.py`. +- [ ] Run integration suites, independent review and commit + `test: exercise decision replay across process boundaries`. + +**Done:** the second process reconstructs/validates supported recorded contracts, +or precisely reports why it cannot. Historical acceptance/timing stays reported, +not magically re-proven. No generic causal attribution is emitted. diff --git a/docs/superpowers/plans/2026-11-15-evidence-reliability-p0.md b/docs/superpowers/plans/2026-11-15-evidence-reliability-p0.md index 78a99c4..32e9f63 100644 --- a/docs/superpowers/plans/2026-11-15-evidence-reliability-p0.md +++ b/docs/superpowers/plans/2026-11-15-evidence-reliability-p0.md @@ -2,8 +2,9 @@ > **For agentic workers:** Use `superpowers:executing-plans` or > `superpowers:subagent-driven-development` to execute task by task. Every checkbox -> below is future work. The owner requested review and planning first; this -> document does not claim implementation or authorize a live model run. +> below is future work until recorded complete. The owner first requested review, +> then authorized autonomous implementation on 15 November 2026. Follow the +> [execution map](2026-11-15-autonomous-roadmap.md); this plan makes no live-run claim. **Goal:** Explain current eligibility and apply several admission corrections atomically without weakening read, lineage or accounting guarantees. @@ -267,5 +268,6 @@ and independently useful with the explanation API. No migration is needed. - [ ] Prepare the branch for owner review. Merge, publication, hosted spending and the separately authorized Adaption smoke are not part of this milestone. -Next: write the Store-backed incident bridge plan using section 3 of the review. -Bundle replay, policy activation and portability remain later, separate plans. +Next: follow the [execution map](2026-11-15-autonomous-roadmap.md) into the separate +incident bridge, decision replay, policy activation and offline diagnostics plans. +No Rocket integration. Portable interchange retains its real-consumer prerequisite. diff --git a/docs/superpowers/plans/2026-11-15-incident-store-bridge.md b/docs/superpowers/plans/2026-11-15-incident-store-bridge.md new file mode 100644 index 0000000..4f6e476 --- /dev/null +++ b/docs/superpowers/plans/2026-11-15-incident-store-bridge.md @@ -0,0 +1,173 @@ +# Store-backed incident fixture implementation plan + +> **For agentic workers:** Use `superpowers:subagent-driven-development` or +> `superpowers:executing-plans`. Execute tasks sequentially; checkboxes are future +> work. The owner authorized autonomous implementation and documented rulings. + +**Goal:** Demonstrate real Store eligibility at incident capture, claim and +publication using deterministic adapters and the frozen v1 renderer. +**Architecture:** A pure protocol module and an example-local adapter reuse the +existing timed recovery state machine. Capture one context occurrence followed +by every required report. No separate admission database. +**Tech Stack:** existing Rust/SQLite/serde, qualified `sha2 = "=0.11.1"`, Python +fixture parity. No provider/model/service dependency. +**Spec:** [continuation specification](../specs/2026-11-15-reliability-continuation.md), +section B. Prerequisite: P0 A complete and reviewed. + +## Global Constraints + +- Preserve occurrence identity, candidate/input order, lineage and all holds. +- Exact manifest bytes bind the context, mappings and runtime; hashes are not + proof of execution, truth or authority. +- All required IDs come from the pinned workload definition. Caller flags and + caller-supplied payload replacements cannot authorize preparation. +- Fixture byte accounting is labelled explicitly; no live model calls. +- No Rocket connection. Do not change `scripts/incident_routing.py` or the + frozen `workloads/incident-routing-v1/` files. +- Use admitted Cargo output and retain failed checks; honor the autonomous map. + +## Review Focus + +1. Missing mappings, duplicated IDs or aliases must fail before reservation. +2. A fixture's current flag must not admit an unadmitted Store occurrence. +3. Correction before claim must refuse; correction after claim must refuse + stale publication and preserve incurred usage/holds. +4. Protocol-valid wrong answers remain protocol-valid; labels never enter retry. +5. Restart, expiry or duplicate delivery must not create a second invocation. + +### Task 1: Pure protocol and exact-byte manifest primitives + +**Files:** create `src/incident_protocol.rs`, `src/evidence_digest.rs`, +`tests/incident_protocol.rs`; modify `src/lib.rs`, `Cargo.toml`, `Cargo.lock`. +No Store/schema/state-machine changes in this task. + +**Interfaces:** + +```rust +pub fn sha256_hex(bytes: &[u8]) -> String; // crate-private evidence_digest +pub fn render_incident_request(context: &IncidentContextV1, + reports: &[IncidentReport], phase: IncidentPhase) + -> Result, IncidentProtocolError>; +pub fn validate_incident_output(raw: &[u8], context: &IncidentContextV1) + -> Result; +``` + +`IncidentContextV1` has schema_version=1, version=`incident-store-context/1`, +workload_version, freeze_sha256, prompt_sha256, validator_version, case_id, +question, owner_ids, evidence_ids, system_prompt. Strings/arrays are nonempty; +owner/evidence IDs are unique. `IncidentReport` has external_id and text; +its order and IDs must exactly match evidence_ids. `IncidentOutput` has decision, +owner (optional), reason and evidence_ids, matching frozen v1 rules. Error enum: +TooLarge, InvalidEncoding, InvalidJson, UnknownVersion, InvalidContext, +InvalidOutput. Error text never embeds rejected bytes. +`IncidentPhase` is Initial or ProtocolRetry. Initial renderer `incident-request/1` +matches Python bytes. ProtocolRetry renderer `incident-request-retry/1` adds +`validation_error: "incident-routing-protocol/1:invalid_output"` to the inner user +payload and nothing else. It never accepts raw prior response or validator text. + +- [ ] Write tests for byte-exact rendering, Unicode/escaping, duplicate keys, + unknown fields, empty/duplicate IDs, depth 64/65, 1 MiB limit, and well-formed + semantically wrong output. Assert wrong answers can pass protocol validation. +- [ ] Run `cargo test --locked --test incident_protocol`; retain missing-API + failure. Inspect SHA-256 dependency checksum/license and compatibility; update + only its required lock resolution and record the dependency diff. +- [ ] Implement the pure types/codec with strict typed deserialization and the + existing frozen Python JSON rules; SHA-256 delegates to RustCrypto. +- [ ] Run the target suite and locked all-target regression. Commit + `feat: add pure versioned incident protocol and exact-byte digests`. + +**Done:** a pure library can render and validate v1 without a Store, label or +backend. No application can obtain send permission from these functions. + +### Task 2: Complete timed capture and deterministic runtime integration + +**Files:** create `examples/incident_store_bridge.rs`, +`examples/incident_store_bridge/adapter.rs`, +`examples/incident_store_bridge/tests.rs`, `docs/incident-store-bridge.md`. +Read existing recovery APIs; no new core recovery APIs or migration. + +**Interfaces (example-local):** + +```rust +fn prepare_incident_attempt(store: &mut Store, recovery: i64, + candidates: &[i64], contract: &IncidentContract) + -> Result; +``` + +`IncidentContract` fields are private. Its constructor loads a case from embedded +frozen cases/prompt/contract bytes whose digests match the pinned manifest, then +accepts context occurrence, ordered source-to-occurrence mappings, runtime and +cap. It never reads labels. Reject a missing/unknown case, incomplete mapping, +duplicate/aliased occurrence or mismatched context. Runtime prompt_fingerprint +is derived from the exact manifest; other identities are supplied explicitly. + +`IncidentPreparation` contains attempt:i64, request:Vec, captured_ids:Vec, +manifest_sha256:String and preparation:RecoveryPreparation. `BridgeError`: +UnavailableEvidence, ChangedContract, ExistingRecovery, TerminalRecovery, +InvalidManifest, Protocol(IncidentProtocolError), Store(rusqlite::Error). +Missing/unadmitted evidence share UnavailableEvidence with no payload details. + +Candidate order is context then reports; require equality with the full pinned +mapping before calling `begin_recovery_attempt`. Read `recovery_timing` and +verify exact manifest/runtime/cap identity. For New, validate every captured ID +and context byte, recompute every captured report digest and the complete manifest, +and compare it with the immutable timing binding. Render only captured text and +return preparation; reservation +and invocation remain distinct existing API calls. Existing/Finished refuse. +On an error after New, cancel that newly allocated proven-unsent attempt; never +cancel an Existing attempt or modify a claimed hold. Preserve cancellation errors. + +- [ ] Add failing `fixture_flags_cannot_admit_store_evidence`, + `missing_required_input_prevents_claim`, `duplicate_and_cross_case_mapping_refuse`, + `correction_after_prepare_refuses_send`, and + `labels_never_enter_request_or_feedback`. Verify no reservation/dispatch for + rejected mappings; corrupt captured context and a report payload independently, + not merely caller flags. Full manifest mismatch must prevent reservation. +- [ ] Run `cargo test --locked --example incident_store_bridge`; preserve the + initial missing-interface failure. Implement the preparation boundary above. +- [ ] Use `reserve_recovery_attempt` and `run_timed_recovery` with a counted canned + closure and deterministic clocks. Cap/token counts are UTF-8 bytes in this + fixture. Measured reply validation uses only the captured protocol context. +- [ ] Cover invalid output, reopen while reserved/sent, duplicate settlement, + post-claim expiry, backend error and settlement error. A post-claim refusal + invokes no adapter but retains the existing conservative hold. Repeat runs + cannot claim twice. Correction inside the backend yields stale publication + and full recorded usage; before claim it yields zero invocation. +- [ ] A retry after invalid output selects ProtocolRetry from the new recovery + attempt's retry_reason presence, discarding its free text. Both renderer IDs and + the fixed feedback string are already in the manifest. Preserve original + deadline and at-most-one existing recovery retry; mismatched/new source mappings + cannot silently replace the recovery's bound contract. Test byte-exact retry + requests separately from initial Python parity. +- [ ] Run `cargo test --locked --example incident_store_bridge --test + timed_recovery --test recovery_runtime` and `cargo run --locked --example + incident_store_bridge`. Focused review and commit + `feat: exercise incident preparation through Store timed gates`. + +**Done:** the example proves admission and accounting behavior with zero model +calls. A protocol-valid semantic error does not become a correctness claim. + +### Task 3: Cross-language parity and fixture-only CI + +**Files:** modify `.github/workflows/check.yml`, +`docs/incident-routing-workload.md`, `docs/incident-store-bridge.md`, +`examples/incident_store_bridge.rs`, `examples/incident_store_bridge/tests.rs`; +create `scripts/check_incident_store_bridge.py`, +`scripts/test_incident_store_bridge.py`. + +Expose example `--requests PATH`: write case ID/request/manifest digest records +for all 12 frozen cases, no model calls. Artifact metadata labels Store capture +and fixture mode. A Python checker loads the existing frozen workload and compares +every request byte and order with `incident_routing.prepare(..., split='all')`; +case/family/split IDs remain outside model request bytes. + +- [ ] First make parity tests fail on one changed byte, omission and reordered + report. Implement the checker and explicit output argument. +- [ ] Run the example into the artifact directory, run the checker on it and + `python3 -m unittest discover -s scripts -p 'test_incident_store_bridge.py'`. +- [ ] Add only these offline fixture steps to CI. Run all locked Rust targets + plus existing Python suites. Record exact source/runtime/lock and counts. +- [ ] Independent review and commit `test: verify Store incident byte parity in CI`. + +**Done:** B's exact interfaces and artifacts are available to C. No frozen +workload revision or external provider qualification is claimed. diff --git a/docs/superpowers/plans/2026-11-15-offline-diagnostics.md b/docs/superpowers/plans/2026-11-15-offline-diagnostics.md new file mode 100644 index 0000000..2d2cf8f --- /dev/null +++ b/docs/superpowers/plans/2026-11-15-offline-diagnostics.md @@ -0,0 +1,140 @@ +# Bounded offline decision diagnostics implementation plan + +> **For agentic workers:** Use `superpowers:subagent-driven-development` or +> `superpowers:executing-plans`. Execute Task1 after C; Task2 has an explicit +> consumer prerequisite and must not be auto-enabled by a synthetic fixture. + +**Goal:** Plan bounded ablations and score recorded outcomes without execution; +prepare portable evidence only when a real second consumer requires it. +**Architecture:** Pure Python planning/scoring over authorized bundles, with an +optional future Rust envelope codec that grants no admission. +**Tech Stack:** Python standard library; optional Rust serde and B's SHA-256. +**Spec:** [continuation specification](../specs/2026-11-15-reliability-continuation.md), +section E. No Rocket integration or cross-repository build dependency. + +## Global Constraints + +- At most four evidence items; checked call bound before a plan is emitted. +- CLI plans or scores recorded outcomes only; no network, tool or model execution. +- Missing/failed trials stay in the denominator; unstable baselines are inconclusive. +- Influence is a diagnostic association, not causal attribution or independence. +- Never prune opaque dependencies or change evidence admission from a score. +- Preserve exact bytes, ordered inputs, IDs and all cost/hold information. + +## Review Focus + +1. Pair expansion must fit max_calls; 4 items × 3 trials means 33, not 15. +2. Missing/duplicated/mismatched trials must not become a complete report. +3. An unstable baseline or failed trial must not produce a ranked causal claim. +4. Tied associations remain tied and deterministic. +5. Imported receipts never grant current admission or timed send permission. + +### Task 1: Bounded schedule and recorded-outcome scorer + +**Files:** create `scripts/decision_ablation.py`, +`scripts/test_decision_ablation.py`, `docs/decision-ablation.md`; +modify `.github/workflows/check.yml` for tests only. + +**Interfaces:** + +```python +def plan_ablations(bundle, trials=3, include_pairs=False, max_calls=33, + *, selected_ids=None): ... +def score_ablations(plan, outcomes): ... +``` + +Accept only a complete bounded C bundle with all selected evidence available. +`selected_ids=None` selects all distinct selectable Report/Evidence occurrences +only when there are at most four. Otherwise require an explicit selection. +Context/Policy/InheritedOutput and structural lineage do not count toward that +threshold and are never selected. Sort selected IDs by +original input order and reject unknown/duplicate selections; never silently +select a subset when the bundle contains more than four, +rather than silently truncating dependencies. Repeated occurrences at several +positions are one ablated item whose scheduled removal covers every occurrence +position; identical bytes under distinct IDs remain distinct items. + +Plan schema `decision-ablation-plan/1`: source bundle digest, ordered selected IDs, +trials (positive integer <=100), include_pairs boolean, max_calls integer <=10,000, +scheduled_calls, and rows {trial_id, repeat_index, removed_ids, variant_digest}. +The plan also binds scorer=`recorded-result-equality/1`, root runtime identity and +recorded validator identity. Only declared report/evidence inputs are selectable; +context, policy, inherited output and structural lineage are never removed. A +bundle with unknown input roles or missing comparison identity refuses planning. +`variant_digest` hashes versioned compact sorted-key JSON of source bundle digest, +ordered removed IDs and those three comparison identities, excluding repeat index; +trial_id uniquely combines the variant digest and repeat index. +Schedule baseline, ordered singles, lexicographic pairs, each for all r repetitions. +Omission describes a hypothetical diagnostic request; it does not mutate or +re-authorize the original bundle. The plan emits no executable backend command. + +Outcome schema `decision-ablation-outcomes/1`: plan digest and one row per trial +ID with state (recorded, failed, unknown), response digest, validated semantic +result or null, charged units or unknown, reserved bound and unit. Rows also bind +the plan's scorer/runtime/validator identities. The fixed scorer compares +producer-supplied normalized result values; it is not a scientific-truth validator. +Treat normalized results as recorded attestations. Exact duplicate +trial IDs and unknown IDs refuse; missing trials make report Incomplete. Different +scorer/runtime/validator identities make comparison Incomparable. + +Score only if every baseline trial is recorded and all baseline semantic results +agree. Each variant records changed/unchanged/failed/unknown counts out of all r +scheduled repeats; incomplete variant trials yield no influence rank. A complete +variant's association score is changed/r. Preserve equal scores as ties, sorting +display by original selected order. Pair scores are associations, not additive +causal effects. Report total scheduled/recorded calls, charged and held units; +unknown measurements remain unknown plus bounds, never zero. + +- [ ] Write failing tests for n=4,r=3,pairs=True =>33, max_calls=32 refusal, + invalid bounds/types/overflow, more-than-four without selection, identical bytes + with distinct IDs, repeated positions and redacted bundle refusal. +- [ ] Implement deterministic planning. Add no backend imports or run mode. +- [ ] Write failing unstable-baseline, missing/unknown/failed trial, extra trial, + changed plan digest, identity mismatch, tied score and unknown-cost tests. +- [ ] Implement scoring and `plan`/`score` CLI. Monkeypatch process/network hooks + to raise in tests; both commands must remain pure file input/output. +- [ ] Run `python3 -m unittest discover -s scripts -p 'test_decision_ablation.py'` + plus existing script suites; demonstrate plan and score using explicit recorded + fixtures. Independent review and commit `feat: plan and score bounded offline ablations`. + +**Done:** offline diagnostics are usable independently of an external consumer. + +### Task 2: Conditional portable envelope + +**Entry gate:** name an actual second consumer, its owner identity mapping and +scope/purpose needs in a new intake receipt. The current roadmap names none and +explicitly excludes Rocket. Until this condition is met, record Deferred and +continue other work; do not invent a mock consumer as justification. + +**Allowed files after the gate:** create `src/evidence_envelope.rs`, +`tests/evidence_envelope.rs`, `examples/export_evidence.rs`, +`docs/evidence-envelope.md`; modify `src/lib.rs`. No other repository edits. + +```rust +pub fn parse_envelope(bytes: &[u8]) -> Result; +pub fn verify_payload(reference: &EvidenceReferenceV1, payload: &[u8]) + -> Result<(), EnvelopeError>; +``` + +Reference: producer, kind, id, revision, lowercase SHA-256, media_type. +Envelope: version1, full reference, exact context manifest version/bytes/digest, +typed lineage of full references; payload bytes may be separately carried. +Receipt: full reference, authority, actor, scope, purpose, decision +(eligible/ineligible/unknown), authority_revision, view_epoch, context digest; +optional expiry must specify clock domain and trusted sample semantics. Without +known clock continuity an imported receipt stays unknown. Parse <=16 MiB, +<=1,024 lineage entries, depth<=64; reject unknown/duplicate fields and cycles. +EnvelopeError: InvalidSchema, InvalidReference, InvalidHash, HashMismatch, +LimitExceeded. Parsing/hash verification have no Store mutation capability. + +- [ ] Write failing hash/reference mismatch, identical payload/distinct ID, + wrong scope/purpose, stale receipt, unsupported version and binary-byte tests. +- [ ] Implement strict parsing/export, preserve original manifest bytes and + require receiver authentication/current owner eligibility before any future + admission. No import-admit API is added by this task. +- [ ] Run `cargo test --locked --test evidence_envelope`, the export example and + integration suites. Exchange exact fixture bytes with the named consumer's + owner; record what was actually verified. Review and focused commit. + +**Done:** a deliberately optional contract serves a real documented consumer; +its receipts remain assertions, not cross-store authority. diff --git a/docs/superpowers/plans/2026-11-15-policy-activation.md b/docs/superpowers/plans/2026-11-15-policy-activation.md new file mode 100644 index 0000000..de00d13 --- /dev/null +++ b/docs/superpowers/plans/2026-11-15-policy-activation.md @@ -0,0 +1,242 @@ +# Evaluated policy activation implementation plan + +> **For agentic workers:** Use `superpowers:subagent-driven-development` or +> `superpowers:executing-plans`; execute sequentially, preserving failing evidence. + +**Goal:** Register exact known policies, evaluate comparable complete records and +activate or roll back a version without bypassing evidence or accounting gates. +**Architecture:** Immutable registry/evaluation rows and scoped revision CAS in +the same SQLite writer transaction as policy admission changes and support +publications. Python produces offline attestations; Rust validates and decides. +**Tech Stack:** existing Rust/rusqlite/serde/SHA-256 and Python fixtures. +**Spec:** [continuation specification](../specs/2026-11-15-reliability-continuation.md), +section D. Prerequisites: A-C reviewed. Only this stage introduces a migration. + +## Global Constraints + +- Known policies only: always_stop, affordable_retry, evidence_gated_retry. +- Scope/purpose and authenticated management authority come from trusted runtime. +- No inferred scientific result from failure, no unknown-cost efficiency claim. +- Compare a contemporaneous paired baseline under one frozen comparison manifest. +- Activation atomically changes policy admissions; never changes old send claims, + outstanding holds, historical dispositions or original deadlines. +- Rollback advances history and needs compatibility, not superiority. +- No Rocket integration, live model call, new service or live-database migration. +- Next schema version is determined from integrated source; do not assume v10. + +## Review Focus + +1. A B-beats-A evaluation must not activate B when C is the active baseline. +2. Hash, settings, split, case coverage, unit or scorer mismatches refuse. +3. An invalid policy or inconsistent policy admission cannot activate. +4. Concurrent bootstrap/promotion/rollback has one committed winner and no partial + admission, support, pointer or event state. +5. Changing policy invalidates dependent opaque work but preserves claims/holds. + +### Task 1: Strict candidates and complete comparison records + +**Files:** create `src/policy_registry.rs`, `src/policy_registry/types.rs`, +`src/policy_registry/evaluation.rs`, `tests/policy_evaluation.rs`, +`scripts/policy_evaluation_v2.py`, `scripts/test_policy_evaluation_v2.py`, +`docs/policy-evaluation-v2.md`; modify `src/lib.rs`, +`scripts/recovery_policy_eval.py` only to expose an explicit v2 conversion mode, +and `scripts/test_recovery_policy_eval.py` for preservation of v1 behavior. + +**Interfaces:** + +```rust +pub fn validate_policy_artifact(bytes: &[u8]) -> rusqlite::Result; +pub fn evaluate_policy_comparison(value: &PolicyEvaluation) + -> rusqlite::Result; +``` + +Artifact exact fields: schema_version=1, policy_name (known enum), purpose, +compatibility_contract_sha256. Unknown fields, empty purpose, duplicate keys, +unknown policies, invalid lowercase digests or artifacts over 64 KiB refuse. +No user-supplied executable configuration. + +`PolicyCandidate`: version_id:String, actor:String, scope:String, purpose:String, +artifact:Vec, sha256:String. Store creates its evidence occurrence. +`PolicyEvaluation`: evaluation_id, actor/scope/purpose, baseline_id/candidate_id, +baseline_sha256/candidate_sha256, experiment_manifest:Vec, +experiment_sha256, evaluator_version=`policy-selection/1`, evaluator_sha256, +provenance (synthetic_registry_mechanics or observed_policy_replay), baseline_rows, +candidate_rows. Provenance is included in all digests and persisted with the +evaluation. Synthetic evaluations are eligible only for purpose +`fixture/recovery-policy`; nonfixture activation refuses them. +Manifest format is `policy-comparison/1` with all +identities specified in section D. Evidence/labels/request opportunities are +explicit digested records, not URLs loaded during activation. + +Rows contain case_id, outcome enum {Accepted, Refused, Failed, Unknown}, +correct:Option, eligible:Option, answerable:bool, +completed_ms:Option, charged:Option, reserved_bound:u64, +false_refusal:bool, unsafe_acceptance:bool, deadline_miss:bool. Validate the +redundant flags against outcome, frozen labels, eligibility and timing. Require +all ordered scheduled cases exactly once per side. Cap 10,000 cases/16 MiB. +Manifest accounting unit is model_tokens or fixture_bytes, identical on both +sides; logical_support and unknown refuse policy comparison. +Observed rows additionally contain ordered attempt references with opportunity ID, +request/evidence-manifest/response digests, execution/disposition, measured input +and output or unknown, retained bound, start/completion offsets, validator/scorer +identities and label digest. Check against frozen opportunities and recompute row +accounting/outcome from those observations. Missing or incompatible references +make observed replay incomplete/incomparable. This is consistency checking of +attestations; it does not prove the host executed them. Synthetic mechanics rows +declare simulated counters and are never reported as feasible named-policy replay. + +`SelectionDecision` is Eligible { validated_counters } or Refused(SelectionRefusal), +with typed reasons Incomplete, Incomparable, NoImprovement, IncreasedFalseRefusal, +IncreasedUnsafeAcceptance, IncreasedCost, IncreasedDeadlineMisses. Structural +malformation is an error. Unknown data remains stored unevaluated only where +validly represented; it never yields Eligible. + +- [ ] Add failing tests for every identity/case/unit mismatch, malformed known + artifact, incomplete/unknown result, numeric overflow, fabricated counters and + deadline boundary (completion equal to deadline is on time). +- [ ] Implement strict parse and recomputation of counters from rows. Require + success_candidate > success_baseline and all four nonincrease conditions. +- [ ] Implement explicit Python v2 attestations with the same strict validation. + Legacy v1 rows lacking answerability, timing or unit identity return an explicit + incomplete conversion unless the caller supplies a matching frozen comparison + artifact. Do not infer these fields from a successful protocol parse. +- [ ] Preserve v1 evaluation output. Add cross-language golden positive/negative + v2 fixtures; label the positive case `synthetic_registry_mechanics`. + Test that synthetic evaluation cannot activate a nonfixture purpose. + Existing always_stop paired replay may correctly return NoImprovement; never + select a worse baseline or change the selection rule to make it pass. +- [ ] Run `cargo test --locked --test policy_evaluation`, both Python policy + suites and locked all-targets; review and commit + `feat: validate complete comparable policy evaluation records`. + +**Done:** selection is a deterministic check of exact supplied attestations, +not an assertion that current empirical policies improve performance. + +### Task 2: Registry migration and immutable registration + +**Files:** create `src/policy_registry.sql`, `tests/policy_registry.rs`; +modify `src/policy_registry.rs`, `src/policy_registry/types.rs`, `src/lib.rs`. + +Tables: policy_versions (version_id PK, actor, scope, purpose, evidence_id UNIQUE +FK evidence, contract SHA, artifact SHA); policy_evaluations (evaluation_id PK, +actor/scope/purpose, baseline_id/candidate_id FKs, exact manifest and row bytes, +digests/evaluator identity); policy_heads (actor/scope/purpose composite PK, +version_id FK, revision positive i64, baseline_manifest_sha256); +policy_events (same scoped key plus revision UNIQUE, action, previous/target +version, evaluation ID nullable, manifest SHA, nonempty reason). +Version/evaluation/event rows reject update/delete with triggers. Head revisions +are monotonic and changed only by the activation API. No invented backfill. + +```rust +Store::register_policy_candidate(&mut self, candidate: &PolicyCandidate) + -> rusqlite::Result<()>; +Store::record_policy_evaluation(&mut self, evaluation: &PolicyEvaluation) + -> rusqlite::Result<()>; +``` + +Registration checks exact artifact/digest and matching scope/purpose, creates +one fresh unadmitted evidence occurrence and version row atomically. Identical +repeat of an existing version is a no-op; conflicting bytes or metadata error. +Evaluation insertion binds existing exact candidate versions and immutable +manifest/rows. Identical duplicate is a no-op; conflicting duplicate refuses. +Baseline-only evaluation uses baseline_id == candidate_id and identical rows; +it establishes comparison evidence, not superiority. Incomplete structurally +valid evaluations may be stored but are never promotion eligible. + +- [ ] Write failing migration, duplicate-registration, wrong-scope, artifact/hash + and immutable-row tests. Determine the next free schema version. +- [ ] Add migration within Store's existing Immediate initialization transaction. + Test a populated v9 copy containing accepted/stale work and an unresolved hold: + IDs, output, usage, view epochs and old receipts survive unchanged after reopen. +- [ ] Inject failure after some DDL but before commit using a test-only migration + hook; reopen and verify the old schema/rows remain intact. Reject unknown future + versions. Keep original backup bytes; test recovery from that fixture backup. +- [ ] Implement registration. Run `cargo test --locked --test policy_registry + --test recovery --test timed_recovery`; review and commit + `feat: persist immutable scoped policy versions and evaluations`. + +**Done:** registry data is durable; nothing is active and no sends are authorized. + +### Task 3: Atomic bootstrap, promotion and rollback + +**Files:** modify `src/policy_registry.rs`, `src/policy_registry/types.rs`, +`src/support.rs`, `src/lib.rs`, `tests/policy_registry.rs`; +create `docs/policy-promotion.md`. + +```rust +Store::activate_policy(&mut self, request: &ActivationRequest) + -> rusqlite::Result; +Store::active_policy(&self, actor: &str, scope: &str, purpose: &str) + -> rusqlite::Result>; +``` + +Request contains actor/scope/purpose, expected_active_revision:Option, action: +Bootstrap { baseline_id, baseline_manifest:Vec, baseline_manifest_sha256, +reason }, Promote { candidate_id, evaluation_id }, or Rollback { target_version_id, +reason }. Baseline bytes are explicit here so bootstrap verifies their hash and +structure, not only an uncheckable supplied digest. The manifest does not assert +measured superiority. ActivePolicy returns version ID/digest, evidence ID, +compatibility contract, revision and baseline manifest digest, never send permission. + +Result Activated { revision } or Refused(ActivationRefusal): RevisionConflict, +AlreadyInitialized, MissingHead, InvalidArtifact, ScopeMismatch, Incompatible, +BaselineMismatch, MissingBaselineEvaluation, EvaluationRefused(SelectionRefusal), +NeverActivated, AdmissionMismatch, NoChange. Invalid request syntax is an error. + +Use one Immediate transaction. Check expected revision (None only bootstrap), +scope/purpose, candidate compatibility and current policy admission. Promotion +requires evaluation's baseline ID+digest match current head, its exact manifest +match the bootstrapped comparison manifest, and a baseline-only evaluation with +identical baseline rows/manifest/evaluator. Recompute selection from stored rows. +Rollback verifies a prior activation event and current artifact compatibility; +it does not require an evaluation. Refuse selecting an already active target. + +Extract/reuse A's final-state transaction helper rather than calling a public +method that opens a nested transaction. Withdraw old/admit new, revise final +changed support heads, update pointer and append event in the same transaction. +New revision = old+1 (bootstrap1), checked for overflow. No edits to claims, +reservations or deliveries. Head/evaluation lookup is always scope-bound. + +- [ ] Add failing empty-registry bootstrap, concurrent bootstrap, B-beats-A with + C active, stale revision, missing baseline evaluation, incompatible rollback, + rollback without superiority and inconsistent-admission tests. +- [ ] Implement distinct gates. Test simultaneous promote/promote and + promote/rollback on two connections with barriers: one winner, one conflict. +- [ ] Inject an admission/support/event failure after writes; compare full state + after rollback/reopen. Test a captured opaque/timed attempt refuses after the + policy epoch change, while a previously claimed uncertain hold is unchanged. +- [ ] Mutation checks: omit active-baseline comparison, skip revision CAS, + omit admission replacement, release an old hold. Each targeted test must fail. +- [ ] Run registry, recovery, timed recovery, atomic batch and eligibility tests; + focused independent review and commit `feat: atomically activate evaluated policies`. + +**Done:** activation has one atomic authority boundary and no independent send power. + +### Task 4: Fixture integration and final regression + +**Files:** modify `examples/recovery_policy_eval.rs`, +`scripts/recovery_policy_eval.py`, `scripts/test_recovery_policy_eval.py`, +`docs/policy-promotion.md`, `.github/workflows/check.yml`; +create `examples/policy_activation.rs`. + +Keep existing example modes and archived v1 output unchanged. Add a fixture-only +registry path that bootstraps always_stop, records complete synthetic baseline +and paired evaluation, activates a qualifying named candidate and rolls back. +Demonstrate that an actual existing v1 comparison can refuse promotion. The new +`policy_activation` example uses a separately versioned typed-role input manifest: +one Policy occurrence followed by Report occurrences. Capture every occurrence, +validate the captured policy artifact against the selected active digest, and +render only Report payloads. Do not append policy bytes to the existing v1 +collector, which parses every input as Report. Recheck active revision immediately +before preparation and rely on normal claim/settlement gates. Preserve the +original collector path and archived format byte-for-byte. + +- [ ] Add failing integration tests for policy switch before claim, rollback + revision advance, historical receipts and retained uncertain holds. +- [ ] Implement fixture CLI and explicit offline CI step. Outputs identify + synthetic attestations and zero model calls; never label a policy empirically + superior based on the registry example. +- [ ] Run `cargo test --locked --all-targets`, policy fixture example, all Python + suites and explicit compiled-runner checks from existing CI. Retain artifacts. +- [ ] Independent review of the entire D range and commit + `test: demonstrate policy activation and rollback without model calls`. diff --git a/docs/superpowers/plans/2026-11-15-runtime-qualification.md b/docs/superpowers/plans/2026-11-15-runtime-qualification.md new file mode 100644 index 0000000..05e679f --- /dev/null +++ b/docs/superpowers/plans/2026-11-15-runtime-qualification.md @@ -0,0 +1,94 @@ +# Project runtime qualification implementation plan + +> **For agentic workers:** Use `superpowers:subagent-driven-development` or +> `superpowers:executing-plans`. This is a separate bounded successor to A0; +> do not rewrite its historical verification receipt or host runtime defaults. + +**Goal:** Qualify exact current stable, upcoming and previous project runtimes, +then pin the accepted project launch/CI identities and retain a working rollback. +**Architecture:** Isolated runtime lanes run the same locked fixture suite against +one integrated source; only independently passing stable is the normal target. +**Tech Stack:** Rust/Cargo, Python, existing GitHub Actions and storage tooling. +**Spec:** `/home/neo/Projects/LOCAL_CI_RUNTIME_POLICY.md` and the A0 +`docs/runtime-adoption.md` findings. Run after A-E1 to avoid repeated full matrices. + +## Global Constraints + +- No host-wide default replacement, service deployment, dependency bulk upgrade, + old runtime removal, data deletion or model inference. +- Preserve original CI gates and the exact accepted source/runtime/lock artifacts. +- Use current primary upstream records; floating channel names are lookup inputs, + never final identities. Unknown/unavailable lanes remain explicit. +- Admission budgets cover downloads, unpacking and compilation. A refusal blocks + the heavy lane, not lightweight review. Never extend the temporary exception. + +## Review Focus + +1. A recorded pin must resolve to the executable actually running the tests. +2. A passing stable lane cannot conceal a failing or unexecuted preview lane. +3. Older consumers and the exact rollback set remain distinct from previous stable. +4. The keyed Cargo output must use the selected runtime and unchanged lock. +5. Switching back to the retained runtime/source must actually reproduce a basic + current read and accounting fixture; stored backups alone are not a rehearsal. + +### Task 1: Qualify isolated lanes and document actual outcomes + +**Files:** modify `docs/runtime-adoption.md`; create +`scripts/verify_runtime_matrix.py`, `scripts/test_verify_runtime_matrix.py`. +Toolchain downloads/installations are allowed only into user-owned isolated +runtime directories/toolchain slots with a declared output lifecycle. No default +rustup toolchain change and no write to `/etc` or system packages. + +Initial candidate observations are Rust1.100.0, upcoming nightly2026-11-15 +(and separately recorded beta1.101.0-beta.2), previous1.99.1; Python3.15.2, +preview3.16.0a2 and previous3.14.10. Refresh official release/manifest records at +intake and retain bytes/digests before choosing. The normal accepted baseline +remains A0's exact1.99.1/3.15.2 until the stable candidate passes. + +The verifier accepts an explicit JSON manifest containing source commit, lock +digest and absolute Rust/Cargo/Python executable paths plus expected identities +and binary/archive hashes per lane. It never installs tools or searches mutable +PATH for a replacement. Before each lane it verifies exact identities, runs +locked all-target Rust tests, existing Python suites and compiled-runner checks, +then emits command/exit/count/skip and artifact receipts. Unknown prerequisites +produce Unexecuted, never Pass. Tests use tiny local fake executables for manifest +rejection/command identity only; real qualification uses the real full suites. + +- [ ] Add failing tests for missing lane, wrong executable identity/digest, + changed lock, nonzero subprocess exit, missing receipt and false pass on skip. +- [ ] Implement bounded verification and artifact receipts. Install missing + candidates only after storage admission; verify archive hashes before use. +- [ ] Run each available lane serially on one source. Preserve failures; do not + alter tests or dependency pins to make a lane pass. Scope any genuine migration + fix separately with a reproduced regression and review. +- [ ] Rehearse the retained baseline in an isolated checkout with original lock + and fixture database, checking current read and unchanged uncertain hold. +- [ ] Document exact executed/skipped lanes, upstream source responses (including + Python and previous-line patch freshness), NixOS host build display, supported + consumer inventory limits and rollback receipt; focused review and commit. + +**Done:** actual runtime qualification evidence is available. Unavailable external +toolchains remain an explicit adoption gap; they do not become an invented pass. + +### Task 2: Pin qualified project defaults and CI lanes + +**Files:** create `rust-toolchain.toml`, `.python-version`; +modify `.github/workflows/check.yml`, `docs/runtime-adoption.md`. +No application source or dependency resolution change. + +- [ ] If stable qualification passed, pin its exact Rust release in + rust-toolchain.toml with the minimal required components, and exact Python + stable patch in .python-version. Otherwise preserve the accepted baseline and + report the failed candidate without promoting it. +- [ ] Pin setup actions to verified upstream commits and exact runtime versions; + preserve checkout's accepted pin and every existing CI command. Define named + stable/upcoming/previous roles with explicit identities; include a retained + baseline lane if it is distinct. Do not claim hosted execution from YAML. +- [ ] Verify project-selected tools match qualified identities, then run the + exact stable fixture workload once after pinning. Validate CI configuration + and preserve matrix failures/skips honestly; no shared-branch push in this task. +- [ ] Independent review and commit. Retain old binaries, lock and recovery + evidence; no runtime retirement is authorized. + +**Done:** project defaults reflect the accepted current stable candidate and CI +describes exact lanes. Local evidence and eventual hosted execution remain separate. diff --git a/docs/superpowers/specs/2026-11-15-reliability-continuation.md b/docs/superpowers/specs/2026-11-15-reliability-continuation.md new file mode 100644 index 0000000..15e8105 --- /dev/null +++ b/docs/superpowers/specs/2026-11-15-reliability-continuation.md @@ -0,0 +1,208 @@ +# KairoTensor continuation: executable design decisions + +Date: 15 November 2026. Source inspected: main +`20fab84a6baa525452191ecf80eeee339f0b5ac7`; preceding P0 plan commit `8d4cce8`. +All new names below are proposed interfaces until their implementation commits. +The owner authorized autonomous execution and excluded Rocket integration. + +## Evidence and known limits + +The Store already owns admission history, ordered candidate/input occurrences, +branch publication, timed reservations/claims and settlement. Keep that authority. +The existing incident renderer and scorer deliberately trust declared fixture +eligibility. The new bridge must obtain bytes from complete Store capture. + +Independent source review established these constraints: + +- Accepted/stale legacy deliveries have NULL validator identity by schema. A + later caller cannot invent the validator used for them. +- Delivery receipt time and the delivery-time head/view are not recorded for all + attempts. A claim timestamp is not a completion timestamp. +- A budget can include attempts from several actors/scopes. Scoped diagnostics + cannot automatically disclose its global balances. +- Logical support charges, fixture byte counts and measured model tokens are + different units. `charged_usage()` alone cannot qualify a policy comparison. +- Immutable historical output may still exist after withdrawal. Availability + on disk and historical Accepted do not grant current export permission. + +## B: incident preparation + +Use one context occurrence plus the declared report occurrences in their exact +order. A context payload of version `incident-store-context/1` contains workload +and prompt identities, validator `incident-routing-protocol/1`, case ID, question, +ordered owner IDs, ordered external report IDs, system prompt, and the frozen +workload digest. No labels, family/split annotations or eligibility flags. + +Source mappings are keyed by `(workload_version, case_id, external_id)`; IDs like +`e01` are not globally unique. A trusted pinned workload loader determines the +complete required ID set. A caller cannot shrink that set by supplying a smaller +contract. Require a distinct Store occurrence per mapping; identical payloads +may still be separate occurrences. Context is input position zero. + +The manifest is exact JSON bytes of version `incident-store-manifest/1`, containing +the context occurrence ID and SHA-256, ordered full source mappings and payload +digests, workload/prompt/validator identifiers, output cap, and runtime identity +excluding its derived `prompt_fingerprint`. The latter is set to the manifest +SHA-256 in the immutable `RecoveryTiming` binding. The manifest also names initial +renderer `incident-request/1`, retry renderer `incident-request-retry/1` and fixed +feedback `incident-routing-protocol/1:invalid_output`. Retry ordinal 1 adds only +`validation_error` with that fixed string to the user payload; ordinal 0 retains +exact Python v1 bytes. Both alternatives are bound before the recovery starts, +so retry does not change the immutable timing contract. This exclusion avoids a +self-referential hash. Model/tokenizer/runtime identities and decoding remain +explicit even for the deterministic byte-token fixture. + +Prepare compares that frozen timing binding with its reconstructed manifest, +then reconstructs the complete manifest from every `RecoveryStart::New` captured +payload and compares every digest and the final manifest hash again before +rendering. No request byte +comes from a caller's payload cache. Errors after new capture cancel only that +new provably-unsent attempt through the existing cancellation API. Existing, +sent, denied, terminal or otherwise ambiguous operations never produce another +send opportunity. No recovery state machine or implicit retry is added. + +Canonical fixture JSON matches Python `ensure_ascii=False`, sorted object keys, +compact separators and UTF-8 exactly. Arrays preserve order. Unknown/duplicate +keys, floats/nonfinite values where integers are required, invalid UTF-8 and +depth above 64 refuse. Bound context/request/reply at 1 MiB each and manifests +at 1,024 input occurrences, including context. These are new resource limits. + +Put the pure renderer/protocol validator in `src/incident_protocol.rs` so replay +can reuse it without invoking a Store or tool. It has no admission or send API. +Protocol validation checks format and supplied IDs, not scientific correctness. +Retry feedback uses fixed versioned protocol reason codes and never gold labels. + +The reviewed hash dependency candidate is `sha2 = "=0.11.1"`, observed in the +[RustCrypto crate documentation](https://docs.rs/sha2/0.11.1/sha2/) on this date. +Check registry checksum, license, compiler compatibility and freshness at task +intake; commit the exact resolved lock. No hand-written SHA-256 or unrelated +dependency update. A discovered incompatibility is a documented successor +decision, not permission to change existing accepted pins silently. + +## C: authorized decision bundles and replay + +`DecisionBundleV1` is an export, never a transaction/import authority. Use exact +byte strings encoded as lowercase hexadecimal plus byte length and SHA-256; +never decode and re-encode arbitrary request/output/evidence bytes as UTF-8. +Limit decoded payloads to 4 MiB each, bundle JSON to 16 MiB, dependency closure +to 128 attempts and JSON depth to 64. Preserve repeated positions and occurrence +IDs even when content hashes match. Reject overflow before allocation. + +A read transaction collects the root and required data dependencies: inherited +output, model-reuse origin and support rule inputs/formula. Parent lineage that +was not consumed is a reference, not a request to export all ancestor bytes. +Mutable current witness counters are not historical proof. Cycles, missing +dependencies and limits produce redaction/incomplete results, never substitutes. + +CurrentRead requires the requested attempt to be the current eligible head and +every exported evidence input to be currently admitted in its own actor/scope. +Dependency outputs require the same current-read check. DiagnosticReplay permits +historical outputs only after a trusted synchronous authorizer accepts the exact +root/dependency selection and export purpose within that retained snapshot. +It does not override withdrawn inputs or restore redacted bytes. The callback +must not reenter/mutate this Store or perform network access; no reusable grant +is returned. A later revocation does not claw back an already exported file. + +Denied/missing/withdrawn exports share `Redacted { requested_attempt, reason: +Unavailable }`, with no source strings, hashes, counts, payloads, requests, +validator messages or undisclosed lineage. SQL corruption/I/O is a separate +error; public errors carry no private values. Default diagnostic authorizer denies. +The caller's actor/scope identity remains the embedding runtime's responsibility. + +Export per-attempt charges, reservation quote and retained hold. Omit global +budget balances and cross-scope contributions. Preserve budget ID as an opaque +local reference only where the root is authorized; do not claim that a partial +accounting view equals the whole budget. Units are `logical_support`, +`fixture_bytes`, `model_tokens` or `unknown`; infer a unit only from a recognized +immutable computation/runtime binding, never from numeric size. + +Replay reports four separate facts: ledger consistency, request reconstruction, +validator evaluation, and the recorded historical disposition. It cannot +recompute general historical publication/timeliness without missing observations. +For `report-verdict/v1`, recompute the bound deterministic output. For the bridge, +use the captured declared protocol contract and timed manifest binding. Mark +that validator provenance `declared_contract`, not proof it ran historically. +Recognized invalid-output receipts retain `recorded_rejection` provenance. +Legacy unbound accepted responses return `ValidatorUnavailable` for validation. + +Use a compiled allowlist of pure versioned validators/renderers. Unknown versions +are `NotReproducible`, not dynamic code loading. A changed validator yields a +new linked evaluation; the original disposition remains immutable. Missing raw +delivery with a retained hold is `Incomplete`, with unknown usage and the hold +preserved. Pure replay never opens `Store::open`, resolves URLs, loads models, +executes commands or invokes adapters. + +## D: policy registry and evaluation + +Registry key: `(actor, scope, purpose)`. Candidates are immutable exact artifacts +for `always_stop`, `affordable_retry` or `evidence_gated_retry`; schema version 1, +no programmable expressions or parameters. Each version owns a distinct evidence +occurrence. Registration does not admit it. The trusted runtime authenticates +actor/scope and policy-management access before calling Store; artifacts and +evaluation rows cannot authenticate their issuer. + +An immutable comparison manifest identifies ordered case/family/split IDs, +workload/prompt, allowed exact per-policy request opportunities, initial evidence, +runtime/model/tokenizer, protocol validator, semantic scorer/labels, cap, budget, +original deadline, accounting unit and timing boundaries. Policy choices select +among those predeclared opportunities; changed requests cannot inherit outcomes. + +Evaluation provenance is serialized and digested: `synthetic_registry_mechanics` +or `observed_policy_replay`. Synthetic records can activate only under purpose +`fixture/recovery-policy`; all other purposes refuse that provenance. Observed +replay rows reference the exact frozen request opportunities and attempt records; +case totals alone cannot establish those opportunities were executed. The Store +still treats external observations as attestations, not authenticated model proof. +Evaluation rows cover every scheduled case for both baseline and candidate. +Each row records outcome class, correctness, publication eligibility, completion +time or explicit unknown, false refusal, unsafe acceptance, charged units and +reserved bounds. Count success only when correct, eligible, accepted and completed +at or before the original deadline. A false refusal is refusal on a frozen case +labelled answerable with admissible support. Unsafe acceptance is any accepted +incorrect or ineligible result. Missing/unknown rows or unmeasured usage prevent +promotion, even when a conservative hold bound is present. All sums are checked +nonnegative integers; rows, not caller totals, determine counters. + +Promotion requires strictly more successful cases and no increase in false +refusals, unsafe acceptances, charged-plus-reserved units or deadline misses. +It compares identical manifests, unit, case set and label/scorer versions. A +separately recorded baseline-only evaluation must match the paired baseline +rows exactly. Current v1 replay may yield no improvement: always_stop plus equal +initial outcomes and an added paid retry cannot meet the no-cost-increase rule. +Do not weaken the rule or alter old measurements. Positive activation tests use +explicit synthetic row attestations to test registry mechanics; they are not +empirical evidence that any current policy is superior. + +Bootstrap succeeds only without a head, binds the baseline comparison manifest +and admits its policy occurrence. Promote requires expected revision plus active +baseline identity/digest equality inside the writer transaction. Rollback needs +prior activation, exact compatible contract, current authority and a nonempty +reason, but no superiority evaluation. Revision monotonically increases. + +Every transition atomically withdraws the previous policy occurrence (if any), +admits the target, publishes final changed support verdicts through A's shared +transaction helper, updates the pointer and appends an event. Require the current +head occurrence admitted and an inactive target unadmitted; inconsistent external +membership changes refuse activation instead of silently repairing authority. +New operations capture the selected policy occurrence. Existing opaque/timed +work sees the changed scope epoch through ordinary gates; post-claim work may +still execute but cannot publish stale results. Never rewrite claims or holds. + +Only D needs a new database schema. Allocate the next free version at integration +(v10 only if still free). Migrate under the existing Immediate initialization +transaction, preserve v9 IDs and usage, and reject future versions. Back up and +test using fixture/copy databases; do not migrate an owner's live database here. + +## E: offline diagnostics and optional portability + +Plan ablations for at most four declared evidence items. With r trials and pairs, +the schedule has r * (1 + n + n*(n-1)/2) rows; n=4,r=3 means 33. Bounds apply before +execution; the CLI only plans or scores recorded outcomes. Require every scheduled +row, keep failed/unknown trials and call counts, and require all baseline trials +to agree before computing associations. Ties stay tied; no causal or independence +claim and no pruning of opaque dependency manifests. + +Portable envelopes remain conditional on an actual second consumer. Their parser +and digest checker cannot grant admission; a receiver must authenticate owner +mapping and recheck full reference, scope/purpose, context and current authority. +No cross-store atomicity, implicit TTL, Rocket connection or live ablation runner. From cfa8d3cc6c43948d2f41965530184131ad8302c3 Mon Sep 17 00:00:00 2001 From: Alejandro Castellanos <5405628+k2jac9@users.noreply.github.com> Date: Sun, 15 Nov 2026 14:25:12 -0500 Subject: [PATCH 05/51] fix: reject malformed accepted output in eligibility gate --- src/eligibility.rs | 13 ++++-- tests/eligibility.rs | 97 ++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 106 insertions(+), 4 deletions(-) diff --git a/src/eligibility.rs b/src/eligibility.rs index 1714180..440e800 100644 --- a/src/eligibility.rs +++ b/src/eligibility.rs @@ -3,7 +3,7 @@ use rusqlite::{Connection, Result}; use crate::attempts::{find_branch, Branch}; use crate::support::{inspect_current, SupportStatus}; -use crate::{view_epoch, Store}; +use crate::{invalid, view_epoch, Store}; /// Eligibility under the Store contract, not a claim that an output is true. #[derive(Clone, Copy, Debug, PartialEq, Eq)] @@ -61,13 +61,18 @@ pub(crate) fn evaluate( }; // Missing/nonaccepted delivery or a head belonging to another branch is a // malformed stored head, even if its epoch would otherwise be stale. - let captured_epoch: i64 = db.query_row( - "SELECT a.view_epoch FROM attempts a + let (captured_epoch, output_is_blob): (i64, bool) = db.query_row( + "SELECT a.view_epoch, typeof(d.output) = 'blob' FROM attempts a JOIN deliveries d ON d.attempt_id = a.id AND d.disposition = 'accepted' WHERE a.id = ?1 AND a.branch_id = ?2", rusqlite::params![head, branch.id], - |row| row.get(0), + |row| Ok((row.get(0)?, row.get(1)?)), )?; + // SQLite's BLOB affinity alone does not ensure Vec can decode the value. + // Inspect only its storage class, never the accepted output bytes. + if !output_is_blob { + return Err(invalid("stored accepted output is not a blob")); + } explanation.captured_epoch = Some(captured_epoch); let (state, proof, reason) = if captured_epoch == current_epoch { (EligibilityState::Ready, EligibilityProof::CoarseEpoch, None) diff --git a/tests/eligibility.rs b/tests/eligibility.rs index b376789..9902f38 100644 --- a/tests/eligibility.rs +++ b/tests/eligibility.rs @@ -223,6 +223,11 @@ fn snapshot(db: &Connection) -> Result>> { let mut snapshot = Vec::new(); for table in [ "branches", + "evidence", + "support_bindings", + "support_clauses", + "support_premises", + "attempt_candidates", "attempts", "attempt_inputs", "admission_events", @@ -335,3 +340,95 @@ fn malformed_head_without_accepted_delivery_is_an_error() -> Result<()> { } Ok(()) } + +// Rejecting malformed output must precede both Ready proofs, without repairing +// corruption or altering historical charges and unresolved dispatch reservations. +fn malformed_output_is_rejected(support_fallback: bool) -> Result<()> { + use rusqlite::types::Value; + for malformed in [ + Value::Integer(42), + Value::Real(4.2), + Value::Text("wrong storage class".into()), + ] { + let file = Database::new(); + let mut store = Store::open(&file.0)?; + let witness = store.observe_and_admit_many("a", "s", &[("source", b"evidence")])?[0]; + let budget = store.create_token_budget(100)?; + let accepted = store.begin_attempt("a", "s", "b", &[witness])?; + store.reserve_dispatch(budget, accepted, b"request", 2, 8)?; + store.settle(accepted, b"accepted", 7)?; + let head = if support_fallback { + let head = store.publish_support("a", "s", "b", &[vec![witness]])?; + store.observe_and_admit_many("a", "s", &[("unrelated", b"new")])?; + head + } else { + accepted + }; + let pending = store.begin_attempt("a", "s", "uncertain", &[])?; + store.reserve_dispatch(budget, pending, b"unknown send", 4, 10)?; + let explanation = store.explain_current("a", "s", "b")?; + assert_eq!(explanation.state, State::Ready); + assert_eq!(explanation.head_id, Some(head)); + assert_eq!( + explanation.proof_kind, + Some(if support_fallback { + Proof::SupportV1 + } else { + Proof::CoarseEpoch + }) + ); + assert_eq!( + store.read_current("a", "s", "b")?, + Current::Ready(if support_fallback { + b"supported".to_vec() + } else { + b"accepted".to_vec() + }) + ); + + let audit = Connection::open(&file.0)?; + assert_eq!( + audit.execute( + "UPDATE deliveries SET output=?1 WHERE attempt_id=?2", + rusqlite::params![malformed, head] + )?, + 1 + ); + let before = snapshot(&audit)?; + let before_budget = store.token_budget_state(budget)?; + let before_charge = store.charged_usage()?; + assert_eq!(before_budget.spent, 7); + assert_eq!(before_budget.reserved, 14); + assert_eq!(before_charge, if support_fallback { 8 } else { 7 }); + + let explanation = store.explain_current("a", "s", "b"); + assert!(store.read_current("a", "s", "b").is_err()); + assert!(store.begin_resume("a", "s", "b", &[]).is_err()); + assert_eq!(snapshot(&audit)?, before); + assert_eq!(store.token_budget_state(budget)?, before_budget); + assert_eq!(store.charged_usage()?, before_charge); + assert_eq!( + audit.query_row( + "SELECT output FROM deliveries WHERE attempt_id=?1", + [head], + |row| row.get::<_, Value>(0) + )?, + malformed + ); + assert!( + explanation.is_err(), + "malformed accepted output must not be eligible: {explanation:?}" + ); + } + Ok(()) +} + +#[test] +fn malformed_output_is_rejected_at_unchanged_epoch() -> Result<()> { + malformed_output_is_rejected(false) +} + +#[test] +fn malformed_output_is_rejected_by_support_fallback() -> Result<()> { + malformed_output_is_rejected(true) +} From 1509c38d1a4006c8391279026ff44fba6c813e97 Mon Sep 17 00:00:00 2001 From: Alejandro Castellanos <5405628+k2jac9@users.noreply.github.com> Date: Sun, 15 Nov 2026 14:42:11 -0500 Subject: [PATCH 06/51] feat: atomically revise admissions at an expected epoch --- docs/atomic-correction.md | 60 +++++++ examples/selective_revision.rs | 16 +- src/lib.rs | 2 +- src/support.rs | 109 ++++++++++-- tests/atomic_revision_batch.rs | 315 +++++++++++++++++++++++++++++++++ 5 files changed, 480 insertions(+), 22 deletions(-) create mode 100644 docs/atomic-correction.md create mode 100644 tests/atomic_revision_batch.rs diff --git a/docs/atomic-correction.md b/docs/atomic-correction.md new file mode 100644 index 0000000..a800974 --- /dev/null +++ b/docs/atomic-correction.md @@ -0,0 +1,60 @@ +# Atomic admission corrections + +`Store::revise_admissions_if_current(actor, scope, expected_epoch, edits)` applies +one correction against a known actor/scope view. An `AdmissionEdit` names an +existing occurrence and its requested admission. A correction can withdraw old +observations and admit replacements without publishing intermediate verdicts. +The embedding runtime authenticates the actor, scope and diagnostic access. + +## Request and conflict ordering + +1. Validate shape: actor and scope must be nonempty, expected epoch nonnegative, + and the batch must contain at most **1,024 edits**, each with a unique positive + occurrence ID. Duplicate IDs are errors even when their requested values agree. +2. Acquire SQLite's **Immediate** writer transaction, then compare the view epoch. + A valid stale request returns `BatchRevision::EpochConflict { current_epoch }` + before occurrence existence is inspected. This includes empty batches, + all-no-op batches and requests naming unknown positive IDs. +3. At a matching epoch, validate that every occurrence exists before the first + write. An unknown occurrence is an error and leaves the database unchanged. +4. Apply effective edits in input order. Each effective membership transition + advances the epoch by one and updates the support index in that transaction. +5. Union consumers of effective changes, evaluate their final support/v1 verdicts + once, and publish changed branches in branch-name order. Every new attempt + captures the **final** epoch. Commit all admissions, projections, attempts, + lineage and charges together, or roll them all back on error. + +A matching empty/all-no-op batch returns `Applied { epoch, changes: vec![] }` +without events, index writes, attempts or charges. No-op edits do not repair stale +consumers. A stale consumer may be reconciled if it also consumes an effective +edit in the batch. Consumers with unchanged final verdicts retain their heads and +charges even when a sequential correction would temporarily change the verdict. +Each changed verdict costs one unit, retains its rule identity and points to the +previous accepted head as its parent. Existing incurred charges and unresolved +reservations are preserved. Opaque/model heads retain their existing eligibility +rules; this operation does not certify or silently reuse them. + +## Retry and concurrency + +Two independent connections correcting the same nonempty effective view at the +same expected epoch serialize: one applies, the other observes an epoch conflict. +The epoch counts membership transitions, not batch calls. It advances even when +all final support verdicts remain unchanged. + +After a lost response, **reread the current view and relevant heads** before +choosing another expected epoch. Reusing the original epoch is safe against +reapplying a successful effective correction, including after a later writer +reverses its memberships. A conflict is not proof that the original call succeeded; +another writer could have advanced the view. Do not automatically replace the +expected epoch and replay old intent. Empty or all-no-op calls do not advance the +epoch and can both succeed at the same epoch without writes. + +The older `set_admitted_and_revise` remains an unconditional single-edit API. +Its immediate retry is a no-op, but a retry after an intervening reversal is a new +effective edit. The new API adds explicit optimistic concurrency without changing +that existing behavior. Neither API migrates the schema. + +Run `cargo run --locked --example selective_revision` for a mixed withdrawal and +replacement. `tests/atomic_revision_batch.rs` covers error precedence, rollback +following real writes, independent writer contention, final-view comparison for +all 256 initial/final four-occurrence subsets, and reproducible seeded histories. diff --git a/examples/selective_revision.rs b/examples/selective_revision.rs index e191501..11015b9 100644 --- a/examples/selective_revision.rs +++ b/examples/selective_revision.rs @@ -1,5 +1,5 @@ //! Run with `cargo run --example selective_revision`. -use kairotensor::{Current, Store, SupportRegistration}; +use kairotensor::{AdmissionEdit, BatchRevision, Current, Store, SupportRegistration}; use rusqlite::Result; use std::collections::BTreeSet; @@ -33,6 +33,8 @@ fn main() -> Result<()> { ]; let ids = store.observe_and_admit_many(ACTOR, SCOPE, observations)?; let &[report, manifest, rerun] = ids.as_slice() else { unreachable!() }; + // The replacement occurrence is retained but not yet admitted. + store.set_admitted(ACTOR, SCOPE, rerun, false)?; let report_rule = [vec![report]]; let record_rule = [vec![report, manifest], vec![rerun]]; let heads = store.publish_support_many(ACTOR, SCOPE, &[ @@ -43,7 +45,13 @@ fn main() -> Result<()> { let consumers = store.support_consumers(ACTOR, SCOPE, report)?; assert_eq!(consumers, vec![REPORT.to_owned(), RECORD.to_owned()]); let usage = store.charged_usage()?; - let changes = store.set_admitted_and_revise(ACTOR, SCOPE, report, false)?; + let expected_epoch = store.epoch(ACTOR, SCOPE)?; + let BatchRevision::Applied { epoch, changes } = store.revise_admissions_if_current( + ACTOR, SCOPE, expected_epoch, + &[AdmissionEdit { evidence: report, admitted: false }, + AdmissionEdit { evidence: rerun, admitted: true }], + )? else { panic!("single-writer example unexpectedly conflicted") }; + assert_eq!(epoch, expected_epoch + 2); assert_eq!(changes.len(), 1); let revision = &changes[0]; @@ -52,9 +60,9 @@ fn main() -> Result<()> { assert!(!revision.supported); assert_eq!(store.charged_usage()?, usage + 1); assert_eq!(store.revise_support(ACTOR, SCOPE, RECORD)?, old_record); - println!("Withdrew the original report and revised its dependent branches in one commit."); + println!("Replaced the original report with the second run in one epoch-checked commit."); println!("The primary report lost its only witness. Published attempt {}.", revision.attempt_id); - println!("The second run still supplies a reproduction record. Kept attempt {old_record}."); + println!("The replacement supplies a reproduction record without an intermediate verdict. Kept attempt {old_record}."); for (branch, rule) in [(REPORT, report_rule.as_slice()), (RECORD, record_rule.as_slice())] { let expected = recompute(&store, rule)?; diff --git a/src/lib.rs b/src/lib.rs index 3ab5f11..b635f7e 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -13,7 +13,7 @@ mod recovery; pub use recovery::{RecoveryReservation, RecoveryStart}; pub use attempts::{CapturedInputs, Current, Disposition}; pub use model_reuse::ModelAction; -pub use support::{SupportRegistration, SupportRevision}; +pub use support::{AdmissionEdit, BatchRevision, SupportRegistration, SupportRevision}; pub use token_budget::TokenBudgetState; use rusqlite::{ diff --git a/src/support.rs b/src/support.rs index dc88c1d..bf3630b 100644 --- a/src/support.rs +++ b/src/support.rs @@ -2,7 +2,7 @@ //! observation IDs are all admitted is a witness. Only the resulting flag can //! be reused under this rule. Observation payloads are never read by the rule. use rusqlite::{params, Connection, OptionalExtension, Result, Transaction, TransactionBehavior}; -use std::collections::{hash_map::Entry, HashMap}; +use std::collections::{hash_map::Entry, HashMap, HashSet}; use crate::attempts::{accept, ensure_branch, find_branch, record_attempt, Branch}; use crate::{change_admission, invalid, is_admitted, validate_view, view_epoch, Store}; @@ -28,6 +28,20 @@ pub struct SupportRevision { pub supported: bool, } +/// One occurrence's requested admission in an atomic correction. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct AdmissionEdit { + pub evidence: i64, + pub admitted: bool, +} + +/// An atomic correction either commits completely or observes a newer view. +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum BatchRevision { + Applied { epoch: i64, changes: Vec }, + EpochConflict { current_epoch: i64 }, +} + fn output(supported: bool) -> &'static [u8] { if supported { b"supported" } else { b"unsupported" } } @@ -132,13 +146,19 @@ fn changed_heads( tx: &Transaction<'_>, actor: &str, scope: &str, - evidence: i64, + evidence: &[i64], ) -> Result> { - let mut select = tx.prepare_cached( + if evidence.is_empty() { + return Ok(Vec::new()); + } + // The batch's 1,024-edit limit bounds both bindings and statement size. + let placeholders = (3..3 + evidence.len()).map(|i| format!("?{i}")) + .collect::>().join(","); + let sql = format!( "WITH affected(rule_id) AS ( SELECT DISTINCT c.rule_id FROM support_premises p JOIN support_clauses c ON c.id = p.clause_id - WHERE p.actor = ?1 AND p.scope = ?2 AND p.requested_id = ?3 AND p.watching = 1 + WHERE p.actor = ?1 AND p.scope = ?2 AND p.requested_id IN ({placeholders}) AND p.watching = 1 ) SELECT b.name, b.id, b.head, b.revision, s.rule_id, s.verdict, EXISTS ( SELECT 1 FROM support_clauses c WHERE c.rule_id = s.rule_id AND c.missing = 0 @@ -149,8 +169,11 @@ fn changed_heads( JOIN support_bindings s ON s.attempt_id = b.head AND s.rule_id = affected.rule_id WHERE b.actor = ?1 AND b.scope = ?2 ORDER BY b.name", - )?; - let mut rows = select.query(params![actor, scope, evidence])?; + ); + let mut select = tx.prepare(&sql)?; + let mut bindings = vec![rusqlite::types::Value::from(actor.to_owned()), rusqlite::types::Value::from(scope.to_owned())]; + bindings.extend(evidence.iter().copied().map(rusqlite::types::Value::from)); + let mut rows = select.query(rusqlite::params_from_iter(bindings))?; let mut changed = Vec::new(); while let Some(row) = rows.next()? { let published: bool = row.get(5)?; @@ -171,6 +194,19 @@ fn changed_heads( Ok(changed) } +// Publish from one final view after all effective admission/index changes. +// The single-edit API uses this same transaction-scoped path. +fn publish_changed_heads( + tx: &Transaction<'_>, actor: &str, scope: &str, epoch: i64, evidence: &[i64], +) -> Result> { + let mut revisions = Vec::new(); + for head in changed_heads(tx, actor, scope, evidence)? { + let attempt_id = publish_revision(tx, &head.branch, epoch, head.rule, head.supported)?; + revisions.push(SupportRevision { branch: head.name, attempt_id, supported: head.supported }); + } + Ok(revisions) +} + // A bulk registration never changes admissions. Bind this cache to its single // actor, scope and writer transaction; both missing and visible IDs can repeat. fn cached_admissions(mut lookup: impl FnMut(i64) -> Result) -> impl FnMut(i64) -> Result { @@ -321,20 +357,59 @@ impl Store { let tx = self .conn .transaction_with_behavior(TransactionBehavior::Immediate)?; - let mut revisions = Vec::new(); - if let Some(epoch) = change_admission(&tx, actor, scope, evidence, admitted)? { - for head in changed_heads(&tx, actor, scope, evidence)? { - let attempt_id = - publish_revision(&tx, &head.branch, epoch, head.rule, head.supported)?; - revisions.push(SupportRevision { - branch: head.name, - attempt_id, - supported: head.supported, - }); + let revisions = if let Some(epoch) = change_admission(&tx, actor, scope, evidence, admitted)? { + publish_changed_heads(&tx, actor, scope, epoch, &[evidence])? + } else { + Vec::new() + }; + tx.commit()?; + Ok(revisions) + } + + /// Correct up to 1,024 unique positive occurrence IDs at an expected epoch. + /// Shape errors precede the writer transaction. A valid stale request always + /// returns a conflict, even if empty, all-no-op, or containing unknown IDs. + /// At a matching epoch every occurrence must exist before any mutation. + /// + /// Effective edits follow input order. Only their consumers are reconciled, + /// once, against the final view, in branch-name order. Unchanged verdicts + /// retain their attempts and charges; no-op edits do not repair stale heads. + /// After a lost response reread the view before choosing a new expected epoch. + pub fn revise_admissions_if_current( + &mut self, + actor: &str, + scope: &str, + expected_epoch: i64, + edits: &[AdmissionEdit], + ) -> Result { + validate_view(actor, scope)?; + if expected_epoch < 0 || edits.len() > 1024 { + return Err(invalid("expected epoch must be nonnegative and at most 1,024 edits are allowed")); + } + let mut seen = HashSet::with_capacity(edits.len()); + for edit in edits { + if edit.evidence <= 0 || !seen.insert(edit.evidence) { + return Err(invalid("edits require unique positive occurrence IDs")); } } + let tx = self.conn.transaction_with_behavior(TransactionBehavior::Immediate)?; + let mut epoch = view_epoch(&tx, actor, scope)?; + if epoch != expected_epoch { + return Ok(BatchRevision::EpochConflict { current_epoch: epoch }); + } + for edit in edits { + tx.query_row("SELECT 1 FROM evidence WHERE id = ?1", [edit.evidence], |_| Ok(()))?; + } + let mut effective = Vec::new(); + for edit in edits { + if let Some(next) = change_admission(&tx, actor, scope, edit.evidence, edit.admitted)? { + epoch = next; + effective.push(edit.evidence); + } + } + let changes = publish_changed_heads(&tx, actor, scope, epoch, &effective)?; tx.commit()?; - Ok(revisions) + Ok(BatchRevision::Applied { epoch, changes }) } /// List the active support-rule branch names watching this occurrence in diff --git a/tests/atomic_revision_batch.rs b/tests/atomic_revision_batch.rs new file mode 100644 index 0000000..2528297 --- /dev/null +++ b/tests/atomic_revision_batch.rs @@ -0,0 +1,315 @@ +use kairotensor::{AdmissionEdit, BatchRevision, Current, Disposition, Store, SupportRevision}; +use rusqlite::{types::Value, Connection, Result}; +use std::collections::BTreeMap; +use std::path::PathBuf; +use std::sync::{Arc, Barrier}; +use std::time::{SystemTime, UNIX_EPOCH}; + +const A: &str = "reviewer"; +const S: &str = "release"; +struct Database(PathBuf); +impl Database { + fn new() -> Self { + let stamp = SystemTime::now().duration_since(UNIX_EPOCH).unwrap().as_nanos(); + Self(std::env::temp_dir().join(format!("kairotensor-batch-{}-{stamp}.db", std::process::id()))) + } +} +impl Drop for Database { + fn drop(&mut self) { let _ = std::fs::remove_file(&self.0); } +} +fn edit(evidence: i64, admitted: bool) -> AdmissionEdit { AdmissionEdit { evidence, admitted } } +fn ready(value: bool) -> Current { + Current::Ready(if value { b"supported".to_vec() } else { b"unsupported".to_vec() }) +} +fn applied(result: BatchRevision, expected: i64) -> Vec { + match result { + BatchRevision::Applied { epoch, changes } => { assert_eq!(epoch, expected); changes } + other => panic!("expected Applied at {expected}, got {other:?}"), + } +} +// Include every persistent table, table boundaries and every column, including +// history, indexes, charges, dispatches and unresolved budget reservations. +fn snapshot(db: &Connection) -> Result>>> { + let tables = db.prepare("SELECT name FROM sqlite_schema WHERE type='table' ORDER BY name")? + .query_map([], |r| r.get::<_, String>(0))?.collect::>>()?; + let mut result = BTreeMap::new(); + for table in tables { + let mut stmt = db.prepare(&format!("SELECT * FROM \"{}\"", table.replace('"', "\"\"")))?; + let columns = stmt.column_count(); + let mut rows = stmt.query_map([], |r| (0..columns).map(|i| r.get(i)).collect())? + .collect::>>>()?; + rows.sort_by_key(|row| format!("{row:?}")); + result.insert(table, rows); + } + Ok(result) +} +fn two(store: &mut Store) -> Result<()> { + store.observe_and_admit_many(A, S, &[("one", b"1"), ("two", b"2")])?; + Ok(()) +} + +#[test] +fn replacement_has_no_intermediate_verdict() -> Result<()> { + let file = Database::new(); + let mut store = Store::open(&file.0)?; + two(&mut store)?; + store.set_admitted(A, S, 2, false)?; + let head = store.publish_support(A, S, "either", &[vec![1], vec![2]])?; + let audit = Connection::open(&file.0)?; + let charges = store.charged_usage()?; + assert!(applied(store.revise_admissions_if_current(A, S, 3, &[edit(1, false), edit(2, true)])?, 5).is_empty()); + assert_eq!(store.revise_support(A, S, "either")?, head); + assert_eq!(store.charged_usage()?, charges); + assert_eq!(store.read_current(A, S, "either")?, ready(true)); + let events = audit.prepare("SELECT evidence_id, admitted FROM admission_events WHERE epoch > 3 ORDER BY epoch")? + .query_map([], |r| Ok((r.get::<_, i64>(0)?, r.get::<_, bool>(1)?)))?.collect::>>()?; + assert_eq!(events, [(1, false), (2, true)]); + Ok(()) +} + +#[test] +fn overlapping_consumers_publish_once_at_final_epoch() -> Result<()> { + let file = Database::new(); + let mut store = Store::open(&file.0)?; + two(&mut store)?; + let z = store.publish_support(A, S, "z", &[vec![1, 2], vec![1]])?; + let a = store.publish_support(A, S, "a", &[vec![1, 2]])?; + let before = store.charged_usage()?; + let changes = applied(store.revise_admissions_if_current(A, S, 2, &[edit(2, false), edit(1, false)])?, 4); + assert_eq!(changes.iter().map(|c| c.branch.as_str()).collect::>(), ["a", "z"]); + assert_eq!(store.charged_usage()?, before + 2); + assert!(changes[0].attempt_id < changes[1].attempt_id); + let audit = Connection::open(&file.0)?; + for (change, root) in changes.iter().zip([a, z]) { + let row: (i64, i64, i64, i64) = audit.query_row( + "SELECT a.parent, a.view_epoch, s.rule_id, d.usage FROM attempts a JOIN support_bindings s ON s.attempt_id=a.id JOIN deliveries d ON d.attempt_id=a.id WHERE a.id=?1", + [change.attempt_id], |r| Ok((r.get(0)?, r.get(1)?, r.get(2)?, r.get(3)?)))?; + assert_eq!(row, (root, 4, root, 1)); + assert!(!change.supported); + } + Ok(()) +} + +#[test] +fn duplicate_edit_is_rejected() -> Result<()> { + let mut store = Store::in_memory()?; + two(&mut store)?; + for values in [[false, false], [false, true]] { + assert!(store.revise_admissions_if_current(A, S, 2, &[edit(1, values[0]), edit(1, values[1])]).is_err()); + } + assert_eq!(store.epoch(A, S)?, 2); + Ok(()) +} + +#[test] +fn stale_empty_and_noop_batches_conflict() -> Result<()> { + let file = Database::new(); + let mut store = Store::open(&file.0)?; + two(&mut store)?; + let audit = Connection::open(&file.0)?; + let before = snapshot(&audit)?; + for edits in [vec![], vec![edit(1, true)], vec![edit(9999, false)]] { + assert_eq!(store.revise_admissions_if_current(A, S, 0, &edits)?, BatchRevision::EpochConflict { current_epoch: 2 }); + } + for edits in [vec![], vec![edit(1, true), edit(2, true)]] { + assert!(applied(store.revise_admissions_if_current(A, S, 2, &edits)?, 2).is_empty()); + } + assert_eq!(snapshot(&audit)?, before); + Ok(()) +} + +#[test] +fn noop_leaves_stale_consumers_alone() -> Result<()> { + let mut store = Store::in_memory()?; + two(&mut store)?; + store.publish_support(A, S, "stale", &[vec![1]])?; + store.publish_support(A, S, "changed", &[vec![2]])?; + store.set_admitted(A, S, 1, false)?; + let charge = store.charged_usage()?; + let changes = applied(store.revise_admissions_if_current(A, S, 3, &[edit(1, false), edit(2, false)])?, 4); + assert_eq!(changes.len(), 1); + assert_eq!(changes[0].branch, "changed"); + assert_eq!(store.read_current(A, S, "stale")?, Current::NeedsReview); + assert_eq!(store.charged_usage()?, charge + 1); + Ok(()) +} + +#[test] +fn request_shape_and_epoch_precedence_are_stable() -> Result<()> { + let file = Database::new(); + let mut store = Store::open(&file.0)?; + two(&mut store)?; + let audit = Connection::open(&file.0)?; + let before = snapshot(&audit)?; + for (actor, scope, epoch, edits) in [ + ("", S, 0, vec![]), (A, "", 0, vec![]), (A, S, -1, vec![]), + (A, S, 0, vec![edit(0, true)]), (A, S, 0, vec![edit(-2, true)]), + (A, S, 0, vec![edit(1, false), edit(1, true)]), + (A, S, 0, (1..=1025).map(|id| edit(id, true)).collect()), + ] { assert!(store.revise_admissions_if_current(actor, scope, epoch, &edits).is_err()); } + let boundary: Vec<_> = (1..=1024).map(|id| edit(id, true)).collect(); + assert_eq!(store.revise_admissions_if_current(A, S, 0, &boundary)?, BatchRevision::EpochConflict { current_epoch: 2 }); + // An abort trigger on the first edit distinguishes full prevalidation from + // rollback following a late unknown-ID lookup. + audit.execute_batch("CREATE TRIGGER reject_write BEFORE INSERT ON admission_events BEGIN SELECT RAISE(ABORT, 'mutation before existence validation'); END;")?; + let error = store.revise_admissions_if_current(A, S, 2, &[edit(1, false), edit(9999, true)]).unwrap_err(); + assert!(matches!(error, rusqlite::Error::QueryReturnedNoRows)); + assert_eq!(snapshot(&audit)?, before); + audit.execute_batch("DROP TRIGGER reject_write")?; + for id in 3..=1024 { assert_eq!(store.observe("boundary", b"x")?, id); } + let result = applied(store.revise_admissions_if_current(A, S, 2, &boundary)?, 1024); + assert!(result.is_empty()); + Ok(()) +} + +fn rollback_after(trigger: &str, expected_error: &str) -> Result<()> { + let file = Database::new(); + let mut store = Store::open(&file.0)?; + two(&mut store)?; + for name in ["a", "z"] { store.publish_support(A, S, name, &[vec![1, 2]])?; } + let settled = store.begin_attempt(A, S, "opaque", &[1, 2])?; + let budget = store.create_token_budget(100)?; + store.reserve_dispatch(budget, settled, b"accepted request", 2, 8)?; + assert_eq!(store.settle(settled, b"answer", 7)?, Disposition::Accepted); + let pending = store.begin_attempt(A, S, "unresolved", &[])?; + store.reserve_dispatch(budget, pending, b"unknown send", 4, 10)?; + let audit = Connection::open(&file.0)?; + audit.execute_batch(trigger)?; + let before = snapshot(&audit)?; + let held = store.token_budget_state(budget)?; + assert_eq!((held.spent, held.reserved), (7, 14)); + let error = store.revise_admissions_if_current(A, S, 2, &[edit(1, false), edit(2, false)]).unwrap_err(); + assert!(error.to_string().contains(expected_error), "{error}"); + assert_eq!(snapshot(&audit)?, before); + assert_eq!(store.token_budget_state(budget)?, held); + drop(store); + let mut store = Store::open(&file.0)?; + assert_eq!(snapshot(&audit)?, before); + assert_eq!(store.epoch(A, S)?, 2); + assert_eq!(store.token_budget_state(budget)?, held); + audit.execute_batch("DROP TRIGGER fail_batch")?; + assert_eq!(applied(store.revise_admissions_if_current(A, S, 2, &[edit(1, false), edit(2, false)])?, 4).len(), 2); + assert_eq!(store.token_budget_state(budget)?, held); + Ok(()) +} + +#[test] +fn second_edit_failure_rolls_back_every_write() -> Result<()> { + rollback_after("CREATE TRIGGER fail_batch BEFORE INSERT ON admission_events WHEN NEW.epoch=4 AND EXISTS(SELECT 1 FROM admission_events WHERE epoch=3 AND evidence_id=1 AND admitted=0) BEGIN SELECT RAISE(ABORT, 'second effective edit'); END;", "second effective edit") +} +#[test] +fn later_publication_failure_rolls_back_every_write() -> Result<()> { + rollback_after("CREATE TRIGGER fail_batch BEFORE UPDATE OF head ON branches WHEN NEW.name='z' AND EXISTS(SELECT 1 FROM branches WHERE name='a' AND revision=2) BEGIN SELECT RAISE(ABORT, 'later publication'); END;", "later publication") +} + +#[test] +fn concurrent_correction_has_one_winner() -> Result<()> { + let file = Database::new(); + let mut store = Store::open(&file.0)?; + two(&mut store)?; + for name in ["a", "z"] { store.publish_support(A, S, name, &[vec![1, 2]])?; } + let initial_charge = store.charged_usage()?; + let connections = [Store::open(&file.0)?, Store::open(&file.0)?]; + let start = Arc::new(Barrier::new(3)); + let workers: Vec<_> = connections.into_iter().map(|mut writer| { + let start = Arc::clone(&start); + std::thread::spawn(move || { start.wait(); writer.revise_admissions_if_current(A, S, 2, &[edit(1, false), edit(2, false)]) }) + }).collect(); + start.wait(); + let mut wins = 0; + let mut conflicts = 0; + for worker in workers { + match worker.join().unwrap()? { + BatchRevision::Applied { epoch, changes } => { assert_eq!(epoch, 4); assert_eq!(changes.len(), 2); wins += 1; } + BatchRevision::EpochConflict { current_epoch } => { assert_eq!(current_epoch, 4); conflicts += 1; } + } + } + assert_eq!((wins, conflicts), (1, 1)); + assert_eq!(store.charged_usage()?, initial_charge + 2); + applied(store.revise_admissions_if_current(A, S, 4, &[edit(1, true), edit(2, true)])?, 6); + let audit = Connection::open(&file.0)?; + let before = snapshot(&audit)?; + assert_eq!(store.revise_admissions_if_current(A, S, 2, &[edit(1, false), edit(2, false)])?, BatchRevision::EpochConflict { current_epoch: 6 }); + assert_eq!(snapshot(&audit)?, before); + assert_eq!(store.charged_usage()?, initial_charge + 4); + Ok(()) +} + +fn rules() -> Vec<(&'static str, Vec>)> { + vec![("a", vec![vec![1, 2], vec![3, 4]]), ("b", vec![vec![1], vec![2, 3]]), ("c", vec![vec![1, 2, 3, 4]]), ("d", vec![vec![2], vec![4]])] +} +fn fixture(store: &mut Store, mask: u8) -> Result> { + for id in 1..=4 { + assert_eq!(store.observe("occurrence", b"same payload")?, id); + store.set_admitted(A, S, id, mask & (1 << (id-1)) != 0)?; + } + rules().into_iter().map(|(name, groups)| Ok((name.to_owned(), store.publish_support(A, S, name, &groups)?))).collect() +} +fn verdict(groups: &[Vec], mask: u8) -> bool { + groups.iter().any(|g| g.iter().all(|id| mask & (1 << (id-1)) != 0)) +} +fn compare_transition(store: &mut Store, audit: &Connection, heads: &mut BTreeMap, roots: &BTreeMap, from: u8, to: u8, reverse: bool) -> Result<()> { + let mut reference = Store::in_memory()?; + fixture(&mut reference, to)?; + let mut edits: Vec<_> = (1..=4).map(|id| edit(id, to & (1 << (id-1)) != 0)).collect(); + if reverse { edits.reverse(); } + let before_epoch = store.epoch(A, S)?; + let final_epoch = before_epoch + (from ^ to).count_ones() as i64; + let charge = store.charged_usage()?; + let changes = applied(store.revise_admissions_if_current(A, S, before_epoch, &edits)?, final_epoch); + let expected: Vec<_> = rules().into_iter().filter(|(_, g)| verdict(g, from) != verdict(g, to)).map(|(n, _)| n).collect(); + assert_eq!(changes.iter().map(|c| c.branch.as_str()).collect::>(), expected, "transition {from}->{to}"); + assert_eq!(store.charged_usage()?, charge + expected.len() as i64); + for c in changes { + let row: (i64, i64, i64, i64) = audit.query_row("SELECT a.parent, a.view_epoch, s.rule_id, d.usage FROM attempts a JOIN support_bindings s ON s.attempt_id=a.id JOIN deliveries d ON d.attempt_id=a.id WHERE a.id=?1", [c.attempt_id], |r| Ok((r.get(0)?, r.get(1)?, r.get(2)?, r.get(3)?)))?; + assert_eq!(row, (heads[&c.branch], final_epoch, roots[&c.branch], 1)); + heads.insert(c.branch, c.attempt_id); + } + for (name, groups) in rules() { + assert_eq!(store.read_current(A, S, name)?, reference.read_current(A, S, name)?); + assert_eq!(store.read_current(A, S, name)?, ready(verdict(&groups, to))); + assert_eq!(store.revise_support(A, S, name)?, heads[name]); + } + let events = audit.prepare("SELECT evidence_id, admitted FROM admission_events WHERE epoch>?1 ORDER BY epoch")? + .query_map([before_epoch], |r| Ok((r.get::<_, i64>(0)?, r.get::<_, bool>(1)?)))?.collect::>>()?; + let expected_events: Vec<_> = edits.iter().filter(|e| (from ^ to) & (1 << (e.evidence-1)) != 0).map(|e| (e.evidence, e.admitted)).collect(); + assert_eq!(events, expected_events); + Ok(()) +} + +#[test] +fn every_initial_and_final_four_occurrence_view_matches_fresh_store() -> Result<()> { + for from in 0..16 { + for to in 0..16 { + let file = Database::new(); + let mut store = Store::open(&file.0)?; + let mut heads = fixture(&mut store, from)?; + let roots = heads.clone(); + let audit = Connection::open(&file.0)?; + compare_transition(&mut store, &audit, &mut heads, &roots, from, to, (from + to) % 2 == 0)?; + } + } + Ok(()) +} + +#[test] +fn seeded_edit_sequences_preserve_lineage_charges_and_final_views() -> Result<()> { + for seed in [0x4b54_2026_u64, 0x5eed_1115, 0xdead_beef] { + eprintln!("atomic correction sequence seed={seed:#x}"); + let file = Database::new(); + let mut store = Store::open(&file.0)?; + let mut heads = fixture(&mut store, 0)?; + let roots = heads.clone(); + let audit = Connection::open(&file.0)?; + let mut random = seed; + let mut from = 0; + for step in 0..64 { + random = random.wrapping_mul(6364136223846793005).wrapping_add(1); + let to = ((random >> 32) & 15) as u8; + eprintln!("seed={seed:#x} step={step} transition={from}->{to}"); + compare_transition(&mut store, &audit, &mut heads, &roots, from, to, random & 1 != 0)?; + from = to; + } + } + Ok(()) +} From 261a8636458d08693078a8a06b4f336b08b2da9d Mon Sep 17 00:00:00 2001 From: Alejandro Castellanos <5405628+k2jac9@users.noreply.github.com> Date: Sun, 15 Nov 2026 15:02:49 -0500 Subject: [PATCH 07/51] feat: add pure versioned incident protocol and exact-byte digests --- Cargo.lock | 71 +++++++ Cargo.toml | 1 + src/evidence_digest.rs | 30 +++ src/incident_protocol.rs | 289 +++++++++++++++++++++++++++ src/lib.rs | 3 + tests/incident_protocol.rs | 392 +++++++++++++++++++++++++++++++++++++ 6 files changed, 786 insertions(+) create mode 100644 src/evidence_digest.rs create mode 100644 src/incident_protocol.rs create mode 100644 tests/incident_protocol.rs diff --git a/Cargo.lock b/Cargo.lock index ca1d8fe..5cad425 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -20,6 +20,15 @@ version = "2.13.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3ded4057c258ba199e2d26386d3af3780957ecaee6c4ef4041c6b4b8b97c0b06" +[[package]] +name = "block-buffer" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2f6c7dbe95a6ed67ad9f18e57daf93a2f034c524b99fd2b76d18fdfeb6660aa" +dependencies = [ + "hybrid-array", +] + [[package]] name = "cc" version = "1.8.0" @@ -36,6 +45,41 @@ version = "1.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4e7648175b45a9a48536d676f68d918270699102aa8dab5496df06904c914600" +[[package]] +name = "const-oid" +version = "0.10.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "be95d4a884358a21a199689323c29e9256b4a1b315e22f183d155422041d3a9a" + +[[package]] +name = "cpufeatures" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5ca28b0ae3115b884660db4118d803791fd6756b6e88f39c0f3f7859060d7566" +dependencies = [ + "libc", +] + +[[package]] +name = "crypto-common" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce6e4c961d6cd6c9a86db418387425e8bdeaf05b3c8bc1411e6dca4c252f1453" +dependencies = [ + "hybrid-array", +] + +[[package]] +name = "digest" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1dd6dbb5841937940781866fa1281a1ff7bd3bf827091440879f9994983d5c2" +dependencies = [ + "block-buffer", + "const-oid", + "crypto-common", +] + [[package]] name = "fallible-iterator" version = "0.3.0" @@ -72,6 +116,15 @@ dependencies = [ "hashbrown", ] +[[package]] +name = "hybrid-array" +version = "0.4.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "27f864f10dfb56725ce5ce5472bc52252c8f93a4ab86327122cebf62c5f59a17" +dependencies = [ + "typenum", +] + [[package]] name = "itoa" version = "1.0.18" @@ -86,6 +139,7 @@ dependencies = [ "rusqlite", "serde", "serde_json", + "sha2", ] [[package]] @@ -198,6 +252,17 @@ dependencies = [ "zmij", ] +[[package]] +name = "sha2" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "47d7069beb7d6ac7b9acd1039986e73443f24234f41074da099d6f994ac9ad19" +dependencies = [ + "cfg-if", + "cpufeatures", + "digest", +] + [[package]] name = "shlex" version = "2.0.1" @@ -232,6 +297,12 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "typenum" +version = "1.20.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6f5e870be6c3b371b77fe0ee0bafb859fa4964b4404c27de1d380043c4dda20" + [[package]] name = "unicode-ident" version = "1.0.27" diff --git a/Cargo.toml b/Cargo.toml index dc6985e..6c16435 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -6,6 +6,7 @@ description = "Versioned evidence memory for agent runtimes" publish = false [dependencies] +sha2 = "=0.11.1" rusqlite = { version = "=0.32.1", features = ["bundled"] } serde = { version = "1", features = ["derive"] } diff --git a/src/evidence_digest.rs b/src/evidence_digest.rs new file mode 100644 index 0000000..cb8ab06 --- /dev/null +++ b/src/evidence_digest.rs @@ -0,0 +1,30 @@ +//! Exact-byte digests are identities, not proof of truth or execution. +use sha2::{Digest, Sha256}; + +pub(crate) fn sha256_hex(bytes: &[u8]) -> String { + Sha256::digest(bytes) + .iter() + .map(|byte| format!("{byte:02x}")) + .collect() +} + +#[cfg(test)] +mod tests { + use super::sha256_hex; + + #[test] + fn exact_byte_sha256_known_vectors() { + assert_eq!( + sha256_hex(b""), + "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" + ); + assert_eq!( + sha256_hex(b"abc"), + "ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad" + ); + assert_eq!( + sha256_hex(&[0, 255]), + "06eb7d6a69ee19e5fbdf749018d3d2abfa04bcbd1365db312eb86dc7169389b8" + ); + } +} diff --git a/src/incident_protocol.rs b/src/incident_protocol.rs new file mode 100644 index 0000000..a0e2214 --- /dev/null +++ b/src/incident_protocol.rs @@ -0,0 +1,289 @@ +//! Pure incident v1 codec. Format validation never grants admission, send +//! permission, or scientific correctness. No Store or backend is consulted. +use serde::{Deserialize, Serialize}; +use std::collections::HashSet; +use std::fmt; + +const MAX_BYTES: usize = 1024 * 1024; +const MAX_DEPTH: usize = 64; +const RETRY_FEEDBACK: &str = "incident-routing-protocol/1:invalid_output"; + +/// Stable, payload-free protocol errors. Rejected bytes are never retained. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum IncidentProtocolError { + TooLarge, + InvalidEncoding, + InvalidJson, + UnknownVersion, + InvalidContext, + InvalidOutput, +} +impl fmt::Display for IncidentProtocolError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str(match self { + Self::TooLarge => "incident protocol: size limit exceeded", + Self::InvalidEncoding => "incident protocol: invalid UTF-8", + Self::InvalidJson => "incident protocol: invalid JSON", + Self::UnknownVersion => "incident protocol: unknown version", + Self::InvalidContext => "incident protocol: invalid context", + Self::InvalidOutput => "incident protocol: invalid output", + }) + } +} +impl std::error::Error for IncidentProtocolError {} +use IncidentProtocolError as Error; + +/// Supplied context metadata is not evidence of a trusted workload or authority. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct IncidentContextV1 { + pub schema_version: u32, + pub version: String, + pub workload_version: String, + pub freeze_sha256: String, + pub prompt_sha256: String, + pub validator_version: String, + pub case_id: String, + pub question: String, + pub owner_ids: Vec, + pub evidence_ids: Vec, + pub system_prompt: String, +} + +// Keep deserialization/serialization behind the validated public byte codec. +#[derive(Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +struct ContextWire { + schema_version: u32, + version: String, + workload_version: String, + freeze_sha256: String, + prompt_sha256: String, + validator_version: String, + case_id: String, + question: String, + owner_ids: Vec, + evidence_ids: Vec, + system_prompt: String, +} +impl IncidentContextV1 { + /// Decode bounded UTF-8 JSON with exact fields and unique object keys. + pub fn from_json(raw: &[u8]) -> Result { + let w: ContextWire = decode(raw, Error::InvalidContext)?; + let context = Self { + schema_version: w.schema_version, + version: w.version, + workload_version: w.workload_version, + freeze_sha256: w.freeze_sha256, + prompt_sha256: w.prompt_sha256, + validator_version: w.validator_version, + case_id: w.case_id, + question: w.question, + owner_ids: w.owner_ids, + evidence_ids: w.evidence_ids, + system_prompt: w.system_prompt, + }; + context.validate()?; + Ok(context) + } + + /// Encode sorted-key compact UTF-8 JSON, checking direct constructions too. + pub fn to_json(&self) -> Result, Error> { + self.validate()?; + self.encode() + } + + fn encode(&self) -> Result, Error> { + canonical(&serde_json::json!({ + "schema_version":self.schema_version, "version":self.version, + "workload_version":self.workload_version, "freeze_sha256":self.freeze_sha256, + "prompt_sha256":self.prompt_sha256, "validator_version":self.validator_version, + "case_id":self.case_id, "question":self.question, "owner_ids":self.owner_ids, + "evidence_ids":self.evidence_ids, "system_prompt":self.system_prompt, + })) + } + + fn validate(&self) -> Result<(), Error> { + if self.schema_version != 1 || self.version != "incident-store-context/1" { + return Err(Error::UnknownVersion); + } + if ![ + &self.workload_version, + &self.freeze_sha256, + &self.prompt_sha256, + &self.validator_version, + &self.case_id, + &self.question, + &self.system_prompt, + ] + .into_iter() + .all(|s| nonempty(s)) + || !unique_nonempty(&self.owner_ids) + || !unique_nonempty(&self.evidence_ids) + { + return Err(Error::InvalidContext); + } + // Bounds apply to the whole context, even fields not included in a request. + self.encode()?; + Ok(()) + } +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct IncidentReport { + pub external_id: String, + pub text: String, +} + +/// Each phase has fixed rendering behavior; no prior output or free-form feedback. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum IncidentPhase { + Initial, + ProtocolRetry, +} + +/// Valid format and supplied IDs only, independent of the reports' meaning. +#[derive(Clone, Debug, PartialEq, Eq, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct IncidentOutput { + pub decision: String, + // Option ordinarily permits missing keys; frozen v1 requires an explicit key. + #[serde(deserialize_with = "required_owner")] + pub owner: Option, + pub reason: String, + pub evidence_ids: Vec, +} +fn required_owner<'de, D: serde::Deserializer<'de>>(d: D) -> Result, D::Error> { + Option::::deserialize(d) +} + +/// Render `incident-request/1` or `incident-request-retry/1` with Python v1 bytes. +pub fn render_incident_request( + context: &IncidentContextV1, + reports: &[IncidentReport], + phase: IncidentPhase, +) -> Result, Error> { + context.validate()?; + if reports.len() != context.evidence_ids.len() + || reports + .iter() + .zip(&context.evidence_ids) + .any(|(r, id)| &r.external_id != id || !nonempty(&r.text)) + { + return Err(Error::InvalidContext); + } + let evidence: Vec<_> = reports + .iter() + .map(|r| { + serde_json::json!({ + "id": r.external_id, "text": r.text, + }) + }) + .collect(); + let mut payload = serde_json::json!({ + "question": context.question, "owner_ids": context.owner_ids, "evidence": evidence, + }); + if phase == IncidentPhase::ProtocolRetry { + payload["validation_error"] = RETRY_FEEDBACK.into(); + } + let user = String::from_utf8(canonical(&payload)?).map_err(|_| Error::InvalidEncoding)?; + canonical(&serde_json::json!({"messages": [ + {"role": "system", "content": context.system_prompt}, + {"role": "user", "content": user}, + ]})) +} + +/// Validate frozen v1 structure and supplied IDs, never labels or semantics. +pub fn validate_incident_output( + raw: &[u8], + context: &IncidentContextV1, +) -> Result { + context.validate()?; + let output: IncidentOutput = decode(raw, Error::InvalidOutput)?; + let valid_decision = match output.decision.as_str() { + "route" => { + output + .owner + .as_ref() + .is_some_and(|owner| context.owner_ids.contains(owner)) + && output.reason == "diagnosed_component" + } + "abstain" => { + output.owner.is_none() + && matches!( + output.reason.as_str(), + "undetermined_cause" | "conflicting_diagnoses" | "owner_not_unique" + ) + } + _ => false, + }; + if !valid_decision + || !unique_nonempty(&output.evidence_ids) + || output + .evidence_ids + .iter() + .any(|id| !context.evidence_ids.contains(id)) + { + return Err(Error::InvalidOutput); + } + Ok(output) +} + +fn nonempty(s: &str) -> bool { + // Python str.strip additionally treats the four ASCII information + // separators as whitespace; Rust char::is_whitespace does not. + s.chars() + .any(|c| !c.is_whitespace() && !('\u{1c}'..='\u{1f}').contains(&c)) +} +fn unique_nonempty(ids: &[String]) -> bool { + let mut seen = HashSet::new(); + !ids.is_empty() && ids.iter().all(|s| nonempty(s) && seen.insert(s)) +} +fn canonical(value: &serde_json::Value) -> Result, Error> { + // serde_json's default Map is a BTreeMap: sorted keys, compact separators, + // literal UTF-8, and Python-compatible string escaping. + let raw = serde_json::to_vec(value).map_err(|_| Error::InvalidJson)?; + if raw.len() > MAX_BYTES { + return Err(Error::TooLarge); + } + Ok(raw) +} +fn decode<'a, T: Deserialize<'a>>(raw: &'a [u8], invalid: Error) -> Result { + if raw.len() > MAX_BYTES { + return Err(Error::TooLarge); + } + let source = std::str::from_utf8(raw).map_err(|_| Error::InvalidEncoding)?; + check_depth(source)?; + serde_json::from_str(source).map_err(|e| match e.classify() { + serde_json::error::Category::Data => invalid, + _ => Error::InvalidJson, + }) +} +fn check_depth(source: &str) -> Result<(), Error> { + let (mut depth, mut quoted, mut escaped) = (0usize, false, false); + for byte in source.bytes() { + if quoted { + if escaped { + escaped = false; + } else if byte == b'\\' { + escaped = true; + } else if byte == b'"' { + quoted = false; + } + } else { + match byte { + b'"' => quoted = true, + b'[' | b'{' => { + depth += 1; + if depth > MAX_DEPTH { + return Err(Error::InvalidJson); + } + } + b']' | b'}' => { + depth = depth.checked_sub(1).ok_or(Error::InvalidJson)?; + } + _ => {} + } + } + } + Ok(()) +} diff --git a/src/lib.rs b/src/lib.rs index b635f7e..c612292 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1,5 +1,8 @@ //! Local versioned evidence memory with actor-scoped admission and atomic publication. +mod evidence_digest; +pub mod incident_protocol; + mod attempts; mod eligibility; pub use eligibility::{CurrentExplanation, EligibilityProof, EligibilityReason, EligibilityState}; diff --git a/tests/incident_protocol.rs b/tests/incident_protocol.rs new file mode 100644 index 0000000..dc8e96e --- /dev/null +++ b/tests/incident_protocol.rs @@ -0,0 +1,392 @@ +use kairotensor::incident_protocol::{ + render_incident_request, validate_incident_output, IncidentContextV1, IncidentPhase, + IncidentProtocolError as Error, IncidentReport, +}; + +fn context() -> IncidentContextV1 { + IncidentContextV1 { + schema_version: 1, + version: "incident-store-context/1".into(), + workload_version: "incident-routing/1".into(), + freeze_sha256: "freeze".into(), + prompt_sha256: "prompt".into(), + validator_version: "incident-routing-protocol/1".into(), + case_id: "case-1".into(), + question: "Who?".into(), + owner_ids: vec!["a".into(), "b".into()], + evidence_ids: vec!["e1".into(), "e2".into()], + system_prompt: "Read.".into(), + } +} +fn reports() -> Vec { + vec![ + IncidentReport { + external_id: "e1".into(), + text: "Cause is A.".into(), + }, + IncidentReport { + external_id: "e2".into(), + text: "A belongs to a.".into(), + }, + ] +} +const ROUTE: &[u8] = br#"{"decision":"route","owner":"a","reason":"diagnosed_component","evidence_ids":["e1","e2"]}"#; + +#[test] +fn initial_request_is_exact_python_v1_bytes() { + let expected = br#"{"messages":[{"content":"Read.","role":"system"},{"content":"{\"evidence\":[{\"id\":\"e1\",\"text\":\"Cause is A.\"},{\"id\":\"e2\",\"text\":\"A belongs to a.\"}],\"owner_ids\":[\"a\",\"b\"],\"question\":\"Who?\"}","role":"user"}]}"#; + assert_eq!( + render_incident_request(&context(), &reports(), IncidentPhase::Initial).unwrap(), + expected + ); +} +#[test] +fn retry_adds_only_fixed_versioned_feedback() { + let initial: serde_json::Value = serde_json::from_slice( + &render_incident_request(&context(), &reports(), IncidentPhase::Initial).unwrap(), + ) + .unwrap(); + let mut expected = initial.clone(); + let mut inner: serde_json::Value = + serde_json::from_str(initial["messages"][1]["content"].as_str().unwrap()).unwrap(); + inner["validation_error"] = "incident-routing-protocol/1:invalid_output".into(); + expected["messages"][1]["content"] = serde_json::to_string(&inner).unwrap().into(); + assert_eq!( + render_incident_request(&context(), &reports(), IncidentPhase::ProtocolRetry).unwrap(), + serde_json::to_vec(&expected).unwrap() + ); +} +#[test] +fn unicode_controls_and_escaping_match_python() { + let mut c = context(); + c.question = "é🦀\u{2028}\"\\\n\t\0\u{8}\u{c}\r".into(); + let request = render_incident_request(&c, &reports(), IncidentPhase::Initial).unwrap(); + let text = String::from_utf8(request).unwrap(); + assert_eq!( + text, + r###"{"messages":[{"content":"Read.","role":"system"},{"content":"{\"evidence\":[{\"id\":\"e1\",\"text\":\"Cause is A.\"},{\"id\":\"e2\",\"text\":\"A belongs to a.\"}],\"owner_ids\":[\"a\",\"b\"],\"question\":\"é🦀
\\\"\\\\\\n\\t\\u0000\\b\\f\\r\"}","role":"user"}]}"### + ); +} +#[test] +fn context_roundtrip_is_canonical_and_preserves_order() { + let c = context(); + let raw = c.to_json().unwrap(); + assert_eq!(IncidentContextV1::from_json(&raw).unwrap(), c); + assert!(raw.starts_with(br#"{"case_id":"case-1","evidence_ids":["e1","e2"]"#)); +} +#[test] +fn contexts_refuse_unknown_duplicate_missing_and_wrong_typed_fields() { + let raw = String::from_utf8(context().to_json().unwrap()).unwrap(); + for bad in [ + raw.replacen('{', r#"{"unknown":1,"#, 1), + raw.replacen('{', r#"{"case_id":"other","#, 1), + raw.replace(r#""schema_version":1"#, r#""schema_version":1.0"#), + raw.replace(r#""schema_version":1"#, r#""schema_version":true"#), + raw.replace(r#""case_id":"case-1","#, ""), + ] { + assert!( + IncidentContextV1::from_json(bad.as_bytes()).is_err(), + "{bad}" + ); + } +} +#[test] +fn unknown_context_versions_refuse() { + let mut c = context(); + c.schema_version = 2; + assert_eq!(c.to_json(), Err(Error::UnknownVersion)); + c.schema_version = 1; + c.version = "incident-store-context/2".into(); + assert_eq!( + render_incident_request(&c, &reports(), IncidentPhase::Initial), + Err(Error::UnknownVersion) + ); +} +#[test] +fn all_context_text_and_id_invariants_apply_to_direct_construction() { + for field in [ + "workload_version", + "freeze_sha256", + "prompt_sha256", + "validator_version", + "case_id", + "question", + "system_prompt", + ] { + let mut value: serde_json::Value = + serde_json::from_slice(&context().to_json().unwrap()).unwrap(); + value[field] = " \n\t".into(); + assert_eq!( + IncidentContextV1::from_json(&serde_json::to_vec(&value).unwrap()), + Err(Error::InvalidContext) + ); + } + for ids in [ + vec![], + vec!["".into()], + vec![" \n".into()], + vec!["x".into(), "x".into()], + ] { + for owner in [true, false] { + let mut c = context(); + if owner { + c.owner_ids = ids.clone() + } else { + c.evidence_ids = ids.clone() + } + assert_eq!(c.to_json(), Err(Error::InvalidContext)); + assert_eq!( + render_incident_request(&c, &reports(), IncidentPhase::Initial), + Err(Error::InvalidContext) + ); + assert_eq!( + validate_incident_output(ROUTE, &c), + Err(Error::InvalidContext) + ); + } + } +} +#[test] +fn report_ids_and_order_must_exactly_match_context() { + let original = reports(); + for bad in [ + vec![], + vec![original[0].clone()], + vec![original[1].clone(), original[0].clone()], + vec![original[0].clone(), original[0].clone()], + ] { + assert_eq!( + render_incident_request(&context(), &bad, IncidentPhase::Initial), + Err(Error::InvalidContext) + ); + } + let mut bad = reports(); + bad[0].text = " \n".into(); + assert_eq!( + render_incident_request(&context(), &bad, IncidentPhase::Initial), + Err(Error::InvalidContext) + ); +} +#[test] +fn protocol_accepts_wrong_semantics_and_non_gold_citation_subset() { + let raw = + br#"{"decision":"route","owner":"b","reason":"diagnosed_component","evidence_ids":["e2"]}"#; + let output = validate_incident_output(raw, &context()).unwrap(); + assert_eq!(output.owner.as_deref(), Some("b")); + assert_eq!(output.evidence_ids, vec!["e2"]); + assert_eq!(output.decision, "route"); + assert_eq!(output.reason, "diagnosed_component"); +} +#[test] +fn all_abstention_reasons_and_citation_order_are_allowed() { + for reason in [ + "undetermined_cause", + "conflicting_diagnoses", + "owner_not_unique", + ] { + let raw = format!( + r#"{{"decision":"abstain","owner":null,"reason":"{reason}","evidence_ids":["e2","e1"]}}"# + ); + assert!(validate_incident_output(raw.as_bytes(), &context()).is_ok()); + } +} +#[test] +fn invalid_output_contract_refuses() { + let base = String::from_utf8(ROUTE.to_vec()).unwrap(); + for bad in [ + base.replace("route", "other"), + base.replace("diagnosed_component", "undetermined_cause"), + base.replace(r#""owner":"a""#, r#""owner":null"#), + base.replace(r#""owner":"a","#, ""), + base.replace(r#""owner":"a""#, r#""owner":"unknown""#), + base.replace(r#"["e1","e2"]"#, "[]"), + base.replace(r#"["e1","e2"]"#, r#"["e1","e1"]"#), + base.replace(r#"["e1","e2"]"#, r#"["unknown"]"#), + base.replace(r#"["e1","e2"]"#, "[1]"), + base.replacen('{', r#"{"extra":1,"#, 1), + base.replacen('{', r#"{"decision":"route","#, 1), + base.replace("route", "abstain"), + "[]".into(), + "null".into(), + ] { + assert!( + validate_incident_output(bad.as_bytes(), &context()).is_err(), + "{bad}" + ); + } +} +#[test] +fn invalid_utf8_syntax_nonfinite_and_error_text_are_payload_free() { + for bad in [ + vec![0xff], + b"SECRET not JSON".to_vec(), + br#"{"decision":NaN}"#.to_vec(), + br#"{"decision":"\ud800"}"#.to_vec(), + ] { + let error = validate_incident_output(&bad, &context()).unwrap_err(); + assert!(!error.to_string().contains("SECRET")); + assert!(!format!("{error:?}").contains("SECRET")); + } + assert_eq!( + validate_incident_output(&[0xff], &context()), + Err(Error::InvalidEncoding) + ); +} +#[test] +fn container_depth_counts_strings_and_enforces_64_65_boundary() { + for depth in [64, 65] { + let raw = format!("{}0{}", "[".repeat(depth), "]".repeat(depth)); + assert_eq!( + validate_incident_output(raw.as_bytes(), &context()), + Err(if depth == 64 { + Error::InvalidOutput + } else { + Error::InvalidJson + }) + ); + assert_eq!( + IncidentContextV1::from_json(raw.as_bytes()), + Err(if depth == 64 { + Error::InvalidContext + } else { + Error::InvalidJson + }) + ); + } + let mut c = context(); + c.question = "[{}]\\\"".repeat(100); + assert!(render_incident_request(&c, &reports(), IncidentPhase::Initial).is_ok()); +} +#[test] +fn reply_size_limit_includes_whitespace_and_allows_exactly_one_mib() { + let mut raw = ROUTE.to_vec(); + raw.resize(1024 * 1024, b' '); + assert!(validate_incident_output(&raw, &context()).is_ok()); + raw.push(b' '); + assert_eq!( + validate_incident_output(&raw, &context()), + Err(Error::TooLarge) + ); +} +#[test] +fn context_and_rendered_request_have_independent_size_limits() { + let mut raw = context().to_json().unwrap(); + raw.resize(1024 * 1024, b' '); + assert!(IncidentContextV1::from_json(&raw).is_ok()); + raw.push(b' '); + assert_eq!(IncidentContextV1::from_json(&raw), Err(Error::TooLarge)); + let mut c = context(); + c.question = "x".repeat(1024 * 1024); + assert_eq!(c.to_json(), Err(Error::TooLarge)); + assert_eq!( + render_incident_request(&c, &reports(), IncidentPhase::Initial), + Err(Error::TooLarge) + ); + let mut r = reports(); + r[0].text = "x".repeat(1024 * 1024); + assert_eq!( + render_incident_request(&context(), &r, IncidentPhase::Initial), + Err(Error::TooLarge) + ); +} + +// These literals were produced independently by the unchanged frozen Python v1 +// renderer, not by the Rust renderer or its helpers. +#[test] +fn all_frozen_initial_requests_match_python_byte_digests() { + use sha2::{Digest, Sha256}; + let prompt: serde_json::Value = serde_json::from_slice(include_bytes!( + "../workloads/incident-routing-v1/prompt.json" + )) + .unwrap(); + let cases: serde_json::Value = serde_json::from_slice(include_bytes!( + "../workloads/incident-routing-v1/cases.json" + )) + .unwrap(); + let expected = [ + ( + "cal-01", + "7600121b3d4b7c69fdb12b3f7ceb9acaf3060953fd2f82cb68a8413ac6d7affe", + ), + ( + "cal-02", + "59d5b5c677fd93125f3e70a5174ddc3ee6a4c7c4ca0ce32379333eb22d2988c8", + ), + ( + "cal-03", + "847df04fe5b8b1501366fab4199eba7ce7bb2d7b1d3cc7b95521522bac04f35b", + ), + ( + "cal-04", + "fb8853428685997a53e6f0454d8a6c08d53787e2c7e2b7a2e3ef8a987aa57b25", + ), + ( + "hold-01", + "66c456316a9d96738521be91139fb7c740a5545f54bc64e05b47354bb2e9aa38", + ), + ( + "hold-02", + "e0b6960f8864fe981e1bcb3523e22360cddbfd6c0b80a08f9f71fa6262f85b20", + ), + ( + "hold-03", + "1e32fee2527f579552f3a1047580108976728d32782841b522b1403ef65f284a", + ), + ( + "hold-04", + "a7adac34f6cdb484a22abf2eb84f2b4b30b2ac9234a84c4e13312080900449fe", + ), + ( + "hold-05", + "999cedbd55ee757a642e1e21cae9ead0155f6f840150bfc465be1a2cf281386a", + ), + ( + "hold-06", + "47ff0f1778722ec4e1a0be16c7c144f78bc510235eaddb67153a694fa1ca7e48", + ), + ( + "hold-07", + "cdedc2d6338519c0aebe911afe5eef4dabcb7091502b48b99df85c03afb7f45a", + ), + ( + "hold-08", + "bb63c5e11f46c9dede7858160b5e1a7e4af1a1951e5dd6ac91c8a024dd821106", + ), + ]; + let rows = cases["cases"].as_array().unwrap(); + assert_eq!(rows.len(), expected.len()); + for (row, (id, digest)) in rows.iter().zip(expected) { + let mut c = context(); + c.case_id = row["case_id"].as_str().unwrap().into(); + assert_eq!(c.case_id, id); + c.system_prompt = prompt["system"].as_str().unwrap().into(); + c.question = row["question"].as_str().unwrap().into(); + c.owner_ids = serde_json::from_value(row["owner_ids"].clone()).unwrap(); + let r: Vec<_> = row["evidence"] + .as_array() + .unwrap() + .iter() + .map(|e| IncidentReport { + external_id: e["id"].as_str().unwrap().into(), + text: e["text"].as_str().unwrap().into(), + }) + .collect(); + c.evidence_ids = r.iter().map(|e| e.external_id.clone()).collect(); + let request = render_incident_request(&c, &r, IncidentPhase::Initial).unwrap(); + assert_eq!( + Sha256::digest(&request) + .iter() + .map(|b| format!("{b:02x}")) + .collect::(), + digest, + "{id}" + ); + } +} + +#[test] +fn python_strip_control_whitespace_is_empty_text() { + let mut c = context(); + c.question = "\u{1c}\u{1d}\u{1e}\u{1f}".into(); + assert_eq!(c.to_json(), Err(Error::InvalidContext)); +} From ef42bcf589645d113111337b89806b26e04fa300 Mon Sep 17 00:00:00 2001 From: Alejandro Castellanos <5405628+k2jac9@users.noreply.github.com> Date: Sun, 15 Nov 2026 15:12:19 -0500 Subject: [PATCH 08/51] fix: bound incident protocol allocation before size refusal --- src/incident_protocol.rs | 117 +++++++++++++++++++++++++++++++++++-- tests/incident_protocol.rs | 112 +++++++++++++++++++++++++++++++++++ 2 files changed, 223 insertions(+), 6 deletions(-) diff --git a/src/incident_protocol.rs b/src/incident_protocol.rs index a0e2214..61d5b92 100644 --- a/src/incident_protocol.rs +++ b/src/incident_protocol.rs @@ -3,6 +3,7 @@ use serde::{Deserialize, Serialize}; use std::collections::HashSet; use std::fmt; +use std::io::{self, Write}; const MAX_BYTES: usize = 1024 * 1024; const MAX_DEPTH: usize = 64; @@ -103,6 +104,29 @@ impl IncidentContextV1 { } fn validate(&self) -> Result<(), Error> { + // Borrowed lower bounds reject unbounded caller inputs before text + // scanning, ID deduplication, or owned JSON intermediates are allocated. + let mut size = 0; + for text in [ + &self.version, + &self.workload_version, + &self.freeze_sha256, + &self.prompt_sha256, + &self.validator_version, + &self.case_id, + &self.question, + &self.system_prompt, + ] { + add_size(&mut size, text.len())?; + } + for ids in [&self.owner_ids, &self.evidence_ids] { + // Each JSON array string needs two quotes and a separator, except + // the last separator is replaced by the array's closing bracket. + add_size(&mut size, ids.len().checked_mul(3).ok_or(Error::TooLarge)?)?; + for id in ids { + add_size(&mut size, id.len())?; + } + } if self.schema_version != 1 || self.version != "incident-store-context/1" { return Err(Error::UnknownVersion); } @@ -163,6 +187,14 @@ pub fn render_incident_request( phase: IncidentPhase, ) -> Result, Error> { context.validate()?; + // Avoid allocating a JSON value from arbitrarily large report text. This + // lower bound is followed by the exact bounded serialization below. + let mut size = 0; + add_size(&mut size, reports.len())?; + for report in reports { + add_size(&mut size, report.external_id.len())?; + add_size(&mut size, report.text.len())?; + } if reports.len() != context.evidence_ids.len() || reports .iter() @@ -238,14 +270,56 @@ fn unique_nonempty(ids: &[String]) -> bool { let mut seen = HashSet::new(); !ids.is_empty() && ids.iter().all(|s| nonempty(s) && seen.insert(s)) } -fn canonical(value: &serde_json::Value) -> Result, Error> { - // serde_json's default Map is a BTreeMap: sorted keys, compact separators, - // literal UTF-8, and Python-compatible string escaping. - let raw = serde_json::to_vec(value).map_err(|_| Error::InvalidJson)?; - if raw.len() > MAX_BYTES { +fn add_size(total: &mut usize, additional: usize) -> Result<(), Error> { + let next = total.checked_add(additional).ok_or(Error::TooLarge)?; + if next > MAX_BYTES { return Err(Error::TooLarge); } - Ok(raw) + *total = next; + Ok(()) +} + +#[derive(Default)] +struct JsonBytes { + bytes: Vec, + exceeded: bool, +} +impl Write for JsonBytes { + fn write(&mut self, bytes: &[u8]) -> io::Result { + let mut next = self.bytes.len(); + if add_size(&mut next, bytes.len()).is_err() { + self.exceeded = true; + return Err(io::Error::other("incident JSON size limit exceeded")); + } + if next > self.bytes.capacity() { + // Vec's normal geometric growth may exceed the byte cap. Reserve + // explicitly, capping both stored bytes and requested capacity. + let capacity = next + .max(self.bytes.capacity().saturating_mul(2)) + .min(MAX_BYTES); + self.bytes.reserve_exact(capacity - self.bytes.len()); + } + self.bytes.extend_from_slice(bytes); + Ok(bytes.len()) + } + fn flush(&mut self) -> io::Result<()> { + Ok(()) + } +} +fn canonical(value: &serde_json::Value) -> Result, Error> { + // The borrowed lower-bound preflights bound owned Value construction. This + // writer additionally bounds escaping expansion before buffer allocation. + // serde_json's default Map supplies sorted keys, compact UTF-8 JSON and + // Python-compatible escaping as before. + let mut output = JsonBytes::default(); + if serde_json::to_writer(&mut output, value).is_err() { + return Err(if output.exceeded { + Error::TooLarge + } else { + Error::InvalidJson + }); + } + Ok(output.bytes) } fn decode<'a, T: Deserialize<'a>>(raw: &'a [u8], invalid: Error) -> Result { if raw.len() > MAX_BYTES { @@ -287,3 +361,34 @@ fn check_depth(source: &str) -> Result<(), Error> { } Ok(()) } + +#[cfg(test)] +mod resource_tests { + use super::*; + + #[test] + fn bounded_json_writer_accepts_cap_and_refuses_without_appending_or_growth() { + let mut output = JsonBytes::default(); + output.write_all(&vec![b'x'; MAX_BYTES - 1]).unwrap(); + output.write_all(b"y").unwrap(); + assert_eq!(output.bytes.len(), MAX_BYTES); + assert!(output.bytes.capacity() <= MAX_BYTES); + let capacity = output.bytes.capacity(); + assert!(output.write_all(b"rejected bytes").is_err()); + assert!(output.exceeded); + assert_eq!(output.bytes.len(), MAX_BYTES); + assert_eq!(output.bytes.capacity(), capacity); + assert_eq!(output.bytes.last(), Some(&b'y')); + } + + #[test] + fn preflight_size_rejects_overflow_and_cap_without_mutating_count() { + let mut size = 1; + assert_eq!(add_size(&mut size, usize::MAX), Err(Error::TooLarge)); + assert_eq!(size, 1); + assert_eq!(add_size(&mut size, MAX_BYTES), Err(Error::TooLarge)); + assert_eq!(size, 1); + add_size(&mut size, MAX_BYTES - 1).unwrap(); + assert_eq!(size, MAX_BYTES); + } +} diff --git a/tests/incident_protocol.rs b/tests/incident_protocol.rs index dc8e96e..47bb068 100644 --- a/tests/incident_protocol.rs +++ b/tests/incident_protocol.rs @@ -1,3 +1,55 @@ +// Observe only allocations on this test thread, after oversized caller inputs +// have been constructed. Other parallel tests cannot affect the measurement. +use std::alloc::{GlobalAlloc, Layout, System}; +use std::cell::Cell; + +thread_local! { + static MEASURE: Cell = const { Cell::new(false) }; + static LARGEST: Cell = const { Cell::new(0) }; +} +struct AllocationObserver; +fn observe(size: usize) { + let _ = MEASURE.try_with(|active| { + if active.get() { + let _ = LARGEST.try_with(|largest| largest.set(largest.get().max(size))); + } + }); +} +unsafe impl GlobalAlloc for AllocationObserver { + unsafe fn alloc(&self, layout: Layout) -> *mut u8 { + observe(layout.size()); + unsafe { System.alloc(layout) } + } + unsafe fn alloc_zeroed(&self, layout: Layout) -> *mut u8 { + observe(layout.size()); + unsafe { System.alloc_zeroed(layout) } + } + unsafe fn realloc(&self, ptr: *mut u8, layout: Layout, size: usize) -> *mut u8 { + observe(size); + unsafe { System.realloc(ptr, layout, size) } + } + unsafe fn dealloc(&self, ptr: *mut u8, layout: Layout) { + unsafe { System.dealloc(ptr, layout) } + } +} +#[global_allocator] +static ALLOCATOR: AllocationObserver = AllocationObserver; + +fn largest_allocation_during(f: impl FnOnce() -> T) -> (T, usize) { + struct Stop; + impl Drop for Stop { + fn drop(&mut self) { + MEASURE.with(|active| active.set(false)); + } + } + LARGEST.with(|largest| largest.set(0)); + MEASURE.with(|active| active.set(true)); + let stop = Stop; + let result = f(); + drop(stop); + (result, LARGEST.with(Cell::get)) +} + use kairotensor::incident_protocol::{ render_incident_request, validate_incident_output, IncidentContextV1, IncidentPhase, IncidentProtocolError as Error, IncidentReport, @@ -390,3 +442,63 @@ fn python_strip_control_whitespace_is_empty_text() { c.question = "\u{1c}\u{1d}\u{1e}\u{1f}".into(); assert_eq!(c.to_json(), Err(Error::InvalidContext)); } + +// Catch a late TooLarge check that first clones the caller's oversized String, +// grows a serializer without a bound, or builds a large ID-deduplication table. +#[test] +fn direct_oversize_refuses_without_input_sized_intermediate_allocations() { + const LIMIT: usize = 1024 * 1024; + let mut c = context(); + c.question = "x".repeat(8 * LIMIT); + for call in [0, 1, 2] { + let (result, largest) = largest_allocation_during(|| match call { + 0 => c.to_json().map(|_| ()), + 1 => render_incident_request(&c, &reports(), IncidentPhase::Initial).map(|_| ()), + _ => validate_incident_output(ROUTE, &c).map(|_| ()), + }); + assert_eq!(result, Err(Error::TooLarge)); + assert!( + largest <= LIMIT, + "context entry {call} allocated {largest} bytes before refusal" + ); + } + let mut r = reports(); + r[0].text = "x".repeat(8 * LIMIT); + let c = context(); + let (result, largest) = + largest_allocation_during(|| render_incident_request(&c, &r, IncidentPhase::Initial)); + assert_eq!(result, Err(Error::TooLarge)); + assert!( + largest <= LIMIT, + "report allocated {largest} bytes before refusal" + ); + + let mut c = context(); + c.question = "\0".repeat(LIMIT / 2); + let (result, largest) = largest_allocation_during(|| c.to_json()); + assert_eq!(result, Err(Error::TooLarge)); + assert!( + largest <= LIMIT, + "escaped context allocated {largest} bytes before refusal" + ); + + let mut r = reports(); + r[0].text = "\0".repeat(LIMIT / 2); + let c = context(); + let (result, largest) = + largest_allocation_during(|| render_incident_request(&c, &r, IncidentPhase::Initial)); + assert_eq!(result, Err(Error::TooLarge)); + assert!( + largest <= LIMIT, + "escaped report allocated {largest} bytes before refusal" + ); + + let mut c = context(); + c.owner_ids = (0..65536).map(|i| format!("{i:032}")).collect(); + let (result, largest) = largest_allocation_during(|| c.to_json()); + assert_eq!(result, Err(Error::TooLarge)); + assert!( + largest <= LIMIT, + "aggregate IDs allocated {largest} bytes before refusal" + ); +} From 94a6d439b0300fe95f2e96afb790293e32cb8590 Mon Sep 17 00:00:00 2001 From: Alejandro Castellanos <5405628+k2jac9@users.noreply.github.com> Date: Sun, 15 Nov 2026 15:26:03 -0500 Subject: [PATCH 09/51] feat: exercise incident preparation through Store timed gates --- docs/incident-store-bridge.md | 167 +++++ examples/incident_store_bridge.rs | 9 + examples/incident_store_bridge/adapter.rs | 548 ++++++++++++++++ examples/incident_store_bridge/tests.rs | 726 ++++++++++++++++++++++ 4 files changed, 1450 insertions(+) create mode 100644 docs/incident-store-bridge.md create mode 100644 examples/incident_store_bridge.rs create mode 100644 examples/incident_store_bridge/adapter.rs create mode 100644 examples/incident_store_bridge/tests.rs diff --git a/docs/incident-store-bridge.md b/docs/incident-store-bridge.md new file mode 100644 index 0000000..c341157 --- /dev/null +++ b/docs/incident-store-bridge.md @@ -0,0 +1,167 @@ +# Store-backed incident fixture + +`examples/incident_store_bridge.rs` composes the existing timed Store recovery +APIs with the pure incident protocol. It makes zero model or provider calls. +The canned backend counts UTF-8 bytes as fixture tokens; its timing, CPU identity, +and calibration are synthetic, not measured performance evidence. + +Run the example and focused gates with the admitted project Cargo wrapper: + +```sh +CARGO_PROFILE_DEV_DEBUG=0 CARGO_PROFILE_TEST_DEBUG=0 CARGO_INCREMENTAL=0 \ + /home/neo/.local/bin/kairotensor-cargo --budget-gib 2 test --locked \ + --example incident_store_bridge --test timed_recovery --test recovery_runtime +CARGO_PROFILE_DEV_DEBUG=0 CARGO_PROFILE_TEST_DEBUG=0 CARGO_INCREMENTAL=0 \ + /home/neo/.local/bin/kairotensor-cargo --budget-gib 2 run --locked \ + --example incident_store_bridge +``` + +The example test's independent initial-request oracle invokes `python3` and +reads only frozen cases and prompt bytes. It compares the complete request bytes +with Python's sorted, compact, non-ASCII-escaped v1 JSON encoding. Retry bytes +have a separate exact comparison. + +## Contract and capture + +The example-local `IncidentContract` has private fields. Its constructor verifies +the embedded freeze against SHA-256 +`f71732c193ff889f24709a11855b59820a657c21c2b43ec721cddfbff1ede2c8`, then checks +the embedded cases, prompt and contract against the pinned manifest digests. +It derives the complete required report set from the selected frozen case. +It never loads `labels.json`; the freeze's label digest is integrity metadata, +not label contents. Family, split and eligibility flags are not context fields. + +The caller supplies a context occurrence, its exact canonical context bytes, +ordered source mappings, explicit runtime identity and output cap. Each mapping +is keyed by workload version, case ID and external report ID. The constructor +rejects incomplete, duplicate, cross-case, reordered or aliased mappings and +contexts that differ from the pinned case. A context occurrence cannot also be a +report occurrence. Context is always candidate position zero. + +`incident-store-manifest/1` is sorted-key compact UTF-8 JSON containing: + +- Schema and manifest versions; workload, frozen-workload, prompt and validator identities. +- Context occurrence ID and exact-byte SHA-256. +- Ordered complete source mappings with occurrence IDs and payload SHA-256 values. +- Output cap and the complete runtime identity except `prompt_fingerprint`. +- Initial renderer `incident-request/1`, retry renderer `incident-request-retry/1`, + and fixed feedback `incident-routing-protocol/1:invalid_output`. + +The runtime fingerprint is derived from those exact manifest bytes. Excluding +the fingerprint itself avoids a self-referential hash. Hashes bind identities; +they do not grant admission or prove truth, execution or authority. There are at +most 1,024 input occurrences, including context. The pure protocol bounds +context, rendered request and reply validation to 1 MiB and JSON depth to 64. + +For replay, the exact object shape below is normative; angle-bracket strings +describe values, not literal manifest contents. Serialize recursively with +lexicographically sorted object keys, compact `,` and `:` separators, UTF-8, +no ASCII escaping of Unicode, and no trailing newline. Arrays retain input +order. Numeric fields are JSON integers, never floats or strings. Digests are +64 lowercase hexadecimal characters over exact bytes. + +```json +{ + "context": {"occurrence_id": 1, "sha256": ""}, + "freeze_sha256": "", + "initial_renderer": "incident-request/1", + "output_cap": 512, + "prompt_sha256": "", + "retry_feedback": "incident-routing-protocol/1:invalid_output", + "retry_renderer": "incident-request-retry/1", + "runtime": { + "cpu_fingerprint": "", + "decoding": "", + "dtype": "", + "model": "", + "model_revision": "", + "readiness": "warm", + "runtime_version": "", + "threads": 1, + "tokenizer_revision": "" + }, + "schema_version": 1, + "sources": [ + { + "case_id": "", + "external_id": "", + "occurrence_id": 2, + "payload_sha256": "", + "workload_version": "incident-routing/1" + } + ], + "validator_version": "incident-routing-protocol/1", + "version": "incident-store-manifest/1", + "workload_version": "incident-routing/1" +} +``` + +Use the actual captured occurrence IDs, not the illustrative `1` and `2`. +There is one `sources` object for every input after context, with no omissions. +Its case ID and external IDs derive from the captured context, verified against +the pinned case; its occurrence IDs and hashes derive from ordered captured +inputs. Context SHA-256 hashes the raw stored context bytes, not a re-encoding. +Report SHA-256 hashes the raw report text bytes, not a JSON string or report +wrapper. Freeze and prompt digests come from the verified captured context. +Workload/validator/renderer versions and feedback are the fixed literals shown. +The runtime values come from `RecoveryTiming.runtime`, with only +`prompt_fingerprint` removed, and output cap comes from the bound calibration +cap. Readiness is the actual lowercase `warm` or `cold` value; all other runtime +fields are preserved exactly. This fixture uses cap 512 and threads 1, but these +are bound supplied values, not manifest constants. Reconstructed manifest +SHA-256 must equal the timing runtime's original `prompt_fingerprint`. + +Initial attempt ordinal 0 has no retry reason and uses `IncidentPhase::Initial`. +Retry ordinal 1 is available only after invalid output, has a retry reason and +uses `IncidentPhase::ProtocolRetry`. Both phases share the exact same manifest. +Request rendering retains the captured system prompt and ordered reports; the +retry adds `validation_error` with the fixed feedback to the user payload before +its compact canonical encoding. No stored free-text reason enters the request. + +`prepare_incident_attempt` checks the complete candidate order and immutable +timing runtime/cap/manifest binding before calling `begin_recovery_attempt`. +Only `RecoveryStart::New` provides a preparation opportunity. The timed Store +path captures every required admitted occurrence atomically. Unknown and +unadmitted occurrences return the same payload-free `UnavailableEvidence`; +other Store corruption and I/O errors remain `BridgeError::Store`. + +After capture, preparation checks every captured ID, decodes the captured +context, hashes the captured context and every report, reconstructs the whole +manifest, and compares it with the original immutable timing fingerprint. +Only then does it render captured report text. It does not render caller caches. +An error cancels only the newly allocated unsent attempt. Cancellation refusal +or SQL failure preserves both the original error and typed cancellation failure +in `CancellationFailed`; claimed holds remain conservative. + +## Reservation, dispatch and retry + +Preparation does not reserve budget or invoke a backend. The example separately +calls `reserve_recovery_attempt`, then `run_timed_recovery` with deterministic +clocks and a counted canned closure. Reply format validation uses only context +decoded from the capture. Usage reports include both request and reply bytes. + +An admission correction before reservation prevents a hold. A correction after +reservation and before claim prevents dispatch. A correction through another +Store connection inside the backend records stale publication and charges full +measured usage. Post-claim expiry invokes no backend and preserves the hold; +backend uncertainty or settlement failure likewise preserves it. No repeat run +can claim twice. Duplicate settlement does not charge twice. + +Reopening a reserved or sent recovery returns `ExistingRecovery`, never another +prepared request. The restart test retains its original in-process preparation +while reopening the Store; this example does not add a preparation reconstruction +or resend API for crashed callers. Finished, denied and exhausted recoveries +return `TerminalRecovery`. + +After invalid output, the existing Store permits at most one retry. The bridge +selects the retry renderer solely from the new attempt's `retry_reason` presence +and discards the reason text. The retry adds only the fixed `validation_error` +field. The original timing deadline and manifest remain unchanged. Different +source occurrences or runtime/cap identities cannot silently replace that +recovery's contract. + +Tests deliberately use a protocol-valid but semantically wrong canned answer. +Store acceptance here means current evidence and valid format, not correctness +against semantic labels. This work supplies no efficacy, live inference, +production calibration, remote exactly-once delivery, or all-case export claim. +All-case exports and CI integration belong to the separately scoped next task. diff --git a/examples/incident_store_bridge.rs b/examples/incident_store_bridge.rs new file mode 100644 index 0000000..f3a3c37 --- /dev/null +++ b/examples/incident_store_bridge.rs @@ -0,0 +1,9 @@ +#[path = "incident_store_bridge/adapter.rs"] +mod adapter; +#[cfg(test)] +#[path = "incident_store_bridge/tests.rs"] +mod tests; + +fn main() -> Result<(), Box> { + adapter::demo() +} diff --git a/examples/incident_store_bridge/adapter.rs b/examples/incident_store_bridge/adapter.rs new file mode 100644 index 0000000..12af195 --- /dev/null +++ b/examples/incident_store_bridge/adapter.rs @@ -0,0 +1,548 @@ +//! Offline fixture: Store admission and byte accounting, never model efficacy. +use kairotensor::incident_protocol::{ + render_incident_request, validate_incident_output, IncidentContextV1, IncidentPhase, + IncidentProtocolError, IncidentReport, +}; +use kairotensor::{ + MeasuredRecoveryReply, RecoveryCalibration, RecoveryClock, RecoveryPreparation, + RecoveryReadiness, RecoveryRun, RecoveryRuntime, RecoveryStart, RecoveryTiming, + ReplyValidation, Store, +}; +use serde_json::{json, Value}; +use sha2::{Digest, Sha256}; +use std::collections::HashSet; + +pub(super) const ACTOR: &str = "fixture-actor"; +pub(super) const SCOPE: &str = "incident-fixture"; +pub(super) const FEEDBACK: &str = "incident-routing-protocol/1:invalid_output"; +const VERSION: &str = "incident-routing/1"; +const VALIDATOR: &str = "incident-routing-protocol/1"; +const FREEZE_SHA256: &str = "f71732c193ff889f24709a11855b59820a657c21c2b43ec721cddfbff1ede2c8"; +const FREEZE: &[u8] = include_bytes!("../../workloads/incident-routing-v1/freeze.json"); +const CASES: &[u8] = include_bytes!("../../workloads/incident-routing-v1/cases.json"); +const PROMPT: &[u8] = include_bytes!("../../workloads/incident-routing-v1/prompt.json"); +const CONTRACT: &[u8] = include_bytes!("../../workloads/incident-routing-v1/contract.json"); +const MAX_INPUTS: usize = 1024; + +pub(super) fn sha256(bytes: &[u8]) -> String { + Sha256::digest(bytes) + .iter() + .map(|byte| format!("{byte:02x}")) + .collect() +} + +#[derive(Debug)] +pub(super) enum CancellationFailure { + Refused, + Store(rusqlite::Error), +} +#[derive(Debug)] +pub(super) enum BridgeError { + UnavailableEvidence, + ChangedContract, + ExistingRecovery, + TerminalRecovery, + InvalidManifest, + Protocol(IncidentProtocolError), + Store(rusqlite::Error), + // Technical-owner clarification: keep both failures without disguising SQL errors. + CancellationFailed { + original: Box, + cancellation: CancellationFailure, + }, +} +impl std::fmt::Display for BridgeError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Self::Protocol(error) => write!(f, "{error}"), + Self::Store(error) => write!(f, "incident store: {error}"), + Self::CancellationFailed { + original, + cancellation, + } => { + write!(f, "{original}; cancellation: ")?; + match cancellation { + CancellationFailure::Refused => f.write_str("refused; retain hold"), + CancellationFailure::Store(error) => write!(f, "{error}"), + } + } + Self::UnavailableEvidence => f.write_str("incident evidence unavailable"), + Self::ChangedContract => f.write_str("incident contract changed"), + Self::ExistingRecovery => f.write_str("incident recovery already exists"), + Self::TerminalRecovery => f.write_str("incident recovery finished"), + Self::InvalidManifest => f.write_str("invalid incident manifest"), + } + } +} +impl std::error::Error for BridgeError {} +impl From for BridgeError { + fn from(error: rusqlite::Error) -> Self { + Self::Store(error) + } +} +impl From for BridgeError { + fn from(error: IncidentProtocolError) -> Self { + Self::Protocol(error) + } +} +type Result = std::result::Result; + +fn decode(raw: &[u8]) -> Result { + serde_json::from_slice(raw).map_err(|_| BridgeError::InvalidManifest) +} +fn text(value: &Value) -> Result { + value + .as_str() + .map(str::to_owned) + .ok_or(BridgeError::InvalidManifest) +} +fn strings(value: &Value) -> Result> { + value + .as_array() + .ok_or(BridgeError::InvalidManifest)? + .iter() + .map(text) + .collect() +} + +/// Trusted input loader: hashes are checked before parsing. Labels are never loaded. +pub(super) fn pinned_case(case_id: &str) -> Result<(IncidentContextV1, Vec)> { + if sha256(FREEZE) != FREEZE_SHA256 { + return Err(BridgeError::InvalidManifest); + } + let freeze = decode(FREEZE)?; + for (name, raw) in [ + ("cases.json", CASES), + ("prompt.json", PROMPT), + ("contract.json", CONTRACT), + ] { + if freeze["sha256"][name].as_str() != Some(sha256(raw).as_str()) { + return Err(BridgeError::InvalidManifest); + } + } + let cases = decode(CASES)?; + let prompt = decode(PROMPT)?; + let contract = decode(CONTRACT)?; + for document in [&cases, &prompt, &contract] { + if document["version"] != VERSION || document["schema_version"] != 1 { + return Err(BridgeError::InvalidManifest); + } + } + let case = cases["cases"] + .as_array() + .ok_or(BridgeError::InvalidManifest)? + .iter() + .find(|case| case["case_id"].as_str() == Some(case_id)) + .ok_or(BridgeError::InvalidManifest)?; + let reports: Vec<_> = case["evidence"] + .as_array() + .ok_or(BridgeError::InvalidManifest)? + .iter() + .map(|report| { + Ok(IncidentReport { + external_id: text(&report["id"])?, + text: text(&report["text"])?, + }) + }) + .collect::>()?; + let context = IncidentContextV1 { + schema_version: 1, + version: "incident-store-context/1".into(), + workload_version: VERSION.into(), + freeze_sha256: FREEZE_SHA256.into(), + prompt_sha256: sha256(PROMPT), + validator_version: VALIDATOR.into(), + case_id: case_id.into(), + question: text(&case["question"])?, + owner_ids: strings(&case["owner_ids"])?, + evidence_ids: reports.iter().map(|r| r.external_id.clone()).collect(), + system_prompt: text(&prompt["system"])?, + }; + context.to_json()?; + Ok((context, reports)) +} + +#[derive(Clone, Debug)] +pub(super) struct SourceMapping { + pub(super) workload_version: String, + pub(super) case_id: String, + pub(super) external_id: String, + pub(super) occurrence: i64, +} + +/// Private fields prevent a caller from replacing the required set or cached bytes. +pub(super) struct IncidentContract { + context: IncidentContextV1, + context_id: i64, + context_sha256: String, + mappings: Vec, + report_sha256: Vec, + runtime: RecoveryRuntime, + output_cap: i64, + manifest_sha256: String, +} +impl IncidentContract { + pub(super) fn new( + case_id: &str, + context_id: i64, + context_bytes: &[u8], + mappings: Vec, + mut runtime: RecoveryRuntime, + output_cap: i64, + ) -> Result { + let (context, reports) = pinned_case(case_id)?; + if mappings.len() + 1 > MAX_INPUTS + || mappings.len() != reports.len() + || output_cap <= 0 + || context_id <= 0 + { + return Err(BridgeError::InvalidManifest); + } + // Public codec bounds the caller bytes before parsing; byte identity is exact. + if IncidentContextV1::from_json(context_bytes)? != context + || context_bytes != context.to_json()? + { + return Err(BridgeError::ChangedContract); + } + let mut seen = HashSet::from([context_id]); + for (mapping, report) in mappings.iter().zip(&reports) { + if mapping.workload_version != VERSION + || mapping.case_id != case_id + || mapping.external_id != report.external_id + || mapping.occurrence <= 0 + || !seen.insert(mapping.occurrence) + { + return Err(BridgeError::InvalidManifest); + } + } + // The supplied fingerprint is never authority; the exact manifest derives it. + runtime.prompt_fingerprint.clear(); + let mut result = Self { + context, + context_id, + context_sha256: sha256(context_bytes), + mappings, + report_sha256: reports.iter().map(|r| sha256(r.text.as_bytes())).collect(), + runtime, + output_cap, + manifest_sha256: String::new(), + }; + result.manifest_sha256 = + sha256(&result.manifest(&result.context_sha256, &result.report_sha256)?); + result.runtime.prompt_fingerprint = result.manifest_sha256.clone(); + Ok(result) + } + + fn ids(&self) -> Vec { + std::iter::once(self.context_id) + .chain(self.mappings.iter().map(|m| m.occurrence)) + .collect() + } + + fn manifest(&self, context_digest: &str, report_digests: &[String]) -> Result> { + if self.mappings.len() + 1 > MAX_INPUTS || report_digests.len() != self.mappings.len() { + return Err(BridgeError::InvalidManifest); + } + let mut runtime = + serde_json::to_value(&self.runtime).map_err(|_| BridgeError::InvalidManifest)?; + runtime + .as_object_mut() + .ok_or(BridgeError::InvalidManifest)? + .remove("prompt_fingerprint"); + let mappings: Vec<_> = self.mappings.iter().zip(report_digests).map(|(m, digest)| json!({ + "workload_version":m.workload_version, "case_id":m.case_id, + "external_id":m.external_id, "occurrence_id":m.occurrence, "payload_sha256":digest, + })).collect(); + serde_json::to_vec(&json!({ + "version":"incident-store-manifest/1", "schema_version":1, + "context":{"occurrence_id":self.context_id,"sha256":context_digest}, + "sources":mappings, "workload_version":VERSION, + "freeze_sha256":self.context.freeze_sha256,"prompt_sha256":self.context.prompt_sha256, + "validator_version":VALIDATOR, "initial_renderer":"incident-request/1", + "retry_renderer":"incident-request-retry/1", "retry_feedback":FEEDBACK, + "output_cap":self.output_cap, "runtime":runtime, + })) + .map_err(|_| BridgeError::InvalidManifest) + } + + #[cfg(test)] + pub(super) fn mappings_for_test(&self) -> Vec { + self.mappings.clone() + } +} + +#[derive(Debug)] +pub(super) struct IncidentPreparation { + pub(super) attempt: i64, + pub(super) request: Vec, + pub(super) captured_ids: Vec, + pub(super) manifest_sha256: String, + pub(super) preparation: RecoveryPreparation, + // Only context decoded from the captured occurrence reaches reply validation. + context: IncidentContextV1, +} +impl IncidentPreparation { + pub(super) fn measure(&self, request: &[u8], output: &[u8]) -> MeasuredRecoveryReply { + let validation = match validate_incident_output(output, &self.context) { + Ok(_) => ReplyValidation::Valid, + Err(_) => ReplyValidation::Invalid { + validator: VALIDATOR.into(), + reason: FEEDBACK.into(), + }, + }; + MeasuredRecoveryReply { + output: output.to_vec(), + input_tokens: request.len() as i64, + output_tokens: output.len() as i64, + validation, + } + } +} + +/// No reservation or backend invocation occurs here. New captures alone can prepare. +pub(super) fn prepare_incident_attempt( + store: &mut Store, + recovery: i64, + candidates: &[i64], + contract: &IncidentContract, +) -> Result { + if candidates.len() > MAX_INPUTS || candidates != contract.ids() { + return Err(BridgeError::InvalidManifest); + } + let timing = store + .recovery_timing(recovery)? + .ok_or(BridgeError::ChangedContract)?; + let manifest_hash = + sha256(&contract.manifest(&contract.context_sha256, &contract.report_sha256)?); + if manifest_hash != contract.manifest_sha256 + || timing.runtime != contract.runtime + || timing.runtime.prompt_fingerprint != manifest_hash + || timing.calibration.output_cap != contract.output_cap + { + return Err(BridgeError::ChangedContract); + } + let start = store + .begin_recovery_attempt(recovery, candidates) + .map_err(|error| { + // Match just the documented complete-capture refusal, never arbitrary SQL failures. + match &error { + rusqlite::Error::SqliteFailure(code, Some(message)) + if code.extended_code == rusqlite::ffi::SQLITE_CONSTRAINT + && message == "candidate evidence is incomplete or unavailable" => + { + BridgeError::UnavailableEvidence + } + _ => BridgeError::Store(error), + } + })?; + match start { + RecoveryStart::Existing(_) => Err(BridgeError::ExistingRecovery), + RecoveryStart::Finished => Err(BridgeError::TerminalRecovery), + RecoveryStart::New { + attempt, + inputs, + retry_reason, + } => { + let result = + prepare_captured(attempt, &inputs, retry_reason.is_some(), contract, &timing); + result.map_err(|error| cancel_new(store, attempt, error)) + } + } +} + +pub(super) fn cancel_new(store: &mut Store, attempt: i64, original: BridgeError) -> BridgeError { + match store.cancel_recovery_attempt(attempt) { + Ok(true) => original, + Ok(false) => BridgeError::CancellationFailed { + original: Box::new(original), + cancellation: CancellationFailure::Refused, + }, + Err(error) => BridgeError::CancellationFailed { + original: Box::new(original), + cancellation: CancellationFailure::Store(error), + }, + } +} + +fn prepare_captured( + attempt: i64, + inputs: &[(i64, Vec)], + retry: bool, + contract: &IncidentContract, + timing: &RecoveryTiming, +) -> Result { + if inputs.len() != contract.mappings.len() + 1 + || inputs.iter().map(|(id, _)| *id).ne(contract.ids()) + { + return Err(BridgeError::InvalidManifest); + } + let context = IncidentContextV1::from_json(&inputs[0].1)?; + let context_digest = sha256(&inputs[0].1); + let report_digests: Vec<_> = inputs[1..].iter().map(|(_, bytes)| sha256(bytes)).collect(); + let manifest_hash = sha256(&contract.manifest(&context_digest, &report_digests)?); + if context != contract.context + || context_digest != contract.context_sha256 + || report_digests != contract.report_sha256 + || manifest_hash != contract.manifest_sha256 + || manifest_hash != timing.runtime.prompt_fingerprint + { + return Err(BridgeError::ChangedContract); + } + let reports: Vec<_> = inputs[1..] + .iter() + .zip(&contract.mappings) + .map(|((_, bytes), mapping)| { + Ok(IncidentReport { + external_id: mapping.external_id.clone(), + text: std::str::from_utf8(bytes) + .map_err(|_| BridgeError::Protocol(IncidentProtocolError::InvalidEncoding))? + .into(), + }) + }) + .collect::>()?; + let request = render_incident_request( + &context, + &reports, + if retry { + IncidentPhase::ProtocolRetry + } else { + IncidentPhase::Initial + }, + )?; + let preparation = RecoveryPreparation { + runtime: Some(timing.runtime.clone()), + calibration_id: Some(timing.calibration.id.clone()), + input_tokens: request.len() as i64, + output_cap: contract.output_cap, + request: request.clone(), + }; + Ok(IncidentPreparation { + attempt, + request, + captured_ids: inputs.iter().map(|(id, _)| *id).collect(), + manifest_sha256: manifest_hash, + preparation, + context, + }) +} + +pub(super) fn fixture_runtime() -> RecoveryRuntime { + RecoveryRuntime { + model: "canned-no-model".into(), + model_revision: "fixture/1".into(), + tokenizer_revision: "utf8-bytes/1".into(), + runtime_version: "incident-canned/1".into(), + dtype: "bytes".into(), + decoding: "canned-no-inference".into(), + cpu_fingerprint: sha256(b"synthetic-cpu-not-measured"), + threads: 1, + prompt_fingerprint: String::new(), + readiness: RecoveryReadiness::Warm, + } +} +pub(super) fn tick(now_ms: i64) -> rusqlite::Result { + Ok(RecoveryClock { + domain: "synthetic-incident-clock/1".into(), + now_ms, + }) +} + +pub(super) struct Fixture { + pub(super) store: Store, + pub(super) contract: IncidentContract, + pub(super) ids: Vec, + pub(super) recovery: i64, + pub(super) budget: i64, +} +impl Fixture { + pub(super) fn new(mut store: Store) -> Result { + let (context, reports) = pinned_case("cal-01")?; + let raw = context.to_json()?; + let observations: Vec<_> = std::iter::once(("incident-context", raw.as_slice())) + .chain( + reports + .iter() + .map(|r| (r.external_id.as_str(), r.text.as_bytes())), + ) + .collect(); + let ids = store.observe_and_admit_many(ACTOR, SCOPE, &observations)?; + let mappings = reports + .iter() + .zip(&ids[1..]) + .map(|(r, id)| SourceMapping { + workload_version: VERSION.into(), + case_id: context.case_id.clone(), + external_id: r.external_id.clone(), + occurrence: *id, + }) + .collect(); + let contract = IncidentContract::new( + &context.case_id, + ids[0], + &raw, + mappings, + fixture_runtime(), + 512, + )?; + let timing = RecoveryTiming { + clock_domain: "synthetic-incident-clock/1".into(), + start_ms: 100, + deadline_ms: 1000, + safety_margin_ms: 20, + runtime: contract.runtime.clone(), + calibration: RecoveryCalibration { + id: "synthetic-not-measured/1".into(), + max_duration_ms: 100, + sample_count: 2, + max_input_tokens: 1024 * 1024, + output_cap: 512, + measured_at_ms: 0, + expires_at_ms: 2000, + }, + }; + let budget = store.create_token_budget(100_000)?; + let recovery = store.create_timed_recovery( + "incident-demo/1", + budget, + ACTOR, + SCOPE, + "verdict", + &timing, + )?; + Ok(Self { + store, + contract, + ids, + recovery, + budget, + }) + } +} + +pub(super) fn demo() -> std::result::Result<(), Box> { + let mut f = Fixture::new(Store::in_memory()?)?; + let p = prepare_incident_attempt(&mut f.store, f.recovery, &f.ids, &f.contract)?; + assert!(f.store.reserve_recovery_attempt( + p.attempt, + &p.request, + p.preparation.input_tokens, + p.preparation.output_cap + )?); + let mut calls = 0; + let result = f.store.run_timed_recovery(p.attempt, &p.preparation, || tick(200), |request| { + calls += 1; + // Deliberately format-valid only: this is not a claim of semantic correctness. + Ok::<_, String>(p.measure(request, br#"{"decision":"abstain","owner":null,"reason":"undetermined_cause","evidence_ids":["e01"]}"#)) + })?; + assert!(matches!(result, RecoveryRun::Settled { .. })); + let ledger = f.store.token_budget_state(f.budget)?; + println!( + "{}", + json!({"mode":"synthetic_store_incident_fixture","new_model_calls":0, + "fixture_backend_invocations":calls,"accounting_unit":"UTF-8 bytes", + "spent":ledger.spent,"reserved":ledger.reserved,"captured_ids":p.captured_ids, + "manifest_sha256":p.manifest_sha256,"semantic_correctness_claim":false, + "calibration_measured":false}) + ); + Ok(()) +} diff --git a/examples/incident_store_bridge/tests.rs b/examples/incident_store_bridge/tests.rs new file mode 100644 index 0000000..3ea12ba --- /dev/null +++ b/examples/incident_store_bridge/tests.rs @@ -0,0 +1,726 @@ +use super::adapter::*; +use kairotensor::{Disposition, RecoveryRun, Store, TimingRefusal}; +use kairotensor::{MeasuredRecoveryReply, ReplyValidation}; +use rusqlite::Connection; +use serde_json::{json, Value}; + +// Format-valid, deliberately wrong for cal-01; no labels supply this canned output. +const REPLY: &[u8] = + br#"{"decision":"abstain","owner":null,"reason":"undetermined_cause","evidence_ids":["e01"]}"#; + +fn counts(file: &Database) -> (i64, i64, i64) { + Connection::open(&file.0).unwrap().query_row( + "SELECT (SELECT count(*) FROM attempts), (SELECT count(*) FROM token_reservations), (SELECT count(*) FROM dispatches)", + [], |r| Ok((r.get(0)?,r.get(1)?,r.get(2)?))).unwrap() +} + +#[test] +fn fixture_flags_cannot_admit_store_evidence() { + let mut f = Fixture::new(Store::in_memory().unwrap()).unwrap(); + f.store.set_admitted(ACTOR, SCOPE, f.ids[1], false).unwrap(); + let caller_flag_current = true; + assert!(caller_flag_current); + assert!(matches!( + prepare_incident_attempt(&mut f.store, f.recovery, &f.ids, &f.contract), + Err(BridgeError::UnavailableEvidence) + )); + assert_eq!(f.store.token_budget_state(f.budget).unwrap().reserved, 0); +} + +#[test] +fn missing_required_input_prevents_claim() { + let file = Database::new(); + let mut f = Fixture::new(Store::open(&file.0).unwrap()).unwrap(); + assert!(matches!( + prepare_incident_attempt(&mut f.store, f.recovery, &f.ids[..2], &f.contract), + Err(BridgeError::InvalidManifest) + )); + assert_eq!(f.store.token_budget_state(f.budget).unwrap().reserved, 0); + assert_eq!(counts(&file), (0, 0, 0)); +} + +#[test] +fn duplicate_and_cross_case_mapping_refuse() { + let f = Fixture::new(Store::in_memory().unwrap()).unwrap(); + let (context, _) = pinned_case("cal-01").unwrap(); + let raw = context.to_json().unwrap(); + let mut mappings = f.contract.mappings_for_test(); + mappings[1].occurrence = mappings[0].occurrence; + assert!( + IncidentContract::new("cal-01", f.ids[0], &raw, mappings, fixture_runtime(), 512).is_err() + ); + let mut mappings = f.contract.mappings_for_test(); + mappings[0].case_id = "cal-02".into(); + assert!( + IncidentContract::new("cal-01", f.ids[0], &raw, mappings, fixture_runtime(), 512).is_err() + ); + let mut mappings = f.contract.mappings_for_test(); + mappings[0].occurrence = f.ids[0]; + assert!( + IncidentContract::new("cal-01", f.ids[0], &raw, mappings, fixture_runtime(), 512).is_err() + ); + let mut mappings = f.contract.mappings_for_test(); + mappings.pop(); + assert!( + IncidentContract::new("cal-01", f.ids[0], &raw, mappings, fixture_runtime(), 512).is_err() + ); + let mut mappings = f.contract.mappings_for_test(); + mappings.swap(0, 1); + assert!( + IncidentContract::new("cal-01", f.ids[0], &raw, mappings, fixture_runtime(), 512).is_err() + ); + for case in ["", "unknown", "cal-02"] { + assert!(IncidentContract::new( + case, + f.ids[0], + &raw, + f.contract.mappings_for_test(), + fixture_runtime(), + 512 + ) + .is_err()); + } +} + +#[test] +fn correction_after_prepare_refuses_send() { + let file = Database::new(); + let mut f = Fixture::new(Store::open(&file.0).unwrap()).unwrap(); + let p = prepare_incident_attempt(&mut f.store, f.recovery, &f.ids, &f.contract).unwrap(); + reserve(&mut f.store, &p); + let mut other = Store::open(&file.0).unwrap(); + other.set_admitted(ACTOR, SCOPE, f.ids[1], false).unwrap(); + other + .observe_and_admit_many( + ACTOR, + SCOPE, + &[( + "e01", + b"Correction: the earlier diagnosis is withdrawn; cause remains unknown.", + )], + ) + .unwrap(); + let result = f + .store + .run_timed_recovery( + p.attempt, + &p.preparation, + || tick(200), + |_| -> Result<_, String> { panic!("stale request invoked") }, + ) + .unwrap(); + assert!(matches!( + result, + RecoveryRun::Refused { + reason: TimingRefusal::StaleEvidence, + .. + } + )); + assert_eq!(f.store.charged_usage().unwrap(), 0); + assert_eq!(counts(&file).2, 0); +} + +#[test] +fn unknown_and_unadmitted_have_identical_payload_free_errors() { + let mut messages = Vec::new(); + for unknown in [false, true] { + let file = Database::new(); + let mut f = Fixture::new(Store::open(&file.0).unwrap()).unwrap(); + f.store.set_admitted(ACTOR, SCOPE, f.ids[1], false).unwrap(); + if unknown { + let db = Connection::open(&file.0).unwrap(); + db.execute( + "DELETE FROM admission_events WHERE evidence_id=?1", + [f.ids[1]], + ) + .unwrap(); + db.execute("DELETE FROM evidence WHERE id=?1", [f.ids[1]]) + .unwrap(); + } + let error = + prepare_incident_attempt(&mut f.store, f.recovery, &f.ids, &f.contract).unwrap_err(); + assert!(matches!(error, BridgeError::UnavailableEvidence)); + messages.push(error.to_string()); + assert_eq!(counts(&file), (0, 0, 0)); + } + assert_eq!(messages[0], messages[1]); +} + +#[test] +fn captured_context_and_report_corruption_cancel_before_reservation() { + for position in [0, 1] { + let file = Database::new(); + let mut f = Fixture::new(Store::open(&file.0).unwrap()).unwrap(); + let db = Connection::open(&file.0).unwrap(); + let bytes = if position == 0 { + let (mut context, _) = pinned_case("cal-01").unwrap(); + context.question = "CORRUPTED CAPTURE".into(); + context.to_json().unwrap() + } else { + b"CORRUPTED REPORT".to_vec() + }; + // Deliberately corrupt storage, independent of caller flags or caches. + db.execute( + "UPDATE evidence SET payload=?1 WHERE id=?2", + rusqlite::params![bytes, f.ids[position]], + ) + .unwrap(); + assert!(matches!( + prepare_incident_attempt(&mut f.store, f.recovery, &f.ids, &f.contract), + Err(BridgeError::ChangedContract) + )); + let state: String = db + .query_row("SELECT state FROM recovery_attempts", [], |r| r.get(0)) + .unwrap(); + assert_eq!(state, "cancelled"); + assert_eq!(counts(&file), (1, 0, 0)); + assert_eq!(f.store.token_budget_state(f.budget).unwrap().reserved, 0); + } +} + +#[test] +fn full_manifest_runtime_cap_and_source_mismatch_prevent_capture() { + let file = Database::new(); + let mut f = Fixture::new(Store::open(&file.0).unwrap()).unwrap(); + let (context, _) = pinned_case("cal-01").unwrap(); + let raw = context.to_json().unwrap(); + for change in 0..4 { + let mut runtime = fixture_runtime(); + let mut mappings = f.contract.mappings_for_test(); + let mut cap = 512; + match change { + 0 => runtime.model_revision = "other".into(), + 1 => cap = 513, + 2 => mappings[0].occurrence = 9999, + _ => runtime.decoding = "different".into(), + } + let ids: Vec<_> = std::iter::once(f.ids[0]) + .chain(mappings.iter().map(|m| m.occurrence)) + .collect(); + let contract = + IncidentContract::new("cal-01", f.ids[0], &raw, mappings, runtime, cap).unwrap(); + assert!(matches!( + prepare_incident_attempt(&mut f.store, f.recovery, &ids, &contract), + Err(BridgeError::ChangedContract) + )); + assert_eq!(counts(&file), (0, 0, 0)); + } +} + +#[test] +fn sql_corruption_is_not_unavailable_evidence() { + let file = Database::new(); + let mut f = Fixture::new(Store::open(&file.0).unwrap()).unwrap(); + // This test intentionally injects schema corruption outside the Store API. + Connection::open(&file.0) + .unwrap() + .execute_batch("PRAGMA foreign_keys=OFF; DROP TABLE evidence;") + .unwrap(); + assert!(matches!( + prepare_incident_attempt(&mut f.store, f.recovery, &f.ids, &f.contract), + Err(BridgeError::Store(_)) + )); +} + +#[test] +fn correction_before_reservation_has_no_hold_or_dispatch() { + let file = Database::new(); + let mut f = Fixture::new(Store::open(&file.0).unwrap()).unwrap(); + let p = prepare_incident_attempt(&mut f.store, f.recovery, &f.ids, &f.contract).unwrap(); + Store::open(&file.0) + .unwrap() + .set_admitted(ACTOR, SCOPE, f.ids[1], false) + .unwrap(); + assert!(f + .store + .reserve_recovery_attempt(p.attempt, &p.request, p.preparation.input_tokens, 512) + .is_err()); + assert_eq!(counts(&file), (1, 0, 0)); + assert_eq!(f.store.token_budget_state(f.budget).unwrap().reserved, 0); +} + +#[test] +fn retry_cannot_replace_bound_source_occurrences() { + let file = Database::new(); + let mut f = Fixture::new(Store::open(&file.0).unwrap()).unwrap(); + let p = prepare_incident_attempt(&mut f.store, f.recovery, &f.ids, &f.contract).unwrap(); + reserve(&mut f.store, &p); + f.store + .run_timed_recovery( + p.attempt, + &p.preparation, + || tick(200), + |request| Ok::<_, String>(p.measure(request, b"bad")), + ) + .unwrap(); + let (context, reports) = pinned_case("cal-01").unwrap(); + let replacement = f + .store + .observe_and_admit_many(ACTOR, SCOPE, &[("replacement", reports[0].text.as_bytes())]) + .unwrap()[0]; + let mut mappings = f.contract.mappings_for_test(); + mappings[0].occurrence = replacement; + let contract = IncidentContract::new( + "cal-01", + f.ids[0], + &context.to_json().unwrap(), + mappings, + fixture_runtime(), + 512, + ) + .unwrap(); + let mut ids = f.ids.clone(); + ids[1] = replacement; + assert!(matches!( + prepare_incident_attempt(&mut f.store, f.recovery, &ids, &contract), + Err(BridgeError::ChangedContract) + )); + assert_eq!(counts(&file), (1, 1, 1)); + // The bound occurrences can still obtain the one retry, under the original window. + assert!(prepare_incident_attempt(&mut f.store, f.recovery, &f.ids, &f.contract).is_ok()); +} + +#[test] +fn denied_reservation_is_terminal_without_dispatch() { + let mut f = Fixture::new(Store::in_memory().unwrap()).unwrap(); + let timing = f.store.recovery_timing(f.recovery).unwrap().unwrap(); + let budget = f.store.create_token_budget(0).unwrap(); + let recovery = f + .store + .create_timed_recovery("denied", budget, ACTOR, SCOPE, "denied", &timing) + .unwrap(); + let p = prepare_incident_attempt(&mut f.store, recovery, &f.ids, &f.contract).unwrap(); + assert!(!f + .store + .reserve_recovery_attempt(p.attempt, &p.request, p.preparation.input_tokens, 512) + .unwrap()); + assert!(matches!( + f.store + .run_timed_recovery( + p.attempt, + &p.preparation, + || panic!("denied clock"), + |_| panic!("denied backend") + ) + .unwrap(), + RecoveryRun::::NotReserved + )); + assert!(matches!( + prepare_incident_attempt(&mut f.store, recovery, &f.ids, &f.contract), + Err(BridgeError::TerminalRecovery) + )); +} + +#[test] +fn initial_request_matches_independent_python_renderer_bytes() { + let mut f = Fixture::new(Store::in_memory().unwrap()).unwrap(); + let p = prepare_incident_attempt(&mut f.store, f.recovery, &f.ids, &f.contract).unwrap(); + // Python oracle is independent of Rust codec and never opens label bytes. + let script = r#"import json, pathlib, sys +root=pathlib.Path(sys.argv[1]) +cases=json.loads((root/'cases.json').read_text()) +prompt=json.loads((root/'prompt.json').read_text()) +case=next(c for c in cases['cases'] if c['case_id']=='cal-01') +canonical=lambda v: json.dumps(v,ensure_ascii=False,sort_keys=True,separators=(',',':'),allow_nan=False) +payload={k:case[k] for k in ('question','owner_ids','evidence')} +sys.stdout.buffer.write(canonical({'messages':[{'role':'system','content':prompt['system']},{'role':'user','content':canonical(payload)}]}).encode()) +"#; + let result = std::process::Command::new("python3") + .args([ + "-c", + script, + concat!(env!("CARGO_MANIFEST_DIR"), "/workloads/incident-routing-v1"), + ]) + .output() + .unwrap(); + assert!( + result.status.success(), + "{}", + String::from_utf8_lossy(&result.stderr) + ); + assert_eq!(p.request, result.stdout); + assert_eq!(p.captured_ids, f.ids); + assert_eq!( + p.preparation.runtime.as_ref().unwrap().prompt_fingerprint, + p.manifest_sha256 + ); +} + +#[test] +fn retry_is_byte_exact_fixed_feedback_and_keeps_original_deadline() { + let mut f = Fixture::new(Store::in_memory().unwrap()).unwrap(); + let initial = prepare_incident_attempt(&mut f.store, f.recovery, &f.ids, &f.contract).unwrap(); + let timing = f.store.recovery_timing(f.recovery).unwrap(); + reserve(&mut f.store, &initial); + let result = f + .store + .run_timed_recovery( + initial.attempt, + &initial.preparation, + || tick(200), + |request| { + let mut reply = initial.measure(request, b"bad"); + reply.validation = ReplyValidation::Invalid { + validator: "fixture-validator".into(), + reason: "SECRET LABEL FREE TEXT MUST NOT REACH RETRY".into(), + }; + Ok::<_, String>(reply) + }, + ) + .unwrap(); + assert!(matches!( + result, + RecoveryRun::Settled { + disposition: Disposition::InvalidOutput, + .. + } + )); + let retry = prepare_incident_attempt(&mut f.store, f.recovery, &f.ids, &f.contract).unwrap(); + let mut expected: Value = serde_json::from_slice(&initial.request).unwrap(); + let mut user: Value = + serde_json::from_str(expected["messages"][1]["content"].as_str().unwrap()).unwrap(); + user["validation_error"] = json!(FEEDBACK); + expected["messages"][1]["content"] = json!(serde_json::to_string(&user).unwrap()); + assert_eq!(retry.request, serde_json::to_vec(&expected).unwrap()); + assert_eq!(retry.manifest_sha256, initial.manifest_sha256); + assert_eq!(f.store.recovery_timing(f.recovery).unwrap(), timing); + reserve(&mut f.store, &retry); + let result = f + .store + .run_timed_recovery( + retry.attempt, + &retry.preparation, + || tick(300), + |request| Ok::<_, String>(retry.measure(request, b"bad again")), + ) + .unwrap(); + assert!(matches!( + result, + RecoveryRun::Settled { + disposition: Disposition::InvalidOutput, + .. + } + )); + assert!(matches!( + prepare_incident_attempt(&mut f.store, f.recovery, &f.ids, &f.contract), + Err(BridgeError::TerminalRecovery) + )); + assert_eq!( + f.store.token_budget_state(f.budget).unwrap().spent, + (initial.request.len() + 3 + retry.request.len() + 9) as i64 + ); +} + +#[test] +fn correction_inside_backend_records_full_usage_and_stale_publication() { + let file = Database::new(); + let mut f = Fixture::new(Store::open(&file.0).unwrap()).unwrap(); + let p = prepare_incident_attempt(&mut f.store, f.recovery, &f.ids, &f.contract).unwrap(); + reserve(&mut f.store, &p); + let mut other = Store::open(&file.0).unwrap(); + let mut calls = 0; + let result = f.store.run_timed_recovery(p.attempt, &p.preparation, || tick(200), |request| { + calls += 1; + other.set_admitted(ACTOR, SCOPE, f.ids[1], false).unwrap(); + other.observe_and_admit_many(ACTOR, SCOPE, &[("e01", b"Correction: the earlier diagnosis is withdrawn; cause remains unknown.")]).unwrap(); + Ok::<_, String>(p.measure(request, REPLY)) + }).unwrap(); + assert!(matches!( + result, + RecoveryRun::Settled { + disposition: Disposition::StaleView, + .. + } + )); + assert_eq!(calls, 1); + let usage = (p.request.len() + REPLY.len()) as i64; + assert_eq!(f.store.token_budget_state(f.budget).unwrap().spent, usage); + assert_eq!(f.store.token_budget_state(f.budget).unwrap().reserved, 0); + assert_eq!( + f.store.settle(p.attempt, REPLY, usage).unwrap(), + Disposition::StaleView + ); + assert_eq!(f.store.charged_usage().unwrap(), usage); + assert_eq!(counts(&file).2, 1); +} + +#[test] +fn reopen_reserved_then_sent_does_not_reprepare_or_claim_twice() { + let file = Database::new(); + let mut f = Fixture::new(Store::open(&file.0).unwrap()).unwrap(); + let p = prepare_incident_attempt(&mut f.store, f.recovery, &f.ids, &f.contract).unwrap(); + reserve(&mut f.store, &p); + drop(f.store); + f.store = Store::open(&file.0).unwrap(); + assert!(matches!( + prepare_incident_attempt(&mut f.store, f.recovery, &f.ids, &f.contract), + Err(BridgeError::ExistingRecovery) + )); + assert_eq!( + f.store + .recovery_reservation(p.attempt) + .unwrap() + .unwrap() + .request, + p.request + ); + let mut calls = 0; + assert!(matches!( + f.store + .run_timed_recovery( + p.attempt, + &p.preparation, + || tick(200), + |_| { + calls += 1; + Err::("backend uncertain") + } + ) + .unwrap(), + RecoveryRun::BackendFailed { + error: "backend uncertain", + .. + } + )); + drop(f.store); + f.store = Store::open(&file.0).unwrap(); + assert!(matches!( + prepare_incident_attempt(&mut f.store, f.recovery, &f.ids, &f.contract), + Err(BridgeError::ExistingRecovery) + )); + assert!(matches!( + f.store + .run_timed_recovery( + p.attempt, + &p.preparation, + || panic!("repeat clock"), + |_| panic!("repeat backend") + ) + .unwrap(), + RecoveryRun::::NotReserved + )); + assert_eq!(calls, 1); + assert_eq!( + f.store.token_budget_state(f.budget).unwrap().reserved, + p.preparation.input_tokens + 512 + ); + assert!(!f.store.cancel_recovery_attempt(p.attempt).unwrap()); +} + +#[test] +fn post_claim_expiry_retains_hold_without_adapter_invocation() { + let mut f = Fixture::new(Store::in_memory().unwrap()).unwrap(); + let p = prepare_incident_attempt(&mut f.store, f.recovery, &f.ids, &f.contract).unwrap(); + reserve(&mut f.store, &p); + let mut samples = 0; + let result = f + .store + .run_timed_recovery( + p.attempt, + &p.preparation, + || { + samples += 1; + tick(if samples == 1 { 200 } else { 1000 }) + }, + |_| panic!("expired backend"), + ) + .unwrap(); + assert!(matches!( + result, + RecoveryRun::::ClaimedButNotInvoked { + reason: TimingRefusal::DeadlineExpired, + .. + } + )); + assert_eq!( + f.store.token_budget_state(f.budget).unwrap().reserved, + p.preparation.input_tokens + 512 + ); + assert_eq!(f.store.charged_usage().unwrap(), 0); + assert!(!f.store.cancel_recovery_attempt(p.attempt).unwrap()); +} + +#[test] +fn settlement_error_preserves_measured_reply_and_hold() { + let file = Database::new(); + let mut f = Fixture::new(Store::open(&file.0).unwrap()).unwrap(); + let p = prepare_incident_attempt(&mut f.store, f.recovery, &f.ids, &f.contract).unwrap(); + reserve(&mut f.store, &p); + Connection::open(&file.0).unwrap().execute_batch("CREATE TRIGGER reject_delivery BEFORE INSERT ON deliveries BEGIN SELECT RAISE(ABORT, 'fixture settlement failure'); END;").unwrap(); + let result = f + .store + .run_timed_recovery( + p.attempt, + &p.preparation, + || tick(200), + |request| Ok::<_, String>(p.measure(request, REPLY)), + ) + .unwrap(); + match result { + RecoveryRun::SettlementFailed { reply, error, .. } => { + assert_eq!(reply.output, REPLY); + assert_eq!(reply.input_tokens, p.request.len() as i64); + assert!(error.to_string().contains("fixture settlement failure")); + } + other => panic!("{other:?}"), + } + assert_eq!( + f.store.token_budget_state(f.budget).unwrap().reserved, + p.preparation.input_tokens + 512 + ); + assert_eq!(f.store.charged_usage().unwrap(), 0); +} + +#[test] +fn protocol_valid_semantic_error_is_only_format_acceptance_and_duplicate_is_idempotent() { + let mut f = Fixture::new(Store::in_memory().unwrap()).unwrap(); + let p = prepare_incident_attempt(&mut f.store, f.recovery, &f.ids, &f.contract).unwrap(); + reserve(&mut f.store, &p); + assert!(matches!( + f.store + .run_timed_recovery( + p.attempt, + &p.preparation, + || tick(200), + |request| Ok::<_, String>(p.measure(request, REPLY)) + ) + .unwrap(), + RecoveryRun::Settled { + disposition: Disposition::Accepted, + .. + } + )); + let usage = (p.request.len() + REPLY.len()) as i64; + assert_eq!( + f.store.settle(p.attempt, REPLY, usage).unwrap(), + Disposition::Accepted + ); + assert_eq!(f.store.charged_usage().unwrap(), usage); + assert!(matches!( + prepare_incident_attempt(&mut f.store, f.recovery, &f.ids, &f.contract), + Err(BridgeError::TerminalRecovery) + )); +} + +#[test] +fn cancellation_failure_preserves_original_error_and_sql_failure() { + let file = Database::new(); + let mut f = Fixture::new(Store::open(&file.0).unwrap()).unwrap(); + let db = Connection::open(&file.0).unwrap(); + db.execute( + "UPDATE evidence SET payload=?1 WHERE id=?2", + rusqlite::params![b"corrupted".as_slice(), f.ids[1]], + ) + .unwrap(); + db.execute_batch("CREATE TRIGGER reject_cancel BEFORE UPDATE OF state ON recovery_attempts WHEN NEW.state='cancelled' BEGIN SELECT RAISE(ABORT, 'fixture cancellation failure'); END;").unwrap(); + match prepare_incident_attempt(&mut f.store, f.recovery, &f.ids, &f.contract).unwrap_err() { + BridgeError::CancellationFailed { + original, + cancellation: CancellationFailure::Store(error), + } => { + assert!(matches!(*original, BridgeError::ChangedContract)); + assert!(error.to_string().contains("fixture cancellation failure")); + } + other => panic!("{other:?}"), + } + let state: String = db + .query_row("SELECT state FROM recovery_attempts", [], |r| r.get(0)) + .unwrap(); + assert_eq!(state, "planned"); + assert_eq!(counts(&file), (1, 0, 0)); +} + +#[test] +fn cancellation_refusal_retains_claimed_hold_and_original_error() { + let mut f = Fixture::new(Store::in_memory().unwrap()).unwrap(); + let p = prepare_incident_attempt(&mut f.store, f.recovery, &f.ids, &f.contract).unwrap(); + reserve(&mut f.store, &p); + f.store + .run_timed_recovery( + p.attempt, + &p.preparation, + || tick(200), + |_| Err::("uncertain"), + ) + .unwrap(); + match cancel_new(&mut f.store, p.attempt, BridgeError::ChangedContract) { + BridgeError::CancellationFailed { + original, + cancellation: CancellationFailure::Refused, + } => assert!(matches!(*original, BridgeError::ChangedContract)), + other => panic!("{other:?}"), + } + assert_eq!( + f.store.token_budget_state(f.budget).unwrap().reserved, + p.preparation.input_tokens + 512 + ); +} + +#[test] +fn labels_never_enter_request_or_feedback() { + let mut f = Fixture::new(Store::in_memory().unwrap()).unwrap(); + let p = prepare_incident_attempt(&mut f.store, f.recovery, &f.ids, &f.contract).unwrap(); + let text = std::str::from_utf8(&p.request).unwrap(); + for forbidden in [ + "labels", + "family_id", + "split", + "eligibility", + "current_joint_success", + ] { + assert!(!text.contains(forbidden)); + } + reserve(&mut f.store, &p); + let result = f + .store + .run_timed_recovery( + p.attempt, + &p.preparation, + || tick(200), + |request| Ok::<_, String>(p.measure(request, b"invalid")), + ) + .unwrap(); + assert!(matches!( + result, + RecoveryRun::Settled { + disposition: Disposition::InvalidOutput, + .. + } + )); + let retry = prepare_incident_attempt(&mut f.store, f.recovery, &f.ids, &f.contract).unwrap(); + assert!(std::str::from_utf8(&retry.request) + .unwrap() + .contains(FEEDBACK)); +} + +fn reserve(store: &mut Store, p: &IncidentPreparation) { + assert!(store + .reserve_recovery_attempt( + p.attempt, + &p.request, + p.preparation.input_tokens, + p.preparation.output_cap + ) + .unwrap()); +} + +struct Database(std::path::PathBuf); +impl Database { + fn new() -> Self { + static NEXT: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(0); + Self(std::env::temp_dir().join(format!( + "incident-store-{}-{}-{}.db", + std::process::id(), + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap() + .as_nanos(), + NEXT.fetch_add(1, std::sync::atomic::Ordering::Relaxed) + ))) + } +} +impl Drop for Database { + fn drop(&mut self) { + let _ = std::fs::remove_file(&self.0); + } +} From 0dd48f85507570b345d1385e567fc1b53ac569b9 Mon Sep 17 00:00:00 2001 From: Alejandro Castellanos <5405628+k2jac9@users.noreply.github.com> Date: Sun, 15 Nov 2026 15:37:40 -0500 Subject: [PATCH 10/51] test: verify Store incident byte parity in CI --- .github/workflows/check.yml | 9 ++- docs/incident-routing-workload.md | 11 +++ docs/incident-store-bridge.md | 62 +++++++++++++- examples/incident_store_bridge.rs | 55 ++++++++++++- examples/incident_store_bridge/adapter.rs | 8 +- examples/incident_store_bridge/tests.rs | 37 +++++++++ scripts/check_incident_store_bridge.py | 64 +++++++++++++++ scripts/test_incident_store_bridge.py | 98 +++++++++++++++++++++++ 8 files changed, 338 insertions(+), 6 deletions(-) create mode 100644 scripts/check_incident_store_bridge.py create mode 100644 scripts/test_incident_store_bridge.py diff --git a/.github/workflows/check.yml b/.github/workflows/check.yml index 802d562..619937c 100644 --- a/.github/workflows/check.yml +++ b/.github/workflows/check.yml @@ -32,7 +32,8 @@ jobs: run: | python3 -m unittest discover -s reference -p test_memory.py python3 reference/belief_oracle.py - python3 -m unittest discover -s scripts -p 'test_*.py' + cargo build --locked --example recovery_policy_eval + KAIROTENSOR_RECOVERY_RUNNER="$PWD/target/debug/examples/recovery_policy_eval" python3 -m unittest discover -s scripts -p 'test_*.py' - name: Verify archived output-cap measurements run: python3 scripts/summarize_token_caps.py docs/evaluations/token-caps-*-2026-11-13.json @@ -53,6 +54,12 @@ jobs: - name: Prepare frozen semantic calibration requests without inference run: python3 scripts/incident_routing.py prepare --out "$RUNNER_TEMP/incident-routing-calibration" + - name: Verify all frozen incident request bytes through Store capture without inference + run: | + cargo run --locked --example incident_store_bridge -- --requests "$RUNNER_TEMP/incident-store-requests.json" + python3 scripts/check_incident_store_bridge.py --requests "$RUNNER_TEMP/incident-store-requests.json" + python3 -m unittest discover -s scripts -p test_incident_store_bridge.py + - name: Emit bounded prepared-request calibration proposal without inference run: python3 scripts/incident_preflight.py --out "$RUNNER_TEMP/incident-preflight-plan.json" diff --git a/docs/incident-routing-workload.md b/docs/incident-routing-workload.md index 54e514a..00dcf11 100644 --- a/docs/incident-routing-workload.md +++ b/docs/incident-routing-workload.md @@ -112,6 +112,17 @@ A future integration must capture complete authorized evidence and recheck actua Store freshness at claim/publication. Missing or revoked evidence is not grounds to ask a model to infer its hidden content. +The offline [Store-backed fixture](incident-store-bridge.md#all-case-store-request-export-and-parity) +now admits and captures actual Store occurrences for all 12 frozen cases. Run +`incident_store_bridge --requests PATH` through the admitted Cargo wrapper, then +`python3 scripts/check_incident_store_bridge.py --requests PATH`. The destination +file must not exist. The checker compares exact UTF-8 request bytes and order +against `prepare(..., split='all')`, including evidence order, rather than trusting +exported hashes. This fixture reports Store capture, UTF-8 byte accounting and +zero model calls; it performs no live inference or semantic scoring. Case IDs, +manifest digests and captured occurrence IDs stay outside request bytes. See the +linked contract for the complete export schema and occurrence scope. + ## Audit a complete response bundle Run: diff --git a/docs/incident-store-bridge.md b/docs/incident-store-bridge.md index c341157..73f271e 100644 --- a/docs/incident-store-bridge.md +++ b/docs/incident-store-bridge.md @@ -163,5 +163,63 @@ recovery's contract. Tests deliberately use a protocol-valid but semantically wrong canned answer. Store acceptance here means current evidence and valid format, not correctness against semantic labels. This work supplies no efficacy, live inference, -production calibration, remote exactly-once delivery, or all-case export claim. -All-case exports and CI integration belong to the separately scoped next task. +production calibration or remote exactly-once delivery claim. + +## All-case Store request export and parity + +Export every frozen case in corpus order to an explicit, previously nonexistent +file. The parent directory must already exist. The default invocation above +continues to run the original single-case canned demo. + +```sh +mkdir -p /home/neo/Projects/kairotensor-artifacts/my-parity-run +# First obtain fresh storage admission for the retained artifact destination. +/home/neo/.local/bin/kairotensor-storage preflight --budget-gib 2 \ + --output-dir /home/neo/Projects/kairotensor-artifacts/my-parity-run +CARGO_PROFILE_DEV_DEBUG=0 CARGO_PROFILE_TEST_DEBUG=0 CARGO_INCREMENTAL=0 \ + /home/neo/.local/bin/kairotensor-cargo --budget-gib 2 run --locked \ + --example incident_store_bridge -- \ + --requests /home/neo/Projects/kairotensor-artifacts/my-parity-run/requests.json +python3 scripts/check_incident_store_bridge.py --requests \ + /home/neo/Projects/kairotensor-artifacts/my-parity-run/requests.json +python3 -m unittest discover -s scripts -p test_incident_store_bridge.py +``` + +The exporter uses `Fixture::for_case` with a fresh in-memory Store for each case, +admits its context and all reports, binds the complete manifest, and calls +`prepare_incident_attempt`. Requests come from that actual Store capture and +preparation, with context first and reports in pinned order. Occurrence IDs are +local to each fixture Store; equal IDs across records do not identify the same +occurrence. Export performs no reservation, dispatch, backend invocation or model +call. Both calibration and held-out cases are exported solely for offline byte +parity; this is not a held-out inference or scoring run. + +The output is one compact UTF-8 JSON object followed by a newline. Its exact +top-level fields are `schema_version` (integer `1`), `version` +(`incident-store-requests/1`), `mode` (`synthetic_store_incident_fixture`), +`new_model_calls` (integer `0`), `accounting_unit` (`UTF-8 bytes`), `capture` +(`Store`), `freeze_sha256` (the pinned frozen manifest digest above), and +`requests` (12 records in frozen case order). Each record has exactly: + +- `case_id`: frozen case ID, outside request bytes. +- `request`: exact prepared request as a JSON string; send only its UTF-8 bytes + to any separately authorized consumer. +- `request_sha256`: SHA-256 of those UTF-8 request bytes. +- `manifest_sha256`: immutable `incident-store-manifest/1` digest returned by + Store-backed preparation, also bound as the timing runtime fingerprint. +- `captured_ids`: ordered positive occurrence IDs, context followed by every + required report, without duplicates within that record. + +The checker reloads the frozen workload and calls +`incident_routing.prepare(workload, split='all')`. It compares every case ID, +request byte and case/report order, independently of producer digests. It rejects +missing, duplicate, reordered or malformed records, changed bytes even with +recomputed hashes, duplicate JSON keys and incorrect fixture metadata. Success +prints `mode=offline_store_byte_parity`, `cases_checked=12`, +`new_model_calls=0` and `accounting_unit=UTF-8 bytes`; failure exits nonzero. +The manifest digest and occurrence IDs are producer provenance: the checker +validates their shape, while the adapter tests establish Store capture behavior. +Neither hashes nor this checker authenticate an arbitrary report's execution. + +`check.yml` adds only offline export/checker/test steps. Configured CI is not +evidence of hosted execution or qualification of an external provider/runtime. diff --git a/examples/incident_store_bridge.rs b/examples/incident_store_bridge.rs index f3a3c37..23b3d43 100644 --- a/examples/incident_store_bridge.rs +++ b/examples/incident_store_bridge.rs @@ -4,6 +4,59 @@ mod adapter; #[path = "incident_store_bridge/tests.rs"] mod tests; +fn requests_report() -> Result> { + use adapter::{prepare_incident_attempt, sha256, Fixture}; + use serde_json::{json, Value}; + + let cases: Value = serde_json::from_slice(include_bytes!( + "../workloads/incident-routing-v1/cases.json" + ))?; + let mut requests = Vec::new(); + for case in cases["cases"].as_array().ok_or("missing frozen cases")? { + let case_id = case["case_id"].as_str().ok_or("missing frozen case ID")?; + // Each fixture has its own Store. Occurrence IDs are local to that Store. + let mut fixture = Fixture::for_case(kairotensor::Store::in_memory()?, case_id)?; + let prepared = prepare_incident_attempt( + &mut fixture.store, + fixture.recovery, + &fixture.ids, + &fixture.contract, + )?; + requests.push(json!({ + "case_id": case_id, + "request_sha256": sha256(&prepared.request), + "request": String::from_utf8(prepared.request)?, + "manifest_sha256": prepared.manifest_sha256, + "captured_ids": prepared.captured_ids, + })); + } + Ok(json!({ + "schema_version": 1, "version": "incident-store-requests/1", + "mode": "synthetic_store_incident_fixture", "new_model_calls": 0, + "accounting_unit": "UTF-8 bytes", "capture": "Store", + "freeze_sha256": sha256(include_bytes!("../workloads/incident-routing-v1/freeze.json")), + "requests": requests, + })) +} + +fn export_requests(path: &std::path::Path) -> Result<(), Box> { + use std::io::Write; + let report = requests_report()?; + let mut file = std::fs::OpenOptions::new() + .write(true) + .create_new(true) + .open(path)?; + serde_json::to_writer(&mut file, &report)?; + file.write_all(b"\n")?; + file.sync_all()?; + Ok(()) +} + fn main() -> Result<(), Box> { - adapter::demo() + let args: Vec<_> = std::env::args_os().skip(1).collect(); + match args.as_slice() { + [] => adapter::demo(), + [flag, path] if flag == "--requests" => export_requests(std::path::Path::new(path)), + _ => Err("usage: incident_store_bridge [--requests PATH]".into()), + } } diff --git a/examples/incident_store_bridge/adapter.rs b/examples/incident_store_bridge/adapter.rs index 12af195..0ab8cc7 100644 --- a/examples/incident_store_bridge/adapter.rs +++ b/examples/incident_store_bridge/adapter.rs @@ -455,8 +455,12 @@ pub(super) struct Fixture { pub(super) budget: i64, } impl Fixture { - pub(super) fn new(mut store: Store) -> Result { - let (context, reports) = pinned_case("cal-01")?; + pub(super) fn new(store: Store) -> Result { + Self::for_case(store, "cal-01") + } + + pub(super) fn for_case(mut store: Store, case_id: &str) -> Result { + let (context, reports) = pinned_case(case_id)?; let raw = context.to_json()?; let observations: Vec<_> = std::iter::once(("incident-context", raw.as_slice())) .chain( diff --git a/examples/incident_store_bridge/tests.rs b/examples/incident_store_bridge/tests.rs index 3ea12ba..135625f 100644 --- a/examples/incident_store_bridge/tests.rs +++ b/examples/incident_store_bridge/tests.rs @@ -8,6 +8,43 @@ use serde_json::{json, Value}; const REPLY: &[u8] = br#"{"decision":"abstain","owner":null,"reason":"undetermined_cause","evidence_ids":["e01"]}"#; +#[test] +fn export_prepares_all_frozen_cases_in_order_through_store() { + let report = super::requests_report().unwrap(); + let cases: Value = serde_json::from_slice(include_bytes!( + "../../workloads/incident-routing-v1/cases.json" + )) + .unwrap(); + let records = report["requests"].as_array().unwrap(); + assert_eq!(records.len(), 12); + for (record, case) in records.iter().zip(cases["cases"].as_array().unwrap()) { + assert_eq!(record["case_id"], case["case_id"]); + assert_eq!( + record["captured_ids"].as_array().unwrap().len(), + case["evidence"].as_array().unwrap().len() + 1 + ); + assert_eq!( + record["request_sha256"], + sha256(record["request"].as_str().unwrap().as_bytes()) + ); + assert_eq!(record["manifest_sha256"].as_str().unwrap().len(), 64); + } + assert_eq!(report["capture"], "Store"); + assert_eq!(report["new_model_calls"], 0); + assert_eq!(report["accounting_unit"], "UTF-8 bytes"); +} + +#[test] +fn export_exclusively_creates_explicit_destination() { + let path = Database::new(); + super::export_requests(&path.0).unwrap(); + let original = std::fs::read(&path.0).unwrap(); + assert!(super::export_requests(&path.0).is_err()); + assert_eq!(std::fs::read(&path.0).unwrap(), original); + let report: Value = serde_json::from_slice(&original).unwrap(); + assert_eq!(report["requests"].as_array().unwrap().len(), 12); +} + fn counts(file: &Database) -> (i64, i64, i64) { Connection::open(&file.0).unwrap().query_row( "SELECT (SELECT count(*) FROM attempts), (SELECT count(*) FROM token_reservations), (SELECT count(*) FROM dispatches)", diff --git a/scripts/check_incident_store_bridge.py b/scripts/check_incident_store_bridge.py new file mode 100644 index 0000000..f6ee4ff --- /dev/null +++ b/scripts/check_incident_store_bridge.py @@ -0,0 +1,64 @@ +"""Check offline Store-captured incident requests against the frozen Python renderer.""" +import argparse +from pathlib import Path +import re + +import incident_routing as routing + + +def check(report): + """Compare exact UTF-8 bytes and order, never just producer-supplied digests. + + Capture IDs and manifest digests are producer provenance, not independent + proof of Store execution or authorization. The Rust adapter tests cover that. + """ + routing.fields(report, { + 'schema_version', 'version', 'mode', 'new_model_calls', + 'accounting_unit', 'capture', 'freeze_sha256', 'requests', + }, 'Store request export') + expected_metadata = { + 'schema_version': 1, 'version': 'incident-store-requests/1', + 'mode': 'synthetic_store_incident_fixture', 'new_model_calls': 0, + 'accounting_unit': 'UTF-8 bytes', 'capture': 'Store', + 'freeze_sha256': routing.FREEZE_SHA256, + } + for key, value in expected_metadata.items(): + routing.require(type(report[key]) is type(value) and report[key] == value, + f'incorrect {key}') + workload = routing.load_workload() + expected = routing.prepare(workload, split='all')['requests'] + records = report['requests'] + routing.require(type(records) is list and len(records) == len(expected), 'case count mismatch') + for record, wanted, case in zip(records, expected, workload['cases']['cases']): + routing.fields(record, {'case_id', 'request', 'request_sha256', + 'manifest_sha256', 'captured_ids'}, 'request record') + routing.require(record['case_id'] == wanted['case_id'], 'case order mismatch') + routing.require(type(record['request']) is str and + record['request'].encode('utf-8') == wanted['request'].encode('utf-8'), + f"request bytes mismatch: {wanted['case_id']}") + routing.require(record['request_sha256'] == wanted['request_sha256'], 'request digest mismatch') + digest = record['manifest_sha256'] + routing.require(type(digest) is str and re.fullmatch('[0-9a-f]{64}', digest) is not None, + 'invalid manifest digest') + ids = record['captured_ids'] + routing.require(type(ids) is list and len(ids) == len(case['evidence']) + 1 and + all(type(i) is int and i > 0 for i in ids) and len(set(ids)) == len(ids), + 'invalid captured occurrence IDs') + return len(records) + + +def main(argv=None): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('--requests', required=True, type=Path) + args = parser.parse_args(argv) + count = check(routing.read_json(args.requests)) + print(routing.canonical({'mode': 'offline_store_byte_parity', 'cases_checked': count, + 'new_model_calls': 0, 'accounting_unit': 'UTF-8 bytes'})) + return 0 + + +if __name__ == '__main__': + try: + raise SystemExit(main()) + except (ValueError, OSError, UnicodeError, RecursionError) as error: + raise SystemExit(f'Store request parity refused: {error}') diff --git a/scripts/test_incident_store_bridge.py b/scripts/test_incident_store_bridge.py new file mode 100644 index 0000000..8f04e5a --- /dev/null +++ b/scripts/test_incident_store_bridge.py @@ -0,0 +1,98 @@ +"""Cross-language export rejection gates; expected bytes come from frozen Python preparation.""" +import copy +import hashlib +import json +from pathlib import Path +import subprocess +import sys +import tempfile +import unittest + +import incident_routing as routing +import check_incident_store_bridge as bridge + + +class ParityTests(unittest.TestCase): + def setUp(self): + workload = routing.load_workload() + prepared = routing.prepare(workload, split='all') + self.report = { + 'schema_version': 1, 'version': 'incident-store-requests/1', + 'mode': 'synthetic_store_incident_fixture', 'new_model_calls': 0, + 'accounting_unit': 'UTF-8 bytes', 'capture': 'Store', + 'freeze_sha256': routing.FREEZE_SHA256, + 'requests': [dict(case_id=r['case_id'], request=r['request'], + request_sha256=r['request_sha256'], manifest_sha256='a' * 64, + captured_ids=list(range(1, len(c['evidence']) + 2))) + for r, c in zip(prepared['requests'], workload['cases']['cases'])], + } + + def test_accepts_complete_byte_exact_requests(self): + self.assertEqual(bridge.check(self.report), 12) + + def test_changed_byte_with_recomputed_digest_is_rejected(self): + self.report['requests'][0]['request'] += ' ' + self.report['requests'][0]['request_sha256'] = hashlib.sha256( + self.report['requests'][0]['request'].encode()).hexdigest() + with self.assertRaisesRegex(ValueError, 'request bytes'): + bridge.check(self.report) + + def test_omitted_case_is_rejected(self): + self.report['requests'].pop() + with self.assertRaisesRegex(ValueError, 'case count'): + bridge.check(self.report) + + def test_reordered_cases_are_rejected(self): + self.report['requests'][0], self.report['requests'][1] = ( + self.report['requests'][1], self.report['requests'][0]) + with self.assertRaisesRegex(ValueError, 'case order'): + bridge.check(self.report) + + def test_reordered_reports_with_recomputed_digest_are_rejected(self): + record = self.report['requests'][0] + request = json.loads(record['request']) + user = json.loads(request['messages'][1]['content']) + user['evidence'].reverse() + request['messages'][1]['content'] = routing.canonical(user) + record['request'] = routing.canonical(request) + record['request_sha256'] = hashlib.sha256(record['request'].encode()).hexdigest() + with self.assertRaisesRegex(ValueError, 'request bytes'): + bridge.check(self.report) + + def test_malformed_metadata_records_and_capture_are_rejected(self): + for index, mutate in enumerate(( + lambda r: r.update(new_model_calls=True), + lambda r: r.update(mode='live'), + lambda r: r.update(capture='declared_flags'), + lambda r: r.update(accounting_unit='tokens'), + lambda r: r.update(freeze_sha256='0' * 64), + lambda r: r['requests'][0].update(manifest_sha256='bad'), + lambda r: r['requests'][0].update(request_sha256='0' * 64), + lambda r: r['requests'][0].update(captured_ids=[1, 1, 2]), + lambda r: r['requests'][0].update(request=None), + lambda r: r['requests'][0].update(extra='unrecognized'), + )): + report = copy.deepcopy(self.report) + mutate(report) + with self.subTest(mutation=index), self.assertRaises(ValueError): + bridge.check(report) + + def test_cli_rejects_corrupt_and_duplicate_key_files(self): + with tempfile.TemporaryDirectory() as folder: + path = Path(folder) / 'requests.json' + for raw, success in ( + (routing.canonical(self.report), True), + ('{"requests":[],"requests":[]}', False), + ('{', False), + ): + path.write_text(raw, encoding='utf-8') + result = subprocess.run( + [sys.executable, str(Path(bridge.__file__)), '--requests', str(path)], + capture_output=True, text=True) + self.assertEqual(result.returncode == 0, success, result.stderr) + if success: + self.assertEqual(json.loads(result.stdout)['cases_checked'], 12) + + +if __name__ == '__main__': + unittest.main() From 3b457218742a67d3a2afb4e20948962a17c8a2e1 Mon Sep 17 00:00:00 2001 From: Alejandro Castellanos <5405628+k2jac9@users.noreply.github.com> Date: Sun, 15 Nov 2026 15:45:16 -0500 Subject: [PATCH 11/51] docs: bind replay safeguards and SQLite lifetime repair --- .../plans/2026-11-15-decision-replay.md | 16 +++++- .../plans/2026-11-15-sqlite-lifetimes.md | 52 +++++++++++++++++++ 2 files changed, 66 insertions(+), 2 deletions(-) create mode 100644 docs/superpowers/plans/2026-11-15-sqlite-lifetimes.md diff --git a/docs/superpowers/plans/2026-11-15-decision-replay.md b/docs/superpowers/plans/2026-11-15-decision-replay.md index 4c578cc..3e5762d 100644 --- a/docs/superpowers/plans/2026-11-15-decision-replay.md +++ b/docs/superpowers/plans/2026-11-15-decision-replay.md @@ -21,6 +21,8 @@ section C. Prerequisites: A and B reviewed. - Do not export cross-scope/global budget totals or infer missing validators. - Pure replay cannot invoke tools, a model, network, Store initialization or migrations. No Rocket integration; use normal storage admission. +- Resource bounds apply before cloning public constructed values and before + materializing oversized SQLite payload/metadata fields, not only JSON parsing. ## Review Focus @@ -89,8 +91,9 @@ Reevaluation additionally names the new validator and never changes the bundle. Replay compares deterministic output or exact reconstructed request as applicable. Missing legacy validator is NotReproducible. Unknown version never falls back. - [ ] Test changed validator as a separate report, missing inherited/reuse input, - and original disposition preservation. A panic-on-call fake adapter must remain - unused; the replay API takes no backend capability. + and original disposition preservation. The replay API takes no backend capability; + inspect that boundary and observe actual subprocess effects in Task3. Do not add + a disconnected panic-only adapter whose non-use proves nothing. - [ ] Run `cargo test --locked --test decision_bundle` and locked all-targets; review and commit `feat: define bounded pure decision bundle replay`. @@ -133,6 +136,12 @@ historical output bytes. Omit global balances and unrelated budget members. Copy bytes only after access checks; commit the read transaction before return. Metadata/source/validator free text are sensitive too. Physical unavailability or denied access never causes an archive lookup. SQL/I/O errors are not success. +Use held-snapshot length/type checks before copying payloads or metadata beyond +the bundle limits. Unavailable/denied/oversized undisclosed selections redact +generically. CurrentRead authenticates only the request actor/scope: reject +foreign-scope output dependency closures, rather than treating row identities as +caller authority. DiagnosticReplay may authorize the exact cross-scope selection, +but still requires every input's current admission under its owning scope. - [ ] Add failing tests for default denial, authorized historical output with admitted inputs, revocation despite diagnostic approval, missing/withheld target @@ -168,6 +177,9 @@ Rust validator. Unknown modes refuse, no auto-detection that changes legacy mode - [ ] Add failing subprocess tests that compare export/replay results, binary preservation, redacted refusal, unknown validator and unchanged file hashes. - [ ] Implement CLI and versioned reader; do not add a live/generate mode. +- [ ] Retain a separate-process replay process/network syscall trace with the + installed strace where available; distinguish observed paths from a universal + proof. No fake adapter or test-only dispatch capability is needed. - [ ] Run `cargo run --locked --example decision_replay -- export-fixture PATH`, then `cargo run --locked --example decision_replay -- replay PATH` in a second process; run the new Python suite plus `test_audit_recovery_bundle.py`. diff --git a/docs/superpowers/plans/2026-11-15-sqlite-lifetimes.md b/docs/superpowers/plans/2026-11-15-sqlite-lifetimes.md new file mode 100644 index 0000000..5513d7b --- /dev/null +++ b/docs/superpowers/plans/2026-11-15-sqlite-lifetimes.md @@ -0,0 +1,52 @@ +# SQLite connection lifetime repair + +> Execute with superpowers:subagent-driven-development, test-first and independent review. + +**Goal:** close owned Python SQLite connections on every exit without changing +audit decisions, transaction behavior, stored bytes, sends or held resources. +**Architecture:** retain current transaction contexts; add deterministic resource +ownership around them. No new dependency, service, schema or model call. +**Spec:** this document and the proven diagnosis in +`/home/neo/Projects/kairotensor-artifacts/reliability-implementation-20261115/warning-diagnosis/REPORT.md`. + +## Global Constraints + +- Preserve transaction commit/rollback behavior and original raised exceptions. +- Never suppress ResourceWarning, weaken audit checks, modify retained bundles, + release uncertain holds, execute inference or connect to Rocket. +- A connection returned successfully transfers ownership to the caller; a setup + failure must close it. Source evidence and rejected fixture copies remain retained. +- Use existing admitted fixture runner; no rebuild is needed for Python changes. + Any heavy work uses the project storage wrappers and bounded profile. + +### Task 1: Deterministic connection ownership and exception regression tests + +**Allowed files:** `scripts/audit_recovery_bundle.py`, +`scripts/incident_calibration_controller.py`, +`scripts/test_audit_recovery_bundle.py`, +`scripts/test_recovery_policy_runner.py`, and create +`scripts/test_sqlite_lifetimes.py`. No other production or test files. + +- [ ] Add failing focused lifecycle tests for successful, incomplete and rejected + archived audits, proving owned handles close without changing results/errors + or bundle file hashes. Check actual connection closure as well as collected + ResourceWarnings after forced GC; warning absence alone is insufficient. +- [ ] Cover the test helper's normal and exceptional contexts, retaining its + transaction behavior. Repair directly owned fixture connections in the two + allowed existing test files so assertions/exceptions cannot bypass close. +- [ ] Reproduce `_connect` PRAGMA failure on malformed SQLite and assert the + original error survives while the opened connection closes. Also use bounded + fault injection to establish and repair `_initialize` failure after connection + transfer and `audit_run` snapshot query failure. Preserve success ownership, + commit/rollback, incomplete audit behavior and all ledger/hold decisions. +- [ ] Implement minimal closing/finally ownership; do not replace a transactional + context with a closing-only context where commit/rollback semantics are needed. +- [ ] Run new focused tests and the auditor/controller/runner tests with the + precompiled fixture runner set. Run the full script suite once at the final + candidate. Record interpreter, source hashes, runner identity, passes, skips, + residual warnings, exact commands and artifact digests. No warning filtering. +- [ ] Self-review and commit `fix: close SQLite audit and controller connections`. + +**Done:** reproduced owned-handle leaks are closed on success/early return/error, +and offline audit/accounting behavior remains verified. Residual warnings outside +the allowed scope are reported honestly, not treated as fixed. From 062725fd5874241c63446805ef898364ef942f78 Mon Sep 17 00:00:00 2001 From: Alejandro Castellanos <5405628+k2jac9@users.noreply.github.com> Date: Sun, 15 Nov 2026 15:51:52 -0500 Subject: [PATCH 12/51] fix: close SQLite audit and controller connections --- scripts/audit_recovery_bundle.py | 3 +- scripts/incident_calibration_controller.py | 53 +++--- scripts/test_audit_recovery_bundle.py | 10 +- scripts/test_recovery_policy_runner.py | 10 +- scripts/test_sqlite_lifetimes.py | 202 +++++++++++++++++++++ 5 files changed, 246 insertions(+), 32 deletions(-) create mode 100644 scripts/test_sqlite_lifetimes.py diff --git a/scripts/audit_recovery_bundle.py b/scripts/audit_recovery_bundle.py index 26d8b34..57326ba 100644 --- a/scripts/audit_recovery_bundle.py +++ b/scripts/audit_recovery_bundle.py @@ -7,6 +7,7 @@ hardware attestations. SQLite is opened mode=ro, with query_only enabled. """ import argparse +from contextlib import closing import json from pathlib import Path import sqlite3 @@ -257,7 +258,7 @@ def audit_bundle(directory): require(header[18:20] == b'\x01\x01' and not any(Path(str(db_path) + suffix).exists() for suffix in ('-wal', '-shm', '-journal')), 'WAL or journal sidecar bundle unsupported; not opened or scored') try: - with sqlite3.connect(db_path.as_uri() + '?mode=ro', uri=True) as db: + with closing(sqlite3.connect(db_path.as_uri() + '?mode=ro', uri=True)) as db, db: db.row_factory = sqlite3.Row db.execute('PRAGMA query_only=ON') db.execute('BEGIN') diff --git a/scripts/incident_calibration_controller.py b/scripts/incident_calibration_controller.py index 3181701..f4abc19 100644 --- a/scripts/incident_calibration_controller.py +++ b/scripts/incident_calibration_controller.py @@ -6,6 +6,7 @@ import argparse import base64 import copy +from contextlib import closing from dataclasses import dataclass import hashlib import json @@ -79,7 +80,11 @@ def _durable_new(path, payload, *, fail_fsync=False): def _connect(path): db = sqlite3.connect(path, timeout=0.1) - db.execute('PRAGMA journal_mode=DELETE'); db.execute('PRAGMA synchronous=FULL') + try: + db.execute('PRAGMA journal_mode=DELETE'); db.execute('PRAGMA synchronous=FULL') + except BaseException: + db.close() + raise return db @@ -92,24 +97,28 @@ def _initialize(out, plan, mode): fd = os.open(out / 'ledger.sqlite', os.O_RDWR | os.O_CREAT | os.O_EXCL, 0o600) os.close(fd) db = _connect(out / 'ledger.sqlite') - db.executescript(''' - CREATE TABLE slots(slot INTEGER PRIMARY KEY, request_utf8 TEXT NOT NULL, request_sha256 TEXT NOT NULL, - state TEXT NOT NULL CHECK(state IN ('planned','reserved','claimed','received','settled','uncertain')), - input_units INTEGER,output_cap INTEGER,handle TEXT,charged_units INTEGER, - raw_reply BLOB,receipt_ns INTEGER,late INTEGER NOT NULL DEFAULT 0); - CREATE TABLE events(id INTEGER PRIMARY KEY,event TEXT NOT NULL,slot INTEGER,payload TEXT NOT NULL,observed_ns INTEGER NOT NULL); - CREATE TRIGGER immutable_request BEFORE UPDATE OF request_utf8,request_sha256 ON slots - BEGIN SELECT RAISE(ABORT,'request binding immutable'); END; - CREATE TRIGGER immutable_events_update BEFORE UPDATE ON events - BEGIN SELECT RAISE(ABORT,'events immutable'); END; - CREATE TRIGGER immutable_events_delete BEFORE DELETE ON events - BEGIN SELECT RAISE(ABORT,'events immutable'); END; - ''') - with db: - for slot in plan['schedule']: - db.execute('INSERT INTO slots(slot,request_utf8,request_sha256,state) VALUES(?,?,?,?)', - (slot['index'], slot['request_utf8'], slot['request_sha256'], 'planned')) - _event(db, 'planned', None, {'plan_sha256': plan['plan_sha256'], 'manifest_sha256': digest(_loads((out / 'manifest.json').read_bytes()))}, mode) + try: + db.executescript(''' + CREATE TABLE slots(slot INTEGER PRIMARY KEY, request_utf8 TEXT NOT NULL, request_sha256 TEXT NOT NULL, + state TEXT NOT NULL CHECK(state IN ('planned','reserved','claimed','received','settled','uncertain')), + input_units INTEGER,output_cap INTEGER,handle TEXT,charged_units INTEGER, + raw_reply BLOB,receipt_ns INTEGER,late INTEGER NOT NULL DEFAULT 0); + CREATE TABLE events(id INTEGER PRIMARY KEY,event TEXT NOT NULL,slot INTEGER,payload TEXT NOT NULL,observed_ns INTEGER NOT NULL); + CREATE TRIGGER immutable_request BEFORE UPDATE OF request_utf8,request_sha256 ON slots + BEGIN SELECT RAISE(ABORT,'request binding immutable'); END; + CREATE TRIGGER immutable_events_update BEFORE UPDATE ON events + BEGIN SELECT RAISE(ABORT,'events immutable'); END; + CREATE TRIGGER immutable_events_delete BEFORE DELETE ON events + BEGIN SELECT RAISE(ABORT,'events immutable'); END; + ''') + with db: + for slot in plan['schedule']: + db.execute('INSERT INTO slots(slot,request_utf8,request_sha256,state) VALUES(?,?,?,?)', + (slot['index'], slot['request_utf8'], slot['request_sha256'], 'planned')) + _event(db, 'planned', None, {'plan_sha256': plan['plan_sha256'], 'manifest_sha256': digest(_loads((out / 'manifest.json').read_bytes()))}, mode) + except BaseException: + db.close() + raise return db @@ -326,9 +335,9 @@ def snapshot(): path = Path(temporary) / 'ledger.sqlite' for suffix, data in before.items(): Path(str(path)+suffix).write_bytes(data) require(snapshot() == before, 'source changed during audit snapshot') - db = sqlite3.connect(path) - rows = db.execute('SELECT slot,request_utf8,request_sha256,state,input_units,output_cap,charged_units,raw_reply,receipt_ns,late FROM slots ORDER BY slot').fetchall() - events = db.execute('SELECT event,slot,payload,observed_ns FROM events ORDER BY id').fetchall(); db.close() + with closing(sqlite3.connect(path)) as db: + rows = db.execute('SELECT slot,request_utf8,request_sha256,state,input_units,output_cap,charged_units,raw_reply,receipt_ns,late FROM slots ORDER BY slot').fetchall() + events = db.execute('SELECT event,slot,payload,observed_ns FROM events ORDER BY id').fetchall() require([r[0] for r in rows] == list(range(8)), 'exact planned slot set required') require(events and events[0][0:2] == ('planned', None), 'initial event missing') require(_loads(events[0][2]) == {'plan_sha256': manifest['plan']['plan_sha256'], 'manifest_sha256': digest(manifest)}, 'ledger manifest binding') diff --git a/scripts/test_audit_recovery_bundle.py b/scripts/test_audit_recovery_bundle.py index a8c4a04..1dce999 100644 --- a/scripts/test_audit_recovery_bundle.py +++ b/scripts/test_audit_recovery_bundle.py @@ -1,5 +1,6 @@ """Offline auditor regressions; optional real Rust fixture generation, no live calls.""" import hashlib +from contextlib import closing import json import os from pathlib import Path @@ -124,7 +125,7 @@ def corrupt(data): audit_bundle(self.folder) def test_sqlite_delivery_tamper_rejected(self): - with sqlite3.connect(self.folder / 'trace.json.sqlite') as db: + with closing(sqlite3.connect(self.folder / 'trace.json.sqlite')) as db, db: db.execute("UPDATE deliveries SET output=X'7B7D' WHERE attempt_id=1") with self.assertRaises(ValueError): audit_bundle(self.folder) @@ -150,23 +151,22 @@ def corrupt(data): audit_bundle(self.folder) def test_checkpointed_wal_bundle_is_refused_without_creating_sidecars(self): - with sqlite3.connect(self.folder / 'trace.json.sqlite') as db: + with closing(sqlite3.connect(self.folder / 'trace.json.sqlite')) as db, db: self.assertEqual(db.execute('PRAGMA journal_mode=WAL').fetchone()[0], 'wal') db.execute('PRAGMA wal_checkpoint(TRUNCATE)') - db.close() before = {p.name: hashlib.sha256(p.read_bytes()).hexdigest() for p in self.folder.iterdir()} with self.assertRaisesRegex(ValueError, 'WAL'): audit_bundle(self.folder) self.assertEqual(before, {p.name: hashlib.sha256(p.read_bytes()).hexdigest() for p in self.folder.iterdir()}) def test_mutated_database_budget_is_rejected(self): - with sqlite3.connect(self.folder / 'trace.json.sqlite') as db: + with closing(sqlite3.connect(self.folder / 'trace.json.sqlite')) as db, db: db.execute('UPDATE token_budgets SET token_limit=1 WHERE id=1') with self.assertRaisesRegex(ValueError, 'database budget limit'): audit_bundle(self.folder) def test_unresolved_sent_hold_is_not_misreported_as_zero_cost(self): - with sqlite3.connect(self.folder / 'trace.json.sqlite') as db: + with closing(sqlite3.connect(self.folder / 'trace.json.sqlite')) as db, db: db.execute('DELETE FROM deliveries WHERE attempt_id=1') result = audit_bundle(self.folder) self.assertEqual(result['status'], 'incomplete_not_scored') diff --git a/scripts/test_recovery_policy_runner.py b/scripts/test_recovery_policy_runner.py index 11aab02..28f82d0 100644 --- a/scripts/test_recovery_policy_runner.py +++ b/scripts/test_recovery_policy_runner.py @@ -6,6 +6,7 @@ """ import json import hashlib +from contextlib import closing, contextmanager from datetime import datetime import os from pathlib import Path @@ -43,10 +44,11 @@ def setUpClass(cls): def tearDownClass(cls): cls.directory.cleanup() + @contextmanager def database(self): - db = sqlite3.connect(f'{self.db_path.as_uri()}?mode=ro', uri=True) - db.row_factory = sqlite3.Row - return db + with closing(sqlite3.connect(f'{self.db_path.as_uri()}?mode=ro', uri=True)) as db, db: + db.row_factory = sqlite3.Row + yield db def test_collected_trace_scores_and_affordable_totals_reconcile_sqlite(self): self.assertEqual(self.trace['schema_version'], 2) @@ -281,7 +283,7 @@ def test_v2_calibration_only_excludes_heldout_execution_and_corpus_inputs(self): ids = {c['case_id'] for c in selected} self.assertEqual({c['case_id'] for c in raw['completed_cases']}, ids) self.assertTrue(all(e['case_id'] in ids for e in raw['events'])) - with sqlite3.connect(f'{Path(str(trace) + ".sqlite").as_uri()}?mode=ro', uri=True) as db: + with closing(sqlite3.connect(f'{Path(str(trace) + ".sqlite").as_uri()}?mode=ro', uri=True)) as db, db: keys = [r[0] for r in db.execute('SELECT recovery_key FROM recovery_runs')] self.assertEqual(set(keys), {'ci-report-recovery-v2/' + c for c in ids}) calls = db.execute('SELECT COUNT(*) FROM dispatches').fetchone()[0] diff --git a/scripts/test_sqlite_lifetimes.py b/scripts/test_sqlite_lifetimes.py new file mode 100644 index 0000000..3e79c00 --- /dev/null +++ b/scripts/test_sqlite_lifetimes.py @@ -0,0 +1,202 @@ +"""Offline SQLite ownership regressions; real handles and archived bytes only.""" +from contextlib import closing, contextmanager +import gc +import hashlib +import json +from pathlib import Path +import sqlite3 +import tempfile +import unittest +from unittest.mock import patch +import warnings +import zipfile + +from audit_recovery_bundle import audit_bundle +import incident_calibration_controller as controller +from incident_preflight import calibration_plan +import test_recovery_policy_runner as runner_tests + + +class SQLiteLifetimeTests(unittest.TestCase): + def setUp(self): + temporary = tempfile.TemporaryDirectory() + self.addCleanup(temporary.cleanup) + self.out = Path(temporary.name) + + @contextmanager + def owned_connections(self, *, factory=sqlite3.Connection): + """Observe real connections, then collect without retaining their handles.""" + connect = sqlite3.connect + connections = [] + + def open_connection(*args, **kwargs): + db = connect(*args, factory=factory, **kwargs) + connections.append(db) + return db + + gc.collect() + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter('always', ResourceWarning) + with patch.object(sqlite3, 'connect', open_connection): + yield + closed = [] + for db in connections: + try: + db.in_transaction + except sqlite3.ProgrammingError: + closed.append(True) + else: + closed.append(False) + if connections: + del db + connections.clear() + gc.collect() + resource_warnings = [str(w.message) for w in caught + if issubclass(w.category, ResourceWarning)] + self.assertTrue(closed, 'expected an owned connection') + self.assertTrue(all(closed), f'open SQLite handles: {closed}; warnings: {resource_warnings}') + self.assertEqual(resource_warnings, []) + + def archived_bundle(self): + archive = Path(__file__).resolve().parents[1] / 'docs/evaluations/recovery-policy-v1-2026-11-14.zip' + self.assertEqual(hashlib.sha256(archive.read_bytes()).hexdigest(), + '067b7b00a8ecf725b11e900eb62ea81ec72c63616e994b3e62ddae4eafec57fa') + with zipfile.ZipFile(archive) as bundle: + bundle.extractall(self.out) + + def hashes(self): + return {p.name: hashlib.sha256(p.read_bytes()).hexdigest() + for p in self.out.iterdir() if p.is_file()} + + def test_complete_archive_closes_and_preserves_score_and_bytes(self): + self.archived_bundle() + before = self.hashes() + with self.owned_connections(): + result = audit_bundle(self.out) + self.assertEqual(result['status'], 'verified_complete') + self.assertEqual(result['charged_tokens'], 2978) + self.assertEqual(result['outstanding_tokens'], 0) + # Historical producer count; this read-only audit makes no calls. + self.assertEqual(result['new_model_calls'], 20) + self.assertEqual(self.hashes(), before) + + def test_incomplete_archive_closes_without_scoring_or_changing_bytes(self): + self.archived_bundle() + path = self.out / 'trace.json.raw.json' + raw = json.loads(path.read_bytes()) + raw['status'] = 'failed' + path.write_text(json.dumps(raw)) + before = self.hashes() + with self.owned_connections(): + result = audit_bundle(self.out) + self.assertEqual(result['status'], 'incomplete_not_scored') + self.assertEqual(result['unresolved_attempts'], 0) + self.assertNotIn('summary', result) + self.assertNotIn('charged_tokens', result) + self.assertEqual(self.hashes(), before) + + def test_rejected_archive_closes_and_preserves_error_and_bytes(self): + self.archived_bundle() + path = self.out / 'trace.json.raw.json' + raw = json.loads(path.read_bytes()) + raw['actual_backend_calls'] = 999 + path.write_text(json.dumps(raw)) + before = self.hashes() + with self.owned_connections(): + with self.assertRaisesRegex(ValueError, 'bundle mismatch: backend call count'): + audit_bundle(self.out) + self.assertEqual(self.hashes(), before) + + def helper_transaction(self, exceptional): + path = self.out / 'helper.sqlite' + writable = self.out / 'writable.sqlite' + with closing(sqlite3.connect(path)): + pass + with closing(sqlite3.connect(writable)) as db: + db.execute('CREATE TABLE writes(value TEXT)') + helper = runner_tests.RecoveryPolicyRunnerTests('test_collected_trace_scores_and_affordable_totals_reconcile_sqlite') + helper.db_path = path + original = AssertionError('assertion inside helper context') + + def write(): + with helper.database() as db: + db.execute('ATTACH DATABASE ? AS writable', (str(writable),)) + db.execute("INSERT INTO writable.writes VALUES('kept')") + if exceptional: + raise original + + with self.owned_connections(): + if exceptional: + with self.assertRaises(AssertionError) as caught: + write() + self.assertIs(caught.exception, original) + original.__traceback__ = None + else: + write() + with closing(sqlite3.connect(writable)) as db: + self.assertEqual(db.execute('SELECT value FROM writes').fetchall(), + [] if exceptional else [('kept',)]) + + def test_helper_commits_and_closes_on_success(self): + self.helper_transaction(False) + + def test_helper_rolls_back_and_closes_on_assertion(self): + self.helper_transaction(True) + + def test_connect_pragma_failure_closes_and_keeps_original_database_error(self): + path = self.out / 'malformed.sqlite' + path.write_bytes(b'this is not a SQLite database') + with self.owned_connections(): + with self.assertRaisesRegex(sqlite3.DatabaseError, 'file is not a database'): + controller._connect(path) + self.assertEqual(path.read_bytes(), b'this is not a SQLite database') + + def test_initialize_event_failure_rolls_back_closes_and_keeps_original_error(self): + (self.out / 'manifest.json').write_text('{}') + original = RuntimeError('injected initial event failure') + with self.owned_connections(), patch.object(controller, '_event', side_effect=original): + with self.assertRaises(RuntimeError) as caught: + controller._initialize(self.out, calibration_plan(), controller._START_MODE) + self.assertIs(caught.exception, original) + original.__traceback__ = None + with closing(sqlite3.connect(self.out / 'ledger.sqlite')) as db: + self.assertEqual(db.execute('SELECT count(*) FROM slots').fetchone()[0], 0) + + def test_initialize_success_transfers_open_committed_connection(self): + (self.out / 'manifest.json').write_text('{}') + with closing(controller._initialize(self.out, calibration_plan(), controller._START_MODE)) as db: + self.assertEqual(db.execute('SELECT count(*) FROM slots').fetchone()[0], 8) + self.assertFalse(db.in_transaction) + self.assertEqual(db.execute('SELECT event FROM events').fetchall(), [('planned',)]) + with closing(sqlite3.connect(self.out / 'ledger.sqlite')) as db: + self.assertEqual(db.execute('SELECT count(*) FROM slots').fetchone()[0], 8) + + def test_connect_success_transfers_open_handle(self): + with closing(controller._connect(self.out / 'normal.sqlite')) as db: + self.assertEqual(db.execute('PRAGMA journal_mode').fetchone()[0], 'delete') + self.assertEqual(db.execute('PRAGMA synchronous').fetchone()[0], 2) + + def test_audit_snapshot_query_failure_closes_and_leaves_unknown_accounting(self): + (self.out / 'manifest.json').write_text('{}') + with closing(controller._initialize(self.out, calibration_plan(), controller._START_MODE)): + pass + before = self.hashes() + + class FailingSnapshot(sqlite3.Connection): + def execute(self, sql, *args, **kwargs): + if sql.startswith('SELECT slot,'): + raise sqlite3.OperationalError('injected snapshot query failure') + return super().execute(sql, *args, **kwargs) + + with self.owned_connections(factory=FailingSnapshot): + report = controller.audit_run(self.out, _manifest_validator=lambda manifest: None) + self.assertFalse(report['audit_complete']) + self.assertFalse(report['complete']) + self.assertIsNone(report['ledger_charged_units']) + self.assertIsNone(report['unresolved_hold_units']) + self.assertIn('injected snapshot query failure', report['audit_error']) + self.assertEqual(self.hashes(), before) + + +if __name__ == '__main__': + unittest.main() From 8a519e19b777cdfbd6975bbde22c95f28a6b33d5 Mon Sep 17 00:00:00 2001 From: Alejandro Castellanos <5405628+k2jac9@users.noreply.github.com> Date: Sun, 15 Nov 2026 15:56:20 -0500 Subject: [PATCH 13/51] docs: define deterministic decision bundle digest encoding --- docs/superpowers/plans/2026-11-15-decision-replay.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/docs/superpowers/plans/2026-11-15-decision-replay.md b/docs/superpowers/plans/2026-11-15-decision-replay.md index 3e5762d..63cbe97 100644 --- a/docs/superpowers/plans/2026-11-15-decision-replay.md +++ b/docs/superpowers/plans/2026-11-15-decision-replay.md @@ -73,6 +73,12 @@ charge and hold bound; unknown measured usage stays null. Never invent times. The integrity manifest binds the versioned ordered record serialization and root selection; include a digest excluding its own digest field. Preserve all original raw byte envelopes; reserialization of embedded bytes is forbidden. +Define that outer serialization as versioned compact UTF-8 JSON with recursively +sorted object keys, unchanged array order, no Unicode normalization and no ASCII +escaping of non-ASCII characters. `source_bundle_sha256` hashes this canonical +complete representation, including the integrity field, rather than claiming to +hash an arbitrary original transport file. File-byte digests remain separate +artifact receipts. Document this encoding and share exact golden bytes with C3. `ReplayReport` has independent consistency, reconstruction and validation results; recorded_disposition, validator_provenance, missing_requirements, From 43556ae6637c22e2815eff3f37092af7ee339590 Mon Sep 17 00:00:00 2001 From: Alejandro Castellanos <5405628+k2jac9@users.noreply.github.com> Date: Sun, 15 Nov 2026 16:17:24 -0500 Subject: [PATCH 14/51] feat: define bounded pure decision bundle replay --- docs/decision-bundle-v1.md | 242 +++++++++++++ src/decision_bundle.rs | 625 ++++++++++++++++++++++++++++++++++ src/decision_bundle/replay.rs | 506 +++++++++++++++++++++++++++ src/decision_bundle/types.rs | 296 ++++++++++++++++ src/lib.rs | 1 + src/report_verdict.rs | 2 +- tests/decision_bundle.rs | 606 ++++++++++++++++++++++++++++++++ 7 files changed, 2277 insertions(+), 1 deletion(-) create mode 100644 docs/decision-bundle-v1.md create mode 100644 src/decision_bundle.rs create mode 100644 src/decision_bundle/replay.rs create mode 100644 src/decision_bundle/types.rs create mode 100644 tests/decision_bundle.rs diff --git a/docs/decision-bundle-v1.md b/docs/decision-bundle-v1.md new file mode 100644 index 0000000..0afb85c --- /dev/null +++ b/docs/decision-bundle-v1.md @@ -0,0 +1,242 @@ +# Decision bundle v1 + +`kairotensor::decision_bundle` is a pure, bounded export codec and replay API. +It cannot authenticate the producer, grant current access, import into a Store, +reproduce general historical timing/publication, or dispatch a backend. SHA-256 +establishes internal consistency only. C2 owns transactional access checks and +payload redaction; C3 owns CLI/process-effect observation. + +## Public API and limits + +```rust +parse_decision_bundle(bytes: &[u8]) -> Result +validate_recorded_bundle(bundle: &DecisionBundleV1) -> ReplayReport +reevaluate_bundle(bundle: &DecisionBundleV1, validator: &str) -> ReplayReport +DecisionBundleV1::seal(&mut self) -> Result<(), BundleError> +DecisionBundleV1::to_json(&self) -> Result, BundleError> +ByteEnvelope::from_bytes(bytes: &[u8]) -> Result +ByteEnvelope::decode(&self) -> Result, BundleError> +``` + +All types are re-exported from `decision_bundle`; exact field definitions are +in `src/decision_bundle/types.rs`. Every wire struct denies unknown and duplicate +fields. Every `Option` field must be present, using JSON `null` for absence. +Arrays preserve their order. Identity values are positive signed 64-bit integers; +positions, epochs, revisions, charges and holds are nonnegative signed 64-bit +integers. Arithmetic is checked. Floating-point identities are invalid. + +Limits: 16 MiB complete JSON, 4 MiB decoded bytes per envelope, 128 attempt +records, JSON nesting depth 64. The incident codec additionally bounds its +context/request/reply to 1 MiB, with at most 1,024 incident input occurrences. +Raw parsing checks size, UTF-8 and depth before deserializing. Directly constructed +public values check envelope lengths and use a bounded counting serializer +before cloning or constructing a JSON value. Envelope bytes are decoded only +after the length and lowercase-hex checks. There is no unbounded JSON `Value` +field in the wire schema. + +Errors are payload-free `InvalidSchema`, `InvalidEncoding`, `LimitExceeded`, +`InconsistentLedger`, or `Store(rusqlite::Error)` for C2's distinct database errors. +Pure parsing and replay never open a database. Missing input/dependency records +are representable for diagnostic completeness reporting; duplicate definitions, +invalid identities, contradictory accounting and invalid digests refuse parsing. + +## Exact outer encoding and digests + +Encode recursively sorted object keys as compact UTF-8 JSON, with comma and +colon separators, no trailing newline, no Unicode normalization, and no ASCII +escaping of non-ASCII characters. Array order is unchanged. Rust's default +`serde_json::Map` ordering implements this encoding. An independent Python +encoder uses `json.dumps(value, sort_keys=True, ensure_ascii=False, +separators=(',', ':')).encode('utf-8')`. + +`integrity` is `{ "version": "decision-bundle-integrity/1", "sha256": ... }`. +Its digest hashes the canonical complete object after removing **only** the +`integrity.sha256` key. Keep `integrity.version`, root selection, all scope +metadata, and every ordered record in that hash. `seal` checks a bounded +candidate before changing this digest; it is not authorization. + +`source_bundle_sha256` hashes the canonical **complete** representation, including +`integrity.sha256`. Pretty printing or transport key order may therefore change +a file's digest without changing `source_bundle_sha256`. Record transport file +SHA-256 separately when retaining an artifact. + +Every original byte field is `{ "hex": "lowercase hex", "byte_len": integer, +"sha256": "64 lowercase hex characters" }`. Embedded bytes are never re-encoded. +In particular, runtime/timing JSON, source bytes, payloads, requests and responses +retain their original bytes, whitespace, key order and non-UTF-8 values. Replaying +a recognized contract may parse copies, but never replaces the stored envelopes. + +## Exact record map for C2 + +Top-level required fields: + +| Field | Wire type and ledger meaning | +| --- | --- | +| `schema_version` | Integer `1` | +| `root_attempt` | Selected attempt ID | +| `actor`, `scope`, `export_purpose` | Nonempty strings for the authorized export | +| `snapshot` | `{root_branch, head: nullable ID, revision, current_epoch}` from the retained root snapshot | +| `attempts` | Ordered `AttemptRecord` array; no duplicate IDs | +| `evidence` | Ordered `EvidenceRecord` array; one definition per global occurrence ID | +| `integrity` | Version and outer digest described above | + +Every `AttemptRecord` has these fields: + +| Field | Wire type / source | +| --- | --- | +| `id`, `branch` | `attempts.id`, `attempts.branch_id` | +| `actor`, `scope` | The attempt branch's own actor/scope; root must equal top-level identity | +| `captured_epoch`, `captured_revision` | `attempts.view_epoch`, `attempts.branch_revision` | +| `parent`, `inherited_from` | Nullable attempt IDs; inherited_from must equal parent if present | +| `candidates` | Ordered `{position, evidence_id}` records from `attempt_candidates.requested_id` | +| `inputs` | Ordered `{position, evidence_id, role}` records from `attempt_inputs` | +| `dispatch_request` | Nullable envelope from **actual** `dispatches.request` | +| `delivery` | Tagged state described below | +| `validator` | Nullable `{version, provenance}` | +| `immutable_runtime` | Nullable exact-byte envelope of retained `runtime_json` | +| `immutable_manifest` | Nullable exact B manifest envelope; if reconstructed, only export after its SHA-256 matches the captured runtime fingerprint | +| `timing_configuration` | Nullable exact-byte envelope of `recovery_timing_bindings.configuration`; no fabricated completion observations | +| `accounting` | Per-attempt object described below | +| `reuse_origin` | Nullable `model_reuses.origin_attempt` | +| `model_contract` | Nullable `model_attempts.contract`, preserved verbatim | +| `deterministic` | Nullable `{version, output: ByteEnvelope}` from computation/output binding | +| `support` | Nullable `{version, rule_id, verdict, clauses}`; see formula below | +| `recovery` | Nullable `{recovery_id, ordinal, state, predecessor_id, request}`; `request` is a nullable envelope of `recovery_attempts.request`, distinct from dispatch | + +Candidate positions and input positions are separate ordered sequences. The +legacy capture API compacts admitted inputs after filtering candidates, and +other exact-input attempts may have no candidate rows. Never join them by +position globally. Repeated occurrence IDs at distinct positions are preserved. +Positions must strictly increase within each sequence. Only recognized complete +contracts enforce their additional candidate/input parity requirements. + +Each `EvidenceRecord` is `{occurrence_id, source: ByteEnvelope, +payload: ByteEnvelope, authorized_scopes: [{actor, scope}, ...]}`. The scope list +is nonempty and has unique pairs. Every consuming attempt must have its own +actor/scope in the list. Shared occurrence bytes are defined once even when +multiple scopes consume them. These assertions are checked for consistency; +they do not authenticate the supplied identity or prove actual admission. + +Roles use `context`, `policy`, `report`, `evidence`, or `unknown`. The recognized +`report-verdict/v1` contract uses `report`; a recognized B manifest uses `context` +then ordered `report` entries; `support/v1` uses `evidence`. Otherwise use +`unknown`. No policy-role derivation exists in C1. Roles never justify omitting +an input or independently authorize access. + +`support` retains the immutable formula, not mutable witness counters. `clauses` +is an ordered array of `{clause_id, premises: [{evidence_id, actor, scope}]}`; +clause IDs and premise occurrence IDs within a clause must be unique. The +formula's `rule_id` is a reference, not an output dependency. A requested premise +may be absent; do not invent an evidence record for it. Support computation is +not a C1 replay validator. The recorded verdict remains historical metadata. + +Only `inherited_from`, `reuse_origin` and the bridge retry predecessor identify +required prior output records. An unconsumed `parent` is lineage only. Explicit +inherited/reuse cycles or absent outputs yield `Incomplete` and +`MissingDependency`, with bounded memoized graph traversal. Included reuse outputs +must match their dispatched accepted origin, contract, candidates and inputs. + +## Delivery, provenance and accounting + +Delivery is either: + +```json +{"state":"unresolved"} +``` + +or: + +```json +{"state":"recorded","raw":{"hex":"","byte_len":0,"sha256":"e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"},"disposition":"accepted","usage":0,"rejection":null} +``` + +Dispositions: `accepted`, `stale_view`, `stale_branch`, `invalid_output`. +A recorded ledger delivery has nonnegative measured `usage`. The Rust type is +nullable so a caller cannot accidentally default unknown usage to zero; null on +a recorded delivery refuses. An unresolved state has no usage fact. Its charge +is zero **recorded charges**, not a claim of zero model usage. + +Only invalid-output deliveries have rejection metadata: +`{validator, reason, applicability}`. Applicability is `current`, `stale_view` +or `stale_branch`. The reason is preserved, never rendered into retry feedback. +Validator provenance values: `declared_contract`, `recorded_rejection`, +`deterministic_binding`, `unknown`. Bind deterministic provenance to its exact +computation version; recorded rejection provenance must match its rejection +record. A bridge declaration does not prove that validator ran historically. + +`accounting` fields are `{unit, quote, granted, budget_reference, charge, +hold_bound}`. Quote is null or `{input_tokens, output_cap}`; quote/granted/budget +reference must all be present or all null. `output_cap > 0`; the checked sum is +the reservation bound. A known delivery's charge equals usage and its hold is +zero; reserved usage must lie between input quote and full bound. A granted +unresolved reservation keeps its full hold except a cancelled unsent recovery. +Denied reservations have no hold. Recovery state, request, dispatch, reservation +and delivery facts must be compatible. Unknown recovery phases can be retained +but cannot select a renderer. + +Units: `logical_support`, `fixture_bytes`, `model_tokens`, `unknown`. C1 recognizes +logical units only with `support/v1` or `report-verdict/v1`; fixture bytes require +the incident manifest fingerprint and immutable runtime identity +`canned-no-model`, `fixture/1`, `utf8-bytes/1`, `incident-canned/1`, `bytes`, +`canned-no-inference`. There is no allowlisted real-model token runtime in C1; +legacy and other runtime accounting stays `unknown`. Do not infer units from +sizes or arbitrary model contract strings. Budget references are opaque local +IDs, not global totals or authority. Report totals sum only exported attempts, +grouped by declared unit; `unknown` totals remain numerically recorded unknown +units and must not be interpreted as model-token or monetary totals. + +## Replay semantics and pure boundary + +Reports contain independent `consistency`, `reconstruction`, `validation`, root +`recorded_disposition`, root `validator_provenance`, `missing_requirements`, +`accounting`, `source_bundle_sha256`, `reevaluated_validator` and per-attempt +`attempts` results. Status serialization is `matched`, `mismatch`, `incomplete`, +`not_reproducible`. Aggregation prioritizes mismatch, then incomplete, then not +reproducible. An invalid directly constructed value yields consistency mismatch +without cloning its private metadata into the report. + +Missing requirements serialize as `validator_unavailable`, `renderer_unavailable`, +`missing_input`, `missing_delivery`, `missing_dependency`, +`unknown_accounting_unit`, `historical_timing_unavailable`. Historical timing is +always marked unavailable; this API does not reconstruct general past publication +eligibility from a current snapshot. + +`report-verdict/v1` invokes the existing crate-private pure report computation +and compares its exact output with the immutable output and recorded delivery. +Malformed report evidence yields mismatch, not an unavailable-validator claim. + +The incident allowlist uses the exact manifest described in +`docs/incident-store-bridge.md`: complete canonical manifest bytes, runtime +fingerprint excluding its own fingerprint field, exact runtime and calibration +cap binding, context hash, ordered complete source mappings and raw report hashes. +Ordinal 0 with no predecessor selects `Initial`; ordinal 1 with the initial +invalid-output predecessor selects `ProtocolRetry`. Fixed feedback is always +`incident-routing-protocol/1:invalid_output`. Unknown version/phase returns +renderer unavailable. No legacy or unknown-version fallback exists. +Reconstruction compares an actual dispatch, or an unsent retained recovery +preparation when no dispatch exists. Neither proves it was sent. + +Recorded validation compares actual format validity with accepted/stale versus +invalid-output disposition. Missing legacy validator bindings are not reproducible. +A changed-validator evaluation evaluates validity under that named allowlist +entry, carries the same source digest and historical disposition, and does not +mutate the bundle. It is not a revision of the historical receipt. + +The pure API receives only bytes, bundle records and a validator name. It has no +Store or backend capability, closure, filesystem, process, network or dynamic +loader argument. The implementation calls pure codecs/hash functions and the +existing pure deterministic derive function. C3 must still observe real process +and network effects; no unused fake adapter test is offered as effect evidence. + +## Frozen independent encoding example + +The following is **825 UTF-8 bytes**, without a newline. It includes literal +`é` (U+00E9). Generated independently using Python's standard `json` and `hashlib`, +and frozen in `tests/decision_bundle.rs::GOLDEN` for C3's exact oracle. + +- Integrity digest, excluding only its own key: `765fbaa97312a42a4e5da7022d6350efd2fb4811767bf763f327f6ddf0832093`. +- Complete canonical/source digest: `569b219c44295bb8e1a42358b1bbbd66c153ff379694dd55ef5decf229dbe471`. + +```json +{"actor":"é","attempts":[{"accounting":{"budget_reference":null,"charge":0,"granted":null,"hold_bound":0,"quote":null,"unit":"unknown"},"actor":"é","branch":1,"candidates":[],"captured_epoch":0,"captured_revision":0,"delivery":{"state":"unresolved"},"deterministic":null,"dispatch_request":null,"id":1,"immutable_manifest":null,"immutable_runtime":null,"inherited_from":null,"inputs":[],"model_contract":null,"parent":null,"recovery":null,"reuse_origin":null,"scope":"s","support":null,"timing_configuration":null,"validator":null}],"evidence":[],"export_purpose":"golden","integrity":{"sha256":"765fbaa97312a42a4e5da7022d6350efd2fb4811767bf763f327f6ddf0832093","version":"decision-bundle-integrity/1"},"root_attempt":1,"schema_version":1,"scope":"s","snapshot":{"current_epoch":0,"head":null,"revision":0,"root_branch":1}} +``` diff --git a/src/decision_bundle.rs b/src/decision_bundle.rs new file mode 100644 index 0000000..c01d31d --- /dev/null +++ b/src/decision_bundle.rs @@ -0,0 +1,625 @@ +//! Bounded, pure decision exports. Digests establish internal consistency only; +//! neither parsing nor replay authenticates execution or grants access. +mod replay; +mod types; +use crate::evidence_digest::sha256_hex; +pub use replay::{reevaluate_bundle, validate_recorded_bundle}; +use serde::{Deserialize, Serialize}; +use std::{ + collections::HashSet, + fmt, + io::{self, Write}, +}; +pub use types::*; + +pub const MAX_BUNDLE_BYTES: usize = 16 * 1024 * 1024; +pub const MAX_PAYLOAD_BYTES: usize = 4 * 1024 * 1024; +pub const MAX_ATTEMPTS: usize = 128; +pub const MAX_JSON_DEPTH: usize = 64; + +#[derive(Debug)] +pub enum BundleError { + InvalidSchema, + InvalidEncoding, + LimitExceeded, + InconsistentLedger, + Store(rusqlite::Error), +} +impl fmt::Display for BundleError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str(match self { + Self::InvalidSchema => "invalid decision bundle schema", + Self::InvalidEncoding => "invalid decision bundle encoding", + Self::LimitExceeded => "decision bundle limit exceeded", + Self::InconsistentLedger => "inconsistent decision ledger", + Self::Store(_) => "decision bundle store error", + }) + } +} +impl std::error::Error for BundleError {} +impl From for BundleError { + fn from(error: rusqlite::Error) -> Self { + Self::Store(error) + } +} +use BundleError as Error; + +impl ByteEnvelope { + pub fn from_bytes(bytes: &[u8]) -> Result { + if bytes.len() > MAX_PAYLOAD_BYTES { + return Err(Error::LimitExceeded); + } + let mut hex = String::with_capacity(bytes.len() * 2); + for byte in bytes { + use std::fmt::Write; + write!(hex, "{byte:02x}").unwrap(); + } + Ok(Self { + hex, + byte_len: bytes.len() as u64, + sha256: sha256_hex(bytes), + }) + } + pub fn decode(&self) -> Result, Error> { + if self.byte_len > MAX_PAYLOAD_BYTES as u64 || self.hex.len() > MAX_PAYLOAD_BYTES * 2 { + return Err(Error::LimitExceeded); + } + if self.hex.len() != self.byte_len as usize * 2 + || !lower_hex(&self.hex) + || !digest_shape(&self.sha256) + { + return Err(Error::InvalidEncoding); + } + let bytes: Vec = self + .hex + .as_bytes() + .chunks_exact(2) + .map(|pair| nibble(pair[0]) * 16 + nibble(pair[1])) + .collect(); + if sha256_hex(&bytes) != self.sha256 { + return Err(Error::InconsistentLedger); + } + Ok(bytes) + } +} +fn nibble(b: u8) -> u8 { + if b <= b'9' { + b - b'0' + } else { + b - b'a' + 10 + } +} +fn lower_hex(s: &str) -> bool { + s.bytes() + .all(|b| b.is_ascii_digit() || (b'a'..=b'f').contains(&b)) +} +fn digest_shape(s: &str) -> bool { + s.len() == 64 && lower_hex(s) +} + +// This first pass writes to a counter, not an owned JSON tree or cloned record. +// Serde stops on the first overflowing write, including escape expansion. +struct CountWriter { + size: usize, +} +impl Write for CountWriter { + fn write(&mut self, bytes: &[u8]) -> io::Result { + self.size = self + .size + .checked_add(bytes.len()) + .filter(|&n| n <= MAX_BUNDLE_BYTES) + .ok_or_else(|| io::Error::other("bundle limit"))?; + Ok(bytes.len()) + } + fn flush(&mut self) -> io::Result<()> { + Ok(()) + } +} +fn bounded_value(value: &T) -> Result { + serde_json::to_writer(&mut CountWriter { size: 0 }, value).map_err(|_| Error::LimitExceeded)?; + serde_json::to_value(value).map_err(|_| Error::InvalidSchema) +} +fn canonical(value: &serde_json::Value) -> Result, Error> { + // Default serde_json maps are BTreeMaps: recursively sorted UTF-8 keys. + serde_json::to_vec(value).map_err(|_| Error::InvalidSchema) +} +fn check_depth(raw: &[u8]) -> Result<(), Error> { + let (mut depth, mut quoted, mut escaped) = (0usize, false, false); + for &b in raw { + if quoted { + if escaped { + escaped = false; + } else if b == b'\\' { + escaped = true; + } else if b == b'"' { + quoted = false; + } + } else { + match b { + b'"' => quoted = true, + b'[' | b'{' => { + depth += 1; + if depth > MAX_JSON_DEPTH { + return Err(Error::LimitExceeded); + } + } + b']' | b'}' => { + depth = depth.checked_sub(1).ok_or(Error::InvalidSchema)?; + } + _ => {} + } + } + } + Ok(()) +} +pub(super) fn decode_strict<'a, T: Deserialize<'a>>(raw: &'a [u8]) -> Result { + if raw.len() > MAX_BUNDLE_BYTES { + return Err(Error::LimitExceeded); + } + std::str::from_utf8(raw).map_err(|_| Error::InvalidEncoding)?; + check_depth(raw)?; + serde_json::from_slice(raw).map_err(|_| Error::InvalidSchema) +} +pub fn parse_decision_bundle(bytes: &[u8]) -> Result { + let bundle: DecisionBundleV1 = decode_strict(bytes)?; + check_bundle(&bundle)?; + Ok(bundle) +} +impl DecisionBundleV1 { + /// Canonical complete bytes, with integrity checked. Not the transport file digest. + pub fn to_json(&self) -> Result, Error> { + check_bundle(self).and_then(|v| canonical(&v)) + } + /// Bind the current ordered export. This is an integrity operation, not authorization. + pub fn seal(&mut self) -> Result<(), Error> { + if self.attempts.len() > MAX_ATTEMPTS { + return Err(Error::LimitExceeded); + } + preflight_envelopes(self)?; + let mut value = bounded_value(self)?; + value["integrity"] + .as_object_mut() + .ok_or(Error::InvalidSchema)? + .remove("sha256"); + let hash = sha256_hex(&canonical(&value)?); + // Check a bounded candidate before mutating the caller's manifest. + value["integrity"]["sha256"] = hash.into(); + let candidate: Self = serde_json::from_value(value).map_err(|_| Error::InvalidSchema)?; + check_bundle(&candidate)?; + self.integrity.sha256 = candidate.integrity.sha256; + Ok(()) + } +} +fn positive(id: i64) -> Result<(), Error> { + if id > 0 { + Ok(()) + } else { + Err(Error::InconsistentLedger) + } +} +fn nonnegative(n: i64) -> Result<(), Error> { + if n >= 0 { + Ok(()) + } else { + Err(Error::InconsistentLedger) + } +} +fn nonempty(s: &str) -> Result<(), Error> { + if !s.is_empty() { + Ok(()) + } else { + Err(Error::InconsistentLedger) + } +} +fn unique_positions<'a>(positions: impl Iterator + 'a) -> Result<(), Error> { + let mut prior = None; + for (position, id) in positions { + nonnegative(position)?; + positive(id)?; + if prior.is_some_and(|p| p >= position) { + return Err(Error::InconsistentLedger); + } + prior = Some(position); + } + Ok(()) +} +fn check_accounting(a: &AttemptRecord) -> Result<(), Error> { + let q = &a.accounting; + nonnegative(q.charge)?; + nonnegative(q.hold_bound)?; + let bound = match (&q.quote, q.granted, q.budget_reference) { + (None, None, None) => None, + (Some(quote), Some(granted), Some(budget)) => { + positive(budget)?; + nonnegative(quote.input_tokens)?; + positive(quote.output_cap)?; + let cap = quote + .input_tokens + .checked_add(quote.output_cap) + .ok_or(Error::InconsistentLedger)?; + Some((quote.input_tokens, cap, granted)) + } + _ => return Err(Error::InconsistentLedger), + }; + match &a.delivery { + DeliveryState::Unresolved => { + if q.charge != 0 + || q.hold_bound + != bound + .filter(|b| { + b.2 && !a.recovery.as_ref().is_some_and(|r| r.state == "cancelled") + }) + .map_or(0, |b| b.1) + { + return Err(Error::InconsistentLedger); + } + } + DeliveryState::Recorded { + usage, + disposition, + rejection, + .. + } => { + let usage = usage.ok_or(Error::InconsistentLedger)?; + nonnegative(usage)?; + if usage != q.charge || q.hold_bound != 0 { + return Err(Error::InconsistentLedger); + } + if bound.is_some_and(|(input, cap, granted)| !granted || usage < input || usage > cap) { + return Err(Error::InconsistentLedger); + } + match (disposition, rejection) { + (RecordedDisposition::InvalidOutput, Some(r)) => { + nonempty(&r.validator)?; + nonempty(&r.reason)?; + if !matches!( + r.applicability.as_str(), + "current" | "stale_view" | "stale_branch" + ) { + return Err(Error::InconsistentLedger); + } + } + (RecordedDisposition::InvalidOutput, None) => { + return Err(Error::InconsistentLedger) + } + (_, Some(_)) => return Err(Error::InconsistentLedger), + _ => {} + } + } + } + Ok(()) +} +fn check_bundle(b: &DecisionBundleV1) -> Result { + if b.attempts.len() > MAX_ATTEMPTS { + return Err(Error::LimitExceeded); + } + // Bound direct constructions before any clone, hash buffer, or owned Value. + preflight_envelopes(b)?; + let mut value = bounded_value(b)?; + if b.schema_version != 1 || b.integrity.version != "decision-bundle-integrity/1" { + return Err(Error::InvalidSchema); + } + positive(b.root_attempt)?; + positive(b.snapshot.root_branch)?; + if let Some(id) = b.snapshot.head { + positive(id)?; + } + nonnegative(b.snapshot.revision)?; + nonnegative(b.snapshot.current_epoch)?; + nonempty(&b.actor)?; + nonempty(&b.scope)?; + nonempty(&b.export_purpose)?; + if !digest_shape(&b.integrity.sha256) { + return Err(Error::InvalidEncoding); + } + let mut ids = HashSet::new(); + for e in &b.evidence { + positive(e.occurrence_id)?; + if !ids.insert(e.occurrence_id) { + return Err(Error::InconsistentLedger); + } + let mut scopes = HashSet::new(); + if e.authorized_scopes.is_empty() { + return Err(Error::InconsistentLedger); + } + for scope in &e.authorized_scopes { + nonempty(&scope.actor)?; + nonempty(&scope.scope)?; + if !scopes.insert((&scope.actor, &scope.scope)) { + return Err(Error::InconsistentLedger); + } + } + e.source.decode()?; + e.payload.decode()?; + } + ids.clear(); + for a in &b.attempts { + nonempty(&a.actor)?; + nonempty(&a.scope)?; + positive(a.id)?; + positive(a.branch)?; + nonnegative(a.captured_epoch)?; + nonnegative(a.captured_revision)?; + if !ids.insert(a.id) { + return Err(Error::InconsistentLedger); + } + for id in [a.parent, a.inherited_from, a.reuse_origin] + .into_iter() + .flatten() + { + positive(id)?; + } + if a.inherited_from.is_some() && a.inherited_from != a.parent { + return Err(Error::InconsistentLedger); + } + unique_positions(a.inputs.iter().map(|i| (i.position, i.evidence_id)))?; + unique_positions(a.candidates.iter().map(|i| (i.position, i.evidence_id)))?; + for input in &a.inputs { + if b.evidence + .iter() + .find(|e| e.occurrence_id == input.evidence_id) + .is_some_and(|e| { + !e.authorized_scopes + .iter() + .any(|s| s.actor == a.actor && s.scope == a.scope) + }) + { + return Err(Error::InconsistentLedger); + } + } + for raw in [ + &a.dispatch_request, + &a.immutable_runtime, + &a.immutable_manifest, + &a.timing_configuration, + ] + .into_iter() + .flatten() + { + raw.decode()?; + } + if let DeliveryState::Recorded { raw, .. } = &a.delivery { + raw.decode()?; + } + check_accounting(a)?; + check_metadata(a)?; + if let Some(d) = &a.deterministic { + nonempty(&d.version)?; + d.output.decode()?; + if a.dispatch_request.is_some() + || a.accounting.quote.is_some() + || a.model_contract.is_some() + || a.recovery.is_some() + || a.accounting.charge != 0 + { + return Err(Error::InconsistentLedger); + } + if let DeliveryState::Recorded { + raw, disposition, .. + } = &a.delivery + { + if *raw != d.output || *disposition == RecordedDisposition::InvalidOutput { + return Err(Error::InconsistentLedger); + } + } + } + if a.reuse_origin.is_some() + && (a.model_contract.is_none() + || a.dispatch_request.is_some() + || a.accounting.charge != 0) + { + return Err(Error::InconsistentLedger); + } + if let Some(s) = &a.support { + nonempty(&s.version)?; + positive(s.rule_id)?; + let mut clauses = HashSet::new(); + for c in &s.clauses { + positive(c.clause_id)?; + if !clauses.insert(c.clause_id) || c.premises.is_empty() { + return Err(Error::InconsistentLedger); + } + let mut premises = HashSet::new(); + for p in &c.premises { + positive(p.evidence_id)?; + nonempty(&p.actor)?; + nonempty(&p.scope)?; + if !premises.insert(p.evidence_id) { + return Err(Error::InconsistentLedger); + } + } + } + } + if let Some(r) = &a.recovery { + positive(r.recovery_id)?; + nonnegative(r.ordinal)?; + nonempty(&r.state)?; + if let Some(request) = &r.request { + request.decode()?; + } + if let Some(dispatched) = &a.dispatch_request { + if r.request.as_ref() != Some(dispatched) || r.state != "sent" { + return Err(Error::InconsistentLedger); + } + } + if matches!(r.state.as_str(), "reserved" | "sent" | "denied") && r.request.is_none() { + return Err(Error::InconsistentLedger); + } + if r.state == "sent" && a.dispatch_request.is_none() { + return Err(Error::InconsistentLedger); + } + let reservation_valid = match r.state.as_str() { + "planned" => a.accounting.quote.is_none() && r.request.is_none(), + "reserved" | "sent" => a.accounting.granted == Some(true), + "denied" => a.accounting.granted == Some(false), + "cancelled" => a.accounting.granted != Some(false), + _ => true, // Retain unknown historical phase; renderer will refuse. + }; + if !reservation_valid { + return Err(Error::InconsistentLedger); + } + + if let Some(id) = r.predecessor_id { + positive(id)?; + } + if matches!(a.delivery, DeliveryState::Recorded { .. }) && r.state != "sent" { + return Err(Error::InconsistentLedger); + } + } + } + for a in &b.attempts { + if let Some(origin) = a + .reuse_origin + .and_then(|id| b.attempts.iter().find(|d| d.id == id)) + { + if let ( + DeliveryState::Recorded { raw, .. }, + DeliveryState::Recorded { + raw: prior, + disposition, + .. + }, + ) = (&a.delivery, &origin.delivery) + { + if raw != prior + || *disposition != RecordedDisposition::Accepted + || origin.dispatch_request.is_none() + || a.model_contract != origin.model_contract + || a.inputs != origin.inputs + || a.candidates != origin.candidates + { + return Err(Error::InconsistentLedger); + } + } + } + } + let root = b + .attempts + .iter() + .find(|a| a.id == b.root_attempt) + .ok_or(Error::InconsistentLedger)?; + if root.branch != b.snapshot.root_branch || root.actor != b.actor || root.scope != b.scope { + return Err(Error::InconsistentLedger); + } + value["integrity"].as_object_mut().unwrap().remove("sha256"); + if sha256_hex(&canonical(&value)?) != b.integrity.sha256 { + return Err(Error::InconsistentLedger); + } + value["integrity"]["sha256"] = b.integrity.sha256.clone().into(); + // Aggregate arithmetic checked independently for each unit; no global budget totals. + for unit in [ + AccountingUnit::LogicalSupport, + AccountingUnit::FixtureBytes, + AccountingUnit::ModelTokens, + AccountingUnit::Unknown, + ] { + let (mut charge, mut held) = (0i64, 0i64); + for a in b.attempts.iter().filter(|a| a.accounting.unit == unit) { + charge = charge + .checked_add(a.accounting.charge) + .ok_or(Error::InconsistentLedger)?; + held = held + .checked_add(a.accounting.hold_bound) + .ok_or(Error::InconsistentLedger)?; + } + charge.checked_add(held).ok_or(Error::InconsistentLedger)?; + } + Ok(value) +} + +// Check every envelope's borrowed lengths before the outer JSON allocation. +fn preflight_envelopes(b: &DecisionBundleV1) -> Result<(), Error> { + fn check(e: &ByteEnvelope) -> Result<(), Error> { + if e.byte_len > MAX_PAYLOAD_BYTES as u64 || e.hex.len() > MAX_PAYLOAD_BYTES * 2 { + Err(Error::LimitExceeded) + } else { + Ok(()) + } + } + for e in &b.evidence { + check(&e.source)?; + check(&e.payload)?; + } + for a in &b.attempts { + for e in [ + &a.dispatch_request, + &a.immutable_runtime, + &a.immutable_manifest, + &a.timing_configuration, + ] + .into_iter() + .flatten() + { + check(e)?; + } + if let DeliveryState::Recorded { raw, .. } = &a.delivery { + check(raw)?; + } + if let Some(d) = &a.deterministic { + check(&d.output)?; + } + if let Some(r) = &a.recovery { + if let Some(raw) = &r.request { + check(raw)?; + } + } + } + Ok(()) +} +fn check_metadata(a: &AttemptRecord) -> Result<(), Error> { + let report = a + .deterministic + .as_ref() + .is_some_and(|d| d.version == "report-verdict/v1"); + let support = a + .support + .as_ref() + .is_some_and(|s| s.version == "support/v1"); + let incident = a + .immutable_manifest + .as_ref() + .and_then(|e| e.decode().ok()) + .is_some_and(|raw| replay::recognized_incident_manifest(&raw)); + for (index, input) in a.inputs.iter().enumerate() { + let expected = if report { + InputRole::Report + } else if incident { + if index == 0 { + InputRole::Context + } else { + InputRole::Report + } + } else if support { + InputRole::Evidence + } else { + InputRole::Unknown + }; + if input.role != expected { + return Err(Error::InconsistentLedger); + } + } + match a.accounting.unit { + AccountingUnit::Unknown => {} + AccountingUnit::LogicalSupport if report || support => {} + AccountingUnit::FixtureBytes if incident && replay::fixture_runtime(a) => {} + _ => return Err(Error::InconsistentLedger), + } + if let Some(v) = &a.validator { + nonempty(&v.version)?; + let valid = match v.provenance { + ValidatorProvenance::Unknown => true, + ValidatorProvenance::DeterministicBinding => a + .deterministic + .as_ref() + .is_some_and(|d| d.version == v.version), + ValidatorProvenance::DeclaredContract => a.immutable_manifest.is_some(), + ValidatorProvenance::RecordedRejection => { + matches!(&a.delivery,DeliveryState::Recorded{rejection:Some(r),..} if r.validator==v.version) + } + }; + if !valid { + return Err(Error::InconsistentLedger); + } + } + Ok(()) +} diff --git a/src/decision_bundle/replay.rs b/src/decision_bundle/replay.rs new file mode 100644 index 0000000..79f208f --- /dev/null +++ b/src/decision_bundle/replay.rs @@ -0,0 +1,506 @@ +//! Pure allowlist replay. No Store, adapter, callback, filesystem or process API. +use super::*; +use crate::incident_protocol::{ + render_incident_request, validate_incident_output, IncidentContextV1, IncidentPhase, + IncidentReport, +}; +use crate::{RecoveryRuntime, RecoveryTiming}; +use serde::Deserialize; + +fn missing(report: &mut ReplayReport, requirement: MissingRequirement) { + if !report.missing_requirements.contains(&requirement) { + report.missing_requirements.push(requirement); + } +} +fn merge(a: ReplayStatus, b: ReplayStatus) -> ReplayStatus { + use ReplayStatus::*; + if a == Mismatch || b == Mismatch { + Mismatch + } else if a == Incomplete || b == Incomplete { + Incomplete + } else if a == NotReproducible || b == NotReproducible { + NotReproducible + } else { + Matched + } +} +fn disposition(a: &AttemptRecord) -> Option { + match a.delivery { + DeliveryState::Recorded { disposition, .. } => Some(disposition), + DeliveryState::Unresolved => None, + } +} +fn provenance(a: &AttemptRecord) -> Option { + if let DeliveryState::Recorded { + rejection: Some(_), .. + } = &a.delivery + { + Some(ValidatorProvenance::RecordedRejection) + } else { + a.validator.as_ref().map(|v| v.provenance) + } +} +/// Evaluate captured versions only; historical publication and timing are not replayed. +pub fn validate_recorded_bundle(bundle: &DecisionBundleV1) -> ReplayReport { + replay(bundle, None) +} +/// A separate linked evaluation; the original bundle and disposition stay unchanged. +pub fn reevaluate_bundle(bundle: &DecisionBundleV1, validator: &str) -> ReplayReport { + replay(bundle, Some(validator)) +} +fn replay(b: &DecisionBundleV1, new_validator: Option<&str>) -> ReplayReport { + let mut result = ReplayReport { + consistency: ReplayStatus::Mismatch, + reconstruction: ReplayStatus::NotReproducible, + validation: ReplayStatus::NotReproducible, + recorded_disposition: None, + validator_provenance: None, + missing_requirements: vec![], + accounting: vec![], + source_bundle_sha256: None, + reevaluated_validator: None, + attempts: vec![], + }; + // Even report metadata is cloned only after bounded public-value validation. + if new_validator.is_some_and(|v| v.len() > MAX_PAYLOAD_BYTES) { + return result; + } + let Ok(value) = check_bundle(b) else { + return result; + }; + result.source_bundle_sha256 = canonical(&value).ok().map(|bytes| sha256_hex(&bytes)); + result.reevaluated_validator = new_validator.map(str::to_owned); + result.consistency = ReplayStatus::Matched; + result.reconstruction = ReplayStatus::Matched; + result.validation = ReplayStatus::Matched; + missing(&mut result, MissingRequirement::HistoricalTimingUnavailable); + for a in &b.attempts { + let unit = a.accounting.unit; + if unit == AccountingUnit::Unknown { + missing(&mut result, MissingRequirement::UnknownAccountingUnit); + } + if let Some(t) = result.accounting.iter_mut().find(|t| t.unit == unit) { + t.charged += a.accounting.charge; + t.held += a.accounting.hold_bound; + } else { + result.accounting.push(UnitAccounting { + unit, + charged: a.accounting.charge, + held: a.accounting.hold_bound, + }); + } + let mut item = AttemptReplay { + attempt_id: a.id, + reconstruction: ReplayStatus::NotReproducible, + validation: ReplayStatus::NotReproducible, + recorded_disposition: disposition(a), + validator_provenance: provenance(a), + }; + let inputs_missing = a + .inputs + .iter() + .any(|i| !b.evidence.iter().any(|e| e.occurrence_id == i.evidence_id)); + let dependencies_missing = + !dependencies_complete(b, a, &mut Vec::new(), &mut HashSet::new()); + if inputs_missing { + missing(&mut result, MissingRequirement::MissingInput); + } + if dependencies_missing { + missing(&mut result, MissingRequirement::MissingDependency); + } + if inputs_missing || dependencies_missing { + item.reconstruction = ReplayStatus::Incomplete; + item.validation = ReplayStatus::Incomplete; + } else { + let (reconstruction, validation) = evaluate(b, a, new_validator, &mut result); + item.reconstruction = reconstruction; + item.validation = validation; + } + if matches!(a.delivery, DeliveryState::Unresolved) { + missing(&mut result, MissingRequirement::MissingDelivery); + item.validation = ReplayStatus::Incomplete; + } + result.reconstruction = merge(result.reconstruction, item.reconstruction); + result.validation = merge(result.validation, item.validation); + if a.id == b.root_attempt { + result.recorded_disposition = item.recorded_disposition; + result.validator_provenance = item.validator_provenance; + } + result.attempts.push(item); + } + result +} +fn dependencies_complete( + b: &DecisionBundleV1, + a: &AttemptRecord, + path: &mut Vec, + complete_ids: &mut HashSet, +) -> bool { + if complete_ids.contains(&a.id) { + return true; + } + if path.contains(&a.id) { + return false; + } + path.push(a.id); + let complete = [a.inherited_from, a.reuse_origin] + .into_iter() + .flatten() + .all(|id| { + b.attempts.iter().find(|d| d.id == id).is_some_and(|d| { + matches!(d.delivery, DeliveryState::Recorded { .. }) + && dependencies_complete(b, d, path, complete_ids) + }) + }); + path.pop(); + if complete { + complete_ids.insert(a.id); + } + complete +} +fn payload(b: &DecisionBundleV1, id: i64) -> Result, Error> { + b.evidence + .iter() + .find(|e| e.occurrence_id == id) + .ok_or(Error::InconsistentLedger)? + .payload + .decode() +} +fn evaluate( + b: &DecisionBundleV1, + a: &AttemptRecord, + new_validator: Option<&str>, + report: &mut ReplayReport, +) -> (ReplayStatus, ReplayStatus) { + use ReplayStatus::*; + let mut reconstruction = NotReproducible; + let mut derived = None; + let mut incident = None; + if let Some(d) = &a.deterministic { + if d.version == "report-verdict/v1" { + derived = match a.inputs.as_slice() { + [input] + if a.candidates.len() == 1 + && a.candidates[0].evidence_id == input.evidence_id => + { + payload(b, input.evidence_id) + .ok() + .and_then(|raw| crate::report_verdict::derive(&raw).ok()) + } + _ => None, + }; + reconstruction = if derived + .as_ref() + .is_some_and(|bytes| d.output.decode().ok().as_ref() == Some(bytes)) + { + Matched + } else { + Mismatch + }; + } else { + missing(report, MissingRequirement::RendererUnavailable); + } + } else if a.immutable_manifest.is_some() { + match reconstruct_incident(b, a) { + Ok((context, request)) => { + reconstruction = match a + .dispatch_request + .as_ref() + .or_else(|| a.recovery.as_ref().and_then(|r| r.request.as_ref())) + { + Some(raw) => { + if raw.decode().ok().as_ref() == Some(&request) { + Matched + } else { + Mismatch + } + } + None => { + missing(report, MissingRequirement::MissingInput); + Incomplete + } + }; + incident = Some(context); + } + Err(IncidentFailure::Unavailable) => { + missing(report, MissingRequirement::RendererUnavailable); + } + Err(IncidentFailure::Missing) => { + missing(report, MissingRequirement::MissingDependency); + reconstruction = Incomplete; + } + Err(IncidentFailure::Mismatch) => { + reconstruction = Mismatch; + } + } + } else { + missing(report, MissingRequirement::RendererUnavailable); + } + let selected = new_validator.or_else(|| match &a.delivery { + DeliveryState::Recorded { + rejection: Some(r), .. + } => Some(r.validator.as_str()), + _ => a + .validator + .as_ref() + .map(|v| v.version.as_str()) + .or_else(|| a.deterministic.as_ref().map(|d| d.version.as_str())), + }); + let DeliveryState::Recorded { + raw, disposition, .. + } = &a.delivery + else { + return (reconstruction, Incomplete); + }; + let raw = raw.decode().expect("checked byte envelope"); + let evaluation = match selected { + Some("report-verdict/v1") + if a.deterministic + .as_ref() + .is_some_and(|d| d.version == "report-verdict/v1") => + { + Some(derived.as_ref().is_some_and(|output| *output == raw)) + } + Some("incident-routing-protocol/1") => incident + .as_ref() + .map(|context| validate_incident_output(&raw, context).is_ok()), + _ => None, + }; + let validation = match evaluation { + Some(valid) => { + // Changed-validator evaluation describes validity itself. Recorded replay + // compares format validity with the recorded acceptance/rejection class. + let expected = if new_validator.is_some() { + true + } else { + *disposition != RecordedDisposition::InvalidOutput + }; + if valid == expected { + Matched + } else { + Mismatch + } + } + None => { + missing(report, MissingRequirement::ValidatorUnavailable); + NotReproducible + } + }; + (reconstruction, validation) +} + +#[derive(Deserialize)] +#[serde(deny_unknown_fields)] +struct Manifest { + context: ManifestContext, + freeze_sha256: String, + initial_renderer: String, + output_cap: i64, + prompt_sha256: String, + retry_feedback: String, + retry_renderer: String, + runtime: ManifestRuntime, + schema_version: u32, + sources: Vec, + validator_version: String, + version: String, + workload_version: String, +} +#[derive(Deserialize)] +#[serde(deny_unknown_fields)] +struct ManifestContext { + occurrence_id: i64, + sha256: String, +} +#[derive(Deserialize)] +#[serde(deny_unknown_fields)] +struct ManifestSource { + case_id: String, + external_id: String, + occurrence_id: i64, + payload_sha256: String, + workload_version: String, +} +#[derive(Deserialize, Serialize)] +#[serde(deny_unknown_fields)] +struct ManifestRuntime { + cpu_fingerprint: String, + decoding: String, + dtype: String, + model: String, + model_revision: String, + readiness: crate::RecoveryReadiness, + runtime_version: String, + threads: i64, + tokenizer_revision: String, +} +enum IncidentFailure { + Unavailable, + Missing, + Mismatch, +} +fn reconstruct_incident( + b: &DecisionBundleV1, + a: &AttemptRecord, +) -> Result<(IncidentContextV1, Vec), IncidentFailure> { + use IncidentFailure::*; + let envelope = a.immutable_manifest.as_ref().ok_or(Missing)?; + let manifest_bytes = envelope.decode().map_err(|_| Mismatch)?; + let manifest: Manifest = decode_strict(&manifest_bytes).map_err(|_| Unavailable)?; + if manifest.schema_version != 1 + || manifest.version != "incident-store-manifest/1" + || manifest.workload_version != "incident-routing/1" + || manifest.validator_version != "incident-routing-protocol/1" + || manifest.initial_renderer != "incident-request/1" + || manifest.retry_renderer != "incident-request-retry/1" + || manifest.retry_feedback != "incident-routing-protocol/1:invalid_output" + { + return Err(Unavailable); + } + let recovery = a.recovery.as_ref().ok_or(Missing)?; + if !matches!( + recovery.state.as_str(), + "planned" | "reserved" | "sent" | "cancelled" | "denied" + ) { + return Err(Unavailable); + } + let phase = match recovery.ordinal { + 0 if recovery.predecessor_id.is_none() => IncidentPhase::Initial, + 1 if recovery.predecessor_id.is_some() => IncidentPhase::ProtocolRetry, + _ => return Err(Unavailable), + }; + if let Some(id) = recovery.predecessor_id { + let previous = b.attempts.iter().find(|p| p.id == id).ok_or(Missing)?; + if !previous + .recovery + .as_ref() + .is_some_and(|r| r.recovery_id == recovery.recovery_id && r.ordinal == 0) + || disposition(previous) != Some(RecordedDisposition::InvalidOutput) + || previous.immutable_manifest != a.immutable_manifest + { + return Err(Mismatch); + } + } + let runtime_bytes = a + .immutable_runtime + .as_ref() + .ok_or(Missing)? + .decode() + .map_err(|_| Mismatch)?; + let runtime: RecoveryRuntime = decode_strict(&runtime_bytes).map_err(|_| Mismatch)?; + let timing_bytes = a + .timing_configuration + .as_ref() + .ok_or(Missing)? + .decode() + .map_err(|_| Mismatch)?; + let timing: RecoveryTiming = decode_strict(&timing_bytes).map_err(|_| Mismatch)?; + if runtime != timing.runtime + || runtime.prompt_fingerprint != sha256_hex(&manifest_bytes) + || manifest.output_cap != timing.calibration.output_cap + || manifest.output_cap <= 0 + || runtime.threads <= 0 + { + return Err(Mismatch); + } + let mut runtime_value = serde_json::to_value(&runtime).map_err(|_| Mismatch)?; + runtime_value + .as_object_mut() + .unwrap() + .remove("prompt_fingerprint"); + if runtime_value != serde_json::to_value(&manifest.runtime).map_err(|_| Mismatch)? { + return Err(Mismatch); + } + // Exact canonical manifest encoding is part of B's fingerprint binding. + let manifest_value: serde_json::Value = + serde_json::from_slice(&manifest_bytes).map_err(|_| Mismatch)?; + if canonical(&manifest_value).map_err(|_| Mismatch)? != manifest_bytes { + return Err(Mismatch); + } + let Some(first) = a.inputs.first() else { + return Err(Missing); + }; + if a.inputs.len() > 1024 + || a.inputs.len() != manifest.sources.len() + 1 + || first.evidence_id != manifest.context.occurrence_id + || a.candidates.len() != a.inputs.len() + || a.candidates + .iter() + .zip(&a.inputs) + .any(|(c, i)| c.evidence_id != i.evidence_id || c.position != i.position) + || a.inputs + .iter() + .enumerate() + .any(|(position, i)| i.position != position as i64) + { + return Err(Mismatch); + } + let context_bytes = payload(b, first.evidence_id).map_err(|_| Missing)?; + if sha256_hex(&context_bytes) != manifest.context.sha256 { + return Err(Mismatch); + } + let context = IncidentContextV1::from_json(&context_bytes).map_err(|_| Mismatch)?; + if context.freeze_sha256 != manifest.freeze_sha256 + || context.prompt_sha256 != manifest.prompt_sha256 + || context.workload_version != manifest.workload_version + || context.validator_version != manifest.validator_version + || context.evidence_ids.len() != manifest.sources.len() + { + return Err(Mismatch); + } + let mut reports = Vec::new(); + let mut ids = HashSet::new(); + ids.insert(first.evidence_id); + for ((input, source), external) in a.inputs[1..] + .iter() + .zip(&manifest.sources) + .zip(&context.evidence_ids) + { + if !ids.insert(input.evidence_id) + || input.evidence_id != source.occurrence_id + || source.case_id != context.case_id + || source.external_id != *external + || source.workload_version != manifest.workload_version + { + return Err(Mismatch); + } + let raw = payload(b, input.evidence_id).map_err(|_| Missing)?; + if sha256_hex(&raw) != source.payload_sha256 { + return Err(Mismatch); + } + let text = String::from_utf8(raw).map_err(|_| Mismatch)?; + reports.push(IncidentReport { + external_id: source.external_id.clone(), + text, + }); + } + let request = render_incident_request(&context, &reports, phase).map_err(|_| Mismatch)?; + Ok((context, request)) +} + +pub(super) fn recognized_incident_manifest(raw: &[u8]) -> bool { + decode_strict::(raw).is_ok_and(|m| { + m.schema_version == 1 + && m.version == "incident-store-manifest/1" + && m.workload_version == "incident-routing/1" + && m.validator_version == "incident-routing-protocol/1" + }) +} + +pub(super) fn fixture_runtime(a: &AttemptRecord) -> bool { + let Some(raw) = a.immutable_runtime.as_ref().and_then(|e| e.decode().ok()) else { + return false; + }; + let Ok(runtime) = decode_strict::(&raw) else { + return false; + }; + runtime.model == "canned-no-model" + && runtime.model_revision == "fixture/1" + && runtime.tokenizer_revision == "utf8-bytes/1" + && runtime.runtime_version == "incident-canned/1" + && runtime.dtype == "bytes" + && runtime.decoding == "canned-no-inference" + && a.immutable_manifest + .as_ref() + .is_some_and(|m| runtime.prompt_fingerprint == m.sha256) +} diff --git a/src/decision_bundle/types.rs b/src/decision_bundle/types.rs new file mode 100644 index 0000000..d078c5b --- /dev/null +++ b/src/decision_bundle/types.rs @@ -0,0 +1,296 @@ +//! Version 1 wire records. Optional facts are explicit JSON nulls, never defaults. +use serde::{Deserialize, Serialize}; + +fn required<'de, D: serde::Deserializer<'de>, T: Deserialize<'de>>( + d: D, +) -> Result, D::Error> { + Option::::deserialize(d) +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct DecisionBundleV1 { + pub schema_version: u32, + pub root_attempt: i64, + pub actor: String, + pub scope: String, + pub export_purpose: String, + pub snapshot: BundleSnapshot, + pub attempts: Vec, + pub evidence: Vec, + pub integrity: IntegrityManifest, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct BundleSnapshot { + pub root_branch: i64, + #[serde(deserialize_with = "required")] + pub head: Option, + pub revision: i64, + pub current_epoch: i64, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct IntegrityManifest { + pub version: String, + pub sha256: String, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct ByteEnvelope { + pub hex: String, + pub byte_len: u64, + pub sha256: String, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct AuthorizedScope { + pub actor: String, + pub scope: String, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct EvidenceRecord { + pub occurrence_id: i64, + pub source: ByteEnvelope, + pub payload: ByteEnvelope, + pub authorized_scopes: Vec, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct CandidateRecord { + pub position: i64, + pub evidence_id: i64, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct InputRecord { + pub position: i64, + pub evidence_id: i64, + pub role: InputRole, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct AttemptRecord { + pub actor: String, + pub scope: String, + pub id: i64, + pub branch: i64, + pub captured_epoch: i64, + pub captured_revision: i64, + #[serde(deserialize_with = "required")] + pub parent: Option, + #[serde(deserialize_with = "required")] + pub inherited_from: Option, + pub candidates: Vec, + pub inputs: Vec, + #[serde(deserialize_with = "required")] + pub dispatch_request: Option, + pub delivery: DeliveryState, + #[serde(deserialize_with = "required")] + pub validator: Option, + #[serde(deserialize_with = "required")] + pub immutable_runtime: Option, + #[serde(deserialize_with = "required")] + pub immutable_manifest: Option, + #[serde(deserialize_with = "required")] + pub timing_configuration: Option, + pub accounting: AttemptAccounting, + #[serde(deserialize_with = "required")] + pub reuse_origin: Option, + #[serde(deserialize_with = "required")] + pub model_contract: Option, + #[serde(deserialize_with = "required")] + pub deterministic: Option, + #[serde(deserialize_with = "required")] + pub support: Option, + #[serde(deserialize_with = "required")] + pub recovery: Option, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct ValidatorBinding { + pub version: String, + pub provenance: ValidatorProvenance, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct RejectionMetadata { + pub validator: String, + pub reason: String, + pub applicability: String, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct AttemptAccounting { + pub unit: AccountingUnit, + #[serde(deserialize_with = "required")] + pub quote: Option, + #[serde(deserialize_with = "required")] + pub granted: Option, + #[serde(deserialize_with = "required")] + pub budget_reference: Option, + pub charge: i64, + pub hold_bound: i64, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct ReservationQuote { + pub input_tokens: i64, + pub output_cap: i64, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct DeterministicContract { + pub version: String, + pub output: ByteEnvelope, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct SupportContract { + pub version: String, + pub rule_id: i64, + pub verdict: bool, + pub clauses: Vec, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct SupportClause { + pub clause_id: i64, + pub premises: Vec, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct SupportPremise { + pub evidence_id: i64, + pub actor: String, + pub scope: String, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct RecoveryRecord { + #[serde(deserialize_with = "required")] + pub request: Option, + pub recovery_id: i64, + pub ordinal: i64, + pub state: String, + #[serde(deserialize_with = "required")] + pub predecessor_id: Option, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum InputRole { + Context, + Policy, + Report, + Evidence, + Unknown, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum RecordedDisposition { + Accepted, + StaleView, + StaleBranch, + InvalidOutput, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum ValidatorProvenance { + DeclaredContract, + RecordedRejection, + DeterministicBinding, + Unknown, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum AccountingUnit { + LogicalSupport, + FixtureBytes, + ModelTokens, + Unknown, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum ReplayStatus { + Matched, + Mismatch, + Incomplete, + NotReproducible, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum MissingRequirement { + ValidatorUnavailable, + RendererUnavailable, + MissingInput, + MissingDelivery, + MissingDependency, + UnknownAccountingUnit, + HistoricalTimingUnavailable, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(tag = "state", rename_all = "snake_case", deny_unknown_fields)] +pub enum DeliveryState { + Recorded { + raw: ByteEnvelope, + disposition: RecordedDisposition, + #[serde(deserialize_with = "required")] + usage: Option, + #[serde(deserialize_with = "required")] + rejection: Option, + }, + Unresolved, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize)] +pub struct UnitAccounting { + pub unit: AccountingUnit, + pub charged: i64, + pub held: i64, +} +#[derive(Clone, Debug, PartialEq, Eq, Serialize)] +pub struct AttemptReplay { + pub attempt_id: i64, + pub reconstruction: ReplayStatus, + pub validation: ReplayStatus, + pub recorded_disposition: Option, + pub validator_provenance: Option, +} +#[derive(Clone, Debug, PartialEq, Eq, Serialize)] +pub struct ReplayReport { + pub consistency: ReplayStatus, + pub reconstruction: ReplayStatus, + pub validation: ReplayStatus, + pub recorded_disposition: Option, + pub validator_provenance: Option, + pub missing_requirements: Vec, + pub accounting: Vec, + pub source_bundle_sha256: Option, + pub reevaluated_validator: Option, + pub attempts: Vec, +} diff --git a/src/lib.rs b/src/lib.rs index c612292..0944003 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -2,6 +2,7 @@ mod evidence_digest; pub mod incident_protocol; +pub mod decision_bundle; mod attempts; mod eligibility; diff --git a/src/report_verdict.rs b/src/report_verdict.rs index cb78319..5c7fd57 100644 --- a/src/report_verdict.rs +++ b/src/report_verdict.rs @@ -26,7 +26,7 @@ struct Verdict { status: &'static str, } -fn derive(payload: &[u8]) -> Result> { +pub(crate) fn derive(payload: &[u8]) -> Result> { // Deserialize directly into a strict struct: a Value would lose duplicate keys. let report: Report = serde_json::from_slice(payload).map_err(|_| invalid("invalid report evidence"))?; diff --git a/tests/decision_bundle.rs b/tests/decision_bundle.rs new file mode 100644 index 0000000..5bc2dce --- /dev/null +++ b/tests/decision_bundle.rs @@ -0,0 +1,606 @@ +use kairotensor::decision_bundle::*; +use serde_json::{json, Value}; +use sha2::{Digest, Sha256}; + +fn digest(raw: &[u8]) -> String { + Sha256::digest(raw) + .iter() + .map(|b| format!("{b:02x}")) + .collect() +} +fn bytes(raw: &[u8]) -> Value { + json!({"hex":raw.iter().map(|b|format!("{b:02x}")).collect::(),"byte_len":raw.len(),"sha256":digest(raw)}) +} +fn seal(mut value: Value) -> Vec { + value["integrity"].as_object_mut().unwrap().remove("sha256"); + let hash = digest(&serde_json::to_vec(&value).unwrap()); + value["integrity"]["sha256"] = hash.into(); + serde_json::to_vec(&value).unwrap() +} +fn fixture() -> Value { + json!({ + "schema_version":1,"root_attempt":1,"actor":"a","scope":"s","export_purpose":"diagnostic", + "snapshot":{"root_branch":1,"head":1,"revision":1,"current_epoch":0}, + "attempts":[{"id":1,"branch":1,"actor":"a","scope":"s","captured_epoch":0,"captured_revision":0, + "parent":null,"inherited_from":null,"candidates":[{"position":0,"evidence_id":1}], + "inputs":[{"position":0,"evidence_id":1,"role":"unknown"}],"dispatch_request":null, + "delivery":{"state":"recorded","raw":bytes(b"\xff\0"),"disposition":"accepted","usage":0,"rejection":null}, + "validator":null,"immutable_runtime":null,"immutable_manifest":null,"timing_configuration":null, + "accounting":{"unit":"unknown","quote":null,"granted":null,"budget_reference":null,"charge":0,"hold_bound":0}, + "reuse_origin":null,"model_contract":null,"deterministic":null,"support":null,"recovery":null}], + "evidence":[{"occurrence_id":1,"source":bytes(b"\xfe"),"payload":bytes(b"\xff\0"),"authorized_scopes":[{"actor":"a","scope":"s"}]}], + "integrity":{"version":"decision-bundle-integrity/1","sha256":""} + }) +} +fn parsed(v: Value) -> DecisionBundleV1 { + parse_decision_bundle(&seal(v)).unwrap() +} +fn report_fixture() -> Value { + let mut v = fixture(); + v["evidence"][0]["payload"] = bytes(br#"{"release":"r1","passed":2,"total":2}"#); + let a = &mut v["attempts"][0]; + a["inputs"][0]["role"] = "report".into(); + a["delivery"]["raw"] = bytes(br#"{"release":"r1","tests":"2/2","status":"pass"}"#); + a["deterministic"] = json!({"version":"report-verdict/v1","output":a["delivery"]["raw"]}); + a["validator"] = json!({"version":"report-verdict/v1","provenance":"deterministic_binding"}); + a["accounting"]["unit"] = "logical_support".into(); + v +} +#[test] +fn missing_unknown_duplicate_and_wrong_schema_refuse() { + assert!(matches!( + parse_decision_bundle(b"{}"), + Err(BundleError::InvalidSchema) + )); + for change in [json!(2), json!("1"), json!(-1)] { + let mut v = fixture(); + v["schema_version"] = change; + assert!(parse_decision_bundle(&seal(v)).is_err()); + } + let mut v = fixture(); + v["extra"] = true.into(); + assert!(parse_decision_bundle(&seal(v)).is_err()); + let raw = String::from_utf8(seal(fixture())).unwrap().replacen( + "\"schema_version\":1", + "\"schema_version\":1,\"schema_version\":1", + 1, + ); + assert!(parse_decision_bundle(raw.as_bytes()).is_err()); + let mut v = fixture(); + v["attempts"][0].as_object_mut().unwrap().remove("parent"); + assert!(parse_decision_bundle(&seal(v)).is_err()); +} +#[test] +fn non_utf8_bytes_round_trip_without_reencoding_and_legacy_unknowns() { + let b = parsed(fixture()); + assert_eq!(b.evidence[0].payload.decode().unwrap(), [255, 0]); + let r = validate_recorded_bundle(&b); + assert_eq!(r.consistency, ReplayStatus::Matched); + assert_eq!(r.validation, ReplayStatus::NotReproducible); + assert!(r + .missing_requirements + .contains(&MissingRequirement::ValidatorUnavailable)); + assert!(r + .missing_requirements + .contains(&MissingRequirement::HistoricalTimingUnavailable)); + assert_eq!(r.recorded_disposition, Some(RecordedDisposition::Accepted)); + assert_eq!(r.source_bundle_sha256, Some(digest(&seal(fixture())))); + assert_eq!(parse_decision_bundle(&b.to_json().unwrap()).unwrap(), b); +} +#[test] +fn altered_raw_or_outer_digest_refuses() { + let mut v = fixture(); + v["evidence"][0]["payload"]["hex"] = "fe00".into(); + assert!(matches!( + parse_decision_bundle(&seal(v)), + Err(BundleError::InconsistentLedger) + )); + let mut v: Value = serde_json::from_slice(&seal(fixture())).unwrap(); + v["actor"] = "other".into(); + assert!(matches!( + parse_decision_bundle(&serde_json::to_vec(&v).unwrap()), + Err(BundleError::InconsistentLedger) + )); + let mut v = fixture(); + v["evidence"][0]["payload"]["hex"] = "FF00".into(); + assert!(parse_decision_bundle(&seal(v)).is_err()); +} +#[test] +fn occurrence_identity_and_repeated_occurrences_at_distinct_positions_survive() { + let mut v = fixture(); + let mut e = v["evidence"][0].clone(); + e["occurrence_id"] = 2.into(); + v["evidence"].as_array_mut().unwrap().push(e); + v["attempts"][0]["inputs"] = json!([{"position":0,"evidence_id":1,"role":"unknown"},{"position":1,"evidence_id":2,"role":"unknown"},{"position":2,"evidence_id":1,"role":"unknown"}]); + v["attempts"][0]["candidates"] = json!([{"position":0,"evidence_id":1},{"position":1,"evidence_id":2},{"position":2,"evidence_id":1}]); + let b = parsed(v.clone()); + assert_eq!( + b.attempts[0] + .inputs + .iter() + .map(|i| i.evidence_id) + .collect::>(), + [1, 2, 1] + ); + v["attempts"][0]["inputs"][2]["position"] = 1.into(); + assert!(parse_decision_bundle(&seal(v)).is_err()); + let mut v = fixture(); + let e = v["evidence"][0].clone(); + v["evidence"].as_array_mut().unwrap().push(e); + assert!(parse_decision_bundle(&seal(v)).is_err()); +} +#[test] +fn json_and_constructed_values_are_bounded() { + assert!(matches!( + parse_decision_bundle(&vec![b' '; 16 * 1024 * 1024 + 1]), + Err(BundleError::LimitExceeded) + )); + let nested = format!("{}0{}", "[".repeat(65), "]".repeat(65)); + assert!(matches!( + parse_decision_bundle(nested.as_bytes()), + Err(BundleError::LimitExceeded) + )); + assert!(matches!( + parse_decision_bundle(&[255]), + Err(BundleError::InvalidEncoding) + )); + let mut b = parsed(fixture()); + b.evidence[0].payload.byte_len = 4 * 1024 * 1024 + 1; + assert_eq!( + validate_recorded_bundle(&b).consistency, + ReplayStatus::Mismatch + ); + let mut b = parsed(fixture()); + b.actor = "x".repeat(16 * 1024 * 1024 + 1); + assert!(matches!(b.to_json(), Err(BundleError::LimitExceeded))); + assert_eq!(validate_recorded_bundle(&b).source_bundle_sha256, None); +} +#[test] +fn accounting_rejects_inconsistent_usage_quote_and_holds() { + for (usage, charge, hold, granted) in [ + (3, 2, 0, true), + (1, 1, 0, true), + (9, 9, 0, true), + (3, 3, 1, true), + (3, 3, 0, false), + ] { + let mut v = fixture(); + let a = &mut v["attempts"][0]; + a["delivery"]["usage"] = usage.into(); + a["accounting"] = json!({"unit":"unknown","quote":{"input_tokens":2,"output_cap":4},"granted":granted,"budget_reference":1,"charge":charge,"hold_bound":hold}); + assert!(parse_decision_bundle(&seal(v)).is_err()); + } + let mut v = fixture(); + v["attempts"][0]["accounting"]["quote"] = json!({"input_tokens":i64::MAX,"output_cap":1}); + assert!(parse_decision_bundle(&seal(v)).is_err()); +} +#[test] +fn unresolved_delivery_retains_hold_and_unknown_usage() { + let mut v = fixture(); + let a = &mut v["attempts"][0]; + a["delivery"] = json!({"state":"unresolved"}); + a["accounting"] = json!({"unit":"unknown","quote":{"input_tokens":2,"output_cap":4},"granted":true,"budget_reference":1,"charge":0,"hold_bound":6}); + let b = parsed(v); + let r = validate_recorded_bundle(&b); + assert_eq!(r.consistency, ReplayStatus::Matched); + assert_eq!(r.validation, ReplayStatus::Incomplete); + assert_eq!(r.accounting[0].held, 6); + assert_eq!(r.accounting[0].charged, 0); + assert!(r + .missing_requirements + .contains(&MissingRequirement::MissingDelivery)); +} +#[test] +fn deterministic_replay_and_changed_validator_preserve_original_disposition() { + let b = parsed(report_fixture()); + let r = validate_recorded_bundle(&b); + assert_eq!(r.reconstruction, ReplayStatus::Matched); + assert_eq!(r.validation, ReplayStatus::Matched); + let changed = reevaluate_bundle(&b, "report-verdict/v2"); + assert_eq!(changed.validation, ReplayStatus::NotReproducible); + assert_eq!( + changed.reevaluated_validator.as_deref(), + Some("report-verdict/v2") + ); + assert_eq!(changed.source_bundle_sha256, r.source_bundle_sha256); + assert_eq!(changed.recorded_disposition, r.recorded_disposition); + let mut v = report_fixture(); + v["attempts"][0]["deterministic"]["output"] = bytes(b"wrong"); + v["attempts"][0]["delivery"]["raw"] = bytes(b"wrong"); + let r = validate_recorded_bundle(&parsed(v)); + assert_eq!(r.consistency, ReplayStatus::Matched); + assert_eq!(r.reconstruction, ReplayStatus::Mismatch); +} +#[test] +fn missing_data_dependencies_are_incomplete_but_unconsumed_parent_is_only_lineage() { + let mut v = fixture(); + v["attempts"][0]["parent"] = 2.into(); + let b = parsed(v.clone()); + assert_eq!( + validate_recorded_bundle(&b).consistency, + ReplayStatus::Matched + ); + v["attempts"][0]["inherited_from"] = 2.into(); + let r = validate_recorded_bundle(&parsed(v)); + assert_eq!(r.reconstruction, ReplayStatus::Incomplete); + assert!(r + .missing_requirements + .contains(&MissingRequirement::MissingDependency)); + let mut v = fixture(); + v["attempts"][0]["reuse_origin"] = 2.into(); + v["attempts"][0]["model_contract"] = "legacy".into(); + let r = validate_recorded_bundle(&parsed(v)); + assert_eq!(r.reconstruction, ReplayStatus::Incomplete); + let mut v = report_fixture(); + v["evidence"] = json!([]); + let r = validate_recorded_bundle(&parsed(v)); + assert_eq!(r.validation, ReplayStatus::Incomplete); + assert!(r + .missing_requirements + .contains(&MissingRequirement::MissingInput)); +} + +#[test] +fn legacy_filtered_candidates_and_exact_inputs_are_independent_positions() { + let mut v = fixture(); + v["attempts"][0]["candidates"] = + json!([{"position":0,"evidence_id":99},{"position":1,"evidence_id":1}]); + assert!(parse_decision_bundle(&seal(v)).is_ok()); + let mut v = fixture(); + v["attempts"][0]["candidates"] = json!([]); + assert!(parse_decision_bundle(&seal(v)).is_ok()); +} +#[test] +fn cancelled_unsent_recovery_releases_granted_hold() { + let mut v = fixture(); + let a = &mut v["attempts"][0]; + a["delivery"] = json!({"state":"unresolved"}); + a["recovery"] = json!({"recovery_id":1,"ordinal":0,"state":"cancelled","predecessor_id":null,"request":bytes(b"request")}); + a["accounting"] = json!({"unit":"unknown","quote":{"input_tokens":2,"output_cap":4},"granted":true,"budget_reference":1,"charge":0,"hold_bound":0}); + let r = validate_recorded_bundle(&parsed(v)); + assert_eq!(r.accounting[0].held, 0); +} +#[test] +fn shared_occurrence_requires_each_consuming_attempt_scope() { + let mut v = fixture(); + let mut dependency = v["attempts"][0].clone(); + dependency["id"] = 2.into(); + dependency["branch"] = 2.into(); + dependency["scope"] = "foreign".into(); + v["attempts"][0]["parent"] = 2.into(); + v["attempts"][0]["inherited_from"] = 2.into(); + v["attempts"].as_array_mut().unwrap().push(dependency); + v["evidence"][0]["authorized_scopes"] + .as_array_mut() + .unwrap() + .push(json!({"actor":"a","scope":"foreign"})); + assert!(parse_decision_bundle(&seal(v.clone())).is_ok()); + v["evidence"][0]["authorized_scopes"] + .as_array_mut() + .unwrap() + .pop(); + assert!(parse_decision_bundle(&seal(v)).is_err()); + let mut v = fixture(); + v["attempts"][0]["actor"] = "wrong".into(); + assert!(parse_decision_bundle(&seal(v)).is_err()); + let mut v = fixture(); + v["evidence"][0]["authorized_scopes"] + .as_array_mut() + .unwrap() + .push(json!({"actor":"a","scope":"s"})); + assert!(parse_decision_bundle(&seal(v)).is_err()); +} +#[test] +fn descriptive_roles_and_validator_provenance_cannot_be_invented() { + let mut v = fixture(); + v["attempts"][0]["inputs"][0]["role"] = "report".into(); + assert!(parse_decision_bundle(&seal(v)).is_err()); + let mut v = fixture(); + v["attempts"][0]["validator"] = + json!({"version":"report-verdict/v1","provenance":"deterministic_binding"}); + assert!(parse_decision_bundle(&seal(v)).is_err()); +} + +fn incident_fixture() -> Value { + let context = json!({"schema_version":1,"version":"incident-store-context/1","workload_version":"incident-routing/1","freeze_sha256":"f".repeat(64),"prompt_sha256":"e".repeat(64),"validator_version":"incident-routing-protocol/1","case_id":"c1","question":"Where?","owner_ids":["network"],"evidence_ids":["r1"],"system_prompt":"Route."}); + let context_raw = serde_json::to_vec(&context).unwrap(); + let report_raw = b"network lost"; + let runtime = json!({"cpu_fingerprint":"c".repeat(64),"decoding":"canned-no-inference","dtype":"bytes","model":"canned-no-model","model_revision":"fixture/1","readiness":"warm","runtime_version":"incident-canned/1","threads":1,"tokenizer_revision":"utf8-bytes/1"}); + let manifest = json!({"context":{"occurrence_id":1,"sha256":digest(&context_raw)},"freeze_sha256":context["freeze_sha256"],"initial_renderer":"incident-request/1","output_cap":512,"prompt_sha256":context["prompt_sha256"],"retry_feedback":"incident-routing-protocol/1:invalid_output","retry_renderer":"incident-request-retry/1","runtime":runtime,"schema_version":1,"sources":[{"case_id":"c1","external_id":"r1","occurrence_id":2,"payload_sha256":digest(report_raw),"workload_version":"incident-routing/1"}],"validator_version":"incident-routing-protocol/1","version":"incident-store-manifest/1","workload_version":"incident-routing/1"}); + let manifest_raw = serde_json::to_vec(&manifest).unwrap(); + let mut runtime = runtime; + runtime["prompt_fingerprint"] = digest(&manifest_raw).into(); + let timing = json!({"clock_domain":"synthetic","start_ms":0,"deadline_ms":1000,"runtime":runtime,"calibration":{"id":"fixture","max_duration_ms":5,"sample_count":1,"max_input_tokens":1024,"output_cap":512,"measured_at_ms":0,"expires_at_ms":1001},"safety_margin_ms":1}); + // Independent literal oracle: no call to the Rust incident renderer. + let request=br#"{"messages":[{"content":"Route.","role":"system"},{"content":"{\"evidence\":[{\"id\":\"r1\",\"text\":\"network lost\"}],\"owner_ids\":[\"network\"],\"question\":\"Where?\"}","role":"user"}]}"#; + let mut v = fixture(); + v["evidence"][0]["payload"] = bytes(&context_raw); + let mut e = v["evidence"][0].clone(); + e["occurrence_id"] = 2.into(); + e["payload"] = bytes(report_raw); + v["evidence"].as_array_mut().unwrap().push(e); + let a = &mut v["attempts"][0]; + a["inputs"] = json!([{"position":0,"evidence_id":1,"role":"context"},{"position":1,"evidence_id":2,"role":"report"}]); + a["candidates"] = json!([{"position":0,"evidence_id":1},{"position":1,"evidence_id":2}]); + a["dispatch_request"] = bytes(request); + a["delivery"]["raw"]=bytes(br#"{"decision":"route","owner":"network","reason":"diagnosed_component","evidence_ids":["r1"]}"#); + a["validator"] = + json!({"version":"incident-routing-protocol/1","provenance":"declared_contract"}); + a["immutable_runtime"] = bytes(&serde_json::to_vec(&runtime).unwrap()); + a["immutable_manifest"] = bytes(&manifest_raw); + a["timing_configuration"] = bytes(&serde_json::to_vec(&timing).unwrap()); + a["recovery"] = json!({"recovery_id":1,"ordinal":0,"state":"sent","predecessor_id":null,"request":bytes(request)}); + let usage = request.len() as i64 + a["delivery"]["raw"]["byte_len"].as_i64().unwrap(); + a["delivery"]["usage"] = usage.into(); + a["accounting"] = json!({"unit":"unknown","quote":{"input_tokens":request.len(),"output_cap":512},"granted":true,"budget_reference":1,"charge":usage,"hold_bound":0}); + v +} +fn retry_fixture() -> Value { + let mut v = incident_fixture(); + let mut next = v["attempts"][0].clone(); + let retry=br#"{"messages":[{"content":"Route.","role":"system"},{"content":"{\"evidence\":[{\"id\":\"r1\",\"text\":\"network lost\"}],\"owner_ids\":[\"network\"],\"question\":\"Where?\",\"validation_error\":\"incident-routing-protocol/1:invalid_output\"}","role":"user"}]}"#; + next["id"] = 2.into(); + next["dispatch_request"] = bytes(retry); + next["recovery"]["ordinal"] = 1.into(); + next["recovery"]["predecessor_id"] = 1.into(); + next["recovery"]["request"] = bytes(retry); + let usage = retry.len() as i64 + next["delivery"]["raw"]["byte_len"].as_i64().unwrap(); + next["delivery"]["usage"] = usage.into(); + next["accounting"]["charge"] = usage.into(); + next["accounting"]["quote"]["input_tokens"] = retry.len().into(); + let first_usage = v["attempts"][0]["accounting"]["quote"]["input_tokens"] + .as_i64() + .unwrap() + + 7; + v["attempts"][0]["delivery"]["usage"] = first_usage.into(); + v["attempts"][0]["accounting"]["charge"] = first_usage.into(); + v["attempts"][0]["delivery"]["raw"] = bytes(b"invalid"); + v["attempts"][0]["delivery"]["disposition"] = "invalid_output".into(); + v["attempts"][0]["delivery"]["rejection"] = json!({"validator":"incident-routing-protocol/1","reason":"free text must never enter retry request","applicability":"current"}); + v["attempts"].as_array_mut().unwrap().push(next); + v["root_attempt"] = 2.into(); + v["snapshot"]["head"] = 2.into(); + v +} +#[test] +fn incident_manifest_initial_and_retry_match_exact_oracles() { + for v in [incident_fixture(), retry_fixture()] { + let r = validate_recorded_bundle(&parsed(v)); + assert_eq!(r.consistency, ReplayStatus::Matched); + assert_eq!(r.reconstruction, ReplayStatus::Matched); + assert_eq!(r.validation, ReplayStatus::Matched); + assert_eq!( + r.validator_provenance, + Some(ValidatorProvenance::DeclaredContract) + ); + } + let r = validate_recorded_bundle(&parsed(retry_fixture())); + assert_eq!( + r.attempts[0].validator_provenance, + Some(ValidatorProvenance::RecordedRejection) + ); + assert_eq!( + r.attempts[0].recorded_disposition, + Some(RecordedDisposition::InvalidOutput) + ); +} +#[test] +fn incident_changed_request_order_runtime_and_unknown_phase_do_not_fallback() { + let mut v = incident_fixture(); + v["attempts"][0]["dispatch_request"] = bytes(b"wrong"); + v["attempts"][0]["recovery"]["request"] = bytes(b"wrong"); + assert_eq!( + validate_recorded_bundle(&parsed(v)).reconstruction, + ReplayStatus::Mismatch + ); + let mut v = incident_fixture(); + v["attempts"][0]["immutable_runtime"] = bytes(b"{}"); + assert_eq!( + validate_recorded_bundle(&parsed(v)).reconstruction, + ReplayStatus::Mismatch + ); + let mut v = incident_fixture(); + v["attempts"][0]["recovery"]["ordinal"] = 2.into(); + let r = validate_recorded_bundle(&parsed(v)); + assert_eq!(r.reconstruction, ReplayStatus::NotReproducible); + assert!(r + .missing_requirements + .contains(&MissingRequirement::RendererUnavailable)); + let mut v = incident_fixture(); + v["attempts"][0]["candidates"][0]["evidence_id"] = 2.into(); + v["attempts"][0]["candidates"][1]["evidence_id"] = 1.into(); + assert_eq!( + validate_recorded_bundle(&parsed(v)).reconstruction, + ReplayStatus::Mismatch + ); + let mut v = retry_fixture(); + v["attempts"].as_array_mut().unwrap().remove(0); + let r = validate_recorded_bundle(&parsed(v)); + assert_eq!(r.reconstruction, ReplayStatus::Incomplete); +} +#[test] +fn protocol_rejection_matches_but_new_evaluation_reports_invalid() { + let mut v = retry_fixture(); + v["attempts"].as_array_mut().unwrap().pop(); + v["root_attempt"] = 1.into(); + v["snapshot"]["head"] = 1.into(); + let b = parsed(v); + assert_eq!( + validate_recorded_bundle(&b).validation, + ReplayStatus::Matched + ); + let r = reevaluate_bundle(&b, "incident-routing-protocol/1"); + assert_eq!(r.validation, ReplayStatus::Mismatch); + assert_eq!( + r.recorded_disposition, + Some(RecordedDisposition::InvalidOutput) + ); +} +#[test] +fn typed_schema_rejects_nested_duplicates_and_record_count_overflow() { + let raw = String::from_utf8(seal(fixture())).unwrap(); + for (find, replace) in [ + ("\"usage\":0", "\"usage\":0,\"usage\":0"), + ( + "\"state\":\"recorded\"", + "\"state\":\"recorded\",\"state\":\"recorded\"", + ), + ("\"position\":0", "\"position\":0,\"position\":0"), + ] { + assert!(parse_decision_bundle(raw.replacen(find, replace, 1).as_bytes()).is_err()); + } + let mut b = parsed(fixture()); + let a = b.attempts[0].clone(); + b.attempts = vec![a; 129]; + assert!(matches!(b.to_json(), Err(BundleError::LimitExceeded))); + assert!(ByteEnvelope::from_bytes(&vec![0; 4 * 1024 * 1024 + 1]).is_err()); +} + +#[test] +fn dependency_cycles_are_incomplete_and_support_rule_owner_is_not_an_output_dependency() { + let mut v = fixture(); + v["attempts"][0]["parent"] = 2.into(); + v["attempts"][0]["inherited_from"] = 2.into(); + let mut d = v["attempts"][0].clone(); + d["id"] = 2.into(); + d["parent"] = 1.into(); + d["inherited_from"] = 1.into(); + v["attempts"].as_array_mut().unwrap().push(d); + let r = validate_recorded_bundle(&parsed(v)); + assert_eq!(r.reconstruction, ReplayStatus::Incomplete); + assert!(r + .missing_requirements + .contains(&MissingRequirement::MissingDependency)); + let mut v = fixture(); + v["attempts"][0]["support"] = json!({"version":"support/v1","rule_id":999,"verdict":true,"clauses":[{"clause_id":1,"premises":[{"evidence_id":1,"actor":"a","scope":"s"}]}]}); + v["attempts"][0]["inputs"][0]["role"] = "evidence".into(); + let r = validate_recorded_bundle(&parsed(v)); + assert!(!r + .missing_requirements + .contains(&MissingRequirement::MissingDependency)); +} +#[test] +fn fixture_units_require_recognized_runtime_and_deterministic_units_require_binding() { + let mut v = fixture(); + v["attempts"][0]["accounting"]["unit"] = "logical_support".into(); + assert!(parse_decision_bundle(&seal(v)).is_err()); + let mut v = incident_fixture(); + v["attempts"][0]["accounting"]["unit"] = "fixture_bytes".into(); + assert!(parse_decision_bundle(&seal(v.clone())).is_ok()); + v["attempts"][0]["immutable_runtime"] = bytes(b"{}"); + assert!(parse_decision_bundle(&seal(v)).is_err()); + let mut v = fixture(); + v["attempts"][0]["accounting"]["unit"] = "model_tokens".into(); + assert!(parse_decision_bundle(&seal(v)).is_err()); +} + +#[test] +fn invalid_deterministic_input_is_a_validator_mismatch_not_an_unavailable_validator() { + let mut v = report_fixture(); + v["evidence"][0]["payload"] = bytes(b"invalid report"); + let r = validate_recorded_bundle(&parsed(v)); + assert_eq!(r.reconstruction, ReplayStatus::Mismatch); + assert_eq!(r.validation, ReplayStatus::Mismatch); + assert!(!r + .missing_requirements + .contains(&MissingRequirement::ValidatorUnavailable)); +} +#[test] +fn reserved_unsent_incident_reconstructs_recorded_preparation_without_claiming_delivery() { + let mut v = incident_fixture(); + let a = &mut v["attempts"][0]; + a["dispatch_request"] = Value::Null; + a["delivery"] = json!({"state":"unresolved"}); + a["recovery"]["state"] = "reserved".into(); + a["accounting"] = json!({"unit":"unknown","quote":{"input_tokens":2,"output_cap":512},"granted":true,"budget_reference":1,"charge":0,"hold_bound":514}); + let r = validate_recorded_bundle(&parsed(v)); + assert_eq!(r.reconstruction, ReplayStatus::Matched); + assert_eq!(r.validation, ReplayStatus::Incomplete); + assert_eq!(r.accounting[0].held, 514); +} + +#[test] +fn recovery_state_and_reservation_cannot_contradict_each_other() { + for state in ["denied", "reserved", "planned", "cancelled"] { + let mut v = incident_fixture(); + v["attempts"][0]["recovery"]["state"] = state.into(); + assert!(parse_decision_bundle(&seal(v)).is_err()); + } + let mut v = incident_fixture(); + let a = &mut v["attempts"][0]; + a["accounting"]["quote"] = Value::Null; + a["accounting"]["granted"] = Value::Null; + a["accounting"]["budget_reference"] = Value::Null; + assert!(parse_decision_bundle(&seal(v)).is_err()); +} + +#[test] +fn reuse_delivery_must_equal_its_explicit_dispatched_origin() { + let mut v = fixture(); + let mut origin = v["attempts"][0].clone(); + origin["dispatch_request"] = bytes(b"request"); + origin["model_contract"] = "legacy".into(); + v["attempts"][0]["id"] = 2.into(); + v["attempts"][0]["reuse_origin"] = 1.into(); + v["attempts"][0]["model_contract"] = "legacy".into(); + v["root_attempt"] = 2.into(); + v["snapshot"]["head"] = 2.into(); + v["attempts"].as_array_mut().unwrap().push(origin); + assert!(parse_decision_bundle(&seal(v.clone())).is_ok()); + v["attempts"][0]["delivery"]["raw"] = bytes(b"invented"); + assert!(parse_decision_bundle(&seal(v)).is_err()); +} + +// Frozen Python json/hashlib oracle, shared verbatim with the v1 format document. +const GOLDEN: &str = r#"{"actor":"é","attempts":[{"accounting":{"budget_reference":null,"charge":0,"granted":null,"hold_bound":0,"quote":null,"unit":"unknown"},"actor":"é","branch":1,"candidates":[],"captured_epoch":0,"captured_revision":0,"delivery":{"state":"unresolved"},"deterministic":null,"dispatch_request":null,"id":1,"immutable_manifest":null,"immutable_runtime":null,"inherited_from":null,"inputs":[],"model_contract":null,"parent":null,"recovery":null,"reuse_origin":null,"scope":"s","support":null,"timing_configuration":null,"validator":null}],"evidence":[],"export_purpose":"golden","integrity":{"sha256":"765fbaa97312a42a4e5da7022d6350efd2fb4811767bf763f327f6ddf0832093","version":"decision-bundle-integrity/1"},"root_attempt":1,"schema_version":1,"scope":"s","snapshot":{"current_epoch":0,"head":null,"revision":0,"root_branch":1}}"#; +#[test] +fn canonical_golden_bytes_digest_and_transport_whitespace_are_distinct() { + let bundle = parse_decision_bundle(GOLDEN.as_bytes()).unwrap(); + assert_eq!(bundle.to_json().unwrap(), GOLDEN.as_bytes()); + let report = validate_recorded_bundle(&bundle); + assert_eq!( + report.source_bundle_sha256.as_deref(), + Some("569b219c44295bb8e1a42358b1bbbd66c153ff379694dd55ef5decf229dbe471") + ); + let pretty = + serde_json::to_vec_pretty(&serde_json::from_str::(GOLDEN).unwrap()).unwrap(); + let again = parse_decision_bundle(&pretty).unwrap(); + assert_eq!( + validate_recorded_bundle(&again).source_bundle_sha256, + report.source_bundle_sha256 + ); + assert_ne!(digest(&pretty), report.source_bundle_sha256.unwrap()); + let mut changed = bundle.clone(); + changed.export_purpose = "changed".into(); + changed.seal().unwrap(); + assert_ne!(changed.integrity.sha256, bundle.integrity.sha256); + assert!(parse_decision_bundle(&changed.to_json().unwrap()).is_ok()); +} + +#[test] +fn unknown_manifest_or_renderer_version_never_falls_back() { + for field in ["version", "initial_renderer", "retry_renderer"] { + let mut bundle = parsed(incident_fixture()); + let raw = bundle.attempts[0] + .immutable_manifest + .as_ref() + .unwrap() + .decode() + .unwrap(); + let mut manifest: Value = serde_json::from_slice(&raw).unwrap(); + manifest[field] = "unknown/2".into(); + bundle.attempts[0].immutable_manifest = + Some(ByteEnvelope::from_bytes(&serde_json::to_vec(&manifest).unwrap()).unwrap()); + if field == "version" { + for input in &mut bundle.attempts[0].inputs { + input.role = InputRole::Unknown; + } + } + bundle.seal().unwrap(); + let report = validate_recorded_bundle(&bundle); + assert_eq!(report.reconstruction, ReplayStatus::NotReproducible); + assert!(report + .missing_requirements + .contains(&MissingRequirement::RendererUnavailable)); + } +} From b5756e094c14231ec0632ccaa31c3b9d4aa8811b Mon Sep 17 00:00:00 2001 From: Alejandro Castellanos <5405628+k2jac9@users.noreply.github.com> Date: Sun, 15 Nov 2026 16:26:33 -0500 Subject: [PATCH 15/51] fix: enforce recovery bundle ledger invariants --- docs/decision-bundle-v1.md | 13 ++++- src/decision_bundle.rs | 58 ++++++++++++++++++- tests/decision_bundle.rs | 113 +++++++++++++++++++++++++++++++++++++ 3 files changed, 182 insertions(+), 2 deletions(-) diff --git a/docs/decision-bundle-v1.md b/docs/decision-bundle-v1.md index 0afb85c..59c637e 100644 --- a/docs/decision-bundle-v1.md +++ b/docs/decision-bundle-v1.md @@ -171,7 +171,18 @@ the reservation bound. A known delivery's charge equals usage and its hold is zero; reserved usage must lie between input quote and full bound. A granted unresolved reservation keeps its full hold except a cancelled unsent recovery. Denied reservations have no hold. Recovery state, request, dispatch, reservation -and delivery facts must be compatible. Unknown recovery phases can be retained +and delivery facts must be compatible. A sent timed phase must also have a quote +whose output cap equals its immutable calibration cap and whose input is covered +by that calibration's maximum input. This check uses no historical clock data. +An unsent reserved, denied or cancelled phase may retain an uncovered quote: timed +admission can refuse after reservation, so such a quote is not itself corruption. + +The composite `(recovery_id, ordinal)` key is unique even across different attempt +IDs. Included phases of one run must agree on actor/scope/branch, exact timing +configuration, immutable runtime and manifest. All recorded budget references +for that recovery must agree. Quotes themselves remain per-attempt and can differ; +a planned phase with null quote/granted/budget reference does not contradict +another phase's recorded reservation. Unknown recovery phases can be retained but cannot select a renderer. Units: `logical_support`, `fixture_bytes`, `model_tokens`, `unknown`. C1 recognizes diff --git a/src/decision_bundle.rs b/src/decision_bundle.rs index c01d31d..a651d98 100644 --- a/src/decision_bundle.rs +++ b/src/decision_bundle.rs @@ -6,7 +6,7 @@ use crate::evidence_digest::sha256_hex; pub use replay::{reevaluate_bundle, validate_recorded_bundle}; use serde::{Deserialize, Serialize}; use std::{ - collections::HashSet, + collections::{HashMap, HashSet}, fmt, io::{self, Write}, }; @@ -467,6 +467,7 @@ fn check_bundle(b: &DecisionBundleV1) -> Result { } } } + check_recovery_runs(b)?; for a in &b.attempts { if let Some(origin) = a .reuse_origin @@ -527,6 +528,61 @@ fn check_bundle(b: &DecisionBundleV1) -> Result { Ok(value) } +// A recovery's primary key and immutable run facts are shared across phases. +// Quotes are per-attempt: an unsent phase may have no reservation, or a quote +// that timed admission refused. Only a recorded sent phase requires coverage. +fn check_recovery_runs(bundle: &DecisionBundleV1) -> Result<(), Error> { + let mut keys = HashSet::new(); + let mut runs: HashMap = HashMap::new(); + let mut budgets = HashMap::new(); + for attempt in &bundle.attempts { + let Some(recovery) = &attempt.recovery else { + continue; + }; + if !keys.insert((recovery.recovery_id, recovery.ordinal)) { + return Err(Error::InconsistentLedger); + } + if let Some(prior) = runs.insert(recovery.recovery_id, attempt) { + if attempt.actor != prior.actor + || attempt.scope != prior.scope + || attempt.branch != prior.branch + || attempt.timing_configuration != prior.timing_configuration + || attempt.immutable_runtime != prior.immutable_runtime + || attempt.immutable_manifest != prior.immutable_manifest + { + return Err(Error::InconsistentLedger); + } + } + // Absence here means this phase has no token_reservations row; it does + // not erase or contradict a budget reference recorded by another phase. + if let Some(budget) = attempt.accounting.budget_reference { + if budgets + .insert(recovery.recovery_id, budget) + .is_some_and(|prior| prior != budget) + { + return Err(Error::InconsistentLedger); + } + } + if recovery.state == "sent" { + if let Some(configuration) = &attempt.timing_configuration { + let timing: crate::RecoveryTiming = decode_strict(&configuration.decode()?) + .map_err(|_| Error::InconsistentLedger)?; + let quote = attempt + .accounting + .quote + .as_ref() + .ok_or(Error::InconsistentLedger)?; + if quote.output_cap != timing.calibration.output_cap + || quote.input_tokens > timing.calibration.max_input_tokens + { + return Err(Error::InconsistentLedger); + } + } + } + } + Ok(()) +} + // Check every envelope's borrowed lengths before the outer JSON allocation. fn preflight_envelopes(b: &DecisionBundleV1) -> Result<(), Error> { fn check(e: &ByteEnvelope) -> Result<(), Error> { diff --git a/tests/decision_bundle.rs b/tests/decision_bundle.rs index 5bc2dce..45573eb 100644 --- a/tests/decision_bundle.rs +++ b/tests/decision_bundle.rs @@ -604,3 +604,116 @@ fn unknown_manifest_or_renderer_version_never_falls_back() { .contains(&MissingRequirement::RendererUnavailable)); } } + +#[test] +fn sent_timed_quote_cap_must_equal_immutable_calibration_after_rehashing() { + let mut value = incident_fixture(); + value["attempts"][0]["accounting"]["quote"]["output_cap"] = 513.into(); + assert!(matches!( + parse_decision_bundle(&seal(value)), + Err(BundleError::InconsistentLedger) + )); +} + +#[test] +fn sent_timed_quote_input_must_be_covered_even_with_consistent_usage() { + let mut value = incident_fixture(); + let attempt = &mut value["attempts"][0]; + attempt["accounting"]["quote"]["input_tokens"] = 1025.into(); + attempt["accounting"]["charge"] = 1026.into(); + attempt["delivery"]["usage"] = 1026.into(); + assert!(matches!( + parse_decision_bundle(&seal(value)), + Err(BundleError::InconsistentLedger) + )); +} + +#[test] +fn unsent_timed_refusal_keeps_uncovered_reservation_facts() { + for state in ["reserved", "denied", "cancelled"] { + let mut value = incident_fixture(); + let attempt = &mut value["attempts"][0]; + attempt["recovery"]["state"] = state.into(); + attempt["dispatch_request"] = Value::Null; + attempt["delivery"] = json!({"state":"unresolved"}); + attempt["accounting"] = json!({"unit":"unknown","quote":{"input_tokens":1025,"output_cap":513},"granted":state != "denied","budget_reference":1,"charge":0,"hold_bound":if state == "reserved" {1538} else {0}}); + let report = validate_recorded_bundle(&parsed(value)); + assert_eq!(report.consistency, ReplayStatus::Matched, "{state}"); + assert_eq!(report.reconstruction, ReplayStatus::Matched, "{state}"); + assert_eq!(report.validation, ReplayStatus::Incomplete, "{state}"); + assert_eq!( + report.accounting[0].held, + if state == "reserved" { 1538 } else { 0 } + ); + } +} + +#[test] +fn recovery_composite_key_is_unique_across_distinct_attempt_ids() { + let mut value = incident_fixture(); + let mut duplicate = value["attempts"][0].clone(); + duplicate["id"] = 2.into(); + value["attempts"].as_array_mut().unwrap().push(duplicate); + assert!(matches!( + parse_decision_bundle(&seal(value)), + Err(BundleError::InconsistentLedger) + )); +} + +#[test] +fn included_recovery_phases_share_immutable_timing_and_contract_metadata() { + for (field, key, replacement) in [ + ("timing_configuration", "deadline_ms", json!(1002)), + ("immutable_runtime", "threads", json!(2)), + ("immutable_manifest", "output_cap", json!(513)), + ] { + let mut value = retry_fixture(); + let envelope: ByteEnvelope = + serde_json::from_value(value["attempts"][1][field].clone()).unwrap(); + let mut document: Value = serde_json::from_slice(&envelope.decode().unwrap()).unwrap(); + document[key] = replacement; + value["attempts"][1][field] = bytes(&serde_json::to_vec(&document).unwrap()); + assert!( + matches!( + parse_decision_bundle(&seal(value)), + Err(BundleError::InconsistentLedger) + ), + "{field}" + ); + } +} + +#[test] +fn included_recovery_phases_cannot_name_different_known_budgets() { + let mut value = retry_fixture(); + value["attempts"][1]["accounting"]["budget_reference"] = 2.into(); + assert!(matches!( + parse_decision_bundle(&seal(value)), + Err(BundleError::InconsistentLedger) + )); +} + +#[test] +fn planned_recovery_phase_preserves_absent_per_attempt_budget_and_quote() { + let mut value = retry_fixture(); + let planned = &mut value["attempts"][1]; + planned["recovery"]["state"] = "planned".into(); + planned["recovery"]["request"] = Value::Null; + planned["dispatch_request"] = Value::Null; + planned["delivery"] = json!({"state":"unresolved"}); + planned["accounting"] = json!({"unit":"unknown","quote":null,"granted":null,"budget_reference":null,"charge":0,"hold_bound":0}); + for reverse in [false, true] { + let mut ordered = value.clone(); + if reverse { + ordered["attempts"].as_array_mut().unwrap().reverse(); + } + let bundle = parsed(ordered); + let planned = bundle.attempts.iter().find(|a| a.id == 2).unwrap(); + assert_eq!(planned.accounting.quote, None); + assert_eq!(planned.accounting.budget_reference, None); + assert_eq!( + validate_recorded_bundle(&bundle).consistency, + ReplayStatus::Matched + ); + } +} From 37f25c4634dd74f0b50b66eab4bb37603ed9739f Mon Sep 17 00:00:00 2001 From: Alejandro Castellanos <5405628+k2jac9@users.noreply.github.com> Date: Sun, 15 Nov 2026 16:45:59 -0500 Subject: [PATCH 16/51] feat: authorize decision exports within one Store snapshot --- docs/decision-bundle-v1.md | 80 ++- src/decision_bundle.rs | 5 +- src/decision_bundle/export.rs | 949 +++++++++++++++++++++++++++++++++ src/decision_bundle/types.rs | 73 +++ tests/decision_bundle.rs | 961 ++++++++++++++++++++++++++++++++++ 5 files changed, 2063 insertions(+), 5 deletions(-) create mode 100644 src/decision_bundle/export.rs diff --git a/docs/decision-bundle-v1.md b/docs/decision-bundle-v1.md index 59c637e..ecca1f1 100644 --- a/docs/decision-bundle-v1.md +++ b/docs/decision-bundle-v1.md @@ -1,10 +1,10 @@ # Decision bundle v1 -`kairotensor::decision_bundle` is a pure, bounded export codec and replay API. +`kairotensor::decision_bundle` provides a bounded Store export and pure codec/replay API. It cannot authenticate the producer, grant current access, import into a Store, reproduce general historical timing/publication, or dispatch a backend. SHA-256 -establishes internal consistency only. C2 owns transactional access checks and -payload redaction; C3 owns CLI/process-effect observation. +establishes internal consistency only. The Store exporter performs transactional access checks and payload redaction; +C3 owns CLI/process-effect observation. ## Public API and limits @@ -40,6 +40,80 @@ Pure parsing and replay never open a database. Missing input/dependency records are representable for diagnostic completeness reporting; duplicate definitions, invalid identities, contradictory accounting and invalid digests refuse parsing. +## Authorized Store export (C2) + +```rust +Store::export_decision_bundle(&mut self, request: &BundleRequest, + authorizer: &impl DiagnosticAuthorizer) -> Result +``` + +`BundleRequest` contains `actor`, `scope`, `attempt`, and `BundlePurpose`, either +`CurrentRead` or `DiagnosticReplay`. The embedding runtime authenticates that +identity. The root must belong to that exact actor/scope even for diagnostics. +`DenyDiagnostics` is the default-deny implementation passed explicitly by callers +that have no diagnostic policy. CurrentRead never invokes the diagnostic callback. + +The synchronous `DiagnosticAuthorizer::authorize(&DiagnosticReadRequest)` gets +only the exact ordered selection within the open read transaction. Its fields +are `actor`, `scope`, `root_attempt`, `purpose`, and `attempts`. Each +`SelectedAttempt` contains `attempt`, its own `actor` and `scope`, `branch`, +`head`, `revision`, `current_epoch`, `captured_revision`, `captured_epoch`, and +ordered `inputs: Vec`. Inputs inherit +their owning identity from their enclosing selected attempt. Repeated input +positions and occurrence uses remain visible to policy. Traversal is root first, +then depth-first inherited output, model-reuse origin, and retry predecessor; +already selected attempts appear once. Missing required outputs, missing retry +predecessors, cycles and selections exceeding 128 attempts redact. Parent lineage +and support rule owners are references, not automatic output dependencies. +The callback must not reenter or mutate the Store or use the network. It receives +no payload, source, validator reason, reusable grant, or execution authority. + +One deferred SQLite read transaction begins before metadata selection and remains +open through authorization, every input admission check, bounded fetch and sealing. +It commits before returning either a complete result or a redaction. CurrentRead +requires each selected output to be its branch's current eligible head, evaluated +by the existing authoritative eligibility gate. It rejects any selected output +whose actor/scope differs from the authenticated request; stored row identities +are not caller authority. Diagnostics can authorize a precise cross-scope closure, +but neither mode bypasses current input admission in **every consuming scope**. +A shared global evidence occurrence is exported once, with each admitted consuming +scope recorded. A later revocation cannot claw back bytes already exported. + +`BundleExport` serializes as `{"status":"complete","bundle":...}` or +`{"status":"redacted","bundle":{"requested_attempt":ID,"reason":"unavailable"}}`. +The redacted body has only the caller's requested ID and the generic reason. +Missing, denied, withdrawn, physically unavailable and over-limit selections have +this same payload-free result. No archive fallback is performed. SQL errors remain +`BundleError::Store`; its Display message contains no private database values. +Corrupt schema/ledger values fail rather than becoming successful exports. + +SQLite storage class and byte length (including embedded NULs) are checked within +the held snapshot before copying each selected payload or free-text field. An +aggregate allocation/encoding budget bounds row enumeration and field copying; +text metadata reserves worst-case JSON escaping while byte envelopes reserve +hexadecimal expansion. This conservative budget can redact near-limit selections +before the final codec's exact 16 MiB check. Neither row counts nor failure details +are disclosed on redaction. Runtime/timing JSON and source bytes retain their +exact stored byte representation, including whitespace and non-UTF-8 source bytes. + +Public support registration stores formula references without `attempt_inputs`. +Export preserves that fact: it does not synthesize payload captures or historical +witness counters from current support premises. If a persisted support attempt +also has input rows, those inputs must pass admission even when an alternate +witness keeps its current output eligible. Candidate positions remain independent +from the compacted input positions used by legacy capture. + +Accounting reads only selected per-attempt reservations/deliveries, never a global +budget balance or unrelated member. Planned phases have absent quote/grant/budget; +cancelled unsent phases retain any quote but hold zero; unresolved granted sends +retain their full checked bound. Actual dispatch and recovery preparation are +separate envelopes. The existing B manifest is reconstructed from exact captured +context/report bytes plus immutable runtime/calibration cap, and recognized only +when its canonical hash equals the captured runtime fingerprint. No matching +fingerprint means no invented manifest, incident roles or fixture units. This +path does not send, recover, execute validators dynamically or create another +recovery state machine. + ## Exact outer encoding and digests Encode recursively sorted object keys as compact UTF-8 JSON, with comma and diff --git a/src/decision_bundle.rs b/src/decision_bundle.rs index a651d98..201206c 100644 --- a/src/decision_bundle.rs +++ b/src/decision_bundle.rs @@ -1,5 +1,6 @@ -//! Bounded, pure decision exports. Digests establish internal consistency only; -//! neither parsing nor replay authenticates execution or grants access. +//! Bounded authorized Store exports and pure replay. Digests establish internal +//! consistency only; parsing and replay never authenticate or grant access. +mod export; mod replay; mod types; use crate::evidence_digest::sha256_hex; diff --git a/src/decision_bundle/export.rs b/src/decision_bundle/export.rs new file mode 100644 index 0000000..b38da02 --- /dev/null +++ b/src/decision_bundle/export.rs @@ -0,0 +1,949 @@ +//! One retained read snapshot, with authority and admission before payloads. +use super::*; +use crate::{attempts::Branch, eligibility, is_admitted, view_epoch, EligibilityState, Store}; +use rusqlite::{Connection, OptionalExtension}; + +// Bound allocation even for malicious row counts and text with JSON escapes. +// Conservative encoding estimates can refuse a selection before final sealing. +struct Limits { + remaining: usize, +} +impl Limits { + fn new() -> Self { + Self { + remaining: MAX_BUNDLE_BYTES, + } + } + fn take(&mut self, bytes: usize) -> Result<(), Error> { + self.remaining = self + .remaining + .checked_sub(bytes) + .ok_or(Error::LimitExceeded)?; + Ok(()) + } +} +struct Selection { + selected: SelectedAttempt, + parent: Option, + inherited: Option, + reuse: Option, + predecessor: Option, +} +impl Selection { + fn dependencies(&self) -> impl Iterator + '_ { + [self.inherited, self.reuse, self.predecessor] + .into_iter() + .flatten() + } +} + +// All expressions are fixed internal SQL, never caller text. A probe reads only +// SQLite type/byte length; CAST avoids SQLite text-length's embedded-NUL rules. +fn field( + db: &Connection, + from: &str, + column: &str, + id: i64, + kind: &str, + expansion: usize, + limits: &mut Limits, +) -> Result>, Error> { + let probe = format!("SELECT typeof({column}), length(CAST({column} AS BLOB)) FROM {from}"); + let shape: Option<(String, Option)> = db + .query_row(&probe, [id], |r| Ok((r.get(0)?, r.get(1)?))) + .optional()?; + let Some((storage, length)) = shape else { + return Ok(None); + }; + if storage == "null" { + return Ok(None); + } + if storage != kind { + return Err(Error::Store(crate::invalid( + "invalid export field storage class", + ))); + } + let length = usize::try_from(length.ok_or(Error::InconsistentLedger)?) + .map_err(|_| Error::LimitExceeded)?; + if length > MAX_PAYLOAD_BYTES { + return Err(Error::LimitExceeded); + } + limits.take( + length + .checked_mul(expansion) + .and_then(|n| n.checked_add(192)) + .ok_or(Error::LimitExceeded)?, + )?; + let sql = format!("SELECT CAST({column} AS BLOB) FROM {from}"); + Ok(Some(db.query_row(&sql, [id], |r| r.get(0))?)) +} +fn text_field( + db: &Connection, + from: &str, + column: &str, + id: i64, + limits: &mut Limits, +) -> Result, Error> { + field(db, from, column, id, "text", 6, limits)? + .map(|b| String::from_utf8(b).map_err(|_| Error::InvalidEncoding)) + .transpose() +} +fn envelope( + db: &Connection, + from: &str, + column: &str, + id: i64, + kind: &str, + limits: &mut Limits, +) -> Result, Error> { + field(db, from, column, id, kind, 2, limits)? + .map(|b| ByteEnvelope::from_bytes(&b)) + .transpose() +} +fn required(value: Option) -> Result { + value.ok_or(Error::InconsistentLedger) +} +fn pairs( + db: &Connection, + sql: &str, + id: i64, + limits: &mut Limits, +) -> Result, Error> { + let mut statement = db.prepare(sql)?; + let mut rows = statement.query([id])?; + let mut result = Vec::new(); + while let Some(row) = rows.next()? { + limits.take(128)?; + result.push((row.get(0)?, row.get(1)?)); + } + Ok(result) +} +fn select(db: &Connection, id: i64, limits: &mut Limits) -> Result, Error> { + let data: Option<(i64,Option,i64,i64,Option,Option,i64)> = db.query_row( + "SELECT a.branch_id,a.parent,a.branch_revision,a.view_epoch,a.inherited_from,b.head,b.revision FROM attempts a JOIN branches b ON b.id=a.branch_id WHERE a.id=?1",[id], + |r|Ok((r.get(0)?,r.get(1)?,r.get(2)?,r.get(3)?,r.get(4)?,r.get(5)?,r.get(6)?))).optional()?; + let Some((branch, parent, captured_revision, captured_epoch, inherited, head, revision)) = data + else { + return Ok(None); + }; + limits.take(4096)?; + let actor = required(text_field( + db, + "branches WHERE id=?1", + "actor", + branch, + limits, + )?)?; + let scope = required(text_field( + db, + "branches WHERE id=?1", + "scope", + branch, + limits, + )?)?; + let current_epoch = view_epoch(db, &actor, &scope)?; + let inputs = pairs( + db, + "SELECT position,evidence_id FROM attempt_inputs WHERE attempt_id=?1 ORDER BY position", + id, + limits, + )? + .into_iter() + .map(|(position, occurrence_id)| SelectedInput { + position, + occurrence_id, + }) + .collect(); + let reuse = db + .query_row( + "SELECT origin_attempt FROM model_reuses WHERE attempt_id=?1", + [id], + |r| r.get(0), + ) + .optional()?; + let predecessor = db.query_row("SELECT p.attempt_id FROM recovery_attempts a JOIN recovery_attempts p ON p.recovery_id=a.recovery_id AND p.ordinal=a.ordinal-1 WHERE a.attempt_id=?1",[id],|r|r.get(0)).optional()?; + let ordinal: Option = db + .query_row( + "SELECT ordinal FROM recovery_attempts WHERE attempt_id=?1", + [id], + |r| r.get(0), + ) + .optional()?; + if ordinal.is_some_and(|n| n > 0) && predecessor.is_none() { + return Ok(None); + } + Ok(Some(Selection { + selected: SelectedAttempt { + attempt: id, + actor, + scope, + branch, + head, + revision, + current_epoch, + captured_revision, + captured_epoch, + inputs, + }, + parent, + inherited, + reuse, + predecessor, + })) +} +fn visit( + db: &Connection, + id: i64, + limits: &mut Limits, + all: &mut Vec, + path: &mut Vec, +) -> Result { + if path.contains(&id) { + return Ok(false); + } + if all.iter().any(|s| s.selected.attempt == id) { + return Ok(true); + } + if all.len() >= MAX_ATTEMPTS { + return Err(Error::LimitExceeded); + } + let Some(s) = select(db, id, limits)? else { + return Ok(false); + }; + let deps: Vec<_> = s.dependencies().collect(); + all.push(s); + path.push(id); + for dep in deps { + let exists: bool = db.query_row( + "SELECT EXISTS(SELECT 1 FROM deliveries WHERE attempt_id=?1)", + [dep], + |r| r.get(0), + )?; + if !exists || !visit(db, dep, limits, all, path)? { + return Ok(false); + } + } + path.pop(); + Ok(true) +} +impl Store { + /// Export authorized bytes from one snapshot. An exported file never grants + /// access, dispatch or import authority. No archive fallback is performed. + pub fn export_decision_bundle( + &mut self, + request: &BundleRequest, + authorizer: &impl DiagnosticAuthorizer, + ) -> Result { + let tx = self.conn.transaction()?; + let result = export(&tx, request, authorizer); + // Also commit read-only redactions; an SQL error is never a successful export. + tx.commit()?; + match result { + Ok(Some(bundle)) => Ok(BundleExport::Complete(bundle)), + Ok(None) | Err(Error::LimitExceeded) => Ok(BundleExport::Redacted(RedactedBundleV1 { + requested_attempt: request.attempt, + reason: RedactionReason::Unavailable, + })), + Err(error) => Err(error), + } + } +} +fn export( + db: &Connection, + request: &BundleRequest, + authorizer: &impl DiagnosticAuthorizer, +) -> Result, Error> { + let mut limits = Limits::new(); + limits.take( + request + .actor + .len() + .checked_add(request.scope.len()) + .and_then(|n| n.checked_mul(6)) + .ok_or(Error::LimitExceeded)?, + )?; + let mut selection = Vec::new(); + if !visit( + db, + request.attempt, + &mut limits, + &mut selection, + &mut Vec::new(), + )? { + return Ok(None); + } + let root = &selection[0].selected; + if root.actor != request.actor || root.scope != request.scope { + return Ok(None); + } + if request.purpose == BundlePurpose::DiagnosticReplay { + let exact = DiagnosticReadRequest { + actor: request.actor.clone(), + scope: request.scope.clone(), + root_attempt: request.attempt, + purpose: request.purpose, + attempts: selection.iter().map(|s| s.selected.clone()).collect(), + }; + if !authorizer.authorize(&exact)? { + return Ok(None); + } + } else { + for s in &selection { + let a = &s.selected; + if a.actor != request.actor || a.scope != request.scope || a.head != Some(a.attempt) { + return Ok(None); + } + let branch = Branch { + id: a.branch, + head: a.head, + revision: a.revision, + }; + if eligibility::evaluate(db, Some(&branch), a.current_epoch)?.state + != EligibilityState::Ready + { + return Ok(None); + } + } + } + for s in &selection { + for input in &s.selected.inputs { + if !is_admitted( + db, + &s.selected.actor, + &s.selected.scope, + input.occurrence_id, + )? { + return Ok(None); + } + let exists: bool = db.query_row( + "SELECT EXISTS(SELECT 1 FROM evidence WHERE id=?1)", + [input.occurrence_id], + |r| r.get(0), + )?; + if !exists { + return Ok(None); + } + } + } + #[cfg(test)] + snapshot_tests::before_payload(db, &selection); + let mut bundle = DecisionBundleV1 { + schema_version: 1, + root_attempt: request.attempt, + actor: request.actor.clone(), + scope: request.scope.clone(), + export_purpose: match request.purpose { + BundlePurpose::CurrentRead => "current_read", + BundlePurpose::DiagnosticReplay => "diagnostic_replay", + } + .into(), + snapshot: BundleSnapshot { + root_branch: root.branch, + head: root.head, + revision: root.revision, + current_epoch: root.current_epoch, + }, + attempts: Vec::new(), + evidence: Vec::new(), + integrity: IntegrityManifest { + version: "decision-bundle-integrity/1".into(), + sha256: String::new(), + }, + }; + for s in &selection { + let a = &s.selected; + for input in &a.inputs { + let scope = AuthorizedScope { + actor: a.actor.clone(), + scope: a.scope.clone(), + }; + if let Some(e) = bundle + .evidence + .iter_mut() + .find(|e| e.occurrence_id == input.occurrence_id) + { + if !e.authorized_scopes.contains(&scope) { + limits.take(scope.actor.len() * 6 + scope.scope.len() * 6 + 128)?; + e.authorized_scopes.push(scope); + } + } else { + limits.take(scope.actor.len() * 6 + scope.scope.len() * 6 + 256)?; + bundle.evidence.push(EvidenceRecord { + occurrence_id: input.occurrence_id, + source: required(envelope( + db, + "evidence WHERE id=?1", + "source", + input.occurrence_id, + "text", + &mut limits, + )?)?, + payload: required(envelope( + db, + "evidence WHERE id=?1", + "payload", + input.occurrence_id, + "blob", + &mut limits, + )?)?, + authorized_scopes: vec![scope], + }); + } + } + bundle.attempts.push(load_attempt(db, s, &mut limits)?); + } + for attempt in &mut bundle.attempts { + bind_incident(attempt, &bundle.evidence, &mut limits)?; + } + bundle.seal()?; + Ok(Some(bundle)) +} + +fn load_attempt( + db: &Connection, + s: &Selection, + limits: &mut Limits, +) -> Result { + let a = &s.selected; + let id = a.attempt; + let delivery = if let Some(raw) = envelope( + db, + "deliveries WHERE attempt_id=?1", + "output", + id, + "blob", + limits, + )? { + let disposition = required(text_field( + db, + "deliveries WHERE attempt_id=?1", + "disposition", + id, + limits, + )?)?; + let disposition = match disposition.as_str() { + "accepted" => RecordedDisposition::Accepted, + "stale_view" => RecordedDisposition::StaleView, + "stale_branch" => RecordedDisposition::StaleBranch, + "invalid_output" => RecordedDisposition::InvalidOutput, + _ => return Err(Error::InconsistentLedger), + }; + let usage = db.query_row( + "SELECT usage FROM deliveries WHERE attempt_id=?1", + [id], + |r| r.get(0), + )?; + let rejection = if disposition == RecordedDisposition::InvalidOutput { + Some(RejectionMetadata { + validator: required(text_field( + db, + "deliveries WHERE attempt_id=?1", + "validator", + id, + limits, + )?)?, + reason: required(text_field( + db, + "deliveries WHERE attempt_id=?1", + "reason", + id, + limits, + )?)?, + applicability: required(text_field( + db, + "deliveries WHERE attempt_id=?1", + "applicability", + id, + limits, + )?)?, + }) + } else { + None + }; + DeliveryState::Recorded { + raw, + disposition, + usage: Some(usage), + rejection, + } + } else { + DeliveryState::Unresolved + }; + let deterministic = if let Some(version) = text_field( + db, + "deterministic_verdicts WHERE attempt_id=?1", + "computation", + id, + limits, + )? { + Some(DeterministicContract { + version, + output: required(envelope( + db, + "deterministic_verdicts WHERE attempt_id=?1", + "output", + id, + "blob", + limits, + )?)?, + }) + } else { + None + }; + let support = load_support(db, id, limits)?; + let recovery_key: Option<(i64, i64)> = db + .query_row( + "SELECT recovery_id,ordinal FROM recovery_attempts WHERE attempt_id=?1", + [id], + |r| Ok((r.get(0)?, r.get(1)?)), + ) + .optional()?; + let recovery = if let Some((recovery_id, ordinal)) = recovery_key { + Some(RecoveryRecord { + recovery_id, + ordinal, + state: required(text_field( + db, + "recovery_attempts WHERE attempt_id=?1", + "state", + id, + limits, + )?)?, + request: envelope( + db, + "recovery_attempts WHERE attempt_id=?1", + "request", + id, + "blob", + limits, + )?, + predecessor_id: s.predecessor, + }) + } else { + None + }; + let timing_configuration = if let Some((key, _)) = recovery_key { + envelope( + db, + "recovery_timing_bindings WHERE recovery_id=?1", + "configuration", + key, + "text", + limits, + )? + } else { + None + }; + let immutable_runtime = if let Some((key, _)) = recovery_key { + envelope( + db, + "recovery_timing_bindings WHERE recovery_id=?1", + "runtime_json", + key, + "text", + limits, + )? + } else { + None + }; + let reservation:Option<(i64,i64,i64,bool)>=db.query_row("SELECT budget_id,input_tokens,output_cap,granted FROM token_reservations WHERE attempt_id=?1",[id],|r|Ok((r.get(0)?,r.get(1)?,r.get(2)?,r.get(3)?))).optional()?; + let charge = match &delivery { + DeliveryState::Recorded { usage, .. } => required(*usage)?, + DeliveryState::Unresolved => 0, + }; + let hold_bound = match reservation { + Some((_, input, cap, true)) + if matches!(delivery, DeliveryState::Unresolved) + && !recovery.as_ref().is_some_and(|r| r.state == "cancelled") => + { + input.checked_add(cap).ok_or(Error::InconsistentLedger)? + } + _ => 0, + }; + let validator = if let DeliveryState::Recorded { + rejection: Some(r), .. + } = &delivery + { + Some(ValidatorBinding { + version: r.validator.clone(), + provenance: ValidatorProvenance::RecordedRejection, + }) + } else { + deterministic.as_ref().map(|d| ValidatorBinding { + version: d.version.clone(), + provenance: ValidatorProvenance::DeterministicBinding, + }) + }; + let role = if deterministic + .as_ref() + .is_some_and(|d| d.version == "report-verdict/v1") + { + InputRole::Report + } else if support.is_some() { + InputRole::Evidence + } else { + InputRole::Unknown + }; + Ok(AttemptRecord{actor:a.actor.clone(),scope:a.scope.clone(),id,branch:a.branch,captured_epoch:a.captured_epoch,captured_revision:a.captured_revision,parent:s.parent,inherited_from:s.inherited, + candidates:pairs(db,"SELECT position,requested_id FROM attempt_candidates WHERE attempt_id=?1 ORDER BY position",id,limits)?.into_iter().map(|(position,evidence_id)|CandidateRecord{position,evidence_id}).collect(), + inputs:a.inputs.iter().map(|i|InputRecord{position:i.position,evidence_id:i.occurrence_id,role}).collect(), + dispatch_request:envelope(db,"dispatches WHERE attempt_id=?1","request",id,"blob",limits)?,delivery,validator,immutable_runtime,immutable_manifest:None,timing_configuration, + accounting:AttemptAccounting{unit:if role!=InputRole::Unknown{AccountingUnit::LogicalSupport}else{AccountingUnit::Unknown},quote:reservation.map(|(_,input_tokens,output_cap,_)|ReservationQuote{input_tokens,output_cap}),granted:reservation.map(|r|r.3),budget_reference:reservation.map(|r|r.0),charge,hold_bound}, + reuse_origin:s.reuse,model_contract:text_field(db,"model_attempts WHERE attempt_id=?1","contract",id,limits)?,deterministic,support,recovery}) +} +fn load_support( + db: &Connection, + id: i64, + limits: &mut Limits, +) -> Result, Error> { + let bound: Option<(i64, bool)> = db + .query_row( + "SELECT rule_id,verdict FROM support_bindings WHERE attempt_id=?1", + [id], + |r| Ok((r.get(0)?, r.get(1)?)), + ) + .optional()?; + let Some((rule_id, verdict)) = bound else { + return Ok(None); + }; + let clauses = pairs( + db, + "SELECT id,id FROM support_clauses WHERE rule_id=?1 ORDER BY id", + rule_id, + limits, + )?; + let mut result = SupportContract { + version: "support/v1".into(), + rule_id, + verdict, + clauses: Vec::new(), + }; + for (clause_id, _) in clauses { + let premises=pairs(db,"SELECT requested_id,requested_id FROM support_premises WHERE clause_id=?1 ORDER BY requested_id",clause_id,limits)?; + let mut clause = SupportClause { + clause_id, + premises: Vec::new(), + }; + for (evidence_id, _) in premises { + // Composite key bound in SQL literals (integers from SQLite, not text). + let from = format!("support_premises WHERE clause_id={clause_id} AND requested_id=?1"); + clause.premises.push(SupportPremise { + evidence_id, + actor: required(text_field(db, &from, "actor", evidence_id, limits)?)?, + scope: required(text_field(db, &from, "scope", evidence_id, limits)?)?, + }); + } + result.clauses.push(clause); + } + Ok(Some(result)) +} + +// Count borrowed values once before repeated source metadata is cloned. A +// context is individually bounded, but repeating its case ID can otherwise +// produce a much larger candidate manifest even when no fingerprint matches. +fn manifest_bound( + context: &crate::incident_protocol::IncidentContextV1, + runtime: &crate::RecoveryRuntime, +) -> Result { + fn encoded(value: &T) -> Result { + let mut counter = super::CountWriter { size: 0 }; + serde_json::to_writer(&mut counter, value).map_err(|_| Error::LimitExceeded)?; + Ok(counter.size) + } + let mut size = 1024usize; + let mut add = |amount: usize| -> Result<(), Error> { + size = size + .checked_add(amount) + .filter(|n| *n <= MAX_PAYLOAD_BYTES) + .ok_or(Error::LimitExceeded)?; + Ok(()) + }; + add(encoded(runtime)?)?; + for text in [ + &context.workload_version, + &context.freeze_sha256, + &context.prompt_sha256, + ] { + add(encoded(text)?)?; + } + let case = encoded(&context.case_id)?; + let workload = encoded(&context.workload_version)?; + for external in &context.evidence_ids { + add(256)?; // keys, digest, integer occurrence ID, and punctuation + add(case)?; + add(workload)?; + add(encoded(external)?)?; + } + Ok(size) +} + +// B does not persist its manifest separately. Reconstruct its exact canonical +// bytes from retained inputs and bindings; only a matching fingerprint allows +// the declared contract, roles or fixture accounting unit to be attached. +fn bind_incident( + a: &mut AttemptRecord, + evidence: &[EvidenceRecord], + limits: &mut Limits, +) -> Result<(), Error> { + let (Some(runtime), Some(timing), Some(first)) = ( + &a.immutable_runtime, + &a.timing_configuration, + a.inputs.first(), + ) else { + return Ok(()); + }; + let Ok(runtime) = super::decode_strict::(&runtime.decode()?) else { + return Ok(()); + }; + let Ok(timing) = super::decode_strict::(&timing.decode()?) else { + return Ok(()); + }; + let Some(context) = evidence + .iter() + .find(|e| e.occurrence_id == first.evidence_id) + else { + return Ok(()); + }; + let Ok(context_value) = + crate::incident_protocol::IncidentContextV1::from_json(&context.payload.decode()?) + else { + return Ok(()); + }; + if a.inputs.len() != context_value.evidence_ids.len() + 1 || a.inputs.len() > 1024 { + return Ok(()); + } + let bound = manifest_bound(&context_value, &runtime)?; + limits.take( + bound + .checked_mul(2) + .and_then(|n| n.checked_add(192)) + .ok_or(Error::LimitExceeded)?, + )?; + let mut runtime_value = serde_json::to_value(&runtime).map_err(|_| Error::InvalidEncoding)?; + runtime_value + .as_object_mut() + .ok_or(Error::InvalidEncoding)? + .remove("prompt_fingerprint"); + let mut sources = Vec::new(); + for (input, external) in a.inputs.iter().skip(1).zip(&context_value.evidence_ids) { + let Some(e) = evidence + .iter() + .find(|e| e.occurrence_id == input.evidence_id) + else { + return Ok(()); + }; + sources.push(serde_json::json!({"workload_version":context_value.workload_version,"case_id":context_value.case_id,"external_id":external,"occurrence_id":input.evidence_id,"payload_sha256":e.payload.sha256})); + } + let manifest=serde_json::to_vec(&serde_json::json!({ + "version":"incident-store-manifest/1","schema_version":1, + "context":{"occurrence_id":first.evidence_id,"sha256":context.payload.sha256}, + "sources":sources,"workload_version":context_value.workload_version, + "freeze_sha256":context_value.freeze_sha256,"prompt_sha256":context_value.prompt_sha256, + "validator_version":"incident-routing-protocol/1","initial_renderer":"incident-request/1", + "retry_renderer":"incident-request-retry/1","retry_feedback":"incident-routing-protocol/1:invalid_output", + "output_cap":timing.calibration.output_cap,"runtime":runtime_value, + })).map_err(|_|Error::InvalidEncoding)?; + if sha256_hex(&manifest) != runtime.prompt_fingerprint { + return Ok(()); + } + a.immutable_manifest = Some(ByteEnvelope::from_bytes(&manifest)?); + for (index, input) in a.inputs.iter_mut().enumerate() { + input.role = if index == 0 { + InputRole::Context + } else { + InputRole::Report + }; + } + if a.validator.is_none() { + a.validator = Some(ValidatorBinding { + version: "incident-routing-protocol/1".into(), + provenance: ValidatorProvenance::DeclaredContract, + }); + } + if super::replay::fixture_runtime(a) { + a.accounting.unit = AccountingUnit::FixtureBytes; + } + Ok(()) +} + +#[cfg(test)] +mod snapshot_tests { + use super::*; + use std::cell::RefCell; + type Hook = Box; + thread_local! {static BOUNDARY:RefCell>=RefCell::new(None);} + pub(super) fn before_payload(db: &Connection, selection: &[Selection]) { + BOUNDARY.with(|cell| { + if let Some(hook) = cell.borrow_mut().take() { + hook(db, selection) + } + }); + } + + fn dump(db: &Connection) -> Vec { + let mut names = db + .prepare("SELECT name FROM sqlite_schema WHERE type='table' ORDER BY name") + .unwrap(); + let names = names + .query_map([], |r| r.get::<_, String>(0)) + .unwrap() + .collect::>>() + .unwrap(); + let mut result = Vec::new(); + for name in names { + let mut statement = db + .prepare(&format!("SELECT * FROM \"{}\"", name.replace('"', "\"\""))) + .unwrap(); + let columns = statement.column_count(); + let mut rows = statement.query([]).unwrap(); + while let Some(row) = rows.next().unwrap() { + result.push(format!( + "{name}: {:?}", + (0..columns) + .map(|i| row.get_ref(i).unwrap()) + .collect::>() + )); + } + } + result.sort(); + result + } + struct Allow; + impl DiagnosticAuthorizer for Allow { + fn authorize(&self, _: &DiagnosticReadRequest) -> Result { + Ok(true) + } + } + fn concurrent(change: u8) { + use std::{sync::mpsc, time::Duration}; + static NEXT: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(0); + let path = std::env::temp_dir().join(format!( + "kairotensor-export-wal-{}-{}.sqlite", + std::process::id(), + NEXT.fetch_add(1, std::sync::atomic::Ordering::Relaxed) + )); + let mut reader = Store::open(&path).unwrap(); + reader + .conn + .pragma_update(None, "journal_mode", "WAL") + .unwrap(); + let input = reader + .observe_and_admit_many("a", "s", &[("source", b"old evidence")]) + .unwrap()[0]; + let root = reader.begin_attempt("a", "s", "main", &[input]).unwrap(); + let budget = reader.create_token_budget(100).unwrap(); + reader + .reserve_dispatch(budget, root, b"old request", 3, 20) + .unwrap(); + if change != 2 { + reader.settle(root, b"old delivery", 5).unwrap(); + } + let before = dump(&reader.conn); + let expected = before.clone(); + let mut writer = Store::open(&path).unwrap(); + let (start_tx, start_rx) = mpsc::channel(); + let (done_tx, done_rx) = mpsc::channel(); + let worker = std::thread::spawn(move || { + start_rx + .recv_timeout(Duration::from_secs(5)) + .expect("payload barrier reached"); + match change { + 0 => { + writer.set_admitted("a", "s", input, false).unwrap(); + } + 1 => { + let next = writer.begin_attempt("a", "s", "main", &[input]).unwrap(); + writer.settle(next, b"new head", 7).unwrap(); + } + 2 => { + writer.settle(root, b"new delivery", 6).unwrap(); + } + _ => unreachable!(), + } + let committed = dump(&writer.conn); + done_tx.send(committed).unwrap(); + }); + let observed = std::rc::Rc::new(RefCell::new(None)); + let observed_hook = observed.clone(); + BOUNDARY.with(|cell|*cell.borrow_mut()=Some(Box::new(move|db,selection|{ + start_tx.send(()).unwrap(); + let committed=done_rx.recv_timeout(Duration::from_secs(5)).expect("WAL writer commits while reader retains snapshot"); + assert!(!db.is_autocommit(),"payload fetch must retain the transaction"); + assert_eq!(selection.len(),1);let selected=&selection[0].selected; + assert_eq!(selected.attempt,root);assert_eq!(selected.current_epoch,1); + assert_eq!(view_epoch(db,"a","s").unwrap(),1); + assert!(is_admitted(db,"a","s",input).unwrap()); + assert_eq!(dump(db),expected,"all selected rows, heads, admissions, holds and deliveries remain in original snapshot at payload boundary"); + assert_ne!(committed,expected,"writer must actually change committed rows"); + *observed_hook.borrow_mut()=Some(committed); + }))); + let request = BundleRequest { + actor: "a".into(), + scope: "s".into(), + attempt: root, + purpose: if change == 2 { + BundlePurpose::DiagnosticReplay + } else { + BundlePurpose::CurrentRead + }, + }; + let result = reader.export_decision_bundle(&request, &Allow).unwrap(); + worker.join().unwrap(); + let BundleExport::Complete(b) = result else { + panic!("retained admitted snapshot should export") + }; + if change == 2 { + assert_eq!(b.attempts[0].delivery, DeliveryState::Unresolved); + assert_eq!(b.attempts[0].accounting.hold_bound, 23); + } else { + assert!( + matches!(&b.attempts[0].delivery,DeliveryState::Recorded{raw,..} if raw.decode().unwrap()==b"old delivery") + ); + assert_eq!(b.snapshot.head, Some(root)); + } + assert!(reader.conn.is_autocommit()); + let committed = observed + .borrow() + .clone() + .expect("actual payload boundary executed"); + assert_eq!( + dump(&reader.conn), + committed, + "export introduces no writes after concurrent commit" + ); + drop(reader); + let mut reopened = Store::open(&path).unwrap(); + assert_eq!( + dump(&reopened.conn), + committed, + "reopen preserves all rows and holds" + ); + let fresh = reopened.export_decision_bundle(&request, &Allow).unwrap(); + if change < 2 { + assert!(matches!(fresh, BundleExport::Redacted(_))); + } else { + let BundleExport::Complete(b) = fresh else { + panic!() + }; + assert_eq!(b.attempts[0].accounting.hold_bound, 0); + assert_eq!(b.attempts[0].accounting.charge, 6); + } + assert_eq!( + dump(&reopened.conn), + committed, + "fresh export is also read-only" + ); + } + #[test] + fn withdrawal_retains_live_snapshot_through_payload_fetch() { + concurrent(0) + } + #[test] + fn head_change_retains_live_snapshot_through_payload_fetch() { + concurrent(1) + } + #[test] + fn settlement_retains_live_snapshot_through_payload_fetch() { + concurrent(2) + } +} diff --git a/src/decision_bundle/types.rs b/src/decision_bundle/types.rs index d078c5b..f3a64cd 100644 --- a/src/decision_bundle/types.rs +++ b/src/decision_bundle/types.rs @@ -294,3 +294,76 @@ pub struct ReplayReport { pub reevaluated_validator: Option, pub attempts: Vec, } + +/// Caller identity must be authenticated by the embedding runtime. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct BundleRequest { + pub actor: String, + pub scope: String, + pub attempt: i64, + pub purpose: BundlePurpose, +} +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize)] +#[serde(rename_all = "snake_case")] +pub enum BundlePurpose { + CurrentRead, + DiagnosticReplay, +} + +/// Ephemeral exact selection, never a persisted or transferable access grant. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct DiagnosticReadRequest { + pub actor: String, + pub scope: String, + pub root_attempt: i64, + pub purpose: BundlePurpose, + pub attempts: Vec, +} +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct SelectedAttempt { + pub attempt: i64, + pub actor: String, + pub scope: String, + pub branch: i64, + pub head: Option, + pub revision: i64, + pub current_epoch: i64, + pub captured_revision: i64, + pub captured_epoch: i64, + pub inputs: Vec, +} +/// Repeated occurrences retain their consuming attempt and compacted position. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct SelectedInput { + pub position: i64, + pub occurrence_id: i64, +} + +pub trait DiagnosticAuthorizer { + /// Synchronous trusted policy; must not reenter/mutate this Store or use the + /// network. Approval covers precisely this retained-snapshot selection. + fn authorize(&self, request: &DiagnosticReadRequest) -> Result; +} +#[derive(Clone, Copy, Debug, Default)] +pub struct DenyDiagnostics; +impl DiagnosticAuthorizer for DenyDiagnostics { + fn authorize(&self, _: &DiagnosticReadRequest) -> Result { + Ok(false) + } +} +#[derive(Clone, Debug, PartialEq, Eq, Serialize)] +#[serde(tag = "status", content = "bundle", rename_all = "snake_case")] +pub enum BundleExport { + Complete(DecisionBundleV1), + Redacted(RedactedBundleV1), +} +#[derive(Clone, Debug, PartialEq, Eq, Serialize)] +pub struct RedactedBundleV1 { + pub requested_attempt: i64, + pub reason: RedactionReason, +} +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize)] +#[serde(rename_all = "snake_case")] +pub enum RedactionReason { + Unavailable, +} diff --git a/tests/decision_bundle.rs b/tests/decision_bundle.rs index 45573eb..25589cf 100644 --- a/tests/decision_bundle.rs +++ b/tests/decision_bundle.rs @@ -717,3 +717,964 @@ fn planned_recovery_phase_preserves_absent_per_attempt_budget_and_quote() { ); } } + +// Store export authority tests: removing either the diagnostic authorization or +// current admission gate must expose the canaries and fail these tests. +fn export_request(attempt: i64, purpose: BundlePurpose) -> BundleRequest { + BundleRequest { + actor: "a".into(), + scope: "s".into(), + attempt, + purpose, + } +} +struct AllowExport; +impl DiagnosticAuthorizer for AllowExport { + fn authorize(&self, _: &DiagnosticReadRequest) -> Result { + Ok(true) + } +} +fn complete(export: BundleExport) -> DecisionBundleV1 { + match export { + BundleExport::Complete(bundle) => bundle, + other => panic!("expected complete: {other:?}"), + } +} +#[test] +fn export_default_denial_missing_withheld_and_revoked_are_payload_free() { + let mut store = kairotensor::Store::in_memory().unwrap(); + let evidence = store + .observe_and_admit_many("a", "s", &[("SOURCE_CANARY", b"PAYLOAD_CANARY")]) + .unwrap()[0]; + let id = store.begin_attempt("a", "s", "main", &[evidence]).unwrap(); + store.record_dispatch(id, b"REQUEST_CANARY").unwrap(); + store + .reject_invalid( + id, + b"RESPONSE_CANARY", + 3, + "VALIDATOR_CANARY", + "REASON_CANARY", + ) + .unwrap(); + let request = export_request(id, BundlePurpose::DiagnosticReplay); + let denied = store + .export_decision_bundle(&request, &DenyDiagnostics) + .unwrap(); + assert_eq!( + denied, + BundleExport::Redacted(RedactedBundleV1 { + requested_attempt: id, + reason: RedactionReason::Unavailable + }) + ); + let authorized = complete( + store + .export_decision_bundle(&request, &AllowExport) + .unwrap(), + ); + assert_eq!( + authorized.evidence[0].source.decode().unwrap(), + b"SOURCE_CANARY" + ); + assert_eq!( + authorized.attempts[0] + .dispatch_request + .as_ref() + .unwrap() + .decode() + .unwrap(), + b"REQUEST_CANARY" + ); + store.set_admitted("a", "s", evidence, false).unwrap(); + assert_eq!( + store + .export_decision_bundle(&request, &AllowExport) + .unwrap(), + denied + ); + let missing = export_request(999, BundlePurpose::DiagnosticReplay); + assert_eq!( + store + .export_decision_bundle(&missing, &AllowExport) + .unwrap(), + BundleExport::Redacted(RedactedBundleV1 { + requested_attempt: 999, + reason: RedactionReason::Unavailable + }) + ); + let mut wrong = request.clone(); + wrong.actor = "other".into(); + assert_eq!( + store.export_decision_bundle(&wrong, &AllowExport).unwrap(), + denied + ); + let redacted = serde_json::to_string(&denied).unwrap(); + for private in ["CANARY", "sha256", "source", "inputs", "validator", "head"] { + assert!(!redacted.contains(private)); + } +} +#[test] +fn export_current_gate_preserves_filtered_positions_and_legacy_unknowns() { + let mut store = kairotensor::Store::in_memory().unwrap(); + let evidence = store + .observe_and_admit_many("a", "s", &[("raw", b"\xff\0")]) + .unwrap()[0]; + let (id, _) = store + .begin_from_candidates("a", "s", "main", &[999, evidence, evidence]) + .unwrap(); + store.settle(id, b"\xfe\0", 7).unwrap(); + let request = export_request(id, BundlePurpose::CurrentRead); + let b = complete( + store + .export_decision_bundle(&request, &DenyDiagnostics) + .unwrap(), + ); + assert_eq!( + b.attempts[0] + .candidates + .iter() + .map(|c| (c.position, c.evidence_id)) + .collect::>(), + [(0, 999), (1, evidence), (2, evidence)] + ); + assert_eq!( + b.attempts[0] + .inputs + .iter() + .map(|c| (c.position, c.evidence_id)) + .collect::>(), + [(0, evidence), (1, evidence)] + ); + assert_eq!(b.evidence.len(), 1); + assert_eq!(b.evidence[0].payload.decode().unwrap(), [255, 0]); + assert_eq!(b.attempts[0].validator, None); + assert_eq!(b.attempts[0].accounting.unit, AccountingUnit::Unknown); + assert_eq!(parse_decision_bundle(&b.to_json().unwrap()).unwrap(), b); + let next = store.begin_attempt("a", "s", "main", &[]).unwrap(); + store.settle(next, b"new", 1).unwrap(); + assert!(matches!( + store + .export_decision_bundle(&request, &AllowExport) + .unwrap(), + BundleExport::Redacted(_) + )); + let historical = complete( + store + .export_decision_bundle( + &export_request(id, BundlePurpose::DiagnosticReplay), + &AllowExport, + ) + .unwrap(), + ); + assert_eq!(historical.snapshot.head, Some(next)); + // Parent is lineage only: selecting next must not copy the old payload. + let next_bundle = complete( + store + .export_decision_bundle( + &export_request(next, BundlePurpose::CurrentRead), + &DenyDiagnostics, + ) + .unwrap(), + ); + assert_eq!(next_bundle.attempts.len(), 1); + assert!(next_bundle.evidence.is_empty()); +} + +#[test] +fn export_support_ready_still_denies_withdrawn_captured_input() { + let path = export_database(); + let mut store = kairotensor::Store::open(&path).unwrap(); + let ids = store + .observe_and_admit_many("a", "s", &[("one", b"one"), ("two", b"two")]) + .unwrap(); + let id = store + .publish_support("a", "s", "main", &[vec![ids[0]], vec![ids[1]], vec![999]]) + .unwrap(); + let before = complete( + store + .export_decision_bundle( + &export_request(id, BundlePurpose::CurrentRead), + &DenyDiagnostics, + ) + .unwrap(), + ); + assert_eq!( + before.attempts[0].support.as_ref().unwrap().clauses.len(), + 3 + ); + assert_eq!( + before.attempts[0].accounting.unit, + AccountingUnit::LogicalSupport + ); + assert!( + before.evidence.is_empty(), + "support registration captures formula, not evidence bytes" + ); + // Adversarial persisted-state fixture, not produced by register_support: + // retain an additional captured input to isolate the export admission gate. + rusqlite::Connection::open(&path) + .unwrap() + .execute( + "INSERT INTO attempt_inputs(attempt_id,position,evidence_id) VALUES(?1,0,?2)", + rusqlite::params![id, ids[0]], + ) + .unwrap(); + store.set_admitted("a", "s", ids[0], false).unwrap(); + assert!(matches!( + store.read_current("a", "s", "main").unwrap(), + kairotensor::Current::Ready(_) + )); + for purpose in [BundlePurpose::CurrentRead, BundlePurpose::DiagnosticReplay] { + assert!(matches!( + store + .export_decision_bundle(&export_request(id, purpose), &AllowExport) + .unwrap(), + BundleExport::Redacted(_) + )); + } +} +#[test] +fn export_ambiguous_hold_cancelled_and_planned_are_per_attempt() { + let mut store = kairotensor::Store::in_memory().unwrap(); + let budget = store.create_token_budget(1000).unwrap(); + let foreign = store + .begin_attempt("private", "scope", "main", &[]) + .unwrap(); + store + .reserve_dispatch(budget, foreign, b"FOREIGN_REQUEST", 37, 50) + .unwrap(); + store.settle(foreign, b"FOREIGN_OUTPUT", 40).unwrap(); + let id = store.begin_attempt("a", "s", "main", &[]).unwrap(); + store + .reserve_dispatch(budget, id, b"\xff request", 3, 20) + .unwrap(); + let b = complete( + store + .export_decision_bundle( + &export_request(id, BundlePurpose::DiagnosticReplay), + &AllowExport, + ) + .unwrap(), + ); + assert_eq!(b.attempts.len(), 1); + assert_eq!(b.attempts[0].accounting.hold_bound, 23); + assert_eq!(b.attempts[0].accounting.charge, 0); + assert_eq!(b.attempts[0].accounting.budget_reference, Some(budget)); + let wire = String::from_utf8(b.to_json().unwrap()).unwrap(); + for name in [ + "token_limit", + "spent_tokens", + "reserved_tokens", + "FOREIGN", + "private", + ] { + assert!(!wire.contains(name)); + } + let recovery = store + .create_recovery("r", budget, "a", "s", "other") + .unwrap(); + let phase = match store.begin_recovery_attempt(recovery, &[]).unwrap() { + kairotensor::RecoveryStart::New { attempt, .. } => attempt, + _ => panic!(), + }; + let b = complete( + store + .export_decision_bundle( + &export_request(phase, BundlePurpose::DiagnosticReplay), + &AllowExport, + ) + .unwrap(), + ); + assert_eq!(b.attempts[0].recovery.as_ref().unwrap().state, "planned"); + assert_eq!(b.attempts[0].accounting.quote, None); + store + .reserve_recovery_attempt(phase, b"prepared", 4, 10) + .unwrap(); + store.cancel_recovery_attempt(phase).unwrap(); + let b = complete( + store + .export_decision_bundle( + &export_request(phase, BundlePurpose::DiagnosticReplay), + &AllowExport, + ) + .unwrap(), + ); + assert_eq!(b.attempts[0].accounting.hold_bound, 0); + assert_eq!( + b.attempts[0] + .accounting + .quote + .as_ref() + .unwrap() + .input_tokens, + 4 + ); + assert_eq!(b.attempts[0].dispatch_request, None); + assert_eq!(store.token_budget_state(budget).unwrap().reserved, 23); +} +#[test] +fn export_oversized_source_payload_request_and_reason_redact_generically() { + for which in 0..4 { + let mut store = kairotensor::Store::in_memory().unwrap(); + let large = "x".repeat(MAX_PAYLOAD_BYTES + 1); + let id = store + .observe( + if which == 0 { &large } else { "source" }, + if which == 1 { + large.as_bytes() + } else { + b"input" + }, + ) + .unwrap(); + store.set_admitted("a", "s", id, true).unwrap(); + let a = store.begin_attempt("a", "s", "main", &[id]).unwrap(); + store + .record_dispatch( + a, + if which == 2 { + large.as_bytes() + } else { + b"request" + }, + ) + .unwrap(); + store + .reject_invalid( + a, + b"output", + 1, + "v", + if which == 3 { &large } else { "reason" }, + ) + .unwrap(); + assert_eq!( + store + .export_decision_bundle( + &export_request(a, BundlePurpose::DiagnosticReplay), + &AllowExport + ) + .unwrap(), + BundleExport::Redacted(RedactedBundleV1 { + requested_attempt: a, + reason: RedactionReason::Unavailable + }) + ); + } +} +#[test] +fn export_inherited_closure_is_exact_and_current_requires_every_output() { + use std::cell::RefCell; + struct Inspect(RefCell>); + impl DiagnosticAuthorizer for Inspect { + fn authorize(&self, r: &DiagnosticReadRequest) -> Result { + *self.0.borrow_mut() = Some(r.clone()); + Ok(true) + } + } + let mut store = kairotensor::Store::in_memory().unwrap(); + let evidence = store + .observe_and_admit_many("a", "s", &[("source", b"data")]) + .unwrap()[0]; + let first = store + .begin_attempt("a", "s", "main", &[evidence, evidence]) + .unwrap(); + store.settle(first, b"first", 1).unwrap(); + let (next, _) = store.begin_resume("a", "s", "main", &[]).unwrap(); + store.settle(next, b"next", 2).unwrap(); + let inspect = Inspect(RefCell::new(None)); + let b = complete( + store + .export_decision_bundle( + &export_request(next, BundlePurpose::DiagnosticReplay), + &inspect, + ) + .unwrap(), + ); + assert_eq!( + b.attempts.iter().map(|a| a.id).collect::>(), + [next, first] + ); + let selection = inspect.0.borrow(); + let selection = selection.as_ref().unwrap(); + assert_eq!(selection.root_attempt, next); + assert_eq!(selection.purpose, BundlePurpose::DiagnosticReplay); + assert_eq!( + selection + .attempts + .iter() + .map(|a| a.attempt) + .collect::>(), + [next, first] + ); + assert_eq!( + selection.attempts[1].inputs, + [ + SelectedInput { + position: 0, + occurrence_id: evidence + }, + SelectedInput { + position: 1, + occurrence_id: evidence + } + ] + ); + assert_eq!(selection.attempts[1].head, Some(next)); + assert_eq!(selection.attempts[1].revision, 2); + assert_eq!(selection.attempts[1].captured_revision, 0); + assert!(matches!( + store + .export_decision_bundle( + &export_request(next, BundlePurpose::CurrentRead), + &AllowExport + ) + .unwrap(), + BundleExport::Redacted(_) + )); +} + +fn export_database() -> std::path::PathBuf { + static NEXT: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(0); + std::env::temp_dir().join(format!( + "kairotensor-decision-export-{}-{}.sqlite", + std::process::id(), + NEXT.fetch_add(1, std::sync::atomic::Ordering::Relaxed) + )) +} + +#[test] +fn export_cross_scope_closure_requires_exact_authority_and_all_consuming_admissions() { + use std::cell::Cell; + struct Check { + called: Cell, + root: i64, + origin: i64, + evidence: i64, + } + impl DiagnosticAuthorizer for Check { + fn authorize(&self, r: &DiagnosticReadRequest) -> Result { + self.called.set(true); + assert_eq!(r.actor, "a"); + assert_eq!(r.scope, "s"); + assert_eq!( + r.attempts + .iter() + .map(|a| (a.attempt, a.actor.as_str(), a.scope.as_str())) + .collect::>(), + [(self.root, "a", "s"), (self.origin, "foreign", "other")] + ); + for a in &r.attempts { + assert_eq!( + a.inputs, + [SelectedInput { + position: 0, + occurrence_id: self.evidence + }] + ); + } + Ok(true) + } + } + let path = export_database(); + let mut store = kairotensor::Store::open(&path).unwrap(); + let evidence = store + .observe_and_admit_many("a", "s", &[("shared", b"payload")]) + .unwrap()[0]; + store + .set_admitted("foreign", "other", evidence, true) + .unwrap(); + let (origin, _) = store + .begin_model_attempt("foreign", "other", "main", &[evidence], "contract") + .unwrap(); + store.resolve_model_attempt(origin, b"request").unwrap(); + store.settle(origin, b"response", 4).unwrap(); + let (root, _) = store + .begin_model_attempt("a", "s", "main", &[evidence], "contract") + .unwrap(); + // Construct a legal imported historical cross-scope reuse edge. The public + // model-reuse producer is narrower; the exporter must not assume that policy. + let db = rusqlite::Connection::open(&path).unwrap(); + db.execute( + "INSERT INTO model_reuses(attempt_id,origin_attempt) VALUES(?1,?2)", + rusqlite::params![root, origin], + ) + .unwrap(); + db.execute( + "INSERT INTO deliveries(attempt_id,disposition,output,usage) VALUES(?1,'accepted',?2,0)", + rusqlite::params![root, b"response".as_slice()], + ) + .unwrap(); + db.execute( + "UPDATE branches SET head=?1,revision=1 WHERE actor='a' AND scope='s'", + [root], + ) + .unwrap(); + let auth = Check { + called: Cell::new(false), + root, + origin, + evidence, + }; + assert!(matches!( + store + .export_decision_bundle(&export_request(root, BundlePurpose::CurrentRead), &auth) + .unwrap(), + BundleExport::Redacted(_) + )); + assert!(!auth.called.get()); + let b = complete( + store + .export_decision_bundle( + &export_request(root, BundlePurpose::DiagnosticReplay), + &auth, + ) + .unwrap(), + ); + assert!(auth.called.get()); + assert_eq!(b.evidence.len(), 1); + assert_eq!( + b.evidence[0].authorized_scopes, + [ + AuthorizedScope { + actor: "a".into(), + scope: "s".into() + }, + AuthorizedScope { + actor: "foreign".into(), + scope: "other".into() + } + ] + ); + store + .set_admitted("foreign", "other", evidence, false) + .unwrap(); + assert!(matches!( + store + .export_decision_bundle( + &export_request(root, BundlePurpose::DiagnosticReplay), + &auth + ) + .unwrap(), + BundleExport::Redacted(_) + )); +} + +#[allow(dead_code)] +#[path = "../examples/incident_store_bridge/adapter.rs"] +mod export_bridge; +#[test] +fn export_bridge_uses_actual_fixture_manifest_retry_and_immutable_bytes() { + use export_bridge::*; + let path = export_database(); + let mut f = Fixture::new(kairotensor::Store::open(&path).unwrap()).unwrap(); + let initial = prepare_incident_attempt(&mut f.store, f.recovery, &f.ids, &f.contract).unwrap(); + f.store + .reserve_recovery_attempt( + initial.attempt, + &initial.request, + initial.preparation.input_tokens, + initial.preparation.output_cap, + ) + .unwrap(); + let run = f + .store + .run_timed_recovery( + initial.attempt, + &initial.preparation, + || tick(200), + |request| Ok::<_, String>(initial.measure(request, b"bad")), + ) + .unwrap(); + assert!(matches!( + run, + kairotensor::RecoveryRun::Settled { + disposition: kairotensor::Disposition::InvalidOutput, + .. + } + )); + let retry = prepare_incident_attempt(&mut f.store, f.recovery, &f.ids, &f.contract).unwrap(); + // A planned retry has no quote/request in the ledger until reservation. + let request = BundleRequest { + actor: ACTOR.into(), + scope: SCOPE.into(), + attempt: retry.attempt, + purpose: BundlePurpose::DiagnosticReplay, + }; + let planned = complete( + f.store + .export_decision_bundle(&request, &AllowExport) + .unwrap(), + ); + assert_eq!(planned.attempts.len(), 2); + assert_eq!( + planned.attempts[0] + .recovery + .as_ref() + .unwrap() + .predecessor_id, + Some(initial.attempt) + ); + assert_eq!( + planned.attempts[0] + .immutable_manifest + .as_ref() + .unwrap() + .sha256, + initial.manifest_sha256 + ); + f.store + .reserve_recovery_attempt( + retry.attempt, + &retry.request, + retry.preparation.input_tokens, + retry.preparation.output_cap, + ) + .unwrap(); + let b = complete( + f.store + .export_decision_bundle(&request, &AllowExport) + .unwrap(), + ); + assert_eq!( + b.attempts[0] + .recovery + .as_ref() + .unwrap() + .request + .as_ref() + .unwrap() + .decode() + .unwrap(), + retry.request + ); + assert!(b.attempts[0].dispatch_request.is_none()); + assert_eq!( + b.attempts[1] + .dispatch_request + .as_ref() + .unwrap() + .decode() + .unwrap(), + initial.request + ); + assert_eq!(b.attempts[0].accounting.unit, AccountingUnit::FixtureBytes); + let db = rusqlite::Connection::open(&path).unwrap(); + let raw: (String, String) = db + .query_row( + "SELECT configuration,runtime_json FROM recovery_timing_bindings WHERE recovery_id=?1", + [f.recovery], + |r| Ok((r.get(0)?, r.get(1)?)), + ) + .unwrap(); + for a in &b.attempts { + assert_eq!( + a.timing_configuration.as_ref().unwrap().decode().unwrap(), + raw.0.as_bytes() + ); + assert_eq!( + a.immutable_runtime.as_ref().unwrap().decode().unwrap(), + raw.1.as_bytes() + ); + assert_eq!( + a.immutable_manifest.as_ref().unwrap().sha256, + retry.manifest_sha256 + ); + assert_eq!(a.inputs[0].role, InputRole::Context); + } + assert_eq!( + b.attempts[0].validator.as_ref().unwrap().provenance, + ValidatorProvenance::DeclaredContract + ); + assert_eq!( + b.attempts[1].validator.as_ref().unwrap().provenance, + ValidatorProvenance::RecordedRejection + ); + let replay = validate_recorded_bundle(&b); + assert_eq!(replay.consistency, ReplayStatus::Matched); + assert_eq!(replay.reconstruction, ReplayStatus::Matched); + // Data mutation leaves the captured immutable fingerprint unchanged: never + // attach a recognized manifest/roles/fixture units from a near match. + db.execute( + "UPDATE evidence SET payload=?1 WHERE id=?2", + rusqlite::params![b"changed".as_slice(), f.ids[1]], + ) + .unwrap(); + let changed = complete( + f.store + .export_decision_bundle(&request, &AllowExport) + .unwrap(), + ); + for a in changed.attempts { + assert!(a.immutable_manifest.is_none()); + assert_eq!(a.accounting.unit, AccountingUnit::Unknown); + assert!(a.inputs.iter().all(|i| i.role == InputRole::Unknown)); + } +} + +#[test] +fn export_missing_retry_predecessor_redacts_instead_of_inventing_complete_history() { + let path = export_database(); + let mut store = kairotensor::Store::open(&path).unwrap(); + let budget = store.create_token_budget(100).unwrap(); + let run = store + .create_recovery("r", budget, "a", "s", "main") + .unwrap(); + let id = match store.begin_recovery_attempt(run, &[]).unwrap() { + kairotensor::RecoveryStart::New { attempt, .. } => attempt, + _ => panic!(), + }; + // Adversarial persisted state: ordinal 1 exists without its required phase 0. + rusqlite::Connection::open(&path) + .unwrap() + .execute( + "UPDATE recovery_attempts SET ordinal=1 WHERE attempt_id=?1", + [id], + ) + .unwrap(); + assert!(matches!( + store + .export_decision_bundle( + &export_request(id, BundlePurpose::DiagnosticReplay), + &AllowExport + ) + .unwrap(), + BundleExport::Redacted(_) + )); +} +#[test] +fn export_large_text_source_is_enveloped_as_bytes_not_json_text() { + let mut store = kairotensor::Store::in_memory().unwrap(); + let source = "s".repeat(3 * 1024 * 1024); + let e = store.observe(&source, b"input").unwrap(); + store.set_admitted("a", "s", e, true).unwrap(); + let id = store.begin_attempt("a", "s", "main", &[e]).unwrap(); + store.settle(id, b"output", 0).unwrap(); + let b = complete( + store + .export_decision_bundle( + &export_request(id, BundlePurpose::CurrentRead), + &DenyDiagnostics, + ) + .unwrap(), + ); + assert_eq!(b.evidence[0].source.decode().unwrap(), source.as_bytes()); +} +#[test] +fn export_missing_payload_dependency_cycle_and_closure_limit_redact() { + for scenario in 0..3 { + let path = export_database(); + let mut store = kairotensor::Store::open(&path).unwrap(); + let e = store + .observe_and_admit_many("a", "s", &[("secret", b"payload")]) + .unwrap()[0]; + let first = store.begin_attempt("a", "s", "main", &[e]).unwrap(); + store.settle(first, b"output", 0).unwrap(); + let (mut root, _) = store.begin_resume("a", "s", "main", &[]).unwrap(); + store.settle(root, b"output", 0).unwrap(); + let db = rusqlite::Connection::open(&path).unwrap(); + match scenario { + 0 => { + db.execute("DELETE FROM deliveries WHERE attempt_id=?1", [first]) + .unwrap(); + } + 1 => { + db.execute( + "UPDATE attempts SET parent=id,inherited_from=id WHERE id=?1", + [root], + ) + .unwrap(); + } + 2 => { + for _ in 0..127 { + root = store.begin_resume("a", "s", "main", &[]).unwrap().0; + store.settle(root, b"output", 0).unwrap(); + } + } + _ => unreachable!(), + } + assert!(matches!( + store + .export_decision_bundle( + &export_request(root, BundlePurpose::DiagnosticReplay), + &AllowExport + ) + .unwrap(), + BundleExport::Redacted(_) + )); + } +} +#[test] +fn export_aggregate_envelopes_are_bounded_and_sql_corruption_is_an_error() { + let mut store = kairotensor::Store::in_memory().unwrap(); + let payload = vec![0; 3 * 1024 * 1024]; + let mut ids = Vec::new(); + for _ in 0..3 { + let id = store.observe("source", &payload).unwrap(); + store.set_admitted("a", "s", id, true).unwrap(); + ids.push(id); + } + let id = store.begin_attempt("a", "s", "main", &ids).unwrap(); + assert!(matches!( + store + .export_decision_bundle( + &export_request(id, BundlePurpose::DiagnosticReplay), + &AllowExport + ) + .unwrap(), + BundleExport::Redacted(_) + )); + let path = export_database(); + let mut store = kairotensor::Store::open(&path).unwrap(); + let id = store.begin_attempt("a", "s", "main", &[]).unwrap(); + store.settle(id, b"output", 0).unwrap(); + rusqlite::Connection::open(&path) + .unwrap() + .execute( + "UPDATE deliveries SET output='CORRUPTION_CANARY' WHERE attempt_id=?1", + [id], + ) + .unwrap(); + assert!( + matches!( + store + .export_decision_bundle( + &export_request(id, BundlePurpose::DiagnosticReplay), + &DenyDiagnostics + ) + .unwrap(), + BundleExport::Redacted(_) + ), + "denial must not load corrupt payload bytes" + ); + let error = store + .export_decision_bundle( + &export_request(id, BundlePurpose::DiagnosticReplay), + &AllowExport, + ) + .unwrap_err(); + assert!(matches!(error, BundleError::Store(_))); + assert!(!error.to_string().contains("CANARY")); +} +#[test] +fn export_report_binding_replays_exactly_without_invented_units() { + let mut store = kairotensor::Store::in_memory().unwrap(); + let e = store + .observe_and_admit_many( + "a", + "s", + &[("report", br#"{"release":"r1","passed":2,"total":2}"#)], + ) + .unwrap()[0]; + let id = store.begin_report_verdict("a", "s", "main", &[e]).unwrap(); + store.commit_report_verdict(id).unwrap(); + let b = complete( + store + .export_decision_bundle( + &export_request(id, BundlePurpose::CurrentRead), + &DenyDiagnostics, + ) + .unwrap(), + ); + let report = validate_recorded_bundle(&b); + assert_eq!(report.reconstruction, ReplayStatus::Matched); + assert_eq!(report.validation, ReplayStatus::Matched); + assert_eq!( + b.attempts[0].accounting.unit, + AccountingUnit::LogicalSupport + ); + assert_eq!( + b.attempts[0].validator.as_ref().unwrap().provenance, + ValidatorProvenance::DeterministicBinding + ); +} +#[test] +fn export_retains_noncanonical_timing_and_runtime_bytes() { + use export_bridge::*; + let path = export_database(); + let mut f = Fixture::new(kairotensor::Store::open(&path).unwrap()).unwrap(); + let second = f + .store + .create_recovery("raw timing", f.budget, ACTOR, SCOPE, "other") + .unwrap(); + let db = rusqlite::Connection::open(&path).unwrap(); + // Explicit adversarial persisted fixture: valid JSON with retained whitespace, + // inserted before attempts exist, respecting the immutable timing triggers. + db.execute("INSERT INTO recovery_timing_bindings SELECT ?1, ' ' || configuration || char(10),clock_domain,start_ms,deadline_ms, ' ' || runtime_json || char(10),calibration_id,max_duration_ms,safety_margin_ms,max_input_tokens,output_cap,measured_at_ms,expires_at_ms,last_observed_ms FROM recovery_timing_bindings WHERE recovery_id=?2",rusqlite::params![second,f.recovery]).unwrap(); + let id = match f.store.begin_recovery_attempt(second, &f.ids).unwrap() { + kairotensor::RecoveryStart::New { attempt, .. } => attempt, + _ => panic!(), + }; + let request = BundleRequest { + actor: ACTOR.into(), + scope: SCOPE.into(), + attempt: id, + purpose: BundlePurpose::DiagnosticReplay, + }; + let b = complete( + f.store + .export_decision_bundle(&request, &AllowExport) + .unwrap(), + ); + let raw: (String, String) = db + .query_row( + "SELECT configuration,runtime_json FROM recovery_timing_bindings WHERE recovery_id=?1", + [second], + |r| Ok((r.get(0)?, r.get(1)?)), + ) + .unwrap(); + assert_eq!( + b.attempts[0] + .timing_configuration + .as_ref() + .unwrap() + .decode() + .unwrap(), + raw.0.as_bytes() + ); + assert_eq!( + b.attempts[0] + .immutable_runtime + .as_ref() + .unwrap() + .decode() + .unwrap(), + raw.1.as_bytes() + ); + assert!(raw.0.starts_with(" ")); + assert!(raw.1.ends_with('\n')); + assert!(b.attempts[0].immutable_manifest.is_some()); +} + +#[test] +fn export_manifest_expansion_is_bounded_before_replicating_case_metadata() { + use export_bridge::*; + let mut f = Fixture::new(kairotensor::Store::in_memory().unwrap()).unwrap(); + let (mut context, _) = pinned_case("cal-01").unwrap(); + context.case_id = "c".repeat(250_000); + context.evidence_ids = (0..20).map(|n| format!("e{n}")).collect(); + let raw = context.to_json().unwrap(); + let mut ids = f + .store + .observe_and_admit_many(ACTOR, SCOPE, &[("context", raw.as_slice())]) + .unwrap(); + for _ in 0..20 { + ids.extend( + f.store + .observe_and_admit_many(ACTOR, SCOPE, &[("report", b"small")]) + .unwrap(), + ); + } + let id = match f.store.begin_recovery_attempt(f.recovery, &ids).unwrap() { + kairotensor::RecoveryStart::New { attempt, .. } => attempt, + _ => panic!(), + }; + let request = BundleRequest { + actor: ACTOR.into(), + scope: SCOPE.into(), + attempt: id, + purpose: BundlePurpose::DiagnosticReplay, + }; + assert!(matches!(f.store.export_decision_bundle(&request,&AllowExport).unwrap(),BundleExport::Redacted(_)),"a bounded context must not expand into an unbounded candidate manifest before the fingerprint comparison"); +} From b92174349144055ea66f4535dfe7d0b8c5d63b13 Mon Sep 17 00:00:00 2001 From: Alejandro Castellanos <5405628+k2jac9@users.noreply.github.com> Date: Sun, 15 Nov 2026 16:59:59 -0500 Subject: [PATCH 17/51] test: exercise decision replay across process boundaries --- .github/workflows/check.yml | 11 +- docs/decision-replay.md | 93 +++++++++++++++ examples/decision_replay.rs | 91 +++++++++++++++ scripts/audit_recovery_bundle.py | 14 ++- scripts/decision_bundle_v1.py | 159 ++++++++++++++++++++++++++ scripts/test_decision_bundle_v1.py | 178 +++++++++++++++++++++++++++++ 6 files changed, 543 insertions(+), 3 deletions(-) create mode 100644 docs/decision-replay.md create mode 100644 examples/decision_replay.rs create mode 100644 scripts/decision_bundle_v1.py create mode 100644 scripts/test_decision_bundle_v1.py diff --git a/.github/workflows/check.yml b/.github/workflows/check.yml index 619937c..a6a4a10 100644 --- a/.github/workflows/check.yml +++ b/.github/workflows/check.yml @@ -32,8 +32,15 @@ jobs: run: | python3 -m unittest discover -s reference -p test_memory.py python3 reference/belief_oracle.py - cargo build --locked --example recovery_policy_eval - KAIROTENSOR_RECOVERY_RUNNER="$PWD/target/debug/examples/recovery_policy_eval" python3 -m unittest discover -s scripts -p 'test_*.py' + cargo build --locked --offline --example recovery_policy_eval --example decision_replay + KAIROTENSOR_RECOVERY_RUNNER="$PWD/target/debug/examples/recovery_policy_eval" KAIROTENSOR_DECISION_REPLAY="$PWD/target/debug/examples/decision_replay" python3 -m unittest discover -s scripts -p 'test_*.py' + - name: Export and replay a decision in separate offline processes + run: | + set -euo pipefail + cargo run --locked --offline --example decision_replay -- export-fixture "$RUNNER_TEMP/decision-v1.json" + cargo run --locked --offline --example decision_replay -- replay "$RUNNER_TEMP/decision-v1.json" + python3 scripts/audit_recovery_bundle.py --decision-v1 "$RUNNER_TEMP/decision-v1.json" + - name: Verify archived output-cap measurements run: python3 scripts/summarize_token_caps.py docs/evaluations/token-caps-*-2026-11-13.json diff --git a/docs/decision-replay.md b/docs/decision-replay.md new file mode 100644 index 0000000..cff5fd5 --- /dev/null +++ b/docs/decision-replay.md @@ -0,0 +1,93 @@ +# Separate-process decision replay + +`decision_replay` exports one fixed, offline incident fixture through the actual +Store export API. It uses the existing incident adapter, canned response bytes, +synthetic timing and fixture-byte accounting. This exercises request capture, +manifest/runtime binding and format validation; it makes no model-inference, +semantic-correctness or measured-calibration claim. + +```sh +cargo run --locked --offline --example decision_replay -- export-fixture /path/new-bundle.json +cargo run --locked --offline --example decision_replay -- replay /path/new-bundle.json +python3 scripts/audit_recovery_bundle.py --decision-v1 /path/new-bundle.json +``` + +Use the workstation storage/Cargo wrappers and admitted output profile for local +builds. The two Rust commands run separate processes. Both print the recorded +replay report, so their source identities and results can be compared. Fixture +export uses `CurrentRead` and `DenyDiagnostics`, accepts no owner database path, +and creates the output exclusively. It writes the complete canonical payload +using `DecisionBundleV1::to_json()`, not the tagged `BundleExport` wrapper. +Existing files and symbolic links refuse. A failed export can leave its newly +created empty/partial file as a failure artifact; it is never silently replaced. +Only the literal `export-fixture PATH` and `replay PATH` modes exist. + +The replay branch opens its input read-only, checks metadata and reads at most +16 MiB plus one sentinel byte before parsing. The pure codec additionally bounds +nesting, attempt count and decoded envelopes. Replay does not initialize a Store, +run migrations, create an adapter, or invoke any backend callback. Its only +output is the JSON report on stdout. Malformed, oversized and redacted inputs +refuse without modifying the input. A valid bundle can successfully produce an +`incomplete`, `mismatch` or `not_reproducible` report; process success means the +report was produced, not that every contract matched. Unknown validators remain +unavailable. Historical acceptance stays recorded metadata and historical timing +always stays unavailable. See [the exact v1 schema](decision-bundle-v1.md). + +The installed `strace` can observe the compiled executable directly: + +```sh +strace -f -e trace=process,network -o replay.trace /absolute/path/to/decision_replay replay /path/new-bundle.json +``` + +The C3 retained trace observes the exercised compiled replay path, including +process and network syscalls. It does not trace Cargo compilation. Absence of +child-process/network calls in one run is bounded observational evidence; it is +not a universal proof for other paths or a detector of an in-memory Store. +The source boundary supplies the separate evidence for Store initialization. + +## Python reader and E1 helper boundary + +`scripts/decision_bundle_v1.py` uses only the Python standard library. It performs +bounded reads, strict UTF-8 JSON parsing, duplicate-key and depth checks, the v1 +top-level shape check, lowercase hex/length/SHA-256 checks for byte envelopes, +and the documented independent canonical outer hash. It deliberately does not +implement the complete Rust wire/ledger validator, contract renderers, validator +execution, authenticated access checks, historical timing or policy/cost scoring. +An integrity success is not a statement of ledger validity or actual availability. + +Public helpers are `read_bundle(path)` and `parse_bundle(raw_bytes)` for an +integrity-checked recorded dictionary, `decode_envelope(envelope)` for exact +binary bytes, `canonical_bytes(value)` for the documented sorted UTF-8 encoding, +`inspect_bundle(bundle)` for explicit integrity-only facts, and `audit_file(path)` +for those facts plus the separate transport-file SHA-256. `parse_bundle` accepts +only the complete payload; redacted and complete wrappers are not auto-unwrapped. +Unknown schema versions refuse; unknown validator names are retained as declared +names and are never treated as executable validators. Arbitrary binary payloads +are decoded as bytes, without text normalization. Public encoding/inspection +helpers accept in-memory values from trusted bounded callers; file consumers +must use `read_bundle`/`audit_file` to enforce the transport bound before allocation. + +For E1, `attempts`, `inputs`, their ordered `role` strings, `evidence` and envelopes +remain available verbatim. E1 must establish role and contract applicability, +including exact incident manifest/runtime bindings, before choosing an ablation. +A role string or historical Accepted receipt confers no access or applicability. +Python's hash check must not be presented as Rust validation. To obtain supported +recorded reconstruction/validation, invoke the verified Rust replay executable +and retain its source digest and availability report alongside the input. + +The legacy auditor still accepts only an explicit directory in its original +mode. The separate `--decision-v1 FILE` branch never opens SQLite. Combining +both modes refuses; a file is never silently auto-detected as a new mode. + +Subprocess tests require explicitly bound precompiled executables and never +invoke Cargo or install dependencies: + +```sh +KAIROTENSOR_DECISION_REPLAY=/absolute/path/to/decision_replay \ +KAIROTENSOR_RECOVERY_RUNNER=/absolute/path/to/recovery_policy_eval \ +python3 -m unittest discover -s scripts -p 'test_*.py' +``` + +CI builds both before discovery and runs offline export/replay commands while +retaining the prior reference, archived-data, recovery and incident gates. +Configured CI is not evidence of hosted execution or runtime-matrix acceptance. diff --git a/examples/decision_replay.rs b/examples/decision_replay.rs new file mode 100644 index 0000000..21546b8 --- /dev/null +++ b/examples/decision_replay.rs @@ -0,0 +1,91 @@ +//! Separate-process fixture export and capability-free recorded replay. +#[allow(dead_code)] +#[path = "incident_store_bridge/adapter.rs"] +mod adapter; + +use kairotensor::decision_bundle::{ + parse_decision_bundle, validate_recorded_bundle, BundleExport, BundlePurpose, BundleRequest, + DenyDiagnostics, MAX_BUNDLE_BYTES, +}; +use std::io::{Read, Write}; +use std::path::Path; + +type Result = std::result::Result>; + +// This branch holds only an input file and the pure codec/replay API. In +// particular it never constructs a Store, adapter, runtime or backend callback. +fn replay(path: &Path) -> Result<()> { + let file = std::fs::File::open(path)?; + if file.metadata()?.len() > MAX_BUNDLE_BYTES as u64 { + return Err("decision bundle limit exceeded".into()); + } + let mut bytes = Vec::new(); + file.take(MAX_BUNDLE_BYTES as u64 + 1) + .read_to_end(&mut bytes)?; + if bytes.len() > MAX_BUNDLE_BYTES { + return Err("decision bundle limit exceeded".into()); + } + let bundle = parse_decision_bundle(&bytes)?; + println!("{}", serde_json::to_string(&validate_recorded_bundle(&bundle))?); + Ok(()) +} + +fn export_fixture(path: &Path) -> Result<()> { + // Exclusive creation also refuses existing files and symbolic links. + let mut file = std::fs::OpenOptions::new() + .write(true) + .create_new(true) + .open(path)?; + let mut fixture = adapter::Fixture::new(kairotensor::Store::in_memory()?)?; + let prepared = adapter::prepare_incident_attempt( + &mut fixture.store, + fixture.recovery, + &fixture.ids, + &fixture.contract, + )?; + if !fixture.store.reserve_recovery_attempt( + prepared.attempt, + &prepared.request, + prepared.preparation.input_tokens, + prepared.preparation.output_cap, + )? { + return Err("fixture reservation refused".into()); + } + let outcome = fixture.store.run_timed_recovery( + prepared.attempt, + &prepared.preparation, + || adapter::tick(200), + |request| { + // Offline, format-valid canned bytes; no model or semantic claim. + Ok::<_, String>(prepared.measure(request, br#"{"decision":"abstain","owner":null,"reason":"undetermined_cause","evidence_ids":["e01"]}"#)) + }, + )?; + if !matches!(outcome, kairotensor::RecoveryRun::Settled { disposition: kairotensor::Disposition::Accepted, .. }) { + return Err("fixture did not settle accepted".into()); + } + let export = fixture.store.export_decision_bundle( + &BundleRequest { + actor: adapter::ACTOR.into(), + scope: adapter::SCOPE.into(), + attempt: prepared.attempt, + purpose: BundlePurpose::CurrentRead, + }, + &DenyDiagnostics, + )?; + let BundleExport::Complete(bundle) = export else { + return Err("fixture unavailable (redacted)".into()); + }; + file.write_all(&bundle.to_json()?)?; + file.sync_all()?; + println!("{}", serde_json::to_string(&validate_recorded_bundle(&bundle))?); + Ok(()) +} + +fn main() -> Result<()> { + let args: Vec<_> = std::env::args_os().skip(1).collect(); + match args.as_slice() { + [mode, path] if mode == "export-fixture" => export_fixture(Path::new(path)), + [mode, path] if mode == "replay" => replay(Path::new(path)), + _ => Err("usage: decision_replay export-fixture PATH | replay PATH".into()), + } +} diff --git a/scripts/audit_recovery_bundle.py b/scripts/audit_recovery_bundle.py index 57326ba..3584748 100644 --- a/scripts/audit_recovery_bundle.py +++ b/scripts/audit_recovery_bundle.py @@ -274,8 +274,20 @@ def audit_bundle(directory): def main(): parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument('directory', type=Path) + parser.add_argument('directory', type=Path, nargs='?') + parser.add_argument('--decision-v1', type=Path, metavar='FILE') args = parser.parse_args() + if args.decision_v1 is not None: + if args.directory is not None: + parser.error('--decision-v1 FILE cannot be combined with a directory') + from decision_bundle_v1 import audit_file + try: + print(json.dumps(audit_file(args.decision_v1), indent=2, allow_nan=False)) + except (OSError, ValueError) as error: + parser.exit(2, f'decision v1 audit failed: {error}\n') + return + if args.directory is None: + parser.error('directory is required unless --decision-v1 FILE is explicit') try: result = audit_bundle(args.directory) print(json.dumps(result, indent=2, allow_nan=False)) diff --git a/scripts/decision_bundle_v1.py b/scripts/decision_bundle_v1.py new file mode 100644 index 0000000..9b2ff7c --- /dev/null +++ b/scripts/decision_bundle_v1.py @@ -0,0 +1,159 @@ +"""Decision v1 byte integrity reader, not a Rust ledger/contract/access validator. + +Public helpers return recorded facts only. Callers deciding contract applicability +must validate it independently, including roles, manifest and runtime bindings. +""" +import hashlib +import json +from pathlib import Path +import re + +MAX_BUNDLE_BYTES = 16 * 1024 * 1024 +MAX_PAYLOAD_BYTES = 4 * 1024 * 1024 +MAX_JSON_DEPTH = 64 +DIGEST = re.compile(r'[0-9a-f]{64}\Z') +HEX = re.compile(r'[0-9a-f]*\Z') +TOP_FIELDS = {'schema_version', 'root_attempt', 'actor', 'scope', 'export_purpose', + 'snapshot', 'attempts', 'evidence', 'integrity'} + + +def require(condition, reason): + if not condition: + raise ValueError(reason) + + +def canonical_bytes(value): + """Independent canonical v1 encoding. Not an authorization operation.""" + return json.dumps(value, ensure_ascii=False, sort_keys=True, + separators=(',', ':'), allow_nan=False).encode('utf-8') + + +def unique_object(pairs): + result = {} + for key, value in pairs: + require(key not in result, 'duplicate JSON field') + result[key] = value + return result + + +def reject_number(_): + raise ValueError('non-integer JSON number') + + +def bounded_depth(raw): + depth, quoted, escaped = 0, False, False + for byte in raw: + if quoted: + if escaped: + escaped = False + elif byte == 92: + escaped = True + elif byte == 34: + quoted = False + elif byte == 34: + quoted = True + elif byte in (91, 123): + depth += 1 + require(depth <= MAX_JSON_DEPTH, 'JSON depth limit exceeded') + elif byte in (93, 125): + depth -= 1 + + +def decode_envelope(envelope): + """Check bounds, exact lowercase hex, byte length and digest; preserve bytes.""" + require(type(envelope) is dict and set(envelope) == {'hex', 'byte_len', 'sha256'}, 'invalid byte envelope') + length, encoded, digest = envelope['byte_len'], envelope['hex'], envelope['sha256'] + require(type(length) is int and 0 <= length <= MAX_PAYLOAD_BYTES, 'byte length limit or type') + require(type(encoded) is str and len(encoded) <= 2 * MAX_PAYLOAD_BYTES + and len(encoded) == 2 * length and HEX.fullmatch(encoded), 'invalid lowercase hex') + require(type(digest) is str and DIGEST.fullmatch(digest), 'invalid byte digest') + raw = bytes.fromhex(encoded) + require(hashlib.sha256(raw).hexdigest() == digest, 'byte digest mismatch') + return raw + + +def check_envelopes(value): + # Every byte envelope in v1 has these exact three keys. Partial envelopes + # also refuse; other record/ledger semantics remain the Rust validator's job. + if type(value) is dict: + if ('hex' in value or 'byte_len' in value + or ('sha256' in value and 'version' not in value)): + decode_envelope(value) + else: + for child in value.values(): + check_envelopes(child) + elif type(value) is list: + for child in value: + check_envelopes(child) + + +def parse_bundle(raw): + """Read bounded complete canonical-payload JSON; refuse export wrappers. + + Checks duplicate keys, depth, top-level v1 shape, envelope integrity and the + outer canonical digest. Does not implement Rust's full wire or ledger schema. + """ + require(type(raw) is bytes and len(raw) <= MAX_BUNDLE_BYTES, 'transport limit exceeded') + bounded_depth(raw) + try: + value = json.loads(raw.decode('utf-8'), object_pairs_hook=unique_object, + parse_float=reject_number, parse_constant=reject_number) + require(type(value) is dict, 'expected complete decision object') + if value.get('status') == 'redacted': + raise ValueError('redacted export unavailable; not replayable or scored') + require(set(value) == TOP_FIELDS and type(value['schema_version']) is int + and value['schema_version'] == 1, 'unsupported decision v1 shape or version') + require(type(value['attempts']) is list and len(value['attempts']) <= 128 + and all(type(a) is dict for a in value['attempts']) + and type(value['evidence']) is list, 'invalid record arrays') + integrity = value['integrity'] + require(type(integrity) is dict and set(integrity) == {'version', 'sha256'} + and integrity['version'] == 'decision-bundle-integrity/1' + and type(integrity['sha256']) is str and DIGEST.fullmatch(integrity['sha256']), 'invalid integrity manifest') + # Work on the bounded parsed object and restore the original digest even + # on failure; callers always receive the unmodified recorded object. + digest = integrity.pop('sha256') + try: + actual = hashlib.sha256(canonical_bytes(value)).hexdigest() + finally: + integrity['sha256'] = digest + require(actual == digest, 'outer integrity digest mismatch') + check_envelopes(value) + return value + except (UnicodeError, RecursionError, OverflowError) as error: + raise ValueError('invalid decision encoding') from error + + +def read_bundle(path): + """Allocate at most the transport bound plus one before JSON parsing.""" + with Path(path).open('rb') as source: + raw = source.read(MAX_BUNDLE_BYTES + 1) + return parse_bundle(raw) + + +def inspect_bundle(bundle): + """Summarize integrity-checked records without asserting applicability.""" + # Also enforce our checks for public callers passing a constructed object. + bundle = parse_bundle(canonical_bytes(bundle)) + attempts = bundle['attempts'] + validators = set() + for attempt in attempts: + binding = attempt.get('validator') + if type(binding) is dict and type(binding.get('version')) is str: + validators.add(binding['version']) + return dict(status='integrity_checked', schema_version=1, + source_bundle_sha256=hashlib.sha256(canonical_bytes(bundle)).hexdigest(), + attempt_records=len(attempts), evidence_records=len(bundle['evidence']), + declared_validators=sorted(validators), + rust_ledger_validation='not_performed', validator_execution='not_performed', + historical_timing='not_reproduced', current_access='not_authenticated', + limitations='Hashes establish internal byte consistency only; recorded identities, roles, contracts, availability and acceptance require independent validation. No policy or cost score is produced.') + + +def audit_file(path): + """Versioned audit CLI helper, including a distinct transport identity.""" + with Path(path).open('rb') as source: + raw = source.read(MAX_BUNDLE_BYTES + 1) + result = inspect_bundle(parse_bundle(raw)) + result['transport_file_sha256'] = hashlib.sha256(raw).hexdigest() + return result diff --git a/scripts/test_decision_bundle_v1.py b/scripts/test_decision_bundle_v1.py new file mode 100644 index 0000000..9961467 --- /dev/null +++ b/scripts/test_decision_bundle_v1.py @@ -0,0 +1,178 @@ +"""Bounded codec and real precompiled subprocess checks; no build/install helpers.""" +import copy +import hashlib +import importlib +import json +import os +from pathlib import Path +import subprocess +import sys +import tempfile +import unittest + +ROOT = Path(__file__).resolve().parents[1] +GOLDEN = (ROOT / 'docs/decision-bundle-v1.md').read_text().split('```json\n')[-1].split('\n```')[0].encode() + + +def canonical(value): + return json.dumps(value, sort_keys=True, ensure_ascii=False, separators=(',', ':')).encode() + + +def seal(value): + value = copy.deepcopy(value) + value['integrity'].pop('sha256', None) + value['integrity']['sha256'] = hashlib.sha256(canonical(value)).hexdigest() + return canonical(value) + + +class CodecTests(unittest.TestCase): + def reader(self): + self.assertTrue((ROOT / 'scripts/decision_bundle_v1.py').is_file(), 'versioned reader must exist') + return importlib.import_module('decision_bundle_v1') + + def test_independent_golden_and_transport_identity(self): + reader = self.reader() + self.assertEqual(len(GOLDEN), 825) + value = reader.parse_bundle(GOLDEN) + result = reader.inspect_bundle(value) + self.assertEqual(result['source_bundle_sha256'], '569b219c44295bb8e1a42358b1bbbd66c153ff379694dd55ef5decf229dbe471') + self.assertEqual(reader.parse_bundle(json.dumps(value).encode()), value) + self.assertEqual(result['rust_ledger_validation'], 'not_performed') + self.assertEqual(result['validator_execution'], 'not_performed') + self.assertEqual(result['historical_timing'], 'not_reproduced') + self.assertEqual(result['current_access'], 'not_authenticated') + + def test_binary_bytes_and_tamper(self): + reader = self.reader() + raw = b'\x00\xff\x80\r\n' + envelope = dict(hex=raw.hex(), byte_len=len(raw), sha256=hashlib.sha256(raw).hexdigest()) + self.assertEqual(reader.decode_envelope(envelope), raw) + for change in ({'hex': raw.hex().upper()}, {'byte_len': True}, {'sha256': '0' * 64}, + {'byte_len': 4194305}, {'hex': '00 '}, {'extra': 1}, {'version': 'bypass'}): + with self.subTest(change=change), self.assertRaises(ValueError): + reader.decode_envelope(envelope | change) + + def test_duplicate_depth_size_and_digest_refuse(self): + reader = self.reader() + for raw in (b'{"a":1,"a":2}', b'[' * 65 + b'0' + b']' * 65, + b' ' * (16 * 1024 * 1024 + 1), GOLDEN.replace(b'"actor":"\xc3\xa9"', b'"actor":"x"', 1)): + with self.subTest(size=len(raw)), self.assertRaises(ValueError): + reader.parse_bundle(raw) + + def test_redacted_and_unsupported_version_refuse(self): + reader = self.reader() + with self.assertRaisesRegex(ValueError, 'redacted'): + reader.parse_bundle(b'{"status":"redacted","bundle":{"requested_attempt":1,"reason":"unavailable"}}') + value = json.loads(GOLDEN) + value['schema_version'] = 2 + with self.assertRaises(ValueError): + reader.parse_bundle(seal(value)) + + def test_bounded_read_before_materialization(self): + reader = self.reader() + with tempfile.TemporaryDirectory() as folder: + path = Path(folder) / 'oversize' + with path.open('wb') as output: + output.truncate(16 * 1024 * 1024 + 1) + with self.assertRaises(ValueError): + reader.read_bundle(path) + + +class ProcessTests(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.runner = os.environ.get('KAIROTENSOR_DECISION_REPLAY', '') + if not cls.runner or not Path(cls.runner).is_file() or not os.access(cls.runner, os.X_OK): + raise AssertionError('KAIROTENSOR_DECISION_REPLAY must bind a verified precompiled executable; subprocess checks are required') + + def setUp(self): + self.tmp = tempfile.TemporaryDirectory() + self.addCleanup(self.tmp.cleanup) + self.path = Path(self.tmp.name) / 'bundle.json' + + def run_cli(self, *args): + return subprocess.run([self.runner, *map(str, args)], capture_output=True, timeout=30) + + def test_actual_incident_export_then_separate_replay_and_python(self): + exported = self.run_cli('export-fixture', self.path) + self.assertEqual(exported.returncode, 0, exported.stderr) + before = hashlib.sha256(self.path.read_bytes()).hexdigest() + value = json.loads(self.path.read_bytes()) + self.assertEqual(value['export_purpose'], 'current_read') + self.assertIsNotNone(value['attempts'][0]['immutable_manifest']) + self.assertEqual(value['attempts'][0]['inputs'][0]['role'], 'context') + report = self.run_cli('replay', self.path) + self.assertEqual(report.returncode, 0, report.stderr) + report = json.loads(report.stdout) + self.assertEqual(json.loads(exported.stdout), report) + self.assertEqual(report['consistency'], 'matched') + self.assertEqual(report['reconstruction'], 'matched') + self.assertEqual(report['validation'], 'matched') + self.assertEqual(report['recorded_disposition'], 'accepted') + self.assertIn('historical_timing_unavailable', report['missing_requirements']) + python = subprocess.run([sys.executable, str(ROOT / 'scripts/audit_recovery_bundle.py'), '--decision-v1', str(self.path)], capture_output=True, timeout=30) + self.assertEqual(python.returncode, 0, python.stderr) + facts = json.loads(python.stdout) + self.assertEqual(facts['source_bundle_sha256'], report['source_bundle_sha256']) + self.assertEqual(facts['validator_execution'], 'not_performed') + self.assertEqual(before, hashlib.sha256(self.path.read_bytes()).hexdigest()) + + def test_binary_preservation_and_unknown_validator(self): + value = json.loads(GOLDEN) + raw = b'\x00\xff\x80\r\n' + envelope = dict(hex=raw.hex(), byte_len=len(raw), sha256=hashlib.sha256(raw).hexdigest()) + value['evidence'] = [dict(occurrence_id=1, source=envelope, payload=envelope, + authorized_scopes=[dict(actor='é', scope='s')])] + value['attempts'][0]['candidates'] = [dict(position=0, evidence_id=1)] + value['attempts'][0]['inputs'] = [dict(position=0, evidence_id=1, role='unknown')] + value['attempts'][0]['validator'] = dict(version='unavailable/99', provenance='unknown') + value['attempts'][0]['delivery'] = dict(state='recorded', raw=envelope, + disposition='accepted', usage=0, rejection=None) + self.path.write_bytes(seal(value)) + before = self.path.read_bytes() + result = self.run_cli('replay', self.path) + self.assertEqual(result.returncode, 0, result.stderr) + report = json.loads(result.stdout) + self.assertEqual(report['validation'], 'not_reproducible') + self.assertIn('validator_unavailable', report['missing_requirements']) + reader = importlib.import_module('decision_bundle_v1') + decoded = reader.read_bundle(self.path) + self.assertEqual(reader.decode_envelope(decoded['evidence'][0]['payload']), raw) + self.assertEqual(reader.inspect_bundle(decoded)['declared_validators'], ['unavailable/99']) + self.assertEqual(before, self.path.read_bytes()) + + def test_exclusive_export_unknown_mode_and_redacted_refusal(self): + self.path.write_bytes(b'preserved') + self.assertNotEqual(self.run_cli('export-fixture', self.path).returncode, 0) + self.assertEqual(self.path.read_bytes(), b'preserved') + absent = self.path.with_name('absent') + for mode in ('live', 'generate', 'unknown'): + self.assertNotEqual(self.run_cli(mode, absent).returncode, 0) + self.assertFalse(absent.exists()) + self.path.write_bytes(b'{"status":"redacted","bundle":{"requested_attempt":1,"reason":"unavailable"}}') + before = self.path.read_bytes() + self.assertNotEqual(self.run_cli('replay', self.path).returncode, 0) + self.assertEqual(self.path.read_bytes(), before) + + def test_python_mode_is_explicit_and_refuses_redaction(self): + script = str(ROOT / 'scripts/audit_recovery_bundle.py') + self.path.write_bytes(GOLDEN) + for args in ([str(self.path)], ['--decision-v1', str(self.path), str(self.path.parent)]): + result = subprocess.run([sys.executable, script, *args], capture_output=True, timeout=30) + self.assertNotEqual(result.returncode, 0) + self.path.write_bytes(b'{"status":"redacted","bundle":{"requested_attempt":1,"reason":"unavailable"}}') + result = subprocess.run([sys.executable, script, '--decision-v1', str(self.path)], capture_output=True, timeout=30) + self.assertNotEqual(result.returncode, 0) + self.assertIn(b'redacted', result.stderr) + self.assertEqual(sorted(p.name for p in self.path.parent.iterdir()), ['bundle.json']) + + def test_rust_size_depth_and_corruption_refuse_without_mutation(self): + for raw in (b' ' * (16 * 1024 * 1024 + 1), b'[' * 65 + b'0' + b']' * 65, + GOLDEN.replace(b'"scope":"s"', b'"scope":"x"', 1)): + self.path.write_bytes(raw) + self.assertNotEqual(self.run_cli('replay', self.path).returncode, 0) + self.assertEqual(self.path.read_bytes(), raw) + + +if __name__ == '__main__': + unittest.main() From 04cb3b62109e55cd93ae23ebc26596fcdca154a3 Mon Sep 17 00:00:00 2001 From: Alejandro Castellanos <5405628+k2jac9@users.noreply.github.com> Date: Sun, 15 Nov 2026 17:38:26 -0500 Subject: [PATCH 18/51] feat: validate complete comparable policy evaluation records --- docs/policy-evaluation-v2.md | 223 ++++++ scripts/policy_evaluation_v2.py | 998 +++++++++++++++++++++++++++ scripts/recovery_policy_eval.py | 15 +- scripts/test_policy_evaluation_v2.py | 984 ++++++++++++++++++++++++++ scripts/test_recovery_policy_eval.py | 15 + src/lib.rs | 1 + src/policy_registry.rs | 104 +++ src/policy_registry/evaluation.rs | 641 +++++++++++++++++ src/policy_registry/types.rs | 243 +++++++ tests/policy_evaluation.rs | 313 +++++++++ 10 files changed, 3536 insertions(+), 1 deletion(-) create mode 100644 docs/policy-evaluation-v2.md create mode 100644 scripts/policy_evaluation_v2.py create mode 100644 scripts/test_policy_evaluation_v2.py create mode 100644 src/policy_registry.rs create mode 100644 src/policy_registry/evaluation.rs create mode 100644 src/policy_registry/types.rs create mode 100644 tests/policy_evaluation.rs diff --git a/docs/policy-evaluation-v2.md b/docs/policy-evaluation-v2.md new file mode 100644 index 0000000..5c53735 --- /dev/null +++ b/docs/policy-evaluation-v2.md @@ -0,0 +1,223 @@ +# Complete policy comparison attestations + +This module validates supplied records without opening a Store, authenticating an +actor, running a model, fetching a URL, or activating a policy. Management authority +comes from the trusted runtime. SHA-256 establishes byte identity and internal +consistency; it does not prove execution, honest normalization, or semantic truth. + +The Rust API is `policy_registry::{validate_policy_artifact, +validate_policy_candidate, parse_policy_evaluation, evaluate_policy_comparison, +evaluation_sha256, paired_baseline_matches}`. Public types live in +`src/policy_registry/types.rs`. The Python counterpart is +`scripts/policy_evaluation_v2.py`. Structural errors are `rusqlite::Error` / +`ValueError`; a valid but unsuitable comparison returns a typed refusal. No database +schema, policy admission, migration, or dependency changes are part of this format. + +## Encoding and bounds + +All objects have exactly the documented fields: unknown, duplicate and missing +keys refuse. Nullable fields must occur explicitly as JSON `null`; omission is an +error. Integers are unsigned 64-bit, except artifact schema version (u32, exactly +1), and byte-array elements (u8). Booleans cannot stand in for integers. Digests +are exactly 64 lowercase hexadecimal characters. Identifiers and purpose are +nonempty, non-whitespace strings. Unicode must contain valid scalar values. + +A byte payload is a JSON array of integers in 0..255, not text, base64, hex, or a +URL. `DigestedRecord` is exactly `{bytes, sha256}`; the digest covers those original +bytes, including whitespace in an embedded JSON document. Artifact bytes are +limited to 64 KiB. The complete serialized evaluation and manifest are each at +most 16 MiB; JSON nesting is at most 64 levels; each side and manifest contain at +most 10,000 scheduled cases. Caller-constructed values receive bounded checks +before building a serialized JSON tree. The frozen label payload is also limited +to 64 KiB. Every accounting addition is checked for u64 overflow, including +reservation quotes, row costs, and aggregate costs. + +The derived evaluation identity is SHA-256 of ASCII `policy-evaluation/2\n` +followed by compact JSON of the complete typed `PolicyEvaluation`, with recursively +sorted object keys, UTF-8 scalar strings, ordinary JSON escaping, and no spaces. +Arrays retain order. Floats are absent. Python uses `ensure_ascii=False` and +separators `(',', ':')`; Rust uses sorted `serde_json::Value` object maps. All +original payload bytes remain arrays; embedded documents are never normalized +before their original-byte digests are checked. Provenance occurs in both the +comparison manifest and evaluation identity. It does not alter SHA-256 over an +original artifact, request, evidence, response, or label payload. + +## Exact records + +The Rust definitions are the normative field/type listing. The following groups +cover every record and its relationships; names are wire names unless noted. + +- `PolicyArtifactV1`: `schema_version=1`, `policy_name`, `purpose`, + `compatibility_contract_sha256`. Policy names are only `always_stop`, + `affordable_retry`, `evidence_gated_retry`. No executable configuration exists. +- `PolicyCandidate`: `version_id`, `actor`, `scope`, `purpose`, `artifact` (bytes), + `sha256`. The purpose must equal the parsed artifact purpose and its digest must + match. Future Store registration supplies the evidence occurrence. +- `PolicyEvaluation`: `evaluation_id`, `actor`, `scope`, `purpose`, `baseline_id`, + `candidate_id`, `baseline_sha256`, `candidate_sha256`, `experiment_manifest` + (bytes), `experiment_sha256`, `evaluator_version`, `evaluator_sha256`, + `provenance`, `baseline_rows`, `candidate_rows`. Selected version IDs are distinct + and nonempty. Their artifact digests select entries in the shared universe. +- `ComparisonManifest`: `format="policy-comparison/1"`, `actor`, `scope`, `purpose`, + `provenance`, `accounting_unit`, `workload_sha256`, `prompt_sha256`, + `runtime_sha256`, `model_sha256`, `tokenizer_sha256`, `validator_sha256`, + `scorer_sha256`, `compatibility_contract_sha256`, `policies`, `cases`. +- Each frozen policy: `policy_name`, `artifact` (`DigestedRecord`). Names and + artifact digests are unique; parsed artifacts match name, purpose and contract. + The universe contains one to three supported policy choices, independently of + a particular evaluation pair or evaluation ID. +- Each case: `case_id`, `family_id`, `split_id`, `initial_evidence` + (`DigestedRecord`), `label` (`DigestedRecord`), `budget`, `deadline_ms`, + `timing_start_ms`, `timing_end_ms`, `policy_opportunities`. Case IDs are unique; + a family cannot cross split IDs. All offsets share that case's original time + origin; start <= deadline <= end. Every scheduled case stays in the denominator. +- Label bytes strictly encode `{answerable, admissible_support, + normalized_format:"opaque-bytes/1", scorer_sha256, expected_normalized}`. + `expected_normalized` is one exact byte array, frozen before observation. This + format does not infer JSON, booleans, numbers, Unicode normalization, or alternate + spellings from those bytes. The scorer must match the manifest scorer. +- Each `policy_opportunities` entry: `policy_name`, `opportunities`. Every frozen + policy has exactly one entry. `always_stop` has one initial opportunity; each + retry policy has the initial opportunity plus one retry. Initial opportunities + agree across policies except for their local opportunity ID. +- Each opportunity: `opportunity_id`, `request`, `evidence_manifest` (both + `DigestedRecord`), `admitted`, `consistent`, `input_bound`, `output_cap`, + `start_min_ms`, `start_max_ms`. IDs are unique within the path. Initial evidence + matches the case's exact record. The start window lies inside case timing bounds. +- Each row: `case_id`, `outcome`, `correct`, `eligible`, `answerable`, + `completed_ms`, `charged`, `reserved_bound`, `false_refusal`, `unsafe_acceptance`, + `deadline_miss`, `simulated`, `attempts`. Outcomes are `Accepted`, `Refused`, + `Failed`, `Unknown`; correctness, eligibility, charge and completion are nullable. + Both sides cover the same ordered manifest cases exactly once. +- Each observed attempt: `execution_id`, `opportunity_id`, `request_sha256`, + `evidence_manifest_sha256`, `execution`, `disposition`, `response`, + `measured_input`, `measured_output`, `retained_bound`, `start_ms`, `completed_ms`, + `validator_sha256`, `scorer_sha256`, `label_sha256`, `scoring`. `response` is a + nullable `DigestedRecord`. Measurements, completion and scoring are nullable. + Execution is `Executed`, `NotExecuted`, or `Unknown`; disposition is `Accepted`, + `InvalidOutput`, `Stale`, `Failed`, `Uncertain`, or `Refused`. +- A scoring assertion is exactly `{response_sha256, scorer_sha256, label_sha256, + normalized}`. `normalized` is a byte array, compared exactly to the frozen + expected bytes. The assertion binds the actual response record and frozen label + and scorer identities. Raw response -> normalized bytes is an **external + attestation**, not an implemented general scientific scorer. Missing scoring for + an observed response remains incomplete. + +Accounting units are `model_tokens` or `fixture_bytes`. `logical_support` and +`unknown` can be represented but make comparisons incomparable. Units, workload, +requests, labels, runtime, cap, budget and deadlines are frozen in one shared +manifest; the baseline and candidate cannot choose different manifests. A future +bootstrap and later paired evaluations must reuse those exact bytes. +`paired_baseline_matches` checks the exact baseline projection (authority scope, +policy version/digest, manifest bytes/digest, evaluator, provenance and ordered +rows); evaluation IDs and candidate fields are deliberately excluded. It is not +an activation or a substitute for validating either record. + +## Observed replay and unknowns + +Provenance is `observed_policy_replay` or `synthetic_registry_mechanics`. +Observed rows set `simulated=false`. References must match the selected named +policy's frozen ordered opportunity prefix. Only a settled `InvalidOutput` +unlocks the single retry; accepted, stale, failed, uncertain and unknown outcomes +cannot unlock it. Correctness labels never influence retry selection. + +Before execution, admitted evidence, full fresh `input_bound + output_cap`, +remaining budget, and start strictly before the original deadline are required. +The evidence-gated retry also requires frozen consistency. A failed gate requires +an explicit terminal `NotExecuted` refusal with zero usage, no response/scoring, +no hold, and completion equal to its start. An unlocked but missing retry reference +is incomplete, including when the producer must explicitly attest nonexecution. +Attempt starts must respect the frozen window and prior completion. Completion +must be between start and the case timing end. Completion **equal to the deadline +is on time**. + +Settled execution has bounded measured input/output, zero retained hold and a +non-uncertain disposition. Unknown usage retains the full quote, leaves row charge +unknown, and is never eligible. An infrastructure failure without a response is a +failed result, not a semantic success; missing response/scoring for other settled +response outcomes is incomplete. Stale dispositions are ineligible. Invalid-output +exhaustion and explicit refusal produce a refused row. Row charge is the checked +sum of measured input/output, completion is the terminal completion, correctness +comes from the scoring assertion, and eligibility comes from frozen admission and +support (and stale disposition). All supplied redundant row values must agree. + +An execution ID can recur across policy paths only for the same case and identical +observations, excluding the path-local opportunity ID. It cannot be counted for +another scheduled case. Distinct execution IDs can refer to equal request bytes: +request equality is not proof of identical independent realizations. + +Synthetic rows have `simulated=true`, no attempt references, and explicitly +simulated counters. Their eligibility and answerability still match frozen support +and labels. They can qualify only for `fixture/recovery-policy`. The positive +fixture tests registry selection mechanics; it does **not** describe feasible +always-stop/retry measurements or demonstrate superiority of a real policy. + +## Selection and refusals + +Success means accepted, correct, eligible and completed at or before the original +deadline. False refusal means refused on a frozen answerable case with admissible +support; a row-local ineligible flag cannot hide it. Unsafe acceptance means an +accepted incorrect or ineligible result. Deadline miss means completion after the +original deadline. Charge plus retained bound is the cost. Known flags are checked +even on incomplete rows; unknown rows cannot hide fabricated later rows. + +Unknown outcomes, correctness, eligibility, usage or completion and missing +references/cases prevent eligibility (`Incomplete`). Identity/unit/reference +mismatches and unknown evaluator contracts return `Incomparable`. Structural +malformation, impossible counters, timing, budget, flags and overflow are errors. +After completeness, refusals are checked in this order: `NoImprovement`, +`IncreasedFalseRefusal`, `IncreasedUnsafeAcceptance`, `IncreasedCost`, +`IncreasedDeadlineMisses`. Eligible requires strictly more successes and all four +nonincrease conditions. Rust returns `SelectionDecision::Eligible { +validated_counters }` or `SelectionDecision::Refused(SelectionRefusal)`; Python +returns `{decision:"Eligible", validated_counters:{baseline,candidate}}` or +`{decision:"Refused", reason:}`. Each side's counters are `cases`, +`success`, `false_refusal`, `unsafe_acceptance`, `cost`, `deadline_misses`. + +The evaluator version is exactly `policy-selection/1`. Its contract bytes are the +following ASCII line sequence, with LF after each line (including the last): + +```text +policy-selection/1 +complete-paired-frozen-attestations;success=strict-increase;false-refusal,unsafe-acceptance,charged-plus-reserved,deadline-miss=nonincrease;synthetic=fixture/recovery-policy-only +``` + +Its SHA-256 is +`df97ae41b2648bef15a11bf1950883597b057df1940fe43347d485fedf8590b3`. +An arbitrary caller-supplied digest is not accepted as that evaluator identity. + +## Archived input conversion and golden records + +The existing recovery script's default output and `frozen-v1` semantics are +unchanged, including archived input `schema_version=2` (nullable unused retries). +That input schema is unrelated to this complete evaluation format. Explicit use: + +```sh +python3 scripts/recovery_policy_eval.py --fixture --convert-v2 +python3 scripts/recovery_policy_eval.py TRACE --convert-v2 --comparison-v2 COMPLETE.json +``` + +Conversion without a matching complete supplied attestation returns +`policy-evaluation-conversion/2`, `status:"Incomplete"`, a source digest and a +reason; it never fills unknown facts from protocol parse success. With supplied +facts, `workload_sha256` must bind compact sorted-key UTF-8 JSON of the archived +input; cases/families/splits, token unit, exact requests, quotes, gates, deadlines, +timings and old measured row totals must match. Evidence/labels remain supplied +attestations. Missing legacy retry contexts or uncertain legacy usage stay +incomplete. A complete conversion returns the supplied evaluation and its selection +result, which may refuse. The existing positive-retry corpus correctly returns +`IncreasedCost`; no worse baseline or changed selection rule is used. + +The inline `fixture()` and `golden_cases()` in +`scripts/test_policy_evaluation_v2.py` define the exact shared test bytes; Rust +loads these same bytes and checks independent literal expected outcomes, including +positive synthetic and negative malformed/incomplete/incomparable/gate examples. +The positive canonical evaluation is 22,057 bytes; its manifest is 6,477 bytes. +Pinned golden identities: + +| Payload | SHA-256 | +| --- | --- | +| Synthetic evaluation (domain prefix included) | `2b0ed900f956cd8fc2dd215e114a665d4799187dd6610209807fe557a1311a88` | +| Exact manifest bytes | `1373dd5ff2bc8e1a06840f8feb31329b1a6666d434712db2a6fe4a3ef537c265` | +| Baseline artifact bytes | `b8fd122e67c0094b1f7bc1fc91884387249299a603c3e8c9b3e3c56742d676bc` | +| Candidate artifact bytes | `b98a8a7d58c2592bee588fb26884da853832b44688e33ca0ede5648d57904ce5` | diff --git a/scripts/policy_evaluation_v2.py b/scripts/policy_evaluation_v2.py new file mode 100644 index 0000000..c7a40ac --- /dev/null +++ b/scripts/policy_evaluation_v2.py @@ -0,0 +1,998 @@ +"""Bounded, offline policy comparison attestations; never execution authentication.""" + +import hashlib +import json + +MAX_BYTES = 16 * 1024 * 1024 +MAX_CASES = 10_000 +MAX_U64 = (1 << 64) - 1 +POLICIES = ("always_stop", "affordable_retry", "evidence_gated_retry") + + +def require(ok, message): + if not ok: + raise ValueError(message) + + +def unique(pairs): + out = {} + for key, value in pairs: + require(key not in out, "duplicate JSON key") + out[key] = value + return out + + +def loads(raw, limit=MAX_BYTES): + require(type(raw) is bytes and len(raw) <= limit, "policy record size limit") + depth = 0 + quoted = False + escaped = False + for b in raw: + if quoted: + if escaped: + escaped = False + elif b == 92: + escaped = True + elif b == 34: + quoted = False + elif b == 34: + quoted = True + elif b in (91, 123): + depth += 1 + require(depth <= 64, "policy JSON depth limit") + elif b in (93, 125): + depth -= 1 + try: + value = json.loads( + raw.decode("utf-8"), + object_pairs_hook=unique, + parse_constant=lambda _: require(False, "nonfinite JSON"), + ) + bounded_tree(value) + return value + except (UnicodeError, RecursionError, json.JSONDecodeError) as error: + raise ValueError("malformed policy record") from error + + +def bounded_tree(value): + # Count canonical bytes without cloning containers or encoding large strings. + size = 0 + + def visit(item, depth): + nonlocal size + require(depth <= 64, "policy JSON depth limit") + if type(item) is str: + require(len(item) <= MAX_BYTES, "policy record size limit") + size += 2 + for ch in item: + n = ord(ch) + require(not 0xD800 <= n <= 0xDFFF, "invalid Unicode scalar") + if ch in ('"', "\\", "\b", "\f", "\n", "\r", "\t"): + size += 2 + elif n < 32: + size += 6 + else: + size += 1 if n < 128 else 2 if n < 2048 else 3 if n < 65536 else 4 + require(size <= MAX_BYTES, "policy record size limit") + elif type(item) is dict: + require(len(item) <= MAX_BYTES // 2, "policy record size limit") + size += 1 + 2 * len(item) if item else 2 + for k, v in item.items(): + visit(k, depth + 1) + visit(v, depth + 1) + elif type(item) is list: + require(len(item) <= MAX_BYTES // 2, "policy record size limit") + size += 1 + len(item) if item else 2 + for v in item: + visit(v, depth + 1) + elif item is None: + size += 4 + elif type(item) is bool: + size += 4 if item else 5 + elif type(item) is int: + size += len(str(item)) + else: + require(False, "non-integer JSON scalar") + require(size <= MAX_BYTES, "policy record size limit") + + visit(value, 0) + + +def canonical(value): + bounded_tree(value) + try: + # Stream the size check before materializing the encoded representation. + encoder = json.JSONEncoder( + sort_keys=True, separators=(",", ":"), ensure_ascii=False, allow_nan=False + ) + size = 0 + for part in encoder.iterencode(value): + size += len(part.encode("utf-8")) + require(size <= MAX_BYTES, "policy record size limit") + return encoder.encode(value).encode("utf-8") + except (TypeError, UnicodeError, RecursionError) as error: + raise ValueError("malformed policy record") from error + + +def sha(raw): + return hashlib.sha256(raw).hexdigest() + + +def digest(value): + return ( + type(value) is str + and len(value) == 64 + and all(c in "0123456789abcdef" for c in value) + ) + + +def text(value): + # Unicode White_Space, matching Rust str::trim (Python additionally strips FS/GS/RS/US). + whitespace = "\t\n\v\f\r \u0085\u00a0\u1680\u2000\u2001\u2002\u2003\u2004\u2005\u2006\u2007\u2008\u2009\u200a\u2028\u2029\u202f\u205f\u3000" + return type(value) is str and bool(value.strip(whitespace)) + + +def uint(value): + return type(value) is int and 0 <= value <= MAX_U64 + + +def fields(value, names): + require( + type(value) is dict and set(value) == set(names.split()), + "unknown or missing fields", + ) + + +def validate_policy_artifact(raw): + a = loads(raw, 65536) + fields(a, "schema_version policy_name purpose compatibility_contract_sha256") + require( + type(a["schema_version"]) is int + and a["schema_version"] == 1 + and a["policy_name"] in POLICIES + and text(a["purpose"]) + and digest(a["compatibility_contract_sha256"]), + "invalid policy artifact", + ) + return a + + +def byte_array(value): + require( + type(value) is list + and len(value) <= MAX_BYTES + and all(type(x) is int and 0 <= x <= 255 for x in value), + "invalid bytes", + ) + return bytes(value) + + +def validate_policy_candidate(c): + fields(c, "version_id actor scope purpose artifact sha256") + canonical(c) + require( + all(text(c[k]) for k in ("version_id", "actor", "scope", "purpose")), + "empty candidate identity", + ) + raw = byte_array(c["artifact"]) + a = validate_policy_artifact(raw) + require( + a["purpose"] == c["purpose"] + and digest(c["sha256"]) + and sha(raw) == c["sha256"], + "candidate identity mismatch", + ) + return a + + +SELECTION_CONTRACT = b"policy-selection/1\ncomplete-paired-frozen-attestations;success=strict-increase;false-refusal,unsafe-acceptance,charged-plus-reserved,deadline-miss=nonincrease;synthetic=fixture/recovery-policy-only\n" +SELECTION_CONTRACT_SHA256 = ( + "df97ae41b2648bef15a11bf1950883597b057df1940fe43347d485fedf8590b3" +) +PROVENANCE = ("synthetic_registry_mechanics", "observed_policy_replay") +OUTCOMES = ("Accepted", "Refused", "Failed", "Unknown") +DISPOSITIONS = ("Accepted", "InvalidOutput", "Stale", "Failed", "Uncertain", "Refused") + + +def add(a, b): + require(uint(a) and uint(b) and a + b <= MAX_U64, "policy accounting overflow") + return a + b + + +def check_record(r): + fields(r, "bytes sha256") + raw = byte_array(r["bytes"]) + require(digest(r["sha256"]) and sha(raw) == r["sha256"], "record digest mismatch") + return raw + + +def shape(e): + fields( + e, + "evaluation_id actor scope purpose baseline_id candidate_id baseline_sha256 candidate_sha256 experiment_manifest experiment_sha256 evaluator_version evaluator_sha256 provenance baseline_rows candidate_rows", + ) + require( + all( + text(e[k]) + for k in ( + "evaluation_id", + "actor", + "scope", + "purpose", + "baseline_id", + "candidate_id", + "evaluator_version", + ) + ), + "empty evaluation identity", + ) + require( + all( + digest(e[k]) + for k in ( + "baseline_sha256", + "candidate_sha256", + "experiment_sha256", + "evaluator_sha256", + ) + ), + "invalid evaluation digest", + ) + require(e["provenance"] in PROVENANCE, "invalid provenance") + byte_array(e["experiment_manifest"]) + for side in ("baseline_rows", "candidate_rows"): + require(type(e[side]) is list and len(e[side]) <= MAX_CASES, "case limit") + for r in e[side]: + fields( + r, + "case_id outcome correct eligible answerable completed_ms charged reserved_bound false_refusal unsafe_acceptance deadline_miss simulated attempts", + ) + require( + text(r["case_id"]) and r["outcome"] in OUTCOMES, + "invalid row identity or outcome", + ) + require( + all( + type(r[k]) is bool + for k in ( + "answerable", + "false_refusal", + "unsafe_acceptance", + "deadline_miss", + "simulated", + ) + ), + "invalid bool", + ) + require( + all( + r[k] is None or type(r[k]) is bool for k in ("correct", "eligible") + ), + "invalid nullable bool", + ) + require( + all(r[k] is None or uint(r[k]) for k in ("completed_ms", "charged")) + and uint(r["reserved_bound"]), + "invalid row count", + ) + if r["charged"] is not None: + add(r["charged"], r["reserved_bound"]) + require( + type(r["attempts"]) is list and len(r["attempts"]) <= 2, "attempt limit" + ) + for a in r["attempts"]: + fields( + a, + "execution_id opportunity_id request_sha256 evidence_manifest_sha256 execution disposition response measured_input measured_output retained_bound start_ms completed_ms validator_sha256 scorer_sha256 label_sha256 scoring", + ) + require( + text(a["execution_id"]) and text(a["opportunity_id"]), + "empty attempt identity", + ) + require( + all( + digest(a[k]) + for k in ( + "request_sha256", + "evidence_manifest_sha256", + "validator_sha256", + "scorer_sha256", + "label_sha256", + ) + ), + "invalid attempt digest", + ) + require( + a["execution"] in ("Executed", "NotExecuted", "Unknown") + and a["disposition"] in DISPOSITIONS, + "invalid execution or disposition", + ) + require( + uint(a["retained_bound"]) + and uint(a["start_ms"]) + and all( + a[k] is None or uint(a[k]) + for k in ("measured_input", "measured_output", "completed_ms") + ), + "invalid attempt count", + ) + if a["response"] is not None: + check_record(a["response"]) + if a["scoring"] is not None: + s = a["scoring"] + fields(s, "response_sha256 scorer_sha256 label_sha256 normalized") + require( + all( + digest(s[k]) + for k in ( + "response_sha256", + "scorer_sha256", + "label_sha256", + ) + ), + "invalid scoring digest", + ) + byte_array(s["normalized"]) + canonical(e) + + +def parse_policy_evaluation(raw): + e = loads(raw) + shape(e) + return e + + +def evaluation_sha256(e): + shape(e) + return sha(b"policy-evaluation/2\n" + canonical(e)) + + +def paired_baseline_matches(recorded, paired): + shape(recorded) + shape(paired) + return all( + recorded[k] == paired[k] + for k in ( + "actor", + "scope", + "purpose", + "baseline_id", + "baseline_sha256", + "experiment_manifest", + "experiment_sha256", + "evaluator_version", + "evaluator_sha256", + "provenance", + "baseline_rows", + ) + ) + + +def manifest(e): + m = loads(byte_array(e["experiment_manifest"])) + fields( + m, + "format actor scope purpose provenance accounting_unit workload_sha256 prompt_sha256 runtime_sha256 model_sha256 tokenizer_sha256 validator_sha256 scorer_sha256 compatibility_contract_sha256 policies cases", + ) + require( + m["format"] == "policy-comparison/1" + and type(m["cases"]) is list + and 0 < len(m["cases"]) <= MAX_CASES + and type(m["policies"]) is list + and 0 < len(m["policies"]) <= 3, + "invalid manifest format or count", + ) + require( + all(text(m[k]) for k in ("actor", "scope", "purpose")), + "empty manifest identity", + ) + require( + m["provenance"] in PROVENANCE + and m["accounting_unit"] + in ("model_tokens", "fixture_bytes", "logical_support", "unknown"), + "invalid manifest enum", + ) + require( + all( + digest(m[k]) + for k in ( + "workload_sha256", + "prompt_sha256", + "runtime_sha256", + "model_sha256", + "tokenizer_sha256", + "validator_sha256", + "scorer_sha256", + "compatibility_contract_sha256", + ) + ), + "invalid manifest digest", + ) + names = set() + artifacts = set() + for p in m["policies"]: + fields(p, "policy_name artifact") + a = validate_policy_artifact(check_record(p["artifact"])) + require( + p["policy_name"] in POLICIES + and p["policy_name"] not in names + and p["artifact"]["sha256"] not in artifacts, + "duplicate frozen policy", + ) + names.add(p["policy_name"]) + artifacts.add(p["artifact"]["sha256"]) + require( + a["policy_name"] == p["policy_name"] + and a["purpose"] == m["purpose"] + and a["compatibility_contract_sha256"] + == m["compatibility_contract_sha256"], + "frozen artifact identity mismatch", + ) + cases = set() + families = {} + labels = [] + for c in m["cases"]: + fields( + c, + "case_id family_id split_id initial_evidence label budget deadline_ms timing_start_ms timing_end_ms policy_opportunities", + ) + require( + all(text(c[k]) for k in ("case_id", "family_id", "split_id")) + and c["case_id"] not in cases, + "invalid or duplicate case identity", + ) + cases.add(c["case_id"]) + require( + all( + uint(c[k]) + for k in ("budget", "deadline_ms", "timing_start_ms", "timing_end_ms") + ) + and c["timing_start_ms"] <= c["deadline_ms"] <= c["timing_end_ms"], + "invalid timing boundaries", + ) + require( + families.setdefault(c["family_id"], c["split_id"]) == c["split_id"], + "family crosses splits", + ) + check_record(c["initial_evidence"]) + labelraw = check_record(c["label"]) + l = loads(labelraw, 65536) + fields( + l, + "answerable admissible_support normalized_format scorer_sha256 expected_normalized", + ) + require( + type(l["answerable"]) is bool + and type(l["admissible_support"]) is bool + and l["normalized_format"] == "opaque-bytes/1" + and l["scorer_sha256"] == m["scorer_sha256"], + "invalid frozen label identity", + ) + byte_array(l["expected_normalized"]) + require( + type(c["policy_opportunities"]) is list + and len(c["policy_opportunities"]) == len(names), + "missing frozen policy opportunities", + ) + policies = set() + for p in c["policy_opportunities"]: + fields(p, "policy_name opportunities") + require( + p["policy_name"] in names and p["policy_name"] not in policies, + "unknown or duplicate opportunity policy", + ) + policies.add(p["policy_name"]) + require( + type(p["opportunities"]) is list + and len(p["opportunities"]) + == (1 if p["policy_name"] == "always_stop" else 2), + "invalid frozen opportunity count", + ) + ids = set() + for i, o in enumerate(p["opportunities"]): + fields( + o, + "opportunity_id request evidence_manifest admitted consistent input_bound output_cap start_min_ms start_max_ms", + ) + require( + text(o["opportunity_id"]) and o["opportunity_id"] not in ids, + "duplicate opportunity identity", + ) + ids.add(o["opportunity_id"]) + check_record(o["request"]) + check_record(o["evidence_manifest"]) + require( + type(o["admitted"]) is bool and type(o["consistent"]) is bool, + "invalid opportunity bool", + ) + require( + all( + uint(o[k]) + for k in ( + "input_bound", + "output_cap", + "start_min_ms", + "start_max_ms", + ) + ), + "invalid opportunity count", + ) + add(o["input_bound"], o["output_cap"]) + require( + c["timing_start_ms"] + <= o["start_min_ms"] + <= o["start_max_ms"] + <= c["timing_end_ms"], + "invalid opportunity timing", + ) + if i == 0: + require( + o["evidence_manifest"] == c["initial_evidence"], + "initial evidence differs", + ) + first = c["policy_opportunities"][0]["opportunities"][0] + for p in c["policy_opportunities"][1:]: + o = dict(p["opportunities"][0], opportunity_id=first["opportunity_id"]) + require(o == first, "different initial policy opportunities") + labels.append(l) + return m, labels + + +def refused(reason): + return {"decision": "Refused", "reason": reason} + + +def evaluate_policy_comparison(e): + shape(e) + m, labels = manifest(e) + if ( + any(e[k] != m[k] for k in ("actor", "scope", "purpose", "provenance")) + or sha(byte_array(e["experiment_manifest"])) != e["experiment_sha256"] + or e["evaluator_version"] != "policy-selection/1" + or e["evaluator_sha256"] != SELECTION_CONTRACT_SHA256 + or e["baseline_id"] == e["candidate_id"] + or m["accounting_unit"] not in ("model_tokens", "fixture_bytes") + or ( + e["provenance"] == "synthetic_registry_mechanics" + and e["purpose"] != "fixture/recovery-policy" + ) + ): + return refused("Incomparable") + selected = {p["artifact"]["sha256"]: p["policy_name"] for p in m["policies"]} + if any(e[k] not in selected for k in ("baseline_sha256", "candidate_sha256")): + return refused("Incomparable") + executions = {} + b = rows(m, labels, e["baseline_rows"], selected[e["baseline_sha256"]], executions) + c = rows( + m, labels, e["candidate_rows"], selected[e["candidate_sha256"]], executions + ) + if type(b) is str: + return refused(b) + if type(c) is str: + return refused(c) + if c["success"] <= b["success"]: + return refused("NoImprovement") + for key, reason in [ + ("false_refusal", "IncreasedFalseRefusal"), + ("unsafe_acceptance", "IncreasedUnsafeAcceptance"), + ("cost", "IncreasedCost"), + ("deadline_misses", "IncreasedDeadlineMisses"), + ]: + if c[key] > b[key]: + return refused(reason) + return { + "decision": "Eligible", + "validated_counters": {"baseline": b, "candidate": c}, + } + + +def rows(m, labels, records, policy, executions): + issue = "Incomplete" if len(records) != len(m["cases"]) else None + counters = { + "cases": 0, + "success": 0, + "false_refusal": 0, + "unsafe_acceptance": 0, + "cost": 0, + "deadline_misses": 0, + } + for r, c, l in zip(records, m["cases"], labels): + if r["case_id"] != c["case_id"]: + issue = "Incomparable" + continue + require(r["answerable"] == l["answerable"], "fabricated answerability") + require( + r["false_refusal"] + == ( + r["outcome"] == "Refused" + and l["answerable"] + and l["admissible_support"] + ), + "fabricated false refusal", + ) + if r["outcome"] != "Accepted": + require(not r["unsafe_acceptance"], "fabricated unsafe acceptance") + if r["correct"] is not None and r["eligible"] is not None: + require( + r["unsafe_acceptance"] + == ( + r["outcome"] == "Accepted" + and (not r["correct"] or not r["eligible"]) + ), + "fabricated unsafe acceptance", + ) + if r["completed_ms"] is not None: + require( + c["timing_start_ms"] <= r["completed_ms"] <= c["timing_end_ms"] + and r["deadline_miss"] == (r["completed_ms"] > c["deadline_ms"]), + "fabricated deadline miss", + ) + if r["charged"] is not None: + require( + add(r["charged"], r["reserved_bound"]) <= c["budget"], + "row exceeds frozen budget", + ) + if m["provenance"] == "synthetic_registry_mechanics": + require( + r["simulated"] and not r["attempts"], + "synthetic rows require simulated counters only", + ) + if r["eligible"] is not None: + require( + r["eligible"] == l["admissible_support"], + "fabricated synthetic eligibility", + ) + else: + require(not r["simulated"], "observed row cannot simulate counters") + reason = observed(m, c, l, r, policy, executions) + if reason: + issue = issue or reason + continue + if ( + any( + r[k] is None for k in ("correct", "eligible", "completed_ms", "charged") + ) + or r["outcome"] == "Unknown" + ): + issue = issue or "Incomplete" + continue + require( + c["timing_start_ms"] <= r["completed_ms"] <= c["timing_end_ms"], + "row outside timing window", + ) + require( + r["false_refusal"] + == ( + r["outcome"] == "Refused" + and l["answerable"] + and l["admissible_support"] + ), + "fabricated false refusal", + ) + require( + r["unsafe_acceptance"] + == (r["outcome"] == "Accepted" and (not r["correct"] or not r["eligible"])), + "fabricated unsafe acceptance", + ) + require( + r["deadline_miss"] == (r["completed_ms"] > c["deadline_ms"]), + "fabricated deadline miss", + ) + cost = add(r["charged"], r["reserved_bound"]) + require(cost <= c["budget"], "row exceeds frozen budget") + values = { + "cases": 1, + "success": int( + r["outcome"] == "Accepted" + and r["correct"] + and r["eligible"] + and not r["deadline_miss"] + ), + "false_refusal": int(r["false_refusal"]), + "unsafe_acceptance": int(r["unsafe_acceptance"]), + "cost": cost, + "deadline_misses": int(r["deadline_miss"]), + } + for k, v in values.items(): + counters[k] = add(counters[k], v) + return issue or counters + + +def observed(m, c, l, r, policy, executions): + ops = next( + p["opportunities"] + for p in c["policy_opportunities"] + if p["policy_name"] == policy + ) + if not r["attempts"]: + return "Incomplete" + require(len(r["attempts"]) <= len(ops), "too many observed attempts") + charged = 0 + previous_end = c["timing_start_ms"] + result = ("Refused", False, l["admissible_support"], previous_end) + ids = set() + for i, a in enumerate(r["attempts"]): + o = ops[i] + if ( + a["opportunity_id"] != o["opportunity_id"] + or a["request_sha256"] != o["request"]["sha256"] + or a["evidence_manifest_sha256"] != o["evidence_manifest"]["sha256"] + or a["validator_sha256"] != m["validator_sha256"] + or a["scorer_sha256"] != m["scorer_sha256"] + or a["label_sha256"] != c["label"]["sha256"] + ): + return "Incomparable" + require(a["execution_id"] not in ids, "reused execution in one policy path") + ids.add(a["execution_id"]) + if i > 0: + require( + r["attempts"][i - 1]["execution"] == "Executed" + and r["attempts"][i - 1]["disposition"] == "InvalidOutput", + "only settled invalid output unlocks retry", + ) + require( + a["start_ms"] >= previous_end + and o["start_min_ms"] <= a["start_ms"] <= o["start_max_ms"], + "attempt outside timing window", + ) + quote = add(o["input_bound"], o["output_cap"]) + require(charged <= c["budget"], "budget exceeded") + gate = ( + o["admitted"] + and not (i > 0 and policy == "evidence_gated_retry" and not o["consistent"]) + and a["start_ms"] < c["deadline_ms"] + and quote <= c["budget"] - charged + ) + if a["execution"] == "Unknown": + require( + a["measured_input"] is None + and a["measured_output"] is None + and a["response"] is None + and a["scoring"] is None + and a["retained_bound"] == quote + and a["disposition"] == "Uncertain" + and gate + and i + 1 == len(r["attempts"]), + "invalid unknown execution", + ) + require( + r["charged"] is None + and r["reserved_bound"] == quote + and r["outcome"] == "Unknown", + "unknown row accounting", + ) + return "Incomplete" + if a["execution"] == "NotExecuted": + require( + not gate + and a["disposition"] == "Refused" + and a["measured_input"] == 0 + and a["measured_output"] == 0 + and a["retained_bound"] == 0 + and a["response"] is None + and a["scoring"] is None + and a["completed_ms"] == a["start_ms"] + and i + 1 == len(r["attempts"]), + "invalid nonexecution attestation", + ) + result = ( + "Refused", + False, + o["admitted"] and l["admissible_support"], + a["start_ms"], + ) + else: + require(gate, "execution violates frozen admission") + if a["measured_input"] is not None: + require(a["measured_input"] <= o["input_bound"], "input exceeds quote") + if a["measured_output"] is not None: + require(a["measured_output"] <= o["output_cap"], "output exceeds cap") + if a["measured_input"] is None or a["measured_output"] is None: + require( + a["retained_bound"] == quote + and r["charged"] is None + and r["reserved_bound"] == quote + and r["outcome"] == "Unknown", + "unknown usage must retain full bound", + ) + require( + i + 1 == len(r["attempts"]), "unknown usage cannot unlock retry" + ) + return "Incomplete" + require( + a["retained_bound"] == 0 and a["disposition"] != "Uncertain", + "settled usage has retained hold or uncertain disposition", + ) + charged = add(charged, add(a["measured_input"], a["measured_output"])) + end = a["completed_ms"] + if end is None: + return "Incomplete" + require( + a["start_ms"] <= end <= c["timing_end_ms"], "invalid completion time" + ) + previous_end = end + if a["response"] is not None: + s = a["scoring"] + if s is None: + return "Incomplete" + if ( + s["response_sha256"] != a["response"]["sha256"] + or s["scorer_sha256"] != m["scorer_sha256"] + or s["label_sha256"] != c["label"]["sha256"] + ): + return "Incomparable" + correct = s["normalized"] == l["expected_normalized"] + else: + require(a["scoring"] is None, "scoring without response") + if a["disposition"] != "Failed": + return "Incomplete" + correct = False + outcome = ( + "Accepted" + if a["disposition"] == "Accepted" + else ("Failed" if a["disposition"] == "Failed" else "Refused") + ) + result = ( + outcome, + correct, + o["admitted"] + and l["admissible_support"] + and a["disposition"] != "Stale", + end, + ) + if ( + a["disposition"] == "InvalidOutput" + and i + 1 < len(ops) + and i + 1 == len(r["attempts"]) + ): + return "Incomplete" + if a["disposition"] != "InvalidOutput": + require( + i + 1 == len(r["attempts"]), + "terminal disposition cannot unlock retry", + ) + observation = (c["case_id"], dict(a, opportunity_id="")) + if ( + a["execution_id"] in executions + and executions[a["execution_id"]] != observation + ): + return "Incomparable" + executions[a["execution_id"]] = observation + require( + (r["outcome"] == "Unknown" or r["outcome"] == result[0]) + and all( + r[k] is None or r[k] == v + for k, v in zip( + ("correct", "eligible", "completed_ms", "charged"), + (*result[1:], charged), + ) + ) + and r["reserved_bound"] == 0, + "fabricated observed row totals", + ) + if r["outcome"] == "Unknown" or any( + r[k] is None for k in ("correct", "eligible", "completed_ms", "charged") + ): + return "Incomplete" + return None + + +def convert_legacy(document, comparison=None): + """No missing facts are inferred. A supplied complete attestation must bind this input. + + workload_sha256 binds the compact sorted-key UTF-8 encoding of the archived + document. Request strings, scheduling, quotes and legacy measured totals are + checked in addition. Evidence contents, labels and normalization remain the + supplying producer's attestations, not facts created by this converter. + """ + from recovery_policy_eval import evaluate + + legacy = evaluate(document) + source_sha = sha(canonical(document)) + result = { + "format": "policy-evaluation-conversion/2", + "status": "Incomplete", + "legacy_input_sha256": source_sha, + "reason": "A matching complete frozen comparison attestation is required; legacy fields do not establish answerability, eligibility, units or observed references.", + } + if comparison is None: + return result + shape(comparison) + m, _ = manifest(comparison) + expected_provenance = ( + "synthetic_registry_mechanics" + if document["mode"] == "synthetic_policy_fixture" + else "observed_policy_replay" + ) + if ( + m["workload_sha256"] != source_sha + or m["accounting_unit"] != "model_tokens" + or m["provenance"] != expected_provenance + ): + return result + if len(m["cases"]) != len(document["cases"]): + return result + for c, old in zip(m["cases"], document["cases"]): + if any( + c[k] != old[v] + for k, v in [ + ("case_id", "case_id"), + ("family_id", "family"), + ("split_id", "split"), + ("budget", "budget_tokens"), + ("deadline_ms", "deadline_ms"), + ] + ): + return result + if c["timing_start_ms"] != 0: + return result + for p in c["policy_opportunities"]: + for i, o in enumerate(p["opportunities"]): + cell = old["retries"].get(p["policy_name"]) if i else None + if i and cell is None: + return result + ctx = cell["context"] if i else old["initial_context"] + if byte_array(o["request"]["bytes"]) != ctx["request"].encode("utf-8"): + return result + if any( + o[k] != ctx[v] + for k, v in [ + ("admitted", "admitted"), + ("consistent", "consistent"), + ("input_bound", "input_tokens"), + ("output_cap", "output_cap"), + ] + ): + return result + policies = {p["artifact"]["sha256"]: p["policy_name"] for p in m["policies"]} + for side, identity in [ + ("baseline_rows", "baseline_sha256"), + ("candidate_rows", "candidate_sha256"), + ]: + policy = policies.get(comparison[identity]) + oldrows = [r for r in legacy["cases"] if r["policy"] == policy] + if len(oldrows) != len(comparison[side]): + return result + for row, old in zip(comparison[side], oldrows): + if old["reason"] == "uncertain": + return result + if any( + row[k] != old[v] + for k, v in [ + ("case_id", "case_id"), + ("correct", "correct"), + ("completed_ms", "elapsed_ms"), + ("charged", "charged_tokens"), + ("reserved_bound", "reserved_tokens"), + ] + ): + return result + if row["outcome"] != ( + "Refused" + if old["abstained"] or old["reason"] == "stale" + else "Accepted" + ): + return result + success = ( + row["outcome"] == "Accepted" + and row["correct"] is True + and row["eligible"] is True + and row["completed_ms"] + <= next( + c["deadline_ms"] + for c in m["cases"] + if c["case_id"] == row["case_id"] + ) + ) + if success != old["success"]: + return result + selection = evaluate_policy_comparison(comparison) + if selection == refused("Incomplete"): + return result + return { + "format": "policy-evaluation-conversion/2", + "status": "Complete", + "legacy_input_sha256": source_sha, + "evaluation": comparison, + "selection": selection, + } diff --git a/scripts/recovery_policy_eval.py b/scripts/recovery_policy_eval.py index 13cfc15..27a133e 100644 --- a/scripts/recovery_policy_eval.py +++ b/scripts/recovery_policy_eval.py @@ -224,14 +224,27 @@ def main(): parser.add_argument('trace', nargs='?', type=Path) parser.add_argument('--fixture', action='store_true', help='evaluate deterministic synthetic mechanics only') parser.add_argument('--emit-fixture', action='store_true', help='print the full synthetic input schema example') + parser.add_argument('--convert-v2', action='store_true', help='explicit complete-attestation conversion; missing facts remain incomplete') + parser.add_argument('--comparison-v2', type=Path, help='matching complete v2 comparison attestation for --convert-v2') args = parser.parse_args() + if (args.comparison_v2 and not args.convert_v2) or (args.convert_v2 and args.emit_fixture): + parser.error('--comparison-v2 requires --convert-v2; conversion cannot emit a fixture') if sum((args.trace is not None, args.fixture, args.emit_fixture)) != 1: parser.error('choose exactly one of TRACE, --fixture, --emit-fixture') try: document = synthetic_fixture() if args.fixture or args.emit_fixture else json.loads( args.trace.read_text(), object_pairs_hook=unique_object, parse_constant=lambda value: (_ for _ in ()).throw(ValueError(f'nonfinite JSON value: {value}'))) - print(json.dumps(document if args.emit_fixture else evaluate(document), indent=2, allow_nan=False)) + if args.convert_v2: + from policy_evaluation_v2 import convert_legacy, parse_policy_evaluation, MAX_BYTES + comparison = None + if args.comparison_v2: + with args.comparison_v2.open('rb') as stream: + comparison = parse_policy_evaluation(stream.read(MAX_BYTES + 1)) + result = convert_legacy(document, comparison) + else: + result = document if args.emit_fixture else evaluate(document) + print(json.dumps(result, indent=2, allow_nan=False)) except (ValueError, OSError, TypeError) as error: parser.exit(2, f'error: {error}\n') diff --git a/scripts/test_policy_evaluation_v2.py b/scripts/test_policy_evaluation_v2.py new file mode 100644 index 0000000..7a3bf9a --- /dev/null +++ b/scripts/test_policy_evaluation_v2.py @@ -0,0 +1,984 @@ +"""Strict offline comparison contract; no model calls.""" + +import importlib.util +import unittest + + +class PolicyEvaluationTests(unittest.TestCase): + def test_complete_comparison_api_exists(self): + self.assertIsNotNone( + importlib.util.find_spec("policy_evaluation_v2"), + "complete v2 comparison validator is missing", + ) + + +class ArtifactTests(unittest.TestCase): + def test_artifact_refuses_unknown_duplicate_and_injected_configuration(self): + from policy_evaluation_v2 import validate_policy_artifact + + good = ( + b'{"schema_version":1,"policy_name":"always_stop","purpose":"fixture/recovery-policy","compatibility_contract_sha256":"' + + b"a" * 64 + + b'"}' + ) + self.assertEqual(validate_policy_artifact(good)["policy_name"], "always_stop") + for bad in ( + good.replace(b"always_stop", b"evil"), + good.replace(b"fixture/recovery-policy", b" "), + good.replace(b"{", b'{"purpose":"duplicate",'), + good.replace(b"{", b'{"script":"exec",'), + good.replace(b"a" * 64, b"A" * 64), + good.replace(b'"schema_version":1', b'"schema_version":true'), + b" " * 65537, + ): + with self.subTest(bad=bad[:80]), self.assertRaises(ValueError): + validate_policy_artifact(bad) + + +import copy +import hashlib +import json + + +def pack(value): + return json.dumps( + value, sort_keys=True, separators=(",", ":"), ensure_ascii=False + ).encode() + + +def record(raw): + return {"bytes": list(raw), "sha256": hashlib.sha256(raw).hexdigest()} + + +CONTRACT = b"policy-selection/1\ncomplete-paired-frozen-attestations;success=strict-increase;false-refusal,unsafe-acceptance,charged-plus-reserved,deadline-miss=nonincrease;synthetic=fixture/recovery-policy-only\n" +CONTRACT_SHA = "df97ae41b2648bef15a11bf1950883597b057df1940fe43347d485fedf8590b3" + + +def fixture(): + policies = [] + for name in ("always_stop", "affordable_retry", "evidence_gated_retry"): + artifact = { + "schema_version": 1, + "policy_name": name, + "purpose": "fixture/recovery-policy", + "compatibility_contract_sha256": "a" * 64, + } + policies.append({"policy_name": name, "artifact": record(pack(artifact))}) + label = record( + pack( + { + "answerable": True, + "admissible_support": True, + "normalized_format": "opaque-bytes/1", + "scorer_sha256": "1" * 64, + "expected_normalized": list(b"yes"), + } + ) + ) + opportunities = [] + for policy in policies: + name = policy["policy_name"] + ops = [] + for n in range(1 if name == "always_stop" else 2): + ops.append( + { + "opportunity_id": f"{name}/{n}", + "request": record(f"request{n}".encode()), + "evidence_manifest": record(b"evidence"), + "admitted": True, + "consistent": True, + "input_bound": 4, + "output_cap": 6, + "start_min_ms": 0, + "start_max_ms": 99, + } + ) + opportunities.append({"policy_name": name, "opportunities": ops}) + case = { + "case_id": "c1", + "family_id": "family", + "split_id": "heldout", + "initial_evidence": record(b"evidence"), + "label": label, + "budget": 30, + "deadline_ms": 100, + "timing_start_ms": 0, + "timing_end_ms": 200, + "policy_opportunities": opportunities, + } + manifest = { + "format": "policy-comparison/1", + "actor": "actor", + "scope": "scope", + "purpose": "fixture/recovery-policy", + "provenance": "synthetic_registry_mechanics", + "accounting_unit": "fixture_bytes", + "workload_sha256": "a" * 64, + "prompt_sha256": "b" * 64, + "runtime_sha256": "c" * 64, + "model_sha256": "d" * 64, + "tokenizer_sha256": "e" * 64, + "validator_sha256": "f" * 64, + "scorer_sha256": "1" * 64, + "compatibility_contract_sha256": "a" * 64, + "policies": policies, + "cases": [case], + } + row = { + "case_id": "c1", + "outcome": "Refused", + "correct": False, + "eligible": True, + "answerable": True, + "completed_ms": 100, + "charged": 4, + "reserved_bound": 0, + "false_refusal": True, + "unsafe_acceptance": False, + "deadline_miss": False, + "simulated": True, + "attempts": [], + } + candidate = copy.deepcopy(row) + candidate.update(outcome="Accepted", correct=True, false_refusal=False) + raw = pack(manifest) + return { + "evaluation_id": "e1", + "actor": "actor", + "scope": "scope", + "purpose": "fixture/recovery-policy", + "baseline_id": "v1", + "candidate_id": "v2", + "baseline_sha256": policies[0]["artifact"]["sha256"], + "candidate_sha256": policies[1]["artifact"]["sha256"], + "experiment_manifest": list(raw), + "experiment_sha256": hashlib.sha256(raw).hexdigest(), + "evaluator_version": "policy-selection/1", + "evaluator_sha256": CONTRACT_SHA, + "provenance": "synthetic_registry_mechanics", + "baseline_rows": [row], + "candidate_rows": [candidate], + } + + +def manifest_edit(e, change): + m = json.loads(bytes(e["experiment_manifest"])) + change(m) + raw = pack(m) + e.update( + experiment_manifest=list(raw), experiment_sha256=hashlib.sha256(raw).hexdigest() + ) + + +class ComparisonTests(unittest.TestCase): + def evaluate(self, e): + from policy_evaluation_v2 import evaluate_policy_comparison + + return evaluate_policy_comparison(e) + + def test_positive_synthetic_and_equal_deadline(self): + result = self.evaluate(fixture()) + self.assertEqual( + result, + { + "decision": "Eligible", + "validated_counters": { + "baseline": { + "cases": 1, + "success": 0, + "false_refusal": 1, + "unsafe_acceptance": 0, + "cost": 4, + "deadline_misses": 0, + }, + "candidate": { + "cases": 1, + "success": 1, + "false_refusal": 0, + "unsafe_acceptance": 0, + "cost": 4, + "deadline_misses": 0, + }, + }, + }, + ) + + def test_identity_mismatches_are_incomparable(self): + for key, value in [ + ("actor", "other"), + ("scope", "other"), + ("purpose", "other"), + ("experiment_sha256", "0" * 64), + ("baseline_sha256", "0" * 64), + ("candidate_sha256", "0" * 64), + ("evaluator_version", "new"), + ("evaluator_sha256", "0" * 64), + ("provenance", "observed_policy_replay"), + ]: + e = fixture() + e[key] = value + with self.subTest(key=key): + self.assertEqual( + self.evaluate(e), {"decision": "Refused", "reason": "Incomparable"} + ) + for key, value in [ + ("accounting_unit", "logical_support"), + ("accounting_unit", "unknown"), + ]: + e = fixture() + manifest_edit(e, lambda m, k=key, v=value: m.update({k: v})) + self.assertEqual(self.evaluate(e)["reason"], "Incomparable") + + def test_missing_unknown_and_reordered_cases_refuse(self): + for side in ("baseline_rows", "candidate_rows"): + e = fixture() + e[side] = [] + self.assertEqual(self.evaluate(e)["reason"], "Incomplete") + for key in ("correct", "eligible", "completed_ms", "charged"): + e = fixture() + e[side][0][key] = None + self.assertEqual(self.evaluate(e)["reason"], "Incomplete") + e = fixture() + e[side][0].update(outcome="Unknown", false_refusal=False) + self.assertEqual(self.evaluate(e)["reason"], "Incomplete") + e = fixture() + e[side][0]["case_id"] = "other" + self.assertEqual(self.evaluate(e)["reason"], "Incomparable") + + def test_fabricated_flags_labels_and_counters_error(self): + for key in ( + "false_refusal", + "unsafe_acceptance", + "deadline_miss", + "answerable", + "eligible", + ): + e = fixture() + e["candidate_rows"][0][key] = not e["candidate_rows"][0][key] + with self.subTest(key=key), self.assertRaises(ValueError): + self.evaluate(e) + for value in (-1, True, 1.5, 1 << 64): + e = fixture() + e["candidate_rows"][0]["charged"] = value + with self.subTest(value=value), self.assertRaises(ValueError): + self.evaluate(e) + e = fixture() + e["candidate_rows"][0].update(charged=(1 << 64) - 1, reserved_bound=1) + with self.assertRaises(ValueError): + self.evaluate(e) + e = fixture() + e["candidate_rows"][0]["invented_total"] = 100 + with self.assertRaises(ValueError): + self.evaluate(e) + + def test_no_improvement_cost_and_deadline_rules(self): + e = fixture() + e["candidate_rows"] = copy.deepcopy(e["baseline_rows"]) + self.assertEqual(self.evaluate(e)["reason"], "NoImprovement") + e = fixture() + e["candidate_rows"][0]["charged"] = 5 + self.assertEqual(self.evaluate(e)["reason"], "IncreasedCost") + e = fixture() + e["candidate_rows"][0].update(completed_ms=101, deadline_miss=True) + self.assertEqual(self.evaluate(e)["reason"], "NoImprovement") + + def test_synthetic_nonfixture_refuses_even_consistent_identities(self): + e = fixture() + e["purpose"] = "production" + + def edit(m): + m["purpose"] = "production" + for p in m["policies"]: + a = json.loads(bytes(p["artifact"]["bytes"])) + a["purpose"] = "production" + p["artifact"] = record(pack(a)) + e.update( + baseline_sha256=m["policies"][0]["artifact"]["sha256"], + candidate_sha256=m["policies"][1]["artifact"]["sha256"], + ) + + manifest_edit(e, edit) + self.assertEqual(self.evaluate(e)["reason"], "Incomparable") + + def test_parse_required_duplicate_and_unknown_fields(self): + from policy_evaluation_v2 import parse_policy_evaluation + + e = fixture() + self.assertEqual(parse_policy_evaluation(pack(e)), e) + for raw in ( + pack(e).replace(b"{", b'{"actor":"duplicate",', 1), + pack(e).replace(b"{", b'{"extra":0,', 1), + b" " * ((16 << 20) + 1), + ): + with self.assertRaises(ValueError): + parse_policy_evaluation(raw) + del e["candidate_rows"][0]["charged"] + with self.assertRaises(ValueError): + parse_policy_evaluation(pack(e)) + + def test_golden_contract_and_provenance_digest(self): + from policy_evaluation_v2 import SELECTION_CONTRACT, evaluation_sha256 + + self.assertEqual(SELECTION_CONTRACT, CONTRACT) + self.assertEqual(hashlib.sha256(CONTRACT).hexdigest(), CONTRACT_SHA) + e = fixture() + before = evaluation_sha256(e) + e["provenance"] = "observed_policy_replay" + self.assertNotEqual(before, evaluation_sha256(e)) + + +def observed_fixture(): + e = fixture() + e["provenance"] = "observed_policy_replay" + manifest_edit( + e, + lambda m: m.update( + provenance="observed_policy_replay", accounting_unit="model_tokens" + ), + ) + m = json.loads(bytes(e["experiment_manifest"])) + c = m["cases"][0] + for side, index in [("baseline_rows", 0), ("candidate_rows", 1)]: + row = e[side][0] + o = c["policy_opportunities"][index]["opportunities"][0] + response = record(b"yes") + a = { + "execution_id": "shared-execution", + "opportunity_id": o["opportunity_id"], + "request_sha256": o["request"]["sha256"], + "evidence_manifest_sha256": o["evidence_manifest"]["sha256"], + "execution": "Executed", + "disposition": "Accepted", + "response": response, + "measured_input": 2, + "measured_output": 2, + "retained_bound": 0, + "start_ms": 0, + "completed_ms": 100, + "validator_sha256": m["validator_sha256"], + "scorer_sha256": m["scorer_sha256"], + "label_sha256": c["label"]["sha256"], + "scoring": { + "response_sha256": response["sha256"], + "scorer_sha256": m["scorer_sha256"], + "label_sha256": c["label"]["sha256"], + "normalized": list(b"yes"), + }, + } + row.update( + outcome="Accepted", + correct=True, + false_refusal=False, + simulated=False, + attempts=[a], + ) + return e + + +class ObservedTests(unittest.TestCase): + evaluate = ComparisonTests.evaluate + + def test_same_observed_initial_has_no_improvement(self): + self.assertEqual(self.evaluate(observed_fixture())["reason"], "NoImprovement") + + def test_reference_mismatches_and_missing_scoring_refuse(self): + for key in ( + "opportunity_id", + "request_sha256", + "evidence_manifest_sha256", + "validator_sha256", + "scorer_sha256", + "label_sha256", + ): + e = observed_fixture() + e["candidate_rows"][0]["attempts"][0][key] = "0" * 64 + with self.subTest(key=key): + self.assertEqual(self.evaluate(e)["reason"], "Incomparable") + for key in ("response_sha256", "scorer_sha256", "label_sha256"): + e = observed_fixture() + e["candidate_rows"][0]["attempts"][0]["scoring"][key] = "0" * 64 + with self.subTest(key=key): + self.assertEqual(self.evaluate(e)["reason"], "Incomparable") + e = observed_fixture() + e["candidate_rows"][0]["attempts"][0]["scoring"] = None + self.assertEqual(self.evaluate(e)["reason"], "Incomplete") + e = observed_fixture() + e["candidate_rows"][0]["attempts"] = [] + self.assertEqual(self.evaluate(e)["reason"], "Incomplete") + + def test_false_correctness_usage_and_timing_cannot_fabricate_success(self): + for key, value in [ + ("correct", False), + ("charged", 3), + ("completed_ms", 99), + ("reserved_bound", 1), + ]: + e = observed_fixture() + e["candidate_rows"][0][key] = value + with self.subTest(key=key), self.assertRaises(ValueError): + self.evaluate(e) + e = observed_fixture() + a = e["candidate_rows"][0]["attempts"][0] + a["execution_id"] = "independent" + a["scoring"]["normalized"] = list(b"no") + with self.assertRaises(ValueError): + self.evaluate(e) + e = observed_fixture() + a = e["candidate_rows"][0]["attempts"][0] + a["measured_input"] = 5 + a["execution_id"] = "independent" + with self.assertRaises(ValueError): + self.evaluate(e) + e = observed_fixture() + a = e["candidate_rows"][0]["attempts"][0] + a["start_ms"] = 101 + a["execution_id"] = "independent" + with self.assertRaises(ValueError): + self.evaluate(e) + + def test_shared_execution_cannot_contradict_but_equal_requests_may_be_independent( + self, + ): + e = observed_fixture() + r = e["candidate_rows"][0] + a = r["attempts"][0] + a["scoring"]["normalized"] = list(b"no") + r.update(correct=False, unsafe_acceptance=True) + self.assertEqual(self.evaluate(e)["reason"], "Incomparable") + a["execution_id"] = "independent" + self.assertEqual(self.evaluate(e)["reason"], "NoImprovement") + + def test_unknown_row_fields_remain_incomplete_with_known_observations(self): + for key in ("correct", "eligible", "charged", "completed_ms"): + e = observed_fixture() + e["candidate_rows"][0][key] = None + with self.subTest(key=key): + self.assertEqual(self.evaluate(e)["reason"], "Incomplete") + + def test_unknown_usage_retains_quote_and_never_qualifies(self): + e = observed_fixture() + r = e["candidate_rows"][0] + a = r["attempts"][0] + a.update( + execution_id="uncertain", + disposition="Uncertain", + measured_input=None, + measured_output=None, + retained_bound=10, + response=None, + scoring=None, + completed_ms=None, + ) + r.update( + outcome="Unknown", + correct=None, + charged=None, + reserved_bound=10, + completed_ms=None, + ) + self.assertEqual(self.evaluate(e)["reason"], "Incomplete") + a["retained_bound"] = 9 + with self.assertRaises(ValueError): + self.evaluate(e) + + def test_only_invalid_output_unlocks_retry_and_quotes_gate_before_outcome(self): + e = observed_fixture() + r = e["candidate_rows"][0] + first = r["attempts"][0] + first.update(execution_id="first", disposition="InvalidOutput", completed_ms=20) + m = json.loads(bytes(e["experiment_manifest"])) + o = m["cases"][0]["policy_opportunities"][1]["opportunities"][1] + second = copy.deepcopy(first) + second.update( + execution_id="second", + opportunity_id=o["opportunity_id"], + request_sha256=o["request"]["sha256"], + start_ms=20, + completed_ms=100, + disposition="Accepted", + ) + r.update(charged=8, attempts=[first, second]) + self.assertEqual(self.evaluate(e)["reason"], "NoImprovement") + for disposition in ("Accepted", "Stale", "Failed", "Uncertain"): + bad = copy.deepcopy(e) + bad["candidate_rows"][0]["attempts"][0]["disposition"] = disposition + with self.subTest(disposition=disposition), self.assertRaises(ValueError): + self.evaluate(bad) + bad = copy.deepcopy(e) + manifest_edit(bad, lambda m: m["cases"][0].update(budget=13)) + with self.assertRaises(ValueError): + self.evaluate(bad) + bad = copy.deepcopy(e) + bad["candidate_rows"][0]["attempts"] = bad["candidate_rows"][0]["attempts"][:1] + self.assertEqual(self.evaluate(bad)["reason"], "Incomplete") + + def test_baseline_projection_binds_exact_frozen_universe(self): + from policy_evaluation_v2 import paired_baseline_matches + + e = fixture() + b = copy.deepcopy(e) + b["evaluation_id"] = "baseline-only-receipt" + self.assertTrue(paired_baseline_matches(b, e)) + for key in ("family_id", "split_id", "case_id"): + b = copy.deepcopy(e) + manifest_edit(b, lambda m, k=key: m["cases"][0].update({k: "other"})) + self.assertFalse(paired_baseline_matches(b, e)) + b = copy.deepcopy(e) + b["baseline_rows"][0]["charged"] = 3 + self.assertFalse(paired_baseline_matches(b, e)) + + +def legacy_comparison(): + from recovery_policy_eval import evaluate, synthetic_fixture + + old = synthetic_fixture() + out = evaluate(old) + e = fixture() + m = json.loads(bytes(e["experiment_manifest"])) + template = copy.deepcopy(m["cases"][0]) + m["cases"] = [] + m["accounting_unit"] = "model_tokens" + m["workload_sha256"] = hashlib.sha256(pack(old)).hexdigest() + for case in old["cases"]: + c = copy.deepcopy(template) + c.update( + case_id=case["case_id"], + family_id=case["family"], + split_id=case["split"], + budget=case["budget_tokens"], + deadline_ms=case["deadline_ms"], + ) + c["initial_evidence"] = record(case["initial_context"]["evidence_id"].encode()) + for p in c["policy_opportunities"]: + for i, o in enumerate(p["opportunities"]): + ctx = ( + case["initial_context"] + if i == 0 + else case["retries"][p["policy_name"]]["context"] + ) + o.update( + request=record(ctx["request"].encode()), + evidence_manifest=record(ctx["evidence_id"].encode()), + admitted=ctx["admitted"], + consistent=ctx["consistent"], + input_bound=ctx["input_tokens"], + output_cap=ctx["output_cap"], + ) + m["cases"].append(c) + e["baseline_rows"] = [] + e["candidate_rows"] = [] + for side, policy in [ + ("baseline_rows", "always_stop"), + ("candidate_rows", "affordable_retry"), + ]: + for row in out["cases"]: + if row["policy"] != policy: + continue + e[side].append( + { + "case_id": row["case_id"], + "outcome": "Refused" if row["abstained"] else "Accepted", + "correct": row["correct"], + "eligible": True, + "answerable": True, + "completed_ms": row["elapsed_ms"], + "charged": row["charged_tokens"], + "reserved_bound": row["reserved_tokens"], + "false_refusal": row["abstained"], + "unsafe_acceptance": False, + "deadline_miss": not row["on_time"], + "simulated": True, + "attempts": [], + } + ) + raw = pack(m) + e.update( + experiment_manifest=list(raw), experiment_sha256=hashlib.sha256(raw).hexdigest() + ) + return old, e + + +class ConversionTests(unittest.TestCase): + def test_legacy_without_complete_supplied_facts_is_explicitly_incomplete(self): + from policy_evaluation_v2 import convert_legacy + from recovery_policy_eval import synthetic_fixture + + for version in (1, 2): + old = synthetic_fixture() + old["schema_version"] = version + result = convert_legacy(old) + self.assertEqual( + (result["format"], result["status"]), + ("policy-evaluation-conversion/2", "Incomplete"), + ) + self.assertNotIn("evaluation", result) + + def test_matching_frozen_comparison_preserves_no_improvement_and_old_rows(self): + from policy_evaluation_v2 import convert_legacy + + old, e = legacy_comparison() + before = copy.deepcopy(old) + result = convert_legacy(old, e) + self.assertEqual(result["status"], "Complete") + self.assertEqual(result["selection"]["reason"], "IncreasedCost") + self.assertEqual(old, before) + bad = copy.deepcopy(old) + bad["provenance"] = "different source" + self.assertEqual(convert_legacy(bad, e)["status"], "Incomplete") + bad = copy.deepcopy(e) + bad["candidate_rows"][0]["charged"] = 6 + self.assertEqual(convert_legacy(old, bad)["status"], "Incomplete") + + +class AdditionalBoundaries(unittest.TestCase): + evaluate = ComparisonTests.evaluate + + def test_each_nonincrease_gate_independently(self): + for flag, reason in [ + ("false_refusal", "IncreasedFalseRefusal"), + ("unsafe_acceptance", "IncreasedUnsafeAcceptance"), + ("deadline_miss", "IncreasedDeadlineMisses"), + ]: + e = fixture() + + def multiple(m): + c = m["cases"][0] + m["cases"] = [copy.deepcopy(c) for _ in range(3)] + for i, c in enumerate(m["cases"]): + c["case_id"] = str(i) + + manifest_edit(e, multiple) + for side in ("baseline_rows", "candidate_rows"): + row = e[side][0] + e[side] = [copy.deepcopy(row) for _ in range(3)] + for i, r in enumerate(e[side]): + r.update( + case_id=str(i), + outcome="Failed", + correct=False, + false_refusal=False, + ) + for i in (0, 1): + e["candidate_rows"][i].update(outcome="Accepted", correct=True) + r = e["candidate_rows"][2] + if flag == "false_refusal": + r.update(outcome="Refused", false_refusal=True) + if flag == "unsafe_acceptance": + r.update(outcome="Accepted", unsafe_acceptance=True) + if flag == "deadline_miss": + r.update(completed_ms=101, deadline_miss=True) + with self.subTest(flag=flag): + self.assertEqual(self.evaluate(e)["reason"], reason) + + def test_aggregate_overflow_is_error(self): + e = fixture() + + def multiple(m): + m["cases"] = [copy.deepcopy(m["cases"][0]) for _ in range(2)] + for i, c in enumerate(m["cases"]): + c.update(case_id=str(i), budget=(1 << 64) - 1) + + manifest_edit(e, multiple) + for side in ("baseline_rows", "candidate_rows"): + e[side] = [copy.deepcopy(e[side][0]) for _ in range(2)] + for i, r in enumerate(e[side]): + r.update(case_id=str(i), charged=1 << 63) + with self.assertRaises(ValueError): + self.evaluate(e) + + def test_family_split_leakage_and_duplicate_case_are_invalid(self): + for duplicate in (True, False): + e = fixture() + + def edit(m, duplicate=duplicate): + c = copy.deepcopy(m["cases"][0]) + c["split_id"] = "calibration" + if not duplicate: + c["case_id"] = "c2" + m["cases"].append(c) + + manifest_edit(e, edit) + with self.assertRaises(ValueError): + self.evaluate(e) + + +class StrictUnknownTests(unittest.TestCase): + evaluate = ComparisonTests.evaluate + + def test_unknown_does_not_hide_known_false_refusal_flags(self): + e = fixture() + r = e["baseline_rows"][0] + r["charged"] = None + r["false_refusal"] = False + with self.assertRaises(ValueError): + self.evaluate(e) + + def test_same_execution_cannot_count_for_two_scheduled_cases(self): + e = observed_fixture() + + def duplicate(m): + c = copy.deepcopy(m["cases"][0]) + c["case_id"] = "c2" + m["cases"].append(c) + + manifest_edit(e, duplicate) + for side in ("baseline_rows", "candidate_rows"): + r = copy.deepcopy(e[side][0]) + r["case_id"] = "c2" + e[side].append(r) + self.assertEqual(self.evaluate(e)["reason"], "Incomparable") + + def test_unknown_first_case_cannot_hide_fabricated_later_row(self): + e = fixture() + + def duplicate(m): + c = copy.deepcopy(m["cases"][0]) + c["case_id"] = "c2" + m["cases"].append(c) + + manifest_edit(e, duplicate) + for side in ("baseline_rows", "candidate_rows"): + r = copy.deepcopy(e[side][0]) + r["case_id"] = "c2" + e[side].append(r) + e["baseline_rows"][0]["charged"] = None + e["baseline_rows"][1]["false_refusal"] = False + with self.assertRaises(ValueError): + self.evaluate(e) + + +class EncodingTests(unittest.TestCase): + def test_canonical_limit_counts_json_escapes_exactly(self): + from unittest.mock import patch + + from policy_evaluation_v2 import canonical + + expected = b'{"text":"\\n"}' + with patch("policy_evaluation_v2.MAX_BYTES", len(expected)): + self.assertEqual(canonical({"text": "\n"}), expected) + with ( + patch("policy_evaluation_v2.MAX_BYTES", len(expected) - 1), + self.assertRaises(ValueError), + ): + canonical({"text": "\n"}) + + def test_deep_json_and_lone_surrogates_are_rejected(self): + from policy_evaluation_v2 import loads, validate_policy_artifact + + with self.assertRaises(ValueError): + loads(b"[" * 65 + b"0" + b"]" * 65) + raw = ( + b'{"schema_version":1,"policy_name":"always_stop","purpose":"\\ud800","compatibility_contract_sha256":"' + + b"a" * 64 + + b'"}' + ) + with self.assertRaises(ValueError): + validate_policy_artifact(raw) + + +def golden_cases(): + cases = [ + ("synthetic-positive", fixture(), "Eligible"), + ("observed-equal", observed_fixture(), "NoImprovement"), + ] + + def changed(name, change, want, observed=False): + e = observed_fixture() if observed else fixture() + change(e) + cases.append((name, e, want)) + + for key in ( + "actor", + "scope", + "purpose", + "evaluator_version", + "evaluator_sha256", + "baseline_sha256", + "candidate_sha256", + "experiment_sha256", + ): + changed(key, lambda e, k=key: e.update({k: "0" * 64}), "Incomparable") + for key in ("correct", "eligible", "charged", "completed_ms"): + changed( + "unknown-" + key, + lambda e, k=key: e["candidate_rows"][0].update({k: None}), + "Incomplete", + ) + for key in ("correct", "eligible", "charged", "completed_ms"): + changed( + "observed-unknown-" + key, + lambda e, k=key: e["candidate_rows"][0].update({k: None}), + "Incomplete", + True, + ) + for key in ( + "false_refusal", + "unsafe_acceptance", + "deadline_miss", + "answerable", + "eligible", + ): + changed( + "fabricated-" + key, + lambda e, k=key: e["candidate_rows"][0].update( + {k: not e["candidate_rows"][0][k]} + ), + "Error", + ) + for unit in ("logical_support", "unknown"): + changed( + "unit-" + unit, + lambda e, u=unit: manifest_edit(e, lambda m: m.update(accounting_unit=u)), + "Incomparable", + ) + changed( + "over-u64", lambda e: e["candidate_rows"][0].update(charged=1 << 64), "Error" + ) + changed( + "boolean-count", lambda e: e["candidate_rows"][0].update(charged=True), "Error" + ) + changed( + "overflow", + lambda e: e["candidate_rows"][0].update( + charged=(1 << 64) - 1, reserved_bound=1 + ), + "Error", + ) + changed("missing-cases", lambda e: e.update(candidate_rows=[]), "Incomplete") + changed( + "wrong-case", + lambda e: e["candidate_rows"][0].update(case_id="wrong"), + "Incomparable", + ) + changed("missing-field", lambda e: e["candidate_rows"][0].pop("charged"), "Error") + changed("extra-field", lambda e: e["candidate_rows"][0].update(total=0), "Error") + changed("cost", lambda e: e["candidate_rows"][0].update(charged=5), "IncreasedCost") + changed( + "late", + lambda e: e["candidate_rows"][0].update(completed_ms=101, deadline_miss=True), + "NoImprovement", + ) + for key in ( + "request_sha256", + "opportunity_id", + "evidence_manifest_sha256", + "validator_sha256", + "scorer_sha256", + "label_sha256", + ): + changed( + "observed-" + key, + lambda e, k=key: e["candidate_rows"][0]["attempts"][0].update( + {k: "0" * 64} + ), + "Incomparable", + True, + ) + changed( + "missing-scoring", + lambda e: e["candidate_rows"][0]["attempts"][0].update(scoring=None), + "Incomplete", + True, + ) + changed( + "observed-cost", + lambda e: e["candidate_rows"][0].update(charged=3), + "Error", + True, + ) + changed( + "unknown-hidden-flag", + lambda e: e["baseline_rows"][0].update(charged=None, false_refusal=False), + "Error", + ) + for kind, reason in [ + ("false_refusal", "IncreasedFalseRefusal"), + ("unsafe_acceptance", "IncreasedUnsafeAcceptance"), + ("deadline_miss", "IncreasedDeadlineMisses"), + ]: + e = fixture() + + def multiple(m): + m["cases"] = [copy.deepcopy(m["cases"][0]) for _ in range(3)] + for i, c in enumerate(m["cases"]): + c["case_id"] = str(i) + + manifest_edit(e, multiple) + for side in ("baseline_rows", "candidate_rows"): + e[side] = [copy.deepcopy(e[side][0]) for _ in range(3)] + for i, r in enumerate(e[side]): + r.update( + case_id=str(i), outcome="Failed", correct=False, false_refusal=False + ) + for i in (0, 1): + e["candidate_rows"][i].update(outcome="Accepted", correct=True) + r = e["candidate_rows"][2] + if kind == "false_refusal": + r.update(outcome="Refused", false_refusal=True) + if kind == "unsafe_acceptance": + r.update(outcome="Accepted", unsafe_acceptance=True) + if kind == "deadline_miss": + r.update(completed_ms=101, deadline_miss=True) + cases.append((kind, e, reason)) + return cases + + +class GoldenTests(unittest.TestCase): + def test_frozen_golden_bytes_and_decisions(self): + from policy_evaluation_v2 import ( + evaluate_policy_comparison, + evaluation_sha256, + parse_policy_evaluation, + ) + + e = fixture() + self.assertEqual( + evaluation_sha256(e), + "2b0ed900f956cd8fc2dd215e114a665d4799187dd6610209807fe557a1311a88", + ) + self.assertEqual( + e["experiment_sha256"], + "1373dd5ff2bc8e1a06840f8feb31329b1a6666d434712db2a6fe4a3ef537c265", + ) + for name, value, want in golden_cases(): + with self.subTest(name=name): + try: + result = evaluate_policy_comparison( + parse_policy_evaluation(pack(value)) + ) + actual = result.get("reason", result["decision"]) + except ValueError: + actual = "Error" + self.assertEqual(actual, want) + + +class CandidateTests(unittest.TestCase): + def test_candidate_requires_exact_artifact_digest_purpose_and_identity(self): + from policy_evaluation_v2 import validate_policy_candidate + + e = fixture() + m = json.loads(bytes(e["experiment_manifest"])) + a = m["policies"][0]["artifact"] + c = { + "version_id": "v1", + "actor": e["actor"], + "scope": e["scope"], + "purpose": e["purpose"], + "artifact": a["bytes"], + "sha256": a["sha256"], + } + self.assertEqual(validate_policy_candidate(c)["policy_name"], "always_stop") + for key, value in [ + ("actor", " "), + ("version_id", ""), + ("purpose", "other"), + ("sha256", "0" * 64), + ]: + bad = copy.deepcopy(c) + bad[key] = value + with self.subTest(key=key), self.assertRaises(ValueError): + validate_policy_candidate(bad) + + +if __name__ == "__main__": + unittest.main() diff --git a/scripts/test_recovery_policy_eval.py b/scripts/test_recovery_policy_eval.py index 866b21e..0b06644 100644 --- a/scripts/test_recovery_policy_eval.py +++ b/scripts/test_recovery_policy_eval.py @@ -278,5 +278,20 @@ def test_v2_ordinary_replay_matches_v1_scores(self): self.assertEqual(original['summary'], newer['summary']) +class V2ConversionModeTests(unittest.TestCase): + def test_explicit_conversion_does_not_change_default_output(self): + path = Path(__file__).with_name('recovery_policy_eval.py') + default = subprocess.run([sys.executable, str(path), '--fixture'], + capture_output=True, text=True, check=True) + self.assertEqual(json.loads(default.stdout), evaluate(synthetic_fixture())) + converted = subprocess.run([sys.executable, str(path), '--fixture', '--convert-v2'], + capture_output=True, text=True) + self.assertEqual(converted.returncode, 0, converted.stderr) + result = json.loads(converted.stdout) + self.assertEqual(result['format'], 'policy-evaluation-conversion/2') + self.assertEqual(result['status'], 'Incomplete') + self.assertEqual(json.loads(default.stdout)['policy_version'], 'frozen-v1') + + if __name__ == '__main__': unittest.main() diff --git a/src/lib.rs b/src/lib.rs index 0944003..76261c7 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1,6 +1,7 @@ //! Local versioned evidence memory with actor-scoped admission and atomic publication. mod evidence_digest; +pub mod policy_registry; pub mod incident_protocol; pub mod decision_bundle; diff --git a/src/policy_registry.rs b/src/policy_registry.rs new file mode 100644 index 0000000..451f914 --- /dev/null +++ b/src/policy_registry.rs @@ -0,0 +1,104 @@ +//! Pure policy attestations. Validation does not authenticate execution or activate a Store. +mod evaluation; +mod types; +use crate::evidence_digest::sha256_hex; +pub use evaluation::*; +use serde::{Deserialize, Serialize}; +use std::io::{self, Write}; +pub use types::*; +pub const MAX_ARTIFACT_BYTES: usize = 64 * 1024; +pub const MAX_EVALUATION_BYTES: usize = 16 * 1024 * 1024; +pub const MAX_CASES: usize = 10_000; +fn bad(message: &str) -> rusqlite::Error { + crate::invalid(message) +} +fn require(ok: bool, message: &str) -> rusqlite::Result<()> { + if ok { + Ok(()) + } else { + Err(bad(message)) + } +} +fn digest(s: &str) -> bool { + s.len() == 64 + && s.bytes() + .all(|b| b.is_ascii_digit() || (b'a'..=b'f').contains(&b)) +} +fn nonempty(s: &str) -> bool { + !s.trim().is_empty() +} +fn parse(bytes: &[u8], limit: usize) -> rusqlite::Result { + require(bytes.len() <= limit, "policy record size limit")?; + let (mut depth, mut quoted, mut escaped) = (0usize, false, false); + for &b in bytes { + if quoted { + if escaped { + escaped = false; + } else if b == b'\\' { + escaped = true; + } else if b == b'"' { + quoted = false; + } + } else { + match b { + b'"' => quoted = true, + b'[' | b'{' => { + depth += 1; + require(depth <= 64, "policy JSON depth limit")?; + } + b']' | b'}' => { + depth = depth + .checked_sub(1) + .ok_or_else(|| bad("malformed policy record"))?; + } + _ => {} + } + } + } + serde_json::from_slice(bytes).map_err(|_| bad("malformed policy record")) +} +struct SizeWriter(usize); +impl Write for SizeWriter { + fn write(&mut self, bytes: &[u8]) -> io::Result { + self.0 = self + .0 + .checked_add(bytes.len()) + .filter(|n| *n <= MAX_EVALUATION_BYTES) + .ok_or_else(|| io::Error::other("policy record size limit"))?; + Ok(bytes.len()) + } + fn flush(&mut self) -> io::Result<()> { + Ok(()) + } +} +fn bounded(v: &T) -> rusqlite::Result<()> { + serde_json::to_writer(SizeWriter(0), v).map_err(|_| bad("policy record size limit")) +} +fn nullable<'de, D: serde::Deserializer<'de>, T: Deserialize<'de>>( + d: D, +) -> Result, D::Error> { + Option::deserialize(d) +} +pub fn validate_policy_artifact(bytes: &[u8]) -> rusqlite::Result { + let a: PolicyArtifactV1 = parse(bytes, MAX_ARTIFACT_BYTES)?; + require( + a.schema_version == 1 && nonempty(&a.purpose) && digest(&a.compatibility_contract_sha256), + "invalid policy artifact", + )?; + Ok(a) +} +pub fn validate_policy_candidate(c: &PolicyCandidate) -> rusqlite::Result { + bounded(c)?; + require( + [&c.version_id, &c.actor, &c.scope, &c.purpose] + .iter() + .all(|s| nonempty(s)), + "empty candidate identity", + )?; + let a = validate_policy_artifact(&c.artifact)?; + require( + c.purpose == a.purpose && digest(&c.sha256) && sha256_hex(&c.artifact) == c.sha256, + "candidate identity mismatch", + )?; + Ok(a) +} diff --git a/src/policy_registry/evaluation.rs b/src/policy_registry/evaluation.rs new file mode 100644 index 0000000..0087724 --- /dev/null +++ b/src/policy_registry/evaluation.rs @@ -0,0 +1,641 @@ +use super::*; +use std::collections::{HashMap, HashSet}; +pub const SELECTION_CONTRACT: &[u8] = b"policy-selection/1\ncomplete-paired-frozen-attestations;success=strict-increase;false-refusal,unsafe-acceptance,charged-plus-reserved,deadline-miss=nonincrease;synthetic=fixture/recovery-policy-only\n"; +pub const SELECTION_CONTRACT_SHA256: &str = + "df97ae41b2648bef15a11bf1950883597b057df1940fe43347d485fedf8590b3"; +fn add(a: u64, b: u64) -> rusqlite::Result { + a.checked_add(b) + .ok_or_else(|| bad("policy accounting overflow")) +} +fn record(r: &DigestedRecord) -> rusqlite::Result<()> { + require( + r.bytes.len() <= MAX_EVALUATION_BYTES + && digest(&r.sha256) + && sha256_hex(&r.bytes) == r.sha256, + "record digest mismatch", + ) +} +fn shape(e: &PolicyEvaluation) -> rusqlite::Result<()> { + require( + e.baseline_rows.len() <= MAX_CASES && e.candidate_rows.len() <= MAX_CASES, + "case limit", + )?; + require( + e.experiment_manifest.len() <= MAX_EVALUATION_BYTES, + "manifest size limit", + )?; + bounded(e)?; + require( + [ + &e.evaluation_id, + &e.actor, + &e.scope, + &e.purpose, + &e.baseline_id, + &e.candidate_id, + &e.evaluator_version, + ] + .iter() + .all(|s| nonempty(s)), + "empty evaluation identity", + )?; + require( + [ + &e.baseline_sha256, + &e.candidate_sha256, + &e.experiment_sha256, + &e.evaluator_sha256, + ] + .iter() + .all(|s| digest(s)), + "invalid evaluation digest", + )?; + for row in e.baseline_rows.iter().chain(&e.candidate_rows) { + require( + nonempty(&row.case_id) && row.attempts.len() <= 2, + "invalid row identity or attempt count", + )?; + if let Some(c) = row.charged { + add(c, row.reserved_bound)?; + } + for a in &row.attempts { + require( + nonempty(&a.execution_id) && nonempty(&a.opportunity_id), + "empty attempt identity", + )?; + require( + [ + &a.request_sha256, + &a.evidence_manifest_sha256, + &a.validator_sha256, + &a.scorer_sha256, + &a.label_sha256, + ] + .iter() + .all(|s| digest(s)), + "invalid attempt digest", + )?; + if let Some(r) = &a.response { + record(r)?; + } + if let Some(s) = &a.scoring { + require( + [&s.response_sha256, &s.scorer_sha256, &s.label_sha256] + .iter() + .all(|v| digest(v)), + "invalid scoring digest", + )?; + } + } + } + Ok(()) +} +pub fn parse_policy_evaluation(bytes: &[u8]) -> rusqlite::Result { + let e = parse(bytes, MAX_EVALUATION_BYTES)?; + shape(&e)?; + Ok(e) +} +/// Digest of domain prefix plus sorted-key, compact UTF-8 JSON; exact byte payloads stay arrays. +pub fn evaluation_sha256(e: &PolicyEvaluation) -> rusqlite::Result { + shape(e)?; + let value = serde_json::to_value(e).map_err(|_| bad("evaluation encoding"))?; + let mut raw = b"policy-evaluation/2\n".to_vec(); + raw.extend(serde_json::to_vec(&value).map_err(|_| bad("evaluation encoding"))?); + Ok(sha256_hex(&raw)) +} +/// Exact frozen-baseline projection; does not imply either comparison is eligible. +pub fn paired_baseline_matches( + recorded: &PolicyEvaluation, + paired: &PolicyEvaluation, +) -> rusqlite::Result { + shape(recorded)?; + shape(paired)?; + Ok(recorded.actor == paired.actor + && recorded.scope == paired.scope + && recorded.purpose == paired.purpose + && recorded.baseline_id == paired.baseline_id + && recorded.baseline_sha256 == paired.baseline_sha256 + && recorded.experiment_manifest == paired.experiment_manifest + && recorded.experiment_sha256 == paired.experiment_sha256 + && recorded.evaluator_version == paired.evaluator_version + && recorded.evaluator_sha256 == paired.evaluator_sha256 + && recorded.provenance == paired.provenance + && recorded.baseline_rows == paired.baseline_rows) +} +fn manifest(e: &PolicyEvaluation) -> rusqlite::Result<(ComparisonManifest, Vec)> { + let m: ComparisonManifest = parse(&e.experiment_manifest, MAX_EVALUATION_BYTES)?; + require( + m.format == "policy-comparison/1" + && !m.cases.is_empty() + && m.cases.len() <= MAX_CASES + && !m.policies.is_empty() + && m.policies.len() <= 3, + "invalid manifest format or count", + )?; + require( + [&m.actor, &m.scope, &m.purpose].iter().all(|s| nonempty(s)), + "empty manifest identity", + )?; + require( + [ + &m.workload_sha256, + &m.prompt_sha256, + &m.runtime_sha256, + &m.model_sha256, + &m.tokenizer_sha256, + &m.validator_sha256, + &m.scorer_sha256, + &m.compatibility_contract_sha256, + ] + .iter() + .all(|s| digest(s)), + "invalid manifest digest", + )?; + let mut names = HashSet::new(); + let mut artifacts = HashSet::new(); + for p in &m.policies { + record(&p.artifact)?; + let a = validate_policy_artifact(&p.artifact.bytes)?; + require( + names.insert(p.policy_name) && artifacts.insert(&p.artifact.sha256), + "duplicate frozen policy", + )?; + require( + a.policy_name == p.policy_name + && a.purpose == m.purpose + && a.compatibility_contract_sha256 == m.compatibility_contract_sha256, + "frozen artifact identity mismatch", + )?; + } + let mut cases = HashSet::new(); + let mut families = HashMap::new(); + let mut labels = Vec::with_capacity(m.cases.len()); + for c in &m.cases { + require( + [&c.case_id, &c.family_id, &c.split_id] + .iter() + .all(|s| nonempty(s)) + && cases.insert(&c.case_id), + "invalid or duplicate case identity", + )?; + require( + c.timing_start_ms <= c.deadline_ms && c.deadline_ms <= c.timing_end_ms, + "invalid timing boundaries", + )?; + if let Some(split) = families.insert(&c.family_id, &c.split_id) { + require(split == &c.split_id, "family crosses splits")?; + } + record(&c.initial_evidence)?; + record(&c.label)?; + let l: FrozenLabel = parse(&c.label.bytes, MAX_ARTIFACT_BYTES)?; + require( + l.normalized_format == "opaque-bytes/1" && l.scorer_sha256 == m.scorer_sha256, + "invalid frozen label identity", + )?; + let mut policies = HashSet::new(); + require( + c.policy_opportunities.len() == names.len(), + "missing frozen policy opportunities", + )?; + for p in &c.policy_opportunities { + require( + names.contains(&p.policy_name) && policies.insert(p.policy_name), + "unknown or duplicate opportunity policy", + )?; + let expected = if p.policy_name == PolicyName::AlwaysStop { + 1 + } else { + 2 + }; + require( + p.opportunities.len() == expected, + "invalid frozen opportunity count", + )?; + let mut ids = HashSet::new(); + for (i, o) in p.opportunities.iter().enumerate() { + require( + nonempty(&o.opportunity_id) && ids.insert(&o.opportunity_id), + "duplicate opportunity identity", + )?; + record(&o.request)?; + record(&o.evidence_manifest)?; + add(o.input_bound, o.output_cap)?; + require( + o.start_min_ms >= c.timing_start_ms + && o.start_min_ms <= o.start_max_ms + && o.start_max_ms <= c.timing_end_ms, + "invalid opportunity timing", + )?; + if i == 0 { + require( + o.evidence_manifest == c.initial_evidence, + "initial evidence differs", + )?; + } + } + } + // All named policies start from the same frozen opportunity, excluding local ID. + let first = &c.policy_opportunities[0].opportunities[0]; + for p in &c.policy_opportunities[1..] { + let mut o = p.opportunities[0].clone(); + o.opportunity_id = first.opportunity_id.clone(); + require(o == *first, "different initial policy opportunities")?; + } + labels.push(l); + } + Ok((m, labels)) +} +fn refused(r: SelectionRefusal) -> SelectionDecision { + SelectionDecision::Refused(r) +} + +pub fn evaluate_policy_comparison(e: &PolicyEvaluation) -> rusqlite::Result { + shape(e)?; + let (m, labels) = manifest(e)?; + if e.actor != m.actor + || e.scope != m.scope + || e.purpose != m.purpose + || e.provenance != m.provenance + || sha256_hex(&e.experiment_manifest) != e.experiment_sha256 + || e.evaluator_version != "policy-selection/1" + || e.evaluator_sha256 != SELECTION_CONTRACT_SHA256 + || e.baseline_id == e.candidate_id + || !matches!( + m.accounting_unit, + AccountingUnit::ModelTokens | AccountingUnit::FixtureBytes + ) + || (e.provenance == Provenance::SyntheticRegistryMechanics + && e.purpose != "fixture/recovery-policy") + { + return Ok(refused(SelectionRefusal::Incomparable)); + } + let base = m + .policies + .iter() + .find(|p| p.artifact.sha256 == e.baseline_sha256); + let cand = m + .policies + .iter() + .find(|p| p.artifact.sha256 == e.candidate_sha256); + let (Some(base), Some(cand)) = (base, cand) else { + return Ok(refused(SelectionRefusal::Incomparable)); + }; + let mut executions = HashMap::new(); + let b = rows( + &m, + &labels, + &e.baseline_rows, + base.policy_name, + &mut executions, + )?; + let c = rows( + &m, + &labels, + &e.candidate_rows, + cand.policy_name, + &mut executions, + )?; + let (b, c) = match (b, c) { + (Ok(b), Ok(c)) => (b, c), + (Err(r), _) | (_, Err(r)) => return Ok(refused(r)), + }; + let reason = if c.success <= b.success { + Some(SelectionRefusal::NoImprovement) + } else if c.false_refusal > b.false_refusal { + Some(SelectionRefusal::IncreasedFalseRefusal) + } else if c.unsafe_acceptance > b.unsafe_acceptance { + Some(SelectionRefusal::IncreasedUnsafeAcceptance) + } else if c.cost > b.cost { + Some(SelectionRefusal::IncreasedCost) + } else if c.deadline_misses > b.deadline_misses { + Some(SelectionRefusal::IncreasedDeadlineMisses) + } else { + None + }; + Ok(match reason { + Some(r) => refused(r), + None => SelectionDecision::Eligible { + validated_counters: ValidatedCounters { + baseline: b, + candidate: c, + }, + }, + }) +} +type CheckedCounters = Result; +fn rows( + m: &ComparisonManifest, + labels: &[FrozenLabel], + rows: &[PolicyRow], + policy: PolicyName, + executions: &mut HashMap, +) -> rusqlite::Result { + let mut refusal = if rows.len() != m.cases.len() { + Some(SelectionRefusal::Incomplete) + } else { + None + }; + let mut counters = PolicyCounters::default(); + for ((r, c), l) in rows.iter().zip(&m.cases).zip(labels) { + if r.case_id != c.case_id { + refusal = Some(SelectionRefusal::Incomparable); + continue; + } + require(r.answerable == l.answerable, "fabricated answerability")?; + require( + r.false_refusal + == (r.outcome == Outcome::Refused && l.answerable && l.admissible_support), + "fabricated false refusal", + )?; + if r.outcome != Outcome::Accepted { + require(!r.unsafe_acceptance, "fabricated unsafe acceptance")?; + } + if let (Some(correct), Some(eligible)) = (r.correct, r.eligible) { + require( + r.unsafe_acceptance == (r.outcome == Outcome::Accepted && (!correct || !eligible)), + "fabricated unsafe acceptance", + )?; + } + if let Some(completed) = r.completed_ms { + require( + completed >= c.timing_start_ms + && completed <= c.timing_end_ms + && r.deadline_miss == (completed > c.deadline_ms), + "fabricated deadline miss", + )?; + } + if let Some(charged) = r.charged { + require( + add(charged, r.reserved_bound)? <= c.budget, + "row exceeds frozen budget", + )?; + } + if m.provenance == Provenance::SyntheticRegistryMechanics { + require( + r.simulated && r.attempts.is_empty(), + "synthetic rows require simulated counters only", + )?; + if let Some(eligible) = r.eligible { + require( + eligible == l.admissible_support, + "fabricated synthetic eligibility", + )?; + } + } else { + require(!r.simulated, "observed row cannot simulate counters")?; + if let Some(reason) = observed(m, c, l, r, policy, executions)? { + refusal.get_or_insert(reason); + continue; + } + } + let (Some(correct), Some(eligible), Some(completed), Some(charged)) = + (r.correct, r.eligible, r.completed_ms, r.charged) + else { + refusal.get_or_insert(SelectionRefusal::Incomplete); + continue; + }; + if r.outcome == Outcome::Unknown { + refusal.get_or_insert(SelectionRefusal::Incomplete); + continue; + } + require( + completed >= c.timing_start_ms && completed <= c.timing_end_ms, + "row outside timing window", + )?; + require( + r.false_refusal + == (r.outcome == Outcome::Refused && l.answerable && l.admissible_support), + "fabricated false refusal", + )?; + require( + r.unsafe_acceptance == (r.outcome == Outcome::Accepted && (!correct || !eligible)), + "fabricated unsafe acceptance", + )?; + require( + r.deadline_miss == (completed > c.deadline_ms), + "fabricated deadline miss", + )?; + let cost = add(charged, r.reserved_bound)?; + require(cost <= c.budget, "row exceeds frozen budget")?; + counters.cases = add(counters.cases, 1)?; + counters.success = add( + counters.success, + u64::from(r.outcome == Outcome::Accepted && correct && eligible && !r.deadline_miss), + )?; + counters.false_refusal = add(counters.false_refusal, r.false_refusal as u64)?; + counters.unsafe_acceptance = add(counters.unsafe_acceptance, r.unsafe_acceptance as u64)?; + counters.deadline_misses = add(counters.deadline_misses, r.deadline_miss as u64)?; + counters.cost = add(counters.cost, cost)?; + } + Ok(match refusal { + Some(r) => Err(r), + None => Ok(counters), + }) +} +fn observed( + m: &ComparisonManifest, + c: &FrozenCase, + l: &FrozenLabel, + r: &PolicyRow, + policy: PolicyName, + executions: &mut HashMap, +) -> rusqlite::Result> { + let ops = &c + .policy_opportunities + .iter() + .find(|p| p.policy_name == policy) + .unwrap() + .opportunities; + if r.attempts.is_empty() { + return Ok(Some(SelectionRefusal::Incomplete)); + } + require(r.attempts.len() <= ops.len(), "too many observed attempts")?; + let mut charged = 0; + let mut previous_end = c.timing_start_ms; + let mut result = ( + Outcome::Refused, + false, + l.admissible_support, + c.timing_start_ms, + ); + let mut ids = HashSet::new(); + for (i, a) in r.attempts.iter().enumerate() { + let o = &ops[i]; + if a.opportunity_id != o.opportunity_id + || a.request_sha256 != o.request.sha256 + || a.evidence_manifest_sha256 != o.evidence_manifest.sha256 + || a.validator_sha256 != m.validator_sha256 + || a.scorer_sha256 != m.scorer_sha256 + || a.label_sha256 != c.label.sha256 + { + return Ok(Some(SelectionRefusal::Incomparable)); + } + require( + ids.insert(&a.execution_id), + "reused execution in one policy path", + )?; + if i > 0 { + require( + r.attempts[i - 1].execution == Execution::Executed + && r.attempts[i - 1].disposition == AttemptDisposition::InvalidOutput, + "only settled invalid output unlocks retry", + )?; + } + require( + a.start_ms >= previous_end + && a.start_ms >= o.start_min_ms + && a.start_ms <= o.start_max_ms, + "attempt outside timing window", + )?; + let quote = add(o.input_bound, o.output_cap)?; + let gate = o.admitted + && !(i > 0 && policy == PolicyName::EvidenceGatedRetry && !o.consistent) + && a.start_ms < c.deadline_ms + && quote + <= c.budget + .checked_sub(charged) + .ok_or_else(|| bad("budget exceeded"))?; + if a.execution == Execution::Unknown { + require( + a.measured_input.is_none() + && a.measured_output.is_none() + && a.response.is_none() + && a.scoring.is_none() + && a.retained_bound == quote + && a.disposition == AttemptDisposition::Uncertain + && gate + && i + 1 == r.attempts.len(), + "invalid unknown execution", + )?; + require( + r.charged.is_none() && r.reserved_bound == quote && r.outcome == Outcome::Unknown, + "unknown row accounting", + )?; + return Ok(Some(SelectionRefusal::Incomplete)); + } + if a.execution == Execution::NotExecuted { + require( + !gate + && a.disposition == AttemptDisposition::Refused + && a.measured_input == Some(0) + && a.measured_output == Some(0) + && a.retained_bound == 0 + && a.response.is_none() + && a.scoring.is_none() + && a.completed_ms == Some(a.start_ms) + && i + 1 == r.attempts.len(), + "invalid nonexecution attestation", + )?; + result = ( + Outcome::Refused, + false, + o.admitted && l.admissible_support, + a.start_ms, + ); + } else { + require(gate, "execution violates frozen admission")?; + if let Some(v) = a.measured_input { + require(v <= o.input_bound, "input exceeds quote")?; + } + if let Some(v) = a.measured_output { + require(v <= o.output_cap, "output exceeds cap")?; + } + let (Some(input), Some(output)) = (a.measured_input, a.measured_output) else { + require( + a.retained_bound == quote + && r.charged.is_none() + && r.reserved_bound == quote + && r.outcome == Outcome::Unknown, + "unknown usage must retain full bound", + )?; + require( + i + 1 == r.attempts.len(), + "unknown usage cannot unlock retry", + )?; + return Ok(Some(SelectionRefusal::Incomplete)); + }; + require( + a.retained_bound == 0 && a.disposition != AttemptDisposition::Uncertain, + "settled usage has retained hold or uncertain disposition", + )?; + charged = add(charged, add(input, output)?)?; + let Some(end) = a.completed_ms else { + return Ok(Some(SelectionRefusal::Incomplete)); + }; + require( + end >= a.start_ms && end <= c.timing_end_ms, + "invalid completion time", + )?; + previous_end = end; + let correct = if let Some(response) = &a.response { + let Some(scoring) = &a.scoring else { + return Ok(Some(SelectionRefusal::Incomplete)); + }; + if scoring.response_sha256 != response.sha256 + || scoring.scorer_sha256 != m.scorer_sha256 + || scoring.label_sha256 != c.label.sha256 + { + return Ok(Some(SelectionRefusal::Incomparable)); + } + scoring.normalized == l.expected_normalized + } else { + require(a.scoring.is_none(), "scoring without response")?; + if a.disposition != AttemptDisposition::Failed { + return Ok(Some(SelectionRefusal::Incomplete)); + } + false + }; + let outcome = match a.disposition { + AttemptDisposition::Accepted => Outcome::Accepted, + AttemptDisposition::Failed => Outcome::Failed, + _ => Outcome::Refused, + }; + result = ( + outcome, + correct, + o.admitted && l.admissible_support && a.disposition != AttemptDisposition::Stale, + end, + ); + if a.disposition == AttemptDisposition::InvalidOutput + && i + 1 < ops.len() + && i + 1 == r.attempts.len() + { + return Ok(Some(SelectionRefusal::Incomplete)); + } + if a.disposition != AttemptDisposition::InvalidOutput { + require( + i + 1 == r.attempts.len(), + "terminal disposition cannot unlock retry", + )?; + } + } + // Opportunity IDs are local to a policy path; execution IDs identify realizations. + let mut observation = a.clone(); + observation.opportunity_id.clear(); + if let Some(prior) = executions.get(&a.execution_id) { + if *prior != (c.case_id.clone(), observation.clone()) { + return Ok(Some(SelectionRefusal::Incomparable)); + } + } else { + executions.insert(a.execution_id.clone(), (c.case_id.clone(), observation)); + } + } + require( + (r.outcome == Outcome::Unknown || r.outcome == result.0) + && r.correct.is_none_or(|v| v == result.1) + && r.eligible.is_none_or(|v| v == result.2) + && r.completed_ms.is_none_or(|v| v == result.3) + && r.charged.is_none_or(|v| v == charged) + && r.reserved_bound == 0, + "fabricated observed row totals", + )?; + if r.outcome == Outcome::Unknown + || r.correct.is_none() + || r.eligible.is_none() + || r.completed_ms.is_none() + || r.charged.is_none() + { + return Ok(Some(SelectionRefusal::Incomplete)); + } + Ok(None) +} diff --git a/src/policy_registry/types.rs b/src/policy_registry/types.rs new file mode 100644 index 0000000..8931a1d --- /dev/null +++ b/src/policy_registry/types.rs @@ -0,0 +1,243 @@ +use serde::{Deserialize, Serialize}; +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum PolicyName { + AlwaysStop, + AffordableRetry, + EvidenceGatedRetry, +} +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct PolicyArtifactV1 { + pub schema_version: u32, + pub policy_name: PolicyName, + pub purpose: String, + pub compatibility_contract_sha256: String, +} +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct PolicyCandidate { + pub version_id: String, + pub actor: String, + pub scope: String, + pub purpose: String, + pub artifact: Vec, + pub sha256: String, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum Provenance { + SyntheticRegistryMechanics, + ObservedPolicyReplay, +} +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum AccountingUnit { + ModelTokens, + FixtureBytes, + LogicalSupport, + Unknown, +} +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub enum Outcome { + Accepted, + Refused, + Failed, + Unknown, +} +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub enum Execution { + Executed, + NotExecuted, + Unknown, +} +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub enum AttemptDisposition { + Accepted, + InvalidOutput, + Stale, + Failed, + Uncertain, + Refused, +} +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct DigestedRecord { + pub bytes: Vec, + pub sha256: String, +} +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct FrozenPolicy { + pub policy_name: PolicyName, + pub artifact: DigestedRecord, +} +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct FrozenLabel { + pub answerable: bool, + pub admissible_support: bool, + pub normalized_format: String, + pub scorer_sha256: String, + pub expected_normalized: Vec, +} +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct RequestOpportunity { + pub opportunity_id: String, + pub request: DigestedRecord, + pub evidence_manifest: DigestedRecord, + pub admitted: bool, + pub consistent: bool, + pub input_bound: u64, + pub output_cap: u64, + pub start_min_ms: u64, + pub start_max_ms: u64, +} +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct PolicyOpportunities { + pub policy_name: PolicyName, + pub opportunities: Vec, +} +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct FrozenCase { + pub case_id: String, + pub family_id: String, + pub split_id: String, + pub initial_evidence: DigestedRecord, + pub label: DigestedRecord, + pub budget: u64, + pub deadline_ms: u64, + pub timing_start_ms: u64, + pub timing_end_ms: u64, + pub policy_opportunities: Vec, +} +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct ComparisonManifest { + pub format: String, + pub actor: String, + pub scope: String, + pub purpose: String, + pub provenance: Provenance, + pub accounting_unit: AccountingUnit, + pub workload_sha256: String, + pub prompt_sha256: String, + pub runtime_sha256: String, + pub model_sha256: String, + pub tokenizer_sha256: String, + pub validator_sha256: String, + pub scorer_sha256: String, + pub compatibility_contract_sha256: String, + pub policies: Vec, + pub cases: Vec, +} +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct ScoringAssertion { + pub response_sha256: String, + pub scorer_sha256: String, + pub label_sha256: String, + pub normalized: Vec, +} +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct ObservedAttempt { + pub execution_id: String, + pub opportunity_id: String, + pub request_sha256: String, + pub evidence_manifest_sha256: String, + pub execution: Execution, + pub disposition: AttemptDisposition, + #[serde(deserialize_with = "super::nullable")] + pub response: Option, + #[serde(deserialize_with = "super::nullable")] + pub measured_input: Option, + #[serde(deserialize_with = "super::nullable")] + pub measured_output: Option, + pub retained_bound: u64, + pub start_ms: u64, + #[serde(deserialize_with = "super::nullable")] + pub completed_ms: Option, + pub validator_sha256: String, + pub scorer_sha256: String, + pub label_sha256: String, + #[serde(deserialize_with = "super::nullable")] + pub scoring: Option, +} +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct PolicyRow { + pub case_id: String, + pub outcome: Outcome, + #[serde(deserialize_with = "super::nullable")] + pub correct: Option, + #[serde(deserialize_with = "super::nullable")] + pub eligible: Option, + pub answerable: bool, + #[serde(deserialize_with = "super::nullable")] + pub completed_ms: Option, + #[serde(deserialize_with = "super::nullable")] + pub charged: Option, + pub reserved_bound: u64, + pub false_refusal: bool, + pub unsafe_acceptance: bool, + pub deadline_miss: bool, + pub simulated: bool, + pub attempts: Vec, +} +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct PolicyEvaluation { + pub evaluation_id: String, + pub actor: String, + pub scope: String, + pub purpose: String, + pub baseline_id: String, + pub candidate_id: String, + pub baseline_sha256: String, + pub candidate_sha256: String, + pub experiment_manifest: Vec, + pub experiment_sha256: String, + pub evaluator_version: String, + pub evaluator_sha256: String, + pub provenance: Provenance, + pub baseline_rows: Vec, + pub candidate_rows: Vec, +} +#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct PolicyCounters { + pub cases: u64, + pub success: u64, + pub false_refusal: u64, + pub unsafe_acceptance: u64, + pub cost: u64, + pub deadline_misses: u64, +} +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct ValidatedCounters { + pub baseline: PolicyCounters, + pub candidate: PolicyCounters, +} +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub enum SelectionRefusal { + Incomplete, + Incomparable, + NoImprovement, + IncreasedFalseRefusal, + IncreasedUnsafeAcceptance, + IncreasedCost, + IncreasedDeadlineMisses, +} +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum SelectionDecision { + Eligible { + validated_counters: ValidatedCounters, + }, + Refused(SelectionRefusal), +} diff --git a/tests/policy_evaluation.rs b/tests/policy_evaluation.rs new file mode 100644 index 0000000..38ca410 --- /dev/null +++ b/tests/policy_evaluation.rs @@ -0,0 +1,313 @@ +use kairotensor::policy_registry::validate_policy_artifact; + +#[test] +fn exact_known_artifact_only() { + let good = br#"{"schema_version":1,"policy_name":"always_stop","purpose":"fixture/recovery-policy","compatibility_contract_sha256":"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"}"#; + assert!(validate_policy_artifact(good).is_ok()); + for bad in [ + String::from_utf8(good.to_vec()) + .unwrap() + .replace("always_stop", "user_script"), + String::from_utf8(good.to_vec()) + .unwrap() + .replace("fixture/recovery-policy", " "), + String::from_utf8(good.to_vec()) + .unwrap() + .replace("schema_version\":1", "schema_version\":2"), + String::from_utf8(good.to_vec()) + .unwrap() + .replace("aaaaaaaa", "AAAAAAAA"), + String::from_utf8(good.to_vec()) + .unwrap() + .replace("{", "{\"extra\":0,"), + String::from_utf8(good.to_vec()) + .unwrap() + .replace("{", "{\"purpose\":\"duplicate\","), + " ".repeat(65537), + ] { + assert!(validate_policy_artifact(bad.as_bytes()).is_err(), "{bad}"); + } +} + +use kairotensor::policy_registry::*; +use serde_json::Value; +use std::process::Command; +fn fixture() -> PolicyEvaluation { + let output = Command::new("python3").args(["-c", "import sys,json;sys.path.insert(0,'scripts');from test_policy_evaluation_v2 import fixture;print(json.dumps(fixture()))"]).output().unwrap(); + assert!(output.status.success()); + serde_json::from_slice(&output.stdout).unwrap() +} +#[test] +fn synthetic_selection_uses_all_cases_and_original_deadline() { + let e = fixture(); + match evaluate_policy_comparison(&e).unwrap() { + SelectionDecision::Eligible { + validated_counters: c, + } => { + assert_eq!( + ( + c.baseline.cases, + c.baseline.success, + c.baseline.false_refusal, + c.baseline.cost + ), + (1, 0, 1, 4) + ); + assert_eq!( + ( + c.candidate.cases, + c.candidate.success, + c.candidate.false_refusal, + c.candidate.cost, + c.candidate.deadline_misses + ), + (1, 1, 0, 4, 0) + ); + } + other => panic!("{other:?}"), + } +} +#[test] +fn malformed_unknown_incomparable_and_fabricated_records() { + let e = fixture(); + for key in [ + "actor", + "scope", + "purpose", + "evaluator_version", + "evaluator_sha256", + "experiment_sha256", + "baseline_sha256", + "candidate_sha256", + ] { + let mut v = serde_json::to_value(&e).unwrap(); + v[key] = Value::String(if key.ends_with("sha256") { + "0".repeat(64) + } else { + "other".into() + }); + let changed: PolicyEvaluation = serde_json::from_value(v).unwrap(); + assert_eq!( + evaluate_policy_comparison(&changed).unwrap(), + SelectionDecision::Refused(SelectionRefusal::Incomparable), + "{key}" + ); + } + for key in ["correct", "eligible", "completed_ms", "charged"] { + let mut v = serde_json::to_value(&e).unwrap(); + v["candidate_rows"][0][key] = Value::Null; + let changed = serde_json::from_value(v).unwrap(); + assert_eq!( + evaluate_policy_comparison(&changed).unwrap(), + SelectionDecision::Refused(SelectionRefusal::Incomplete) + ); + } + for key in [ + "answerable", + "eligible", + "false_refusal", + "unsafe_acceptance", + "deadline_miss", + ] { + let mut v = serde_json::to_value(&e).unwrap(); + let b = v["candidate_rows"][0][key].as_bool().unwrap(); + v["candidate_rows"][0][key] = Value::Bool(!b); + let changed = serde_json::from_value(v).unwrap(); + assert!(evaluate_policy_comparison(&changed).is_err(), "{key}"); + } + let mut changed = e.clone(); + changed.candidate_rows[0].charged = Some(u64::MAX); + changed.candidate_rows[0].reserved_bound = 1; + assert!(evaluate_policy_comparison(&changed).is_err()); + let mut changed = e.clone(); + changed.candidate_rows.clear(); + assert_eq!( + evaluate_policy_comparison(&changed).unwrap(), + SelectionDecision::Refused(SelectionRefusal::Incomplete) + ); + let mut changed = e.clone(); + changed.candidate_rows[0].case_id = "other".into(); + assert_eq!( + evaluate_policy_comparison(&changed).unwrap(), + SelectionDecision::Refused(SelectionRefusal::Incomparable) + ); + let raw = serde_json::to_vec(&e).unwrap(); + assert_eq!(parse_policy_evaluation(&raw).unwrap(), e); + let bad = String::from_utf8(raw) + .unwrap() + .replacen('{', "{\"actor\":\"duplicate\",", 1); + assert!(parse_policy_evaluation(bad.as_bytes()).is_err()); + let mut v = serde_json::to_value(&e).unwrap(); + v["candidate_rows"][0] + .as_object_mut() + .unwrap() + .remove("charged"); + assert!(parse_policy_evaluation(&serde_json::to_vec(&v).unwrap()).is_err()); + let mut changed = e.clone(); + changed.evaluation_id = "a".repeat(MAX_EVALUATION_BYTES); + assert!(evaluate_policy_comparison(&changed).is_err()); + let mut changed = e.clone(); + changed.candidate_rows = vec![changed.candidate_rows[0].clone(); MAX_CASES + 1]; + assert!(evaluate_policy_comparison(&changed).is_err()); +} +#[test] +fn no_improvement_and_no_cost_increase() { + let mut e = fixture(); + e.candidate_rows = e.baseline_rows.clone(); + assert_eq!( + evaluate_policy_comparison(&e).unwrap(), + SelectionDecision::Refused(SelectionRefusal::NoImprovement) + ); + let mut e = fixture(); + e.candidate_rows[0].charged = Some(5); + assert_eq!( + evaluate_policy_comparison(&e).unwrap(), + SelectionDecision::Refused(SelectionRefusal::IncreasedCost) + ); +} +fn observed_fixture() -> PolicyEvaluation { + let output=Command::new("python3").args(["-c","import sys,json;sys.path.insert(0,'scripts');from test_policy_evaluation_v2 import observed_fixture;print(json.dumps(observed_fixture()))"]).output().unwrap(); + assert!(output.status.success()); + serde_json::from_slice(&output.stdout).unwrap() +} +#[test] +fn observed_records_require_bound_references_and_recomputed_counters() { + let e = observed_fixture(); + assert_eq!( + evaluate_policy_comparison(&e).unwrap(), + SelectionDecision::Refused(SelectionRefusal::NoImprovement) + ); + for key in [ + "opportunity_id", + "request_sha256", + "evidence_manifest_sha256", + "validator_sha256", + "scorer_sha256", + "label_sha256", + ] { + let mut v = serde_json::to_value(&e).unwrap(); + v["candidate_rows"][0]["attempts"][0][key] = Value::String("0".repeat(64)); + let changed = serde_json::from_value(v).unwrap(); + assert_eq!( + evaluate_policy_comparison(&changed).unwrap(), + SelectionDecision::Refused(SelectionRefusal::Incomparable), + "{key}" + ); + } + let mut changed = e.clone(); + changed.candidate_rows[0].attempts[0].scoring = None; + assert_eq!( + evaluate_policy_comparison(&changed).unwrap(), + SelectionDecision::Refused(SelectionRefusal::Incomplete) + ); + let mut changed = e.clone(); + changed.candidate_rows[0].charged = Some(3); + assert!(evaluate_policy_comparison(&changed).is_err()); + let mut changed = e.clone(); + changed.candidate_rows[0].attempts[0] + .scoring + .as_mut() + .unwrap() + .normalized = b"no".to_vec(); + changed.candidate_rows[0].correct = Some(false); + changed.candidate_rows[0].unsafe_acceptance = true; + assert_eq!( + evaluate_policy_comparison(&changed).unwrap(), + SelectionDecision::Refused(SelectionRefusal::Incomparable) + ); + changed.candidate_rows[0].attempts[0].execution_id = "independent".into(); + assert_eq!( + evaluate_policy_comparison(&changed).unwrap(), + SelectionDecision::Refused(SelectionRefusal::NoImprovement) + ); +} +#[test] +fn unknown_usage_retains_full_quote_and_never_qualifies() { + let mut e = observed_fixture(); + let r = &mut e.candidate_rows[0]; + let a = &mut r.attempts[0]; + a.execution_id = "uncertain".into(); + a.disposition = AttemptDisposition::Uncertain; + a.measured_input = None; + a.measured_output = None; + a.retained_bound = 10; + a.response = None; + a.scoring = None; + a.completed_ms = None; + r.outcome = Outcome::Unknown; + r.correct = None; + r.charged = None; + r.reserved_bound = 10; + r.completed_ms = None; + assert_eq!( + evaluate_policy_comparison(&e).unwrap(), + SelectionDecision::Refused(SelectionRefusal::Incomplete) + ); + e.candidate_rows[0].attempts[0].retained_bound = 9; + assert!(evaluate_policy_comparison(&e).is_err()); +} +#[test] +fn baseline_projection_uses_exact_manifest_and_rows() { + let e = fixture(); + let mut b = e.clone(); + b.evaluation_id = "baseline-only".into(); + assert!(paired_baseline_matches(&b, &e).unwrap()); + b.experiment_manifest.push(b' '); + assert!(!paired_baseline_matches(&b, &e).unwrap()); + b = e.clone(); + b.baseline_rows[0].charged = Some(3); + assert!(!paired_baseline_matches(&b, &e).unwrap()); +} + +#[test] +fn cross_language_golden_bytes_and_decisions() { + let e = fixture(); + assert_eq!( + evaluation_sha256(&e).unwrap(), + "2b0ed900f956cd8fc2dd215e114a665d4799187dd6610209807fe557a1311a88" + ); + assert_eq!( + e.experiment_sha256, + "1373dd5ff2bc8e1a06840f8feb31329b1a6666d434712db2a6fe4a3ef537c265" + ); + let output=Command::new("python3").args(["-c","import sys,json;sys.path.insert(0,'scripts');from test_policy_evaluation_v2 import golden_cases;print(json.dumps(golden_cases()))"]).output().unwrap(); + assert!(output.status.success()); + let cases: Vec<(String, Value, String)> = serde_json::from_slice(&output.stdout).unwrap(); + assert!(cases.len() >= 40); + for (name, value, want) in cases { + let actual = match parse_policy_evaluation(&serde_json::to_vec(&value).unwrap()) + .and_then(|e| evaluate_policy_comparison(&e)) + { + Ok(SelectionDecision::Eligible { .. }) => "Eligible".into(), + Ok(SelectionDecision::Refused(reason)) => format!("{reason:?}"), + Err(_) => "Error".into(), + }; + assert_eq!(actual, want, "{name}"); + } +} + +#[test] +fn candidate_binds_exact_artifact_purpose_and_identity() { + let e = fixture(); + let m: ComparisonManifest = serde_json::from_slice(&e.experiment_manifest).unwrap(); + let mut c = PolicyCandidate { + version_id: "v1".into(), + actor: e.actor, + scope: e.scope, + purpose: e.purpose, + artifact: m.policies[0].artifact.bytes.clone(), + sha256: m.policies[0].artifact.sha256.clone(), + }; + assert_eq!( + validate_policy_candidate(&c).unwrap().policy_name, + PolicyName::AlwaysStop + ); + c.purpose = "other".into(); + assert!(validate_policy_candidate(&c).is_err()); + c.purpose = "fixture/recovery-policy".into(); + c.sha256 = "0".repeat(64); + assert!(validate_policy_candidate(&c).is_err()); + c.sha256 = m.policies[0].artifact.sha256.clone(); + c.actor.clear(); + assert!(validate_policy_candidate(&c).is_err()); +} From 62f5646c48128f6b62efb79b7579c36284998f25 Mon Sep 17 00:00:00 2001 From: Alejandro Castellanos <5405628+k2jac9@users.noreply.github.com> Date: Sun, 15 Nov 2026 18:03:12 -0500 Subject: [PATCH 19/51] fix: validate partial policy facts and legacy attempt timing --- docs/policy-evaluation-v2.md | 45 +++- scripts/policy_evaluation_v2.py | 365 ++++++++++++++++++++++++++- scripts/test_policy_evaluation_v2.py | 334 ++++++++++++++++++++++++ scripts/test_recovery_policy_eval.py | 22 ++ src/policy_registry/evaluation.rs | 317 +++++++++++++++++++++-- tests/policy_evaluation.rs | 21 ++ 6 files changed, 1065 insertions(+), 39 deletions(-) diff --git a/docs/policy-evaluation-v2.md b/docs/policy-evaluation-v2.md index 5c53735..e23e383 100644 --- a/docs/policy-evaluation-v2.md +++ b/docs/policy-evaluation-v2.md @@ -56,8 +56,11 @@ cover every record and its relationships; names are wire names unless noted. - `PolicyEvaluation`: `evaluation_id`, `actor`, `scope`, `purpose`, `baseline_id`, `candidate_id`, `baseline_sha256`, `candidate_sha256`, `experiment_manifest` (bytes), `experiment_sha256`, `evaluator_version`, `evaluator_sha256`, - `provenance`, `baseline_rows`, `candidate_rows`. Selected version IDs are distinct - and nonempty. Their artifact digests select entries in the shared universe. + `provenance`, `baseline_rows`, `candidate_rows`. Selected version IDs are nonempty. + Their artifact digests select entries in the shared universe. A baseline-only receipt uses equal IDs, equal digests and + identical ordered rows on both sides. It undergoes full validation and returns + `NoImprovement` when complete, never `Eligible`; unknown facts remain incomplete + and malformed facts error. Conflicting same-ID digests or rows are incomparable. - `ComparisonManifest`: `format="policy-comparison/1"`, `actor`, `scope`, `purpose`, `provenance`, `accounting_unit`, `workload_sha256`, `prompt_sha256`, `runtime_sha256`, `model_sha256`, `tokenizer_sha256`, `validator_sha256`, @@ -141,10 +144,23 @@ sum of measured input/output, completion is the terminal completion, correctness comes from the scoring assertion, and eligibility comes from frozen admission and support (and stale disposition). All supplied redundant row values must agree. -An execution ID can recur across policy paths only for the same case and identical -observations, excluding the path-local opportunity ID. It cannot be counted for -another scheduled case. Distinct execution IDs can refer to equal request bytes: -request equality is not proof of identical independent realizations. +An execution ID can recur across policy paths only for the same case and compatible +present facts. Fixed request/evidence/validator/scorer/label identities, start time, +and opportunity input bound/output cap must agree; path-local opportunity IDs may +differ. Two present optional facts or known execution/terminal states cannot +conflict. A null optional fact, `Unknown` execution, or `Uncertain` disposition +remains missing knowledge. Known facts accumulate solely for order-independent +conflict detection, including response/scoring associations supplied separately. +They never complete a local record, infer a measurement, or release its hold. +Each local hold is validated independently: missing usage retains the full quote; +settled measured usage retains zero. Their resulting hold difference is permitted. +All independently known timing, reference, scoring, gate and transition facts are +checked before an incompleteness refusal; a missing completion cannot hide an +accepted-to-retry transition or a denied later gate. Incomparable shared facts take +precedence over incompleteness independent of which side is checked first. +An execution cannot be counted for another scheduled case. Distinct execution IDs +can refer to equal request bytes: request equality is not proof of identical +independent realizations. Synthetic rows have `simulated=true`, no attempt references, and explicitly simulated counters. Their eligibility and answerability still match frozen support @@ -202,10 +218,19 @@ Conversion without a matching complete supplied attestation returns reason; it never fills unknown facts from protocol parse success. With supplied facts, `workload_sha256` must bind compact sorted-key UTF-8 JSON of the archived input; cases/families/splits, token unit, exact requests, quotes, gates, deadlines, -timings and old measured row totals must match. Evidence/labels remain supplied -attestations. Missing legacy retry contexts or uncertain legacy usage stay -incomplete. A complete conversion returns the supplied evaluation and its selection -result, which may refuse. The existing positive-retry corpus correctly returns +timings and old measured row totals must match. For observed conversion, each +supplied executed or nonexecuted opportunity is additionally reconciled with the +legacy path: dispatch offset equals prior completion plus recorded preparation +duration; completion adds the recorded attempt duration for executed attempts, or +equals dispatch for a gated nonexecution. Execution/disposition, per-attempt +input/output split and charge, zero/nonzero holds, and the recorded correctness +assertion must agree. Equal aggregate time or cost cannot hide changed attempt +facts. Frozen evidence-manifest bytes in this legacy binding must equal the exact +UTF-8 archived `evidence_id`; this binds the old opaque identity, not reconstructed +evidence contents. Labels, raw responses and normalization remain supplied +attestations. Missing legacy retry contexts, unrecorded events or uncertain legacy +usage stay incomplete; an omitted event is never assigned an invented duration. +A complete conversion returns the supplied evaluation and its selection result, which may refuse. The existing positive-retry corpus correctly returns `IncreasedCost`; no worse baseline or changed selection rule is used. The inline `fixture()` and `golden_cases()` in diff --git a/scripts/policy_evaluation_v2.py b/scripts/policy_evaluation_v2.py index c7a40ac..7134589 100644 --- a/scripts/policy_evaluation_v2.py +++ b/scripts/policy_evaluation_v2.py @@ -550,7 +550,6 @@ def evaluate_policy_comparison(e): or sha(byte_array(e["experiment_manifest"])) != e["experiment_sha256"] or e["evaluator_version"] != "policy-selection/1" or e["evaluator_sha256"] != SELECTION_CONTRACT_SHA256 - or e["baseline_id"] == e["candidate_id"] or m["accounting_unit"] not in ("model_tokens", "fixture_bytes") or ( e["provenance"] == "synthetic_registry_mechanics" @@ -566,6 +565,13 @@ def evaluate_policy_comparison(e): c = rows( m, labels, e["candidate_rows"], selected[e["candidate_sha256"]], executions ) + if e["baseline_id"] == e["candidate_id"] and ( + e["baseline_sha256"] != e["candidate_sha256"] + or e["baseline_rows"] != e["candidate_rows"] + ): + return refused("Incomparable") + if b == "Incomparable" or c == "Incomparable": + return refused("Incomparable") if type(b) is str: return refused(b) if type(c) is str: @@ -646,7 +652,7 @@ def rows(m, labels, records, policy, executions): require(not r["simulated"], "observed row cannot simulate counters") reason = observed(m, c, l, r, policy, executions) if reason: - issue = issue or reason + issue = reason if reason == "Incomparable" else issue or reason continue if ( any( @@ -707,6 +713,9 @@ def observed(m, c, l, r, policy, executions): if not r["attempts"]: return "Incomplete" require(len(r["attempts"]) <= len(ops), "too many observed attempts") + reason = check_observed_facts(m, c, l, r, ops, policy, executions) + if reason: + return reason charged = 0 previous_end = c["timing_start_ms"] result = ("Refused", False, l["admissible_support"], previous_end) @@ -823,8 +832,7 @@ def observed(m, c, l, r, policy, executions): return "Incomparable" correct = s["normalized"] == l["expected_normalized"] else: - require(a["scoring"] is None, "scoring without response") - if a["disposition"] != "Failed": + if a["scoring"] is not None or a["disposition"] != "Failed": return "Incomplete" correct = False outcome = ( @@ -851,13 +859,7 @@ def observed(m, c, l, r, policy, executions): i + 1 == len(r["attempts"]), "terminal disposition cannot unlock retry", ) - observation = (c["case_id"], dict(a, opportunity_id="")) - if ( - a["execution_id"] in executions - and executions[a["execution_id"]] != observation - ): - return "Incomparable" - executions[a["execution_id"]] = observation + require( (r["outcome"] == "Unknown" or r["outcome"] == result[0]) and all( @@ -877,6 +879,332 @@ def observed(m, c, l, r, policy, executions): return None +def shared_execution_consistent(c, o, a, executions): + # Retained holds remain local; cumulative facts never fill another row. + import copy + + identity = (c["case_id"], o["input_bound"], o["output_cap"]) + if a["execution_id"] not in executions: + executions[a["execution_id"]] = (identity, copy.deepcopy(a)) + return True + prior_identity, prior = executions[a["execution_id"]] + if prior_identity != identity or any( + prior[k] != a[k] + for k in ( + "request_sha256", + "evidence_manifest_sha256", + "validator_sha256", + "scorer_sha256", + "label_sha256", + "start_ms", + ) + ): + return False + for k, unknown in [("execution", "Unknown"), ("disposition", "Uncertain")]: + if prior[k] != unknown and a[k] != unknown and prior[k] != a[k]: + return False + if prior[k] == unknown: + prior[k] = a[k] + for k in ( + "response", + "scoring", + "measured_input", + "measured_output", + "completed_ms", + ): + if prior[k] is not None and a[k] is not None and prior[k] != a[k]: + return False + if prior[k] is None: + prior[k] = copy.deepcopy(a[k]) + return ( + prior["response"] is None + or prior["scoring"] is None + or prior["response"]["sha256"] == prior["scoring"]["response_sha256"] + ) + + +def check_observed_facts(m, c, l, r, ops, policy, executions): + ids = set() + previous_start = c["timing_start_ms"] + previous_end = previous_start + known_charge = 0 + held = 0 + for i, (a, o) in enumerate(zip(r["attempts"], ops)): + if i: + previous = r["attempts"][i - 1] + require( + previous["execution"] == "Executed" + and previous["disposition"] == "InvalidOutput" + and previous["measured_input"] is not None + and previous["measured_output"] is not None, + "only settled invalid output unlocks retry", + ) + if i + 1 < len(r["attempts"]): + require( + a["execution"] == "Executed" and a["disposition"] == "InvalidOutput", + "terminal disposition cannot unlock retry", + ) + require(a["execution_id"] not in ids, "reused execution in one policy path") + ids.add(a["execution_id"]) + if ( + a["opportunity_id"] != o["opportunity_id"] + or a["request_sha256"] != o["request"]["sha256"] + or a["evidence_manifest_sha256"] != o["evidence_manifest"]["sha256"] + or a["validator_sha256"] != m["validator_sha256"] + or a["scorer_sha256"] != m["scorer_sha256"] + or a["label_sha256"] != c["label"]["sha256"] + ): + return "Incomparable" + require( + a["start_ms"] >= previous_start + and (previous_end is None or a["start_ms"] >= previous_end) + and o["start_min_ms"] <= a["start_ms"] <= o["start_max_ms"], + "attempt outside timing window", + ) + if a["completed_ms"] is not None: + require( + a["start_ms"] <= a["completed_ms"] <= c["timing_end_ms"], + "invalid completion time", + ) + previous_start = a["start_ms"] + previous_end = a["completed_ms"] + if a["measured_input"] is not None: + require(a["measured_input"] <= o["input_bound"], "input exceeds quote") + if a["measured_output"] is not None: + require(a["measured_output"] <= o["output_cap"], "output exceeds cap") + if a["scoring"] is not None: + scoring = a["scoring"] + if ( + scoring["scorer_sha256"] != m["scorer_sha256"] + or scoring["label_sha256"] != c["label"]["sha256"] + or ( + a["response"] is not None + and a["response"]["sha256"] != scoring["response_sha256"] + ) + ): + return "Incomparable" + quote = add(o["input_bound"], o["output_cap"]) + if known_charge is not None: + require(known_charge <= c["budget"], "budget exceeded") + gate = ( + o["admitted"] + and not ( + i > 0 and policy == "evidence_gated_retry" and not o["consistent"] + ) + and a["start_ms"] < c["deadline_ms"] + and quote <= c["budget"] - known_charge + ) + require( + gate == (a["execution"] != "NotExecuted"), + "execution violates frozen admission", + ) + if a["execution"] == "Unknown": + require( + a["measured_input"] is None + and a["measured_output"] is None + and a["response"] is None + and a["scoring"] is None + and a["disposition"] == "Uncertain" + and a["retained_bound"] == quote, + "invalid unknown execution", + ) + elif a["execution"] == "NotExecuted": + require( + a["measured_input"] == 0 + and a["measured_output"] == 0 + and a["retained_bound"] == 0 + and a["response"] is None + and a["scoring"] is None + and a["disposition"] == "Refused" + and a["completed_ms"] == a["start_ms"], + "invalid nonexecution attestation", + ) + elif a["measured_input"] is not None and a["measured_output"] is not None: + require( + a["retained_bound"] == 0 and a["disposition"] != "Uncertain", + "invalid settled usage", + ) + else: + require( + a["retained_bound"] == quote, "unknown usage must retain full bound" + ) + known_charge = ( + add(known_charge, add(a["measured_input"], a["measured_output"])) + if all( + x is not None + for x in (known_charge, a["measured_input"], a["measured_output"]) + ) + else None + ) + held = add(held, a["retained_bound"]) + if not shared_execution_consistent(c, o, a, executions): + return "Incomparable" + terminal = r["attempts"][-1] + missing_retry = ( + terminal["execution"] == "Executed" + and terminal["disposition"] == "InvalidOutput" + and len(r["attempts"]) < len(ops) + ) + if r["charged"] is not None and known_charge is not None: + require(r["charged"] >= known_charge, "row charge below known attempt prefix") + if r["completed_ms"] is not None and terminal["completed_ms"] is not None: + require( + r["completed_ms"] >= terminal["completed_ms"], + "row completion before known attempt prefix", + ) + if not missing_retry: + known_outcome = ( + "Unknown" + if terminal["execution"] == "Unknown" + or terminal["disposition"] == "Uncertain" + else "Refused" + if terminal["execution"] == "NotExecuted" + else "Accepted" + if terminal["disposition"] == "Accepted" + else "Failed" + if terminal["disposition"] == "Failed" + else "Refused" + ) + require( + r["outcome"] == "Unknown" or r["outcome"] == known_outcome, + "fabricated observed outcome", + ) + if ( + terminal["disposition"] == "Failed" + and terminal["response"] is None + and terminal["scoring"] is None + ): + require( + r["correct"] is None or not r["correct"], + "failed response cannot imply correctness", + ) + if ( + terminal["execution"] != "Unknown" + and terminal["disposition"] != "Uncertain" + and r["eligible"] is not None + ): + opportunity = ops[len(r["attempts"]) - 1] + require( + r["eligible"] + == ( + opportunity["admitted"] + and l["admissible_support"] + and terminal["disposition"] != "Stale" + ), + "fabricated observed eligibility", + ) + require(r["reserved_bound"] == held, "fabricated observed hold") + if r["charged"] is not None and known_charge is not None: + require(r["charged"] == known_charge, "fabricated observed charge") + if known_charge is None: + require(r["charged"] is None, "unknown charge must remain unknown") + if r["completed_ms"] is not None and terminal["completed_ms"] is not None: + require( + r["completed_ms"] == terminal["completed_ms"], + "fabricated observed completion", + ) + if r["correct"] is not None and terminal["scoring"] is not None: + require( + r["correct"] + == (terminal["scoring"]["normalized"] == l["expected_normalized"]), + "fabricated observed correctness", + ) + return None + + +def legacy_observations_match(case, label, legacy_case, row, policy): + """Check a supplied observed prefix against facts the archived trace records. + + Missing measurements/references remain missing. This computes expected offsets + from recorded preparation/execution durations; it never supplies new observations. + """ + ops = next( + p["opportunities"] + for p in case["policy_opportunities"] + if p["policy_name"] == policy + ) + end = 0 + charged = 0 + expected = [] + disposition_names = { + "accepted": "Accepted", + "invalid_output": "InvalidOutput", + "stale": "Stale", + "uncertain": "Uncertain", + } + for i, o in enumerate(ops): + if i: + # Legacy stops before preparing a retry once its deadline has elapsed. + if policy == "always_stop" or end >= legacy_case["deadline_ms"]: + break + cell = legacy_case["retries"][policy] + if cell is None: + return False + context, receipt = cell["context"], cell["attempt"] + else: + context, receipt = legacy_case["initial_context"], legacy_case["initial"] + start = add(end, context["elapsed_ms"]) + quote = add(context["input_tokens"], context["output_cap"]) + gate = ( + context["admitted"] + and not ( + i and policy == "evidence_gated_retry" and not context["consistent"] + ) + and start < legacy_case["deadline_ms"] + and quote <= legacy_case["budget_tokens"] - charged + ) + if not gate: + expected.append((o, context, None, start, start)) + break + if receipt is None or receipt["disposition"] == "uncertain": + return False + end = add(start, receipt["elapsed_ms"]) + charged = add(charged, receipt["charged_tokens"]) + expected.append((o, context, receipt, start, end)) + # Only invalid_output reaches another iteration. + if receipt["disposition"] != "invalid_output": + break + if len(row["attempts"]) != len(expected): + return False + for a, (o, context, receipt, start, end) in zip(row["attempts"], expected): + if ( + a["opportunity_id"] != o["opportunity_id"] + or a["request_sha256"] != o["request"]["sha256"] + or a["evidence_manifest_sha256"] != o["evidence_manifest"]["sha256"] + or a["start_ms"] != start + or a["completed_ms"] != end + ): + return False + if receipt is None: + if ( + a["execution"] != "NotExecuted" + or a["disposition"] != "Refused" + or a["measured_input"] != 0 + or a["measured_output"] != 0 + or a["retained_bound"] != 0 + or a["response"] is not None + or a["scoring"] is not None + ): + return False + else: + if ( + a["execution"] != "Executed" + or a["disposition"] != disposition_names[receipt["disposition"]] + or a["measured_input"] != context["input_tokens"] + or a["measured_output"] + != receipt["charged_tokens"] - context["input_tokens"] + or a["retained_bound"] != 0 + or a["response"] is None + or a["scoring"] is None + ): + return False + if (a["scoring"]["normalized"] == label["expected_normalized"]) != receipt[ + "correct" + ]: + return False + return True + + def convert_legacy(document, comparison=None): """No missing facts are inferred. A supplied complete attestation must bind this input. @@ -898,7 +1226,7 @@ def convert_legacy(document, comparison=None): if comparison is None: return result shape(comparison) - m, _ = manifest(comparison) + m, labels = manifest(comparison) expected_provenance = ( "synthetic_registry_mechanics" if document["mode"] == "synthetic_policy_fixture" @@ -934,6 +1262,10 @@ def convert_legacy(document, comparison=None): ctx = cell["context"] if i else old["initial_context"] if byte_array(o["request"]["bytes"]) != ctx["request"].encode("utf-8"): return result + if byte_array(o["evidence_manifest"]["bytes"]) != ctx[ + "evidence_id" + ].encode("utf-8"): + return result if any( o[k] != ctx[v] for k, v in [ @@ -953,7 +1285,14 @@ def convert_legacy(document, comparison=None): oldrows = [r for r in legacy["cases"] if r["policy"] == policy] if len(oldrows) != len(comparison[side]): return result - for row, old in zip(comparison[side], oldrows): + for row, old, case, label, legacy_case in zip( + comparison[side], oldrows, m["cases"], labels, document["cases"] + ): + if ( + expected_provenance == "observed_policy_replay" + and not legacy_observations_match(case, label, legacy_case, row, policy) + ): + return result if old["reason"] == "uncertain": return result if any( diff --git a/scripts/test_policy_evaluation_v2.py b/scripts/test_policy_evaluation_v2.py index 7a3bf9a..b2d40b0 100644 --- a/scripts/test_policy_evaluation_v2.py +++ b/scripts/test_policy_evaluation_v2.py @@ -923,6 +923,340 @@ def multiple(m): return cases +def baseline_only_fixture(observed=False): + e = observed_fixture() if observed else fixture() + e.update( + candidate_id=e["baseline_id"], + candidate_sha256=e["baseline_sha256"], + candidate_rows=copy.deepcopy(e["baseline_rows"]), + ) + return e + + +def incomplete_observation_fixture(): + e = observed_fixture() + r = e["candidate_rows"][0] + a = r["attempts"][0] + a.update(execution_id="unknown-measurement", measured_input=None, retained_bound=10) + r.update( + outcome="Unknown", + correct=None, + charged=None, + reserved_bound=10, + completed_ms=None, + ) + return e + + +def legacy_observed_comparison(): + old, e = legacy_comparison() + old["mode"] = "recorded_policy_replay" + e["provenance"] = "observed_policy_replay" + manifest_edit( + e, + lambda m: m.update( + provenance="observed_policy_replay", + workload_sha256=hashlib.sha256(pack(old)).hexdigest(), + ), + ) + m = json.loads(bytes(e["experiment_manifest"])) + for side, pi in [("baseline_rows", 0), ("candidate_rows", 1)]: + for row, c, oldc in zip(e[side], m["cases"], old["cases"]): + count = ( + 2 if pi and oldc["initial"]["disposition"] == "invalid_output" else 1 + ) + attempts = [] + end = 0 + for i in range(count): + cell = ( + oldc["retries"]["affordable_retry"] + if i + else { + "context": oldc["initial_context"], + "attempt": oldc["initial"], + } + ) + ctx, prior = cell["context"], cell["attempt"] + op = c["policy_opportunities"][pi]["opportunities"][i] + start = end + ctx["elapsed_ms"] + end = start + prior["elapsed_ms"] + response = record(b"yes" if prior["correct"] else b"no") + attempts.append( + { + "execution_id": c["case_id"] + "/" + str(i), + "opportunity_id": op["opportunity_id"], + "request_sha256": op["request"]["sha256"], + "evidence_manifest_sha256": op["evidence_manifest"]["sha256"], + "execution": "Executed", + "disposition": "Accepted" + if prior["disposition"] == "accepted" + else "InvalidOutput", + "response": response, + "measured_input": ctx["input_tokens"], + "measured_output": prior["charged_tokens"] + - ctx["input_tokens"], + "retained_bound": 0, + "start_ms": start, + "completed_ms": end, + "validator_sha256": m["validator_sha256"], + "scorer_sha256": m["scorer_sha256"], + "label_sha256": c["label"]["sha256"], + "scoring": { + "response_sha256": response["sha256"], + "scorer_sha256": m["scorer_sha256"], + "label_sha256": c["label"]["sha256"], + "normalized": list(b"yes" if prior["correct"] else b"no"), + }, + } + ) + row.update(simulated=False, attempts=attempts) + return old, e + + +def fix_round_one_goldens(): + cases = [] + for observed in (False, True): + e = baseline_only_fixture(observed) + cases.append(("baseline-only-" + str(observed), e, "NoImprovement")) + e = baseline_only_fixture(observed) + for side in ("baseline_rows", "candidate_rows"): + e[side][0]["charged"] = None + cases.append(("baseline-unknown-" + str(observed), e, "Incomplete")) + e = baseline_only_fixture() + for side in ("baseline_rows", "candidate_rows"): + e[side][0]["false_refusal"] = False + cases.append(("baseline-invalid-identical", e, "Error")) + e = baseline_only_fixture() + e["candidate_sha256"] = "0" * 64 + cases.append(("baseline-conflicting-digest", e, "Incomparable")) + e = baseline_only_fixture() + e["candidate_rows"][0]["charged"] = 3 + cases.append(("baseline-conflicting-rows", e, "Incomparable")) + e = incomplete_observation_fixture() + cases.append(("unknown-usage-valid", e, "Incomplete")) + for end in (201,): + e = incomplete_observation_fixture() + e["candidate_rows"][0]["attempts"][0]["completed_ms"] = end + cases.append(("unknown-usage-invalid-completion", e, "Error")) + e = incomplete_observation_fixture() + a = e["candidate_rows"][0]["attempts"][0] + a.update(start_ms=20, completed_ms=19) + cases.append(("unknown-usage-completion-before-start", e, "Error")) + for key in ("response_sha256", "scorer_sha256", "label_sha256"): + e = incomplete_observation_fixture() + e["candidate_rows"][0]["attempts"][0]["scoring"][key] = "0" * 64 + cases.append(("unknown-usage-scoring-" + key, e, "Incomparable")) + e = incomplete_observation_fixture() + a = e["candidate_rows"][0]["attempts"][0] + a["execution_id"] = "shared-execution" + a["response"] = record(b"different raw response") + a["scoring"].update(response_sha256=a["response"]["sha256"], normalized=list(b"no")) + cases.append(("unknown-usage-conflicting-execution", e, "Incomparable")) + e = incomplete_observation_fixture() + a = e["candidate_rows"][0]["attempts"][0] + a.update( + execution="Unknown", + disposition="Uncertain", + measured_output=None, + response=None, + scoring=None, + completed_ms=201, + ) + cases.append(("unknown-execution-invalid-completion", e, "Error")) + for reverse in (False, True): + e = incomplete_observation_fixture() + e["candidate_rows"][0]["attempts"][0]["execution_id"] = "shared-execution" + if reverse: + for left, right in [ + ("baseline_id", "candidate_id"), + ("baseline_sha256", "candidate_sha256"), + ("baseline_rows", "candidate_rows"), + ]: + e[left], e[right] = e[right], e[left] + cases.append( + ("compatible-partial-order-" + str(reverse), copy.deepcopy(e), "Incomplete") + ) + partial = e["baseline_rows" if reverse else "candidate_rows"][0]["attempts"][0] + partial["response"] = record(b"different response") + partial["scoring"].update( + response_sha256=partial["response"]["sha256"], normalized=list(b"no") + ) + cases.append(("conflicting-partial-order-" + str(reverse), e, "Incomparable")) + e = incomplete_observation_fixture() + a = e["candidate_rows"][0]["attempts"][0] + a.update(execution_id="shared-execution", start_ms=1) + cases.append(("partial-fixed-start-conflict", e, "Incomparable")) + e = incomplete_observation_fixture() + a = e["candidate_rows"][0]["attempts"][0] + a.update(execution_id="shared-execution", disposition="Stale") + e["candidate_rows"][0]["eligible"] = False + cases.append(("partial-known-terminal-conflict", e, "Incomparable")) + for reverse in (False, True): + for conflict in (False, True): + e = observed_fixture() + e["baseline_rows"][0]["attempts"][0]["response"] = None + e["baseline_rows"][0]["correct"] = None + e["candidate_rows"][0]["attempts"][0]["scoring"] = None + e["candidate_rows"][0]["correct"] = None + if conflict: + e["candidate_rows"][0]["attempts"][0]["response"] = record(b"other") + if reverse: + for left, right in [ + ("baseline_id", "candidate_id"), + ("baseline_sha256", "candidate_sha256"), + ("baseline_rows", "candidate_rows"), + ]: + e[left], e[right] = e[right], e[left] + cases.append( + ( + "complementary-partials-" + str(reverse) + "-" + str(conflict), + e, + "Incomparable" if conflict else "Incomplete", + ) + ) + for changed_bound in ("input_bound", "output_cap"): + _, e = legacy_observed_comparison() + m = json.loads(bytes(e["experiment_manifest"])) + e.update( + baseline_id="v3", + baseline_sha256=m["policies"][2]["artifact"]["sha256"], + baseline_rows=copy.deepcopy(e["candidate_rows"]), + ) + for row, c in zip(e["baseline_rows"], m["cases"]): + for a, o in zip( + row["attempts"], c["policy_opportunities"][2]["opportunities"] + ): + a["opportunity_id"] = o["opportunity_id"] + manifest_edit( + e, + lambda m, k=changed_bound: m["cases"][1]["policy_opportunities"][2][ + "opportunities" + ][1].update({k: 6 if k == "input_bound" else 9}), + ) + cases.append(("shared-execution-" + changed_bound, e, "Incomparable")) + e = incomplete_observation_fixture() + e["candidate_rows"][0]["eligible"] = False + cases.append(("unknown-usage-known-eligibility", e, "Error")) + _, e = legacy_observed_comparison() + a = e["candidate_rows"][1]["attempts"][0] + a.update( + execution_id="independent-terminal", disposition="Accepted", completed_ms=None + ) + cases.append(("unknown-completion-terminal-cannot-retry", e, "Error")) + _, e = legacy_observed_comparison() + e["candidate_rows"][1]["attempts"][0]["completed_ms"] = None + e["candidate_rows"][1].update(eligible=False, unsafe_acceptance=True) + manifest_edit( + e, + lambda m: m["cases"][1]["policy_opportunities"][1]["opportunities"][1].update( + admitted=False + ), + ) + cases.append(("unknown-completion-cannot-hide-next-denied-gate", e, "Error")) + for key, value in [("charged", 1), ("completed_ms", 5)]: + _, e = legacy_observed_comparison() + r = e["candidate_rows"][1] + r["attempts"] = r["attempts"][:1] + r[key] = value + cases.append(("missing-retry-below-known-prefix-" + key, e, "Error")) + return cases + + +class FixRoundOneTests(unittest.TestCase): + evaluate = ComparisonTests.evaluate + + def test_baseline_only_and_incomplete_known_fact_regressions(self): + for name, e, want in fix_round_one_goldens(): + with self.subTest(name=name): + try: + result = self.evaluate(e) + actual = result.get("reason", result["decision"]) + except ValueError: + actual = "Error" + self.assertEqual(actual, want) + + def test_observed_conversion_preserves_known_per_attempt_facts(self): + from policy_evaluation_v2 import convert_legacy + + old, e = legacy_observed_comparison() + self.assertEqual(convert_legacy(old, e)["status"], "Complete") + changes = [ + ("start", lambda a: a.update(start_ms=3)), + ( + "input-output-split", + lambda a: a.update(measured_input=3, measured_output=4), + ), + ("disposition", lambda a: a.update(disposition="Stale")), + ("correctness", lambda a: a["scoring"].update(normalized=list(b"no"))), + ] + for name, change in changes: + bad = copy.deepcopy(e) + for side in ("baseline_rows", "candidate_rows"): + change(bad[side][0]["attempts"][0]) + with self.subTest(name=name): + self.assertEqual(convert_legacy(old, bad)["status"], "Incomplete") + bad = copy.deepcopy(e) + row = bad["candidate_rows"][1] + row["attempts"][0]["completed_ms"] = 5 + row["attempts"][1]["start_ms"] = 6 + self.assertEqual(convert_legacy(old, bad)["status"], "Incomplete") + + +class LegacyNonexecutionTests(unittest.TestCase): + def test_nonexecuted_attempt_preserves_preparation_and_zero_usage(self): + from policy_evaluation_v2 import convert_legacy + + old, e = legacy_observed_comparison() + old["cases"][0]["initial_context"]["admitted"] = False + + def edit(m): + m["workload_sha256"] = hashlib.sha256(pack(old)).hexdigest() + for p in m["cases"][0]["policy_opportunities"]: + p["opportunities"][0]["admitted"] = False + + manifest_edit(e, edit) + for side in ("baseline_rows", "candidate_rows"): + r = e[side][0] + a = r["attempts"][0] + a.update( + execution="NotExecuted", + disposition="Refused", + response=None, + scoring=None, + measured_input=0, + measured_output=0, + completed_ms=1, + ) + r.update( + outcome="Refused", + correct=False, + eligible=False, + charged=0, + completed_ms=1, + false_refusal=True, + ) + self.assertEqual(convert_legacy(old, e)["status"], "Complete") + for key, value in [ + ("start_ms", 0), + ("measured_input", 1), + ("execution", "Executed"), + ]: + bad = copy.deepcopy(e) + bad["candidate_rows"][0]["attempts"][0][key] = value + with self.subTest(key=key): + self.assertEqual(convert_legacy(old, bad)["status"], "Incomplete") + + def test_same_total_charge_cannot_shift_usage_between_attempts(self): + from policy_evaluation_v2 import convert_legacy + + old, e = legacy_observed_comparison() + row = e["candidate_rows"][1] + row["attempts"][0]["execution_id"] = "independent-initial" + row["attempts"][0]["measured_output"] -= 1 + row["attempts"][1]["measured_output"] += 1 + self.assertEqual(convert_legacy(old, e)["status"], "Incomplete") + + class GoldenTests(unittest.TestCase): def test_frozen_golden_bytes_and_decisions(self): from policy_evaluation_v2 import ( diff --git a/scripts/test_recovery_policy_eval.py b/scripts/test_recovery_policy_eval.py index 0b06644..d7d295d 100644 --- a/scripts/test_recovery_policy_eval.py +++ b/scripts/test_recovery_policy_eval.py @@ -293,5 +293,27 @@ def test_explicit_conversion_does_not_change_default_output(self): self.assertEqual(json.loads(default.stdout)['policy_version'], 'frozen-v1') +class ObservedConversionTimingTests(unittest.TestCase): + def test_explicit_conversion_rejects_changed_preparation_with_same_completion(self): + from test_policy_evaluation_v2 import legacy_observed_comparison + old, comparison = legacy_observed_comparison() + script = Path(__file__).with_name('recovery_policy_eval.py') + with tempfile.TemporaryDirectory() as directory: + source = Path(directory) / 'legacy.json' + supplied = Path(directory) / 'complete.json' + source.write_text(json.dumps(old)) + supplied.write_text(json.dumps(comparison)) + command = [sys.executable, str(script), str(source), '--convert-v2', '--comparison-v2', str(supplied)] + good = subprocess.run(command, capture_output=True, text=True, check=True) + self.assertEqual(json.loads(good.stdout)['status'], 'Complete') + for side in ('baseline_rows', 'candidate_rows'): + for row in comparison[side]: + row['attempts'][0]['start_ms'] = 3 + supplied.write_text(json.dumps(comparison)) + bad = subprocess.run(command, capture_output=True, text=True, check=True) + self.assertEqual(json.loads(bad.stdout)['status'], 'Incomplete') + self.assertEqual(evaluate(old)['policy_version'], 'frozen-v1') + + if __name__ == '__main__': unittest.main() diff --git a/src/policy_registry/evaluation.rs b/src/policy_registry/evaluation.rs index 0087724..178f882 100644 --- a/src/policy_registry/evaluation.rs +++ b/src/policy_registry/evaluation.rs @@ -259,7 +259,6 @@ pub fn evaluate_policy_comparison(e: &PolicyEvaluation) -> rusqlite::Result rusqlite::Result (b, c), + (Err(SelectionRefusal::Incomparable), _) | (_, Err(SelectionRefusal::Incomparable)) => { + return Ok(refused(SelectionRefusal::Incomparable)) + } (Err(r), _) | (_, Err(r)) => return Ok(refused(r)), }; let reason = if c.success <= b.success { @@ -328,7 +335,7 @@ fn rows( labels: &[FrozenLabel], rows: &[PolicyRow], policy: PolicyName, - executions: &mut HashMap, + executions: &mut HashMap, ) -> rusqlite::Result { let mut refusal = if rows.len() != m.cases.len() { Some(SelectionRefusal::Incomplete) @@ -384,7 +391,11 @@ fn rows( } else { require(!r.simulated, "observed row cannot simulate counters")?; if let Some(reason) = observed(m, c, l, r, policy, executions)? { - refusal.get_or_insert(reason); + if reason == SelectionRefusal::Incomparable { + refusal = Some(reason); + } else { + refusal.get_or_insert(reason); + } continue; } } @@ -438,7 +449,7 @@ fn observed( l: &FrozenLabel, r: &PolicyRow, policy: PolicyName, - executions: &mut HashMap, + executions: &mut HashMap, ) -> rusqlite::Result> { let ops = &c .policy_opportunities @@ -450,6 +461,9 @@ fn observed( return Ok(Some(SelectionRefusal::Incomplete)); } require(r.attempts.len() <= ops.len(), "too many observed attempts")?; + if let Some(reason) = check_observed_facts(m, c, l, r, ops, policy, executions)? { + return Ok(Some(reason)); + } let mut charged = 0; let mut previous_end = c.timing_start_ms; let mut result = ( @@ -579,8 +593,7 @@ fn observed( } scoring.normalized == l.expected_normalized } else { - require(a.scoring.is_none(), "scoring without response")?; - if a.disposition != AttemptDisposition::Failed { + if a.scoring.is_some() || a.disposition != AttemptDisposition::Failed { return Ok(Some(SelectionRefusal::Incomplete)); } false @@ -609,16 +622,6 @@ fn observed( )?; } } - // Opportunity IDs are local to a policy path; execution IDs identify realizations. - let mut observation = a.clone(); - observation.opportunity_id.clear(); - if let Some(prior) = executions.get(&a.execution_id) { - if *prior != (c.case_id.clone(), observation.clone()) { - return Ok(Some(SelectionRefusal::Incomparable)); - } - } else { - executions.insert(a.execution_id.clone(), (c.case_id.clone(), observation)); - } } require( (r.outcome == Outcome::Unknown || r.outcome == result.0) @@ -639,3 +642,285 @@ fn observed( } Ok(None) } + +// Cumulative facts are used only to detect contradictions, never to complete a row. +struct ExecutionFacts { + case_id: String, + input_bound: u64, + output_cap: u64, + observation: ObservedAttempt, +} +fn merge_known(known: &mut Option, supplied: &Option) -> bool { + match (known.as_ref(), supplied.as_ref()) { + (Some(left), Some(right)) => left == right, + (None, Some(value)) => { + *known = Some(value.clone()); + true + } + _ => true, + } +} +fn shared_execution_consistent( + c: &FrozenCase, + o: &RequestOpportunity, + a: &ObservedAttempt, + executions: &mut HashMap, +) -> bool { + let Some(facts) = executions.get_mut(&a.execution_id) else { + executions.insert( + a.execution_id.clone(), + ExecutionFacts { + case_id: c.case_id.clone(), + input_bound: o.input_bound, + output_cap: o.output_cap, + observation: a.clone(), + }, + ); + return true; + }; + let prior = &mut facts.observation; + if facts.case_id != c.case_id + || facts.input_bound != o.input_bound + || facts.output_cap != o.output_cap + || prior.request_sha256 != a.request_sha256 + || prior.evidence_manifest_sha256 != a.evidence_manifest_sha256 + || prior.validator_sha256 != a.validator_sha256 + || prior.scorer_sha256 != a.scorer_sha256 + || prior.label_sha256 != a.label_sha256 + || prior.start_ms != a.start_ms + { + return false; + } + if prior.execution != Execution::Unknown + && a.execution != Execution::Unknown + && prior.execution != a.execution + { + return false; + } + if prior.disposition != AttemptDisposition::Uncertain + && a.disposition != AttemptDisposition::Uncertain + && prior.disposition != a.disposition + { + return false; + } + if !merge_known(&mut prior.response, &a.response) + || !merge_known(&mut prior.scoring, &a.scoring) + || !merge_known(&mut prior.measured_input, &a.measured_input) + || !merge_known(&mut prior.measured_output, &a.measured_output) + || !merge_known(&mut prior.completed_ms, &a.completed_ms) + { + return false; + } + if prior.execution == Execution::Unknown { + prior.execution = a.execution; + } + if prior.disposition == AttemptDisposition::Uncertain { + prior.disposition = a.disposition; + } + // A response and its scoring assertion can arrive in separate partial records. + match (&prior.response, &prior.scoring) { + (Some(response), Some(scoring)) => response.sha256 == scoring.response_sha256, + _ => true, + } +} +fn check_observed_facts( + m: &ComparisonManifest, + c: &FrozenCase, + l: &FrozenLabel, + r: &PolicyRow, + ops: &[RequestOpportunity], + policy: PolicyName, + executions: &mut HashMap, +) -> rusqlite::Result> { + let mut ids = HashSet::new(); + let mut previous_start = c.timing_start_ms; + let mut previous_end = Some(c.timing_start_ms); + let mut known_charge = Some(0u64); + let mut held = 0; + for (i, (a, o)) in r.attempts.iter().zip(ops).enumerate() { + if i > 0 { + let previous = &r.attempts[i - 1]; + require( + previous.execution == Execution::Executed + && previous.disposition == AttemptDisposition::InvalidOutput + && previous.measured_input.is_some() + && previous.measured_output.is_some(), + "only settled invalid output unlocks retry", + )?; + } + if i + 1 < r.attempts.len() { + require( + a.execution == Execution::Executed + && a.disposition == AttemptDisposition::InvalidOutput, + "terminal disposition cannot unlock retry", + )?; + } + require( + ids.insert(&a.execution_id), + "reused execution in one policy path", + )?; + if a.opportunity_id != o.opportunity_id + || a.request_sha256 != o.request.sha256 + || a.evidence_manifest_sha256 != o.evidence_manifest.sha256 + || a.validator_sha256 != m.validator_sha256 + || a.scorer_sha256 != m.scorer_sha256 + || a.label_sha256 != c.label.sha256 + { + return Ok(Some(SelectionRefusal::Incomparable)); + } + require( + a.start_ms >= previous_start + && previous_end.is_none_or(|end| a.start_ms >= end) + && a.start_ms >= o.start_min_ms + && a.start_ms <= o.start_max_ms, + "attempt outside timing window", + )?; + if let Some(end) = a.completed_ms { + require( + end >= a.start_ms && end <= c.timing_end_ms, + "invalid completion time", + )?; + } + previous_start = a.start_ms; + previous_end = a.completed_ms; + if let Some(input) = a.measured_input { + require(input <= o.input_bound, "input exceeds quote")?; + } + if let Some(output) = a.measured_output { + require(output <= o.output_cap, "output exceeds cap")?; + } + if let Some(scoring) = &a.scoring { + if scoring.scorer_sha256 != m.scorer_sha256 + || scoring.label_sha256 != c.label.sha256 + || a.response + .as_ref() + .is_some_and(|response| response.sha256 != scoring.response_sha256) + { + return Ok(Some(SelectionRefusal::Incomparable)); + } + } + let quote = add(o.input_bound, o.output_cap)?; + if let Some(spent) = known_charge { + let remaining = c + .budget + .checked_sub(spent) + .ok_or_else(|| bad("budget exceeded"))?; + let gate = o.admitted + && !(i > 0 && policy == PolicyName::EvidenceGatedRetry && !o.consistent) + && a.start_ms < c.deadline_ms + && quote <= remaining; + require( + gate == (a.execution != Execution::NotExecuted), + "execution violates frozen admission", + )?; + } + match a.execution { + Execution::Unknown => require( + a.measured_input.is_none() + && a.measured_output.is_none() + && a.response.is_none() + && a.scoring.is_none() + && a.disposition == AttemptDisposition::Uncertain + && a.retained_bound == quote, + "invalid unknown execution", + )?, + Execution::NotExecuted => require( + a.measured_input == Some(0) + && a.measured_output == Some(0) + && a.retained_bound == 0 + && a.response.is_none() + && a.scoring.is_none() + && a.disposition == AttemptDisposition::Refused + && a.completed_ms == Some(a.start_ms), + "invalid nonexecution attestation", + )?, + Execution::Executed => { + if a.measured_input.is_some() && a.measured_output.is_some() { + require( + a.retained_bound == 0 && a.disposition != AttemptDisposition::Uncertain, + "invalid settled usage", + )?; + } else { + require( + a.retained_bound == quote, + "unknown usage must retain full bound", + )?; + } + } + } + known_charge = match (known_charge, a.measured_input, a.measured_output) { + (Some(total), Some(input), Some(output)) => Some(add(total, add(input, output)?)?), + _ => None, + }; + held = add(held, a.retained_bound)?; + if !shared_execution_consistent(c, o, a, executions) { + return Ok(Some(SelectionRefusal::Incomparable)); + } + } + let terminal = r.attempts.last().unwrap(); + let missing_retry = terminal.execution == Execution::Executed + && terminal.disposition == AttemptDisposition::InvalidOutput + && r.attempts.len() < ops.len(); + if let (Some(supplied), Some(known)) = (r.charged, known_charge) { + require(supplied >= known, "row charge below known attempt prefix")?; + } + if let (Some(supplied), Some(known)) = (r.completed_ms, terminal.completed_ms) { + require( + supplied >= known, + "row completion before known attempt prefix", + )?; + } + if !missing_retry { + let known_outcome = match (terminal.execution, terminal.disposition) { + (Execution::Unknown, _) | (_, AttemptDisposition::Uncertain) => Outcome::Unknown, + (Execution::NotExecuted, _) => Outcome::Refused, + (_, AttemptDisposition::Accepted) => Outcome::Accepted, + (_, AttemptDisposition::Failed) => Outcome::Failed, + _ => Outcome::Refused, + }; + require( + r.outcome == Outcome::Unknown || r.outcome == known_outcome, + "fabricated observed outcome", + )?; + if terminal.disposition == AttemptDisposition::Failed + && terminal.response.is_none() + && terminal.scoring.is_none() + { + require( + r.correct.is_none_or(|v| !v), + "failed response cannot imply correctness", + )?; + } + if terminal.execution != Execution::Unknown + && terminal.disposition != AttemptDisposition::Uncertain + { + if let Some(eligible) = r.eligible { + let opportunity = &ops[r.attempts.len() - 1]; + require( + eligible + == (opportunity.admitted + && l.admissible_support + && terminal.disposition != AttemptDisposition::Stale), + "fabricated observed eligibility", + )?; + } + } + require(r.reserved_bound == held, "fabricated observed hold")?; + if let (Some(supplied), Some(known)) = (r.charged, known_charge) { + require(supplied == known, "fabricated observed charge")?; + } + if known_charge.is_none() { + require(r.charged.is_none(), "unknown charge must remain unknown")?; + } + if let (Some(supplied), Some(known)) = (r.completed_ms, terminal.completed_ms) { + require(supplied == known, "fabricated observed completion")?; + } + if let (Some(supplied), Some(scoring)) = (r.correct, &terminal.scoring) { + require( + supplied == (scoring.normalized == l.expected_normalized), + "fabricated observed correctness", + )?; + } + } + Ok(None) +} diff --git a/tests/policy_evaluation.rs b/tests/policy_evaluation.rs index 38ca410..1c52a57 100644 --- a/tests/policy_evaluation.rs +++ b/tests/policy_evaluation.rs @@ -311,3 +311,24 @@ fn candidate_binds_exact_artifact_purpose_and_identity() { c.actor.clear(); assert!(validate_policy_candidate(&c).is_err()); } + +#[test] +fn fix_round_one_baseline_and_incomplete_observation_regressions() { + let output=Command::new("python3").args(["-c","import sys,json;sys.path.insert(0,'scripts');from test_policy_evaluation_v2 import fix_round_one_goldens;print(json.dumps(fix_round_one_goldens()))"]).output().unwrap(); + assert!(output.status.success()); + let cases: Vec<(String, Value, String)> = serde_json::from_slice(&output.stdout).unwrap(); + let mut failures = Vec::new(); + for (name, value, want) in cases { + let actual = match parse_policy_evaluation(&serde_json::to_vec(&value).unwrap()) + .and_then(|e| evaluate_policy_comparison(&e)) + { + Ok(SelectionDecision::Eligible { .. }) => "Eligible".into(), + Ok(SelectionDecision::Refused(reason)) => format!("{reason:?}"), + Err(_) => "Error".into(), + }; + if actual != want { + failures.push(format!("{name}: got {actual}, expected {want}")); + } + } + assert!(failures.is_empty(), "{}", failures.join("\n")); +} From 0923e13cfe77b32917baa8151dce2b8adddda6ce Mon Sep 17 00:00:00 2001 From: Alejandro Castellanos <5405628+k2jac9@users.noreply.github.com> Date: Sun, 15 Nov 2026 18:10:17 -0500 Subject: [PATCH 20/51] docs: bind policy migration and activation integration scopes --- .../plans/2026-11-15-policy-activation.md | 20 ++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/docs/superpowers/plans/2026-11-15-policy-activation.md b/docs/superpowers/plans/2026-11-15-policy-activation.md index de00d13..5cf4512 100644 --- a/docs/superpowers/plans/2026-11-15-policy-activation.md +++ b/docs/superpowers/plans/2026-11-15-policy-activation.md @@ -115,7 +115,11 @@ not an assertion that current empirical policies improve performance. ### Task 2: Registry migration and immutable registration **Files:** create `src/policy_registry.sql`, `tests/policy_registry.rs`; -modify `src/policy_registry.rs`, `src/policy_registry/types.rs`, `src/lib.rs`. +modify `src/policy_registry.rs`, `src/policy_registry/types.rs`, `src/lib.rs`, +`tests/dispatch.rs`, `tests/deterministic_verdict.rs`, `tests/timed_recovery.rs`, +and `README.md`. The three legacy test edits are limited to the exact final +schema-version expectation; preserve their fixtures and every data/accounting/FK +assertion. README changes only its current migration-version sentence. Tables: policy_versions (version_id PK, actor, scope, purpose, evidence_id UNIQUE FK evidence, contract SHA, artifact SHA); policy_evaluations (evaluation_id PK, @@ -151,8 +155,12 @@ valid evaluations may be stored but are never promotion eligible. - [ ] Inject failure after some DDL but before commit using a test-only migration hook; reopen and verify the old schema/rows remain intact. Reject unknown future versions. Keep original backup bytes; test recovery from that fixture backup. -- [ ] Implement registration. Run `cargo test --locked --test policy_registry - --test recovery --test timed_recovery`; review and commit +- [ ] Update the three existing migration tests to assert the newly allocated + exact final version, and update the README migration sentence. Do not replace + version equality with a permissive range or weaken old invariants. +- [ ] Implement registration. Run `cargo test --locked --lib --test policy_registry + --test recovery --test timed_recovery --test dispatch --test deterministic_verdict`; + the lib run must execute the test-only interrupted-migration hook. Review and commit `feat: persist immutable scoped policy versions and evaluations`. **Done:** registry data is durable; nothing is active and no sends are authorized. @@ -160,8 +168,10 @@ valid evaluations may be stored but are never promotion eligible. ### Task 3: Atomic bootstrap, promotion and rollback **Files:** modify `src/policy_registry.rs`, `src/policy_registry/types.rs`, -`src/support.rs`, `src/lib.rs`, `tests/policy_registry.rs`; -create `docs/policy-promotion.md`. +`src/policy_registry/evaluation.rs`, `src/support.rs`, `src/lib.rs`, +`tests/policy_registry.rs`; create `docs/policy-promotion.md`. The evaluation-module +change is limited to extracting/exposing its existing strict manifest validator +for reuse by bootstrap; do not duplicate the validator or change selection rules. ```rust Store::activate_policy(&mut self, request: &ActivationRequest) From 2cc5f9dc87af21abf174ecd9520b8402ba46a983 Mon Sep 17 00:00:00 2001 From: Alejandro Castellanos <5405628+k2jac9@users.noreply.github.com> Date: Sun, 15 Nov 2026 18:26:45 -0500 Subject: [PATCH 21/51] feat: persist immutable scoped policy versions and evaluations --- README.md | 2 +- src/lib.rs | 241 ++++++++++++++++- src/policy_registry.rs | 250 ++++++++++++++++- src/policy_registry.sql | 86 ++++++ tests/deterministic_verdict.rs | 2 +- tests/dispatch.rs | 2 +- tests/policy_registry.rs | 479 +++++++++++++++++++++++++++++++++ tests/timed_recovery.rs | 2 +- 8 files changed, 1057 insertions(+), 7 deletions(-) create mode 100644 src/policy_registry.sql create mode 100644 tests/policy_registry.rs diff --git a/README.md b/README.md index 22bbcd3..a527beb 100644 --- a/README.md +++ b/README.md @@ -48,7 +48,7 @@ The second example asks whether a test run has a reproduction record. An origina The built-in rule checks **availability of declared witnesses**. The caller defines which observations qualify for the question. `unsupported` means every declared alternative lacks at least one admitted observation. Missing and withheld IDs have the same effect. The store never needs the payload bytes to answer this rule. -SQLite updates the affected clause counts in the same transaction as admission or withdrawal. Reads and resumes check the verdict against the current view. In-flight opaque work still goes through the original epoch check. Opening a v1 through v8 database migrates it to schema v9. Existing heads and usage records retain their identities. +SQLite updates the affected clause counts in the same transaction as admission or withdrawal. Reads and resumes check the verdict against the current view. In-flight opaque work still goes through the original epoch check. Opening a v1 through v9 database migrates it to schema v10. Existing heads and usage records retain their identities. Use `set_admitted_and_revise` when readers should see the membership update and all affected support heads in the same database snapshot. Any failure rolls back the entire edit. A witness that keeps the same verdict retains its head and cost. Repeating the desired membership while it is already current returns an empty change list. An intervening reversal makes a subsequent call a new edit. The original `set_admitted` and `revise_support` remain available for deferred revision. diff --git a/src/lib.rs b/src/lib.rs index 76261c7..f432b82 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -29,7 +29,7 @@ use std::time::Duration; // "KTSM", stored in the SQLite header. const APPLICATION_ID: i32 = 0x4b54_534d; -const SCHEMA_VERSION: i32 = 9; +const SCHEMA_VERSION: i32 = 10; const INSERT_OBSERVATION: &str = "INSERT INTO evidence(source, payload) VALUES (?1, ?2)"; /// A local trusted runtime owns the database and supplies authenticated actor/scope names. @@ -47,7 +47,18 @@ impl Store { Self::initialize(Connection::open_in_memory()?) } - fn initialize(mut conn: Connection) -> Result { + fn initialize(conn: Connection) -> Result { + Self::initialize_inner( + conn, + #[cfg(test)] + None, + ) + } + + fn initialize_inner( + mut conn: Connection, + #[cfg(test)] after_registry_ddl: Option) -> Result<()>>, + ) -> Result { conn.busy_timeout(Duration::from_secs(5))?; conn.pragma_update(None, "foreign_keys", true)?; conn.pragma_update(None, "synchronous", "FULL")?; @@ -97,6 +108,13 @@ impl Store { if version < 9 { tx.execute_batch(include_str!("timed_recovery.sql"))?; } + if version < 10 { + tx.execute_batch(include_str!("policy_registry.sql"))?; + #[cfg(test)] + if let Some(hook) = after_registry_ddl { + hook(&tx)?; + } + } if version < SCHEMA_VERSION { tx.pragma_update(None, "user_version", SCHEMA_VERSION)?; } @@ -426,3 +444,222 @@ pub use recovery_runtime::{ReplyValidation, MeasuredRecoveryReply, RecoveryRun}; mod boot_clock; pub use boot_clock::LinuxBootClock; + +#[cfg(test)] +mod registry_migration_tests { + use super::*; + use std::path::PathBuf; + + fn packet() -> PathBuf { + let root = std::env::var_os("KAIROTENSOR_D2_ARTIFACTS") + .map(PathBuf::from) + .unwrap_or_else(std::env::temp_dir); + let stamp = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap() + .as_nanos(); + let path = root.join(format!("registry-v9-{}-{stamp}", std::process::id())); + std::fs::create_dir_all(&path).unwrap(); + path + } + // Build the actual historical layout, never a v10 file with a relabelled header. + fn populated_v9(path: &Path) -> Result<()> { + let conn = Connection::open(path)?; + conn.pragma_update(None, "foreign_keys", true)?; + for sql in [ + include_str!("schema.sql"), + include_str!("attempts.sql"), + include_str!("support.sql"), + include_str!("dispatch.sql"), + include_str!("model_reuse.sql"), + include_str!("validation.sql"), + include_str!("token_budget.sql"), + include_str!("recovery.sql"), + include_str!("report_verdict.sql"), + include_str!("timed_recovery.sql"), + ] { + conn.execute_batch(sql)?; + } + conn.pragma_update(None, "application_id", APPLICATION_ID)?; + conn.pragma_update(None, "user_version", 9)?; + // Existing Store APIs populate the old schema without initializing/migrating it. + let mut store = Store { conn }; + let evidence = store.observe("v9-fixture", b"old exact evidence\0\xff")?; + store.set_admitted("actor", "scope", evidence, true)?; + let budget = store.create_token_budget(100)?; + for (key, disposition) in [ + ("accepted", Some(Disposition::Accepted)), + ("stale", Some(Disposition::StaleView)), + ("unresolved", None), + ] { + let run = store.create_recovery(key, budget, "actor", "scope", key)?; + let attempt = match store.begin_recovery_attempt(run, &[evidence])? { + RecoveryStart::New { attempt, .. } => attempt, + other => panic!("{other:?}"), + }; + assert!(store.reserve_recovery_attempt(attempt, b"v9 exact request\0\xff", 4, 8)?); + assert_eq!( + store.claim_recovery_dispatch(attempt)?, + Some(b"v9 exact request\0\xff".to_vec()) + ); + if disposition == Some(Disposition::StaleView) { + store.set_admitted("actor", "scope", evidence, false)?; + } + if let Some(expected) = disposition { + assert_eq!( + store.settle(attempt, b"v9 exact output\0\xff", 7)?, + expected + ); + } + } + assert_eq!(store.charged_usage()?, 14); + assert_eq!( + store.token_budget_state(budget)?, + TokenBudgetState { + limit: 100, + spent: 14, + reserved: 12 + } + ); + assert_eq!(store.epoch("actor", "scope")?, 2); + Ok(()) + } + // Every old table, column, row and schema object, including dispatch receipts/holds. + fn snapshot(conn: &Connection) -> Result>)>> { + let mut names = conn.prepare("SELECT name FROM sqlite_schema WHERE type='table' AND name NOT LIKE 'policy_%' ORDER BY name")?; + let tables = names + .query_map([], |r| r.get::<_, String>(0))? + .collect::>>()?; + let mut result = Vec::new(); + for table in tables { + let mut stmt = conn.prepare(&format!( + "SELECT * FROM \"{}\" ORDER BY {}", + table.replace('"', "\"\""), + (1..=conn + .prepare(&format!("SELECT * FROM \"{}\"", table.replace('"', "\"\"")))? + .column_count()) + .map(|v| v.to_string()) + .collect::>() + .join(",") + ))?; + let cols = stmt.column_count(); + let rows = stmt + .query_map([], |r| (0..cols).map(|i| r.get(i)).collect())? + .collect::>>()?; + result.push((table, rows)); + } + let mut stmt = conn.prepare("SELECT type,name,tbl_name,sql FROM sqlite_schema WHERE name NOT LIKE 'policy_%' AND tbl_name NOT LIKE 'policy_%' ORDER BY type,name")?; + result.push(( + "schema".into(), + stmt.query_map([], |r| (0..4).map(|i| r.get(i)).collect())? + .collect::>>()?, + )); + Ok(result) + } + #[test] + fn populated_v9_migration_reopen_and_backup_restore_preserve_every_old_row() -> Result<()> { + let packet = packet(); + let backup = packet.join("original-v9.sqlite"); + populated_v9(&backup)?; + let bytes = std::fs::read(&backup).unwrap(); + let hash = crate::evidence_digest::sha256_hex(&bytes); + std::fs::write(packet.join("original-v9.sha256"), &hash).unwrap(); + let before = snapshot(&Connection::open(&backup)?)?; + std::fs::write(packet.join("old-state.txt"), format!("{before:#?}")).unwrap(); + for name in ["migrated.sqlite", "restored-from-backup.sqlite"] { + let copy = packet.join(name); + std::fs::copy(&backup, ©).unwrap(); + for _ in 0..2 { + let store = Store::open(©)?; + assert_eq!( + store + .conn + .pragma_query_value(None, "user_version", |r| r.get::<_, i32>(0))?, + 10 + ); + assert_eq!(snapshot(&store.conn)?, before); + assert_eq!(store.charged_usage()?, 14); + assert_eq!( + store.token_budget_state(1)?, + TokenBudgetState { + limit: 100, + spent: 14, + reserved: 12 + } + ); + assert_eq!(store.epoch("actor", "scope")?, 2); + assert_eq!( + store + .conn + .query_row("SELECT COUNT(*) FROM policy_versions", [], |r| r + .get::<_, i64>(0))?, + 0 + ); + assert_eq!( + store.conn.query_row( + "SELECT COUNT(*) FROM pragma_foreign_key_check", + [], + |r| r.get::<_, i64>(0) + )?, + 0 + ); + } + } + assert_eq!(std::fs::read(backup).unwrap(), bytes); + eprintln!( + "D2 v9 backup/restore evidence: {} sha256={hash}", + packet.display() + ); + Ok(()) + } + #[test] + fn interrupted_registry_migration_rolls_back_ddl_and_preserves_v9_backup() -> Result<()> { + let packet = packet(); + let backup = packet.join("original-v9.sqlite"); + populated_v9(&backup)?; + let bytes = std::fs::read(&backup).unwrap(); + let hash = crate::evidence_digest::sha256_hex(&bytes); + std::fs::write(packet.join("original-v9.sha256"), &hash).unwrap(); + let copy = packet.join("interrupted.sqlite"); + std::fs::copy(&backup, ©).unwrap(); + let before = snapshot(&Connection::open(©)?)?; + let failure = Store::initialize_inner( + Connection::open(©)?, + Some(|tx| { + assert_eq!( + tx.pragma_query_value(None, "user_version", |r| r.get::<_, i32>(0))?, + 9 + ); + assert_eq!(tx.query_row("SELECT COUNT(*) FROM sqlite_schema WHERE type='table' AND name LIKE 'policy_%'", [], |r|r.get::<_,i64>(0))?, 4); + Err(invalid( + "injected after registry DDL before migration commit", + )) + }), + ); + assert!(failure.is_err()); + let old = Connection::open(©)?; + assert_eq!( + old.pragma_query_value(None, "user_version", |r| r.get::<_, i32>(0))?, + 9 + ); + assert_eq!(snapshot(&old)?, before); + assert_eq!( + old.query_row( + "SELECT COUNT(*) FROM sqlite_schema WHERE name LIKE 'policy_%'", + [], + |r| r.get::<_, i64>(0) + )?, + 0 + ); + drop(old); + assert_eq!(std::fs::read(©).unwrap(), bytes); + let reopened = Store::open(©)?; + assert_eq!(snapshot(&reopened.conn)?, before); + assert_eq!(std::fs::read(&backup).unwrap(), bytes); + eprintln!( + "D2 interrupted migration evidence: {} sha256={hash}", + packet.display() + ); + Ok(()) + } +} diff --git a/src/policy_registry.rs b/src/policy_registry.rs index 451f914..c605549 100644 --- a/src/policy_registry.rs +++ b/src/policy_registry.rs @@ -1,4 +1,4 @@ -//! Pure policy attestations. Validation does not authenticate execution or activate a Store. +//! Immutable policy attestations. Registration neither authenticates execution nor activates a policy. mod evaluation; mod types; use crate::evidence_digest::sha256_hex; @@ -102,3 +102,251 @@ pub fn validate_policy_candidate(c: &PolicyCandidate) -> rusqlite::Result rusqlite::Result { + let columns = fields + .iter() + .map(|(name, _, _)| format!("typeof({name}), length(CAST({name} AS BLOB))")) + .collect::>() + .join(","); + conn.query_row( + &format!("SELECT {columns} FROM {from} WHERE {key} = ?1"), + [id], + |row| { + for (i, (_, kind, max)) in fields.iter().enumerate() { + let actual: String = row.get(2 * i)?; + let len: Option = row.get(2 * i + 1)?; + require( + actual == *kind && len.is_some_and(|n| n >= 0 && n <= *max as i64), + "invalid persisted policy field type or byte length", + )?; + } + Ok(true) + }, + ) + .optional() + .map(|v| v.unwrap_or(false)) +} +fn encoded(value: &T) -> rusqlite::Result> { + bounded(value)?; + serde_json::to_vec(value).map_err(|_| bad("policy record encoding")) +} + +fn stored_candidate(conn: &Connection, id: &str) -> rusqlite::Result> { + let from = "policy_versions v LEFT JOIN evidence e ON e.id = v.evidence_id"; + if !preflight_fields( + conn, + from, + "v.version_id", + id, + &[ + ("v.version_id", "text", MAX_EVALUATION_BYTES), + ("v.actor", "text", MAX_EVALUATION_BYTES), + ("v.scope", "text", MAX_EVALUATION_BYTES), + ("v.purpose", "text", MAX_EVALUATION_BYTES), + ("v.artifact_sha256", "text", 64), + ("v.compatibility_contract_sha256", "text", 64), + ("e.source", "text", 64), + ("e.payload", "blob", MAX_ARTIFACT_BYTES), + ], + )? { + return Ok(None); + } + let (candidate, contract, source) = conn.query_row( + &format!("SELECT v.version_id,v.actor,v.scope,v.purpose,v.artifact_sha256,e.payload,v.compatibility_contract_sha256,e.source FROM {from} WHERE v.version_id=?1"), + [id], + |r| { + Ok(( + PolicyCandidate { + version_id: r.get(0)?, actor: r.get(1)?, scope: r.get(2)?, + purpose: r.get(3)?, sha256: r.get(4)?, artifact: r.get(5)?, + }, + r.get::<_, String>(6)?, + r.get::<_, String>(7)?, + )) + }, + )?; + let artifact = validate_policy_candidate(&candidate)?; + require( + artifact.compatibility_contract_sha256 == contract && source == "policy-registry/1", + "persisted policy artifact metadata mismatch", + )?; + Ok(Some(candidate)) +} + +// The D1 validator checks shape and strict manifest structure even when it refuses +// comparison early. Unknown evaluator/unit row semantics remain unevaluated. The +// original object is always passed unchanged; no identity is relabelled to approve it. +fn bind_evaluation(conn: &Connection, e: &PolicyEvaluation) -> rusqlite::Result<()> { + let _decision = evaluate_policy_comparison(e)?; + let manifest: ComparisonManifest = parse(&e.experiment_manifest, MAX_EVALUATION_BYTES)?; + require( + manifest.actor == e.actor + && manifest.scope == e.scope + && manifest.purpose == e.purpose + && manifest.provenance == e.provenance + && sha256_hex(&e.experiment_manifest) == e.experiment_sha256, + "evaluation manifest identity mismatch", + )?; + require( + e.baseline_id != e.candidate_id + || (e.baseline_sha256 == e.candidate_sha256 && e.baseline_rows == e.candidate_rows), + "baseline-only evaluation must contain identical rows", + )?; + for (id, digest) in [ + (&e.baseline_id, &e.baseline_sha256), + (&e.candidate_id, &e.candidate_sha256), + ] { + let candidate = + stored_candidate(conn, id)?.ok_or_else(|| bad("unregistered policy version"))?; + require( + candidate.actor == e.actor + && candidate.scope == e.scope + && candidate.purpose == e.purpose + && &candidate.sha256 == digest, + "evaluation registered version identity mismatch", + )?; + require( + manifest + .policies + .iter() + .any(|p| &p.artifact.sha256 == digest && p.artifact.bytes == candidate.artifact), + "evaluation must bind exact registered artifact bytes", + )?; + } + Ok(()) +} + +fn stored_evaluation(conn: &Connection, id: &str) -> rusqlite::Result> { + if !preflight_fields( + conn, + "policy_evaluations", + "evaluation_id", + id, + &[ + ("evaluation_id", "text", MAX_EVALUATION_BYTES), + ("actor", "text", MAX_EVALUATION_BYTES), + ("scope", "text", MAX_EVALUATION_BYTES), + ("purpose", "text", MAX_EVALUATION_BYTES), + ("baseline_id", "text", MAX_EVALUATION_BYTES), + ("candidate_id", "text", MAX_EVALUATION_BYTES), + ("baseline_sha256", "text", 64), + ("candidate_sha256", "text", 64), + ("experiment_sha256", "text", 64), + ("evaluator_version", "text", MAX_EVALUATION_BYTES), + ("evaluator_sha256", "text", 64), + ("evaluation_sha256", "text", 64), + ("evaluation_bytes", "blob", MAX_EVALUATION_BYTES), + ("experiment_manifest", "blob", MAX_EVALUATION_BYTES), + ("baseline_rows", "blob", MAX_EVALUATION_BYTES), + ("candidate_rows", "blob", MAX_EVALUATION_BYTES), + ], + )? { + return Ok(None); + } + let e = conn.query_row( + "SELECT evaluation_bytes, evaluation_id, actor, scope, purpose, + baseline_id, candidate_id, baseline_sha256, candidate_sha256, experiment_sha256, + evaluator_version, evaluator_sha256, evaluation_sha256, experiment_manifest, + baseline_rows, candidate_rows FROM policy_evaluations WHERE evaluation_id=?1", + [id], + |r| { + let raw: Vec = r.get(0)?; + let e = parse_policy_evaluation(&raw)?; + for (i, expected) in [ + &e.evaluation_id, + &e.actor, + &e.scope, + &e.purpose, + &e.baseline_id, + &e.candidate_id, + &e.baseline_sha256, + &e.candidate_sha256, + &e.experiment_sha256, + &e.evaluator_version, + &e.evaluator_sha256, + ] + .iter() + .enumerate() + { + require( + r.get::<_, String>(i + 1)? == **expected, + "persisted evaluation metadata mismatch", + )?; + } + require( + r.get::<_, String>(12)? == evaluation_sha256(&e)? + && r.get::<_, Vec>(13)? == e.experiment_manifest + && r.get::<_, Vec>(14)? == encoded(&e.baseline_rows)? + && r.get::<_, Vec>(15)? == encoded(&e.candidate_rows)? + && raw == encoded(&e)?, + "persisted evaluation bytes or digest mismatch", + )?; + Ok(e) + }, + )?; + bind_evaluation(conn, &e)?; + Ok(Some(e)) +} + +impl crate::Store { + /// Register one immutable scoped artifact and its fresh, unadmitted occurrence. + /// The trusted runtime must authenticate management authority before calling. + pub fn register_policy_candidate( + &mut self, + candidate: &PolicyCandidate, + ) -> rusqlite::Result<()> { + let artifact = validate_policy_candidate(candidate)?; + let tx = self + .conn + .transaction_with_behavior(TransactionBehavior::Immediate)?; + if let Some(existing) = stored_candidate(&tx, &candidate.version_id)? { + require( + existing == *candidate, + "conflicting policy version registration", + )?; + } else { + let evidence_id = tx + .prepare_cached(crate::INSERT_OBSERVATION)? + .insert(params!["policy-registry/1", candidate.artifact])?; + tx.execute("INSERT INTO policy_versions(version_id,actor,scope,purpose,evidence_id,compatibility_contract_sha256,artifact_sha256) VALUES (?1,?2,?3,?4,?5,?6,?7)", + params![candidate.version_id,candidate.actor,candidate.scope,candidate.purpose,evidence_id,artifact.compatibility_contract_sha256,candidate.sha256])?; + } + tx.commit() + } + + /// Persist exact attestation contents. A stored evaluation does not imply + /// superiority, authenticated observation, semantic validation of unknown + /// evaluator assertions, or admission. Identical retries do not append rows. + pub fn record_policy_evaluation( + &mut self, + evaluation: &PolicyEvaluation, + ) -> rusqlite::Result<()> { + let digest = evaluation_sha256(evaluation)?; + let tx = self + .conn + .transaction_with_behavior(TransactionBehavior::Immediate)?; + bind_evaluation(&tx, evaluation)?; + if let Some(existing) = stored_evaluation(&tx, &evaluation.evaluation_id)? { + require( + existing == *evaluation, + "conflicting policy evaluation registration", + )?; + } else { + let e = evaluation; + tx.execute("INSERT INTO policy_evaluations(evaluation_id,actor,scope,purpose,baseline_id,candidate_id,baseline_sha256,candidate_sha256,experiment_manifest,experiment_sha256,evaluator_version,evaluator_sha256,baseline_rows,candidate_rows,evaluation_bytes,evaluation_sha256) VALUES (?1,?2,?3,?4,?5,?6,?7,?8,?9,?10,?11,?12,?13,?14,?15,?16)", + params![e.evaluation_id,e.actor,e.scope,e.purpose,e.baseline_id,e.candidate_id,e.baseline_sha256,e.candidate_sha256,e.experiment_manifest,e.experiment_sha256,e.evaluator_version,e.evaluator_sha256,encoded(&e.baseline_rows)?,encoded(&e.candidate_rows)?,encoded(e)?,digest])?; + } + tx.commit() + } +} diff --git a/src/policy_registry.sql b/src/policy_registry.sql new file mode 100644 index 0000000..01a30fe --- /dev/null +++ b/src/policy_registry.sql @@ -0,0 +1,86 @@ +-- Scoped immutable attestations. Only the trusted runtime authenticates the caller. +CREATE TABLE policy_versions ( + version_id TEXT PRIMARY KEY NOT NULL CHECK (length(trim(version_id)) > 0), + actor TEXT NOT NULL CHECK (length(trim(actor)) > 0), + scope TEXT NOT NULL CHECK (length(trim(scope)) > 0), + purpose TEXT NOT NULL CHECK (length(trim(purpose)) > 0), + evidence_id INTEGER NOT NULL UNIQUE REFERENCES evidence(id), + compatibility_contract_sha256 TEXT NOT NULL CHECK (length(compatibility_contract_sha256) = 64), + artifact_sha256 TEXT NOT NULL CHECK (length(artifact_sha256) = 64), + UNIQUE (version_id, actor, scope, purpose) +); +CREATE TABLE policy_evaluations ( + evaluation_id TEXT PRIMARY KEY NOT NULL CHECK (length(trim(evaluation_id)) > 0), + actor TEXT NOT NULL CHECK (length(trim(actor)) > 0), + scope TEXT NOT NULL CHECK (length(trim(scope)) > 0), + purpose TEXT NOT NULL CHECK (length(trim(purpose)) > 0), + baseline_id TEXT NOT NULL, + candidate_id TEXT NOT NULL, + baseline_sha256 TEXT NOT NULL CHECK (length(baseline_sha256) = 64), + candidate_sha256 TEXT NOT NULL CHECK (length(candidate_sha256) = 64), + experiment_manifest BLOB NOT NULL CHECK (typeof(experiment_manifest) = 'blob' AND length(experiment_manifest) <= 16777216), + experiment_sha256 TEXT NOT NULL CHECK (length(experiment_sha256) = 64), + evaluator_version TEXT NOT NULL CHECK (length(trim(evaluator_version)) > 0), + evaluator_sha256 TEXT NOT NULL CHECK (length(evaluator_sha256) = 64), + baseline_rows BLOB NOT NULL CHECK (typeof(baseline_rows) = 'blob' AND length(baseline_rows) <= 16777216), + candidate_rows BLOB NOT NULL CHECK (typeof(candidate_rows) = 'blob' AND length(candidate_rows) <= 16777216), + evaluation_bytes BLOB NOT NULL CHECK (typeof(evaluation_bytes) = 'blob' AND length(evaluation_bytes) <= 16777216), + evaluation_sha256 TEXT NOT NULL CHECK (length(evaluation_sha256) = 64), + FOREIGN KEY (baseline_id, actor, scope, purpose) REFERENCES policy_versions(version_id, actor, scope, purpose), + FOREIGN KEY (candidate_id, actor, scope, purpose) REFERENCES policy_versions(version_id, actor, scope, purpose), + UNIQUE (evaluation_id, actor, scope, purpose) +); +CREATE TABLE policy_heads ( + actor TEXT NOT NULL, + scope TEXT NOT NULL, + purpose TEXT NOT NULL, + version_id TEXT NOT NULL, + revision INTEGER NOT NULL CHECK (typeof(revision) = 'integer' AND revision > 0), + baseline_manifest_sha256 TEXT NOT NULL CHECK (length(baseline_manifest_sha256) = 64), + PRIMARY KEY (actor, scope, purpose), + FOREIGN KEY (version_id, actor, scope, purpose) REFERENCES policy_versions(version_id, actor, scope, purpose) +) WITHOUT ROWID; +CREATE TABLE policy_events ( + actor TEXT NOT NULL, + scope TEXT NOT NULL, + purpose TEXT NOT NULL, + revision INTEGER NOT NULL CHECK (typeof(revision) = 'integer' AND revision > 0), + action TEXT NOT NULL CHECK (action IN ('bootstrap', 'promote', 'rollback')), + previous_version_id TEXT, + target_version_id TEXT NOT NULL, + evaluation_id TEXT, + manifest_sha256 TEXT NOT NULL CHECK (length(manifest_sha256) = 64), + reason TEXT NOT NULL CHECK (length(trim(reason)) > 0), + PRIMARY KEY (actor, scope, purpose, revision), + FOREIGN KEY (previous_version_id, actor, scope, purpose) REFERENCES policy_versions(version_id, actor, scope, purpose), + FOREIGN KEY (target_version_id, actor, scope, purpose) REFERENCES policy_versions(version_id, actor, scope, purpose), + FOREIGN KEY (evaluation_id, actor, scope, purpose) REFERENCES policy_evaluations(evaluation_id, actor, scope, purpose) +) WITHOUT ROWID; + +CREATE TRIGGER policy_versions_no_update BEFORE UPDATE ON policy_versions BEGIN SELECT RAISE(ABORT, 'immutable policy version'); END; +CREATE TRIGGER policy_versions_no_delete BEFORE DELETE ON policy_versions BEGIN SELECT RAISE(ABORT, 'immutable policy version'); END; +CREATE TRIGGER policy_evaluations_no_update BEFORE UPDATE ON policy_evaluations BEGIN SELECT RAISE(ABORT, 'immutable policy evaluation'); END; +CREATE TRIGGER policy_evaluations_no_delete BEFORE DELETE ON policy_evaluations BEGIN SELECT RAISE(ABORT, 'immutable policy evaluation'); END; +CREATE TRIGGER policy_events_no_update BEFORE UPDATE ON policy_events BEGIN SELECT RAISE(ABORT, 'immutable policy event'); END; +CREATE TRIGGER policy_events_no_delete BEFORE DELETE ON policy_events BEGIN SELECT RAISE(ABORT, 'immutable policy event'); END; +CREATE TRIGGER policy_heads_initial_revision BEFORE INSERT ON policy_heads WHEN NEW.revision <> 1 +BEGIN SELECT RAISE(ABORT, 'policy head must start at revision one'); END; +CREATE TRIGGER policy_heads_monotonic BEFORE UPDATE ON policy_heads +WHEN NEW.actor <> OLD.actor OR NEW.scope <> OLD.scope OR NEW.purpose <> OLD.purpose + OR OLD.revision = 9223372036854775807 OR NEW.revision <> OLD.revision + 1 +BEGIN SELECT RAISE(ABORT, 'policy head revision must advance exactly once'); END; +CREATE TRIGGER policy_heads_no_delete BEFORE DELETE ON policy_heads BEGIN SELECT RAISE(ABORT, 'policy head history cannot reset'); END; +-- SQLite REPLACE deletes conflicting rows without delete triggers unless recursive +-- triggers are enabled. Refuse that path even with the default connection setting. +CREATE TRIGGER policy_versions_no_replace BEFORE INSERT ON policy_versions +WHEN EXISTS (SELECT 1 FROM policy_versions WHERE version_id = NEW.version_id OR evidence_id = NEW.evidence_id) +BEGIN SELECT RAISE(ABORT, 'immutable policy version'); END; +CREATE TRIGGER policy_evaluations_no_replace BEFORE INSERT ON policy_evaluations +WHEN EXISTS (SELECT 1 FROM policy_evaluations WHERE evaluation_id = NEW.evaluation_id) +BEGIN SELECT RAISE(ABORT, 'immutable policy evaluation'); END; +CREATE TRIGGER policy_events_no_replace BEFORE INSERT ON policy_events +WHEN EXISTS (SELECT 1 FROM policy_events WHERE actor = NEW.actor AND scope = NEW.scope AND purpose = NEW.purpose AND revision = NEW.revision) +BEGIN SELECT RAISE(ABORT, 'immutable policy event'); END; +CREATE TRIGGER policy_heads_no_replace BEFORE INSERT ON policy_heads +WHEN EXISTS (SELECT 1 FROM policy_heads WHERE actor = NEW.actor AND scope = NEW.scope AND purpose = NEW.purpose) +BEGIN SELECT RAISE(ABORT, 'policy head history cannot reset'); END; diff --git a/tests/deterministic_verdict.rs b/tests/deterministic_verdict.rs index 209fe9f..b571fa9 100644 --- a/tests/deterministic_verdict.rs +++ b/tests/deterministic_verdict.rs @@ -426,7 +426,7 @@ fn v7_migration_preserves_accepted_usage_and_unknown_reservations() -> Result<() let db = Connection::open(&file.0)?; assert_eq!( db.pragma_query_value(None, "user_version", |r| r.get::<_, i64>(0))?, - 9 + 10 ); assert_eq!( db.query_row("SELECT COUNT(*) FROM pragma_foreign_key_check", [], |r| r diff --git a/tests/dispatch.rs b/tests/dispatch.rs index d0a617a..927e377 100644 --- a/tests/dispatch.rs +++ b/tests/dispatch.rs @@ -102,7 +102,7 @@ fn migration_from_v2_enables_dispatch_on_old_pending_attempts() -> Result<()> { drop(store); let audit = Connection::open(&file.0)?; let version: i32 = audit.pragma_query_value(None, "user_version", |row| row.get(0))?; - assert_eq!(version, 9); + assert_eq!(version, 10); let request: Vec = audit.query_row( "SELECT request FROM dispatches WHERE attempt_id = 1", [], |row| row.get(0), )?; diff --git a/tests/policy_registry.rs b/tests/policy_registry.rs new file mode 100644 index 0000000..ae49442 --- /dev/null +++ b/tests/policy_registry.rs @@ -0,0 +1,479 @@ +use kairotensor::policy_registry::*; +use kairotensor::Store; +use rusqlite::{params, Connection, Result}; +use sha2::{Digest, Sha256}; +use std::path::PathBuf; + +fn sha(bytes: &[u8]) -> String { + Sha256::digest(bytes) + .iter() + .map(|b| format!("{b:02x}")) + .collect() +} +struct Database(PathBuf); +impl Database { + fn new() -> Self { + let stamp = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap() + .as_nanos(); + Self(std::env::temp_dir().join(format!( + "policy-registry-{}-{stamp}.sqlite", + std::process::id() + ))) + } + fn conn(&self) -> Result { + let c = Connection::open(&self.0)?; + c.pragma_update(None, "foreign_keys", true)?; + Ok(c) + } +} +impl Drop for Database { + fn drop(&mut self) { + let _ = std::fs::remove_file(&self.0); + } +} +fn candidate(id: &str) -> PolicyCandidate { + let artifact = br#"{"schema_version":1,"policy_name":"always_stop","purpose":"fixture/recovery-policy","compatibility_contract_sha256":"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"}"#.to_vec(); + PolicyCandidate { + version_id: id.into(), + actor: "actor".into(), + scope: "scope".into(), + purpose: "fixture/recovery-policy".into(), + sha256: sha(&artifact), + artifact, + } +} +fn fixture() -> PolicyEvaluation { + let output = std::process::Command::new("python3").args(["-c", "import sys,json;sys.path.insert(0,'scripts');from test_policy_evaluation_v2 import fixture;print(json.dumps(fixture()))"]).output().unwrap(); + assert!(output.status.success()); + serde_json::from_slice(&output.stdout).unwrap() +} +fn register_pair(store: &mut Store, e: &PolicyEvaluation) -> Result<()> { + let m: ComparisonManifest = serde_json::from_slice(&e.experiment_manifest).unwrap(); + for (id, digest) in [ + (&e.baseline_id, &e.baseline_sha256), + (&e.candidate_id, &e.candidate_sha256), + ] { + let artifact = m + .policies + .iter() + .find(|p| &p.artifact.sha256 == digest) + .unwrap() + .artifact + .bytes + .clone(); + store.register_policy_candidate(&PolicyCandidate { + version_id: id.clone(), + actor: e.actor.clone(), + scope: e.scope.clone(), + purpose: e.purpose.clone(), + artifact, + sha256: digest.clone(), + })?; + } + Ok(()) +} +fn count(c: &Connection, table: &str) -> Result { + c.query_row(&format!("SELECT COUNT(*) FROM {table}"), [], |r| r.get(0)) +} +fn inactive(c: &Connection) -> Result<()> { + for table in [ + "policy_heads", + "policy_events", + "admission_events", + "attempts", + "dispatches", + "token_reservations", + ] { + assert_eq!(count(c, table)?, 0, "{table}"); + } + Ok(()) +} +#[test] +fn registration_is_durable_exact_idempotent_and_allocates_distinct_unadmitted_occurrences( +) -> Result<()> { + let db = Database::new(); + let mut s = Store::open(&db.0)?; + let c = candidate("v1"); + s.register_policy_candidate(&c)?; + s.register_policy_candidate(&c)?; + drop(s); + let mut s = Store::open(&db.0)?; + s.register_policy_candidate(&c)?; + let mut other = c.clone(); + other.version_id = "v2".into(); + s.register_policy_candidate(&other)?; + let sql = db.conn()?; + assert_eq!(count(&sql, "evidence")?, 2); + assert_eq!(count(&sql, "policy_versions")?, 2); + let mut stmt=sql.prepare("SELECT e.id,e.payload FROM policy_versions v JOIN evidence e ON e.id=v.evidence_id ORDER BY v.version_id")?; + let rows = stmt + .query_map([], |r| Ok((r.get::<_, i64>(0)?, r.get::<_, Vec>(1)?)))? + .collect::>>()?; + assert_ne!(rows[0].0, rows[1].0); + assert_eq!(rows[0].1, c.artifact); + assert_eq!(rows[1].1, c.artifact); + assert_eq!(s.read_evidence("actor", "scope", rows[0].0)?, None); + assert_eq!(s.epoch("actor", "scope")?, 0); + inactive(&sql) +} +#[test] +fn conflicting_registration_and_invalid_artifacts_leave_no_occurrences() -> Result<()> { + let db = Database::new(); + let mut s = Store::open(&db.0)?; + let c = candidate("v1"); + s.register_policy_candidate(&c)?; + for field in ["actor", "scope", "purpose", "artifact", "sha256"] { + let mut changed = c.clone(); + match field { + "actor" => changed.actor = "other".into(), + "scope" => changed.scope = "other".into(), + "purpose" => changed.purpose = "other".into(), + "artifact" => { + changed.artifact.push(b' '); + changed.sha256 = sha(&changed.artifact); + } + _ => changed.sha256 = "0".repeat(64), + } + assert!(s.register_policy_candidate(&changed).is_err(), "{field}"); + } + for artifact in [b"{}".to_vec(), vec![b' '; MAX_ARTIFACT_BYTES + 1]] { + let mut changed = candidate("new"); + changed.artifact = artifact; + changed.sha256 = sha(&changed.artifact); + assert!(s.register_policy_candidate(&changed).is_err()); + } + let sql = db.conn()?; + assert_eq!(count(&sql, "evidence")?, 1); + assert_eq!(count(&sql, "policy_versions")?, 1); + inactive(&sql) +} +#[test] +fn failed_version_insert_rolls_back_fresh_evidence() -> Result<()> { + let db = Database::new(); + let mut s = Store::open(&db.0)?; + let sql = db.conn()?; + sql.execute_batch("CREATE TRIGGER fail_version BEFORE INSERT ON policy_versions BEGIN SELECT RAISE(ABORT,'injected constraint'); END;")?; + assert!(s.register_policy_candidate(&candidate("v1")).is_err()); + assert_eq!(count(&sql, "evidence")?, 0); + inactive(&sql) +} +#[test] +fn evaluations_bind_exact_registered_versions_and_preserve_manifest_and_rows() -> Result<()> { + let db = Database::new(); + let mut s = Store::open(&db.0)?; + let e = fixture(); + assert!(s.record_policy_evaluation(&e).is_err()); + register_pair(&mut s, &e)?; + s.record_policy_evaluation(&e)?; + s.record_policy_evaluation(&e)?; + let sql = db.conn()?; + let (raw,manifest,b,c):(Vec,Vec,Vec,Vec)=sql.query_row("SELECT evaluation_bytes,experiment_manifest,baseline_rows,candidate_rows FROM policy_evaluations",[],|r|Ok((r.get(0)?,r.get(1)?,r.get(2)?,r.get(3)?)))?; + assert_eq!(parse_policy_evaluation(&raw)?, e); + assert_eq!(manifest, e.experiment_manifest); + assert_eq!( + serde_json::from_slice::>(&b).unwrap(), + e.baseline_rows + ); + assert_eq!( + serde_json::from_slice::>(&c).unwrap(), + e.candidate_rows + ); + drop(s); + let mut s = Store::open(&db.0)?; + s.record_policy_evaluation(&e)?; + for field in [ + "actor", + "scope", + "purpose", + "digest", + "rows", + "manifest", + "candidate_id", + ] { + let mut bad = e.clone(); + match field { + "actor" => bad.actor = "other".into(), + "scope" => bad.scope = "other".into(), + "purpose" => bad.purpose = "other".into(), + "digest" => bad.candidate_sha256 = "0".repeat(64), + "rows" => bad.candidate_rows[0].charged = Some(5), + "manifest" => { + bad.experiment_manifest.push(b' '); + bad.experiment_sha256 = sha(&bad.experiment_manifest); + } + _ => bad.candidate_id = "missing".into(), + } + assert!(s.record_policy_evaluation(&bad).is_err(), "{field}"); + } + assert_eq!(count(&sql, "policy_evaluations")?, 1); + inactive(&sql) +} +#[test] +fn baseline_only_and_incomplete_evaluations_are_evidence_without_activation() -> Result<()> { + let db = Database::new(); + let mut s = Store::open(&db.0)?; + let e = fixture(); + register_pair(&mut s, &e)?; + let mut baseline = e.clone(); + baseline.evaluation_id = "baseline".into(); + baseline.candidate_id = baseline.baseline_id.clone(); + baseline.candidate_sha256 = baseline.baseline_sha256.clone(); + baseline.candidate_rows = baseline.baseline_rows.clone(); + assert_eq!( + evaluate_policy_comparison(&baseline)?, + SelectionDecision::Refused(SelectionRefusal::NoImprovement) + ); + s.record_policy_evaluation(&baseline)?; + let mut bad = baseline.clone(); + bad.evaluation_id = "bad-baseline".into(); + bad.candidate_rows[0].charged = Some(5); + assert!(s.record_policy_evaluation(&bad).is_err()); + let mut incomplete = e.clone(); + incomplete.evaluation_id = "incomplete".into(); + incomplete.candidate_rows[0].charged = None; + assert_eq!( + evaluate_policy_comparison(&incomplete)?, + SelectionDecision::Refused(SelectionRefusal::Incomplete) + ); + s.record_policy_evaluation(&incomplete)?; + let mut malformed = e.clone(); + malformed.evaluation_id = "malformed".into(); + malformed.candidate_rows[0].false_refusal = true; + assert!(s.record_policy_evaluation(&malformed).is_err()); + let mut unknown = e.clone(); + unknown.evaluation_id = "unknown-evaluator".into(); + unknown.evaluator_version = "future/2".into(); + s.record_policy_evaluation(&unknown)?; + assert_eq!(count(&db.conn()?, "policy_evaluations")?, 3); + inactive(&db.conn()?) +} +#[test] +fn immutable_registry_rows_and_monotonic_head_constraints() -> Result<()> { + let db = Database::new(); + let mut s = Store::open(&db.0)?; + let e = fixture(); + register_pair(&mut s, &e)?; + s.record_policy_evaluation(&e)?; + let sql = db.conn()?; + for table in ["policy_versions", "policy_evaluations"] { + assert!(sql + .execute(&format!("UPDATE {table} SET actor='other'"), []) + .is_err()); + assert!(sql.execute(&format!("DELETE FROM {table}"), []).is_err()); + } + // Direct SQL here tests schema constraints only, never an activation API or admission. + sql.execute( + "INSERT INTO policy_heads VALUES ('actor','scope','fixture/recovery-policy','v1',1,?1)", + [&e.experiment_sha256], + )?; + assert!(sql + .execute("UPDATE policy_heads SET revision=1", []) + .is_err()); + assert!(sql + .execute("UPDATE policy_heads SET revision=3", []) + .is_err()); + assert!(sql.execute("DELETE FROM policy_heads", []).is_err()); + sql.execute("INSERT INTO policy_events(actor,scope,purpose,revision,action,previous_version_id,target_version_id,evaluation_id,manifest_sha256,reason) VALUES ('actor','scope','fixture/recovery-policy',1,'bootstrap',NULL,'v1','e1',?1,'fixture only')",[&e.experiment_sha256])?; + assert!(sql + .execute("UPDATE policy_events SET reason='changed'", []) + .is_err()); + assert!(sql.execute("DELETE FROM policy_events", []).is_err()); + assert!(sql + .execute("INSERT INTO policy_events SELECT * FROM policy_events", []) + .is_err()); + assert_eq!(count(&sql, "admission_events")?, 0); + Ok(()) +} +#[test] +fn existing_artifact_payload_is_bounded_and_verified_before_duplicate_acceptance() -> Result<()> { + let db = Database::new(); + let mut s = Store::open(&db.0)?; + let c = candidate("v1"); + s.register_policy_candidate(&c)?; + let sql = db.conn()?; + for payload in [b"corrupt".to_vec(), vec![0; MAX_ARTIFACT_BYTES + 1]] { + sql.execute("UPDATE evidence SET payload=?1", params![payload])?; + assert!(s.register_policy_candidate(&c).is_err()); + } + sql.execute("UPDATE evidence SET payload='wrong-type'", [])?; + assert!(s.register_policy_candidate(&c).is_err()); + assert_eq!(count(&sql, "evidence")?, 1); + inactive(&sql) +} + +#[test] +fn unknown_evaluator_is_retained_unchanged_and_never_relabelled_as_known() -> Result<()> { + let db = Database::new(); + let mut s = Store::open(&db.0)?; + let mut e = fixture(); + register_pair(&mut s, &e)?; + e.evaluator_version = "unknown/99".into(); + e.evaluator_sha256 = "b".repeat(64); + // Unsupported assertions are explicitly unevaluated, not interpreted by a cloned selector. + e.candidate_rows[0].false_refusal = true; + assert_eq!( + evaluate_policy_comparison(&e)?, + SelectionDecision::Refused(SelectionRefusal::Incomparable) + ); + s.record_policy_evaluation(&e)?; + let raw: Vec = + db.conn()? + .query_row("SELECT evaluation_bytes FROM policy_evaluations", [], |r| { + r.get(0) + })?; + assert_eq!(raw, serde_json::to_vec(&e).unwrap()); + let persisted = parse_policy_evaluation(&raw)?; + assert_eq!( + evaluate_policy_comparison(&persisted)?, + SelectionDecision::Refused(SelectionRefusal::Incomparable) + ); + for kind in ["shape", "manifest", "identity"] { + let mut bad = e.clone(); + bad.evaluation_id = format!("invalid-{kind}"); + match kind { + "shape" => bad.candidate_rows[0].case_id = "".into(), + "manifest" => bad.experiment_manifest = b"{}".to_vec(), + _ => bad.scope = "other".into(), + } + assert!(s.record_policy_evaluation(&bad).is_err(), "{kind}"); + } + assert_eq!(count(&db.conn()?, "policy_evaluations")?, 1); + inactive(&db.conn()?) +} +#[test] +fn corrupted_persisted_evaluation_is_not_accepted_as_identical() -> Result<()> { + let db = Database::new(); + let mut s = Store::open(&db.0)?; + let e = fixture(); + register_pair(&mut s, &e)?; + s.record_policy_evaluation(&e)?; + let sql = db.conn()?; + // Simulate host corruption outside the ordinary immutable-write boundary. + sql.execute_batch( + "DROP TRIGGER policy_evaluations_no_update; PRAGMA ignore_check_constraints=ON;", + )?; + for (column, original) in [ + ("evaluation_bytes", serde_json::to_vec(&e).unwrap()), + ("experiment_manifest", e.experiment_manifest.clone()), + ( + "baseline_rows", + serde_json::to_vec(&e.baseline_rows).unwrap(), + ), + ( + "candidate_rows", + serde_json::to_vec(&e.candidate_rows).unwrap(), + ), + ] { + for expression in ["X'00'", "'text'", "zeroblob(16777217)"] { + sql.execute( + &format!("UPDATE policy_evaluations SET {column}={expression}"), + [], + )?; + assert!( + s.record_policy_evaluation(&e).is_err(), + "{column} {expression}" + ); + } + sql.execute( + &format!("UPDATE policy_evaluations SET {column}=?1"), + [original], + )?; + } + sql.execute( + "UPDATE policy_evaluations SET evaluation_sha256=?1", + ["0".repeat(64)], + )?; + assert!(s.record_policy_evaluation(&e).is_err()); + assert_eq!(count(&sql, "policy_evaluations")?, 1); + inactive(&sql) +} +#[test] +fn sqlite_replace_cannot_rewrite_immutable_registry_or_reset_head_history() -> Result<()> { + let db = Database::new(); + let mut s = Store::open(&db.0)?; + let e = fixture(); + register_pair(&mut s, &e)?; + s.record_policy_evaluation(&e)?; + let sql = db.conn()?; + for table in ["policy_versions", "policy_evaluations"] { + assert!( + sql.execute( + &format!("INSERT OR REPLACE INTO {table} SELECT * FROM {table}"), + [] + ) + .is_err(), + "{table}" + ); + } + sql.execute( + "INSERT INTO policy_heads VALUES ('actor','scope','fixture/recovery-policy','v1',1,?1)", + [&e.experiment_sha256], + )?; + sql.execute("UPDATE policy_heads SET revision=2", [])?; + assert!(sql.execute("INSERT OR REPLACE INTO policy_heads VALUES ('actor','scope','fixture/recovery-policy','v1',1,?1)",[&e.experiment_sha256]).is_err()); + sql.execute("INSERT INTO policy_events VALUES ('actor','scope','fixture/recovery-policy',1,'bootstrap',NULL,'v1','e1',?1,'fixture only')",[&e.experiment_sha256])?; + assert!(sql + .execute( + "INSERT OR REPLACE INTO policy_events SELECT * FROM policy_events", + [] + ) + .is_err()); + Ok(()) +} +#[test] +fn concurrent_identical_registrations_allocate_only_one_occurrence() -> Result<()> { + let db = Database::new(); + drop(Store::open(&db.0)?); + let barrier = std::sync::Arc::new(std::sync::Barrier::new(2)); + let workers = (0..2) + .map(|_| { + let path = db.0.clone(); + let barrier = barrier.clone(); + std::thread::spawn(move || -> Result<()> { + let mut s = Store::open(path)?; + barrier.wait(); + s.register_policy_candidate(&candidate("same")) + }) + }) + .collect::>(); + for worker in workers { + worker.join().unwrap()?; + } + assert_eq!(count(&db.conn()?, "policy_versions")?, 1); + assert_eq!(count(&db.conn()?, "evidence")?, 1); + inactive(&db.conn()?) +} + +#[test] +fn evaluation_cannot_borrow_same_artifact_from_another_scope() -> Result<()> { + let db = Database::new(); + let mut s = Store::open(&db.0)?; + let mut e = fixture(); + register_pair(&mut s, &e)?; + let manifest: ComparisonManifest = serde_json::from_slice(&e.experiment_manifest).unwrap(); + let frozen = manifest + .policies + .iter() + .find(|p| p.artifact.sha256 == e.candidate_sha256) + .unwrap(); + let foreign = PolicyCandidate { + version_id: "foreign-version".into(), + actor: e.actor.clone(), + scope: "other-scope".into(), + purpose: e.purpose.clone(), + artifact: frozen.artifact.bytes.clone(), + sha256: frozen.artifact.sha256.clone(), + }; + s.register_policy_candidate(&foreign)?; + e.candidate_id = foreign.version_id; + // The pure evaluator cannot see registration scope; the Store must bind it. + assert!(matches!( + evaluate_policy_comparison(&e)?, + SelectionDecision::Eligible { .. } + )); + assert!(s.record_policy_evaluation(&e).is_err()); + assert_eq!(count(&db.conn()?, "policy_evaluations")?, 0); + inactive(&db.conn()?) +} diff --git a/tests/timed_recovery.rs b/tests/timed_recovery.rs index 2593619..fd36b51 100644 --- a/tests/timed_recovery.rs +++ b/tests/timed_recovery.rs @@ -939,7 +939,7 @@ fn schema8_migration_keeps_legacy_sent_denied_cancelled_and_deterministic_record let db = Connection::open(&file.0)?; assert_eq!( db.pragma_query_value(None, "user_version", |r| r.get::<_, i64>(0))?, - 9 + 10 ); assert_eq!( db.query_row("SELECT COUNT(*) FROM pragma_foreign_key_check", [], |r| r From f1a0af00b6e420e46fd381b0b07688563311b3e1 Mon Sep 17 00:00:00 2001 From: Alejandro Castellanos <5405628+k2jac9@users.noreply.github.com> Date: Sun, 15 Nov 2026 18:33:48 -0500 Subject: [PATCH 22/51] fix: prevent rowid replacement of immutable policy records --- src/policy_registry.sql | 5 +- tests/policy_registry.rs | 113 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 116 insertions(+), 2 deletions(-) diff --git a/src/policy_registry.sql b/src/policy_registry.sql index 01a30fe..20b040a 100644 --- a/src/policy_registry.sql +++ b/src/policy_registry.sql @@ -1,4 +1,5 @@ -- Scoped immutable attestations. Only the trusted runtime authenticates the caller. +-- WITHOUT ROWID removes undeclared rowid aliases as a REPLACE conflict target. CREATE TABLE policy_versions ( version_id TEXT PRIMARY KEY NOT NULL CHECK (length(trim(version_id)) > 0), actor TEXT NOT NULL CHECK (length(trim(actor)) > 0), @@ -8,7 +9,7 @@ CREATE TABLE policy_versions ( compatibility_contract_sha256 TEXT NOT NULL CHECK (length(compatibility_contract_sha256) = 64), artifact_sha256 TEXT NOT NULL CHECK (length(artifact_sha256) = 64), UNIQUE (version_id, actor, scope, purpose) -); +) WITHOUT ROWID; CREATE TABLE policy_evaluations ( evaluation_id TEXT PRIMARY KEY NOT NULL CHECK (length(trim(evaluation_id)) > 0), actor TEXT NOT NULL CHECK (length(trim(actor)) > 0), @@ -29,7 +30,7 @@ CREATE TABLE policy_evaluations ( FOREIGN KEY (baseline_id, actor, scope, purpose) REFERENCES policy_versions(version_id, actor, scope, purpose), FOREIGN KEY (candidate_id, actor, scope, purpose) REFERENCES policy_versions(version_id, actor, scope, purpose), UNIQUE (evaluation_id, actor, scope, purpose) -); +) WITHOUT ROWID; CREATE TABLE policy_heads ( actor TEXT NOT NULL, scope TEXT NOT NULL, diff --git a/tests/policy_registry.rs b/tests/policy_registry.rs index ae49442..a9c37c7 100644 --- a/tests/policy_registry.rs +++ b/tests/policy_registry.rs @@ -477,3 +477,116 @@ fn evaluation_cannot_borrow_same_artifact_from_another_scope() -> Result<()> { assert_eq!(count(&db.conn()?, "policy_evaluations")?, 0); inactive(&db.conn()?) } + +fn all_table_rows(conn: &Connection) -> Result>)>> { + let mut names = + conn.prepare("SELECT name FROM sqlite_schema WHERE type='table' ORDER BY name")?; + let tables = names + .query_map([], |r| r.get::<_, String>(0))? + .collect::>>()?; + let mut snapshot = Vec::new(); + for table in tables { + let quoted = format!("\"{}\"", table.replace('"', "\"\"")); + let columns = conn + .prepare(&format!("SELECT * FROM {quoted}"))? + .column_count(); + let order = (1..=columns) + .map(|n| n.to_string()) + .collect::>() + .join(","); + let mut query = conn.prepare(&format!("SELECT * FROM {quoted} ORDER BY {order}"))?; + let rows = query + .query_map([], |r| (0..columns).map(|n| r.get(n)).collect())? + .collect::>>()?; + snapshot.push((table, rows)); + } + Ok(snapshot) +} + +fn rowid_replacement_refused(table: &str, alias: &str) -> Result<()> { + let db = Database::new(); + let mut store = Store::open(&db.0)?; + let sql = db.conn()?; + sql.pragma_update(None, "recursive_triggers", false)?; + assert!(sql.pragma_query_value(None, "foreign_keys", |r| r.get::<_, bool>(0))?); + assert!(!sql.pragma_query_value(None, "recursive_triggers", |r| r.get::<_, bool>(0))?); + let original = candidate("original"); + let evaluation = fixture(); + let statement = if table == "policy_versions" { + store.register_policy_candidate(&original)?; + // A valid fresh evidence ID avoids the declared evidence uniqueness guard. + let fresh_evidence = store.observe("rowid-regression", &original.artifact)?; + format!("INSERT OR REPLACE INTO policy_versions({alias},version_id,actor,scope,purpose,evidence_id,compatibility_contract_sha256,artifact_sha256) + SELECT 1,'fresh-version',actor,scope,purpose,{fresh_evidence},compatibility_contract_sha256,artifact_sha256 FROM policy_versions WHERE version_id='original'") + } else { + register_pair(&mut store, &evaluation)?; + store.record_policy_evaluation(&evaluation)?; + format!("INSERT OR REPLACE INTO policy_evaluations({alias},evaluation_id,actor,scope,purpose,baseline_id,candidate_id,baseline_sha256,candidate_sha256,experiment_manifest,experiment_sha256,evaluator_version,evaluator_sha256,baseline_rows,candidate_rows,evaluation_bytes,evaluation_sha256) + SELECT 1,'fresh-evaluation',actor,scope,purpose,baseline_id,candidate_id,baseline_sha256,candidate_sha256,experiment_manifest,experiment_sha256,evaluator_version,evaluator_sha256,baseline_rows,candidate_rows,evaluation_bytes,evaluation_sha256 FROM policy_evaluations WHERE evaluation_id='e1'") + }; + assert_eq!(count(&sql, "pragma_foreign_key_check")?, 0); + let before = all_table_rows(&sql)?; + let result = sql.execute(&statement, []); + assert!( + result.is_err(), + "{table} replacement through {alias} succeeded: {result:?}" + ); + assert_eq!( + all_table_rows(&sql)?, + before, + "{table} {alias} altered database rows" + ); + assert_eq!(count(&sql, "pragma_foreign_key_check")?, 0); + // The old immutable record remains usable through the ordinary exact retry API. + if table == "policy_versions" { + store.register_policy_candidate(&original)?; + } else { + store.record_policy_evaluation(&evaluation)?; + } + assert_eq!(all_table_rows(&sql)?, before); + drop(store); + drop(sql); + let _reopened = Store::open(&db.0)?; + assert_eq!(all_table_rows(&db.conn()?)?, before); + assert_eq!(count(&db.conn()?, "pragma_foreign_key_check")?, 0); + Ok(()) +} + +macro_rules! rowid_replacement_test { + ($name:ident, $table:literal, $alias:literal) => { + #[test] + fn $name() -> Result<()> { + rowid_replacement_refused($table, $alias) + } + }; +} +rowid_replacement_test!( + version_explicit_rowid_replace_preserves_history, + "policy_versions", + "rowid" +); +rowid_replacement_test!( + version_explicit_underscore_rowid_replace_preserves_history, + "policy_versions", + "_rowid_" +); +rowid_replacement_test!( + version_explicit_oid_replace_preserves_history, + "policy_versions", + "oid" +); +rowid_replacement_test!( + evaluation_explicit_rowid_replace_preserves_history, + "policy_evaluations", + "rowid" +); +rowid_replacement_test!( + evaluation_explicit_underscore_rowid_replace_preserves_history, + "policy_evaluations", + "_rowid_" +); +rowid_replacement_test!( + evaluation_explicit_oid_replace_preserves_history, + "policy_evaluations", + "oid" +); From 33b330334da3c2acb22fd7a36b49998b2df7dc4f Mon Sep 17 00:00:00 2001 From: Alejandro Castellanos <5405628+k2jac9@users.noreply.github.com> Date: Sun, 15 Nov 2026 19:00:57 -0500 Subject: [PATCH 23/51] docs: bind activation consistency and diagnostic runtime boundaries --- .../plans/2026-11-15-offline-diagnostics.md | 18 ++++++++++++++++++ .../plans/2026-11-15-policy-activation.md | 5 +++++ .../plans/2026-11-15-runtime-qualification.md | 16 +++++++++++++++- 3 files changed, 38 insertions(+), 1 deletion(-) diff --git a/docs/superpowers/plans/2026-11-15-offline-diagnostics.md b/docs/superpowers/plans/2026-11-15-offline-diagnostics.md index 2d2cf8f..3eb4141 100644 --- a/docs/superpowers/plans/2026-11-15-offline-diagnostics.md +++ b/docs/superpowers/plans/2026-11-15-offline-diagnostics.md @@ -44,6 +44,14 @@ def score_ablations(plan, outcomes): ... ``` Accept only a complete bounded C bundle with all selected evidence available. +Initial public applicability is the direct root of the known initial incident/1 +contract: no retry, inherited/reused output, policy-bearing or opaque contract. +Validate roles from the exact context/source manifest, runtime/timing binding, +recorded validator provenance and reconstructed request bytes; the existing +Python byte reader alone is insufficient. Preserve full source comparison +identities and refuse unsupported contracts. Ancestor computations are never +scheduled implicitly. These checks establish declared-contract consistency, +not full Rust ledger validation or renewed access/send authority. `selected_ids=None` selects all distinct selectable Report/Evidence occurrences only when there are at most four. Otherwise require an explicit selection. Context/Policy/InheritedOutput and structural lineage do not count toward that @@ -53,6 +61,9 @@ select a subset when the bundle contains more than four, rather than silently truncating dependencies. Repeated occurrences at several positions are one ablated item whose scheduled removal covers every occurrence position; identical bytes under distinct IDs remain distinct items. +The known incident/1 contract requires unique occurrence IDs, so its public +planner refuses repeated IDs. Test repeated-position grouping in the internal +schedule helper only; do not invent a new public contract or relax incident/1. Plan schema `decision-ablation-plan/1`: source bundle digest, ordered selected IDs, trials (positive integer <=100), include_pairs boolean, max_calls integer <=10,000, @@ -77,6 +88,13 @@ Treat normalized results as recorded attestations. Exact duplicate trial IDs and unknown IDs refuse; missing trials make report Incomplete. Different scorer/runtime/validator identities make comparison Incomparable. +Before scoring, independently regenerate the complete bounded plan schedule, +including every baseline/single/enabled-pair repeat, variant digest and trial ID. +Require exact rows and scheduled count within max_calls even if a caller rehashed +a mutated plan. Reject removed baseline repetitions/pairs, duplicate repeat +indices, incorrect variant digests or false counts. This verifies plan consistency; +without the source bundle the scorer does not reauthenticate source applicability. + Score only if every baseline trial is recorded and all baseline semantic results agree. Each variant records changed/unchanged/failed/unknown counts out of all r scheduled repeats; incomplete variant trials yield no influence rank. A complete diff --git a/docs/superpowers/plans/2026-11-15-policy-activation.md b/docs/superpowers/plans/2026-11-15-policy-activation.md index 5cf4512..a01b851 100644 --- a/docs/superpowers/plans/2026-11-15-policy-activation.md +++ b/docs/superpowers/plans/2026-11-15-policy-activation.md @@ -206,6 +206,11 @@ method that opens a nested transaction. Withdraw old/admit new, revise final changed support heads, update pointer and append event in the same transaction. New revision = old+1 (bootstrap1), checked for overflow. No edits to claims, reservations or deliveries. Head/evaluation lookup is always scope-bound. +The admitted registered-policy set in this exact actor/scope/purpose must be +empty before bootstrap and exactly the current head occurrence before other +transitions. A third inactive version admitted externally also yields +AdmissionMismatch; do not silently repair it. Other purposes are outside this +consistency check. Test that case alongside withdrawn-head/admitted-target cases. - [ ] Add failing empty-registry bootstrap, concurrent bootstrap, B-beats-A with C active, stale revision, missing baseline evaluation, incompatible rollback, diff --git a/docs/superpowers/plans/2026-11-15-runtime-qualification.md b/docs/superpowers/plans/2026-11-15-runtime-qualification.md index 05e679f..107fb41 100644 --- a/docs/superpowers/plans/2026-11-15-runtime-qualification.md +++ b/docs/superpowers/plans/2026-11-15-runtime-qualification.md @@ -72,10 +72,24 @@ toolchains remain an explicit adoption gap; they do not become an invented pass. ### Task 2: Pin qualified project defaults and CI lanes -**Files:** create `rust-toolchain.toml`, `.python-version`; +**Files:** create `rust-toolchain.toml`, `.python-version`, +`scripts/project_runtime.py`, `scripts/test_project_runtime.py`; modify `.github/workflows/check.yml`, `docs/runtime-adoption.md`. No application source or dependency resolution change. +The project launcher uses an explicit accepted local runtime-selection manifest +with absolute executable paths, hashes and versions, distinct from the immutable +source-bound qualification receipt. Verify all identities before launch, bind +selected Rust/Cargo and child python3 on a private PATH, and route Cargo through +the approved storage wrapper. Ordinary editable source is recorded honestly and +does not inherit the old source's qualification. Never silently install/fallback, +change host defaults or commit workstation paths as portable CI identities. +The system Rust commands are rustup proxies and may honor pins; test actual +resolution instead of assuming they override them. Python child selection still +requires explicit verification. Document the durable project invocation. +Meaningful launcher tests cover absent/wrong identity, selected child Python, +editable source without false qualification, and admitted Cargo routing. + - [ ] If stable qualification passed, pin its exact Rust release in rust-toolchain.toml with the minimal required components, and exact Python stable patch in .python-version. Otherwise preserve the accepted baseline and From ee5beb6b8437723c7e25e4db621c53aa279adfba Mon Sep 17 00:00:00 2001 From: Alejandro Castellanos <5405628+k2jac9@users.noreply.github.com> Date: Sun, 15 Nov 2026 19:13:12 -0500 Subject: [PATCH 24/51] feat: atomically activate evaluated policies --- docs/policy-promotion.md | 64 +++ src/policy_registry.rs | 292 ++++++++++++ src/policy_registry/evaluation.rs | 8 +- src/policy_registry/types.rs | 56 +++ src/support.rs | 28 +- tests/policy_registry.rs | 719 ++++++++++++++++++++++++++++++ 6 files changed, 1155 insertions(+), 12 deletions(-) create mode 100644 docs/policy-promotion.md diff --git a/docs/policy-promotion.md b/docs/policy-promotion.md new file mode 100644 index 0000000..eb37bad --- /dev/null +++ b/docs/policy-promotion.md @@ -0,0 +1,64 @@ +# Policy activation + +`Store::activate_policy` is a trusted management operation for one +`(actor, scope, purpose)`. The embedding runtime authenticates the caller and +supplies that scope. Imported artifacts and evaluation attestations cannot grant +management authority or permission to send model work. `active_policy` reads +metadata in one consistent SQLite snapshot; its result is not a dispatch claim. + +Registration preserves immutable exact bytes and allocates an unadmitted evidence +occurrence. Bootstrap explicitly selects one registered baseline with no active +head, supplies the original frozen comparison-manifest bytes and SHA256, and +requires a nonempty reason and `expected_active_revision: None`. Strict manifest +validation checks its structure, scoped identity and exact compatible artifact. +The original bytes are hashed without normalization; the head and event retain +that digest. Bootstrap establishes a comparison baseline, with no assertion of +measured superiority. It does not require an evaluation or known accounting unit. +Synthetic manifests are permitted only for `fixture/recovery-policy`; bootstrap +refuses another synthetic purpose as `Incompatible`, without writes. + +Promotion requires a positive expected revision, the requested candidate and a +scoped stored evaluation. The evaluation's baseline version and digest must equal +the current active head. Its original manifest bytes must match the bootstrapped +digest. Selection is recomputed from the original stored attestation: strictly +more successes, no increase in false refusals, unsafe acceptances, cost or deadline +misses. A separately stored baseline-only evaluation must fully validate as +complete `NoImprovement`, identify the same baseline on both sides with identical +rows, and exactly match the comparison's baseline projection, manifest, provenance +and evaluator. Unknown evaluator attestations remain unevaluated; neither their +storage nor a matching projection makes them sufficient baseline evidence. + +Rollback requires a positive expected revision, a nonempty reason, a target that +was previously activated in this exact scope/purpose and a currently valid artifact +with the same compatibility contract. It needs no superiority evaluation. Selecting +the active target refuses as `NoChange`. Rollback advances revision like promotion. + +All activation gates and writes run in one Immediate transaction. Before bootstrap, +the admitted registered-policy set in this exact actor/scope/purpose must be empty; +for later transitions it must consist of precisely the current head occurrence. +Externally withdrawing the head, admitting the target or admitting a third inactive +version causes `AdmissionMismatch`, without repair. Admissions in other purposes +are independent. Ordinary separately authorized admission corrections can repair +membership before a fresh activation request. + +The transaction withdraws the old occurrence, admits the target, reconciles support +heads against the final combined view, updates the pointer and appends one immutable +event. An unchanged support verdict retains its old attempt and charge; each changed +verdict receives one new publication. Failure at any stage rolls back all changes. +The initial revision is 1; subsequent revisions are checked additions. Concurrent +bootstrap losers receive `AlreadyInitialized`; stale transition requests receive +`RevisionConflict`. Read fresh state before constructing another request. + +Existing dispatch claims, deliveries, outstanding reservations, original deadlines +and attempt dispositions remain untouched. The scope epoch changes, so captured +opaque and timed work must pass the existing freshness gates before dispatch or +publication. An uncertain previously claimed send keeps its hold across activation +and reopen. Different purposes sharing a scope can invalidate opaque work coarsely. + +Malformed management request syntax produces an error. Valid requests that fail a +semantic activation gate return `Refused`; underlying persisted corruption and +SQLite failures remain errors. Refusals and errors never partially activate. + +Positive fixtures demonstrate synthetic registry mechanics only. They are not +observed policy superiority, live model results, production release acceptance or +a migration of an owner's live database. Existing acceptance gates remain required. diff --git a/src/policy_registry.rs b/src/policy_registry.rs index c605549..490f3ef 100644 --- a/src/policy_registry.rs +++ b/src/policy_registry.rs @@ -350,3 +350,295 @@ impl crate::Store { tx.commit() } } + +// All callers hold a read snapshot or the activation writer transaction. Preflight +// the scoped head before copying strings, then reuse exact artifact validation. +fn read_head( + conn: &Connection, + actor: &str, + scope: &str, + purpose: &str, +) -> rusqlite::Result> { + let valid = conn.query_row("SELECT typeof(version_id),length(CAST(version_id AS BLOB)),typeof(revision),revision,typeof(baseline_manifest_sha256),length(CAST(baseline_manifest_sha256 AS BLOB)) FROM policy_heads WHERE actor=?1 AND scope=?2 AND purpose=?3",params![actor,scope,purpose],|r| { + require(r.get::<_,String>(0)? == "text" && r.get::<_,i64>(1)? <= MAX_EVALUATION_BYTES as i64 && r.get::<_,String>(2)? == "integer" && r.get::<_,i64>(3)? > 0 && r.get::<_,String>(4)? == "text" && r.get::<_,i64>(5)? == 64,"invalid policy head") + }).optional()?; + if valid.is_none() { + return Ok(None); + } + let (id, revision, manifest): (String,i64,String) = conn.query_row("SELECT version_id,revision,baseline_manifest_sha256 FROM policy_heads WHERE actor=?1 AND scope=?2 AND purpose=?3",params![actor,scope,purpose],|r|Ok((r.get(0)?,r.get(1)?,r.get(2)?)))?; + let c = stored_candidate(conn, &id)?.ok_or_else(|| bad("missing active artifact"))?; + require( + c.actor == actor && c.scope == scope && c.purpose == purpose && digest(&manifest), + "invalid scoped policy head", + )?; + let a = validate_policy_candidate(&c)?; + let evidence_id = conn.query_row( + "SELECT evidence_id FROM policy_versions WHERE version_id=?1", + [&id], + |r| r.get(0), + )?; + Ok(Some(ActivePolicy { + version_id: id, + sha256: c.sha256, + evidence_id, + compatibility_contract_sha256: a.compatibility_contract_sha256, + revision, + baseline_manifest_sha256: manifest, + })) +} +fn scope_matches(c: &PolicyCandidate, r: &ActivationRequest) -> bool { + c.actor == r.actor && c.scope == r.scope && c.purpose == r.purpose +} +fn consistent_admissions( + conn: &Connection, + r: &ActivationRequest, + head: Option<&ActivePolicy>, +) -> rusqlite::Result { + // An inactive third occurrence is just as inconsistent as an admitted target. + let mut stmt=conn.prepare("SELECT v.evidence_id FROM policy_versions v WHERE v.actor=?1 AND v.scope=?2 AND v.purpose=?3 AND (SELECT admitted FROM admission_events a WHERE a.actor=v.actor AND a.scope=v.scope AND a.evidence_id=v.evidence_id ORDER BY epoch DESC LIMIT 1)=1 ORDER BY v.evidence_id LIMIT 2")?; + let ids = stmt + .query_map(params![r.actor, r.scope, r.purpose], |row| { + row.get::<_, i64>(0) + })? + .collect::>>()?; + Ok(match head { + Some(h) => ids == [h.evidence_id], + None => ids.is_empty(), + }) +} +fn baseline_recorded(conn: &Connection, e: &PolicyEvaluation) -> rusqlite::Result { + // Stream IDs with bounded preflight, so a malformed stored attestation never + // becomes valid evidence through its matching projection alone. + let mut stmt=conn.prepare("SELECT evaluation_id,typeof(evaluation_id),length(CAST(evaluation_id AS BLOB)) FROM policy_evaluations WHERE actor=?1 AND scope=?2 AND purpose=?3 AND baseline_id=?4 AND candidate_id=?4")?; + let mut rows = stmt.query(params![e.actor, e.scope, e.purpose, e.baseline_id])?; + while let Some(row) = rows.next()? { + require( + row.get::<_, String>(1)? == "text" + && row.get::<_, i64>(2)? <= MAX_EVALUATION_BYTES as i64, + "invalid baseline evaluation identity", + )?; + let id: String = row.get(0)?; + let b = stored_evaluation(conn, &id)?.ok_or_else(|| bad("missing baseline evaluation"))?; + if b.baseline_id == b.candidate_id + && b.baseline_sha256 == b.candidate_sha256 + && b.baseline_rows == b.candidate_rows + && evaluate_policy_comparison(&b)? + == SelectionDecision::Refused(SelectionRefusal::NoImprovement) + && paired_baseline_matches(&b, e)? + { + return Ok(true); + } + } + Ok(false) +} + +impl crate::Store { + /// Metadata from one consistent snapshot; this confers no dispatch authority. + pub fn active_policy( + &self, + actor: &str, + scope: &str, + purpose: &str, + ) -> rusqlite::Result> { + crate::validate_view(actor, scope)?; + require(nonempty(purpose), "empty policy purpose")?; + let tx = self.conn.unchecked_transaction()?; + let head = read_head(&tx, actor, scope, purpose)?; + tx.commit()?; + Ok(head) + } + + /// Trusted embedding code authenticates management authority. Every gate and + /// write shares one Immediate transaction; old claims and holds are untouched. + pub fn activate_policy(&mut self, r: &ActivationRequest) -> rusqlite::Result { + use ActivationAction::*; + use ActivationRefusal::*; + bounded(r)?; + crate::validate_view(&r.actor, &r.scope)?; + require(nonempty(&r.purpose), "empty policy purpose")?; + let (target, action, reason, evaluation_id) = match &r.action { + Bootstrap { + baseline_id, + baseline_manifest_sha256, + reason, + .. + } => { + require( + r.expected_active_revision.is_none() + && digest(baseline_manifest_sha256) + && nonempty(reason), + "invalid bootstrap request", + )?; + (baseline_id, "bootstrap", reason.as_str(), None) + } + Promote { + candidate_id, + evaluation_id, + } => { + require(nonempty(evaluation_id), "empty evaluation identity")?; + ( + candidate_id, + "promote", + "validated paired comparison", + Some(evaluation_id.as_str()), + ) + } + Rollback { + target_version_id, + reason, + } => { + require(nonempty(reason), "empty rollback reason")?; + (target_version_id, "rollback", reason.as_str(), None) + } + }; + require(nonempty(target), "empty policy target")?; + if action != "bootstrap" { + require( + r.expected_active_revision.is_some_and(|v| v > 0), + "expected positive active revision required", + )?; + } + let tx = self + .conn + .transaction_with_behavior(TransactionBehavior::Immediate)?; + macro_rules! refuse { + ($r:expr) => { + return Ok(ActivationResult::Refused($r)) + }; + } + let head = read_head(&tx, &r.actor, &r.scope, &r.purpose)?; + if action == "bootstrap" { + if head.is_some() { + refuse!(AlreadyInitialized); + } + } else { + let Some(h) = head.as_ref() else { + refuse!(MissingHead); + }; + if r.expected_active_revision != Some(h.revision) { + refuse!(RevisionConflict); + } + if target == &h.version_id { + refuse!(NoChange); + } + } + let Some(c) = stored_candidate(&tx, target)? else { + refuse!(InvalidArtifact); + }; + if !scope_matches(&c, r) { + refuse!(ScopeMismatch); + } + let artifact = validate_policy_candidate(&c)?; + if head.as_ref().is_some_and(|h| { + h.compatibility_contract_sha256 != artifact.compatibility_contract_sha256 + }) { + refuse!(Incompatible); + } + if !consistent_admissions(&tx, r, head.as_ref())? { + refuse!(AdmissionMismatch); + } + let manifest_sha = match &r.action { + Bootstrap { + baseline_manifest, + baseline_manifest_sha256, + .. + } => { + if sha256_hex(baseline_manifest) != *baseline_manifest_sha256 { + refuse!(InvalidArtifact); + } + let (m, _) = match evaluation::validate_manifest(baseline_manifest) { + Ok(m) => m, + Err(_) => { + refuse!(InvalidArtifact); + } + }; + if m.actor != r.actor || m.scope != r.scope || m.purpose != r.purpose { + refuse!(ScopeMismatch); + } + if m.provenance == Provenance::SyntheticRegistryMechanics + && m.purpose != "fixture/recovery-policy" + { + refuse!(Incompatible); + } + if m.compatibility_contract_sha256 != artifact.compatibility_contract_sha256 { + refuse!(Incompatible); + } + if !m + .policies + .iter() + .any(|p| p.artifact.sha256 == c.sha256 && p.artifact.bytes == c.artifact) + { + refuse!(InvalidArtifact); + } + baseline_manifest_sha256.clone() + } + Promote { evaluation_id, .. } => { + // Scope-bound lookup precedes decoding the persisted evaluation. + let in_scope:bool=tx.query_row("SELECT EXISTS(SELECT 1 FROM policy_evaluations WHERE evaluation_id=?1 AND actor=?2 AND scope=?3 AND purpose=?4)",params![evaluation_id,r.actor,r.scope,r.purpose],|r|r.get(0))?; + if !in_scope { + refuse!(ScopeMismatch); + } + let Some(e) = stored_evaluation(&tx, evaluation_id)? else { + refuse!(InvalidArtifact); + }; + let h = head.as_ref().unwrap(); + if e.baseline_id != h.version_id || e.baseline_sha256 != h.sha256 { + refuse!(BaselineMismatch); + } + if e.candidate_id != c.version_id || e.candidate_sha256 != c.sha256 { + refuse!(InvalidArtifact); + } + if e.experiment_sha256 != h.baseline_manifest_sha256 { + refuse!(BaselineMismatch); + } + match evaluate_policy_comparison(&e)? { + SelectionDecision::Eligible { .. } => {} + SelectionDecision::Refused(reason) => { + refuse!(EvaluationRefused(reason)); + } + } + if !baseline_recorded(&tx, &e)? { + refuse!(MissingBaselineEvaluation); + } + h.baseline_manifest_sha256.clone() + } + Rollback { .. } => { + let activated:bool=tx.query_row("SELECT EXISTS(SELECT 1 FROM policy_events WHERE actor=?1 AND scope=?2 AND purpose=?3 AND target_version_id=?4)",params![r.actor,r.scope,r.purpose,target],|r|r.get(0))?; + if !activated { + refuse!(NeverActivated); + } + head.as_ref().unwrap().baseline_manifest_sha256.clone() + } + }; + let revision = head + .as_ref() + .map_or(Some(1), |h| h.revision.checked_add(1)) + .ok_or_else(|| bad("policy revision overflow"))?; + let evidence_id = tx.query_row( + "SELECT evidence_id FROM policy_versions WHERE version_id=?1", + [target], + |r| r.get(0), + )?; + let mut edits = Vec::with_capacity(2); + if let Some(h) = &head { + edits.push(crate::AdmissionEdit { + evidence: h.evidence_id, + admitted: false, + }); + } + edits.push(crate::AdmissionEdit { + evidence: evidence_id, + admitted: true, + }); + crate::support::apply_admission_edits(&tx, &r.actor, &r.scope, &edits)?; + if let Some(h) = &head { + let changed=tx.execute("UPDATE policy_heads SET version_id=?4,revision=?5 WHERE actor=?1 AND scope=?2 AND purpose=?3 AND revision=?6",params![r.actor,r.scope,r.purpose,target,revision,h.revision])?; + require(changed == 1, "policy head CAS failed")?; + } else { + tx.execute("INSERT INTO policy_heads(actor,scope,purpose,version_id,revision,baseline_manifest_sha256) VALUES(?1,?2,?3,?4,?5,?6)",params![r.actor,r.scope,r.purpose,target,revision,manifest_sha])?; + } + tx.execute("INSERT INTO policy_events(actor,scope,purpose,revision,action,previous_version_id,target_version_id,evaluation_id,manifest_sha256,reason) VALUES(?1,?2,?3,?4,?5,?6,?7,?8,?9,?10)",params![r.actor,r.scope,r.purpose,revision,action,head.as_ref().map(|h|h.version_id.as_str()),target,evaluation_id,manifest_sha,reason])?; + tx.commit()?; + Ok(ActivationResult::Activated { revision }) + } +} diff --git a/src/policy_registry/evaluation.rs b/src/policy_registry/evaluation.rs index 178f882..d2538f1 100644 --- a/src/policy_registry/evaluation.rs +++ b/src/policy_registry/evaluation.rs @@ -122,8 +122,10 @@ pub fn paired_baseline_matches( && recorded.provenance == paired.provenance && recorded.baseline_rows == paired.baseline_rows) } -fn manifest(e: &PolicyEvaluation) -> rusqlite::Result<(ComparisonManifest, Vec)> { - let m: ComparisonManifest = parse(&e.experiment_manifest, MAX_EVALUATION_BYTES)?; +pub(super) fn validate_manifest( + bytes: &[u8], +) -> rusqlite::Result<(ComparisonManifest, Vec)> { + let m: ComparisonManifest = parse(bytes, MAX_EVALUATION_BYTES)?; require( m.format == "policy-comparison/1" && !m.cases.is_empty() @@ -251,7 +253,7 @@ fn refused(r: SelectionRefusal) -> SelectionDecision { pub fn evaluate_policy_comparison(e: &PolicyEvaluation) -> rusqlite::Result { shape(e)?; - let (m, labels) = manifest(e)?; + let (m, labels) = validate_manifest(&e.experiment_manifest)?; if e.actor != m.actor || e.scope != m.scope || e.purpose != m.purpose diff --git a/src/policy_registry/types.rs b/src/policy_registry/types.rs index 8931a1d..b775ac0 100644 --- a/src/policy_registry/types.rs +++ b/src/policy_registry/types.rs @@ -241,3 +241,59 @@ pub enum SelectionDecision { }, Refused(SelectionRefusal), } + +/// Trusted management request, never an authorization to send model work. +#[derive(Clone, Debug, Serialize)] +pub struct ActivationRequest { + pub actor: String, + pub scope: String, + pub purpose: String, + pub expected_active_revision: Option, + pub action: ActivationAction, +} +#[derive(Clone, Debug, Serialize)] +pub enum ActivationAction { + Bootstrap { + baseline_id: String, + baseline_manifest: Vec, + baseline_manifest_sha256: String, + reason: String, + }, + Promote { + candidate_id: String, + evaluation_id: String, + }, + Rollback { + target_version_id: String, + reason: String, + }, +} +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct ActivePolicy { + pub version_id: String, + pub sha256: String, + pub evidence_id: i64, + pub compatibility_contract_sha256: String, + pub revision: i64, + pub baseline_manifest_sha256: String, +} +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum ActivationResult { + Activated { revision: i64 }, + Refused(ActivationRefusal), +} +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum ActivationRefusal { + RevisionConflict, + AlreadyInitialized, + MissingHead, + InvalidArtifact, + ScopeMismatch, + Incompatible, + BaselineMismatch, + MissingBaselineEvaluation, + EvaluationRefused(SelectionRefusal), + NeverActivated, + AdmissionMismatch, + NoChange, +} diff --git a/src/support.rs b/src/support.rs index bf3630b..e27927b 100644 --- a/src/support.rs +++ b/src/support.rs @@ -207,6 +207,23 @@ fn publish_changed_heads( Ok(revisions) } +// Internal callers validate shape, occurrences and their own CAS before using +// this final-state path. This helper never opens or commits a transaction. +pub(crate) fn apply_admission_edits( + tx: &Transaction<'_>, actor: &str, scope: &str, edits: &[AdmissionEdit], +) -> Result<(i64, Vec)> { + let mut epoch = view_epoch(tx, actor, scope)?; + let mut effective = Vec::new(); + for edit in edits { + if let Some(next) = change_admission(tx, actor, scope, edit.evidence, edit.admitted)? { + epoch = next; + effective.push(edit.evidence); + } + } + let changes = publish_changed_heads(tx, actor, scope, epoch, &effective)?; + Ok((epoch, changes)) +} + // A bulk registration never changes admissions. Bind this cache to its single // actor, scope and writer transaction; both missing and visible IDs can repeat. fn cached_admissions(mut lookup: impl FnMut(i64) -> Result) -> impl FnMut(i64) -> Result { @@ -393,21 +410,14 @@ impl Store { } } let tx = self.conn.transaction_with_behavior(TransactionBehavior::Immediate)?; - let mut epoch = view_epoch(&tx, actor, scope)?; + let epoch = view_epoch(&tx, actor, scope)?; if epoch != expected_epoch { return Ok(BatchRevision::EpochConflict { current_epoch: epoch }); } for edit in edits { tx.query_row("SELECT 1 FROM evidence WHERE id = ?1", [edit.evidence], |_| Ok(()))?; } - let mut effective = Vec::new(); - for edit in edits { - if let Some(next) = change_admission(&tx, actor, scope, edit.evidence, edit.admitted)? { - epoch = next; - effective.push(edit.evidence); - } - } - let changes = publish_changed_heads(&tx, actor, scope, epoch, &effective)?; + let (epoch, changes) = apply_admission_edits(&tx, actor, scope, edits)?; tx.commit()?; Ok(BatchRevision::Applied { epoch, changes }) } diff --git a/tests/policy_registry.rs b/tests/policy_registry.rs index a9c37c7..c391c94 100644 --- a/tests/policy_registry.rs +++ b/tests/policy_registry.rs @@ -590,3 +590,722 @@ rowid_replacement_test!( "policy_evaluations", "oid" ); + +fn request( + e: &PolicyEvaluation, + revision: Option, + action: ActivationAction, +) -> ActivationRequest { + ActivationRequest { + actor: e.actor.clone(), + scope: e.scope.clone(), + purpose: e.purpose.clone(), + expected_active_revision: revision, + action, + } +} +fn bootstrap(e: &PolicyEvaluation) -> ActivationRequest { + request( + e, + None, + ActivationAction::Bootstrap { + baseline_id: e.baseline_id.clone(), + baseline_manifest: e.experiment_manifest.clone(), + baseline_manifest_sha256: e.experiment_sha256.clone(), + reason: "explicit initial baseline".into(), + }, + ) +} +fn promote(e: &PolicyEvaluation, revision: i64) -> ActivationRequest { + request( + e, + Some(revision), + ActivationAction::Promote { + candidate_id: e.candidate_id.clone(), + evaluation_id: e.evaluation_id.clone(), + }, + ) +} +fn baseline(e: &PolicyEvaluation) -> PolicyEvaluation { + let mut b = e.clone(); + b.evaluation_id = format!("{}-baseline", e.evaluation_id); + b.candidate_id = b.baseline_id.clone(); + b.candidate_sha256 = b.baseline_sha256.clone(); + b.candidate_rows = b.baseline_rows.clone(); + b +} +fn refusal( + s: &mut Store, + db: &Database, + r: &ActivationRequest, + want: ActivationRefusal, +) -> Result<()> { + let before = all_table_rows(&db.conn()?)?; + assert_eq!(s.activate_policy(r)?, ActivationResult::Refused(want)); + assert_eq!(all_table_rows(&db.conn()?)?, before); + Ok(()) +} +#[test] +fn activation_bootstrap_requires_registered_baseline_and_binds_exact_manifest() -> Result<()> { + let db = Database::new(); + let mut s = Store::open(&db.0)?; + let e = fixture(); + assert_eq!(s.active_policy(&e.actor, &e.scope, &e.purpose)?, None); + refusal( + &mut s, + &db, + &bootstrap(&e), + ActivationRefusal::InvalidArtifact, + )?; + register_pair(&mut s, &e)?; + let mut bad = bootstrap(&e); + if let ActivationAction::Bootstrap { + baseline_manifest, .. + } = &mut bad.action + { + baseline_manifest.push(b' '); + } + refusal(&mut s, &db, &bad, ActivationRefusal::InvalidArtifact)?; + assert_eq!( + s.activate_policy(&bootstrap(&e))?, + ActivationResult::Activated { revision: 1 } + ); + let h = s.active_policy(&e.actor, &e.scope, &e.purpose)?.unwrap(); + assert_eq!(h.version_id, e.baseline_id); + assert_eq!(h.sha256, e.baseline_sha256); + assert_eq!(h.baseline_manifest_sha256, e.experiment_sha256); + assert!(s + .read_evidence(&e.actor, &e.scope, h.evidence_id)? + .is_some()); + refusal( + &mut s, + &db, + &bootstrap(&e), + ActivationRefusal::AlreadyInitialized, + )?; + Ok(()) +} +#[test] +fn activation_requires_current_baseline_revision_and_complete_baseline_only_evaluation( +) -> Result<()> { + let db = Database::new(); + let mut s = Store::open(&db.0)?; + let e = fixture(); + register_pair(&mut s, &e)?; + s.record_policy_evaluation(&e)?; + s.activate_policy(&bootstrap(&e))?; + refusal( + &mut s, + &db, + &promote(&e, 2), + ActivationRefusal::RevisionConflict, + )?; + refusal( + &mut s, + &db, + &promote(&e, 1), + ActivationRefusal::MissingBaselineEvaluation, + )?; + s.record_policy_evaluation(&baseline(&e))?; + assert_eq!( + s.activate_policy(&promote(&e, 1))?, + ActivationResult::Activated { revision: 2 } + ); + let rollback = request( + &e, + Some(2), + ActivationAction::Rollback { + target_version_id: e.baseline_id.clone(), + reason: "return to accepted baseline".into(), + }, + ); + assert_eq!( + s.activate_policy(&rollback)?, + ActivationResult::Activated { revision: 3 } + ); + refusal( + &mut s, + &db, + &promote(&e, 1), + ActivationRefusal::RevisionConflict, + )?; + Ok(()) +} +#[test] +fn activation_b_beats_a_cannot_replace_current_c() -> Result<()> { + let db = Database::new(); + let mut s = Store::open(&db.0)?; + let e = fixture(); + register_pair(&mut s, &e)?; + let m: ComparisonManifest = serde_json::from_slice(&e.experiment_manifest).unwrap(); + let a = m + .policies + .iter() + .find(|p| p.artifact.sha256 == e.baseline_sha256) + .unwrap(); + s.register_policy_candidate(&PolicyCandidate { + version_id: "C".into(), + actor: e.actor.clone(), + scope: e.scope.clone(), + purpose: e.purpose.clone(), + artifact: a.artifact.bytes.clone(), + sha256: a.artifact.sha256.clone(), + })?; + s.record_policy_evaluation(&e)?; + s.record_policy_evaluation(&baseline(&e))?; + let mut init = bootstrap(&e); + if let ActivationAction::Bootstrap { baseline_id, .. } = &mut init.action { + *baseline_id = "C".into(); + } + s.activate_policy(&init)?; + refusal( + &mut s, + &db, + &promote(&e, 1), + ActivationRefusal::BaselineMismatch, + ) +} +#[test] +fn activation_inconsistent_admission_never_repairs_withdrawn_head_target_or_third_version( +) -> Result<()> { + for variant in ["head", "target", "third", "bootstrap"] { + let db = Database::new(); + let mut s = Store::open(&db.0)?; + let e = fixture(); + register_pair(&mut s, &e)?; + s.record_policy_evaluation(&e)?; + s.record_policy_evaluation(&baseline(&e))?; + let mut third = candidate("third"); + third.actor = e.actor.clone(); + third.scope = e.scope.clone(); + s.register_policy_candidate(&third)?; + if variant != "bootstrap" { + s.activate_policy(&bootstrap(&e))?; + } + let id = match variant { + "head" => &e.baseline_id, + "target" => &e.candidate_id, + _ => "third", + }; + let eid = db.conn()?.query_row( + "SELECT evidence_id FROM policy_versions WHERE version_id=?1", + [id], + |r| r.get(0), + )?; + s.set_admitted(&e.actor, &e.scope, eid, variant != "head")?; + let r = if variant == "bootstrap" { + bootstrap(&e) + } else { + promote(&e, 1) + }; + refusal(&mut s, &db, &r, ActivationRefusal::AdmissionMismatch)?; + } + Ok(()) +} + +fn activated_fixture() -> Result<(Database, Store, PolicyEvaluation)> { + let db = Database::new(); + let mut s = Store::open(&db.0)?; + let e = fixture(); + register_pair(&mut s, &e)?; + s.record_policy_evaluation(&e)?; + s.record_policy_evaluation(&baseline(&e))?; + assert_eq!( + s.activate_policy(&bootstrap(&e))?, + ActivationResult::Activated { revision: 1 } + ); + Ok((db, s, e)) +} +fn activation_race( + s: &mut Store, + db: &Database, + requests: [ActivationRequest; 2], + revision: i64, + loser: ActivationRefusal, +) -> Result<()> { + let barrier = std::sync::Arc::new(std::sync::Barrier::new(2)); + let threads = requests + .into_iter() + .map(|r| { + let path = db.0.clone(); + let barrier = barrier.clone(); + std::thread::spawn(move || -> Result<_> { + let mut s = Store::open(path)?; + barrier.wait(); + s.activate_policy(&r) + }) + }) + .collect::>(); + let results = threads + .into_iter() + .map(|t| t.join().unwrap()) + .collect::>>()?; + assert_eq!( + results + .iter() + .filter(|r| **r == ActivationResult::Activated { revision }) + .count(), + 1, + "{results:?}" + ); + assert_eq!( + results + .iter() + .filter(|r| **r == ActivationResult::Refused(loser.clone())) + .count(), + 1, + "{results:?}" + ); + let e = fixture(); + assert_eq!( + s.active_policy(&e.actor, &e.scope, &e.purpose)? + .unwrap() + .revision, + revision + ); + Ok(()) +} +#[test] +fn activation_concurrent_bootstrap_has_one_winner() -> Result<()> { + let db = Database::new(); + let mut s = Store::open(&db.0)?; + let e = fixture(); + register_pair(&mut s, &e)?; + activation_race( + &mut s, + &db, + [bootstrap(&e), bootstrap(&e)], + 1, + ActivationRefusal::AlreadyInitialized, + ) +} +#[test] +fn activation_concurrent_promotions_have_one_winner() -> Result<()> { + let (db, mut s, e) = activated_fixture()?; + activation_race( + &mut s, + &db, + [promote(&e, 1), promote(&e, 1)], + 2, + ActivationRefusal::RevisionConflict, + ) +} +#[test] +fn activation_promote_rollback_race_has_one_winner() -> Result<()> { + let (db, mut s, e) = activated_fixture()?; + s.activate_policy(&promote(&e, 1))?; + s.activate_policy(&request( + &e, + Some(2), + ActivationAction::Rollback { + target_version_id: e.baseline_id.clone(), + reason: "return".into(), + }, + ))?; + activation_race( + &mut s, + &db, + [ + promote(&e, 3), + request( + &e, + Some(3), + ActivationAction::Rollback { + target_version_id: e.candidate_id.clone(), + reason: "restore previous activation".into(), + }, + ), + ], + 4, + ActivationRefusal::RevisionConflict, + ) +} +#[test] +fn activation_replaces_admissions_and_publishes_only_final_changed_support() -> Result<()> { + let (db, mut s, e) = activated_fixture()?; + let old = s + .active_policy(&e.actor, &e.scope, &e.purpose)? + .unwrap() + .evidence_id; + let new = db.conn()?.query_row( + "SELECT evidence_id FROM policy_versions WHERE version_id=?1", + [&e.candidate_id], + |r| r.get(0), + )?; + let union = s.publish_support(&e.actor, &e.scope, "union", &[vec![old], vec![new]])?; + let left = s.publish_support(&e.actor, &e.scope, "left", &[vec![old]])?; + let right = s.publish_support(&e.actor, &e.scope, "right", &[vec![new]])?; + let usage = s.charged_usage()?; + s.activate_policy(&promote(&e, 1))?; + assert_eq!(s.read_evidence(&e.actor, &e.scope, old)?, None); + assert!(s.read_evidence(&e.actor, &e.scope, new)?.is_some()); + let conn = db.conn()?; + for (branch, old_id, changed) in [ + ("union", union, false), + ("left", left, true), + ("right", right, true), + ] { + let head: i64 = conn.query_row( + "SELECT head FROM branches WHERE actor=?1 AND scope=?2 AND name=?3", + params![e.actor, e.scope, branch], + |r| r.get(0), + )?; + assert_eq!(head != old_id, changed); + } + assert_eq!(s.charged_usage()?, usage + 2); + Ok(()) +} +#[test] +fn activation_failures_after_admission_support_and_event_writes_roll_back_every_table() -> Result<()> +{ + for (table, when) in [ + ("admission_events", "AFTER INSERT"), + ("support_bindings", "AFTER INSERT"), + ("policy_events", "AFTER INSERT"), + ] { + let (db, mut s, e) = activated_fixture()?; + let old = s + .active_policy(&e.actor, &e.scope, &e.purpose)? + .unwrap() + .evidence_id; + s.publish_support(&e.actor, &e.scope, "changed", &[vec![old]])?; + let sql = db.conn()?; + sql.execute_batch(&format!("CREATE TRIGGER fail_activation {when} ON {table} BEGIN SELECT RAISE(ABORT,'injected activation failure'); END;"))?; + let before = all_table_rows(&sql)?; + assert!(s.activate_policy(&promote(&e, 1)).is_err(), "{table}"); + assert_eq!(all_table_rows(&sql)?, before, "{table}"); + drop(s); + drop(sql); + let _s = Store::open(&db.0)?; + assert_eq!(all_table_rows(&db.conn()?)?, before, "{table}"); + } + Ok(()) +} +#[test] +fn activation_unknown_evaluators_are_not_promotable_or_baseline_evidence() -> Result<()> { + let db = Database::new(); + let mut s = Store::open(&db.0)?; + let mut e = fixture(); + register_pair(&mut s, &e)?; + let mut unknown = baseline(&e); + unknown.evaluator_version = "unknown/1".into(); + s.record_policy_evaluation(&unknown)?; + s.record_policy_evaluation(&e)?; + s.activate_policy(&bootstrap(&e))?; + refusal( + &mut s, + &db, + &promote(&e, 1), + ActivationRefusal::MissingBaselineEvaluation, + )?; + e.evaluation_id = "unknown-pair".into(); + e.evaluator_version = "unknown/1".into(); + s.record_policy_evaluation(&e)?; + refusal( + &mut s, + &db, + &promote(&e, 1), + ActivationRefusal::EvaluationRefused(SelectionRefusal::Incomparable), + ) +} +#[test] +fn activation_rollback_checks_prior_activation_current_compatibility_and_no_change() -> Result<()> { + let (db, mut s, e) = activated_fixture()?; + let rb = |id: String, rev| { + request( + &e, + Some(rev), + ActivationAction::Rollback { + target_version_id: id, + reason: "explicit rollback".into(), + }, + ) + }; + refusal( + &mut s, + &db, + &rb(e.baseline_id.clone(), 1), + ActivationRefusal::NoChange, + )?; + refusal( + &mut s, + &db, + &rb(e.candidate_id.clone(), 1), + ActivationRefusal::NeverActivated, + )?; + // A valid registered version with a different contract cannot be a rollback target. + let mut other = candidate("incompatible"); + other.actor = e.actor.clone(); + other.scope = e.scope.clone(); + let mut artifact = validate_policy_artifact(&other.artifact)?; + artifact.compatibility_contract_sha256 = "f".repeat(64); + other.artifact = serde_json::to_vec(&artifact).unwrap(); + other.sha256 = sha(&other.artifact); + s.register_policy_candidate(&other)?; + refusal( + &mut s, + &db, + &rb(other.version_id, 1), + ActivationRefusal::Incompatible, + )?; + Ok(()) +} + +#[test] +fn activation_epoch_invalidates_captured_work_without_releasing_uncertain_claim() -> Result<()> { + use kairotensor::{ + RecoveryCalibration, RecoveryClock, RecoveryPreparation, RecoveryReadiness, + RecoveryRuntime, RecoveryStart, RecoveryTiming, TimedRecoveryClaim, TimingRefusal, + }; + let (db, mut s, e) = activated_fixture()?; + let old = s + .active_policy(&e.actor, &e.scope, &e.purpose)? + .unwrap() + .evidence_id; + let budget = s.create_token_budget(100)?; + let mut attempts = Vec::new(); + for name in ["uncertain", "opaque"] { + let r = s.create_recovery(name, budget, &e.actor, &e.scope, name)?; + let RecoveryStart::New { attempt, .. } = s.begin_recovery_attempt(r, &[old])? else { + panic!() + }; + assert!(s.reserve_recovery_attempt(attempt, b"exact", 2, 8)?); + attempts.push(attempt); + } + assert_eq!( + s.claim_recovery_dispatch(attempts[0])?, + Some(b"exact".to_vec()) + ); + let timing = RecoveryTiming { + clock_domain: "boot".into(), + start_ms: 100, + deadline_ms: 1000, + safety_margin_ms: 20, + runtime: RecoveryRuntime { + model: "m".into(), + model_revision: "r".into(), + tokenizer_revision: "t".into(), + runtime_version: "v".into(), + dtype: "f32".into(), + decoding: "greedy".into(), + cpu_fingerprint: "a".repeat(64), + threads: 1, + prompt_fingerprint: "b".repeat(64), + readiness: RecoveryReadiness::Warm, + }, + calibration: RecoveryCalibration { + id: "cal".into(), + max_duration_ms: 100, + sample_count: 2, + max_input_tokens: 20, + output_cap: 8, + measured_at_ms: 50, + expires_at_ms: 2000, + }, + }; + let r = s.create_timed_recovery("timed", budget, &e.actor, &e.scope, "timed", &timing)?; + let RecoveryStart::New { attempt: timed, .. } = s.begin_recovery_attempt(r, &[old])? else { + panic!() + }; + assert!(s.reserve_recovery_attempt(timed, b"exact", 2, 8)?); + let prep = RecoveryPreparation { + runtime: Some(timing.runtime), + calibration_id: Some("cal".into()), + request: b"exact".to_vec(), + input_tokens: 2, + output_cap: 8, + }; + let before = all_table_rows(&db.conn()?)?; + assert_eq!(s.token_budget_state(budget)?.reserved, 30); + s.activate_policy(&promote(&e, 1))?; + let after = all_table_rows(&db.conn()?)?; + for table in [ + "dispatches", + "token_reservations", + "recovery_attempts", + "deliveries", + "attempts", + "recovery_timing_bindings", + ] { + assert_eq!( + before.iter().find(|r| r.0 == table), + after.iter().find(|r| r.0 == table), + "{table}" + ); + } + assert_eq!(s.token_budget_state(budget)?.reserved, 30); + assert!(matches!(s.claim_recovery_dispatch(attempts[1]), + Err(rusqlite::Error::SqliteFailure(code, Some(message))) + if code.code == rusqlite::ErrorCode::ConstraintViolation + && message == "cannot dispatch a stale attempt")); + assert!(matches!( + s.claim_timed_recovery_dispatch(timed, &prep, || Ok(RecoveryClock { + domain: "boot".into(), + now_ms: 200 + }))?, + TimedRecoveryClaim::Refused { + reason: TimingRefusal::StaleEvidence, + .. + } + )); + assert_eq!( + db.conn()?.query_row( + "SELECT state FROM recovery_attempts WHERE attempt_id=?1", + [attempts[0]], + |r| r.get::<_, String>(0) + )?, + "sent" + ); + assert_eq!(s.token_budget_state(budget)?.reserved, 30); + drop(s); + let s = Store::open(&db.0)?; + assert_eq!(s.token_budget_state(budget)?.reserved, 30); + Ok(()) +} + +#[test] +fn activation_scope_manifest_and_request_gates_leave_state_unchanged() -> Result<()> { + let db = Database::new(); + let mut s = Store::open(&db.0)?; + let e = fixture(); + register_pair(&mut s, &e)?; + refusal(&mut s, &db, &promote(&e, 1), ActivationRefusal::MissingHead)?; + let mut invalid = bootstrap(&e); + invalid.expected_active_revision = Some(1); + let before = all_table_rows(&db.conn()?)?; + assert!(s.activate_policy(&invalid).is_err()); + assert_eq!(before, all_table_rows(&db.conn()?)?); + let mut foreign = bootstrap(&e); + foreign.scope = "foreign".into(); + refusal(&mut s, &db, &foreign, ActivationRefusal::ScopeMismatch)?; + let mut malformed = bootstrap(&e); + if let ActivationAction::Bootstrap { + baseline_manifest, + baseline_manifest_sha256, + .. + } = &mut malformed.action + { + *baseline_manifest = b"{}".to_vec(); + *baseline_manifest_sha256 = sha(baseline_manifest); + } + refusal(&mut s, &db, &malformed, ActivationRefusal::InvalidArtifact)?; + s.activate_policy(&bootstrap(&e))?; + // Equivalent parsed JSON is still a different frozen byte manifest. + let mut changed = e.clone(); + changed.experiment_manifest.push(b' '); + changed.experiment_sha256 = sha(&changed.experiment_manifest); + s.record_policy_evaluation(&changed)?; + s.record_policy_evaluation(&baseline(&changed))?; + refusal( + &mut s, + &db, + &promote(&changed, 1), + ActivationRefusal::BaselineMismatch, + )?; + Ok(()) +} +#[test] +fn activation_other_purpose_admissions_are_independent() -> Result<()> { + let db = Database::new(); + let mut s = Store::open(&db.0)?; + let e = fixture(); + register_pair(&mut s, &e)?; + let mut other = candidate("other-purpose"); + other.actor = e.actor.clone(); + other.scope = e.scope.clone(); + other.purpose = "another-purpose".into(); + let mut a = validate_policy_artifact(&other.artifact)?; + a.purpose = other.purpose.clone(); + other.artifact = serde_json::to_vec(&a).unwrap(); + other.sha256 = sha(&other.artifact); + s.register_policy_candidate(&other)?; + let id = db.conn()?.query_row( + "SELECT evidence_id FROM policy_versions WHERE version_id=?1", + [&other.version_id], + |r| r.get(0), + )?; + s.set_admitted(&e.actor, &e.scope, id, true)?; + assert_eq!( + s.activate_policy(&bootstrap(&e))?, + ActivationResult::Activated { revision: 1 } + ); + s.record_policy_evaluation(&e)?; + s.record_policy_evaluation(&baseline(&e))?; + assert_eq!( + s.activate_policy(&promote(&e, 1))?, + ActivationResult::Activated { revision: 2 } + ); + assert!(s.read_evidence(&e.actor, &e.scope, id)?.is_some()); + Ok(()) +} + +#[test] +fn activation_incomplete_baseline_only_record_does_not_satisfy_projection_gate() -> Result<()> { + let db = Database::new(); + let mut s = Store::open(&db.0)?; + let e = fixture(); + register_pair(&mut s, &e)?; + let mut incomplete = baseline(&e); + incomplete.baseline_rows[0].charged = None; + incomplete.candidate_rows = incomplete.baseline_rows.clone(); + s.record_policy_evaluation(&incomplete)?; + s.record_policy_evaluation(&e)?; + s.activate_policy(&bootstrap(&e))?; + refusal( + &mut s, + &db, + &promote(&e, 1), + ActivationRefusal::MissingBaselineEvaluation, + ) +} + +#[test] +fn activation_bootstrap_rejects_synthetic_production_purpose_without_writes() -> Result<()> { + let db = Database::new(); + let mut s = Store::open(&db.0)?; + let mut e = fixture(); + let mut m: ComparisonManifest = serde_json::from_slice(&e.experiment_manifest).unwrap(); + e.purpose = "production/recovery-policy".into(); + m.purpose = e.purpose.clone(); + for policy in &mut m.policies { + let old = policy.artifact.sha256.clone(); + let mut a = validate_policy_artifact(&policy.artifact.bytes)?; + a.purpose = e.purpose.clone(); + policy.artifact.bytes = serde_json::to_vec(&a).unwrap(); + policy.artifact.sha256 = sha(&policy.artifact.bytes); + if e.baseline_sha256 == old { + e.baseline_sha256 = policy.artifact.sha256.clone(); + } + if e.candidate_sha256 == old { + e.candidate_sha256 = policy.artifact.sha256.clone(); + } + } + e.experiment_manifest = serde_json::to_vec(&m).unwrap(); + e.experiment_sha256 = sha(&e.experiment_manifest); + register_pair(&mut s, &e)?; + refusal(&mut s, &db, &bootstrap(&e), ActivationRefusal::Incompatible) +} + +#[test] +fn activation_bootstrap_unknown_unit_binds_baseline_without_comparison_claim() -> Result<()> { + let db = Database::new(); + let mut s = Store::open(&db.0)?; + let mut e = fixture(); + let mut m: ComparisonManifest = serde_json::from_slice(&e.experiment_manifest).unwrap(); + m.accounting_unit = AccountingUnit::Unknown; + e.experiment_manifest = serde_json::to_vec(&m).unwrap(); + e.experiment_sha256 = sha(&e.experiment_manifest); + register_pair(&mut s, &e)?; + assert_eq!( + s.activate_policy(&bootstrap(&e))?, + ActivationResult::Activated { revision: 1 } + ); + s.record_policy_evaluation(&e)?; + s.record_policy_evaluation(&baseline(&e))?; + refusal( + &mut s, + &db, + &promote(&e, 1), + ActivationRefusal::EvaluationRefused(SelectionRefusal::Incomparable), + ) +} From a8f8c4bd6b7b358b6c0066e7a5cf40cde497c536 Mon Sep 17 00:00:00 2001 From: Alejandro Castellanos <5405628+k2jac9@users.noreply.github.com> Date: Sun, 15 Nov 2026 19:33:17 -0500 Subject: [PATCH 25/51] test: demonstrate policy activation and rollback without model calls --- .github/workflows/check.yml | 5 + docs/policy-promotion.md | 60 +++ examples/policy_activation.rs | 625 +++++++++++++++++++++++++++ scripts/recovery_policy_eval.py | 68 ++- scripts/test_recovery_policy_eval.py | 49 +++ 5 files changed, 805 insertions(+), 2 deletions(-) create mode 100644 examples/policy_activation.rs diff --git a/.github/workflows/check.yml b/.github/workflows/check.yml index a6a4a10..b293080 100644 --- a/.github/workflows/check.yml +++ b/.github/workflows/check.yml @@ -46,6 +46,11 @@ jobs: - name: Verify synthetic recovery policy comparison run: python3 scripts/recovery_policy_eval.py --fixture + - name: Exercise typed policy activation and rollback without model calls + run: | + python3 scripts/recovery_policy_eval.py --activation-fixture > "$RUNNER_TEMP/policy-activation-fixture.json" + cargo run --locked --offline --example policy_activation -- --fixture "$RUNNER_TEMP/policy-activation-fixture.json" --out "$RUNNER_TEMP/policy-activation" + python3 scripts/recovery_policy_eval.py --archived-v1-refusal > "$RUNNER_TEMP/archived-v1-promotion-refusal.json" - name: Collect and score Store-backed synthetic recovery trajectories run: | cargo run --locked --example recovery_policy_eval -- --trace "$RUNNER_TEMP/recovery-policy-trace.json" diff --git a/docs/policy-promotion.md b/docs/policy-promotion.md index eb37bad..b484671 100644 --- a/docs/policy-promotion.md +++ b/docs/policy-promotion.md @@ -62,3 +62,63 @@ SQLite failures remain errors. Refusals and errors never partially activate. Positive fixtures demonstrate synthetic registry mechanics only. They are not observed policy superiority, live model results, production release acceptance or a migration of an owner's live database. Existing acceptance gates remain required. + + +## Offline typed fixture + +Run the separate fixture with no model or provider calls: + +```sh +python3 scripts/recovery_policy_eval.py --activation-fixture > /tmp/policy-activation-fixture.json +cargo run --locked --offline --example policy_activation -- \ + --fixture /tmp/policy-activation-fixture.json --out /tmp/policy-activation-demo +python3 scripts/recovery_policy_eval.py --archived-v1-refusal +``` + +Choose a new output directory; the example refuses an existing directory instead +of opening an owner database. Local builds still require the workstation storage +admission wrapper. The example retains its exact synthetic input, SQLite fixture, +prepared requests and result JSON. CI runs it explicitly alongside the existing +offline steps. These are configured checks; remote CI execution is separate. + +The `policy-activation-fixture/1` envelope contains a complete deliberately +synthetic `PolicyEvaluation`, generated with the existing D1 codec and evaluator +identity. The example registers known `always_stop` and `affordable_retry` +artifacts, bootstraps the baseline, records a complete identical-row baseline-only +evaluation and a qualifying paired evaluation, promotes, then rolls back to the +previous occurrence at revision 3. Synthetic row counters are attestations for +registry mechanics: they are **not feasible named-policy replay or measured +superiority**. Their frozen semantic labels never enter execution requests. + +Execution uses a separate `policy-report-fixture/1` manifest: exactly one `Policy` +occurrence in position 0, followed by `Report` occurrences. It binds the active +version, revision, exact evidence occurrence ID, artifact digest, compatibility +contract and original baseline manifest digest, plus each ordered input occurrence +and payload hash. All occurrences pass through ordinary Store recovery capture. +Preparation refuses filtering, reordering, role/hash mismatch, unsupported contract +or an identical-byte policy acquired as a different occurrence. It validates the +captured policy artifact against a fresh active selection immediately before +preparation. Only the captured Reports become `ci_reports`; policy bytes are never +parsed as a Report or inserted into that payload. + +The immutable request includes this manifest, a computation digest, the fixed +protocol feedback and `fixture-byte-runtime/1` identity with a byte quote and +output cap. Normal Store claim and settlement freshness gates still decide +permission to dispatch and publish after preparation. The demonstration shows a +policy switch fencing an old prepared claim, an old duplicate `Accepted` receipt +labelled `historicalAccepted` while the current head needs review, and an unresolved +claimed attempt retaining its exact hold through promotion, rollback and database +reopen. Unsent work may be cancelled; an uncertain claimed send cannot be reclaimed +or cancelled. Rollback advances revision and does not make old work current. + +`--archived-v1-refusal` reads the retained actual v1 ZIP's `trace.json`, preserves +its archive/trace hashes, and invokes the unchanged explicit v2 converter without +supplying invented binding facts. It returns `Incomplete`: v1 fields alone do not +establish frozen answerability, eligibility, units or observed references. This +is a refusal at the conversion boundary, so no insertable `PolicyEvaluation` or +Store promotion request is manufactured. Frozen-v1 default modes, archived schemas +1/2, output and the Report-only collector remain unchanged. + +The typed policy fixture has its own role and validation contract. Existing +`incident/1` decision export/replay and diagnostic support do not cover it; no +incident validator binding or unsupported replay claim is attached to this demo. diff --git a/examples/policy_activation.rs b/examples/policy_activation.rs new file mode 100644 index 0000000..fd7caa9 --- /dev/null +++ b/examples/policy_activation.rs @@ -0,0 +1,625 @@ +//! Offline typed Policy + Report fixture. No provider, adapter, or model calls. +use kairotensor::policy_registry::*; +use kairotensor::{CapturedInputs, Current, Disposition, RecoveryStart, Store}; +use serde::{Deserialize, Serialize}; +use serde_json::{json, Value}; +use sha2::{Digest, Sha256}; +use std::{error::Error, path::Path}; +type Result = std::result::Result>; +const PURPOSE: &str = "fixture/recovery-policy"; +const CONTRACT: &str = "policy-report-fixture/1"; +const CONTRACT_BYTES: &[u8] = + b"policy-report-fixture/1:one-policy-then-reports;fixture-byte-runtime/1"; +const FEEDBACK: &str = "Return complete JSON matching the captured Reports; do not invent a verdict for conflicting or incomplete reports."; +fn sha(bytes: &[u8]) -> String { + Sha256::digest(bytes) + .iter() + .map(|b| format!("{b:02x}")) + .collect() +} +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +struct Occurrence { + role: Role, + evidence_id: i64, + sha256: String, +} +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +enum Role { + Policy, + Report, +} +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +struct Manifest { + format: String, + actor: String, + scope: String, + purpose: String, + active_version_id: String, + active_revision: i64, + active_evidence_id: i64, + active_sha256: String, + compatibility_contract_sha256: String, + baseline_manifest_sha256: String, + occurrences: Vec, +} +struct Prepared { + attempt: i64, + branch: String, + request: Vec, +} +struct Fixture { + store: Store, + evaluation: PolicyEvaluation, + budget: i64, + reports: Vec, +} +#[derive(Deserialize)] +#[serde(deny_unknown_fields)] +struct FixtureInput { + format: String, + new_model_calls: u64, + evaluation: PolicyEvaluation, + #[serde(rename = "limitations")] + _limitations: String, +} +#[derive(Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +struct Report { + release: String, + passed: u64, + total: u64, +} +impl Fixture { + fn open(path: &Path, evaluation: PolicyEvaluation) -> Result { + if evaluation.purpose != PURPOSE + || evaluation.provenance != Provenance::SyntheticRegistryMechanics + || !matches!( + evaluate_policy_comparison(&evaluation)?, + SelectionDecision::Eligible { .. } + ) + { + return Err("fixture requires a qualifying explicitly synthetic evaluation".into()); + } + let m: ComparisonManifest = serde_json::from_slice(&evaluation.experiment_manifest)?; + if m.compatibility_contract_sha256 != sha(CONTRACT_BYTES) { + return Err("unsupported fixture compatibility contract".into()); + } + let mut store = Store::open(path)?; + for (version_id, digest, expected) in [ + ( + &evaluation.baseline_id, + &evaluation.baseline_sha256, + PolicyName::AlwaysStop, + ), + ( + &evaluation.candidate_id, + &evaluation.candidate_sha256, + PolicyName::AffordableRetry, + ), + ] { + let policy = m + .policies + .iter() + .find(|p| &p.artifact.sha256 == digest) + .ok_or("missing fixture policy")?; + if policy.policy_name != expected { + return Err("unexpected fixture policy name".into()); + } + store.register_policy_candidate(&PolicyCandidate { + version_id: version_id.clone(), + actor: evaluation.actor.clone(), + scope: evaluation.scope.clone(), + purpose: evaluation.purpose.clone(), + artifact: policy.artifact.bytes.clone(), + sha256: digest.clone(), + })?; + } + let request = ActivationRequest { + actor: evaluation.actor.clone(), + scope: evaluation.scope.clone(), + purpose: PURPOSE.into(), + expected_active_revision: None, + action: ActivationAction::Bootstrap { + baseline_id: evaluation.baseline_id.clone(), + baseline_manifest: evaluation.experiment_manifest.clone(), + baseline_manifest_sha256: evaluation.experiment_sha256.clone(), + reason: "explicit synthetic fixture baseline; no superiority assertion".into(), + }, + }; + if store.activate_policy(&request)? != (ActivationResult::Activated { revision: 1 }) { + return Err("fixture bootstrap refused".into()); + } + let mut baseline = evaluation.clone(); + baseline.evaluation_id = format!("{}-baseline", evaluation.evaluation_id); + baseline.candidate_id = baseline.baseline_id.clone(); + baseline.candidate_sha256 = baseline.baseline_sha256.clone(); + baseline.candidate_rows = baseline.baseline_rows.clone(); + store.record_policy_evaluation(&baseline)?; + store.record_policy_evaluation(&evaluation)?; + let reports = store.observe_and_admit_many( + &evaluation.actor, + &evaluation.scope, + &[( + "fixture-report", + br#"{"release":"fixture","passed":3,"total":3}"#, + )], + )?; + let budget = store.create_token_budget(100_000)?; + Ok(Self { + store, + evaluation, + budget, + reports, + }) + } + fn manifest(&self) -> Result { + let e = &self.evaluation; + let head = self + .store + .active_policy(&e.actor, &e.scope, PURPOSE)? + .ok_or("missing active fixture policy")?; + let mut occurrences = vec![Occurrence { + role: Role::Policy, + evidence_id: head.evidence_id, + sha256: head.sha256.clone(), + }]; + for id in &self.reports { + let bytes = self + .store + .read_evidence(&e.actor, &e.scope, *id)? + .ok_or("report unavailable")?; + occurrences.push(Occurrence { + role: Role::Report, + evidence_id: *id, + sha256: sha(&bytes), + }); + } + Ok(Manifest { + format: CONTRACT.into(), + actor: e.actor.clone(), + scope: e.scope.clone(), + purpose: PURPOSE.into(), + active_version_id: head.version_id, + active_revision: head.revision, + active_evidence_id: head.evidence_id, + active_sha256: head.sha256, + compatibility_contract_sha256: head.compatibility_contract_sha256, + baseline_manifest_sha256: head.baseline_manifest_sha256, + occurrences, + }) + } + fn transition(&mut self, action: ActivationAction) -> Result { + let e = &self.evaluation; + let revision = self + .store + .active_policy(&e.actor, &e.scope, PURPOSE)? + .ok_or("missing fixture head")? + .revision; + Ok(self.store.activate_policy(&ActivationRequest { + actor: e.actor.clone(), + scope: e.scope.clone(), + purpose: PURPOSE.into(), + expected_active_revision: Some(revision), + action, + })?) + } + fn promote(&mut self) -> Result { + self.transition(ActivationAction::Promote { + candidate_id: self.evaluation.candidate_id.clone(), + evaluation_id: self.evaluation.evaluation_id.clone(), + }) + } + fn rollback(&mut self) -> Result { + self.transition(ActivationAction::Rollback { + target_version_id: self.evaluation.baseline_id.clone(), + reason: "fixture rollback to previously active compatible baseline".into(), + }) + } +} +fn render(manifest: &Manifest, inputs: &CapturedInputs, head: &ActivePolicy) -> Result { + if manifest.format != CONTRACT + || manifest.purpose != PURPOSE + || manifest.active_version_id != head.version_id + || manifest.active_revision != head.revision + || manifest.active_evidence_id != head.evidence_id + || manifest.active_sha256 != head.sha256 + || manifest.compatibility_contract_sha256 != head.compatibility_contract_sha256 + || manifest.compatibility_contract_sha256 != sha(CONTRACT_BYTES) + || manifest.baseline_manifest_sha256 != head.baseline_manifest_sha256 + { + return Err("selected active policy changed or incompatible contract".into()); + } + if manifest.occurrences.len() < 2 + || manifest.occurrences[0].role != Role::Policy + || manifest.occurrences[0].evidence_id != head.evidence_id + || manifest.occurrences[0].sha256 != head.sha256 + { + return Err("position zero must be the exact active policy occurrence".into()); + } + // The legacy candidate API intentionally filters unavailable inputs. This + // new complete typed contract refuses any filtering, reorder, or replacement. + if inputs.len() != manifest.occurrences.len() { + return Err("incomplete captured inputs".into()); + } + let mut reports = Vec::new(); + for (position, (occurrence, (id, bytes))) in manifest.occurrences.iter().zip(inputs).enumerate() + { + if *id != occurrence.evidence_id || sha(bytes) != occurrence.sha256 { + return Err("captured occurrence/digest mismatch".into()); + } + if position == 0 { + let artifact = validate_policy_artifact(bytes)?; + if artifact.purpose != PURPOSE + || artifact.compatibility_contract_sha256 != head.compatibility_contract_sha256 + || sha(bytes) != head.sha256 + { + return Err("captured policy artifact is incompatible".into()); + } + } else { + if occurrence.role != Role::Report { + return Err("only Report roles may follow the Policy".into()); + } + let report: Report = serde_json::from_slice(bytes)?; + if report.release.is_empty() || report.total == 0 || report.passed > report.total { + return Err("invalid captured Report".into()); + } + reports.push(report); + } + } + Ok(json!({"ci_reports": reports})) +} +fn prepare(store: &mut Store, budget: i64, key: &str, manifest: &Manifest) -> Result { + let binding = sha(&serde_json::to_vec(manifest)?); + let branch = format!("{CONTRACT}/{binding}/{key}"); + let recovery = + store.create_recovery(&branch, budget, &manifest.actor, &manifest.scope, &branch)?; + let candidates: Vec<_> = manifest.occurrences.iter().map(|o| o.evidence_id).collect(); + let RecoveryStart::New { + attempt, inputs, .. + } = store.begin_recovery_attempt(recovery, &candidates)? + else { + return Err("fixture recovery already exists; never resend an ambiguous claim".into()); + }; + let preparation = (|| -> Result> { + // Read again after capture, immediately before preparing. A subsequent + // policy change is fenced by normal Store claim/settlement freshness. + let head = store + .active_policy(&manifest.actor, &manifest.scope, &manifest.purpose)? + .ok_or("active policy unavailable")?; + let content = render(manifest, &inputs, &head)?; + Ok(serde_json::to_vec( + &json!({"contract": CONTRACT, "computation_sha256": binding, + "input_manifest": manifest, "runtime": {"identity":"fixture-byte-runtime/1", "accounting_unit":"fixture_bytes", "output_cap":96, "model":null}, + "feedback": FEEDBACK, "messages":[{"role":"user", "content":serde_json::to_string(&content)?}]}), + )?) + })(); + let request = match preparation { + Ok(request) => request, + Err(error) => { + store.cancel_recovery_attempt(attempt)?; + return Err(error); + } + }; + if !store.reserve_recovery_attempt(attempt, &request, i64::try_from(request.len())?, 96)? { + return Err("fixture budget refused".into()); + } + Ok(Prepared { + attempt, + branch, + request, + }) +} +fn save_request(out: &Path, label: &str, prepared: &Prepared) -> Result<()> { + std::fs::write(out.join(format!("{label}.request.json")), &prepared.request)?; + Ok(()) +} +fn main() -> Result<()> { + let args: Vec<_> = std::env::args().skip(1).collect(); + if args.len() != 4 || args[0] != "--fixture" || args[2] != "--out" { + return Err("usage: policy_activation --fixture SYNTHETIC.json --out NEW_DIRECTORY".into()); + } + let bytes = std::fs::read(&args[1])?; + if bytes.len() > MAX_EVALUATION_BYTES { + return Err("fixture size limit".into()); + } + let input: FixtureInput = serde_json::from_slice(&bytes)?; + if input.format != "policy-activation-fixture/1" || input.new_model_calls != 0 { + return Err("not an offline synthetic fixture".into()); + } + let out = Path::new(&args[3]); + // Create a new directory, refusing reuse of any existing/owner database. + std::fs::create_dir(out)?; + std::fs::write(out.join("fixture.json"), &bytes)?; + let db = out.join("fixture.sqlite"); + let mut f = Fixture::open(&db, input.evaluation)?; + let baseline = f.manifest()?; + let historical = prepare(&mut f.store, f.budget, "historical", &baseline)?; + save_request(out, "historical", &historical)?; + if f.store.claim_recovery_dispatch(historical.attempt)? != Some(historical.request.clone()) { + return Err("fixture claim refused".into()); + } + // The fixed response matches the single captured Report. No semantic label + // from the synthetic comparison is in any execution request. + let answer = br#"{"release":"fixture","tests":"3/3","status":"pass"}"#; + let usage = i64::try_from(historical.request.len() + answer.len())?; + if f.store.settle(historical.attempt, answer, usage)? != Disposition::Accepted { + return Err("fixture publication refused".into()); + } + let pending = prepare(&mut f.store, f.budget, "preclaim", &baseline)?; + save_request(out, "preclaim", &pending)?; + let uncertain = prepare(&mut f.store, f.budget, "uncertain", &baseline)?; + save_request(out, "uncertain", &uncertain)?; + if f.store.claim_recovery_dispatch(uncertain.attempt)? != Some(uncertain.request.clone()) { + return Err("uncertain fixture claim refused".into()); + } + let before = f.store.token_budget_state(f.budget)?; + let promoted = f.promote()?; + if promoted != (ActivationResult::Activated { revision: 2 }) { + return Err("fixture promotion refused".into()); + } + let stale_claim = f + .store + .claim_recovery_dispatch(pending.attempt) + .expect_err("policy change must fence old prepared work"); + if !stale_claim + .to_string() + .contains("cannot dispatch a stale attempt") + { + return Err(stale_claim.into()); + } + let historical_receipt = f.store.settle(historical.attempt, answer, usage)?; + let current = f + .store + .read_current(&baseline.actor, &baseline.scope, &historical.branch)?; + if historical_receipt != Disposition::Accepted || current != Current::NeedsReview { + return Err("historical/current gate mismatch".into()); + } + if f.store.token_budget_state(f.budget)? != before { + return Err("activation changed old accounting".into()); + } + if !f.store.cancel_recovery_attempt(pending.attempt)? { + return Err("unsent fixture cancellation refused".into()); + } + let held = f.store.token_budget_state(f.budget)?; + let rolled_back = f.rollback()?; + if rolled_back != (ActivationResult::Activated { revision: 3 }) { + return Err("fixture rollback refused".into()); + } + let rollback = f.manifest()?; + if rollback.active_evidence_id != baseline.active_evidence_id || rollback.active_revision != 3 { + return Err("rollback identity/revision mismatch".into()); + } + let final_prepared = prepare(&mut f.store, f.budget, "after-rollback", &rollback)?; + save_request(out, "after-rollback", &final_prepared)?; + f.store.cancel_recovery_attempt(final_prepared.attempt)?; + if f.store + .claim_recovery_dispatch(uncertain.attempt)? + .is_some() + || f.store.cancel_recovery_attempt(uncertain.attempt)? + { + return Err("ambiguous claim was reclaimed".into()); + } + let budget = f.budget; + drop(f); + let reopened = Store::open(&db)?; + if reopened.token_budget_state(budget)? != held { + return Err("uncertain hold changed across reopen".into()); + } + let summary = json!({"format":"policy-activation-result/1", "provenance":"synthetic_registry_mechanics", "new_model_calls":0, + "limitations":"Synthetic registry attestations only; not feasible named-policy replay or measured superiority.", "bootstrap_revision":1, "promotion_revision":2, "rollback_revision":3, + "baseline_manifest":baseline, "rollback_manifest":rollback, "historical_receipt":"historicalAccepted", + "current_head":"NeedsReview", "preclaim_refusal":stale_claim.to_string(), "uncertain_attempt":uncertain.attempt, + "budget_id":budget, "spent_fixture_bytes":held.spent, "retained_uncertain_bound":held.reserved, + "database":db, "scope":"offline fixture only; incident/1 decision replay and diagnostics do not support this contract"}); + std::fs::write( + out.join("result.json"), + serde_json::to_vec_pretty(&summary)?, + )?; + println!("{}", serde_json::to_string_pretty(&summary)?); + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + fn fixture() -> Result { + let output = std::process::Command::new("python3") + .args(["scripts/recovery_policy_eval.py", "--activation-fixture"]) + .output()?; + assert!( + output.status.success(), + "{}", + String::from_utf8_lossy(&output.stderr) + ); + let value: Value = serde_json::from_slice(&output.stdout)?; + Fixture::open( + Path::new(":memory:"), + serde_json::from_value(value["evaluation"].clone())?, + ) + } + #[test] + fn captures_policy_and_reports_but_renders_only_reports() -> Result<()> { + let mut f = fixture()?; + let m = f.manifest()?; + let p = prepare(&mut f.store, f.budget, "render", &m)?; + let request: Value = serde_json::from_slice(&p.request)?; + assert_eq!(request["contract"], CONTRACT); + assert_eq!(request["input_manifest"], serde_json::to_value(&m)?); + let content: Value = + serde_json::from_str(request["messages"][0]["content"].as_str().unwrap())?; + assert_eq!( + content, + json!({"ci_reports":[{"release":"fixture", "passed":3, "total":3}]}) + ); + assert_eq!(f.store.claim_recovery_dispatch(p.attempt)?, Some(p.request)); + assert_eq!(m.occurrences.len(), 2); + assert_eq!(m.occurrences[0].role, Role::Policy); + Ok(()) + } + #[test] + fn substituted_identical_policy_occurrence_is_refused() -> Result<()> { + let mut f = fixture()?; + let mut m = f.manifest()?; + let head = f + .store + .active_policy(&m.actor, &m.scope, &m.purpose)? + .unwrap(); + let bytes = f + .store + .read_evidence(&m.actor, &m.scope, head.evidence_id)? + .unwrap(); + let substitute = + f.store + .observe_and_admit_many(&m.actor, &m.scope, &[("substitute", &bytes)])?[0]; + m.occurrences[0].evidence_id = substitute; + let err = prepare(&mut f.store, f.budget, "substitute", &m) + .err() + .unwrap() + .to_string(); + assert!(err.contains("active policy occurrence"), "{err}"); + assert_eq!(f.store.token_budget_state(f.budget)?.reserved, 0); + Ok(()) + } + #[test] + fn malformed_roles_hashes_and_incomplete_capture_refuse_preparation() -> Result<()> { + let mut f = fixture()?; + let m = f.manifest()?; + for n in 0..4 { + let mut bad = m.clone(); + match n { + 0 => bad.occurrences.swap(0, 1), + 1 => bad.occurrences[1].role = Role::Policy, + 2 => bad.occurrences[1].sha256 = "0".repeat(64), + _ => bad.occurrences[1].evidence_id = i64::MAX, + } + assert!(prepare(&mut f.store, f.budget, &format!("bad/{n}"), &bad).is_err()); + } + assert_eq!(f.store.token_budget_state(f.budget)?.reserved, 0); + Ok(()) + } + #[test] + fn rechecks_selection_after_lookup_and_before_preparation() -> Result<()> { + let mut f = fixture()?; + let old = f.manifest()?; + assert_eq!(f.promote()?, ActivationResult::Activated { revision: 2 }); + assert!(prepare(&mut f.store, f.budget, "old-selection", &old).is_err()); + assert_eq!(f.store.token_budget_state(f.budget)?.reserved, 0); + Ok(()) + } + #[test] + fn actual_store_refuses_policy_switch_between_prepare_and_claim() -> Result<()> { + let mut f = fixture()?; + let m = f.manifest()?; + let p = prepare(&mut f.store, f.budget, "switch", &m)?; + f.promote()?; + let error = f + .store + .claim_recovery_dispatch(p.attempt) + .unwrap_err() + .to_string(); + assert!(error.contains("cannot dispatch a stale attempt"), "{error}"); + assert!(f.store.cancel_recovery_attempt(p.attempt)?); + Ok(()) + } + #[test] + fn policy_switch_after_claim_refuses_stale_publication_with_known_usage() -> Result<()> { + let mut f = fixture()?; + let m = f.manifest()?; + let p = prepare(&mut f.store, f.budget, "known-delivery", &m)?; + assert_eq!( + f.store.claim_recovery_dispatch(p.attempt)?, + Some(p.request.clone()) + ); + f.promote()?; + let answer = br#"{"release":"fixture","tests":"3/3","status":"pass"}"#; + let usage = i64::try_from(p.request.len() + answer.len())?; + assert_eq!( + f.store.settle(p.attempt, answer, usage)?, + Disposition::StaleView + ); + assert_eq!( + f.store.read_current(&m.actor, &m.scope, &p.branch)?, + Current::Empty + ); + // This separate known receipt charges real fixture bytes; an unresolved + // claimed attempt in the other test retains its bound instead. + assert_eq!(f.store.token_budget_state(f.budget)?.spent, usage); + assert_eq!(f.store.token_budget_state(f.budget)?.reserved, 0); + Ok(()) + } + #[test] + fn rollback_advances_revision_and_old_receipt_is_historical_only() -> Result<()> { + let mut f = fixture()?; + let m = f.manifest()?; + let p = prepare(&mut f.store, f.budget, "historical", &m)?; + f.store.claim_recovery_dispatch(p.attempt)?.unwrap(); + let answer = br#"{"release":"fixture","tests":"3/3","status":"pass"}"#; + let usage = i64::try_from(p.request.len() + answer.len())?; + assert_eq!( + f.store.settle(p.attempt, answer, usage)?, + Disposition::Accepted + ); + assert_eq!( + f.store.read_current(&m.actor, &m.scope, &p.branch)?, + Current::Ready(answer.to_vec()) + ); + f.promote()?; + assert_eq!( + f.store.settle(p.attempt, answer, usage)?, + Disposition::Accepted + ); + assert_eq!( + f.store.read_current(&m.actor, &m.scope, &p.branch)?, + Current::NeedsReview + ); + assert_eq!(f.rollback()?, ActivationResult::Activated { revision: 3 }); + let rolled = f.manifest()?; + assert_eq!(rolled.active_version_id, m.active_version_id); + assert_eq!(rolled.active_evidence_id, m.active_evidence_id); + assert_eq!(rolled.active_revision, m.active_revision + 2); + assert_eq!( + f.store.read_current(&m.actor, &m.scope, &p.branch)?, + Current::NeedsReview + ); + Ok(()) + } + #[test] + fn uncertain_claim_keeps_exact_hold_across_promotion_rollback_and_reopen() -> Result<()> { + let stamp = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH)? + .as_nanos(); + let path = std::env::temp_dir().join(format!( + "policy-activation-{}-{stamp}.sqlite", + std::process::id() + )); + let mut f = fixture()?; + // Reuse the exact fixture attestation in a separate disposable database. + f = Fixture::open(&path, f.evaluation.clone())?; + let m = f.manifest()?; + let p = prepare(&mut f.store, f.budget, "uncertain", &m)?; + assert_eq!( + f.store.claim_recovery_dispatch(p.attempt)?, + Some(p.request.clone()) + ); + let before = f.store.token_budget_state(f.budget)?; + assert!(before.reserved > 0); + f.promote()?; + f.rollback()?; + assert_eq!(f.store.token_budget_state(f.budget)?, before); + assert_eq!(f.store.claim_recovery_dispatch(p.attempt)?, None); + assert!(!f.store.cancel_recovery_attempt(p.attempt)?); + assert_eq!( + f.store.read_current(&m.actor, &m.scope, &p.branch)?, + Current::Empty + ); + let budget = f.budget; + drop(f); + let reopened = Store::open(&path)?; + assert_eq!(reopened.token_budget_state(budget)?, before); + drop(reopened); + std::fs::remove_file(path)?; + Ok(()) + } +} diff --git a/scripts/recovery_policy_eval.py b/scripts/recovery_policy_eval.py index 27a133e..dc66e48 100644 --- a/scripts/recovery_policy_eval.py +++ b/scripts/recovery_policy_eval.py @@ -211,6 +211,62 @@ def attempt(ctx, disposition, correct, charge): provenance='Hand-authored byte-independent token counts and durations; no model measurements.', cases=cases) +def activation_fixture(): + """Complete hand-authored registry counters, never feasible policy trajectories.""" + from policy_evaluation_v2 import canonical, sha, SELECTION_CONTRACT_SHA256 + + def record(raw): + return dict(bytes=list(raw), sha256=sha(raw)) + + compatibility = sha(b'policy-report-fixture/1:one-policy-then-reports;fixture-byte-runtime/1') + policies = [dict(policy_name=name, artifact=record(canonical(dict( + schema_version=1, policy_name=name, purpose='fixture/recovery-policy', + compatibility_contract_sha256=compatibility)))) for name in POLICIES] + scorer = sha(b'synthetic-opaque-scorer/1') + label = record(canonical(dict(answerable=True, admissible_support=True, + normalized_format='opaque-bytes/1', scorer_sha256=scorer, expected_normalized=list(b'yes')))) + opportunities = [dict(policy_name=p['policy_name'], opportunities=[dict( + opportunity_id=p['policy_name'] + '/' + str(n), request=record(('synthetic request ' + str(n)).encode()), + evidence_manifest=record(b'synthetic evidence'), admitted=True, consistent=True, + input_bound=4, output_cap=6, start_min_ms=0, start_max_ms=99) + for n in range(1 if p['policy_name'] == 'always_stop' else 2)]) for p in policies] + case = dict(case_id='synthetic-counter-case', family_id='synthetic-family', split_id='heldout', + initial_evidence=record(b'synthetic evidence'), label=label, budget=30, deadline_ms=100, + timing_start_ms=0, timing_end_ms=200, policy_opportunities=opportunities) + manifest = dict(format='policy-comparison/1', actor='policy-activation-fixture', scope='ci-report', + purpose='fixture/recovery-policy', provenance='synthetic_registry_mechanics', + accounting_unit='fixture_bytes', compatibility_contract_sha256=compatibility, + policies=policies, cases=[case], **{key + '_sha256': sha(('synthetic-' + key + '/1').encode()) + for key in ('workload', 'prompt', 'runtime', 'model', 'tokenizer', 'validator')}, scorer_sha256=scorer) + baseline = dict(case_id=case['case_id'], outcome='Refused', correct=False, eligible=True, + answerable=True, completed_ms=100, charged=4, reserved_bound=0, false_refusal=True, + unsafe_acceptance=False, deadline_miss=False, simulated=True, attempts=[]) + candidate = dict(baseline, outcome='Accepted', correct=True, false_refusal=False) + raw = canonical(manifest) + evaluation = dict(evaluation_id='synthetic-paired/1', actor=manifest['actor'], scope=manifest['scope'], + purpose=manifest['purpose'], baseline_id='always-stop/1', candidate_id='affordable-retry/1', + baseline_sha256=policies[0]['artifact']['sha256'], candidate_sha256=policies[1]['artifact']['sha256'], + experiment_manifest=list(raw), experiment_sha256=sha(raw), evaluator_version='policy-selection/1', + evaluator_sha256=SELECTION_CONTRACT_SHA256, provenance=manifest['provenance'], + baseline_rows=[baseline], candidate_rows=[candidate]) + return dict(format='policy-activation-fixture/1', new_model_calls=0, evaluation=evaluation, + limitations='Synthetic registry attestations only; not feasible named-policy replay or measured superiority.') + + +def archived_v1_refusal(): + """Exercise the unchanged converter on the retained actual archive.""" + import zipfile + from policy_evaluation_v2 import convert_legacy, sha + archive = Path(__file__).resolve().parents[1] / 'docs/evaluations/recovery-policy-v1-2026-11-14.zip' + with zipfile.ZipFile(archive) as source: + raw = source.read('trace.json') + conversion = convert_legacy(json.loads(raw, object_pairs_hook=unique_object)) + return dict(format='archived-policy-promotion-refusal/1', archive=str(archive), + archive_sha256=sha(archive.read_bytes()), trace_sha256=sha(raw), conversion=conversion, + promotion_requested=False, new_model_calls=0, + limitations='Incomplete conversion supplies no insertable PolicyEvaluation; promotion is withheld before Store insertion.') + + def unique_object(pairs): result = {} for key, value in pairs: @@ -224,14 +280,22 @@ def main(): parser.add_argument('trace', nargs='?', type=Path) parser.add_argument('--fixture', action='store_true', help='evaluate deterministic synthetic mechanics only') parser.add_argument('--emit-fixture', action='store_true', help='print the full synthetic input schema example') + parser.add_argument('--activation-fixture', action='store_true', help='emit a separate complete synthetic registry fixture') + parser.add_argument('--archived-v1-refusal', action='store_true', help='show actual archived v1 conversion refusal without inventing facts') parser.add_argument('--convert-v2', action='store_true', help='explicit complete-attestation conversion; missing facts remain incomplete') parser.add_argument('--comparison-v2', type=Path, help='matching complete v2 comparison attestation for --convert-v2') args = parser.parse_args() + if (args.activation_fixture or args.archived_v1_refusal) and (args.convert_v2 or args.comparison_v2): + parser.error('registry fixture/refusal modes cannot convert legacy inputs') if (args.comparison_v2 and not args.convert_v2) or (args.convert_v2 and args.emit_fixture): parser.error('--comparison-v2 requires --convert-v2; conversion cannot emit a fixture') - if sum((args.trace is not None, args.fixture, args.emit_fixture)) != 1: - parser.error('choose exactly one of TRACE, --fixture, --emit-fixture') + if sum((args.trace is not None, args.fixture, args.emit_fixture, args.activation_fixture, args.archived_v1_refusal)) != 1: + parser.error('choose exactly one input/fixture/refusal mode') try: + if args.activation_fixture or args.archived_v1_refusal: + result = activation_fixture() if args.activation_fixture else archived_v1_refusal() + print(json.dumps(result, indent=2, allow_nan=False)) + return document = synthetic_fixture() if args.fixture or args.emit_fixture else json.loads( args.trace.read_text(), object_pairs_hook=unique_object, parse_constant=lambda value: (_ for _ in ()).throw(ValueError(f'nonfinite JSON value: {value}'))) diff --git a/scripts/test_recovery_policy_eval.py b/scripts/test_recovery_policy_eval.py index d7d295d..a2b95d0 100644 --- a/scripts/test_recovery_policy_eval.py +++ b/scripts/test_recovery_policy_eval.py @@ -315,5 +315,54 @@ def test_explicit_conversion_rejects_changed_preparation_with_same_completion(se self.assertEqual(evaluate(old)['policy_version'], 'frozen-v1') +class ActivationFixtureTests(unittest.TestCase): + def command(self, mode): + return subprocess.run([sys.executable, str(Path(__file__).with_name('recovery_policy_eval.py')), mode], + capture_output=True, text=True) + + def test_separate_fixture_supplies_complete_synthetic_baseline_and_comparison(self): + from policy_evaluation_v2 import evaluate_policy_comparison + completed = self.command('--activation-fixture') + self.assertEqual(completed.returncode, 0, completed.stderr) + fixture = json.loads(completed.stdout) + self.assertEqual(fixture['format'], 'policy-activation-fixture/1') + self.assertEqual(fixture['new_model_calls'], 0) + self.assertIn('not feasible named-policy replay', fixture['limitations']) + paired = fixture['evaluation'] + self.assertEqual(paired['purpose'], 'fixture/recovery-policy') + self.assertEqual(evaluate_policy_comparison(paired)['decision'], 'Eligible') + baseline = copy.deepcopy(paired) + baseline.update(candidate_id=paired['baseline_id'], candidate_sha256=paired['baseline_sha256'], + candidate_rows=copy.deepcopy(paired['baseline_rows'])) + self.assertEqual(evaluate_policy_comparison(baseline), {'decision': 'Refused', 'reason': 'NoImprovement'}) + self.assertTrue(all(r['simulated'] and not r['attempts'] for r in paired['baseline_rows'] + paired['candidate_rows'])) + + def test_actual_archived_v1_refuses_at_conversion_without_invented_attestation(self): + import hashlib + import zipfile + from policy_evaluation_v2 import convert_legacy + archive = Path(__file__).resolve().parents[1] / 'docs/evaluations/recovery-policy-v1-2026-11-14.zip' + completed = self.command('--archived-v1-refusal') + self.assertEqual(completed.returncode, 0, completed.stderr) + result = json.loads(completed.stdout) + with zipfile.ZipFile(archive) as source: + raw = source.read('trace.json') + archived_summary = json.loads(source.read('summary.json')) + self.assertEqual(evaluate(json.loads(raw)), archived_summary) + self.assertEqual(result['archive_sha256'], hashlib.sha256(archive.read_bytes()).hexdigest()) + self.assertEqual(result['trace_sha256'], hashlib.sha256(raw).hexdigest()) + self.assertEqual(result['conversion'], convert_legacy(json.loads(raw))) + self.assertEqual(result['conversion']['status'], 'Incomplete') + self.assertFalse(result['promotion_requested']) + self.assertEqual(result['new_model_calls'], 0) + self.assertNotIn('evaluation', result) + + def test_fixture_modes_refuse_mixed_legacy_conversion_flags(self): + script = str(Path(__file__).with_name('recovery_policy_eval.py')) + for mode in ('--activation-fixture', '--archived-v1-refusal'): + completed = subprocess.run([sys.executable, script, mode, '--convert-v2'], capture_output=True, text=True) + self.assertEqual(completed.returncode, 2) + + if __name__ == '__main__': unittest.main() From 9329950c950a29e09c5d825b838fa2f5a9630cce Mon Sep 17 00:00:00 2001 From: Alejandro Castellanos <5405628+k2jac9@users.noreply.github.com> Date: Sun, 15 Nov 2026 19:46:21 -0500 Subject: [PATCH 26/51] fix: bound policy fixture input before deserialization --- examples/policy_activation.rs | 66 ++++++++++++++++++++++++++++++++--- 1 file changed, 61 insertions(+), 5 deletions(-) diff --git a/examples/policy_activation.rs b/examples/policy_activation.rs index fd7caa9..9174766 100644 --- a/examples/policy_activation.rs +++ b/examples/policy_activation.rs @@ -4,7 +4,7 @@ use kairotensor::{CapturedInputs, Current, Disposition, RecoveryStart, Store}; use serde::{Deserialize, Serialize}; use serde_json::{json, Value}; use sha2::{Digest, Sha256}; -use std::{error::Error, path::Path}; +use std::{error::Error, io::Read, path::Path}; type Result = std::result::Result>; const PURPOSE: &str = "fixture/recovery-policy"; const CONTRACT: &str = "policy-report-fixture/1"; @@ -315,15 +315,23 @@ fn save_request(out: &Path, label: &str, prepared: &Prepared) -> Result<()> { std::fs::write(out.join(format!("{label}.request.json")), &prepared.request)?; Ok(()) } +fn read_fixture(reader: impl Read) -> Result> { + let mut bytes = Vec::new(); + // A sentinel detects overflow without draining an oversized or continuing stream. + reader + .take(MAX_EVALUATION_BYTES as u64 + 1) + .read_to_end(&mut bytes)?; + if bytes.len() > MAX_EVALUATION_BYTES { + return Err("fixture size limit".into()); + } + Ok(bytes) +} fn main() -> Result<()> { let args: Vec<_> = std::env::args().skip(1).collect(); if args.len() != 4 || args[0] != "--fixture" || args[2] != "--out" { return Err("usage: policy_activation --fixture SYNTHETIC.json --out NEW_DIRECTORY".into()); } - let bytes = std::fs::read(&args[1])?; - if bytes.len() > MAX_EVALUATION_BYTES { - return Err("fixture size limit".into()); - } + let bytes = read_fixture(std::fs::File::open(&args[1])?)?; let input: FixtureInput = serde_json::from_slice(&bytes)?; if input.format != "policy-activation-fixture/1" || input.new_model_calls != 0 { return Err("not an offline synthetic fixture".into()); @@ -439,6 +447,54 @@ mod tests { serde_json::from_value(value["evaluation"].clone())?, ) } + struct CountingReader { + remaining: usize, + consumed: usize, + } + impl Read for CountingReader { + fn read(&mut self, buffer: &mut [u8]) -> std::io::Result { + let count = buffer.len().min(self.remaining); + buffer[..count].fill(b' '); + self.remaining -= count; + self.consumed += count; + Ok(count) + } + } + #[test] + fn fixture_reader_accepts_empty_and_below_limit_inputs() -> Result<()> { + for length in [0, MAX_EVALUATION_BYTES - 1] { + let mut reader = CountingReader { + remaining: length, + consumed: 0, + }; + assert_eq!(read_fixture(&mut reader)?.len(), length); + assert_eq!(reader.consumed, length); + } + Ok(()) + } + #[test] + fn fixture_reader_accepts_exact_limit() -> Result<()> { + let mut reader = CountingReader { + remaining: MAX_EVALUATION_BYTES, + consumed: 0, + }; + assert_eq!(read_fixture(&mut reader)?.len(), MAX_EVALUATION_BYTES); + assert_eq!(reader.consumed, MAX_EVALUATION_BYTES); + Ok(()) + } + #[test] + fn fixture_reader_refuses_overflow_after_only_one_sentinel_byte() { + for overflow in [1, 4096] { + let mut reader = CountingReader { + remaining: MAX_EVALUATION_BYTES + overflow, + consumed: 0, + }; + let error = read_fixture(&mut reader).unwrap_err().to_string(); + assert_eq!(error, "fixture size limit"); + assert_eq!(reader.consumed, MAX_EVALUATION_BYTES + 1); + assert_eq!(reader.remaining, overflow - 1); + } + } #[test] fn captures_policy_and_reports_but_renders_only_reports() -> Result<()> { let mut f = fixture()?; From a621ad825feed06183d369d551b81c142100467c Mon Sep 17 00:00:00 2001 From: Alejandro Castellanos <5405628+k2jac9@users.noreply.github.com> Date: Sun, 15 Nov 2026 20:00:31 -0500 Subject: [PATCH 27/51] docs: retain policy activation fixture packet --- docs/policy-promotion.md | 26 ++++++++++++++++---------- 1 file changed, 16 insertions(+), 10 deletions(-) diff --git a/docs/policy-promotion.md b/docs/policy-promotion.md index b484671..1ef4122 100644 --- a/docs/policy-promotion.md +++ b/docs/policy-promotion.md @@ -66,20 +66,26 @@ a migration of an owner's live database. Existing acceptance gates remain requir ## Offline typed fixture -Run the separate fixture with no model or provider calls: +Run the separate fixture with no model or provider calls. On the owner workstation, +use: ```sh -python3 scripts/recovery_policy_eval.py --activation-fixture > /tmp/policy-activation-fixture.json -cargo run --locked --offline --example policy_activation -- \ - --fixture /tmp/policy-activation-fixture.json --out /tmp/policy-activation-demo -python3 scripts/recovery_policy_eval.py --archived-v1-refusal +artifact_root=/home/neo/Projects/kairotensor-artifacts +mkdir -p "$artifact_root" +packet="$(mktemp -d "$artifact_root/policy-activation-XXXXXXXX")" +python3 scripts/recovery_policy_eval.py --activation-fixture > "$packet/fixture.json" +kairotensor-cargo --budget-gib 2 run --locked --offline --example policy_activation -- \ + --fixture "$packet/fixture.json" --out "$packet/run" +python3 scripts/recovery_policy_eval.py --archived-v1-refusal > "$packet/archived-v1-refusal.json" ``` -Choose a new output directory; the example refuses an existing directory instead -of opening an owner database. Local builds still require the workstation storage -admission wrapper. The example retains its exact synthetic input, SQLite fixture, -prepared requests and result JSON. CI runs it explicitly alongside the existing -offline steps. These are configured checks; remote CI execution is separate. +The unique packet retains the input, SQLite database, prepared requests, result +and archived-refusal evidence under the project artifact root. The CLI's run +directory is new inside the packet. The example refuses an existing run directory +instead of opening an owner database. Other hosts should choose their project's +retained artifact directory and use its required admitted Cargo invocation. CI +runs it explicitly alongside the existing offline steps. These are configured +checks; remote CI execution is separate. The `policy-activation-fixture/1` envelope contains a complete deliberately synthetic `PolicyEvaluation`, generated with the existing D1 codec and evaluator From 31e0dd14106a5452cbd23d7ac104ce39386b5633 Mon Sep 17 00:00:00 2001 From: Alejandro Castellanos <5405628+k2jac9@users.noreply.github.com> Date: Sun, 15 Nov 2026 20:17:36 -0500 Subject: [PATCH 28/51] feat: plan and score bounded offline ablations --- .github/workflows/check.yml | 2 + docs/decision-ablation.md | 178 +++++++++++ scripts/decision_ablation.py | 514 ++++++++++++++++++++++++++++++ scripts/test_decision_ablation.py | 358 +++++++++++++++++++++ 4 files changed, 1052 insertions(+) create mode 100644 docs/decision-ablation.md create mode 100644 scripts/decision_ablation.py create mode 100644 scripts/test_decision_ablation.py diff --git a/.github/workflows/check.yml b/.github/workflows/check.yml index b293080..7c1e26e 100644 --- a/.github/workflows/check.yml +++ b/.github/workflows/check.yml @@ -34,6 +34,8 @@ jobs: python3 reference/belief_oracle.py cargo build --locked --offline --example recovery_policy_eval --example decision_replay KAIROTENSOR_RECOVERY_RUNNER="$PWD/target/debug/examples/recovery_policy_eval" KAIROTENSOR_DECISION_REPLAY="$PWD/target/debug/examples/decision_replay" python3 -m unittest discover -s scripts -p 'test_*.py' + - name: Verify bounded offline ablation planning and recorded scoring + run: KAIROTENSOR_DECISION_REPLAY="$PWD/target/debug/examples/decision_replay" python3 -m unittest discover -s scripts -p test_decision_ablation.py - name: Export and replay a decision in separate offline processes run: | set -euo pipefail diff --git a/docs/decision-ablation.md b/docs/decision-ablation.md new file mode 100644 index 0000000..6245490 --- /dev/null +++ b/docs/decision-ablation.md @@ -0,0 +1,178 @@ +# Bounded offline decision ablations + +`scripts/decision_ablation.py` plans hypothetical omissions and scores explicit +recorded outcome attestations. It performs file input/output only. It has no run +mode, backend imports, executable command output, process calls, model loading, +provider requests or network access. E2 portability remains Deferred; Rocket is +outside this work. + +```sh +python3 scripts/decision_ablation.py plan complete-bundle.json > plan.json +python3 scripts/decision_ablation.py plan complete-bundle.json --trials 3 --include-pairs --max-calls 33 > pairs.json +python3 scripts/decision_ablation.py plan larger-bundle.json --selected-id 7 --selected-id 2 > selected.json +python3 scripts/decision_ablation.py score plan.json recorded-outcomes.json > report.json +``` + +The corresponding pure interfaces are `plan_ablations(bundle, trials=3, +include_pairs=False, max_calls=33, *, selected_ids=None)` and +`score_ablations(plan, outcomes)`. Refusals raise `ValueError`; the CLI prints a +reason on stderr and exits 1. JSON goes to stdout; callers choose output paths. +No new request bytes or send permission are produced. + +## Supported source and its limits + +Planning accepts a complete, integrity-checked `DecisionBundleV1` for one direct +root of the initial `incident-store-manifest/1` contract only. It requires ordinal +zero, no predecessor, no inherited/reused output, model contract, deterministic +contract or support/policy contract. An opaque parent lineage reference may remain; +it is never selected, expanded or recomputed. Retry and unsupported contracts refuse. + +Checks include exact nested field sets and scalar types; positive i64 IDs; +unique evidence occurrences and scopes; matching root ID, branch, actor and scope; +every selected/input payload and the recorded actor/scope authorization entry; +compact ordered candidates/inputs; context at position zero; and report roles +derived independently from the known context and full ordered manifest source map. +Free-form evidence `source` bytes and fixture eligibility claims do not determine +roles. Context, policy and lineage cannot become selectable by changing a role +label and resealing the outer digest. Incident/1 report roles must be exactly +`report`; an `evidence` label cannot replace them. + +The planner verifies exact manifest versions, canonical manifest bytes, context +payload digest, case/workload/external source mappings and report hashes, +context/manifest freeze/prompt/validator agreement, fixed initial/retry renderer +and feedback identifiers, full immutable runtime, timing runtime equality, +manifest runtime identity, manifest-derived prompt fingerprint, output cap, +typed calibration/timing and local request/state/accounting consistency. +It accepts `incident-routing-protocol/1` provenance `declared_contract` with a +valid manifest, or `recorded_rejection` with matching recorded invalid-output +rejection metadata. Other provenance or conflicts refuse. Source accounting +matches the narrow C contract: `unknown`, or exact canned-runtime `fixture_bytes`. + +It reconstructs the original initial request from captured context and report +bytes, without loading workload labels, then requires equality with the captured +recovery request and also dispatch request when sent. An unsent reserved, denied +or cancelled prepared root may use its captured recovery request; a planned root +without request bytes is unavailable. Unknown or inconsistent states refuse. +These checks establish declared-contract consistency. They do not run a Rust +ledger/protocol validator, authenticate historical metadata, replay timing, +renew current access or authorize export/send. Scoring without the source bundle +checks plan consistency only; it cannot reauthenticate source applicability. + +All public bundle/plan/outcome objects are counted before cloning/serialization: +exact compact UTF-8 size at most 16 MiB, depth at most 64, finite JSON types with +signed i64 integers and no floats, cycles, arbitrary Python objects or surrogates. +Files are read with the 16 MiB transport cap before parsing; duplicate keys refuse. +Byte envelopes retain their independent 4 MiB cap, lowercase hex, length and hash. +Context and reconstructed requests retain the separate 1 MiB incident cap. +At most 1,024 incident input occurrences including context are supported. + +## Deterministic plan + +`decision-ablation-plan/1` contains `source_bundle_digest` (SHA-256 of the full +canonical source bundle including its integrity record), `selected_ids`, +`selected_positions`, `trials`, `include_pairs`, `max_calls`, `scheduled_calls`, +`scorer`, `runtime`, `validator`, and `rows`. `runtime` preserves the original +full immutable-runtime byte envelope; `validator` preserves version and provenance. +They are source comparison identities of hypothetical omissions. In particular, +the original prompt fingerprint does not authorize a changed request. + +`selected_ids=None` selects all report occurrences when there are at most four; +a larger bundle requires an explicit list of up to four distinct selectable IDs. +The list is sorted into original input order. Unknown IDs, context IDs and duplicate +selections refuse. An explicit empty list schedules baseline only. Identical +payload bytes under distinct IDs remain distinct selected items. `selected_positions` +records each ID and its original positions. Incident/1 forbids repeated IDs, so the +public planner refuses them; only the internal grouping helper supports collecting +several positions for a future generic scheduler. No new public contract is implied. + +Trials must be an integer from 1 to 100, `include_pairs` a boolean, and `max_calls` +an integer from 1 to 10,000. The complete schedule must fit before a plan is emitted: + +``` +r * (1 + n + (n * (n - 1) / 2 if include_pairs else 0)) +``` + +Baseline, source-ordered singles and lexicographically ordered pairs each get every +repeat index `0..r-1`. Each removal list retains source order. For four items, three +repeats and pairs, there are exactly 33 calls; a bound of 32 refuses. No ancestor +computations are implicit. Omission leaves the source bundle unchanged. + +Rows contain exactly `trial_id`, `repeat_index`, `removed_ids`, `variant_digest`. +The variant digest hashes compact sorted-key UTF-8 JSON with fields +`version="decision-ablation-variant/1"`, `source_bundle_digest`, ordered `removed_ids`, +`scorer="recorded-result-equality/1"`, full source `runtime` and source `validator`. +Repeat index is excluded; the unique trial ID is `:`. +The plan digest used by outcomes is SHA-256 of the full canonical plan. + +## Recorded outcomes and scoring + +The exact outcome object is: + +```json +{"version":"decision-ablation-outcomes/1","plan_digest":"<64 lowercase hex>","rows":[]} +``` + +Each supplied row has these exact fields (one row for every scheduled trial for +complete coverage): + +```json +{"trial_id":":0","state":"recorded","response_digest":"<64 lowercase hex>","semantic_result":{"version":"recorded-normalized-result/1","value":{"decision":"abstain"}},"charged_units":2,"reserved_bound":10,"unit":"fixture_bytes","scorer":"recorded-result-equality/1","runtime":{"hex":"","byte_len":0,"sha256":""},"validator":{"version":"incident-routing-protocol/1","provenance":"declared_contract"}} +``` + +The displayed runtime is a shape placeholder: copy the exact plan runtime envelope, +including the correct length, lowercase hex and SHA-256. These examples assert no +provider/model execution. Outcomes are supplied attestations, not synthesized +measurements or proof of execution. + +`state` is `recorded`, `failed` or `unknown`. A recorded row requires a response +digest and the versioned semantic-result object. Its `value` is any bounded strict +JSON value: null, boolean, signed i64 integer, string, array or string-key object; +no floats. Equality compares canonical encoded bytes, so boolean `true` differs +from integer `1`, object key order is irrelevant, and array order matters. The +full normalized-result object has a 1 MiB cap. A failed/unknown row has null +`semantic_result`; its response digest may be null or retained if bytes exist. +No result normalization or scientific-truth validator runs. A producer claims that +its supplied result was validated; this tool compares that recorded attestation. + +`charged_units` is a nonnegative integer or null for unknown measurement; +`reserved_bound` is a nonnegative integer bounding any known charge. Unit is +`model_tokens`, `fixture_bytes`, `logical_support` or `unknown`. Outcome units +are reported separately, even though this initial source contract is narrower. +All scalar and per-unit aggregate sums must fit i64. Unknown charges stay null, +with known partial charges and unknown counts retained. `reserved_bound` sums all +observed row bounds; `held_units` conservatively retains the entire bound for +failed/unknown rows or any unknown charge. It is a reported uncertainty bound, +not a reconstructed Store hold. Recorded known-charge rows contribute no hold. +Missing outcome rows have unknown costs and bounds, explicitly counted by +`unreported_cost_trials`; their costs are never silently assigned zero. + +Before semantic scoring the scorer independently regenerates every row of the +bounded baseline/singles/enabled-pairs schedule and checks exact equality and +count. Removing baseline repetitions or pairs, duplicating repeat indices, +changing variant digests/removals or lying about counts refuses even after the +caller rehashes the plan. Unknown/duplicate outcome trial IDs or a changed plan +digest refuse. Every row must bind the plan's scorer, full runtime and validator. +Different identities yield `comparison_status="Incomparable"` and no scores/ranks. + +Reports keep separate facts: `coverage` is `Complete` or `Incomplete` according +to missing rows; `baseline_status` is `Stable` or `Inconclusive` with unavailable +and/or unstable reasons; `comparison_status` is `Comparable` or `Incomparable` +according to identities. They may coexist. No absent/failed/unknown baseline or +unstable baseline supports scoring. Scheduled calls, observed outcome rows and +recorded rows are separate counts; none authenticates actual provider execution. + +Every variant retains changed/unchanged/failed/unknown/missing/unscored counts out +of all scheduled repeats. Recorded rows are `unscored` when the baseline or +comparison is unusable. A variant with any missing/failed/unknown trial gets no +association score or rank. Otherwise its score is `changed / r`. Scores are +associations, including pairs, and are not additive causal effects or proof of +independent evidence. Equal scores keep equal competition ranks; display order +remains original singles followed by ordered pairs. Scores never change admission, +prune dependencies or grant authority. + +Verification uses a real precompiled C3 fixture exporter/replayer identified by +`KAIROTENSOR_DECISION_REPLAY`, explicit synthetic declared-contract mutations and +recorded fixtures, plus process/network hooks that raise during actual APIs and +both CLI commands. The test harness may prepare/replay fixtures in separate offline +processes before those hooks; public plan/score never invokes them. CI requires the +precompiled runner and does not silently build or skip these checks. diff --git a/scripts/decision_ablation.py b/scripts/decision_ablation.py new file mode 100644 index 0000000..24b9005 --- /dev/null +++ b/scripts/decision_ablation.py @@ -0,0 +1,514 @@ +"""Bounded offline hypothetical schedules and recorded-result attestations. + +No backend, process, model, network or validator execution. Contract checks below +are deliberately narrower than Rust ledger validation and confer no authority. +""" +import argparse +import hashlib +import itertools +import json +from pathlib import Path +import sys + +from decision_bundle_v1 import (MAX_BUNDLE_BYTES, MAX_JSON_DEPTH, DIGEST, + canonical_bytes, decode_envelope, bounded_depth, unique_object, reject_number, + parse_bundle, require) + +SCORER = 'recorded-result-equality/1' +VALIDATOR = 'incident-routing-protocol/1' +PLAN = 'decision-ablation-plan/1' +OUTCOMES = 'decision-ablation-outcomes/1' +RESULT = 'recorded-normalized-result/1' +INCIDENT_LIMIT = 1024 * 1024 +I64 = (1 << 63) - 1 +RUNTIME_FIELDS = {'model', 'model_revision', 'tokenizer_revision', 'runtime_version', + 'dtype', 'decoding', 'cpu_fingerprint', 'threads', 'prompt_fingerprint', 'readiness'} +ATTEMPT_FIELDS = {'actor', 'scope', 'id', 'branch', 'captured_epoch', 'captured_revision', + 'parent', 'inherited_from', 'candidates', 'inputs', 'dispatch_request', 'delivery', + 'validator', 'immutable_runtime', 'immutable_manifest', 'timing_configuration', + 'accounting', 'reuse_origin', 'model_contract', 'deterministic', 'support', 'recovery'} +IDENTITIES = ('scorer', 'runtime', 'validator') + + +def integer(value, low=0, high=I64): + require(type(value) is int and low <= value <= high, 'integer type or bound') + + +def text(value): + require(type(value) is str and bool(value.strip()), 'nonempty text required') + + +def fields(value, keys): + require(type(value) is dict and set(value) == set(keys), 'incorrect record fields') + + +def digest(value): + require(type(value) is str and DIGEST.fullmatch(value), 'invalid digest') + + +def sha(raw): + return hashlib.sha256(raw).hexdigest() + + +def preflight(value, limit=MAX_BUNDLE_BYTES): + """Count exact compact UTF-8 JSON size before serialization or cloning. + + Reject arbitrary objects, cycles, depth, wide arrays/maps and oversized strings + without allocating encoded strings/JSON intermediates. Integers are i64. + """ + size = 0 + ancestors = set() + + def add(count): + nonlocal size + size += count + require(size <= limit, 'object byte limit exceeded') + + def string(s): + require(type(s) is str and len(s) <= limit, 'string type or byte limit') + add(2) + for char in s: + n = ord(char) + require(not 0xd800 <= n <= 0xdfff, 'invalid UTF-8 surrogate') + add(2 if char in '\"\\\b\f\n\r\t' else 6 if n < 32 else + 1 if n < 128 else 2 if n < 2048 else 3 if n < 65536 else 4) + + def visit(v, depth): + require(depth <= MAX_JSON_DEPTH, 'object depth limit exceeded') + kind = type(v) + if kind is str: + string(v) + elif kind is bool: + add(4 if v else 5) + elif v is None: + add(4) + elif kind is int: + require(-I64-1 <= v <= I64, 'integer bound') + add(len(str(v))) + elif kind in (dict, list): + require(id(v) not in ancestors and len(v) <= limit // 2, 'cycle or container limit') + add(2 + max(0, len(v)-1)) + ancestors.add(id(v)) + if kind is dict: + for k, child in v.items(): + string(k); add(1); visit(child, depth+1) + else: + for child in v: + visit(child, depth+1) + ancestors.remove(id(v)) + else: + raise ValueError('unsupported JSON type') + visit(value, 1) + + +def parse_json(raw, limit=MAX_BUNDLE_BYTES): + require(type(raw) is bytes and len(raw) <= limit, 'transport byte limit exceeded') + bounded_depth(raw) + try: + value = json.loads(raw.decode('utf-8'), object_pairs_hook=unique_object, + parse_float=reject_number, parse_constant=reject_number) + preflight(value, limit) + return value + except (UnicodeError, RecursionError, OverflowError) as error: + raise ValueError('invalid JSON encoding') from error + + +def read_json(path): + with Path(path).open('rb') as stream: + return parse_json(stream.read(MAX_BUNDLE_BYTES+1)) + + +def _embedded(envelope, limit=None): + raw = decode_envelope(envelope) + return parse_json(raw, limit or MAX_BUNDLE_BYTES), raw + + +def _runtime(runtime): + fields(runtime, RUNTIME_FIELDS) + for key in RUNTIME_FIELDS - {'threads'}: + text(runtime[key]) + integer(runtime['threads'], 1) + require(runtime['readiness'] in ('cold', 'warm'), 'unknown readiness') + digest(runtime['prompt_fingerprint']) + + +def _validator(binding, supported=True): + fields(binding, {'version', 'provenance'}) + text(binding['version']); text(binding['provenance']) + if supported: + require(binding['version'] == VALIDATOR and binding['provenance'] in + ('declared_contract', 'recorded_rejection'), 'unsupported validator identity') + + +def _group_positions(inputs): + """Internal occurrence grouping; supported public incident/1 forbids repeats.""" + grouped = {} + for row in inputs: + integer(row['evidence_id'], 1); integer(row['position']) + grouped.setdefault(row['evidence_id'], []).append(row['position']) + return [dict(evidence_id=i, positions=p) for i,p in grouped.items()] + + +def _accounting(a): + q = a['accounting'] + fields(q, {'unit','quote','granted','budget_reference','charge','hold_bound'}) + require(q['unit'] in ('logical_support','fixture_bytes','model_tokens','unknown'), 'accounting unit') + integer(q['charge']); integer(q['hold_bound']) + require(q['granted'] is None or type(q['granted']) is bool, 'grant type') + if q['budget_reference'] is not None: + integer(q['budget_reference'], 1) + bound = None + if q['quote'] is not None: + fields(q['quote'], {'input_tokens','output_cap'}) + integer(q['quote']['input_tokens']);integer(q['quote']['output_cap'], 1) + bound = q['quote']['input_tokens'] + q['quote']['output_cap'] + integer(bound) + require(type(q['granted']) is bool and q['budget_reference'] is not None, 'incomplete reservation') + else: + require(q['granted'] is None and q['budget_reference'] is None, 'unbound reservation') + d = a['delivery'] + require(type(d) is dict, 'delivery record') + if d.get('state') == 'unresolved': + fields(d, {'state'}) + require(q['charge'] == 0, 'unresolved charge') + expected = bound if q['granted'] is True and a['recovery']['state'] != 'cancelled' else 0 + require(q['hold_bound'] == expected, 'unresolved hold mismatch') + else: + fields(d, {'state','raw','disposition','usage','rejection'}) + require(d['state'] == 'recorded' and d['disposition'] in + ('accepted','stale_view','stale_branch','invalid_output'), 'unknown delivery') + decode_envelope(d['raw']);integer(d['usage']) + require(d['usage'] == q['charge'] and q['hold_bound'] == 0, 'recorded accounting mismatch') + if bound is not None: + require(q['granted'] is True and q['quote']['input_tokens'] <= d['usage'] <= bound, + 'recorded usage outside reservation') + if d['disposition'] == 'invalid_output': + fields(d['rejection'], {'validator','reason','applicability'}) + text(d['rejection']['validator']);text(d['rejection']['reason']) + require(d['rejection']['applicability'] in ('current','stale_view','stale_branch'), 'rejection applicability') + else: + require(d['rejection'] is None, 'unexpected rejection') + return bound + + +def _applicable(bundle): + # Public constructed objects are bounded before the reader serializes/clones. + preflight(bundle) + b = parse_bundle(canonical_bytes(bundle)) + require(len(b['attempts']) == 1, 'only direct initial root supported') + integer(b['root_attempt'], 1);text(b['actor']);text(b['scope']) + require(b['export_purpose'] in ('current_read','diagnostic_replay'), 'unknown export purpose') + snapshot = b['snapshot'] + fields(snapshot, {'root_branch','head','revision','current_epoch'}) + integer(snapshot['root_branch'],1);integer(snapshot['revision']);integer(snapshot['current_epoch']) + if snapshot['head'] is not None:integer(snapshot['head'],1) + a = b['attempts'][0] + fields(a, ATTEMPT_FIELDS) + for key in ('id','branch'):integer(a[key],1) + for key in ('captured_epoch','captured_revision'):integer(a[key]) + if a['parent'] is not None:integer(a['parent'],1) + require(a['id'] == b['root_attempt'] and a['branch'] == snapshot['root_branch'] + and a['actor'] == b['actor'] and a['scope'] == b['scope'], 'root identity mismatch') + require(all(a[k] is None for k in ('inherited_from','reuse_origin','model_contract','deterministic','support')), + 'unsupported dependency or policy contract') + r = a['recovery'] + fields(r, {'request','recovery_id','ordinal','state','predecessor_id'}) + integer(r['recovery_id'],1);integer(r['ordinal']) + require(r['ordinal'] == 0 and r['predecessor_id'] is None, 'retry unsupported') + require(r['state'] in ('planned','reserved','sent','cancelled','denied'), 'unknown recovery state') + bound = _accounting(a) + q = a['accounting'] + if r['state'] == 'planned': + require(q['quote'] is None and r['request'] is None, 'planned request inconsistent') + if r['state'] in ('reserved','sent'): + require(q['granted'] is True, 'reservation state mismatch') + if r['state'] == 'denied':require(q['granted'] is False, 'denied reservation mismatch') + if r['state'] == 'cancelled':require(q['granted'] is not False, 'cancelled reservation mismatch') + if r['state'] in ('reserved','sent','denied'):require(r['request'] is not None, 'missing recovery request') + require((a['dispatch_request'] is not None) == (r['state'] == 'sent'), 'dispatch state mismatch') + require(a['delivery']['state'] != 'recorded' or r['state'] == 'sent', 'unsent delivery') + require(r['request'] is not None, 'captured request unavailable') + + manifest, manifest_raw = _embedded(a['immutable_manifest']) + fields(manifest, {'context','freeze_sha256','initial_renderer','output_cap','prompt_sha256', + 'retry_feedback','retry_renderer','runtime','schema_version','sources','validator_version','version','workload_version'}) + integer(manifest['schema_version'],1,1) + for key, expected in dict(version='incident-store-manifest/1',workload_version='incident-routing/1', + validator_version=VALIDATOR,initial_renderer='incident-request/1',retry_renderer='incident-request-retry/1', + retry_feedback=VALIDATOR+':invalid_output').items(): + require(manifest[key] == expected, 'unsupported manifest contract') + require(manifest_raw == canonical_bytes(manifest), 'manifest must be canonical') + integer(manifest['output_cap'],1) + digest(manifest['freeze_sha256']);digest(manifest['prompt_sha256']) + runtime, _ = _embedded(a['immutable_runtime']);_runtime(runtime) + timing, _ = _embedded(a['timing_configuration']) + fields(timing, {'clock_domain','start_ms','deadline_ms','runtime','calibration','safety_margin_ms'}) + _runtime(timing['runtime']);text(timing['clock_domain']) + for key in ('start_ms','deadline_ms','safety_margin_ms'):integer(timing[key]) + cal=timing['calibration'] + fields(cal, {'id','max_duration_ms','sample_count','max_input_tokens','output_cap','measured_at_ms','expires_at_ms'}) + text(cal['id']) + for key in ('max_duration_ms','sample_count','output_cap'):integer(cal[key],1) + for key in ('max_input_tokens','measured_at_ms','expires_at_ms'):integer(cal[key]) + require(timing['deadline_ms'] > timing['start_ms'] and cal['expires_at_ms'] > cal['measured_at_ms'], 'invalid timing bounds') + require(timing['runtime'] == runtime and runtime['prompt_fingerprint'] == sha(manifest_raw) + and cal['output_cap'] == manifest['output_cap'], 'runtime/timing/manifest mismatch') + fields(manifest['runtime'], RUNTIME_FIELDS - {'prompt_fingerprint'}) + _runtime(dict(manifest['runtime'], prompt_fingerprint=runtime['prompt_fingerprint'])) + require(manifest['runtime'] == {k:v for k,v in runtime.items() if k != 'prompt_fingerprint'}, 'manifest runtime mismatch') + fixture_identity = dict(model='canned-no-model', model_revision='fixture/1', tokenizer_revision='utf8-bytes/1', + runtime_version='incident-canned/1', dtype='bytes', decoding='canned-no-inference') + require(q['unit']=='unknown' or (q['unit']=='fixture_bytes' and + all(runtime[k]==v for k,v in fixture_identity.items())), 'unsupported incident accounting unit') + if r['state']=='sent': + require(q['quote'] is not None and q['quote']['input_tokens']<=cal['max_input_tokens'], 'sent quote outside calibration') + _validator(a['validator']) + rejection = a['delivery'].get('rejection') + if rejection is not None: + require(rejection['validator'] == VALIDATOR, 'recorded rejection validator mismatch') + if a['validator']['provenance'] == 'recorded_rejection': + require(rejection is not None and a['delivery']['disposition'] == 'invalid_output', 'missing recorded rejection') + else: + require(rejection is None, 'rejection provenance mismatch') + + require(type(b['evidence']) is list and len(b['evidence']) <= 1024, 'evidence record bound') + evidence={} + for e in b['evidence']: + fields(e, {'occurrence_id','source','payload','authorized_scopes'}) + integer(e['occurrence_id'],1) + require(e['occurrence_id'] not in evidence, 'duplicate evidence occurrence') + evidence[e['occurrence_id']]=e + decode_envelope(e['source']);decode_envelope(e['payload']) + scopes=e['authorized_scopes'] + require(type(scopes) is list and bool(scopes), 'missing authorized scopes') + seen=set() + for s in scopes: + fields(s, {'actor','scope'});text(s['actor']);text(s['scope']) + pair=(s['actor'],s['scope']);require(pair not in seen, 'duplicate scope');seen.add(pair) + inputs=a['inputs'];candidates=a['candidates'];sources=manifest['sources'] + require(type(inputs) is list and 2 <= len(inputs) <= 1024 and type(candidates) is list + and type(sources) is list and len(inputs)==len(candidates)==len(sources)+1, 'input/source alignment') + seen=set() + for position,(i,c) in enumerate(zip(inputs,candidates)): + fields(i, {'position','evidence_id','role'});fields(c, {'position','evidence_id'}) + integer(i['position']);integer(c['position']);integer(i['evidence_id'],1);integer(c['evidence_id'],1) + require(i['position'] == c['position'] == position and i['evidence_id'] == c['evidence_id'], 'candidate alignment') + require(i['evidence_id'] not in seen, 'incident/1 duplicate occurrence');seen.add(i['evidence_id']) + require(i['role'] == ('context' if position==0 else 'report'), 'role differs from manifest-derived role') + require(i['evidence_id'] in evidence and dict(actor=a['actor'],scope=a['scope']) in + evidence[i['evidence_id']]['authorized_scopes'], 'missing or unauthorized input') + fields(manifest['context'], {'occurrence_id','sha256'}) + integer(manifest['context']['occurrence_id'],1);digest(manifest['context']['sha256']) + first=evidence[inputs[0]['evidence_id']] + require(manifest['context']['occurrence_id'] == first['occurrence_id'] and + manifest['context']['sha256'] == first['payload']['sha256'], 'context identity mismatch') + context,_ = _embedded(first['payload'], INCIDENT_LIMIT) + fields(context, {'schema_version','version','workload_version','freeze_sha256','prompt_sha256', + 'validator_version','case_id','question','owner_ids','evidence_ids','system_prompt'}) + integer(context['schema_version'],1,1) + require(context['version']=='incident-store-context/1', 'unsupported context') + for key in ('workload_version','freeze_sha256','prompt_sha256','validator_version'): + require(context[key] == manifest[key], 'context manifest mismatch') + for key in ('case_id','question','system_prompt'):text(context[key]) + for key in ('owner_ids','evidence_ids'): + ids=context[key];require(type(ids) is list and bool(ids), 'empty context IDs') + for i in ids:text(i) + require(len(set(ids))==len(ids), 'duplicate external ID') + require(len(context['evidence_ids']) == len(sources), 'context source count') + preflight(context, INCIDENT_LIMIT) + reports=[] + for i,s,external in zip(inputs[1:], sources, context['evidence_ids']): + fields(s, {'case_id','external_id','occurrence_id','payload_sha256','workload_version'}) + integer(s['occurrence_id'],1);digest(s['payload_sha256']) + e=evidence[i['evidence_id']] + require(s['occurrence_id']==i['evidence_id'] and s['case_id']==context['case_id'] + and s['external_id']==external and s['workload_version']==manifest['workload_version'] + and s['payload_sha256']==e['payload']['sha256'], 'scoped source mapping mismatch') + try: report=decode_envelope(e['payload']).decode('utf-8') + except UnicodeError as error: raise ValueError('invalid report UTF-8') from error + text(report);reports.append(dict(id=external,text=report)) + user=dict(question=context['question'],owner_ids=context['owner_ids'],evidence=reports) + preflight(user, INCIDENT_LIMIT) + request=dict(messages=[dict(role='system',content=context['system_prompt']), + dict(role='user',content=canonical_bytes(user).decode('utf-8'))]) + preflight(request, INCIDENT_LIMIT);request_raw=canonical_bytes(request) + require(decode_envelope(r['request']) == request_raw, 'reconstructed recovery request mismatch') + if a['dispatch_request'] is not None: + require(decode_envelope(a['dispatch_request']) == request_raw, 'reconstructed dispatch request mismatch') + if bound is not None: + require(q['quote']['output_cap']==manifest['output_cap'], 'quote output cap mismatch') + return b, a + + +def _bounds(trials, include_pairs, max_calls): + integer(trials,1,100);integer(max_calls,1,10000) + require(type(include_pairs) is bool, 'include_pairs must be boolean') + + +def _variants(ids, include_pairs): + return [[]] + [[i] for i in ids] + (sorted(map(list,itertools.combinations(ids,2))) if include_pairs else []) + + +def _schedule(source, ids, trials, pairs, max_calls, identities): + variants=_variants(ids,pairs) + count=len(variants)*trials + require(count <= max_calls, 'scheduled calls exceed max_calls') + rows=[] + for removed in variants: + variant=sha(canonical_bytes(dict(version='decision-ablation-variant/1',source_bundle_digest=source, + removed_ids=removed,**identities))) + for repeat in range(trials): + rows.append(dict(trial_id=f'{variant}:{repeat}',repeat_index=repeat,removed_ids=removed[:],variant_digest=variant)) + return rows + + +def plan_ablations(bundle, trials=3, include_pairs=False, max_calls=33, *, selected_ids=None): + """Validate declared initial incident/1 applicability and emit a bounded plan.""" + _bounds(trials,include_pairs,max_calls) + if selected_ids is not None: + require(type(selected_ids) is list and len(selected_ids)<=4, 'select at most four IDs explicitly') + for i in selected_ids:integer(i,1) + require(len(set(selected_ids))==len(selected_ids), 'duplicate selection') + b,a=_applicable(bundle) + selectable=_group_positions(a['inputs'][1:]) + if selected_ids is None: + require(len(selectable)<=4, 'more than four items require explicit selection') + selected_ids=[i['evidence_id'] for i in selectable] + all_ids={i['evidence_id'] for i in selectable} + require(all(i in all_ids for i in selected_ids), 'unknown or unselectable ID') + positions=[i for i in selectable if i['evidence_id'] in selected_ids] + ids=[i['evidence_id'] for i in positions] + source=sha(canonical_bytes(b)) + identities=dict(scorer=SCORER,runtime=a['immutable_runtime'],validator=a['validator']) + rows=_schedule(source,ids,trials,include_pairs,max_calls,identities) + result=dict(version=PLAN,source_bundle_digest=source,selected_ids=ids,selected_positions=positions, + trials=trials,include_pairs=include_pairs,max_calls=max_calls,scheduled_calls=len(rows),rows=rows,**identities) + preflight(result) + return result + + +def _checked_plan(plan): + preflight(plan) + fields(plan, {'version','source_bundle_digest','selected_ids','selected_positions','trials','include_pairs', + 'max_calls','scheduled_calls','rows',*IDENTITIES}) + require(plan['version']==PLAN and plan['scorer']==SCORER, 'unsupported plan/scorer') + digest(plan['source_bundle_digest']);_bounds(plan['trials'],plan['include_pairs'],plan['max_calls']) + runtime,_=_embedded(plan['runtime']);_runtime(runtime);_validator(plan['validator']) + ids=plan['selected_ids'];positions=plan['selected_positions'] + require(type(ids) is list and len(ids)<=4 and type(positions) is list and len(positions)==len(ids), 'selected ID shape') + for i in ids:integer(i,1) + require(len(set(ids))==len(ids), 'duplicate selected ID') + last=0 + for i,p in zip(ids,positions): + fields(p, {'evidence_id','positions'});integer(p['evidence_id'],1) + require(p['evidence_id']==i and type(p['positions']) is list and len(p['positions'])==1, + 'incident/1 position identity') + integer(p['positions'][0],1,1023) + require(p['positions'][0]>last, 'selected positions must follow original order');last=p['positions'][0] + integer(plan['scheduled_calls'],1,10000) + identities={k:plan[k] for k in IDENTITIES} + expected=_schedule(plan['source_bundle_digest'],ids,plan['trials'],plan['include_pairs'],plan['max_calls'],identities) + require(type(plan['rows']) is list and len(plan['rows'])==len(expected), 'incomplete plan schedule') + # Canonical comparison also refuses bool-as-int in nested schedule rows. + require(plan['scheduled_calls']==len(expected) and canonical_bytes(plan['rows'])==canonical_bytes(expected), 'inconsistent plan schedule') + return expected + + +def _result(value): + fields(value, {'version','value'}) + require(value['version']==RESULT, 'unsupported normalized-result encoding') + preflight(value, INCIDENT_LIMIT) + return canonical_bytes(value) + + +def score_ablations(plan, outcomes): + """Compare producer-supplied attestations after independently checking schedule. + + Without the source bundle this checks plan consistency, not applicability. + """ + preflight(plan);preflight(outcomes) + rows=_checked_plan(plan) + fields(outcomes, {'version','plan_digest','rows'}) + require(outcomes['version']==OUTCOMES, 'unsupported outcomes') + digest(outcomes['plan_digest']) + require(outcomes['plan_digest']==sha(canonical_bytes(plan)), 'changed plan digest') + require(type(outcomes['rows']) is list and len(outcomes['rows'])<=len(rows), 'outcome count bound') + scheduled={r['trial_id']:r for r in rows};observed={};mismatches=[];costs={} + for o in outcomes['rows']: + fields(o, {'trial_id','state','response_digest','semantic_result','charged_units','reserved_bound','unit',*IDENTITIES}) + require(type(o['trial_id']) is str and o['trial_id'] in scheduled and o['trial_id'] not in observed, 'unknown or duplicate trial ID') + require(o['state'] in ('recorded','failed','unknown'), 'unknown outcome state') + if o['response_digest'] is not None:digest(o['response_digest']) + if o['state']=='recorded': + digest(o['response_digest']);_result(o['semantic_result']) + else: + require(o['semantic_result'] is None, 'unrecorded semantic result') + integer(o['reserved_bound']) + if o['charged_units'] is not None: + integer(o['charged_units']);require(o['charged_units']<=o['reserved_bound'], 'charge exceeds bound') + require(o['unit'] in ('logical_support','fixture_bytes','model_tokens','unknown'), 'unknown unit label') + text(o['scorer']);decode_envelope(o['runtime']);_validator(o['validator'],False) + for key in IDENTITIES: + if canonical_bytes(o[key]) != canonical_bytes(plan[key]): + mismatches.append(dict(trial_id=o['trial_id'],identity=key)) + observed[o['trial_id']]=o + c=costs.setdefault(o['unit'],dict(unit=o['unit'],charged_units=0,known_charged_units=0,unknown_charges=0,reserved_bound=0,held_units=0)) + if o['charged_units'] is None: + c['unknown_charges']+=1 + else:c['known_charged_units']+=o['charged_units'] + c['reserved_bound']+=o['reserved_bound'] + if o['charged_units'] is None or o['state']!='recorded':c['held_units']+=o['reserved_bound'] + for c in costs.values(): + for key in ('known_charged_units','reserved_bound','held_units'):integer(c[key]) + integer(c['known_charged_units']+c['held_units']) + c['charged_units']=None if c['unknown_charges'] else c['known_charged_units'] + baseline=[observed.get(r['trial_id']) for r in rows if not r['removed_ids']] + baseline_reasons=[] + if any(o is None or o['state']!='recorded' for o in baseline):baseline_reasons.append('baseline_unavailable') + baseline_values=[_result(o['semantic_result']) for o in baseline if o is not None and o['state']=='recorded'] + if len(set(baseline_values))>1:baseline_reasons.append('baseline_unstable') + usable=not baseline_reasons and not mismatches + variants=[] + for removed in _variants(plan['selected_ids'],plan['include_pairs'])[1:]: + v=dict(removed_ids=removed,scheduled=plan['trials'],changed=0,unchanged=0,failed=0,unknown=0,missing=0, + unscored=0,association_score=None,rank=None) + for r in rows: + if r['removed_ids']!=removed:continue + o=observed.get(r['trial_id']) + if o is None:v['missing']+=1 + elif o['state']!='recorded':v[o['state']]+=1 + elif not usable:v['unscored']+=1 + else:v['unchanged' if _result(o['semantic_result'])==baseline_values[0] else 'changed']+=1 + if usable and not (v['failed'] or v['unknown'] or v['missing']):v['association_score']=v['changed']/plan['trials'] + variants.append(v) + # Preserve source-order display, assign equal exact numerator scores equal ranks. + ranked=[v for v in variants if v['association_score'] is not None] + for v in ranked:v['rank']=1+sum(other['changed']>v['changed'] for other in ranked) + missing=[r['trial_id'] for r in rows if r['trial_id'] not in observed] + return dict(version='decision-ablation-report/1',plan_digest=outcomes['plan_digest'], + coverage='Incomplete' if missing else 'Complete',baseline_status='Inconclusive' if baseline_reasons else 'Stable', + baseline_reasons=baseline_reasons,comparison_status='Incomparable' if mismatches else 'Comparable', + identity_mismatches=mismatches,scheduled_calls=len(rows),observed_outcomes=len(observed), + recorded_calls=sum(o['state']=='recorded' for o in observed.values()),missing_trials=missing, + costs=[costs[k] for k in sorted(costs)],unreported_cost_trials=len(missing),variants=variants, + interpretation='Recorded attestations only; association is not causal influence or scientific truth. Execution and current authorization are not authenticated.') + + +def main(argv=None): + parser=argparse.ArgumentParser(description=__doc__) + sub=parser.add_subparsers(dest='command',required=True) + p=sub.add_parser('plan');p.add_argument('bundle');p.add_argument('--trials',type=int,default=3) + p.add_argument('--include-pairs',action='store_true');p.add_argument('--max-calls',type=int,default=33) + p.add_argument('--selected-id',type=int,action='append',dest='selected_ids') + s=sub.add_parser('score');s.add_argument('plan');s.add_argument('outcomes') + args=parser.parse_args(argv) + try: + if args.command=='plan': + result=plan_ablations(read_json(args.bundle),args.trials,args.include_pairs,args.max_calls,selected_ids=args.selected_ids) + else:result=score_ablations(read_json(args.plan),read_json(args.outcomes)) + print(canonical_bytes(result).decode('utf-8'));return 0 + except (ValueError,OSError) as error: + print(f'decision ablation refused: {error}',file=sys.stderr);return 1 + + +if __name__=='__main__':sys.exit(main()) diff --git a/scripts/test_decision_ablation.py b/scripts/test_decision_ablation.py new file mode 100644 index 0000000..999c953 --- /dev/null +++ b/scripts/test_decision_ablation.py @@ -0,0 +1,358 @@ +"""Offline diagnostics: real C export, resealed adversaries and no-execution checks.""" +import copy +from contextlib import ExitStack, redirect_stdout, redirect_stderr +import hashlib +import importlib +import io +import json +import os +from pathlib import Path +import socket +import subprocess +import tempfile +import unittest +from unittest.mock import patch +import urllib.request + +from decision_bundle_v1 import canonical_bytes, decode_envelope + +ROOT = Path(__file__).resolve().parents[1] + + +def sha(raw): + return hashlib.sha256(raw).hexdigest() + + +def envelope(raw): + return dict(hex=raw.hex(), byte_len=len(raw), sha256=sha(raw)) + + +def seal(bundle): + bundle = copy.deepcopy(bundle) + bundle['integrity'].pop('sha256', None) + bundle['integrity']['sha256'] = sha(canonical_bytes(bundle)) + return bundle + + +def rebuild(bundle, count=None, identical=False): + """Create explicitly synthetic declared-contract variants of a real C export.""" + b = copy.deepcopy(bundle) + a = b['attempts'][0] + context = json.loads(decode_envelope(b['evidence'][0]['payload'])) + m = json.loads(decode_envelope(a['immutable_manifest'])) + count = count or len(m['sources']) + while len(m['sources']) < count: + index = len(m['sources']) + source = copy.deepcopy(m['sources'][0]) + source.update(external_id=f'e{index+1:02}', occurrence_id=index+2) + m['sources'].append(source) + e = copy.deepcopy(b['evidence'][1]) + e['occurrence_id'] = index+2 + b['evidence'].append(e) + context['evidence_ids'] = [s['external_id'] for s in m['sources']] + b['evidence'][0]['payload'] = envelope(canonical_bytes(context)) + m['context']['sha256'] = b['evidence'][0]['payload']['sha256'] + a['inputs'] = [dict(position=0, evidence_id=m['context']['occurrence_id'], role='context')] + reports = [] + for i,s in enumerate(m['sources'], 1): + e = b['evidence'][i] + if identical: + e['payload'] = copy.deepcopy(b['evidence'][1]['payload']) + s['payload_sha256'] = e['payload']['sha256'] + a['inputs'].append(dict(position=i, evidence_id=s['occurrence_id'], role='report')) + reports.append(dict(id=s['external_id'], text=decode_envelope(e['payload']).decode())) + a['candidates'] = [dict(position=i['position'], evidence_id=i['evidence_id']) for i in a['inputs']] + request = canonical_bytes(dict(messages=[dict(role='system', content=context['system_prompt']), + dict(role='user', content=canonical_bytes(dict(question=context['question'], owner_ids=context['owner_ids'], evidence=reports)).decode())])) + a['immutable_manifest'] = envelope(canonical_bytes(m)) + runtime = json.loads(decode_envelope(a['immutable_runtime'])) + runtime['prompt_fingerprint'] = a['immutable_manifest']['sha256'] + a['immutable_runtime'] = envelope(canonical_bytes(runtime)) + timing = json.loads(decode_envelope(a['timing_configuration'])) + timing['runtime'] = runtime + a['timing_configuration'] = envelope(canonical_bytes(timing)) + a['dispatch_request'] = a['recovery']['request'] = envelope(request) + # Synthetic accounting remains consistent with captured fixture-byte requests. + a['accounting']['quote']['input_tokens'] = len(request) + a['accounting']['charge'] = a['delivery']['usage'] = len(request) + a['delivery']['raw']['byte_len'] + return seal(b) + + +def outcomes(plan): + return dict(version='decision-ablation-outcomes/1', plan_digest=sha(canonical_bytes(plan)), rows=[ + dict(trial_id=r['trial_id'], state='recorded', response_digest=sha(b'fixture response'), + semantic_result=dict(version='recorded-normalized-result/1', value={'decision': 'abstain'}), + charged_units=2, reserved_bound=10, unit='fixture_bytes', + scorer=plan['scorer'], runtime=copy.deepcopy(plan['runtime']), validator=copy.deepcopy(plan['validator'])) + for r in plan['rows']]) + + +class AblationTests(unittest.TestCase): + @classmethod + def setUpClass(cls): + runner = os.environ.get('KAIROTENSOR_DECISION_REPLAY', '') + if not runner or not Path(runner).is_file() or not os.access(runner, os.X_OK): + raise AssertionError('KAIROTENSOR_DECISION_REPLAY must bind a verified precompiled executable; no Cargo fallback or skips') + with tempfile.TemporaryDirectory() as folder: + path = Path(folder)/'real.json' + result = subprocess.run([runner, 'export-fixture', str(path)], capture_output=True, timeout=30) + if result.returncode: + raise AssertionError(result.stderr) + cls.actual = json.loads(path.read_bytes()) + replay = subprocess.run([runner, 'replay', str(path)], capture_output=True, timeout=30) + if replay.returncode or json.loads(replay.stdout)['reconstruction'] != 'matched': + raise AssertionError(replay.stderr) + cls.runner = runner + + def module(self): + self.assertTrue((ROOT/'scripts/decision_ablation.py').exists(), 'bounded offline planner/scorer must exist') + return importlib.import_module('decision_ablation') + + def test_actual_c_export_plans_without_modification(self): + m = self.module() + before = canonical_bytes(self.actual) + p = m.plan_ablations(self.actual) + self.assertEqual(p['selected_ids'], [2,3,4]) + self.assertEqual(p['scheduled_calls'], 12) + self.assertEqual(p['source_bundle_digest'], sha(before)) + self.assertEqual(p['runtime'], self.actual['attempts'][0]['immutable_runtime']) + self.assertEqual(p['selected_positions'], [dict(evidence_id=i, positions=[i-1]) for i in [2,3,4]]) + self.assertEqual(before, canonical_bytes(self.actual)) + + def test_four_three_pairs_exact_complete_bound(self): + m = self.module(); b = rebuild(self.actual, 4) + p = m.plan_ablations(b, include_pairs=True) + self.assertEqual(p['scheduled_calls'], 33) + self.assertEqual([r['removed_ids'] for r in p['rows'][::3]], [[],[2],[3],[4],[5],[2,3],[2,4],[2,5],[3,4],[3,5],[4,5]]) + self.assertEqual(len({r['trial_id'] for r in p['rows']}), 33) + self.assertEqual(len({r['variant_digest'] for r in p['rows']}), 11) + with self.assertRaises(ValueError): m.plan_ablations(b, include_pairs=True, max_calls=32) + self.assertEqual(p, m.plan_ablations(b, include_pairs=True)) + + def test_explicit_selection_threshold_order_and_ids(self): + m = self.module(); b = rebuild(self.actual, 5) + with self.assertRaises(ValueError): m.plan_ablations(b) + self.assertEqual(m.plan_ablations(b, selected_ids=[5,2])['selected_ids'], [2,5]) + for ids in ([1], [999], [2,2], [True], [2,3,4,5,6], '2'): + with self.subTest(ids=ids), self.assertRaises(ValueError): m.plan_ablations(b, selected_ids=ids) + + def test_invalid_bounds_types_overflow(self): + m = self.module() + for kwargs in ({'trials':0},{'trials':101},{'trials':True},{'trials':1.0}, {'trials':2**100}, + {'max_calls':True},{'max_calls':10001},{'max_calls':0},{'include_pairs':1}, {'max_calls':1}): + with self.subTest(kwargs=kwargs), self.assertRaises(ValueError): m.plan_ablations(self.actual, **kwargs) + + def test_identical_bytes_distinct_ids_and_internal_grouping(self): + m = self.module() + self.assertEqual(m.plan_ablations(rebuild(self.actual, identical=True))['selected_ids'], [2,3,4]) + self.assertEqual(m._group_positions([dict(position=1,evidence_id=7),dict(position=3,evidence_id=8),dict(position=5,evidence_id=7)]), + [dict(evidence_id=7,positions=[1,5]),dict(evidence_id=8,positions=[3])]) + b=copy.deepcopy(self.actual); b['attempts'][0]['inputs'][2]['evidence_id']=2 + with self.assertRaises(ValueError): m.plan_ablations(seal(b)) + + def test_redacted_unknown_contract_roles_and_dependencies_refuse(self): + m=self.module() + with self.assertRaises(ValueError): m.plan_ablations(dict(status='redacted', bundle={})) + mutations=[('inherited_from',1),('reuse_origin',1),('support',{}),('deterministic',{}),('model_contract','opaque')] + for key,value in mutations: + b=copy.deepcopy(self.actual); b['attempts'][0][key]=value + with self.subTest(key=key),self.assertRaises(ValueError): m.plan_ablations(seal(b)) + for role in ('unknown','evidence','policy','context'): + b=copy.deepcopy(self.actual); b['attempts'][0]['inputs'][1]['role']=role + with self.subTest(role=role),self.assertRaises(ValueError):m.plan_ablations(seal(b)) + b=copy.deepcopy(self.actual);b['attempts'][0]['inputs'][0]['role']='report' + with self.assertRaises(ValueError):m.plan_ablations(seal(b)) + + def test_resealed_metadata_and_request_adversaries(self): + m=self.module() + def mutate_manifest(b): + a=b['attempts'][0]; v=json.loads(decode_envelope(a['immutable_manifest']));v['sources'][0]['external_id']='forged';a['immutable_manifest']=envelope(canonical_bytes(v)) + def mutate_timing(b): + a=b['attempts'][0];v=json.loads(decode_envelope(a['timing_configuration']));v['runtime']['threads']=True;a['timing_configuration']=envelope(canonical_bytes(v)) + changes=[lambda b:b.update(root_attempt=True),lambda b:b['snapshot'].update(root_branch=2), + lambda b:b['attempts'][0].update(actor='forged'),lambda b:b['evidence'][1].update(authorized_scopes=[]), + lambda b:b['evidence'].append(copy.deepcopy(b['evidence'][1])), + lambda b:b['attempts'][0]['recovery'].update(ordinal=1,predecessor_id=5), + lambda b:b['attempts'][0]['recovery'].update(state='unknown'), + lambda b:b['attempts'][0].update(dispatch_request=envelope(b'forged')), + lambda b:b['attempts'][0]['recovery'].update(request=envelope(b'forged')), + lambda b:b['attempts'][0]['validator'].update(provenance='unknown'), + lambda b:b['attempts'][0]['validator'].update(version='forged'), mutate_manifest,mutate_timing] + for change in changes: + b=copy.deepcopy(self.actual);change(b) + with self.subTest(change=changes.index(change)),self.assertRaises(ValueError): m.plan_ablations(seal(b)) + + def test_prepared_unsent_and_rejection_provenance(self): + m=self.module();b=copy.deepcopy(self.actual);a=b['attempts'][0] + a['dispatch_request']=None;a['recovery']['state']='reserved';a['delivery']={'state':'unresolved'} + a['accounting']['charge']=0;a['accounting']['hold_bound']=sum(a['accounting']['quote'].values()) + self.assertEqual(m.plan_ablations(seal(b))['scheduled_calls'],12) + a['recovery']['state']='planned' + with self.assertRaises(ValueError):m.plan_ablations(seal(b)) + b=copy.deepcopy(self.actual);a=b['attempts'][0];a['validator']['provenance']='recorded_rejection' + with self.assertRaises(ValueError):m.plan_ablations(seal(b)) + a['delivery'].update(disposition='invalid_output', rejection=dict(validator='incident-routing-protocol/1',reason='invalid_output',applicability='current')) + self.assertEqual(m.plan_ablations(seal(b))['validator']['provenance'],'recorded_rejection') + a['delivery']['rejection']['validator']='forged' + with self.assertRaises(ValueError):m.plan_ablations(seal(b)) + + def test_nested_manifest_runtime_types_and_cost_overflow_refuse(self): + m=self.module();b=copy.deepcopy(self.actual);a=b['attempts'][0] + manifest=json.loads(decode_envelope(a['immutable_manifest'])) + manifest['runtime']['threads']=True + a['immutable_manifest']=envelope(canonical_bytes(manifest)) + runtime=json.loads(decode_envelope(a['immutable_runtime'])) + runtime['prompt_fingerprint']=a['immutable_manifest']['sha256'] + a['immutable_runtime']=envelope(canonical_bytes(runtime)) + timing=json.loads(decode_envelope(a['timing_configuration']));timing['runtime']=runtime + a['timing_configuration']=envelope(canonical_bytes(timing)) + with self.subTest(case='manifest_bool'),self.assertRaises(ValueError):m.plan_ablations(seal(b)) + p=m.plan_ablations(self.actual);o=outcomes(p) + for row in o['rows']:row.update(reserved_bound=(1<<63)-1,charged_units=(1<<63)-1) + with self.subTest(case='aggregate_overflow'),self.assertRaises(ValueError):m.score_ablations(p,o) + b=copy.deepcopy(self.actual);b['attempts'][0]['accounting']['unit']='logical_support' + with self.subTest(case='source_unit'),self.assertRaises(ValueError):m.plan_ablations(seal(b)) + + def test_scores_ties_denominators_and_unit_totals(self): + m=self.module();p=m.plan_ablations(self.actual);o=outcomes(p) + for r in o['rows'][3:9]: r['semantic_result']['value']={'decision':'route'} + report=m.score_ablations(p,o) + self.assertEqual(report['coverage'],'Complete');self.assertEqual(report['baseline_status'],'Stable') + self.assertEqual([v['association_score'] for v in report['variants']],[1.0,1.0,0.0]) + self.assertEqual([v['rank'] for v in report['variants']],[1,1,3]) + self.assertEqual([v['removed_ids'] for v in report['variants']],[[2],[3],[4]]) + self.assertEqual(report['recorded_calls'],12) + self.assertEqual(report['costs'],[dict(unit='fixture_bytes',charged_units=24,known_charged_units=24,unknown_charges=0,reserved_bound=120,held_units=0)]) + + def test_unstable_failed_unknown_missing_baselines_are_inconclusive(self): + m=self.module();p=m.plan_ablations(self.actual) + for change in ('unstable','failed','unknown','missing'): + o=outcomes(p) + if change=='unstable':o['rows'][0]['semantic_result']['value']=True + elif change=='missing':o['rows'].pop(0) + else:o['rows'][0].update(state=change,response_digest=None,semantic_result=None,charged_units=None) + r=m.score_ablations(p,o) + self.assertEqual(r['baseline_status'],'Inconclusive',change) + self.assertTrue(all(v['rank'] is None and v['association_score'] is None for v in r['variants'])) + self.assertEqual(r['coverage'],'Incomplete' if change=='missing' else 'Complete') + + def test_variant_missing_failed_unknown_no_rank(self): + m=self.module();p=m.plan_ablations(self.actual) + for state in ('missing','failed','unknown'): + o=outcomes(p) + if state=='missing':o['rows'].pop(3) + else:o['rows'][3].update(state=state,response_digest=None,semantic_result=None,charged_units=None) + r=m.score_ablations(p,o);v=r['variants'][0] + self.assertIsNone(v['rank']);self.assertIsNone(v['association_score']) + self.assertEqual(v['scheduled'],3);self.assertEqual(v['unchanged'],2) + self.assertEqual(v['missing'] if state=='missing' else v[state],1) + + def test_unknown_cost_and_mixed_units_remain_distinct(self): + m=self.module();p=m.plan_ablations(self.actual);o=outcomes(p) + o['rows'][0].update(charged_units=None);o['rows'][1].update(unit='model_tokens');o['rows'][2].update(unit='unknown',charged_units=None) + r=m.score_ablations(p,o);costs={c['unit']:c for c in r['costs']} + self.assertIsNone(costs['fixture_bytes']['charged_units']);self.assertEqual(costs['fixture_bytes']['unknown_charges'],1) + self.assertEqual(costs['model_tokens']['charged_units'],2);self.assertIsNone(costs['unknown']['charged_units']) + self.assertEqual(costs['fixture_bytes']['reserved_bound'],100) + + def test_outcome_duplicates_unknown_ids_digest_and_nested_types_refuse(self): + m=self.module();p=m.plan_ablations(self.actual) + changes=[lambda o:o['rows'].append(copy.deepcopy(o['rows'][0])),lambda o:o['rows'][0].update(trial_id='unknown'), + lambda o:o.update(plan_digest='0'*64), lambda o:o['rows'][0].update(charged_units=True), + lambda o:o['rows'][0].update(semantic_result={'version':'arbitrary','value':None}), + lambda o:o['rows'][0].update(semantic_result={'version':'recorded-normalized-result/1','value':1.5}), + lambda o:o['rows'][0].update(response_digest=None),lambda o:o['rows'][0].update(charged_units=11)] + for index,change in enumerate(changes): + o=outcomes(p);change(o) + with self.subTest(index=index),self.assertRaises(ValueError):m.score_ablations(p,o) + + def test_comparison_identity_mismatch_preserves_other_statuses(self): + m=self.module();p=m.plan_ablations(self.actual) + for field in ('scorer','runtime','validator'): + o=outcomes(p) + if field=='scorer':o['rows'][0][field]='other/1' + elif field=='validator':o['rows'][0][field]['provenance']='unknown' + else:o['rows'][0][field]=envelope(b'other runtime') + o['rows'].pop() + r=m.score_ablations(p,o) + self.assertEqual(r['comparison_status'],'Incomparable');self.assertEqual(r['coverage'],'Incomplete') + self.assertTrue(all(v['rank'] is None for v in r['variants'])) + + def test_rehashed_malformed_plans_cannot_shrink_schedule(self): + m=self.module();p=m.plan_ablations(rebuild(self.actual,4),include_pairs=True) + changes=[lambda p:p['rows'].pop(0),lambda p:p['rows'].__delitem__(slice(-3,None)), + lambda p:p['rows'][0].update(repeat_index=1), lambda p:p['rows'][0].update(variant_digest='0'*64), + lambda p:p.update(scheduled_calls=1),lambda p:p.update(trials=True), + lambda p:p.update(selected_ids=[2,2]),lambda p:p['rows'][0].update(removed_ids=[2])] + for index,change in enumerate(changes): + bad=copy.deepcopy(p);change(bad) + with self.subTest(index=index),self.assertRaises(ValueError):m.score_ablations(bad,outcomes(bad)) + + def test_normalized_result_strict_type_equality(self): + m=self.module();p=m.plan_ablations(self.actual);o=outcomes(p) + for r in o['rows']:r['semantic_result']['value']=1 + o['rows'][3]['semantic_result']['value']=True + self.assertEqual(m.score_ablations(p,o)['variants'][0]['changed'],1) + + def test_public_objects_and_transport_bounded_before_serialization(self): + m=self.module() + huge={'x':'x'*(16*1024*1024+1)} + for func,args in ((m.plan_ablations,(huge,)),(m.score_ablations,(huge,{})),(m.score_ablations,({},huge))): + with patch.object(m,'canonical_bytes',side_effect=AssertionError('serialized unbounded object')): + with self.assertRaises(ValueError):func(*args) + cycle=[];cycle.append(cycle) + with self.assertRaises(ValueError):m.plan_ablations(cycle) + with tempfile.TemporaryDirectory() as folder: + path=Path(folder)/'big' + with path.open('wb') as stream:stream.truncate(16*1024*1024+1) + with self.assertRaises(ValueError):m.read_json(path) + for raw in (b'{"a":1,"a":2}',b'['*65+b'0'+b']'*65,b'{"x":1.0}'): + with self.assertRaises(ValueError):m.parse_json(raw) + + def test_api_and_both_cli_commands_cannot_execute_process_or_network(self): + m=self.module() + with tempfile.TemporaryDirectory() as folder: + path=Path(folder);(path/'bundle').write_bytes(canonical_bytes(self.actual)) + with ExitStack() as stack: + for obj,name in ((subprocess,'Popen'),(subprocess,'run'),(os,'system'),(os,'popen'),(os,'posix_spawn'),(socket,'socket'),(socket,'create_connection'),(urllib.request,'urlopen')): + stack.enter_context(patch.object(obj,name,side_effect=AssertionError('execution forbidden'))) + p=m.plan_ablations(self.actual);o=outcomes(p);m.score_ablations(p,o) + (path/'plan').write_bytes(canonical_bytes(p));(path/'outcomes').write_bytes(canonical_bytes(o)) + for args in (['plan',str(path/'bundle')],['score',str(path/'plan'),str(path/'outcomes')]): + output=io.StringIO() + with redirect_stdout(output):self.assertEqual(m.main(args),0) + self.assertIsInstance(json.loads(output.getvalue()),dict) + bad=copy.deepcopy(self.actual);bad['attempts'][0]['inputs'][0]['role']='report';(path/'bad').write_bytes(canonical_bytes(seal(bad))) + with redirect_stderr(io.StringIO()):self.assertNotEqual(m.main(['plan',str(path/'bad')]),0) + with redirect_stderr(io.StringIO()),self.assertRaises(SystemExit):m.main(['run',str(path/'bundle')]) + + def test_resealed_request_and_role_attacks_differential_rust_refuse(self): + m=self.module() + with tempfile.TemporaryDirectory() as folder: + path=Path(folder)/'mutated' + for attack in ('request','role','source','missing_identity'): + b=copy.deepcopy(self.actual);a=b['attempts'][0] + if attack=='request':a['dispatch_request']=a['recovery']['request']=envelope(b'forged request') + elif attack=='role':a['inputs'][0]['role']='report' + elif attack=='source':b['evidence'][1]['payload']=envelope(b'forged payload') + else:a['immutable_runtime']=None + b=seal(b);path.write_bytes(canonical_bytes(b)) + result=subprocess.run([self.runner,'replay',str(path)],capture_output=True,timeout=30) + if result.returncode==0: + self.assertNotEqual(json.loads(result.stdout)['reconstruction'],'matched',attack) + with self.subTest(attack=attack),patch.object(subprocess,'run',side_effect=AssertionError('no validation process')): + with self.assertRaises(ValueError):m.plan_ablations(b) + + def test_synthetic_supported_variants_differential_rust_reconstruction(self): + m=self.module() + with tempfile.TemporaryDirectory() as folder: + path=Path(folder)/'synthetic' + for count in (3,4,5): + b=rebuild(self.actual,count,identical=True);path.write_bytes(canonical_bytes(b)) + result=subprocess.run([self.runner,'replay',str(path)],capture_output=True,timeout=30) + self.assertEqual(result.returncode,0,result.stderr) + self.assertEqual(json.loads(result.stdout)['reconstruction'],'matched') + self.assertEqual(len(m.plan_ablations(b,selected_ids=[2])['selected_ids']),1) + + +if __name__=='__main__':unittest.main() From 794f64e5f9702987cd2502b86776b28f8a8d97c5 Mon Sep 17 00:00:00 2001 From: Alejandro Castellanos <5405628+k2jac9@users.noreply.github.com> Date: Sun, 15 Nov 2026 20:25:09 -0500 Subject: [PATCH 29/51] fix: preserve unspecified-unit accounting observations --- docs/decision-ablation.md | 26 ++++++++++++++++---- scripts/decision_ablation.py | 23 +++++++++++++----- scripts/test_decision_ablation.py | 40 +++++++++++++++++++++++++++++++ 3 files changed, 78 insertions(+), 11 deletions(-) diff --git a/docs/decision-ablation.md b/docs/decision-ablation.md index 6245490..830ecfe 100644 --- a/docs/decision-ablation.md +++ b/docs/decision-ablation.md @@ -138,11 +138,27 @@ its supplied result was validated; this tool compares that recorded attestation. `reserved_bound` is a nonnegative integer bounding any known charge. Unit is `model_tokens`, `fixture_bytes`, `logical_support` or `unknown`. Outcome units are reported separately, even though this initial source contract is narrower. -All scalar and per-unit aggregate sums must fit i64. Unknown charges stay null, -with known partial charges and unknown counts retained. `reserved_bound` sums all -observed row bounds; `held_units` conservatively retains the entire bound for -failed/unknown rows or any unknown charge. It is a reported uncertainty bound, -not a reconstructed Store hold. Recorded known-charge rows contribute no hold. +All scalar measurements and named-unit aggregate sums must fit i64. In each +known named-unit bucket, unknown charges keep `charged_units` null, with +`known_charged_units` retaining the known partial sum and `unknown_charges` +counting unknown measurements. `reserved_bound` sums that named unit's observed +row bounds; `held_units` conservatively retains the entire bound for failed/unknown +rows or any unknown charge. It is a reported uncertainty bound, not a reconstructed +Store hold. Recorded known-charge rows contribute no hold. + +The label `unknown` does not establish commensurable units, even when rows share a +plan or runtime and each has a numeric charge. Its cost bucket therefore keeps +`charged_units`, `known_charged_units`, `reserved_bound` and `held_units` all null. +Only `observed_trials` and `unknown_charges` are counted. Its `observations` list +preserves each unspecified-unit row in supplied order with exactly `trial_id`, +`state`, original numeric-or-null `charged_units`, original `reserved_bound` and +conservative per-trial `held_units` (the full bound for failed/unknown rows or an +unknown charge, otherwise zero). No unknown-unit magnitudes, bounds or holds are +summed, including for overflow checks; each individual measurement remains bounded. +For example, charges 3 and 7 with unspecified units and bounds 17 and 23 remain +two observations with null aggregate measurements, rather than charges totaling 10 +or bounds totaling 40. + Missing outcome rows have unknown costs and bounds, explicitly counted by `unreported_cost_trials`; their costs are never silently assigned zero. diff --git a/scripts/decision_ablation.py b/scripts/decision_ablation.py index 24b9005..96ec967 100644 --- a/scripts/decision_ablation.py +++ b/scripts/decision_ablation.py @@ -452,13 +452,24 @@ def score_ablations(plan, outcomes): if canonical_bytes(o[key]) != canonical_bytes(plan[key]): mismatches.append(dict(trial_id=o['trial_id'],identity=key)) observed[o['trial_id']]=o - c=costs.setdefault(o['unit'],dict(unit=o['unit'],charged_units=0,known_charged_units=0,unknown_charges=0,reserved_bound=0,held_units=0)) - if o['charged_units'] is None: - c['unknown_charges']+=1 - else:c['known_charged_units']+=o['charged_units'] - c['reserved_bound']+=o['reserved_bound'] - if o['charged_units'] is None or o['state']!='recorded':c['held_units']+=o['reserved_bound'] + held = o['reserved_bound'] if o['charged_units'] is None or o['state']!='recorded' else 0 + if o['unit']=='unknown': + # Equal unspecified labels do not establish a shared measurement unit. + c=costs.setdefault('unknown',dict(unit='unknown',charged_units=None,known_charged_units=None, + unknown_charges=0,reserved_bound=None,held_units=None,observed_trials=0,observations=[])) + c['observed_trials']+=1 + c['unknown_charges']+=o['charged_units'] is None + c['observations'].append(dict(trial_id=o['trial_id'],state=o['state'], + charged_units=o['charged_units'],reserved_bound=o['reserved_bound'],held_units=held)) + else: + c=costs.setdefault(o['unit'],dict(unit=o['unit'],charged_units=0,known_charged_units=0,unknown_charges=0,reserved_bound=0,held_units=0)) + if o['charged_units'] is None: + c['unknown_charges']+=1 + else:c['known_charged_units']+=o['charged_units'] + c['reserved_bound']+=o['reserved_bound'] + c['held_units']+=held for c in costs.values(): + if c['unit']=='unknown':continue for key in ('known_charged_units','reserved_bound','held_units'):integer(c[key]) integer(c['known_charged_units']+c['held_units']) c['charged_units']=None if c['unknown_charges'] else c['known_charged_units'] diff --git a/scripts/test_decision_ablation.py b/scripts/test_decision_ablation.py index 999c953..49cd521 100644 --- a/scripts/test_decision_ablation.py +++ b/scripts/test_decision_ablation.py @@ -255,6 +255,46 @@ def test_unknown_cost_and_mixed_units_remain_distinct(self): self.assertEqual(costs['model_tokens']['charged_units'],2);self.assertIsNone(costs['unknown']['charged_units']) self.assertEqual(costs['fixture_bytes']['reserved_bound'],100) + def test_unspecified_units_preserve_numeric_rows_without_shared_aggregate(self): + m=self.module();p=m.plan_ablations(self.actual);o=outcomes(p) + o['rows'][0].update(unit='unknown',charged_units=3,reserved_bound=17) + o['rows'][1].update(unit='unknown',charged_units=7,reserved_bound=23) + before=canonical_bytes(o) + report=m.score_ablations(p,o);c=next(c for c in report['costs'] if c['unit']=='unknown') + self.assertEqual(c,dict(unit='unknown',charged_units=None,known_charged_units=None, + unknown_charges=0,reserved_bound=None,held_units=None,observed_trials=2,observations=[ + dict(trial_id=o['rows'][0]['trial_id'],state='recorded',charged_units=3,reserved_bound=17,held_units=0), + dict(trial_id=o['rows'][1]['trial_id'],state='recorded',charged_units=7,reserved_bound=23,held_units=0)])) + named=next(c for c in report['costs'] if c['unit']=='fixture_bytes') + self.assertEqual(named['charged_units'],20);self.assertEqual(named['reserved_bound'],100) + self.assertEqual(before,canonical_bytes(o)) + self.assertEqual(report['coverage'],'Complete');self.assertEqual(report['baseline_status'],'Stable') + + def test_unspecified_units_preserve_uncertain_charges_and_per_trial_holds(self): + m=self.module();p=m.plan_ablations(self.actual);o=outcomes(p) + o['rows'][3].update(unit='unknown',state='failed',response_digest=None,semantic_result=None,charged_units=2,reserved_bound=31) + o['rows'][4].update(unit='unknown',state='unknown',response_digest=None,semantic_result=None,charged_units=None,reserved_bound=47) + o['rows'][5].update(unit='unknown',charged_units=None,reserved_bound=59) + c=next(c for c in m.score_ablations(p,o)['costs'] if c['unit']=='unknown') + self.assertEqual(c.get('observed_trials'),3);self.assertEqual(c['unknown_charges'],2) + for field in ('charged_units','known_charged_units','reserved_bound','held_units'):self.assertIsNone(c[field]) + self.assertEqual(c['observations'],[ + dict(trial_id=o['rows'][3]['trial_id'],state='failed',charged_units=2,reserved_bound=31,held_units=31), + dict(trial_id=o['rows'][4]['trial_id'],state='unknown',charged_units=None,reserved_bound=47,held_units=47), + dict(trial_id=o['rows'][5]['trial_id'],state='recorded',charged_units=None,reserved_bound=59,held_units=59)]) + variant=m.score_ablations(p,o)['variants'][0] + self.assertEqual((variant['failed'],variant['unknown'],variant['unchanged']),(1,1,1)) + self.assertIsNone(variant['rank']) + + def test_unspecified_unit_magnitudes_are_not_aggregated_for_overflow(self): + m=self.module();p=m.plan_ablations(self.actual);o=outcomes(p) + for row in o['rows']:row.update(unit='unknown',charged_units=(1<<63)-1,reserved_bound=(1<<63)-1) + try:report=m.score_ablations(p,o) + except ValueError as error:self.fail(f'unspecified-unit rows must not be summed: {error}') + c=report['costs'][0];self.assertIsNone(c['known_charged_units']);self.assertEqual(c['observed_trials'],12) + self.assertEqual(len(c['observations']),12) + self.assertTrue(all(r['charged_units']==(1<<63)-1 and r['reserved_bound']==(1<<63)-1 for r in c['observations'])) + def test_outcome_duplicates_unknown_ids_digest_and_nested_types_refuse(self): m=self.module();p=m.plan_ablations(self.actual) changes=[lambda o:o['rows'].append(copy.deepcopy(o['rows'][0])),lambda o:o['rows'][0].update(trial_id='unknown'), From 5bf3c752b0e9805ce0909b524023776bf1be4e52 Mon Sep 17 00:00:00 2001 From: Alejandro Castellanos <5405628+k2jac9@users.noreply.github.com> Date: Sun, 15 Nov 2026 20:36:39 -0500 Subject: [PATCH 30/51] Add source-bound isolated runtime matrix verifier and refusal tests --- scripts/test_verify_runtime_matrix.py | 175 +++++++++++++ scripts/verify_runtime_matrix.py | 364 ++++++++++++++++++++++++++ 2 files changed, 539 insertions(+) create mode 100644 scripts/test_verify_runtime_matrix.py create mode 100644 scripts/verify_runtime_matrix.py diff --git a/scripts/test_verify_runtime_matrix.py b/scripts/test_verify_runtime_matrix.py new file mode 100644 index 0000000..511daf4 --- /dev/null +++ b/scripts/test_verify_runtime_matrix.py @@ -0,0 +1,175 @@ +"""Behavioral checks use tiny executables, never qualification evidence.""" +import copy +import hashlib +import importlib.util +import json +import os +from pathlib import Path +import subprocess +import sys +import tempfile +import unittest + +SPEC = importlib.util.spec_from_file_location('matrix', Path(__file__).with_name('verify_runtime_matrix.py')) + + +class RuntimeMatrixTests(unittest.TestCase): + def setUp(self): + self.assertTrue(Path(SPEC.origin).exists(), 'runtime verifier must exist') + self.m = importlib.util.module_from_spec(SPEC) + SPEC.loader.exec_module(self.m) + self.tmp = tempfile.TemporaryDirectory() + self.addCleanup(self.tmp.cleanup) + self.root = Path(self.tmp.name) + self.tool = self.root / 'fake' + self.tool.write_text(f'#!{sys.executable}\nimport sys\nprint("fixture version 1")\n') + self.tool.chmod(0o755) + self.identity = {'path': str(self.tool), 'sha256': self.sha(self.tool), + 'version_args': ['--version'], 'version': 'fixture version 1'} + + @staticmethod + def sha(p): + return hashlib.sha256(Path(p).read_bytes()).hexdigest() + + def command(self, code, kind='python', contains=None): + p = self.root / 'command' + p.write_text(f'#!{sys.executable}\n{code}\n'); p.chmod(0o755) + return {'id': 'test', 'argv': [str(p)], 'kind': kind, 'contains': contains or [], + 'timeout': 5, 'artifacts': []} + + def run_command(self, command): + return self.m.execute_command(command, self.root, os.environ.copy(), self.root / 'logs') + + def test_wrong_binary_digest_refused(self): + self.identity['sha256'] = '0' * 64 + with self.assertRaisesRegex(ValueError, 'digest'): + self.m.check_identity(self.identity) + + def test_wrong_executable_version_refused(self): + self.identity['version'] = 'different' + with self.assertRaisesRegex(ValueError, 'version'): + self.m.check_identity(self.identity) + + def test_missing_executable_is_unavailable(self): + self.identity['path'] = str(self.root / 'absent') + with self.assertRaises(FileNotFoundError): + self.m.check_identity(self.identity) + + def test_exact_executable_is_executed(self): + self.assertEqual(self.m.check_identity(self.identity)['version'], 'fixture version 1') + + def test_relative_runtime_refused(self): + self.identity['path'] = 'fake' + with self.assertRaisesRegex(ValueError, 'absolute'): + self.m.check_identity(self.identity) + + def test_missing_role_refused(self): + with self.assertRaisesRegex(ValueError, 'roles'): + self.m.validate_roles([{'role': 'stable'}]) + + def test_two_changed_runtimes_refused(self): + lanes = [] + for role in ['stable', 'rust-upcoming', 'rust-previous', 'python-upcoming', 'python-previous']: + lanes.append({'role': role, 'rustc': {'sha256': role if role.startswith('rust-') else 's'}, + 'python': {'sha256': role if role.startswith('python-') else 'p'}}) + self.m.validate_roles(lanes) + lanes[1]['python']['sha256'] = 'other' + with self.assertRaisesRegex(ValueError, 'independent'): + self.m.validate_roles(lanes) + + def test_changed_lock_or_tracked_file_refused(self): + subprocess.run(['git','init','-q',str(self.root)], check=True) + (self.root/'Cargo.lock').write_text('original') + subprocess.run(['git','-C',str(self.root),'add','.'],check=True) + subprocess.run(['git','-C',str(self.root),'-c','user.name=Fixture','-c','user.email=fixture@invalid','commit','-qm','fixture'],check=True) + commit=subprocess.check_output(['git','-C',str(self.root),'rev-parse','HEAD'],text=True).strip() + files={p:self.sha(self.root/p) for p in ['Cargo.lock','fake']} + source={'commit':commit,'files':files,'lock_sha256':files['Cargo.lock']} + self.m.check_source(self.root,source) + (self.root/'Cargo.lock').write_text('changed') + with self.assertRaises(ValueError): self.m.check_source(self.root,source) + + def test_nonzero_exit_cannot_pass(self): + r=self.run_command(self.command('print("Ran 2 tests\\nOK"); raise SystemExit(7)')) + self.assertEqual(r['status'],'Failed'); self.assertEqual(r['exit'],7) + + def test_python_skip_cannot_pass(self): + r=self.run_command(self.command('print("Ran 2 tests\\nOK (skipped=1)")')) + self.assertEqual(r['status'],'Failed'); self.assertEqual(r['skips'],1) + + def test_rust_filter_cannot_pass(self): + r=self.run_command(self.command('print("test result: ok. 2 passed; 0 failed; 0 ignored; 0 measured; 1 filtered out")','rust')) + self.assertEqual(r['status'],'Failed') + + def test_zero_or_missing_test_count_cannot_pass(self): + for output in ['Ran 0 tests\\nOK','all fine']: + r=self.run_command(self.command(f'print({output!r})')) + self.assertEqual(r['status'],'Failed') + + def test_output_predicate_required(self): + r=self.run_command(self.command('print("wrong")','fixture',['expected'])) + self.assertEqual(r['status'],'Failed') + + def test_missing_artifact_cannot_pass(self): + c=self.command('print("expected")','fixture',['expected']);c['artifacts']=[str(self.root/'missing.json')] + self.assertEqual(self.run_command(c)['status'],'Failed') + + def test_timeout_is_failed_and_retained(self): + c=self.command('import time; print("started",flush=True); time.sleep(5)');c['timeout']=0.05 + r=self.run_command(c) + self.assertEqual(r['status'],'Failed');self.assertTrue(r['timeout']) + self.assertTrue(Path(r['stdout']['path']).is_file()) + + def test_success_binds_real_logs_and_counts(self): + r=self.run_command(self.command('print("Ran 2 tests\\nOK")')) + self.assertEqual(r['status'],'Pass');self.assertEqual(r['tests'],2) + self.assertEqual(r['stdout']['sha256'],self.sha(r['stdout']['path'])) + + def test_receipt_checks_logs_not_reported_status(self): + c=self.command('print("Ran 2 tests\\nOK")');r=self.run_command(c) + self.m.check_command_receipt(c,r) + Path(r['stdout']['path']).write_text('tampered') + with self.assertRaises(ValueError):self.m.check_command_receipt(c,r) + + def test_missing_receipt_refused(self): + with self.assertRaisesRegex(ValueError,'receipt'): + self.m.check_command_receipt(self.command('pass'),{}) + + def test_forged_pass_with_skip_rejected(self): + c=self.command('print("Ran 2 tests\\nOK (skipped=1)")');r=self.run_command(c);r['status']='Pass' + with self.assertRaises(ValueError):self.m.check_command_receipt(c,r) + + def test_private_alias_selects_child_python(self): + lane={'rustc':self.identity,'cargo':self.identity,'python':self.identity} + env=self.m.lane_environment(lane,self.root/'aliases',{}, {'PATH':os.environ.get('PATH','')}) + r=subprocess.run(['python3','--version'],env=env,text=True,capture_output=True,check=True) + self.assertEqual(r.stdout.strip(),'fixture version 1') + self.assertEqual(Path(env['PATH'].split(os.pathsep)[0],'python3').resolve(),self.tool) + + def test_inherited_compiler_override_refused(self): + lane={'rustc':self.identity,'cargo':self.identity,'python':self.identity} + with self.assertRaisesRegex(ValueError,'override'): + self.m.lane_environment(lane,self.root/'aliases',{},dict(os.environ,RUSTC_WRAPPER='/wrong')) + + def test_failed_unittest_text_cannot_be_forged_as_ok(self): + r=self.run_command(self.command('print("Ran 2 tests\\nFAILED (failures=1)\\nOK")')) + self.assertEqual(r['status'],'Failed') + + def test_unknown_skip_text_cannot_pass(self): + r=self.run_command(self.command('print("Ran 2 tests\\nOK\\nSKIP required runner unavailable")')) + self.assertEqual(r['status'],'Failed') + + def test_native_override_refused(self): + lane={'rustc':self.identity,'cargo':self.identity,'python':self.identity} + with self.assertRaisesRegex(ValueError,'override'): + self.m.lane_environment(lane,self.root/'aliases',{}, {'PATH':os.environ.get('PATH',''),'CC':'/other/cc'}) + + def test_inventory_contains_all_distinct_commands(self): + commands=self.m.command_inventory() + ids=[c['id'] for c in commands] + self.assertEqual(len(ids),len(set(ids))) + self.assertTrue({'all-targets','all-python','compile-required-runners','ablation-plan','ablation-score','verifier-unit'} <= set(ids)) + + +if __name__ == '__main__': + unittest.main() diff --git a/scripts/verify_runtime_matrix.py b/scripts/verify_runtime_matrix.py new file mode 100644 index 0000000..95c6b79 --- /dev/null +++ b/scripts/verify_runtime_matrix.py @@ -0,0 +1,364 @@ +#!/usr/bin/env python3 +"""Run a frozen offline workload with explicitly identified runtimes. + +No installation, mutable runtime lookup, or activation is performed. Manifests +and receipts are local evidence, not signatures or independent acceptance. +""" +import argparse +import datetime +import hashlib +import json +import os +from pathlib import Path +import platform +import re +import signal +import subprocess +import time + +ROLES = {'stable', 'rust-upcoming', 'rust-previous', 'python-upcoming', 'python-previous'} +OVERRIDES = {'RUSTC', 'RUSTDOC', 'RUSTC_WRAPPER', 'RUSTC_WORKSPACE_WRAPPER', + 'RUSTFLAGS', 'CARGO_ENCODED_RUSTFLAGS', 'CARGO_BUILD_RUSTC', + 'CARGO_BUILD_RUSTC_WRAPPER', 'CARGO_BUILD_RUSTFLAGS', 'CARGO_TARGET_DIR', + 'CC', 'CXX', 'AR', 'LD', 'CFLAGS', 'CXXFLAGS', 'LDFLAGS', 'RUSTUP_TOOLCHAIN', + 'PYTHONHOME', 'PYTHONPATH', 'PYTHONSTARTUP', 'VIRTUAL_ENV', 'CONDA_PREFIX'} +PROFILE = {'CARGO_PROFILE_DEV_DEBUG': '0', 'CARGO_PROFILE_TEST_DEBUG': '0', + 'CARGO_INCREMENTAL': '0', 'KAIROTENSOR_REQUIRE_LINUX_CLOCK': '1', + 'RUSTUP_AUTO_INSTALL': '0', 'UV_PYTHON_DOWNLOADS': 'never', + 'PYTHONDONTWRITEBYTECODE': '1'} + + +def sha(path): + with Path(path).open('rb') as f: + return hashlib.file_digest(f, 'sha256').hexdigest() + + +def file_identity(path): + return {'path': str(Path(path).absolute()), 'sha256': sha(path)} + + +def save(path, value): + Path(path).write_text(json.dumps(value, indent=2, sort_keys=True) + '\n') + + +def check_file(identity): + p = Path(identity['path']) + if not p.is_absolute(): + raise ValueError('identity requires absolute path') + if sha(p) != identity['sha256']: + raise ValueError(f'digest mismatch: {p}') + return p + + +def check_identity(identity): + p = check_file(identity) + if not os.access(p, os.X_OK): + raise ValueError(f'not executable: {p}') + r = subprocess.run([str(p), *identity['version_args']], capture_output=True, + text=True, timeout=20, check=True) + version = r.stdout.strip() + if version != identity['version']: + raise ValueError(f'version mismatch: {p}: {version!r}') + return {**file_identity(p), 'resolved_path': str(p.resolve()), 'version': version} + + +def git(root, *args): + return subprocess.check_output(['git', '-C', str(root), *args], text=True).strip() + + +def check_source(root, source): + root = Path(root) + if git(root, 'rev-parse', 'HEAD') != source['commit']: + raise ValueError('source commit mismatch') + names = git(root, 'ls-files').splitlines() + if set(names) != set(source['files']): + raise ValueError('incomplete tracked source manifest') + for name, digest in source['files'].items(): + if sha(root / name) != digest: + raise ValueError(f'changed source: {name}') + if sha(root / 'Cargo.lock') != source['lock_sha256']: + raise ValueError('changed lock') + if git(root, 'status', '--porcelain', '--untracked-files=normal'): + raise ValueError('dirty source') + # Bind hashes to the commit too, not merely to whatever is on disk. + for name, digest in source['files'].items(): + data = subprocess.check_output(['git', '-C', str(root), 'show', f'{source["commit"]}:{name}']) + if hashlib.sha256(data).hexdigest() != digest: + raise ValueError(f'source manifest differs from commit: {name}') + + +def validate_roles(lanes): + if len(lanes) != 5 or {l['role'] for l in lanes} != ROLES: + raise ValueError('exactly five distinct roles required') + stable = next(l for l in lanes if l['role'] == 'stable') + for lane in lanes: + role = lane['role'] + fixed = 'python' if role.startswith('rust-') else 'rustc' + if role != 'stable' and lane[fixed]['sha256'] != stable[fixed]['sha256']: + raise ValueError(f'independent runtime variation required: {role}') + + +def lane_environment(lane, aliases, explicit, inherited=None): + env = dict(os.environ if inherited is None else inherited) + unexpected = sorted(k for k in OVERRIDES if env.get(k)) + unexpected += sorted(k for k in env if (k.startswith('CARGO_TARGET_') or k.startswith('CARGO_PROFILE_')) + and env[k] and (k not in PROFILE or env[k] != PROFILE[k])) + if unexpected: + raise ValueError(f'unbound inherited override: {unexpected}') + aliases = Path(aliases) + aliases.mkdir(exist_ok=False) + for alias, key in [('rustc', 'rustc'), ('cargo', 'cargo'), ('python3', 'python')]: + check_identity(lane[key]) + (aliases / alias).symlink_to(lane[key]['path']) + env.update(PROFILE) + env.update(explicit) + env['PATH'] = str(aliases) + os.pathsep + env.get('PATH', '') + env['RUSTC'] = lane['rustc']['path'] + return env + + +def command_inventory(): + """Source check.yml workload plus E1 demonstrations; placeholders are explicit.""" + commands = [] + def add(name, argv, kind='fixture', contains=(), artifacts=(), stdout=None): + c = dict(id=name, argv=argv, kind=kind, contains=list(contains), + artifacts=list(artifacts), timeout=600 if argv[0] == '{wrapper}' else 300) + if stdout: c['stdout_path'] = stdout + commands.append(c) + cargo = ['{wrapper}', '--budget-gib', '2'] + py = ['{python}'] + add('compile-required-runners', cargo + ['build', '--locked', '--offline', '--example', + 'recovery_policy_eval', '--example', 'decision_replay', '--example', 'policy_activation', + '--message-format=json'], contains=['compiler-artifact']) + add('all-targets', cargo + ['test', '--locked', '--all-targets'], 'rust') + examples = [('withdrawal','Rebuilt stock count'), ('deterministic_verdict','current_read'), + ('deadline_aware_recovery','synthetic_deadline_admission_fixture'), + ('runtime_recovery','claimed_hold_retained'), ('selective_revision','Published attempt'), + ('corrected_handoff','checklist_passed'), ('scoped_model_reuse','reuse_hits'), + ('token_cap_eval','boundary'), ('budgeted_recovery','deterministic_byte_token_fixture')] + for name, predicate in examples: + add(name,cargo+['run','--locked','--example',name],contains=[predicate]) + if name == 'scoped_model_reuse': + add('scoped-model-replay-invalid',cargo+['run','--locked','--example',name,'--','--replay-invalid'],contains=['invalid_output']) + add('reference',py+['-m','unittest','discover','-s','reference','-p','test_memory.py'],'python') + add('oracle',py+['reference/belief_oracle.py'],'python') + add('all-python',py+['-m','unittest','discover','-s','scripts','-p','test_*.py'],'python') + for name, pattern in [('ablation-unit','test_decision_ablation.py'),('verifier-unit','test_verify_runtime_matrix.py')]: + add(name,py+['-m','unittest','discover','-s','scripts','-p',pattern],'python') + for action in ['export-fixture','replay']: + add('decision-'+action,cargo+['run','--locked','--offline','--example','decision_replay','--',action,'{outputs}/decision-v1.json'],contains=['matched'],artifacts=['{outputs}/decision-v1.json']) + add('decision-audit',py+['scripts/audit_recovery_bundle.py','--decision-v1','{outputs}/decision-v1.json'],contains=['integrity_checked']) + add('token-cap-archives',py+['scripts/summarize_token_caps.py']+[ + f'docs/evaluations/token-caps-{n}-2026-11-13.json' for n in ['compatibility','integration','migration','package','unit']],contains=['Calibration']) + add('synthetic-comparison',py+['scripts/recovery_policy_eval.py','--fixture'],contains=['synthetic_policy_fixture']) + add('activation-fixture-input',py+['scripts/recovery_policy_eval.py','--activation-fixture'],contains=['policy-activation-fixture/1'],stdout='{outputs}/policy-activation-fixture.json',artifacts=['{outputs}/policy-activation-fixture.json']) + add('policy-activation',cargo+['run','--locked','--offline','--example','policy_activation','--','--fixture','{outputs}/policy-activation-fixture.json','--out','{outputs}/policy-activation'],contains=['baseline_manifest'],artifacts=['{outputs}/policy-activation']) + add('archived-v1-refusal',py+['scripts/recovery_policy_eval.py','--archived-v1-refusal'],contains=['refus'],stdout='{outputs}/archived-v1-promotion-refusal.json',artifacts=['{outputs}/archived-v1-promotion-refusal.json']) + add('legacy-collector',cargo+['run','--locked','--example','recovery_policy_eval','--','--trace','{outputs}/recovery-policy-trace.json'],contains=['complete_policy_trace'],artifacts=['{outputs}/recovery-policy-trace.json']) + add('legacy-score',py+['scripts/recovery_policy_eval.py','{outputs}/recovery-policy-trace.json'],contains=['synthetic_policy_fixture']) + for name, pattern in [('explicit-recovery-runner','test_recovery_policy_runner.py'),('explicit-audit-runner','test_audit_recovery_bundle.py')]: + add(name,py+['-m','unittest','discover','-s','scripts','-p',pattern],'python') + add('calibration',py+['scripts/recovery_calibration.py','--output','{outputs}/recovery-calibration.json'],contains=['complete'],artifacts=['{outputs}/recovery-calibration.json']) + add('timed-runtime',cargo+['run','--locked','--example','timed_recovery_runner','--','--out','{outputs}/timed-recovery-runner'],contains=['accounting_reconciled'],artifacts=['{outputs}/timed-recovery-runner']) + add('incident-prepare',py+['scripts/incident_routing.py','prepare','--out','{outputs}/incident-routing-calibration'],contains=['offline_preparation'],artifacts=['{outputs}/incident-routing-calibration']) + add('incident-bridge',cargo+['run','--locked','--example','incident_store_bridge','--','--requests','{outputs}/incident-store-requests.json'],artifacts=['{outputs}/incident-store-requests.json']) + add('incident-bridge-check',py+['scripts/check_incident_store_bridge.py','--requests','{outputs}/incident-store-requests.json'],contains=['offline_store_byte_parity']) + add('explicit-incident-bridge-tests',py+['-m','unittest','discover','-s','scripts','-p','test_incident_store_bridge.py'],'python') + add('incident-preflight',py+['scripts/incident_preflight.py','--out','{outputs}/incident-preflight-plan.json'],contains=['offline_calibration_plan'],artifacts=['{outputs}/incident-preflight-plan.json']) + add('incident-controller',py+['scripts/incident_calibration_controller.py','fixture','--out','{outputs}/incident-controller-fixture'],contains=['offline_calibration_diagnostic'],artifacts=['{outputs}/incident-controller-fixture']) + add('incident-transformers',py+['scripts/incident_transformers_controller.py','fixture','--out','{outputs}/incident-transformers-fixture'],contains=['offline_fake_dependencies'],artifacts=['{outputs}/incident-transformers-fixture']) + add('ablation-plan',py+['scripts/decision_ablation.py','plan','{bundle}'],contains=['decision-ablation-plan/1'],stdout='{outputs}/ablation-plan.json',artifacts=['{outputs}/ablation-plan.json']) + add('ablation-score',py+['scripts/decision_ablation.py','score','{outputs}/ablation-plan.json','{attestations}'],contains=['Comparable','Complete','fixture_bytes'],artifacts=[]) + return commands + + +def render(value, variables): + if isinstance(value,str): + for key, replacement in variables.items(): value=value.replace('{'+key+'}',str(replacement)) + return value + if isinstance(value,list): return [render(v,variables) for v in value] + if isinstance(value,dict): return {k:render(v,variables) for k,v in value.items()} + return value + + +def assess(command, record, raw): + counts = re.findall(r'test result: (\w+)\. (\d+) passed; (\d+) failed; (\d+) ignored; (\d+) measured; (\d+) filtered out', raw) + py = re.findall(r'^Ran (\d+) tests?\b', raw, re.M) + skips = sum(int(r[3])+int(r[5]) for r in counts) + skips += sum(map(int,re.findall(r'skipped=(\d+)',raw))) + failed = sum(int(r[2]) for r in counts) + failed += len(re.findall(r'^FAILED\b',raw,re.M)) + skips += len(re.findall(r'^.*\b(?:SKIP|skipped)\b(?![=])[^\n]*$',raw,re.M)) + tests = sum(int(r[1]) for r in counts) if command['kind']=='rust' else sum(map(int,py)) + reasons=[] + if record['exit'] != 0: reasons.append('nonzero exit') + if record['timeout']: reasons.append('timeout') + if skips: reasons.append('skipped/ignored/filtered checks') + if failed: reasons.append('failed tests') + if command['kind'] in ('rust','python') and tests==0: reasons.append('missing/zero tests') + if command['kind']=='python' and not re.search(r'^OK\s*$',raw,re.M): reasons.append('missing unittest success') + if any(s not in raw for s in command['contains']): reasons.append('missing output predicate') + return dict(status='Failed' if reasons else 'Pass', reasons=reasons, tests=tests, + skips=skips, rust_results=counts, python_counts=py) + + +def output_files(paths): + files=[] + for name in paths: + p=Path(name) + if not p.exists(): raise ValueError(f'missing artifact: {p}') + children=sorted(p.rglob('*')) if p.is_dir() else [p] + found=[c for c in children if c.is_file()] + if not found or any(c.stat().st_size==0 for c in found): raise ValueError(f'empty artifact: {p}') + files.extend(file_identity(c) for c in found) + return files + + +def execute_command(command, root, env, logs): + logs=Path(logs);logs.mkdir(parents=True,exist_ok=True) + stdout=Path(command.get('stdout_path',logs/(command['id']+'.stdout'))) + stderr=logs/(command['id']+'.stderr') + start=time.monotonic() + record={'id':command['id'],'argv':command['argv'],'cwd':str(root), + 'started_at':datetime.datetime.now(datetime.timezone.utc).isoformat(), + 'environment':{k:env[k] for k in [*PROFILE,'PATH','RUSTC','RUNNER_TEMP','KAIROTENSOR_RECOVERY_RUNNER','KAIROTENSOR_DECISION_REPLAY'] if k in env}, + 'timeout':False,'exit':None} + with stdout.open('wb') as out,stderr.open('wb') as err: + try: + p=subprocess.Popen(command['argv'],cwd=root,env=env,stdout=out,stderr=err,start_new_session=True) + try: record['exit']=p.wait(timeout=command['timeout']) + except subprocess.TimeoutExpired: + os.killpg(p.pid,signal.SIGKILL);record['exit']=p.wait();record['timeout']=True + except OSError as e: err.write(str(e).encode()) + record.update(elapsed_seconds=time.monotonic()-start,stdout=file_identity(stdout),stderr=file_identity(stderr)) + raw=stdout.read_text(errors='replace')+'\n'+stderr.read_text(errors='replace') + record.update(assess(command,record,raw)) + record['storage_reservations']=re.findall(r'Storage reservation: ([a-f0-9-]+)',raw) + try: record['artifacts']=output_files(command['artifacts']) + except ValueError as e: record.update(status='Failed');record['reasons'].append(str(e));record['artifacts']=[] + return record + + +def check_command_receipt(command, record): + if not record or record.get('id') != command['id'] or record.get('argv') != command['argv']: + raise ValueError('missing/mismatched command receipt') + for key in ['stdout','stderr']: check_file(record[key]) + raw=Path(record['stdout']['path']).read_text(errors='replace')+'\n'+Path(record['stderr']['path']).read_text(errors='replace') + observed=assess(command,record,raw) + if observed['status']!='Pass' or any(record.get(k)!=v for k,v in observed.items()): + raise ValueError('receipt does not prove passing checks') + actual=output_files(command['artifacts']) + if actual!=record['artifacts']: raise ValueError('changed/missing receipt artifact') + + +def check_manifest(m): + if m.get('schema')!='kairotensor-runtime-matrix/1': raise ValueError('unknown manifest schema') + validate_roles(m['lanes']) + if m['commands'] != command_inventory(): raise ValueError('incomplete or changed required command inventory') + for field in ['wrapper','storage_guard','cargo_recipe','bundle','attestations']: + check_file(m[field]) + if not m['native_files']: raise ValueError('missing native identities') + for identity in m['native_files']: check_file(identity) + for lane in m['lanes']: + if not Path(lane['checkout']).is_absolute(): raise ValueError('absolute checkout required') + check_source(lane['checkout'],m['source']) + if not lane.get('provenance'): raise ValueError('missing runtime archive/closure provenance') + for identity in lane['provenance']: check_file(identity) + + +def run_lane(m,lane,out): + out=Path(out);out.mkdir(parents=True,exist_ok=False) + root=Path(lane['checkout']); outputs=out/'outputs';outputs.mkdir() + result={'role':lane['role'],'source':m['source'],'status':'Unexecuted','commands':[], + 'host':{'platform':platform.platform(),'machine':platform.machine()},'new_model_calls':0} + save(out/'receipt.json',result) + try: + check_manifest(m) + tools={k:check_identity(lane[k]) for k in ['rustc','cargo','python']} + result['tools']=tools + env=lane_environment(lane,out/'bin',{'RUNNER_TEMP':str(outputs)}) + # This actual subprocess must find the selected private python3 alias. + probe=subprocess.run([lane['python']['path'],'-c', + 'import sys,subprocess,sqlite3,ssl,ctypes,zlib,bz2,lzma;print(sys.executable);subprocess.run(["python3","--version"],check=True)'], + env=env,capture_output=True,text=True,timeout=20,check=True) + result['python_child_probe']={'stdout':probe.stdout,'stderr':probe.stderr} + variables={'wrapper':m['wrapper']['path'],'python':lane['python']['path'], + 'outputs':outputs,'bundle':m['bundle']['path'],'attestations':m['attestations']['path']} + commands=render(m['commands'],variables);result['resolved_commands']=commands + for command in commands: + check_source(root,m['source']) + for identity in tools.values(): check_file(identity) + result['status']='Failed' # started work cannot become Unexecuted + record=execute_command(command,root,env,out/'logs') + result['commands'].append(record);save(out/'receipt.json',result) + print(lane['role'],command['id'],record['status'],record['exit'],flush=True) + if record['status']!='Pass': break + if command['id']=='compile-required-runners': + runners={} + for line in Path(record['stdout']['path']).read_text().splitlines(): + try: row=json.loads(line) + except ValueError: continue + if row.get('reason')=='compiler-artifact' and row.get('executable'): + runners[row['target']['name']]=file_identity(row['executable']) + for name,key in [('recovery_policy_eval','KAIROTENSOR_RECOVERY_RUNNER'),('decision_replay','KAIROTENSOR_DECISION_REPLAY')]: + env[key]=runners[name]['path'] + if 'policy_activation' not in runners: raise ValueError('missing policy runner') + result['runners']=runners + check_manifest(m);result['source_verified_after']=True + for identity in tools.values(): check_file(identity) + for identity in result.get('runners',{}).values(): check_file(identity) + if len(result['commands'])==len(commands) and all(r['status']=='Pass' for r in result['commands']): + for c,r in zip(commands,result['commands']): check_command_receipt(c,r) + result['status']='Pass' + except (OSError,ValueError,KeyError,subprocess.SubprocessError) as error: + result['error']=str(error) + save(out/'receipt.json',result) + return result + + +def verify_receipt(m,lane,receipt): + check_manifest(m) + if receipt.get('role')!=lane['role'] or receipt.get('source')!=m['source'] or receipt.get('status')!='Pass': + raise ValueError('wrong source/role/status in receipt') + if not receipt.get('source_verified_after'): raise ValueError('missing source verification') + for k in ['rustc','cargo','python']: + if receipt['tools'][k]!=check_identity(lane[k]): raise ValueError('runtime receipt mismatch') + if set(receipt.get('runners',{}))!={'recovery_policy_eval','decision_replay','policy_activation'}: + raise ValueError('missing runner receipts') + for identity in receipt['runners'].values(): check_file(identity) + commands=receipt['resolved_commands'] + variables={'wrapper':m['wrapper']['path'],'python':lane['python']['path'], + 'outputs':str(Path(receipt['commands'][0]['stdout']['path']).parent.parent/'outputs'), + 'bundle':m['bundle']['path'],'attestations':m['attestations']['path']} + if commands!=render(m['commands'],variables) or len(receipt['commands'])!=len(commands): + raise ValueError('missing or changed command receipts') + for c,r in zip(commands,receipt['commands']): check_command_receipt(c,r) + + +def main(): + p=argparse.ArgumentParser(description=__doc__) + p.add_argument('--manifest',type=Path,required=True) + p.add_argument('--output',type=Path) + p.add_argument('--lane',choices=sorted(ROLES)) + p.add_argument('--verify-receipt',type=Path) + args=p.parse_args();m=json.loads(args.manifest.read_text());check_manifest(m) + if args.verify_receipt: + if not args.lane: p.error('--verify-receipt requires --lane') + lane=next(l for l in m['lanes'] if l['role']==args.lane) + verify_receipt(m,lane,json.loads(args.verify_receipt.read_text()));return 0 + if not args.output: p.error('--output required for execution') + results=[] + for lane in m['lanes']: + if args.lane and lane['role']!=args.lane: continue + if lane['role'].startswith('python-'): + stable=args.output/'stable'/'receipt.json' + if not stable.is_file(): raise ValueError('stable qualification required before dependent Python lane') + verify_receipt(m,next(l for l in m['lanes'] if l['role']=='stable'),json.loads(stable.read_text())) + results.append(run_lane(m,lane,args.output/lane['role'])) + return 0 if results and all(r['status']=='Pass' for r in results) else 1 + + +if __name__=='__main__': + try: raise SystemExit(main()) + except (ValueError,OSError,KeyError,subprocess.SubprocessError) as e: + raise SystemExit(f'Runtime matrix refused: {e}') From 26045ce65a3d1b495f50635da28d961ef71058bb Mon Sep 17 00:00:00 2001 From: Alejandro Castellanos <5405628+k2jac9@users.noreply.github.com> Date: Sun, 15 Nov 2026 20:37:22 -0500 Subject: [PATCH 31/51] Bind runtime receipts to the complete qualification manifest --- scripts/test_verify_runtime_matrix.py | 31 +++++++++++++++++++++++++++ scripts/verify_runtime_matrix.py | 5 ++++- 2 files changed, 35 insertions(+), 1 deletion(-) diff --git a/scripts/test_verify_runtime_matrix.py b/scripts/test_verify_runtime_matrix.py index 511daf4..bebd717 100644 --- a/scripts/test_verify_runtime_matrix.py +++ b/scripts/test_verify_runtime_matrix.py @@ -164,6 +164,37 @@ def test_native_override_refused(self): with self.assertRaisesRegex(ValueError,'override'): self.m.lane_environment(lane,self.root/'aliases',{}, {'PATH':os.environ.get('PATH',''),'CC':'/other/cc'}) + def manifest(self): + repo=self.root/'repo';repo.mkdir() + (repo/'Cargo.lock').write_text('fixture lock') + subprocess.run(['git','init','-q',str(repo)],check=True) + subprocess.run(['git','-C',str(repo),'add','.'],check=True) + subprocess.run(['git','-C',str(repo),'-c','user.name=Fixture','-c','user.email=fixture@invalid','commit','-qm','fixture'],check=True) + commit=subprocess.check_output(['git','-C',str(repo),'rev-parse','HEAD'],text=True).strip() + identity={'path':str(self.tool),'sha256':self.sha(self.tool)} + lanes=[{'role':role,'checkout':str(repo),'rustc':self.identity,'cargo':self.identity, + 'python':self.identity,'provenance':[identity]} + for role in ['stable','rust-upcoming','rust-previous','python-upcoming','python-previous']] + return {'schema':'kairotensor-runtime-matrix/1','lanes':lanes, + 'source':{'commit':commit,'files':{'Cargo.lock':self.sha(repo/'Cargo.lock')},'lock_sha256':self.sha(repo/'Cargo.lock')}, + 'commands':self.m.command_inventory(),'native_files':[identity], + **{k:identity for k in ['wrapper','storage_guard','cargo_recipe','bundle','attestations']}} + + def test_complete_manifest_rejects_omitted_command(self): + m=self.manifest();m['commands'].pop() + with self.assertRaisesRegex(ValueError,'inventory'): self.m.check_manifest(m) + + def test_unavailable_lane_emits_no_pass_or_commands(self): + m=self.manifest();lane=copy.deepcopy(m['lanes'][0]);lane['cargo']['path']=str(self.root/'unavailable') + r=self.m.run_lane(m,lane,self.root/'result') + self.assertEqual(r['status'],'Unexecuted');self.assertEqual(r['commands'],[]) + self.assertTrue((self.root/'result'/'receipt.json').is_file()) + + def test_receipt_binds_whole_manifest(self): + m=self.manifest();receipt={'role':'stable','source':m['source'],'status':'Pass','manifest_sha256':'0'*64} + with self.assertRaisesRegex(ValueError,'manifest'): + self.m.verify_receipt(m,m['lanes'][0],receipt) + def test_inventory_contains_all_distinct_commands(self): commands=self.m.command_inventory() ids=[c['id'] for c in commands] diff --git a/scripts/verify_runtime_matrix.py b/scripts/verify_runtime_matrix.py index 95c6b79..14aab01 100644 --- a/scripts/verify_runtime_matrix.py +++ b/scripts/verify_runtime_matrix.py @@ -270,7 +270,8 @@ def run_lane(m,lane,out): out=Path(out);out.mkdir(parents=True,exist_ok=False) root=Path(lane['checkout']); outputs=out/'outputs';outputs.mkdir() result={'role':lane['role'],'source':m['source'],'status':'Unexecuted','commands':[], - 'host':{'platform':platform.platform(),'machine':platform.machine()},'new_model_calls':0} + 'host':{'platform':platform.platform(),'machine':platform.machine()},'new_model_calls':0, + 'manifest_sha256':hashlib.sha256(json.dumps(m,sort_keys=True).encode()).hexdigest()} save(out/'receipt.json',result) try: check_manifest(m) @@ -318,6 +319,8 @@ def run_lane(m,lane,out): def verify_receipt(m,lane,receipt): check_manifest(m) + if receipt.get('manifest_sha256')!=hashlib.sha256(json.dumps(m,sort_keys=True).encode()).hexdigest(): + raise ValueError('receipt manifest identity mismatch') if receipt.get('role')!=lane['role'] or receipt.get('source')!=m['source'] or receipt.get('status')!='Pass': raise ValueError('wrong source/role/status in receipt') if not receipt.get('source_verified_after'): raise ValueError('missing source verification') From 353ff2f2673b59df8402bf8dd3e10e4735a40470 Mon Sep 17 00:00:00 2001 From: Alejandro Castellanos <5405628+k2jac9@users.noreply.github.com> Date: Sun, 15 Nov 2026 20:44:42 -0500 Subject: [PATCH 32/51] Retain empty log hashes alongside nonempty qualification results --- scripts/test_verify_runtime_matrix.py | 15 +++++++++++++++ scripts/verify_runtime_matrix.py | 2 +- 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/scripts/test_verify_runtime_matrix.py b/scripts/test_verify_runtime_matrix.py index bebd717..ee9fee1 100644 --- a/scripts/test_verify_runtime_matrix.py +++ b/scripts/test_verify_runtime_matrix.py @@ -195,6 +195,21 @@ def test_receipt_binds_whole_manifest(self): with self.assertRaisesRegex(ValueError,'manifest'): self.m.verify_receipt(m,m['lanes'][0],receipt) + def test_directory_artifact_retains_empty_log_with_nonempty_result(self): + d=self.root/'outputs';d.mkdir();(d/'result.json').write_text('{"status":"complete"}');(d/'stderr.log').touch() + records=self.m.output_files([str(d)]) + self.assertEqual({r['path'] for r in records},{str(d/'result.json'),str(d/'stderr.log')}) + self.assertEqual(next(r['sha256'] for r in records if r['path'].endswith('stderr.log')), + 'e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855') + + def test_all_empty_directory_artifact_refused(self): + d=self.root/'outputs';d.mkdir();(d/'stderr.log').touch() + with self.assertRaisesRegex(ValueError,'empty'): self.m.output_files([str(d)]) + + def test_explicit_empty_result_file_refused(self): + p=self.root/'result.json';p.touch() + with self.assertRaisesRegex(ValueError,'empty'): self.m.output_files([str(p)]) + def test_inventory_contains_all_distinct_commands(self): commands=self.m.command_inventory() ids=[c['id'] for c in commands] diff --git a/scripts/verify_runtime_matrix.py b/scripts/verify_runtime_matrix.py index 14aab01..39fb373 100644 --- a/scripts/verify_runtime_matrix.py +++ b/scripts/verify_runtime_matrix.py @@ -209,7 +209,7 @@ def output_files(paths): if not p.exists(): raise ValueError(f'missing artifact: {p}') children=sorted(p.rglob('*')) if p.is_dir() else [p] found=[c for c in children if c.is_file()] - if not found or any(c.stat().st_size==0 for c in found): raise ValueError(f'empty artifact: {p}') + if not found or not any(c.stat().st_size>0 for c in found): raise ValueError(f'empty artifact: {p}') files.extend(file_identity(c) for c in found) return files From ab8e162d335eabd11006b4a9d8bd9f3482dbdbb6 Mon Sep 17 00:00:00 2001 From: Alejandro Castellanos <5405628+k2jac9@users.noreply.github.com> Date: Sun, 15 Nov 2026 21:22:01 -0500 Subject: [PATCH 33/51] Normalize runtime receipt counts across JSON persistence --- scripts/test_verify_runtime_matrix.py | 16 ++++++++++++++++ scripts/verify_runtime_matrix.py | 2 +- 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/scripts/test_verify_runtime_matrix.py b/scripts/test_verify_runtime_matrix.py index ee9fee1..f3d067c 100644 --- a/scripts/test_verify_runtime_matrix.py +++ b/scripts/test_verify_runtime_matrix.py @@ -210,6 +210,22 @@ def test_explicit_empty_result_file_refused(self): p=self.root/'result.json';p.touch() with self.assertRaisesRegex(ValueError,'empty'): self.m.output_files([str(p)]) + def test_rust_receipt_survives_json_roundtrip(self): + c=self.command('print("test result: ok. 2 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out")','rust') + r=json.loads(json.dumps(self.run_command(c))) + self.m.check_command_receipt(c,r) + self.assertEqual(r['tests'],2) + + def test_json_rust_receipt_tampered_count_rejected(self): + c=self.command('print("test result: ok. 2 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out")','rust') + r=json.loads(json.dumps(self.run_command(c)));r['tests']=200 + with self.assertRaises(ValueError):self.m.check_command_receipt(c,r) + + def test_json_rust_receipt_forged_pass_on_ignore_rejected(self): + c=self.command('print("test result: ok. 2 passed; 0 failed; 1 ignored; 0 measured; 0 filtered out")','rust') + r=json.loads(json.dumps(self.run_command(c)));r['status']='Pass';r['skips']=0 + with self.assertRaises(ValueError):self.m.check_command_receipt(c,r) + def test_inventory_contains_all_distinct_commands(self): commands=self.m.command_inventory() ids=[c['id'] for c in commands] diff --git a/scripts/verify_runtime_matrix.py b/scripts/verify_runtime_matrix.py index 39fb373..fe24a78 100644 --- a/scripts/verify_runtime_matrix.py +++ b/scripts/verify_runtime_matrix.py @@ -199,7 +199,7 @@ def assess(command, record, raw): if command['kind']=='python' and not re.search(r'^OK\s*$',raw,re.M): reasons.append('missing unittest success') if any(s not in raw for s in command['contains']): reasons.append('missing output predicate') return dict(status='Failed' if reasons else 'Pass', reasons=reasons, tests=tests, - skips=skips, rust_results=counts, python_counts=py) + skips=skips, rust_results=[list(row) for row in counts], python_counts=py) def output_files(paths): From dd52954fff73484c7fe7dafc474ac706af69c59d Mon Sep 17 00:00:00 2001 From: Alejandro Castellanos <5405628+k2jac9@users.noreply.github.com> Date: Sun, 15 Nov 2026 21:33:35 -0500 Subject: [PATCH 34/51] Bind persisted runtime command and child interpreter provenance --- scripts/test_verify_runtime_matrix.py | 68 +++++++++++++++++- scripts/verify_runtime_matrix.py | 99 ++++++++++++++++++++++----- 2 files changed, 149 insertions(+), 18 deletions(-) diff --git a/scripts/test_verify_runtime_matrix.py b/scripts/test_verify_runtime_matrix.py index f3d067c..6cd225f 100644 --- a/scripts/test_verify_runtime_matrix.py +++ b/scripts/test_verify_runtime_matrix.py @@ -175,7 +175,7 @@ def manifest(self): lanes=[{'role':role,'checkout':str(repo),'rustc':self.identity,'cargo':self.identity, 'python':self.identity,'provenance':[identity]} for role in ['stable','rust-upcoming','rust-previous','python-upcoming','python-previous']] - return {'schema':'kairotensor-runtime-matrix/1','lanes':lanes, + return {'schema':'kairotensor-runtime-matrix/1','host_path':os.environ['PATH'],'lanes':lanes, 'source':{'commit':commit,'files':{'Cargo.lock':self.sha(repo/'Cargo.lock')},'lock_sha256':self.sha(repo/'Cargo.lock')}, 'commands':self.m.command_inventory(),'native_files':[identity], **{k:identity for k in ['wrapper','storage_guard','cargo_recipe','bundle','attestations']}} @@ -226,6 +226,72 @@ def test_json_rust_receipt_forged_pass_on_ignore_rejected(self): r=json.loads(json.dumps(self.run_command(c)));r['status']='Pass';r['skips']=0 with self.assertRaises(ValueError):self.m.check_command_receipt(c,r) + def context_fixture(self): + lane={'checkout':str(self.root),'rustc':self.identity,'cargo':self.identity,'python':self.identity} + manifest={'host_path':'/usr/bin:/bin'} + env={'CARGO_PROFILE_DEV_DEBUG':'0','CARGO_PROFILE_TEST_DEBUG':'0','CARGO_INCREMENTAL':'0', + 'KAIROTENSOR_REQUIRE_LINUX_CLOCK':'1','RUSTUP_AUTO_INSTALL':'0','UV_PYTHON_DOWNLOADS':'never', + 'PYTHONDONTWRITEBYTECODE':'1','PATH':str(self.root/'bin')+':/usr/bin:/bin', + 'RUSTC':str(self.tool),'RUNNER_TEMP':str(self.root/'outputs')} + return manifest,lane,{'cwd':str(self.root),'environment':env} + + def test_recorded_context_roundtrip(self): + m,lane,r=self.context_fixture() + self.m.check_command_context(m,lane,self.root,json.loads(json.dumps(r))) + + def test_contradictory_cwd_rejected(self): + m,lane,r=self.context_fixture();r['cwd']='/another/checkout' + with self.assertRaisesRegex(ValueError,'context'):self.m.check_command_context(m,lane,self.root,r) + + def test_contradictory_runtime_profile_or_path_rejected(self): + for key,value in [('RUSTC','/wrong/rustc'),('CARGO_PROFILE_DEV_DEBUG','2'),('PATH','/usr/bin'),('RUNNER_TEMP','/wrong/output')]: + with self.subTest(key=key): + m,lane,r=self.context_fixture();r['environment'][key]=value + with self.assertRaisesRegex(ValueError,'context'):self.m.check_command_context(m,lane,self.root,r) + + def test_missing_or_wrong_runner_environment_rejected(self): + m,lane,r=self.context_fixture();runners={k:self.identity for k in ['recovery_policy_eval','decision_replay','policy_activation']} + with self.assertRaisesRegex(ValueError,'context'):self.m.check_command_context(m,lane,self.root,r,runners) + r['environment'].update(KAIROTENSOR_RECOVERY_RUNNER=str(self.tool),KAIROTENSOR_DECISION_REPLAY='/wrong/runner') + with self.assertRaisesRegex(ValueError,'context'):self.m.check_command_context(m,lane,self.root,r,runners) + + def test_alias_retargeting_rejected(self): + lane={'rustc':self.identity,'cargo':self.identity,'python':self.identity} + aliases=self.root/'aliases';aliases.mkdir() + for name in ['rustc','cargo','python3']:(aliases/name).symlink_to(self.tool) + self.m.check_aliases(lane,aliases) + (aliases/'python3').unlink();(aliases/'python3').symlink_to(sys.executable) + with self.assertRaisesRegex(ValueError,'alias'):self.m.check_aliases(lane,aliases) + + def test_host_path_must_have_nonempty_absolute_components(self): + m=self.manifest() + for path in ['', '/bin::/usr/bin','relative:/bin']: + m['host_path']=path + with self.assertRaisesRegex(ValueError,'host_path'):self.m.check_manifest(m) + + def test_structured_actual_python_parent_child_probe_roundtrip(self): + identity={'path':sys.executable,'sha256':self.sha(sys.executable),'version_args':['--version'], + 'version':subprocess.check_output([sys.executable,'--version'],text=True).strip()} + lane={'python':identity,'rustc':identity,'cargo':identity} + aliases=self.root/'aliases';env=self.m.lane_environment(lane,aliases,{}, {'PATH':os.environ['PATH']}) + probe=json.loads(json.dumps(self.m.probe_python(lane,env))) + self.m.check_python_probe(lane,aliases,probe) + self.assertEqual(probe['observed']['parent']['realpath'],str(Path(sys.executable).resolve())) + self.assertEqual(probe['observed']['child']['realpath'],str(Path(sys.executable).resolve())) + probe['observed']['child']['realpath']='/wrong/python' + with self.assertRaisesRegex(ValueError,'probe'):self.m.check_python_probe(lane,aliases,probe) + + def test_compiler_artifact_runner_paths_are_authoritative(self): + log=self.root/'compiler.stdout' + log.write_text('\n'.join(json.dumps({'reason':'compiler-artifact','target':{'name':name},'executable':str(self.tool)}) + for name in ['recovery_policy_eval','decision_replay','policy_activation'])) + r={'stdout':{'path':str(log),'sha256':self.sha(log)}} + expected={name:{'path':str(self.tool),'sha256':self.sha(self.tool)} + for name in ['recovery_policy_eval','decision_replay','policy_activation']} + self.assertEqual(self.m.compiled_runners(r),expected) + log.write_text('{}') + with self.assertRaises(ValueError):self.m.compiled_runners(r) + def test_inventory_contains_all_distinct_commands(self): commands=self.m.command_inventory() ids=[c['id'] for c in commands] diff --git a/scripts/verify_runtime_matrix.py b/scripts/verify_runtime_matrix.py index fe24a78..9b90847 100644 --- a/scripts/verify_runtime_matrix.py +++ b/scripts/verify_runtime_matrix.py @@ -117,6 +117,69 @@ def lane_environment(lane, aliases, explicit, inherited=None): return env + +def check_aliases(lane, aliases): + for alias, key in [('rustc','rustc'),('cargo','cargo'),('python3','python')]: + path=Path(aliases)/alias + if not path.is_symlink() or path.resolve()!=Path(lane[key]['path']).resolve(): + raise ValueError(f'private alias mismatch: {alias}') + check_file({'path':str(path),'sha256':lane[key]['sha256']}) + + +def check_command_context(m,lane,out,record,runners=None): + expected={**PROFILE,'PATH':str(Path(out)/'bin')+os.pathsep+m['host_path'], + 'RUSTC':lane['rustc']['path'],'RUNNER_TEMP':str(Path(out)/'outputs')} + if runners is not None: + expected.update(KAIROTENSOR_RECOVERY_RUNNER=runners['recovery_policy_eval']['path'], + KAIROTENSOR_DECISION_REPLAY=runners['decision_replay']['path']) + if record.get('cwd')!=lane['checkout'] or record.get('environment')!=expected: + raise ValueError('command context contradicts declared checkout/runtime environment') + + +PYTHON_PROBE_CODE = """import sys,os,json,subprocess,sqlite3,ssl,ctypes,zlib,bz2,lzma +child=json.loads(subprocess.check_output(['python3','-c', +'import sys,os,json;print(json.dumps({"realpath":os.path.realpath(sys.executable),"version":"Python "+sys.version.split()[0]}))'],text=True)) +print(json.dumps({'parent':{'realpath':os.path.realpath(sys.executable),'version':'Python '+sys.version.split()[0]},'child':child})) +""" + + +def probe_python(lane,env): + argv=[lane['python']['path'],'-c',PYTHON_PROBE_CODE] + r=subprocess.run(argv,env=env,capture_output=True,text=True,timeout=20) + probe={'argv':argv,'exit':r.returncode,'stdout':r.stdout,'stderr':r.stderr} + try: probe['observed']=json.loads(r.stdout) + except ValueError: probe['observed']=None + return probe + + +def check_python_probe(lane,aliases,probe): + check_aliases(lane,aliases) + expected={'realpath':str(Path(lane['python']['path']).resolve()),'version':lane['python']['version']} + try: observed=json.loads(probe['stdout']) + except (KeyError,ValueError,TypeError):raise ValueError('invalid Python probe output') from None + if (probe.get('exit')!=0 or probe.get('argv')!=[lane['python']['path'],'-c',PYTHON_PROBE_CODE] + or observed!={'parent':expected,'child':expected} or probe.get('observed')!=observed + or not isinstance(probe.get('stderr'),str)): + raise ValueError('Python probe contradicts selected parent/child interpreter') + + +def compiled_runners(record): + check_file(record['stdout']) + required={'recovery_policy_eval','decision_replay','policy_activation'} + runners={} + for line in Path(record['stdout']['path']).read_text().splitlines(): + try: row=json.loads(line) + except ValueError: continue + if row.get('reason')!='compiler-artifact' or not row.get('executable'):continue + name=row['target']['name'] + if name not in required:continue + if not Path(row['executable']).is_absolute():raise ValueError('runner path must be absolute') + identity=file_identity(row['executable']) + if name in runners and runners[name]!=identity:raise ValueError('conflicting compiled runner paths') + runners[name]=identity + if set(runners)!=required:raise ValueError('missing compiled runner artifacts') + return runners + def command_inventory(): """Source check.yml workload plus E1 demonstrations; placeholders are explicit.""" commands = [] @@ -253,6 +316,9 @@ def check_command_receipt(command, record): def check_manifest(m): if m.get('schema')!='kairotensor-runtime-matrix/1': raise ValueError('unknown manifest schema') + host_path=m.get('host_path','') + if not isinstance(host_path,str) or any(not p or not Path(p).is_absolute() for p in host_path.split(os.pathsep)): + raise ValueError('host_path must contain nonempty absolute directories') validate_roles(m['lanes']) if m['commands'] != command_inventory(): raise ValueError('incomplete or changed required command inventory') for field in ['wrapper','storage_guard','cargo_recipe','bundle','attestations']: @@ -277,12 +343,10 @@ def run_lane(m,lane,out): check_manifest(m) tools={k:check_identity(lane[k]) for k in ['rustc','cargo','python']} result['tools']=tools - env=lane_environment(lane,out/'bin',{'RUNNER_TEMP':str(outputs)}) - # This actual subprocess must find the selected private python3 alias. - probe=subprocess.run([lane['python']['path'],'-c', - 'import sys,subprocess,sqlite3,ssl,ctypes,zlib,bz2,lzma;print(sys.executable);subprocess.run(["python3","--version"],check=True)'], - env=env,capture_output=True,text=True,timeout=20,check=True) - result['python_child_probe']={'stdout':probe.stdout,'stderr':probe.stderr} + env=lane_environment(lane,out/'bin',{'RUNNER_TEMP':str(outputs)},dict(os.environ,PATH=m['host_path'])) + result['python_child_probe']=probe_python(lane,env) + save(out/'receipt.json',result) + check_python_probe(lane,out/'bin',result['python_child_probe']) variables={'wrapper':m['wrapper']['path'],'python':lane['python']['path'], 'outputs':outputs,'bundle':m['bundle']['path'],'attestations':m['attestations']['path']} commands=render(m['commands'],variables);result['resolved_commands']=commands @@ -295,15 +359,9 @@ def run_lane(m,lane,out): print(lane['role'],command['id'],record['status'],record['exit'],flush=True) if record['status']!='Pass': break if command['id']=='compile-required-runners': - runners={} - for line in Path(record['stdout']['path']).read_text().splitlines(): - try: row=json.loads(line) - except ValueError: continue - if row.get('reason')=='compiler-artifact' and row.get('executable'): - runners[row['target']['name']]=file_identity(row['executable']) + runners=compiled_runners(record) for name,key in [('recovery_policy_eval','KAIROTENSOR_RECOVERY_RUNNER'),('decision_replay','KAIROTENSOR_DECISION_REPLAY')]: env[key]=runners[name]['path'] - if 'policy_activation' not in runners: raise ValueError('missing policy runner') result['runners']=runners check_manifest(m);result['source_verified_after']=True for identity in tools.values(): check_file(identity) @@ -311,8 +369,11 @@ def run_lane(m,lane,out): if len(result['commands'])==len(commands) and all(r['status']=='Pass' for r in result['commands']): for c,r in zip(commands,result['commands']): check_command_receipt(c,r) result['status']='Pass' + save(out/'receipt.json',result) + verify_receipt(m,lane,json.loads((out/'receipt.json').read_text())) except (OSError,ValueError,KeyError,subprocess.SubprocessError) as error: result['error']=str(error) + if result['commands']:result['status']='Failed' save(out/'receipt.json',result) return result @@ -326,16 +387,20 @@ def verify_receipt(m,lane,receipt): if not receipt.get('source_verified_after'): raise ValueError('missing source verification') for k in ['rustc','cargo','python']: if receipt['tools'][k]!=check_identity(lane[k]): raise ValueError('runtime receipt mismatch') - if set(receipt.get('runners',{}))!={'recovery_policy_eval','decision_replay','policy_activation'}: - raise ValueError('missing runner receipts') - for identity in receipt['runners'].values(): check_file(identity) + if not receipt.get('commands'):raise ValueError('missing command receipts') + runners=compiled_runners(receipt['commands'][0]) + if receipt.get('runners')!=runners:raise ValueError('runner receipt contradicts compiler artifacts') + out=Path(receipt['commands'][0]['stdout']['path']).parent.parent + check_python_probe(lane,out/'bin',receipt.get('python_child_probe',{})) commands=receipt['resolved_commands'] variables={'wrapper':m['wrapper']['path'],'python':lane['python']['path'], 'outputs':str(Path(receipt['commands'][0]['stdout']['path']).parent.parent/'outputs'), 'bundle':m['bundle']['path'],'attestations':m['attestations']['path']} if commands!=render(m['commands'],variables) or len(receipt['commands'])!=len(commands): raise ValueError('missing or changed command receipts') - for c,r in zip(commands,receipt['commands']): check_command_receipt(c,r) + for index,(c,r) in enumerate(zip(commands,receipt['commands'])): + check_command_context(m,lane,out,r,None if index==0 else runners) + check_command_receipt(c,r) def main(): From fce095d1c68abb29d1ced319e99004f8d3cb0f34 Mon Sep 17 00:00:00 2001 From: Alejandro Castellanos <5405628+k2jac9@users.noreply.github.com> Date: Sun, 15 Nov 2026 21:48:36 -0500 Subject: [PATCH 35/51] docs: record isolated runtime qualification and rollback evidence --- docs/runtime-adoption.md | 154 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 154 insertions(+) diff --git a/docs/runtime-adoption.md b/docs/runtime-adoption.md index c582f1a..d934af0 100644 --- a/docs/runtime-adoption.md +++ b/docs/runtime-adoption.md @@ -136,3 +136,157 @@ rollback exercise, observation window, retirement date and owner acceptance are not established. The project technical owner owns these adoption gaps, with independent review required before promotion. Adding `--locked` does not close them or retire historical pins or rollback assets. + +## G1 isolated qualification — 2026-11-16 + +The preceding A0 record is historical and remains unchanged. This section binds +new local offline checks to their actual source; it does not activate a runtime +selection, claim hosted CI, qualify model workloads or establish a live service. + +Accepted entry/rollback source S0 is +`794f64e5f9702987cd2502b86776b28f8a8d97c5`. The unchanged current lock SHA-256 is +`72282fc2776662e44b2e7b4003c2c582211296d663fdaecbd5b768dc09bc63aa`. +The initial verifier source S1 was +`26045ce65a3d1b495f50635da28d961ef71058bb`. Its previous-Rust attempt remains +Failed: all application commands it launched exited zero, but the new verifier +incorrectly rejected an empty successful controller stderr log. Root authorized +a narrow correction: recursively hash every artifact file, permit empty logs +beside at least one nonempty directory result, and continue rejecting missing or +all-empty artifacts and explicitly required empty result files. Regression tests +were added before the correction (`353ff2f2673b59df8402bf8dd3e10e4735a40470`). +Subsequent retained revisions corrected JSON tuple/list receipt persistence +(`ab8e162d335eabd11006b4a9d8bd9f3482dbdbb6`) and rejected contradictory +recorded cwd, runtime environment, runner and child-Python provenance. The final +common executable source is `dd52954fff73484c7fe7dafc474ac706af69c59d`, with 42 +verifier regressions. Earlier receipts remain historical; the documentation-only +successor does not relabel them. + +The packet is +`/home/neo/Projects/kairotensor-artifacts/reliability-implementation-20261115/task-g1/`. +`task-1-report.md` inventories every executed command and retained failure. +`required-command-inventory.json`, its independent root review, and the final +source manifest bind the unchanged 40-command workload. This includes all of +`check.yml`, compiled recovery/decision/policy runners, all-target locked Rust, +reference/oracle suites, all Python suites with explicit runner paths, offline +examples/audits/activation fixtures, E1 planning/recorded scoring and the new +verifier unit suite. Focused suites repeat subsets of full discovery; their +counts are reported separately and are not added as distinct tests. + +These checks establish internally consistent local attestations, not +cryptographic issuer authentication or proof of host execution. The verifier +accepts an explicit `kairotensor-runtime-matrix/1` manifest. Its +`source` binds a committed revision, every tracked-file digest and the lock. +Five `lanes` bind roles, registered checkout paths, absolute actual Rust/Cargo/ +Python executables, expected version output, SHA-256 and archive/closure +provenance. Wrapper/guard/recipe/native-library and external E1 input identities +are explicit. Mandatory `host_path` freezes ordered nonempty absolute PATH +directories. Command cwd and selected environment are checked; runner identities +come from the retained compiler-artifact log; structured parent/child Python +real paths and exact versions, raw probe output/exit and alias targets are +verified. `commands` must equal the complete built-in inventory. The verifier +never installs a runtime or substitutes a PATH runtime; it creates verified +private aliases for child Rust/Python selection. Missing prerequisites are +Unexecuted, started failures remain Failed, and a Pass requires every required +command/result, nonzero test counts, no skipped/ignored/filtered tests, exact +source before/after, unchanged identities and independently re-readable logs +and artifacts. Receipts bind the complete manifest, not only a reported status. +Fake executable tests exercise refusals and command identity only; they are not +runtime qualification evidence. + +Use `python3 scripts/verify_runtime_matrix.py --manifest +--output ` for a new qualification run, or add +`--lane --verify-receipt ` to validate an existing lane's +bound artifacts. The Python used to invoke the verifier is operational; each +workload interpreter is selected by the manifest. These are qualification +interfaces, not G2's future ordinary-development launcher or accepted selection. + +### Runtime and source evidence + +Intake refreshed primary Rust stable/nightly manifests and official release +records, plus Python's supported-release list and latest previous/preview +redirects. Selections were Rust 1.100.0; nightly-2026-11-16 / 1.102.0-nightly +(commit `5f4effd57`, replacing the prior dated Nov15 research selection); previous +1.99.1; Python 3.15.2, 3.16.0a2 and 3.14.10. Ongoing support for the previous +Rust line remains unknown. Raw bytes, response times and digests are retained +under `upstream-intake/`, with exact component URLs/digests in +`selected-upstream-identities.json`. Download declarations are distinguished from +verified archives and actual launched binaries. + +The measured host is NixOS 26.05 (Yarara), build +`26.05.20261108.ac588ef`, Linux x86_64. The wrapper's `nixos26.05-x86_64` +output component is a static label, not a fresh host measurement. Actual binary, +ELF loader, library, standard-library import and native GCC identities are +retained in runtime inspection receipts. Compiler identities use separate +registered checkouts/keys. Python-varied reuse requires unchanged compiler, +source, lock, native dependencies and profile, with every test process rerun. +All Cargo operations used the approved `kairotensor-cargo --budget-gib 2` route, +zero debug symbols and disabled incremental compilation. Host defaults, existing +runtime slots, source contracts, dependency pins and CI were preserved. + +| Role | Actual Rust | Actual Python | Final source result | +| --- | --- | --- | --- | +| stable | `rustc 1.100.0 (0ef1bc386 2026-11-09)` | `Python 3.15.2` | Pass: 40/40 commands; CLI revalidation exit 0 | +| rust-upcoming | `rustc 1.102.0-nightly (5f4effd57 2026-11-13)` | `Python 3.15.2` | Pass: 40/40 commands; CLI revalidation exit 0 | +| rust-previous | `rustc 1.99.1 (58cf8f9c3 2026-10-13)` | `Python 3.15.2` | Pass: 40/40 commands; CLI revalidation exit 0 | +| python-upcoming | `rustc 1.100.0 (0ef1bc386 2026-11-09)` | `Python 3.16.0a2` | Pass: 40/40 commands; CLI revalidation exit 0 | +| python-previous | `rustc 1.100.0 (0ef1bc386 2026-11-09)` | `Python 3.14.10` | Pass: 40/40 commands; CLI revalidation exit 0 | + +All five rows bind the original immutable `matrix-qualified.json` and their +`qualified-v2/matrix//receipt.json`. Every row ran 357 Rust tests, 18 +reference tests, 8 oracle tests and 335 full Python tests (293 existing plus +42 verifier tests), with zero skips/ignored/filtered tests. Focused counts +24/42/15/19/7 repeat subsets. `final-matrix-rollup.json` binds all five receipt +SHAs, source/lock and unchanged-runtime equality; `all-final-command-results.json` +contains every exact argv/environment/cwd/exit/log/artifact/count record. All +200 final command executions exited zero. Preview Python uses Astral +python-build-standalone release 20261110, a third-party CPython build, and passed +actual loader/standard-library checks without patching. No hosted CI is claimed. + +### Executed recovery and retention + +The isolated registered `g1-rollback-s0` checkout built S0's original locked +library with retained Rust 1.99.1 and Python 3.15.2. An explicitly authorized, +artifact-only `rollback_probe.rs` created one accepted current branch and a +separate claimed unresolved recovery through actual Store APIs. The closed +database was copied to backup and restored, then reopened twice through Store. +Both checks returned the exact accepted bytes, spent 7/reserved 12, the same +unresolved attempt, no second claim and `cancel=false`; no backend was invoked. +Original, backup and restored final bytes remained identical. The fixture digest +is `b32b32f06e8f8fe78211560a3ad76c16aae4d8b55060513323fd7533441a8e29`. +This proves the bounded local fixture restore, not reversal of external effects. + +The full old Rust sysroot was copied under the unique G1 runtime prefix and all +210 file hashes matched. Eight explicit Nix GC roots retain Python and native +closures, with closure metadata and loader identities. Operational receipts are +under `/home/neo/.local/state/kairotensor/runtime/g1-20261116-s0794f64e/`, with +provenance copies in the packet. No retirement date or deletion is authorized. +The first probe compile's incorrect dependency search path and all source/lane/ +acquisition failures remain retained; later success does not overwrite them. + +### Owned gaps and next review + +G2 owns portable pins, normal project launch selection, CI changes, independent +acceptance and any local accepted-selection activation. No such activation +occurred in G1. Seven-day observation dates begin only at an actual activation; +retirement requires a separate owner decision. Root is the technical owner +acting under Alejandro's project authority; the next weekly source-freshness and +gap review is due 2026-11-23, and freshness must be checked again before promotion. + +The six other workflows (`benchmark`, `handoff`, `recovery-calibration`, +`recovery-policy`, `registration-cache`, `token-caps`), historical explicit ML +Python 3.11 environments, hosted execution, model/provider consumers and any +separately promised older consumers remain unqualified by this core matrix. +A future root Rust toolchain pin can affect workflows that invoke rustup-managed +Cargo; each workflow still needs its own runtime/consumer evidence. No model +inference, provider calls, dependency bulk update, storage exception extension, +cleanup, old-runtime removal or live database work occurred. Inherited Rust +warnings and bounded metadata/tooling failures remain in the logs. + +One operational evidence defect is retained in `controller-log-collision.json`: an +older controller progress log and standalone preflight display were overwritten +while adapting an operational script. The progress log was reconstructed into a +new path and matched its previously retained SHA exactly. The old standalone +preflight display bytes could not be recovered; its original exit/SHA and all +inner command/admission receipts remain. The final matrix uses a fresh exclusively +created packet with direct process exits and complete fresh admission evidence. +The missing historical display is not a passing receipt or silently reconstructed. From d60159d6bda1e36ceecace4125f05059c07cbb50 Mon Sep 17 00:00:00 2001 From: Alejandro Castellanos <5405628+k2jac9@users.noreply.github.com> Date: Sun, 15 Nov 2026 22:02:12 -0500 Subject: [PATCH 36/51] fix: isolate unavailable runtime lanes at the matrix CLI --- scripts/test_verify_runtime_matrix.py | 110 ++++++++++++++++++++++++++ scripts/verify_runtime_matrix.py | 47 +++++++++-- 2 files changed, 149 insertions(+), 8 deletions(-) diff --git a/scripts/test_verify_runtime_matrix.py b/scripts/test_verify_runtime_matrix.py index 6cd225f..43219f6 100644 --- a/scripts/test_verify_runtime_matrix.py +++ b/scripts/test_verify_runtime_matrix.py @@ -190,6 +190,116 @@ def test_unavailable_lane_emits_no_pass_or_commands(self): self.assertEqual(r['status'],'Unexecuted');self.assertEqual(r['commands'],[]) self.assertTrue((self.root/'result'/'receipt.json').is_file()) + def cli_fixture(self): + m=copy.deepcopy(self.manifest()) + m['lanes']=[copy.deepcopy(lane) for lane in m['lanes']] + python={'path':sys.executable,'sha256':self.sha(sys.executable), + 'version_args':['--version'], + 'version':subprocess.check_output([sys.executable,'--version'],text=True).strip()} + for lane in m['lanes']:lane['python']=copy.deepcopy(python) + wrapper=self.root/'fixture-wrapper' + wrapper.write_text(f'#!{sys.executable}\nprint("fixture workload reached")\nraise SystemExit(23)\n') + wrapper.chmod(0o755) + m['wrapper']={'path':str(wrapper),'sha256':self.sha(wrapper)} + return m + + def invoke_cli(self,m,label,lane=None): + manifest=self.root/(label+'.json');manifest.write_text(json.dumps(m)) + output=self.root/('run-'+label) + argv=[sys.executable,SPEC.origin,'--manifest',str(manifest),'--output',str(output)] + if lane:argv+=['--lane',lane] + # The matrix itself runs these tests under selected compiler variables. + # This child CLI fixture declares its own runtime environment. + env={k:v for k,v in os.environ.items() if k not in self.m.OVERRIDES + and not k.startswith(('CARGO_TARGET_','CARGO_PROFILE_'))} + result=subprocess.run(argv,capture_output=True,text=True,env=env,timeout=30) + receipts={p.parent.name:json.loads(p.read_text()) for p in output.glob('*/receipt.json')} + return result,receipts + + def missing_prerequisite(self,lane,kind): + missing=str(self.root/('unavailable-'+kind)) + if kind=='checkout':lane['checkout']=missing + elif kind=='archive':lane['provenance']=[{'path':missing,'sha256':'0'*64}] + else:lane['cargo']['path']=missing + + def assert_workload_reached(self,receipt): + self.assertEqual(receipt['status'],'Failed') + self.assertEqual(len(receipt['commands']),1) + self.assertEqual(receipt['commands'][0]['exit'],23) + self.assertIn('fixture workload reached',Path(receipt['commands'][0]['stdout']['path']).read_text()) + + def test_cli_selected_stable_ignores_unavailable_preview_prerequisites(self): + m=self.cli_fixture() + for kind in ['archive','checkout','executable']: + with self.subTest(kind=kind): + changed=copy.deepcopy(m);self.missing_prerequisite(changed['lanes'][1],kind) + result,receipts=self.invoke_cli(changed,'selected-'+kind,'stable') + self.assertEqual(set(receipts),{'stable'},result.stderr) + self.assert_workload_reached(receipts['stable']) + self.assertEqual(result.returncode,1) # intentional fixture workload failure + + def test_cli_requested_unavailable_role_has_explicit_receipt(self): + m=self.cli_fixture() + for kind in ['archive','checkout','executable']: + with self.subTest(kind=kind): + changed=copy.deepcopy(m);self.missing_prerequisite(changed['lanes'][1],kind) + result,receipts=self.invoke_cli(changed,'unavailable-'+kind,'rust-upcoming') + self.assertEqual(set(receipts),{'rust-upcoming'},result.stderr) + self.assertEqual(receipts['rust-upcoming']['status'],'Unexecuted') + self.assertEqual(receipts['rust-upcoming']['commands'],[]) + self.assertNotEqual(result.returncode,0) + + def test_cli_full_matrix_continues_independent_roles_when_stable_unavailable(self): + m=self.cli_fixture();self.missing_prerequisite(m['lanes'][0],'archive') + result,receipts=self.invoke_cli(m,'full-unavailable') + self.assertEqual(set(receipts),self.m.ROLES,result.stderr) + self.assertEqual(receipts['stable']['status'],'Unexecuted') + for role in ['rust-upcoming','rust-previous']:self.assert_workload_reached(receipts[role]) + for role in ['python-upcoming','python-previous']: + self.assertEqual(receipts[role]['status'],'Unexecuted') + self.assertEqual(receipts[role]['commands'],[]) + self.assertIn('stable',receipts[role]['error']) + self.assertNotEqual(result.returncode,0) + + def test_cli_full_matrix_failed_stable_blocks_dependent_rows(self): + result,receipts=self.invoke_cli(self.cli_fixture(),'full-failed') + self.assertEqual(set(receipts),self.m.ROLES,result.stderr) + for role in ['stable','rust-upcoming','rust-previous']:self.assert_workload_reached(receipts[role]) + for role in ['python-upcoming','python-previous']: + self.assertEqual(receipts[role]['status'],'Unexecuted') + self.assertEqual(receipts[role]['commands'],[]) + self.assertNotEqual(result.returncode,0) + + def test_cli_selected_python_without_stable_receipt_is_unexecuted(self): + result,receipts=self.invoke_cli(self.cli_fixture(),'no-stable','python-upcoming') + self.assertEqual(set(receipts),{'python-upcoming'},result.stderr) + self.assertEqual(receipts['python-upcoming']['status'],'Unexecuted') + self.assertEqual(receipts['python-upcoming']['commands'],[]) + self.assertNotEqual(result.returncode,0) + + def test_cli_digest_mismatch_is_failure_not_unavailable(self): + m=self.cli_fixture();m['lanes'][1]['provenance'][0]['sha256']='0'*64 + result,receipts=self.invoke_cli(m,'bad-digest','rust-upcoming') + self.assertEqual(set(receipts),{'rust-upcoming'},result.stderr) + self.assertEqual(receipts['rust-upcoming']['status'],'Failed') + self.assertEqual(receipts['rust-upcoming']['commands'],[]) + self.assertIn('digest',receipts['rust-upcoming']['error']) + self.assertNotEqual(result.returncode,0) + + def test_cli_unselected_structural_contradiction_still_refuses(self): + m=self.cli_fixture();m['lanes'][1]['checkout']='relative-checkout' + result,receipts=self.invoke_cli(m,'invalid-structure','stable') + self.assertEqual(receipts,{}) + self.assertNotEqual(result.returncode,0) + self.assertIn('absolute',result.stderr) + + def test_cli_missing_checkout_does_not_hide_invalid_shared_source_contract(self): + m=self.cli_fixture();self.missing_prerequisite(m['lanes'][0],'checkout') + m['source']['lock_sha256']='0'*64 + result,receipts=self.invoke_cli(m,'invalid-source','stable') + self.assertEqual(receipts,{}) + self.assertNotEqual(result.returncode,0) + def test_receipt_binds_whole_manifest(self): m=self.manifest();receipt={'role':'stable','source':m['source'],'status':'Pass','manifest_sha256':'0'*64} with self.assertRaisesRegex(ValueError,'manifest'): diff --git a/scripts/verify_runtime_matrix.py b/scripts/verify_runtime_matrix.py index 9b90847..89051ea 100644 --- a/scripts/verify_runtime_matrix.py +++ b/scripts/verify_runtime_matrix.py @@ -315,10 +315,18 @@ def check_command_receipt(command, record): def check_manifest(m): + """Check shared inputs and every role's contract, not its availability.""" if m.get('schema')!='kairotensor-runtime-matrix/1': raise ValueError('unknown manifest schema') host_path=m.get('host_path','') if not isinstance(host_path,str) or any(not p or not Path(p).is_absolute() for p in host_path.split(os.pathsep)): raise ValueError('host_path must contain nonempty absolute directories') + source=m['source'];files=source['files'] + if (not re.fullmatch(r'[0-9a-f]{40}',source['commit']) or not isinstance(files,dict) + or files.get('Cargo.lock')!=source['lock_sha256'] + or not re.fullmatch(r'[0-9a-f]{64}',source['lock_sha256']) + or any(Path(name).is_absolute() or '..' in Path(name).parts + or not re.fullmatch(r'[0-9a-f]{64}',digest) for name,digest in files.items())): + raise ValueError('invalid shared source contract') validate_roles(m['lanes']) if m['commands'] != command_inventory(): raise ValueError('incomplete or changed required command inventory') for field in ['wrapper','storage_guard','cargo_recipe','bundle','attestations']: @@ -327,12 +335,26 @@ def check_manifest(m): for identity in m['native_files']: check_file(identity) for lane in m['lanes']: if not Path(lane['checkout']).is_absolute(): raise ValueError('absolute checkout required') - check_source(lane['checkout'],m['source']) if not lane.get('provenance'): raise ValueError('missing runtime archive/closure provenance') - for identity in lane['provenance']: check_file(identity) + for identity in [*lane['provenance'],*(lane[k] for k in ['rustc','cargo','python'])]: + if not Path(identity['path']).is_absolute(): raise ValueError('identity requires absolute path') + if not re.fullmatch(r'[0-9a-f]{64}',identity['sha256']): raise ValueError('invalid identity digest') + for key in ['rustc','cargo','python']: + identity=lane[key] + if (not isinstance(identity['version'],str) or not identity['version'] + or not isinstance(identity['version_args'],list) + or not all(isinstance(arg,str) for arg in identity['version_args'])): + raise ValueError('invalid runtime version contract') -def run_lane(m,lane,out): +def check_lane(m,lane): + root=Path(lane['checkout']) + if not root.exists(): raise FileNotFoundError(f'unavailable checkout: {root}') + check_source(root,m['source']) + for identity in lane['provenance']: check_file(identity) + + +def run_lane(m,lane,out,blocked_reason=None): out=Path(out);out.mkdir(parents=True,exist_ok=False) root=Path(lane['checkout']); outputs=out/'outputs';outputs.mkdir() result={'role':lane['role'],'source':m['source'],'status':'Unexecuted','commands':[], @@ -341,6 +363,11 @@ def run_lane(m,lane,out): save(out/'receipt.json',result) try: check_manifest(m) + check_lane(m,lane) + if blocked_reason: + result['error']=blocked_reason + save(out/'receipt.json',result) + return result tools={k:check_identity(lane[k]) for k in ['rustc','cargo','python']} result['tools']=tools env=lane_environment(lane,out/'bin',{'RUNNER_TEMP':str(outputs)},dict(os.environ,PATH=m['host_path'])) @@ -363,7 +390,7 @@ def run_lane(m,lane,out): for name,key in [('recovery_policy_eval','KAIROTENSOR_RECOVERY_RUNNER'),('decision_replay','KAIROTENSOR_DECISION_REPLAY')]: env[key]=runners[name]['path'] result['runners']=runners - check_manifest(m);result['source_verified_after']=True + check_manifest(m);check_lane(m,lane);result['source_verified_after']=True for identity in tools.values(): check_file(identity) for identity in result.get('runners',{}).values(): check_file(identity) if len(result['commands'])==len(commands) and all(r['status']=='Pass' for r in result['commands']): @@ -373,13 +400,14 @@ def run_lane(m,lane,out): verify_receipt(m,lane,json.loads((out/'receipt.json').read_text())) except (OSError,ValueError,KeyError,subprocess.SubprocessError) as error: result['error']=str(error) - if result['commands']:result['status']='Failed' + if result['commands'] or not isinstance(error,OSError):result['status']='Failed' save(out/'receipt.json',result) return result def verify_receipt(m,lane,receipt): check_manifest(m) + check_lane(m,lane) if receipt.get('manifest_sha256')!=hashlib.sha256(json.dumps(m,sort_keys=True).encode()).hexdigest(): raise ValueError('receipt manifest identity mismatch') if receipt.get('role')!=lane['role'] or receipt.get('source')!=m['source'] or receipt.get('status')!='Pass': @@ -418,11 +446,14 @@ def main(): results=[] for lane in m['lanes']: if args.lane and lane['role']!=args.lane: continue + blocked_reason=None if lane['role'].startswith('python-'): stable=args.output/'stable'/'receipt.json' - if not stable.is_file(): raise ValueError('stable qualification required before dependent Python lane') - verify_receipt(m,next(l for l in m['lanes'] if l['role']=='stable'),json.loads(stable.read_text())) - results.append(run_lane(m,lane,args.output/lane['role'])) + try: + verify_receipt(m,next(l for l in m['lanes'] if l['role']=='stable'),json.loads(stable.read_text())) + except (OSError,ValueError,KeyError,subprocess.SubprocessError) as error: + blocked_reason=f'validated stable qualification required: {error}' + results.append(run_lane(m,lane,args.output/lane['role'],blocked_reason)) return 0 if results and all(r['status']=='Pass' for r in results) else 1 From 7684b4fd4148145f882a1e2e3fc3e2f2833d559e Mon Sep 17 00:00:00 2001 From: Alejandro Castellanos <5405628+k2jac9@users.noreply.github.com> Date: Sun, 15 Nov 2026 22:11:05 -0500 Subject: [PATCH 37/51] docs: record isolated lane fix and five-runtime requalification --- docs/runtime-adoption.md | 56 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 56 insertions(+) diff --git a/docs/runtime-adoption.md b/docs/runtime-adoption.md index d934af0..2d85a1e 100644 --- a/docs/runtime-adoption.md +++ b/docs/runtime-adoption.md @@ -290,3 +290,59 @@ preflight display bytes could not be recovered; its original exit/SHA and all inner command/admission receipts remain. The final matrix uses a fresh exclusively created packet with direct process exits and complete fresh admission evidence. The missing historical display is not a passing receipt or silently reconstructed. + +## G1 review fix 1 — unavailable lane isolation, 2026-11-16 + +The preceding G1 records remain historical. Independent review found that a +missing archive or checkout in an unselected lane could abort the CLI before the +selected available lane received any result. Executable successor `d60159d6bda1e36ceecace4125f05059c07cbb50` +separates shared/structural contracts from each selected lane's availability. +Only requested roles run. Missing lane prerequisites emit Unexecuted receipts; +independent available roles continue. Dependent Python rows emit Unexecuted when +the same-manifest stable receipt is missing, failed or cannot be validated. +Selected-lane digest mismatches are Failed; malformed shared or role contracts +still refuse. + +Eight additional public-CLI regressions bring the verifier suite to 50 tests. +They use actual subprocesses, tiny fixture tools and a real Python parent/child; +the reachable workload intentionally exits 23, so these refusal/isolation tests +cannot be confused with runtime qualification. RED and GREEN logs are retained. +The approved 40-command workload and acceptance predicates are unchanged. + +The new packet is `fix-1/` within the G1 artifact directory above. +`manifest.json`, `source-freeze.json`, `execution/process-receipts.json`, +`final-matrix-rollup.json` and `all-command-results.json` bind these fresh runs: + +| Role | Rust | Python | Commands | Receipt CLI exit | +| --- | --- | --- | --- | --- | +| stable | `rustc 1.100.0 (0ef1bc386 2026-11-09)` | `Python 3.15.2` | 40/40 Pass | 0 | +| rust-upcoming | `rustc 1.102.0-nightly (5f4effd57 2026-11-13)` | `Python 3.15.2` | 40/40 Pass | 0 | +| rust-previous | `rustc 1.99.1 (58cf8f9c3 2026-10-13)` | `Python 3.15.2` | 40/40 Pass | 0 | +| python-upcoming | `rustc 1.100.0 (0ef1bc386 2026-11-09)` | `Python 3.16.0a2` | 40/40 Pass | 0 | +| python-previous | `rustc 1.100.0 (0ef1bc386 2026-11-09)` | `Python 3.14.10` | 40/40 Pass | 0 | + +Each row ran 357 Rust tests, 18 reference tests, 8 oracle tests and 343 full +Python tests (293 existing plus 50 verifier tests), with zero skipped, ignored or +filtered tests. Focused suites repeat subsets. All 200 workload commands, all +five separate persisted-receipt CLI validations and the outer controller exited +zero. Three new registered frozen worktrees bind the same source/lock; Python +rows reuse only the unchanged stable compiler key and rerun every test process. +The documentation-only successor does not relabel these source-bound receipts. + +`provenance-role-mapping.json` proves equality of the entire prior 78-file +provenance/native identity union. Runtime-specific archives and libraries now +belong to their consuming roles; 27 host/compiler native files remain shared. +This prevents accidental cross-lane availability requirements without omitting +an identity. Mapping follows the actual selected tool installation prefixes and +is bounded to this frozen fixture workload, not discovery of arbitrary future +native dependencies. No runtime, dependency, guard, CI or accepted-selection +change occurred, and no download or repeat restore was needed. The retained S0 +restore evidence and eight GC roots/closures were rechecked unchanged. + +The lost historical standalone preflight display remains a preservation defect: +its original SHA-256 is +`8795475d9b4184453f7c0653f9a594ab4690a4113e986f000aa35b69d7f14f86`. +Its bytes have not been recovered. Fix-1 uses exclusively created operation +paths and fresh admission receipts; it makes no claim to reconstruct that loss. +Independent scoped review remains required. Other adoption gaps and G2's +activation/observation responsibilities above are unchanged. From ae4aa0865912ac5f991a08ddfe379e5a180f7bfa Mon Sep 17 00:00:00 2001 From: Alejandro Castellanos <5405628+k2jac9@users.noreply.github.com> Date: Sun, 15 Nov 2026 22:25:06 -0500 Subject: [PATCH 38/51] Pin qualified project runtimes and add verified local launcher --- .github/workflows/check.yml | 76 +++++++++++++ .python-version | 1 + docs/runtime-adoption.md | 118 ++++++++++++++++++++ rust-toolchain.toml | 3 + scripts/project_runtime.py | 185 ++++++++++++++++++++++++++++++++ scripts/test_project_runtime.py | 177 ++++++++++++++++++++++++++++++ 6 files changed, 560 insertions(+) create mode 100644 .python-version create mode 100644 rust-toolchain.toml create mode 100644 scripts/project_runtime.py create mode 100644 scripts/test_project_runtime.py diff --git a/.github/workflows/check.yml b/.github/workflows/check.yml index 7c1e26e..77103e9 100644 --- a/.github/workflows/check.yml +++ b/.github/workflows/check.yml @@ -6,14 +6,90 @@ permissions: jobs: contracts: + name: contracts (${{ matrix.role }}) runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + include: + - role: stable + rust: '1.100.0' + rust-release: '1.100.0' + rust-commit: '0ef1bc386059596dada8fb65b50425aab2ea637a' + cargo-version: 'cargo 1.100.0 (3acc79c0c 2026-11-05)' + python: '3.15.2' + python-version: '3.15.2' + - role: rust-upcoming + rust: 'nightly-2026-11-16' + rust-release: '1.102.0-nightly' + rust-commit: '5f4effd579500175af844241eb179e756190f1e7' + cargo-version: 'cargo 1.102.0-nightly (ede6cefea 2026-11-11)' + python: '3.15.2' + python-version: '3.15.2' + # This pair also retains the accepted local development baseline. + - role: rust-previous-and-retained-baseline + rust: '1.99.1' + rust-release: '1.99.1' + rust-commit: '58cf8f9c35b355199727627d07ae34a4e2846d67' + cargo-version: 'cargo 1.99.1 (5f94df478 2026-08-27)' + python: '3.15.2' + python-version: '3.15.2' + - role: python-upcoming + rust: '1.100.0' + rust-release: '1.100.0' + rust-commit: '0ef1bc386059596dada8fb65b50425aab2ea637a' + cargo-version: 'cargo 1.100.0 (3acc79c0c 2026-11-05)' + python: '3.16.0-alpha.2' + python-version: '3.16.0a2' + - role: python-previous + rust: '1.100.0' + rust-release: '1.100.0' + rust-commit: '0ef1bc386059596dada8fb65b50425aab2ea637a' + cargo-version: 'cargo 1.100.0 (3acc79c0c 2026-11-05)' + python: '3.14.10' + python-version: '3.14.10' timeout-minutes: 10 env: KAIROTENSOR_REQUIRE_LINUX_CLOCK: '1' + # Explicitly overrides rust-toolchain.toml in every compatibility row. + RUSTUP_TOOLCHAIN: ${{ matrix.rust }} + RUSTUP_AUTO_INSTALL: '0' + EXPECTED_RUST_RELEASE: ${{ matrix.rust-release }} + EXPECTED_RUST_COMMIT: ${{ matrix.rust-commit }} + EXPECTED_CARGO_VERSION: ${{ matrix.cargo-version }} + EXPECTED_PYTHON_VERSION: ${{ matrix.python-version }} steps: - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: persist-credentials: false + - name: Select exact Rust lane + uses: dtolnay/rust-toolchain@df2a88e9fe4098bad09cb42c3dec7187f15861fc + with: + toolchain: ${{ matrix.rust }} + - name: Select exact Python lane + uses: actions/setup-python@6d4fd22b20465ccc0f7354476588e8df7c9ca0e0 # v7.1.0 + with: + python-version: ${{ matrix.python }} + check-latest: false + - name: Verify actual Rust, Cargo and parent/child Python identities + shell: bash + run: | + set -euo pipefail + rustc -vV + cargo --version + rustup which rustc + rustup which cargo + python3 - <<'PYTHON' + import json, os, pathlib, subprocess, sys + rust = dict(line.split(': ', 1) for line in subprocess.check_output(['rustc', '-vV'], text=True).splitlines()[1:]) + assert rust['release'] == os.environ['EXPECTED_RUST_RELEASE'], rust + assert rust['commit-hash'] == os.environ['EXPECTED_RUST_COMMIT'], rust + assert subprocess.check_output(['cargo', '--version'], text=True).strip() == os.environ['EXPECTED_CARGO_VERSION'] + assert sys.version.split()[0] == os.environ['EXPECTED_PYTHON_VERSION'], sys.version + child = json.loads(subprocess.check_output(['python3', '-c', 'import json,sys,os;print(json.dumps([os.path.realpath(sys.executable),sys.version.split()[0]]))'], text=True)) + assert child == [str(pathlib.Path(sys.executable).resolve()), os.environ['EXPECTED_PYTHON_VERSION']], child + print(json.dumps({'parent': str(pathlib.Path(sys.executable).resolve()), 'child': child})) + PYTHON - name: Test the Rust store, including the withdrawal example run: cargo test --locked --all-targets - name: Run the revision and model handoff examples diff --git a/.python-version b/.python-version new file mode 100644 index 0000000..861845e --- /dev/null +++ b/.python-version @@ -0,0 +1 @@ +3.15.2 diff --git a/docs/runtime-adoption.md b/docs/runtime-adoption.md index 2d85a1e..c0e7c3a 100644 --- a/docs/runtime-adoption.md +++ b/docs/runtime-adoption.md @@ -346,3 +346,121 @@ Its bytes have not been recovered. Fix-1 uses exclusively created operation paths and fresh admission receipts; it makes no claim to reconstruct that loss. Independent scoped review remains required. Other adoption gaps and G2's activation/observation responsibilities above are unchanged. + +## G2 project runtime selection (2026-11-16) + +The portable defaults are Rust `1.100.0` (`minimal` profile: Rust/Cargo/std; +no optional rustfmt/clippy component) and Python `3.15.2`. They follow accepted +G1 qualification at executable source +`d60159d6bda1e36ceecace4125f05059c07cbb50`, documented by successor +`7684b4fd4148145f882a1e2e3fc3e2f2833d559e`. G2's final source and results are +recorded below after execution. G1's other-role full suites remain S1 evidence. +No local activation or hosted execution is implied by these files. + +Use the durable project launcher from a project checkout: + +```sh +python3 scripts/project_runtime.py identity +python3 scripts/project_runtime.py python -m unittest discover -s reference -p test_memory.py +python3 scripts/project_runtime.py --budget-gib 2 cargo test --locked --all-targets +``` + +The bootstrap Python only reads and validates selection metadata. Actual workload +Python, child `python3`, Rust and Cargo use the verified selected binaries. The +host Python command and `.python-version` alone do not establish child selection. +The host Rust commands are rustup proxies: actual checks with auto-install disabled +returned exit 1, because `1.100.0` is absent in the host rustup home. The qualified +copy exists separately under project development storage. Use the launcher; do not +change host defaults or allow a proxy to download a substitute. Bootstrap requires +Python 3.11 or newer (stdlib `hashlib.file_digest`). + +The default pointer is +`~/.local/state/kairotensor/runtime/accepted-selection.json` (or the equivalent +`XDG_STATE_HOME` path). It is root-owned operational acceptance state, not a file +committed with workstation paths into portable CI. Before independent review use +`--selection /home/neo/.local/state/kairotensor/runtime/candidates/g2-20261116/stable.json`. +Missing selection/tool, changed hash/version, unexpected inherited runtime override, +unbound Cargo config or incompatible qualification binding fails closed with exit +2. A refused storage admission propagates its actual exit. No automatic install, +download, fallback, host update or runtime retirement is performed. + +The local JSON schema is `kairotensor-project-runtime/1`. Required fields: + +- `role`: the exact role in the qualified five-role manifest. +- `tools`: `rustc`, `cargo`, `python`, each with absolute `path`, SHA-256 + `sha256`, exact stdout `version`, and `version_args` array. These must match + that role's immutable qualification identities. +- `qualification_manifest`, `qualification_receipt`, `independent_review`, + `acceptance`: each an absolute `path` and `sha256`. The receipt must be Pass + for the manifest's source and role; the acceptance binds that source and review. + The manifest additionally binds selected native/closure provenance, approved + Cargo wrapper, storage guard and recipe; those files are rechecked before launch. + +The G1 acceptance is `task-g1/fix-1/root-fix1-acceptance.json` in the retained +implementation packet. A selection can identify accepted G1 runtimes during G2 +candidate testing; it does not declare G2 independently accepted. Root controls +activation after G2 review. These are local trusted-owner artifacts: content hashes +bind bytes, not issuer authentication. The launcher is an invocation boundary, not +a security sandbox for arbitrary Python, Cargo build scripts or executable code. + +Each launch creates a private alias directory and receipt under +`~/.local/state/kairotensor/runtime/invocations/`. It records actual resolved paths, +versions/digests, parent/child Python, selected profile, wrapper, current commit, +tracked-file hashes, dirty/untracked status, current Cargo.lock, command and exit. +The inherited environment is not logged. Keep credentials out of command arguments. +Editable source is allowed and always marked `qualification: not assessed`; +it never inherits the frozen source's successful workload. Requested Cargo commands +route through the approved `kairotensor-cargo --budget-gib N` wrapper; debug symbols +and incremental compilation are disabled. Runtime/output/profile overrides and +Cargo configs require a separate reviewed binding. The wrapper handles fresh +storage admission; refusal blocks heavy work. Its existing exception expires +2026-11-17 18:04:38 UTC and must not be extended by this launcher. + +`check.yml` has five independently varied roles: stable `1.100.0 / 3.15.2`, Rust +upcoming `nightly-2026-11-16 / 3.15.2`, Rust previous and retained baseline +`1.99.1 / 3.15.2`, Python upcoming `1.100.0 / 3.16.0a2`, and Python previous +`1.100.0 / 3.14.10`. The setup-python preview selector is exactly +`3.16.0-alpha.2`, as advertised by its build manifest; observed Python must report +`3.16.0a2`. Each row explicitly sets `RUSTUP_TOOLCHAIN` to override the root pin, +checks Rust release/commit and Cargo version, and checks parent/child Python. +The previous-Rust pair also covers the distinct accepted development baseline; +no redundant sixth row is needed. Failures remain visible (`fail-fast: false`, +no `continue-on-error`). Original checkout identity and all command gates remain. + +Setup actions are pinned to verified upstream source: setup-python v7.1.0 at +`6d4fd22b20465ccc0f7354476588e8df7c9ca0e0` and rust-toolchain at +`df2a88e9fe4098bad09cb42c3dec7187f15861fc`. Primary metadata was retained on +2026-11-16 in `runtime-actions-preflight-20261116/` and +`runtime-python-ci-preflight-20261116/`; runtime freshness was reconfirmed by root +before G2 intake. `ubuntu-latest` is mutable; these pins do not prove hosted +binaries equal the locally qualified Nix/standalone artifacts. Hosted CI is +unexecuted; this task neither pushes nor triggers it. + +Six separate workflows remain owned adoption gaps: benchmark, handoff, +recovery-calibration, recovery-policy, registration-cache and token-caps. Their +files and old receipts are unchanged. Future Cargo commands inside a candidate +checkout can now resolve the root Rust pin; the registration-cache workflow runs +from its parent with `--manifest-path`, so candidate/control Rust selection needs +explicit follow-up rather than assuming the nested pin selects it. The four ML +measurement workflows (handoff, recovery-calibration, recovery-policy, token-caps) +retain Python `3.11` and have no real-model qualification under this core matrix. +Accepted checkout v4.2.2 is intentionally retained; adopting latest v7.0.1 is a +separate owned gap. Previous Rust upstream support and the complete separately +promised consumer inventory remain not established. No model, Rocket, deployment, +shared publication or bulk dependency change is covered here. + +Root, acting under Alejandro's autonomous project authority, owns review and +activation. Retain any prior pointer, independently review G2, then atomically +replace `accepted-selection.json` with the reviewed candidate bytes. Run launcher +identity/child-Python verification and the candidate-built disposable Store +current-read/hold probe. Record activation time and an exact review date seven +days later; freshness and the six workflow/ML/consumer gaps are reviewed then and +weekly. Until activation, next review is due 2026-11-23. No retirement date is +authorized. Preserve old binaries/sysroots, Nix roots, source, locks and failed +receipts. If checks fail, retain failure evidence and restore the exact previous +selection pointer; when no prior pointer existed, remove only the new activation +pointer as a root-controlled reversal. For source/data recovery use the retained +S0 source `794f64e5f9702987cd2502b86776b28f8a8d97c5`, exact Rust `1.99.1` / +Python `3.15.2` closure and G1's exercised rollback procedure; switching a pointer +does not undo data mutations. The G2 probe uses a fresh disposable database and +never mutates the original rollback fixture. diff --git a/rust-toolchain.toml b/rust-toolchain.toml new file mode 100644 index 0000000..3bb51d5 --- /dev/null +++ b/rust-toolchain.toml @@ -0,0 +1,3 @@ +[toolchain] +channel = "1.100.0" +profile = "minimal" diff --git a/scripts/project_runtime.py b/scripts/project_runtime.py new file mode 100644 index 0000000..6ce4052 --- /dev/null +++ b/scripts/project_runtime.py @@ -0,0 +1,185 @@ +#!/usr/bin/env python3 +"""Launch project commands with a locally accepted, explicitly identified runtime. + +The trusted owner maintains the selection and acceptance artifacts. Hashes detect +changes; they do not authenticate their issuer. Python programs are not sandboxed. +""" +import argparse +import datetime +import hashlib +import json +import math +import os +from pathlib import Path +import subprocess +import sys +import uuid + +from verify_runtime_matrix import OVERRIDES, PROFILE, check_file, sha + + +def load_identity(identity): + return json.loads(check_file(identity).read_text()) + + +def tool_identity(tool, env): + path = check_file(tool) + if not os.access(path, os.X_OK): + raise ValueError('runtime identity is not executable') + version = subprocess.check_output([str(path), *tool['version_args']], env=env, + text=True, timeout=20).strip() + if version != tool['version']: + raise ValueError('runtime identity version mismatch') + return {'path': str(path), 'resolved_path': str(path.resolve()), + 'sha256': sha(path), 'version': version} + + +def selection_contract(selection): + if selection['schema'] != 'kairotensor-project-runtime/1': + raise ValueError('unknown selection schema') + manifest = load_identity(selection['qualification_manifest']) + receipt = load_identity(selection['qualification_receipt']) + acceptance = load_identity(selection['acceptance']) + check_file(selection['independent_review']) + if (acceptance['result'] != 'G1 accepted' + or acceptance['accepted_executable_source'] != manifest['source']['commit'] + or acceptance['independent_review'] != selection['independent_review']): + raise ValueError('acceptance does not bind the qualification source/review') + lanes = [lane for lane in manifest['lanes'] if lane['role'] == selection['role']] + if len(lanes) != 1: + raise ValueError('selection requires one qualified role') + lane = lanes[0] + if (receipt['status'] != 'Pass' or receipt['role'] != lane['role'] + or receipt['source'] != manifest['source'] + or receipt['manifest_sha256'] != hashlib.sha256(json.dumps(manifest, sort_keys=True).encode()).hexdigest()): + raise ValueError('qualification receipt identity/status mismatch') + for key in ('rustc', 'cargo', 'python'): + expected = {field: lane[key][field] for field in ('path', 'sha256', 'version', 'version_args')} + if selection['tools'][key] != expected: + raise ValueError('selected tool identity differs from qualification') + recorded = receipt['tools'][key] + if any(recorded[field] != expected[field] for field in ('path', 'sha256', 'version')): + raise ValueError('qualification tool identity mismatch') + for item in [*manifest['native_files'], *lane['provenance'], + *(manifest[key] for key in ('wrapper', 'storage_guard', 'cargo_recipe'))]: + check_file(item) + if not manifest['native_files'] or not lane['provenance']: + raise ValueError('missing native provenance') + return manifest + + +def runtime_environment(selection, aliases): + env = dict(os.environ) + forbidden = OVERRIDES | {'LD_PRELOAD', 'LD_LIBRARY_PATH', 'DYLD_INSERT_LIBRARIES', + 'CARGO_BUILD_TARGET', 'CARGO_BUILD_RUSTDOC', 'CARGO_ALIAS_TEST'} + unexpected = [key for key in forbidden if env.get(key)] + unexpected += [key for key in env if key.startswith(('CARGO_TARGET_', 'CARGO_PROFILE_', 'CARGO_ALIAS_')) + and env[key] and env[key] != PROFILE.get(key)] + if unexpected: + raise ValueError('unexpected runtime overrides: ' + ', '.join(sorted(set(unexpected)))) + env.update(PROFILE) + aliases.mkdir(mode=0o700) + for alias, key in [('rustc', 'rustc'), ('cargo', 'cargo'), ('python3', 'python'), ('python', 'python')]: + (aliases / alias).symlink_to(selection['tools'][key]['path']) + env['PATH'] = str(aliases) + os.pathsep + env.get('PATH', '') + env['RUSTC'] = selection['tools']['rustc']['path'] + return env + + +def python_probe(tools, env): + expression = 'import os,sys,json;print(json.dumps({"realpath":os.path.realpath(sys.executable),"version":"Python "+sys.version.split()[0]}))' + parent = json.loads(subprocess.check_output([tools['python']['path'], '-c', expression], env=env, text=True, timeout=20)) + child_code = 'import subprocess;print(subprocess.check_output(["python3","-c",' + repr(expression) + '],text=True),end="")' + child = json.loads(subprocess.check_output([tools['python']['path'], '-c', child_code], env=env, text=True, timeout=20)) + expected = {'realpath': tools['python']['resolved_path'], 'version': tools['python']['version']} + if parent != expected or child != expected: + raise ValueError('selected parent/child Python mismatch') + return {'parent': parent, 'child': child} + + +def source_record(root): + def git(*args): + return subprocess.check_output(['git', '-C', str(root), *args], text=True).strip() + tracked = git('ls-files').splitlines() + return {'root': str(root), 'commit': git('rev-parse', 'HEAD'), + 'dirty': bool(git('status', '--porcelain', '--untracked-files=normal')), + 'status': git('status', '--porcelain', '--untracked-files=normal'), + 'tracked': {name: sha(root / name) if (root / name).is_file() else None for name in tracked}, + 'untracked': git('ls-files', '--others', '--exclude-standard').splitlines(), + 'lock_sha256': sha(root / 'Cargo.lock'), 'qualification': 'not assessed'} + + +def check_cargo_arguments(args, root, env): + if not args or any(arg.startswith('+') or arg.split('=')[0] in + {'--config', '--target-dir', '--manifest-path', '--profile', '--release', '--target'} for arg in args): + raise ValueError('Cargo requires ordinary commands without runtime/output/profile overrides') + # Cargo reads every ancestor config and Cargo home; refuse hidden selectors. + cargo_home = Path(env.get('CARGO_HOME', str(Path.home() / '.cargo'))).absolute() + directories = [root / '.cargo', *(parent / '.cargo' for parent in root.parents), cargo_home] + if any((directory / name).exists() for directory in directories for name in ('config', 'config.toml')): + raise ValueError('unbound Cargo config; review and bind it before use') + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + state_default = Path(os.environ.get('XDG_STATE_HOME', str(Path.home() / '.local/state'))) / 'kairotensor/runtime' + parser.add_argument('--selection', type=Path, default=state_default / 'accepted-selection.json') + parser.add_argument('--state-dir', type=Path, default=state_default) + parser.add_argument('--budget-gib', type=float) + parser.add_argument('command', choices=['identity', 'python', 'cargo']) + parser.add_argument('args', nargs=argparse.REMAINDER) + args = parser.parse_args() + record = None + receipt_path = None + try: + if args.command == 'cargo' and (args.budget_gib is None or not math.isfinite(args.budget_gib) or args.budget_gib <= 0): + raise ValueError('Cargo requires a positive finite --budget-gib') + selection_path = args.selection.absolute() + selection = json.loads(selection_path.read_text()) + manifest = selection_contract(selection) + invocation = args.state_dir.absolute() / 'invocations' / (datetime.datetime.now(datetime.timezone.utc).strftime('%Y%m%dT%H%M%S%fZ-') + uuid.uuid4().hex) + invocation.mkdir(parents=True, mode=0o700) + env = runtime_environment(selection, invocation / 'bin') + tools = {key: tool_identity(value, env) for key, value in selection['tools'].items()} + probe = python_probe(tools, env) + root = Path(subprocess.check_output(['git', 'rev-parse', '--show-toplevel'], text=True).strip()).resolve() + record = {'schema': 'kairotensor-project-invocation/1', 'selection': {'path': str(selection_path), 'sha256': sha(selection_path)}, + 'qualification_source': manifest['source']['commit'], 'source': source_record(root), + 'tools': tools, 'python_probe': probe, 'profile': PROFILE, 'wrapper': manifest['wrapper'], + 'started_at': datetime.datetime.now(datetime.timezone.utc).isoformat(), 'exit': None} + receipt_path = invocation / 'receipt.json' + if args.command == 'identity': + if args.args: + raise ValueError('identity takes no arguments') + command = [] + elif args.command == 'python': + command = [tools['python']['path'], *args.args] + else: + check_cargo_arguments(args.args, root, env) + command = [manifest['wrapper']['path'], '--budget-gib', format(args.budget_gib, 'g'), *args.args] + record['command'] = command + # Only command arguments and selected identities are recorded, never the environment. + # Callers must keep secrets out of command arguments as with ordinary shell history. + receipt_path.write_text(json.dumps(record, indent=2) + '\n') + if command: + result = subprocess.run(command, cwd=root, env=env) + code = result.returncode + else: + print(json.dumps({'tools': tools, 'python_probe': probe, 'receipt': str(receipt_path)}, indent=2)) + code = 0 + for value in tools.values(): + check_file(value) + record['exit'] = code + record['finished_at'] = datetime.datetime.now(datetime.timezone.utc).isoformat() + receipt_path.write_text(json.dumps(record, indent=2) + '\n') + return code + except (OSError, ValueError, KeyError, TypeError, subprocess.SubprocessError) as error: + if record is not None and receipt_path is not None: + record.update(exit=2, error=str(error)) + receipt_path.write_text(json.dumps(record, indent=2) + '\n') + print(f'Project runtime selection refused: {error}', file=sys.stderr) + return 2 + + +if __name__ == '__main__': + sys.exit(main()) diff --git a/scripts/test_project_runtime.py b/scripts/test_project_runtime.py new file mode 100644 index 0000000..838b682 --- /dev/null +++ b/scripts/test_project_runtime.py @@ -0,0 +1,177 @@ +"""Launcher boundary tests; fake compiler/wrapper avoid compiling in unit tests.""" +import hashlib +import json +import os +from pathlib import Path +import subprocess +import sys +import tempfile +import unittest + +LAUNCHER = Path(__file__).with_name('project_runtime.py').resolve() + + +def identity(path): + return {'path': str(path), 'sha256': hashlib.sha256(path.read_bytes()).hexdigest()} + + +class ProjectRuntimeTests(unittest.TestCase): + def setUp(self): + self.temp = tempfile.TemporaryDirectory() + self.addCleanup(self.temp.cleanup) + self.root = Path(self.temp.name) + self.repo = self.root / 'repo' + self.repo.mkdir() + subprocess.run(['git', 'init', '-q', str(self.repo)], check=True) + (self.repo / 'Cargo.lock').write_text('fixture lock\n') + subprocess.run(['git', '-C', str(self.repo), 'add', '.'], check=True) + subprocess.run(['git', '-C', str(self.repo), '-c', 'user.name=Fixture', + '-c', 'user.email=fixture@example.invalid', 'commit', '-qm', 'fixture'], check=True) + self.commit = subprocess.check_output(['git', '-C', str(self.repo), 'rev-parse', 'HEAD'], text=True).strip() + self.state = self.root / 'state' + self.env = {k: v for k, v in os.environ.items() if not k.startswith(('CARGO_', 'RUST', 'PYTHON', 'KAIROTENSOR_')) + and k not in {'CC', 'CXX', 'AR', 'LD', 'CFLAGS', 'CXXFLAGS', 'LDFLAGS', 'VIRTUAL_ENV', 'CONDA_PREFIX'}} + self.env['PYTHONDONTWRITEBYTECODE'] = '1' + # Isolated Cargo home avoids workstation config influencing the fixture. + self.env['CARGO_HOME'] = str(self.root / 'cargo-home') + self.tools = {} + for key in ['rustc', 'cargo']: + path = self.root / key + path.write_text('#!/bin/sh\nprintf "%s\\n" "' + key + ' fixture"\n') + path.chmod(0o700) + self.tools[key] = dict(identity(path), version=key + ' fixture', version_args=['--version']) + python = Path(sys.executable).resolve() + self.tools['python'] = dict(identity(python), version=subprocess.check_output([str(python), '--version'], text=True).strip(), version_args=['--version']) + self.wrapper = self.root / 'wrapper' + self.wrapper.write_text('#!' + str(python) + '\nimport json,sys\nprint(json.dumps(sys.argv[1:]))\nsys.exit(17 if "deny" in sys.argv else 0)\n') + self.wrapper.chmod(0o700) + self.review = self.write('review.md', 'independent fixture review\n') + self.acceptance = self.write('acceptance.json', {'result': 'G1 accepted', 'accepted_executable_source': self.commit, + 'independent_review': identity(self.review)}) + self.manifest = {'source': {'commit': self.commit}, 'lanes': [{'role': 'stable', **self.tools, 'provenance': [identity(self.review)]}], + 'native_files': [identity(self.review)], 'wrapper': identity(self.wrapper), + 'storage_guard': identity(self.review), 'cargo_recipe': identity(self.review)} + self.manifest_path = self.write('manifest.json', self.manifest) + self.receipt = {'status': 'Pass', 'source': self.manifest['source'], 'role': 'stable', + 'manifest_sha256': hashlib.sha256(json.dumps(self.manifest, sort_keys=True).encode()).hexdigest(), + 'tools': {k: {x: v for x, v in t.items() if x != 'version_args'} for k, t in self.tools.items()}} + for t in self.receipt['tools'].values(): + t['resolved_path'] = str(Path(t['path']).resolve()) + self.receipt_path = self.write('receipt.json', self.receipt) + self.selection = {'schema': 'kairotensor-project-runtime/1', 'role': 'stable', 'tools': self.tools, + 'qualification_manifest': identity(self.manifest_path), 'qualification_receipt': identity(self.receipt_path), + 'independent_review': identity(self.review), 'acceptance': identity(self.acceptance)} + self.selection_path = self.root / 'selection.json' + + def write(self, name, value): + path = self.root / name + path.write_text(value if isinstance(value, str) else json.dumps(value)) + return path + + def run_launcher(self, *args, env=None): + self.selection_path.write_text(json.dumps(self.selection)) + return subprocess.run([sys.executable, str(LAUNCHER), '--selection', str(self.selection_path), + '--state-dir', str(self.state), *args], cwd=self.repo, + env=self.env if env is None else env, capture_output=True, text=True) + + def latest(self): + return json.loads(sorted(self.state.glob('invocations/*/receipt.json'))[-1].read_text()) + + def test_selected_parent_and_child_python(self): + r = self.run_launcher('python', '-c', 'import subprocess; print(subprocess.check_output(["python3", "--version"], text=True).strip())') + self.assertEqual(r.returncode, 0, r.stderr) + self.assertEqual(r.stdout.strip(), self.tools['python']['version']) + probe = self.latest()['python_probe'] + self.assertEqual(probe['parent'], probe['child']) + self.assertEqual(probe['child']['realpath'], self.tools['python']['path']) + + def test_missing_selection_refuses_without_install(self): + r = subprocess.run([sys.executable, str(LAUNCHER), '--selection', str(self.root / 'absent'), 'identity'], capture_output=True, text=True, env=self.env) + self.assertEqual(r.returncode, 2) + self.assertIn('selection', r.stderr.lower()) + + def test_wrong_hash_refuses(self): + self.selection['tools']['python']['sha256'] = '0' * 64 + r = self.run_launcher('identity') + self.assertEqual(r.returncode, 2) + self.assertIn('identity', r.stderr.lower()) + + def test_missing_tool_refuses(self): + Path(self.tools['rustc']['path']).unlink() + r = self.run_launcher('identity') + self.assertEqual(r.returncode, 2) + + def test_changed_tool_version_refuses(self): + # Rebind all content hashes, but leave the expected version incorrect. + self.tools['cargo']['version'] = 'cargo wrong' + self.manifest['lanes'][0]['cargo'] = self.tools['cargo'] + self.manifest_path.write_text(json.dumps(self.manifest)) + self.selection['qualification_manifest'] = identity(self.manifest_path) + self.receipt['manifest_sha256'] = hashlib.sha256(json.dumps(self.manifest, sort_keys=True).encode()).hexdigest() + self.receipt['tools']['cargo']['version'] = 'cargo wrong' + self.receipt_path.write_text(json.dumps(self.receipt)) + self.selection['qualification_receipt'] = identity(self.receipt_path) + r = self.run_launcher('identity') + self.assertEqual(r.returncode, 2) + self.assertIn('version mismatch', r.stderr) + + def test_changed_review_refuses(self): + self.review.write_text('changed\n') + self.assertEqual(self.run_launcher('identity').returncode, 2) + + def test_failed_qualification_refuses(self): + self.receipt['status'] = 'Failed' + self.receipt_path.write_text(json.dumps(self.receipt)) + self.selection['qualification_receipt'] = identity(self.receipt_path) + self.assertEqual(self.run_launcher('identity').returncode, 2) + + def test_editable_source_is_recorded_without_inheriting_qualification(self): + (self.repo / 'Cargo.lock').write_text('edited lock\n') + (self.repo / 'new.txt').write_text('new\n') + r = self.run_launcher('identity') + self.assertEqual(r.returncode, 0, r.stderr) + source = self.latest()['source'] + self.assertTrue(source['dirty']) + self.assertIn('new.txt', source['untracked']) + self.assertEqual(source['commit'], self.commit) + self.assertEqual(source['lock_sha256'], identity(self.repo / 'Cargo.lock')['sha256']) + self.assertEqual(source['qualification'], 'not assessed') + + def test_cargo_routes_through_wrapper_with_budget(self): + r = self.run_launcher('--budget-gib', '2', 'cargo', 'test', '--locked') + self.assertEqual(r.returncode, 0, r.stderr) + self.assertEqual(json.loads(r.stdout), ['--budget-gib', '2', 'test', '--locked']) + self.assertEqual(self.latest()['command'][0], str(self.wrapper)) + + def test_admission_failure_is_propagated(self): + r = self.run_launcher('--budget-gib', '2', 'cargo', 'deny') + self.assertEqual(r.returncode, 17, r.stderr) + self.assertEqual(self.latest()['exit'], 17) + + def test_cargo_requires_budget(self): + self.assertEqual(self.run_launcher('cargo', 'test').returncode, 2) + + def test_inherited_runtime_overrides_refuse(self): + for key in ['RUSTUP_TOOLCHAIN', 'RUSTC', 'PYTHONPATH', 'CARGO_TARGET_DIR', 'LD_PRELOAD', 'CARGO_PROFILE_RELEASE_DEBUG']: + with self.subTest(key=key): + r = self.run_launcher('identity', env=dict(self.env, **{key: 'unexpected'})) + self.assertEqual(r.returncode, 2, r.stderr) + + def test_cargo_command_overrides_refuse(self): + for args in [('+nightly', 'test'), ('test', '--config=build.rustc="bad"'), ('test', '--target-dir', '/tmp/bad')]: + with self.subTest(args=args): + self.assertEqual(self.run_launcher('--budget-gib', '2', 'cargo', *args).returncode, 2) + + def test_cargo_config_refuses(self): + (self.repo / '.cargo').mkdir() + (self.repo / '.cargo/config.toml').write_text('[build]\nrustc="unexpected"\n') + self.assertEqual(self.run_launcher('--budget-gib', '2', 'cargo', 'test').returncode, 2) + + def test_child_command_exit_is_preserved(self): + r = self.run_launcher('python', '-c', 'raise SystemExit(7)') + self.assertEqual(r.returncode, 7, r.stderr) + self.assertEqual(self.latest()['exit'], 7) + + +if __name__ == '__main__': + unittest.main() From 64e0b1b4a45857f419ac9539760018e3890d9aa3 Mon Sep 17 00:00:00 2001 From: Alejandro Castellanos <5405628+k2jac9@users.noreply.github.com> Date: Sun, 15 Nov 2026 22:32:23 -0500 Subject: [PATCH 39/51] Bind native helper resolution and reject Cargo selector bypasses --- docs/runtime-adoption.md | 15 +++++++-- scripts/project_runtime.py | 54 +++++++++++++++++++++++++-------- scripts/test_project_runtime.py | 37 ++++++++++++++++++---- 3 files changed, 85 insertions(+), 21 deletions(-) diff --git a/docs/runtime-adoption.md b/docs/runtime-adoption.md index c0e7c3a..94d1ac9 100644 --- a/docs/runtime-adoption.md +++ b/docs/runtime-adoption.md @@ -378,7 +378,7 @@ The default pointer is `~/.local/state/kairotensor/runtime/accepted-selection.json` (or the equivalent `XDG_STATE_HOME` path). It is root-owned operational acceptance state, not a file committed with workstation paths into portable CI. Before independent review use -`--selection /home/neo/.local/state/kairotensor/runtime/candidates/g2-20261116/stable.json`. +`--selection /home/neo/.local/state/kairotensor/runtime/candidates/g2-20261116-r1/stable.json`. Missing selection/tool, changed hash/version, unexpected inherited runtime override, unbound Cargo config or incompatible qualification binding fails closed with exit 2. A refused storage admission propagates its actual exit. No automatic install, @@ -390,6 +390,12 @@ The local JSON schema is `kairotensor-project-runtime/1`. Required fields: - `tools`: `rustc`, `cargo`, `python`, each with absolute `path`, SHA-256 `sha256`, exact stdout `version`, and `version_args` array. These must match that role's immutable qualification identities. +- `host_tools`: exact resolved absolute paths and hashes for `cc`, `gcc`, `c++`, + `ar`, `ld`, `git`, `sh`, `bash`, `dirname`, `tr`, `sha256sum`, `cut`, `mkdir`, + and `mktemp`. Resolution must agree with the qualification manifest's frozen + `host_path`; `cc`/`gcc` must also match its native-file inventory. Private aliases + bind these helpers, and Git source inspection uses that same verified environment. + Ambient PATH additions cannot select replacement native or Git commands. - `qualification_manifest`, `qualification_receipt`, `independent_review`, `acceptance`: each an absolute `path` and `sha256`. The receipt must be Pass for the manifest's source and role; the acceptance binds that source and review. @@ -411,8 +417,11 @@ The inherited environment is not logged. Keep credentials out of command argumen Editable source is allowed and always marked `qualification: not assessed`; it never inherits the frozen source's successful workload. Requested Cargo commands route through the approved `kairotensor-cargo --budget-gib N` wrapper; debug symbols -and incremental compilation are disabled. Runtime/output/profile overrides and -Cargo configs require a separate reviewed binding. The wrapper handles fresh +and incremental compilation are disabled. The launcher accepts Cargo build/test/run/check/metadata/locate-project/version +commands. Install, direct rustc/rustdoc forwarding, runtime/output/profile overrides +(including `-r` and clustered short flags), and Cargo configs require a separate +reviewed binding. Use long option forms; only `-q`, `-v`, `-vv`, and `-V` shorthand +is accepted. Native helper and Git environment overrides also refuse. The wrapper handles fresh storage admission; refusal blocks heavy work. Its existing exception expires 2026-11-17 18:04:38 UTC and must not be extended by this launcher. diff --git a/scripts/project_runtime.py b/scripts/project_runtime.py index 6ce4052..3070055 100644 --- a/scripts/project_runtime.py +++ b/scripts/project_runtime.py @@ -10,6 +10,7 @@ import json import math import os +import shutil from pathlib import Path import subprocess import sys @@ -17,6 +18,9 @@ from verify_runtime_matrix import OVERRIDES, PROFILE, check_file, sha +HOST_TOOLS = {'cc', 'gcc', 'c++', 'ar', 'ld', 'git', 'sh', 'bash', + 'dirname', 'tr', 'sha256sum', 'cut', 'mkdir', 'mktemp'} + def load_identity(identity): return json.loads(check_file(identity).read_text()) @@ -65,15 +69,36 @@ def selection_contract(selection): check_file(item) if not manifest['native_files'] or not lane['provenance']: raise ValueError('missing native provenance') + host_path = manifest['host_path'] + if not isinstance(host_path, str) or any(not part or not Path(part).is_absolute() for part in host_path.split(os.pathsep)): + raise ValueError('qualification host PATH requires absolute entries') + if set(selection['host_tools']) != HOST_TOOLS: + raise ValueError('missing or unexpected native/helper identity') + for name, identity in selection['host_tools'].items(): + path = check_file(identity) + resolved = shutil.which(name, path=host_path) + if not resolved or Path(resolved).resolve() != path.resolve(): + raise ValueError('native/helper resolution differs from qualified PATH: ' + name) + if not os.access(path, os.X_OK): + raise ValueError('native/helper identity is not executable: ' + name) + # The actual cc/gcc used by the qualified bundled SQLite build is also bound + # by G1's native-file inventory, not merely added to a new local selection. + for name in ('cc', 'gcc'): + if selection['host_tools'][name] not in manifest['native_files']: + raise ValueError('native compiler differs from qualification: ' + name) return manifest -def runtime_environment(selection, aliases): +def runtime_environment(selection, manifest, aliases): env = dict(os.environ) forbidden = OVERRIDES | {'LD_PRELOAD', 'LD_LIBRARY_PATH', 'DYLD_INSERT_LIBRARIES', - 'CARGO_BUILD_TARGET', 'CARGO_BUILD_RUSTDOC', 'CARGO_ALIAS_TEST'} + 'CARGO_BUILD_TARGET', 'CARGO_BUILD_RUSTDOC', 'CARGO_ALIAS_TEST', + 'GIT_DIR', 'GIT_WORK_TREE', 'GIT_INDEX_FILE', 'GIT_CONFIG', + 'GIT_CONFIG_COUNT', 'GIT_CONFIG_PARAMETERS', 'BASH_ENV', 'ENV', + 'NIX_CFLAGS_COMPILE', 'NIX_LDFLAGS'} unexpected = [key for key in forbidden if env.get(key)] - unexpected += [key for key in env if key.startswith(('CARGO_TARGET_', 'CARGO_PROFILE_', 'CARGO_ALIAS_')) + unexpected += [key for key in env if key.startswith(('CARGO_TARGET_', 'CARGO_PROFILE_', 'CARGO_ALIAS_', 'CARGO_BUILD_', + 'CC_', 'CXX_', 'AR_', 'CFLAGS_', 'CXXFLAGS_', 'HOST_CC', 'HOST_CXX', 'HOST_AR')) and env[key] and env[key] != PROFILE.get(key)] if unexpected: raise ValueError('unexpected runtime overrides: ' + ', '.join(sorted(set(unexpected)))) @@ -81,7 +106,9 @@ def runtime_environment(selection, aliases): aliases.mkdir(mode=0o700) for alias, key in [('rustc', 'rustc'), ('cargo', 'cargo'), ('python3', 'python'), ('python', 'python')]: (aliases / alias).symlink_to(selection['tools'][key]['path']) - env['PATH'] = str(aliases) + os.pathsep + env.get('PATH', '') + for name, identity in selection['host_tools'].items(): + (aliases / name).symlink_to(identity['path']) + env['PATH'] = str(aliases) + os.pathsep + manifest['host_path'] env['RUSTC'] = selection['tools']['rustc']['path'] return env @@ -97,9 +124,9 @@ def python_probe(tools, env): return {'parent': parent, 'child': child} -def source_record(root): +def source_record(root, env): def git(*args): - return subprocess.check_output(['git', '-C', str(root), *args], text=True).strip() + return subprocess.check_output(['git', '-C', str(root), *args], env=env, text=True).strip() tracked = git('ls-files').splitlines() return {'root': str(root), 'commit': git('rev-parse', 'HEAD'), 'dirty': bool(git('status', '--porcelain', '--untracked-files=normal')), @@ -110,8 +137,11 @@ def git(*args): def check_cargo_arguments(args, root, env): - if not args or any(arg.startswith('+') or arg.split('=')[0] in - {'--config', '--target-dir', '--manifest-path', '--profile', '--release', '--target'} for arg in args): + allowed = {'build', 'test', 'run', 'check', 'metadata', 'locate-project', '--version', '-V', 'version'} + if (not args or args[0] not in allowed or any(arg.startswith('+') + or (arg.startswith('-') and not arg.startswith('--') and arg not in {'-V', '-q', '-v', '-vv'}) + or arg.split('=')[0] in + {'--config', '--target-dir', '--manifest-path', '--profile', '--release', '-r', '--target', '--lockfile-path'} for arg in args)): raise ValueError('Cargo requires ordinary commands without runtime/output/profile overrides') # Cargo reads every ancestor config and Cargo home; refuse hidden selectors. cargo_home = Path(env.get('CARGO_HOME', str(Path.home() / '.cargo'))).absolute() @@ -139,12 +169,12 @@ def main(): manifest = selection_contract(selection) invocation = args.state_dir.absolute() / 'invocations' / (datetime.datetime.now(datetime.timezone.utc).strftime('%Y%m%dT%H%M%S%fZ-') + uuid.uuid4().hex) invocation.mkdir(parents=True, mode=0o700) - env = runtime_environment(selection, invocation / 'bin') + env = runtime_environment(selection, manifest, invocation / 'bin') tools = {key: tool_identity(value, env) for key, value in selection['tools'].items()} probe = python_probe(tools, env) - root = Path(subprocess.check_output(['git', 'rev-parse', '--show-toplevel'], text=True).strip()).resolve() + root = Path(subprocess.check_output(['git', 'rev-parse', '--show-toplevel'], env=env, text=True).strip()).resolve() record = {'schema': 'kairotensor-project-invocation/1', 'selection': {'path': str(selection_path), 'sha256': sha(selection_path)}, - 'qualification_source': manifest['source']['commit'], 'source': source_record(root), + 'qualification_source': manifest['source']['commit'], 'source': source_record(root, env), 'host_tools': selection['host_tools'], 'tools': tools, 'python_probe': probe, 'profile': PROFILE, 'wrapper': manifest['wrapper'], 'started_at': datetime.datetime.now(datetime.timezone.utc).isoformat(), 'exit': None} receipt_path = invocation / 'receipt.json' @@ -167,7 +197,7 @@ def main(): else: print(json.dumps({'tools': tools, 'python_probe': probe, 'receipt': str(receipt_path)}, indent=2)) code = 0 - for value in tools.values(): + for value in [*tools.values(), *selection['host_tools'].values()]: check_file(value) record['exit'] = code record['finished_at'] = datetime.datetime.now(datetime.timezone.utc).isoformat() diff --git a/scripts/test_project_runtime.py b/scripts/test_project_runtime.py index 838b682..c7a68fc 100644 --- a/scripts/test_project_runtime.py +++ b/scripts/test_project_runtime.py @@ -2,6 +2,7 @@ import hashlib import json import os +import shutil from pathlib import Path import subprocess import sys @@ -43,13 +44,15 @@ def setUp(self): python = Path(sys.executable).resolve() self.tools['python'] = dict(identity(python), version=subprocess.check_output([str(python), '--version'], text=True).strip(), version_args=['--version']) self.wrapper = self.root / 'wrapper' - self.wrapper.write_text('#!' + str(python) + '\nimport json,sys\nprint(json.dumps(sys.argv[1:]))\nsys.exit(17 if "deny" in sys.argv else 0)\n') + self.wrapper.write_text('#!' + str(python) + '\nimport json,sys\nprint(json.dumps(sys.argv[1:]))\nsys.exit(17 if "--deny" in sys.argv else 0)\n') self.wrapper.chmod(0o700) self.review = self.write('review.md', 'independent fixture review\n') self.acceptance = self.write('acceptance.json', {'result': 'G1 accepted', 'accepted_executable_source': self.commit, 'independent_review': identity(self.review)}) - self.manifest = {'source': {'commit': self.commit}, 'lanes': [{'role': 'stable', **self.tools, 'provenance': [identity(self.review)]}], - 'native_files': [identity(self.review)], 'wrapper': identity(self.wrapper), + self.host_tools = {name: identity(Path(shutil.which(name, path=self.env['PATH'])).resolve()) + for name in ['cc', 'gcc', 'c++', 'ar', 'ld', 'git', 'sh', 'bash', 'dirname', 'tr', 'sha256sum', 'cut', 'mkdir', 'mktemp']} + self.manifest = {'host_path': self.env['PATH'], 'source': {'commit': self.commit}, 'lanes': [{'role': 'stable', **self.tools, 'provenance': [identity(self.review)]}], + 'native_files': [identity(self.review), self.host_tools['cc']], 'wrapper': identity(self.wrapper), 'storage_guard': identity(self.review), 'cargo_recipe': identity(self.review)} self.manifest_path = self.write('manifest.json', self.manifest) self.receipt = {'status': 'Pass', 'source': self.manifest['source'], 'role': 'stable', @@ -58,7 +61,7 @@ def setUp(self): for t in self.receipt['tools'].values(): t['resolved_path'] = str(Path(t['path']).resolve()) self.receipt_path = self.write('receipt.json', self.receipt) - self.selection = {'schema': 'kairotensor-project-runtime/1', 'role': 'stable', 'tools': self.tools, + self.selection = {'schema': 'kairotensor-project-runtime/1', 'role': 'stable', 'tools': self.tools, 'host_tools': self.host_tools, 'qualification_manifest': identity(self.manifest_path), 'qualification_receipt': identity(self.receipt_path), 'independent_review': identity(self.review), 'acceptance': identity(self.acceptance)} self.selection_path = self.root / 'selection.json' @@ -144,7 +147,7 @@ def test_cargo_routes_through_wrapper_with_budget(self): self.assertEqual(self.latest()['command'][0], str(self.wrapper)) def test_admission_failure_is_propagated(self): - r = self.run_launcher('--budget-gib', '2', 'cargo', 'deny') + r = self.run_launcher('--budget-gib', '2', 'cargo', 'test', '--deny') self.assertEqual(r.returncode, 17, r.stderr) self.assertEqual(self.latest()['exit'], 17) @@ -158,7 +161,7 @@ def test_inherited_runtime_overrides_refuse(self): self.assertEqual(r.returncode, 2, r.stderr) def test_cargo_command_overrides_refuse(self): - for args in [('+nightly', 'test'), ('test', '--config=build.rustc="bad"'), ('test', '--target-dir', '/tmp/bad')]: + for args in [('+nightly', 'test'), ('test', '--config=build.rustc="bad"'), ('test', '--target-dir', '/tmp/bad'), ('build', '-r'), ('build', '-qr'), ('rustc', '--', '-C', 'opt-level=3'), ('install', 'anything'), ('-C', '/tmp', 'build')]: with self.subTest(args=args): self.assertEqual(self.run_launcher('--budget-gib', '2', 'cargo', *args).returncode, 2) @@ -167,6 +170,28 @@ def test_cargo_config_refuses(self): (self.repo / '.cargo/config.toml').write_text('[build]\nrustc="unexpected"\n') self.assertEqual(self.run_launcher('--budget-gib', '2', 'cargo', 'test').returncode, 2) + def test_shadowed_ambient_native_and_git_commands_are_not_used(self): + shadow = self.root / 'shadow' + shadow.mkdir() + for name in ['cc', 'git']: + fake = shadow / name + fake.write_text('#!/bin/sh\necho shadowed\nexit 63\n') + fake.chmod(0o700) + env = dict(self.env, PATH=str(shadow) + os.pathsep + self.env['PATH']) + r = self.run_launcher('python', '-c', 'import subprocess; print(subprocess.check_output(["cc", "--version"], text=True))', env=env) + self.assertEqual(r.returncode, 0, r.stderr) + self.assertNotIn('shadowed', r.stdout) + self.assertEqual(self.latest()['host_tools']['cc'], self.host_tools['cc']) + + def test_host_helper_digest_mismatch_refuses(self): + self.selection['host_tools']['git']['sha256'] = '0' * 64 + r = self.run_launcher('identity') + self.assertEqual(r.returncode, 2, r.stderr) + + def test_git_environment_source_override_refuses(self): + r = self.run_launcher('identity', env=dict(self.env, GIT_DIR=str(self.repo / '.git'))) + self.assertEqual(r.returncode, 2, r.stderr) + def test_child_command_exit_is_preserved(self): r = self.run_launcher('python', '-c', 'raise SystemExit(7)') self.assertEqual(r.returncode, 7, r.stderr) From 9d94dbf655f3d49f334390e627343bf77c70901a Mon Sep 17 00:00:00 2001 From: Alejandro Castellanos <5405628+k2jac9@users.noreply.github.com> Date: Sun, 15 Nov 2026 22:35:57 -0500 Subject: [PATCH 40/51] Record final runtime launcher qualification and activation handoff --- docs/runtime-adoption.md | 50 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 50 insertions(+) diff --git a/docs/runtime-adoption.md b/docs/runtime-adoption.md index 94d1ac9..b1a005a 100644 --- a/docs/runtime-adoption.md +++ b/docs/runtime-adoption.md @@ -473,3 +473,53 @@ S0 source `794f64e5f9702987cd2502b86776b28f8a8d97c5`, exact Rust `1.99.1` / Python `3.15.2` closure and G1's exercised rollback procedure; switching a pointer does not undo data mutations. The G2 probe uses a fresh disposable database and never mutates the original rollback fixture. + + +### Final G2 local verification and source mapping (2026-11-16) + +Executable/configuration source is `64e0b1b4a45857f419ac9539760018e3890d9aa3`. It supersedes +initial S2 `ae4aa0865912ac5f991a08ddfe379e5a180f7bfa`: root identified that ambient +PATH could shadow native/Git helpers. Regression tests reproduced that problem; +the successor binds verified helper aliases from qualification's frozen host PATH +and also rejects Cargo shorthand/forwarded selector bypasses. Both sources and +all their evidence are retained. The final docs-only successor changes only this +file and is mapped in the packet's `docs-only-transition.json`; no old receipt is +relabelled as evidence for new executable code. + +Final evidence is retained under +`/home/neo/Projects/kairotensor-artifacts/reliability-implementation-20261115/task-g2/successor-r1/`. +`verification-rollup.json` binds the complete source, actual command records, +logs, candidate selections and probe identities. The stable manifest and CLI +receipt revalidation passed with the unchanged 40-command G1 workload, all exits +zero: 357 Rust tests, 18 reference tests, 8 oracle tests, 361 full-discovery Python +tests, and explicit suites of 24/50/15/19/7 tests, with zero skips. Counts across +those overlapping commands total 859 executions; they are not 859 unique tests. +Five selected runtime pairs each passed an actual identity/child-Python check and +18 focused launcher tests (90 additional test executions). Other full role suites +remain attributed to accepted G1 source; no final-source five-role full matrix or +hosted execution is claimed. + +CI YAML was parsed, its shell and Python probe syntax checked, five independent +roles verified, and all 16 original command steps plus checkout pin compared +unchanged. Actionlint was unavailable locally and was not downloaded. Actual +hosted action execution remains open. All 196 protected repository files, four +owner rule files and bound intake inputs still matched; the retained rollback +DBs, eight Nix roots/closures and host defaults were verified unchanged. + +The accepted-pointer candidate is `/home/neo/.local/state/kairotensor/runtime/candidates/g2-20261116-r1/stable.json`, +SHA-256 `aaa7438718dbed6e6f9b7ac50dd7fbb02317278c18a8f093a431d488ba998321`. It is prepared for independent review and root +activation; `accepted-selection.json` remained absent at worker closeout. +The final candidate-built activation binary is `/home/neo/Projects/kairotensor-artifacts/reliability-implementation-20261115/task-g2/successor-r1/activation-probe/activation-probe`, +SHA-256 `275b94eef822d82bbfcfa3be1b2e7b0d40a6fb49d737a2ad59cde918e6b8c666`. Its source is the intake-authorized exact +copy of G1's probe. Library build used the candidate launcher and approved Cargo +wrapper; standalone compile used separate fresh storage admission. Disposable +create/reopen checks passed exact current bytes, spent=7, reserved=12, unresolved +attempt=2, claim=None, cancel=false, and zero backend calls. This is preactivation +candidate health; root must run the post-activation check on a new database. + +The initial probe compile failed (exit 1, E0463): its old harness assumed +`debug/deps`, while Cargo 1.100 emitted per-package `debug/build/*/*/out` +dependency directories. The failed receipt remains under `activation-probe/`. +An artifact-only correction used actual Cargo-emitted paths; both its retained +S2 retry and the final successor probe passed. No application source, dependency, +storage guard or acceptance test was changed to clear the failure. From e32c96814bbf0e841b9bacda7b5edc241e72fbf4 Mon Sep 17 00:00:00 2001 From: Alejandro Castellanos <5405628+k2jac9@users.noreply.github.com> Date: Sun, 15 Nov 2026 22:48:25 -0500 Subject: [PATCH 41/51] Reject inherited native flag families and target-specific selectors --- docs/runtime-adoption.md | 27 +++++++++++++ scripts/project_runtime.py | 35 +++++++++++++++-- scripts/test_project_runtime.py | 70 +++++++++++++++++++++++++++++++++ 3 files changed, 128 insertions(+), 4 deletions(-) diff --git a/docs/runtime-adoption.md b/docs/runtime-adoption.md index b1a005a..7996bc7 100644 --- a/docs/runtime-adoption.md +++ b/docs/runtime-adoption.md @@ -523,3 +523,30 @@ dependency directories. The failed receipt remains under `activation-probe/`. An artifact-only correction used actual Cargo-emitted paths; both its retained S2 retry and the final successor probe passed. No application source, dependency, storage guard or acceptance test was changed to clear the failure. + +### G2 independent-review corrections (2026-11-16, fix round 1) + +Independent review I1 reproduced an inherited native flag outside the declared +runtime boundary: the selected Nix gcc wrapper consumed +`NIX_CFLAGS_COMPILE_x86_64_unknown_linux_gnu` while its executable hash stayed +unchanged. Acceptance and activation remained held. The correction rejects +native input families before creating execution aliases, including arbitrary +Nix target/role suffixes for compile/before/link flags, C++/GNAT flags, hardening, +wrapper role/initialization controls, linker selection, rpath/build-ID/purity and +response-file controls. It also rejects the cc crate's bare, HOST_/TARGET_, raw +and normalized target-specific compiler/archiver/tool/flag selectors, native +search-path controls and the bundled SQLite configuration inputs. Empty native +selectors are refused too because some explicitly disable defaults. No target +triple is hardcoded into the refusal policy. Selection schema and accepted +G1/storage code are unchanged; this remains an invocation boundary rather than +a sandbox for arbitrary workload code. Exact audited native source identities, +pre-fix preprocessing reproduction and RED/GREEN results are in `task-g2/fix-1/`. + +Review M1 also found retained compiler warnings that the prior summary omitted: +17 warning lines, including repeated diagnostics and Cargo-generated summaries, +cover three existing dead-code categories in the protected shared examples: +unused `Prepared`, unused `PreparedReply`, and unused associated methods. +Passing exits do not mean warning-free builds. These warnings remain visible in +the raw Rust suite and handoff logs. Application cleanup is explicitly deferred +to the integrated review owner; G2 neither changes protected application code +nor suppresses warnings or weakens acceptance gates. diff --git a/scripts/project_runtime.py b/scripts/project_runtime.py index 3070055..b9569de 100644 --- a/scripts/project_runtime.py +++ b/scripts/project_runtime.py @@ -22,6 +22,34 @@ 'dirname', 'tr', 'sha256sum', 'cut', 'mkdir', 'mktemp'} +# Selected Nix cc/ld wrappers accept both role and arbitrary target suffixes. +# cc-rs likewise accepts bare inputs, HOST_/TARGET_ inputs and target suffixes. +# Match families rather than a host triple. Presence matters: an empty selector +# such as CXXSTDLIB can disable a native default, so it is not equivalent to unset. +NATIVE_INPUT_FAMILIES = { + 'CC', 'CXX', 'CPP', 'AR', 'RANLIB', 'NVCC', 'AS', 'LD', + 'CXXSTDLIB', 'CFLAGS', 'CXXFLAGS', 'CPPFLAGS', 'ARFLAGS', 'RANLIBFLAGS', + 'ASFLAGS', 'LDFLAGS', 'CRATE_CC_NO_DEFAULTS', 'CROSS_COMPILE', 'RUSTC_LINKER', + 'GCC_EXEC_PREFIX', 'COMPILER_PATH', 'CPATH', 'C_INCLUDE_PATH', + 'CPLUS_INCLUDE_PATH', 'OBJC_INCLUDE_PATH', 'LIBRARY_PATH', + 'DEPENDENCIES_OUTPUT', 'SUNPRO_DEPENDENCIES', 'SOURCE_DATE_EPOCH', + 'LIBSQLITE3', 'SQLITE_MAX', 'SQLITE3', 'SQLCIPHER', 'PKG_CONFIG', + 'NIX_CFLAGS', 'NIX_LDFLAGS', 'NIX_CXXSTDLIB', 'NIX_GNATFLAGS', + 'NIX_CC_WRAPPER', 'NIX_BINTOOLS_WRAPPER', 'NIX_HARDENING', 'NIX_ENFORCE', + 'NIX_DYNAMIC_LINKER', 'NIX_IGNORE_LD_THROUGH_GCC', 'NIX_LINK_TYPE', + 'NIX_DONT_SET_RPATH', 'NIX_SET_BUILD_ID', 'NIX_BUILD_ID_STYLE', + 'NIX_CC_USE_RESPONSE_FILE', 'NIX_LD_USE_RESPONSE_FILE', 'NIX_DEBUG', + 'NIX_STORE', 'NIX_BUILD_TOP', +} + + +def native_override(name): + if name.startswith(('HOST_', 'TARGET_')): + name = name.split('_', 1)[1] + return any(name == family or name.startswith(family + '_') + for family in NATIVE_INPUT_FAMILIES) + + def load_identity(identity): return json.loads(check_file(identity).read_text()) @@ -94,11 +122,10 @@ def runtime_environment(selection, manifest, aliases): forbidden = OVERRIDES | {'LD_PRELOAD', 'LD_LIBRARY_PATH', 'DYLD_INSERT_LIBRARIES', 'CARGO_BUILD_TARGET', 'CARGO_BUILD_RUSTDOC', 'CARGO_ALIAS_TEST', 'GIT_DIR', 'GIT_WORK_TREE', 'GIT_INDEX_FILE', 'GIT_CONFIG', - 'GIT_CONFIG_COUNT', 'GIT_CONFIG_PARAMETERS', 'BASH_ENV', 'ENV', - 'NIX_CFLAGS_COMPILE', 'NIX_LDFLAGS'} + 'GIT_CONFIG_COUNT', 'GIT_CONFIG_PARAMETERS', 'BASH_ENV', 'ENV'} unexpected = [key for key in forbidden if env.get(key)] - unexpected += [key for key in env if key.startswith(('CARGO_TARGET_', 'CARGO_PROFILE_', 'CARGO_ALIAS_', 'CARGO_BUILD_', - 'CC_', 'CXX_', 'AR_', 'CFLAGS_', 'CXXFLAGS_', 'HOST_CC', 'HOST_CXX', 'HOST_AR')) + unexpected += [key for key in env if native_override(key)] + unexpected += [key for key in env if key.startswith(('CARGO_TARGET_', 'CARGO_PROFILE_', 'CARGO_ALIAS_', 'CARGO_BUILD_')) and env[key] and env[key] != PROFILE.get(key)] if unexpected: raise ValueError('unexpected runtime overrides: ' + ', '.join(sorted(set(unexpected)))) diff --git a/scripts/test_project_runtime.py b/scripts/test_project_runtime.py index c7a68fc..d1ba809 100644 --- a/scripts/test_project_runtime.py +++ b/scripts/test_project_runtime.py @@ -8,6 +8,9 @@ import sys import tempfile import unittest +from unittest.mock import patch + +import project_runtime LAUNCHER = Path(__file__).with_name('project_runtime.py').resolve() @@ -192,6 +195,73 @@ def test_git_environment_source_override_refuses(self): r = self.run_launcher('identity', env=dict(self.env, GIT_DIR=str(self.repo / '.git'))) self.assertEqual(r.returncode, 2, r.stderr) + def assert_native_overrides_refused(self, cases): + for index, (key, value) in enumerate(cases): + with self.subTest(key=key, value=value): + aliases = self.root / ('native-refusal-' + str(index)) + with patch.dict(os.environ, dict(self.env, **{key: value}), clear=True): + with self.assertRaisesRegex(ValueError, 'unexpected runtime overrides') as caught: + project_runtime.runtime_environment(self.selection, self.manifest, aliases) + self.assertIn(key, str(caught.exception)) + self.assertFalse(aliases.exists(), 'refuse before creating selected execution aliases') + + def test_nix_native_flags_and_wrapper_controls_refuse_all_target_forms(self): + # Inputs audited from selected GCC/ld add-flags, utils and hardening scripts. + families = [ + 'NIX_CFLAGS_COMPILE', 'NIX_CFLAGS_COMPILE_BEFORE', 'NIX_CFLAGS_LINK', + 'NIX_LDFLAGS', 'NIX_LDFLAGS_BEFORE', 'NIX_LDFLAGS_AFTER', 'NIX_LDFLAGS_HARDEN', + 'NIX_CXXSTDLIB_COMPILE', 'NIX_CXXSTDLIB_LINK', 'NIX_GNATFLAGS_COMPILE', + 'NIX_HARDENING_ENABLE', 'NIX_ENFORCE_NO_NATIVE', 'NIX_DYNAMIC_LINKER', + 'NIX_IGNORE_LD_THROUGH_GCC', 'NIX_LINK_TYPE', 'NIX_DONT_SET_RPATH', + 'NIX_SET_BUILD_ID', 'NIX_CC_WRAPPER_FLAGS_SET', 'NIX_BINTOOLS_WRAPPER_FLAGS_SET', + 'NIX_CC_WRAPPER_TARGET_BUILD', 'NIX_CC_WRAPPER_TARGET_HOST', 'NIX_CC_WRAPPER_TARGET_TARGET', + 'NIX_BINTOOLS_WRAPPER_TARGET_BUILD', 'NIX_BINTOOLS_WRAPPER_TARGET_HOST', + 'NIX_BINTOOLS_WRAPPER_TARGET_TARGET', + ] + suffixes = ['', '_x86_64_unknown_linux_gnu', '_aarch64_unknown_linux_gnu', '_FOR_BUILD', '_FOR_TARGET'] + cases = [(name + suffix, '1') for name in families for suffix in suffixes] + cases += [(name, '1') for name in ['NIX_CC_USE_RESPONSE_FILE', 'NIX_LD_USE_RESPONSE_FILE', + 'NIX_BUILD_ID_STYLE', 'NIX_DEBUG', 'NIX_ENFORCE_PURITY', + 'NIX_STORE', 'NIX_BUILD_TOP']] + self.assert_native_overrides_refused(cases) + + def test_native_tool_and_flag_selectors_refuse_cc_crate_target_forms(self): + # cc 1.8.0 target_envs consumes raw and underscore target suffixes, + # then HOST_/TARGET_ prefixes, then the bare variable. Empty values can + # still be meaningful (for example CXXSTDLIB disables its default). + families = ['CC', 'CXX', 'AR', 'RANLIB', 'NVCC', 'CC_MASM_ASM', + 'CXXSTDLIB', 'CXXSTDLIB_STATIC', 'CFLAGS', 'CXXFLAGS', + 'ARFLAGS', 'RANLIBFLAGS', 'LDFLAGS', 'AS', 'ASFLAGS', 'CPPFLAGS'] + cases = [] + for name in families: + cases += [(key, 'unexpected') for key in [name, name + '_x86_64-unknown-linux-gnu', + name + '_aarch64_unknown_linux_gnu', name + '_thumbv8m.main-none-eabi', + 'HOST_' + name, 'TARGET_' + name]] + cases += [('CXXSTDLIB', ''), ('CFLAGS_x86_64_unknown_linux_gnu', '')] + self.assert_native_overrides_refused(cases) + + def test_native_search_controls_and_sqlite_inputs_refuse(self): + cases = [(name, 'unexpected') for name in [ + 'CRATE_CC_NO_DEFAULTS', 'CROSS_COMPILE', 'RUSTC_LINKER', + 'CC_KNOWN_WRAPPER_CUSTOM', 'CC_FORCE_DISABLE', 'CC_SHELL_ESCAPED_FLAGS', + 'GCC_EXEC_PREFIX', 'COMPILER_PATH', 'CPATH', 'C_INCLUDE_PATH', + 'CPLUS_INCLUDE_PATH', 'OBJC_INCLUDE_PATH', 'LIBRARY_PATH', + 'DEPENDENCIES_OUTPUT', 'SUNPRO_DEPENDENCIES', 'SOURCE_DATE_EPOCH', + 'LIBSQLITE3_FLAGS', 'LIBSQLITE3_SYS_USE_PKG_CONFIG', + 'SQLITE_MAX_VARIABLE_NUMBER', 'SQLITE_MAX_EXPR_DEPTH', 'SQLITE_MAX_COLUMN', + 'SQLITE3_LIB_DIR', 'SQLITE3_INCLUDE_DIR', 'SQLITE3_STATIC', + 'PKG_CONFIG', 'PKG_CONFIG_PATH', 'TARGET_PKG_CONFIG', + ]] + self.assert_native_overrides_refused(cases) + + def test_observed_nix_override_refuses_before_workload_side_effect(self): + marker = self.root / 'should-not-run' + r = self.run_launcher('python', '-c', 'from pathlib import Path; Path(' + repr(str(marker)) + ').touch()', + env=dict(self.env, NIX_CFLAGS_COMPILE_x86_64_unknown_linux_gnu='-DUNBOUND_NATIVE_FLAG=1')) + self.assertEqual(r.returncode, 2, r.stderr) + self.assertIn('NIX_CFLAGS_COMPILE_x86_64_unknown_linux_gnu', r.stderr) + self.assertFalse(marker.exists()) + def test_child_command_exit_is_preserved(self): r = self.run_launcher('python', '-c', 'raise SystemExit(7)') self.assertEqual(r.returncode, 7, r.stderr) From 1870b604fc7861645946a894ff936b349eadbd31 Mon Sep 17 00:00:00 2001 From: Alejandro Castellanos <5405628+k2jac9@users.noreply.github.com> Date: Sun, 15 Nov 2026 22:52:15 -0500 Subject: [PATCH 42/51] Record native boundary correction verification and retained warnings --- docs/runtime-adoption.md | 40 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 40 insertions(+) diff --git a/docs/runtime-adoption.md b/docs/runtime-adoption.md index 7996bc7..ad3492e 100644 --- a/docs/runtime-adoption.md +++ b/docs/runtime-adoption.md @@ -550,3 +550,43 @@ Passing exits do not mean warning-free builds. These warnings remain visible in the raw Rust suite and handoff logs. Application cleanup is explicitly deferred to the integrated review owner; G2 neither changes protected application code nor suppresses warnings or weakens acceptance gates. + + +### G2 fix-round-1 source-bound verification + +Corrected executable source `e32c96814bbf0e841b9bacda7b5edc241e72fbf4` supersedes the earlier +`64e0b1b` proposal; earlier sources and receipts remain historical. A subsequent +docs-only commit changes only this file and is mapped in +`task-g2/fix-1/docs-only-transition.json`. Independent re-review and root activation +remain pending; no accepted pointer was written. + +`task-g2/fix-1/verification-rollup.json` binds a fresh unchanged stable +40-command workload (all exits 0, zero skips), with Rust 357, reference 18, +oracle 8, full Python discovery 365, and explicit suites 24/50/15/19/7. +These overlapping suites total 863 test executions. Five actual selected-runtime +identity probes and five focused 22-test runs also passed (110 executions). +The source-bound final negative control refused the exact reviewed suffixed +Nix flag with exit 2 before launching the selected compiler. The new regression +coverage checks 258 native-input cases, including the end-to-end observed case; +220 subcases failed in the retained pre-fix RED and all 22 tests passed GREEN. +The audited selected shell wrappers have 50 native-variable read occurrences +across the inspected files, all classified by the new refusal boundary; this is +an audit of those selected wrappers, not a claim about arbitrary external code. + +Fresh candidate library and standalone probe builds used unchanged admitted +wrappers. The final candidate binary is `/home/neo/Projects/kairotensor-artifacts/reliability-implementation-20261115/task-g2/fix-1/activation-probe/activation-probe`, +SHA-256 `275b94eef822d82bbfcfa3be1b2e7b0d40a6fb49d737a2ad59cde918e6b8c666`. Its current-read/hold create/reopen checks +passed on a new disposable database, with zero backend calls. This replaces the +earlier probe proposal for root's post-activation check. The accepted-runtime +candidate selection remains byte-identical at +`/home/neo/.local/state/kairotensor/runtime/candidates/g2-20261116-r1/stable.json`; +root must bind the corrected executable source and this binary in its own +reviewed activation receipt. + +The fresh warning audit (`task-g2/fix-1/compiler-warnings.json`) confirms the +same 17 lines, including repetitions and generated summaries, in the Rust suite +and corrected/scoped handoff stderr. The three protected-code dead-code +categories remain deferred to integrated review. All 199 files protected by the +fix intake, four owner policies, review/context identities and candidate selection +bytes were checked unchanged. Hosted CI, other full S2 roles, real-model/ML +consumers and activation remain unexecuted, as before. From d370b41ff91e9bb0af9f5be7c62cf9e7feba8353 Mon Sep 17 00:00:00 2001 From: Alejandro Castellanos <5405628+k2jac9@users.noreply.github.com> Date: Sun, 15 Nov 2026 22:52:49 -0500 Subject: [PATCH 43/51] Clarify units of the retained native wrapper audit --- docs/runtime-adoption.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/runtime-adoption.md b/docs/runtime-adoption.md index ad3492e..f13f373 100644 --- a/docs/runtime-adoption.md +++ b/docs/runtime-adoption.md @@ -569,8 +569,8 @@ The source-bound final negative control refused the exact reviewed suffixed Nix flag with exit 2 before launching the selected compiler. The new regression coverage checks 258 native-input cases, including the end-to-end observed case; 220 subcases failed in the retained pre-fix RED and all 22 tests passed GREEN. -The audited selected shell wrappers have 50 native-variable read occurrences -across the inspected files, all classified by the new refusal boundary; this is +The audited selected shell wrappers expose 50 distinct native-variable-name/file pairs +in the inspected files, all classified by the new refusal boundary; this is an audit of those selected wrappers, not a claim about arbitrary external code. Fresh candidate library and standalone probe builds used unchanged admitted From 5e9c76e0b95fa2559fc594523fd9739263577a1b Mon Sep 17 00:00:00 2001 From: Alejandro Castellanos <5405628+k2jac9@users.noreply.github.com> Date: Sun, 15 Nov 2026 23:11:34 -0500 Subject: [PATCH 44/51] Bind rustdoc selection and version unknown-unit replay observations --- scripts/project_runtime.py | 23 ++++++- scripts/test_project_runtime.py | 55 +++++++++++++++- src/decision_bundle.rs | 4 +- src/decision_bundle/replay.rs | 10 ++- src/decision_bundle/types.rs | 9 +++ tests/decision_bundle.rs | 107 ++++++++++++++++++++++++++++++-- 6 files changed, 195 insertions(+), 13 deletions(-) diff --git a/scripts/project_runtime.py b/scripts/project_runtime.py index b9569de..73d2fa9 100644 --- a/scripts/project_runtime.py +++ b/scripts/project_runtime.py @@ -67,8 +67,10 @@ def tool_identity(tool, env): def selection_contract(selection): - if selection['schema'] != 'kairotensor-project-runtime/1': + if selection['schema'] != 'kairotensor-project-runtime/2': raise ValueError('unknown selection schema') + if set(selection['tools']) != {'rustc', 'cargo', 'python', 'rustdoc'}: + raise ValueError('missing or unexpected runtime tool identity') manifest = load_identity(selection['qualification_manifest']) receipt = load_identity(selection['qualification_receipt']) acceptance = load_identity(selection['acceptance']) @@ -92,6 +94,17 @@ def selection_contract(selection): recorded = receipt['tools'][key] if any(recorded[field] != expected[field] for field in ('path', 'sha256', 'version')): raise ValueError('qualification tool identity mismatch') + # Supplementary V2 binding: G1 never exercised or recorded rustdoc. + rustc = selection['tools']['rustc'] + rustdoc = selection['tools']['rustdoc'] + sibling = Path(rustc['path']).with_name('rustdoc') + expected_version = rustc['version'].splitlines()[0].replace('rustc ', 'rustdoc ', 1) + if (set(rustdoc) != {'path', 'sha256', 'version', 'version_args'} + or rustdoc['path'] != str(sibling) + or rustdoc['version_args'] != ['--version'] + or rustdoc['version'] != expected_version): + raise ValueError('rustdoc identity must bind selected rustc sibling and version') + check_file(rustdoc) for item in [*manifest['native_files'], *lane['provenance'], *(manifest[key] for key in ('wrapper', 'storage_guard', 'cargo_recipe'))]: check_file(item) @@ -123,7 +136,9 @@ def runtime_environment(selection, manifest, aliases): 'CARGO_BUILD_TARGET', 'CARGO_BUILD_RUSTDOC', 'CARGO_ALIAS_TEST', 'GIT_DIR', 'GIT_WORK_TREE', 'GIT_INDEX_FILE', 'GIT_CONFIG', 'GIT_CONFIG_COUNT', 'GIT_CONFIG_PARAMETERS', 'BASH_ENV', 'ENV'} + rustdoc_inputs = {'RUSTDOC', 'RUSTDOCFLAGS', 'CARGO_ENCODED_RUSTDOCFLAGS', 'CARGO_BUILD_RUSTDOC'} unexpected = [key for key in forbidden if env.get(key)] + unexpected += [key for key in rustdoc_inputs if key in env] unexpected += [key for key in env if native_override(key)] unexpected += [key for key in env if key.startswith(('CARGO_TARGET_', 'CARGO_PROFILE_', 'CARGO_ALIAS_', 'CARGO_BUILD_')) and env[key] and env[key] != PROFILE.get(key)] @@ -131,12 +146,13 @@ def runtime_environment(selection, manifest, aliases): raise ValueError('unexpected runtime overrides: ' + ', '.join(sorted(set(unexpected)))) env.update(PROFILE) aliases.mkdir(mode=0o700) - for alias, key in [('rustc', 'rustc'), ('cargo', 'cargo'), ('python3', 'python'), ('python', 'python')]: + for alias, key in [('rustc', 'rustc'), ('rustdoc', 'rustdoc'), ('cargo', 'cargo'), ('python3', 'python'), ('python', 'python')]: (aliases / alias).symlink_to(selection['tools'][key]['path']) for name, identity in selection['host_tools'].items(): (aliases / name).symlink_to(identity['path']) env['PATH'] = str(aliases) + os.pathsep + manifest['host_path'] env['RUSTC'] = selection['tools']['rustc']['path'] + env['RUSTDOC'] = selection['tools']['rustdoc']['path'] return env @@ -202,7 +218,8 @@ def main(): root = Path(subprocess.check_output(['git', 'rev-parse', '--show-toplevel'], env=env, text=True).strip()).resolve() record = {'schema': 'kairotensor-project-invocation/1', 'selection': {'path': str(selection_path), 'sha256': sha(selection_path)}, 'qualification_source': manifest['source']['commit'], 'source': source_record(root, env), 'host_tools': selection['host_tools'], - 'tools': tools, 'python_probe': probe, 'profile': PROFILE, 'wrapper': manifest['wrapper'], + 'tools': tools, 'supplementary_tools': {'rustdoc': 'selection-v2; not historical G1 qualification'}, + 'python_probe': probe, 'profile': PROFILE, 'wrapper': manifest['wrapper'], 'started_at': datetime.datetime.now(datetime.timezone.utc).isoformat(), 'exit': None} receipt_path = invocation / 'receipt.json' if args.command == 'identity': diff --git a/scripts/test_project_runtime.py b/scripts/test_project_runtime.py index d1ba809..2032d4e 100644 --- a/scripts/test_project_runtime.py +++ b/scripts/test_project_runtime.py @@ -64,9 +64,13 @@ def setUp(self): for t in self.receipt['tools'].values(): t['resolved_path'] = str(Path(t['path']).resolve()) self.receipt_path = self.write('receipt.json', self.receipt) - self.selection = {'schema': 'kairotensor-project-runtime/1', 'role': 'stable', 'tools': self.tools, 'host_tools': self.host_tools, + self.selection = {'schema': 'kairotensor-project-runtime/2', 'role': 'stable', 'tools': self.tools, 'host_tools': self.host_tools, 'qualification_manifest': identity(self.manifest_path), 'qualification_receipt': identity(self.receipt_path), 'independent_review': identity(self.review), 'acceptance': identity(self.acceptance)} + rustdoc = self.root / 'rustdoc' + rustdoc.write_text('#!/bin/sh\nprintf "%s\\n" "rustdoc fixture"\n') + rustdoc.chmod(0o700) + self.tools['rustdoc'] = dict(identity(rustdoc), version='rustdoc fixture', version_args=['--version']) self.selection_path = self.root / 'selection.json' def write(self, name, value): @@ -83,6 +87,55 @@ def run_launcher(self, *args, env=None): def latest(self): return json.loads(sorted(self.state.glob('invocations/*/receipt.json'))[-1].read_text()) + def test_selected_rustdoc_shadows_host_and_is_recorded(self): + shadow = self.root / 'shadow-rustdoc' + shadow.mkdir() + fake = shadow / 'rustdoc' + fake.write_text('#!/bin/sh\necho host-shadow\nexit 63\n') + fake.chmod(0o700) + env = dict(self.env, PATH=str(shadow) + os.pathsep + self.env['PATH']) + r = self.run_launcher('python', '-c', 'import os,shutil,subprocess,json; print(json.dumps([os.environ["RUSTDOC"],os.path.realpath(shutil.which("rustdoc")),subprocess.check_output(["rustdoc","--version"],text=True).strip(),os.environ["RUSTC"]]))', env=env) + self.assertEqual(r.returncode, 0, r.stderr) + self.assertEqual(json.loads(r.stdout), [self.tools['rustdoc']['path'], self.tools['rustdoc']['path'], 'rustdoc fixture', self.tools['rustc']['path']]) + self.assertEqual(self.latest()['tools']['rustdoc']['sha256'], self.tools['rustdoc']['sha256']) + + def test_rustdoc_binding_refuses_missing_hash_version_path_and_args(self): + original = dict(self.tools['rustdoc']) + cases = [('sha256', '0' * 64), ('version', 'rustdoc wrong'), ('path', self.tools['cargo']['path']), ('version_args', ['-vV'])] + for key, value in cases: + with self.subTest(key=key): + self.tools['rustdoc'] = dict(original, **{key: value}) + self.assertEqual(self.run_launcher('identity').returncode, 2) + self.tools['rustdoc'] = original + Path(original['path']).unlink() + self.assertEqual(self.run_launcher('identity').returncode, 2) + + def test_rustdoc_actual_version_is_verified_after_hash_rebind(self): + path = Path(self.tools['rustdoc']['path']) + path.write_text('#!/bin/sh\necho "rustdoc wrong"\n') + self.tools['rustdoc'].update(identity(path)) + r = self.run_launcher('identity') + self.assertEqual(r.returncode, 2) + self.assertIn('version mismatch', r.stderr) + + def test_runtime_tool_set_is_exact(self): + for mode in ['missing', 'additional']: + with self.subTest(mode=mode): + original = dict(self.selection['tools']) + if mode == 'missing': + del self.selection['tools']['rustdoc'] + else: + self.selection['tools']['unqualified'] = self.tools['cargo'] + self.assertEqual(self.run_launcher('identity').returncode, 2) + self.selection['tools'] = original + + def test_old_selection_requires_explicit_migration(self): + self.selection['schema'] = 'kairotensor-project-runtime/1' + self.assertEqual(self.run_launcher('identity').returncode, 2) + + def test_rustdoc_selectors_and_flags_refuse_before_aliases(self): + self.assert_native_overrides_refused([(key, value) for key in ['RUSTDOC', 'RUSTDOCFLAGS', 'CARGO_ENCODED_RUSTDOCFLAGS', 'CARGO_BUILD_RUSTDOC'] for value in ['unexpected', '']]) + def test_selected_parent_and_child_python(self): r = self.run_launcher('python', '-c', 'import subprocess; print(subprocess.check_output(["python3", "--version"], text=True).strip())') self.assertEqual(r.returncode, 0, r.stderr) diff --git a/src/decision_bundle.rs b/src/decision_bundle.rs index 201206c..85b1118 100644 --- a/src/decision_bundle.rs +++ b/src/decision_bundle.rs @@ -508,12 +508,12 @@ fn check_bundle(b: &DecisionBundleV1) -> Result { return Err(Error::InconsistentLedger); } value["integrity"]["sha256"] = b.integrity.sha256.clone().into(); - // Aggregate arithmetic checked independently for each unit; no global budget totals. + // Only established common units admit aggregate arithmetic. Unknown observations + // retain their per-attempt bounds above without inventing a shared dimension. for unit in [ AccountingUnit::LogicalSupport, AccountingUnit::FixtureBytes, AccountingUnit::ModelTokens, - AccountingUnit::Unknown, ] { let (mut charge, mut held) = (0i64, 0i64); for a in b.attempts.iter().filter(|a| a.accounting.unit == unit) { diff --git a/src/decision_bundle/replay.rs b/src/decision_bundle/replay.rs index 79f208f..f50e470 100644 --- a/src/decision_bundle/replay.rs +++ b/src/decision_bundle/replay.rs @@ -50,6 +50,7 @@ pub fn reevaluate_bundle(bundle: &DecisionBundleV1, validator: &str) -> ReplayRe } fn replay(b: &DecisionBundleV1, new_validator: Option<&str>) -> ReplayReport { let mut result = ReplayReport { + report_schema: "decision-replay-report/2", consistency: ReplayStatus::Mismatch, reconstruction: ReplayStatus::NotReproducible, validation: ReplayStatus::NotReproducible, @@ -57,6 +58,7 @@ fn replay(b: &DecisionBundleV1, new_validator: Option<&str>) -> ReplayReport { validator_provenance: None, missing_requirements: vec![], accounting: vec![], + unknown_accounting: vec![], source_bundle_sha256: None, reevaluated_validator: None, attempts: vec![], @@ -78,8 +80,12 @@ fn replay(b: &DecisionBundleV1, new_validator: Option<&str>) -> ReplayReport { let unit = a.accounting.unit; if unit == AccountingUnit::Unknown { missing(&mut result, MissingRequirement::UnknownAccountingUnit); - } - if let Some(t) = result.accounting.iter_mut().find(|t| t.unit == unit) { + result.unknown_accounting.push(UnknownAccountingObservation { + attempt_id: a.id, + charged: a.accounting.charge, + held: a.accounting.hold_bound, + }); + } else if let Some(t) = result.accounting.iter_mut().find(|t| t.unit == unit) { t.charged += a.accounting.charge; t.held += a.accounting.hold_bound; } else { diff --git a/src/decision_bundle/types.rs b/src/decision_bundle/types.rs index f3a64cd..c50419e 100644 --- a/src/decision_bundle/types.rs +++ b/src/decision_bundle/types.rs @@ -273,6 +273,13 @@ pub struct UnitAccounting { pub charged: i64, pub held: i64, } +/// An original ledger observation without an established shared measurement unit. +#[derive(Clone, Debug, PartialEq, Eq, Serialize)] +pub struct UnknownAccountingObservation { + pub attempt_id: i64, + pub charged: i64, + pub held: i64, +} #[derive(Clone, Debug, PartialEq, Eq, Serialize)] pub struct AttemptReplay { pub attempt_id: i64, @@ -283,6 +290,7 @@ pub struct AttemptReplay { } #[derive(Clone, Debug, PartialEq, Eq, Serialize)] pub struct ReplayReport { + pub report_schema: &'static str, pub consistency: ReplayStatus, pub reconstruction: ReplayStatus, pub validation: ReplayStatus, @@ -290,6 +298,7 @@ pub struct ReplayReport { pub validator_provenance: Option, pub missing_requirements: Vec, pub accounting: Vec, + pub unknown_accounting: Vec, pub source_bundle_sha256: Option, pub reevaluated_validator: Option, pub attempts: Vec, diff --git a/tests/decision_bundle.rs b/tests/decision_bundle.rs index 25589cf..c087e41 100644 --- a/tests/decision_bundle.rs +++ b/tests/decision_bundle.rs @@ -46,6 +46,103 @@ fn report_fixture() -> Value { a["accounting"]["unit"] = "logical_support".into(); v } +// Distinct opaque computations can have unrelated measurement dimensions. +fn accounting_pair(first: i64, second: i64) -> Value { + let mut v = fixture(); + v["attempts"][0]["accounting"]["charge"] = first.into(); + v["attempts"][0]["delivery"]["usage"] = first.into(); + let mut next = v["attempts"][0].clone(); + next["id"] = 2.into(); + next["branch"] = 2.into(); + next["inherited_from"] = 1.into(); + next["parent"] = 1.into(); + next["accounting"]["charge"] = second.into(); + next["delivery"]["usage"] = second.into(); + v["attempts"].as_array_mut().unwrap().push(next); + v +} +#[test] +fn unknown_observations_are_ordered_per_attempt_not_summed() { + let raw = seal(accounting_pair(10, 20)); + let bundle = parse_decision_bundle(&raw).unwrap(); + let report = validate_recorded_bundle(&bundle); + let json = serde_json::to_value(&report).unwrap(); + assert_eq!(json["report_schema"], "decision-replay-report/2"); + assert_eq!(json["accounting"], json!([])); + assert_eq!(json["unknown_accounting"], json!([ + {"attempt_id":1,"charged":10,"held":0}, {"attempt_id":2,"charged":20,"held":0} + ])); + assert!(report.missing_requirements.contains(&MissingRequirement::UnknownAccountingUnit)); + assert_eq!(bundle.to_json().unwrap(), raw); +} +#[test] +fn independent_large_unknown_charges_do_not_overflow_an_invented_total() { + let bundle = parsed(accounting_pair(i64::MAX, 1)); + let report = serde_json::to_value(validate_recorded_bundle(&bundle)).unwrap(); + assert_eq!(report["consistency"], "matched"); + assert_eq!(report["accounting"], json!([])); + assert_eq!(report["unknown_accounting"], json!([ + {"attempt_id":1,"charged":i64::MAX,"held":0}, {"attempt_id":2,"charged":1,"held":0} + ])); +} +#[test] +fn unknown_holds_remain_independent_and_known_totals_survive() { + let mut v = accounting_pair(0, 0); + for (attempt, hold) in v["attempts"].as_array_mut().unwrap().iter_mut().zip([i64::MAX, 1]) { + attempt["delivery"] = json!({"state":"unresolved"}); + attempt["inherited_from"] = Value::Null; + attempt["accounting"] = json!({"unit":"unknown","quote":{"input_tokens":0,"output_cap":hold},"granted":true,"budget_reference":1,"charge":0,"hold_bound":hold}); + } + let mut known = fixture()["attempts"][0].clone(); + known["inputs"][0]["role"] = "evidence".into(); + known["accounting"]["unit"] = "logical_support".into(); + known["support"] = json!({"version":"support/v1","rule_id":999,"verdict":true,"clauses":[{"clause_id":1,"premises":[{"evidence_id":1,"actor":"a","scope":"s"}]}]}); + known["id"] = 3.into(); + known["branch"] = 3.into(); + known["accounting"]["charge"] = 7.into(); + known["delivery"]["usage"] = 7.into(); + v["attempts"].as_array_mut().unwrap().push(known); + let report = serde_json::to_value(validate_recorded_bundle(&parsed(v))).unwrap(); + assert_eq!(report["consistency"], "matched"); + assert_eq!(report["accounting"], json!([{"unit":"logical_support","charged":7,"held":0}])); + assert_eq!(report["unknown_accounting"], json!([ + {"attempt_id":1,"charged":0,"held":i64::MAX}, {"attempt_id":2,"charged":0,"held":1} + ])); +} +#[test] +fn known_unit_aggregate_and_combined_hold_overflow_still_refuse() { + for held in [false, true] { + let mut v = accounting_pair(i64::MAX, 1); + for a in v["attempts"].as_array_mut().unwrap() { + a["accounting"]["unit"] = "logical_support".into(); + a["inputs"][0]["role"] = "evidence".into(); + a["support"] = json!({"version":"support/v1","rule_id":999,"verdict":true,"clauses":[{"clause_id":1,"premises":[{"evidence_id":1,"actor":"a","scope":"s"}]}]}); + } + if held { + v["attempts"][1]["delivery"] = json!({"state":"unresolved"}); + v["attempts"][1]["accounting"] = json!({"unit":"logical_support","quote":{"input_tokens":0,"output_cap":1},"granted":true,"budget_reference":1,"charge":0,"hold_bound":1}); + } + let mut valid = v.clone(); + valid["attempts"][0]["accounting"]["charge"] = 10.into(); + valid["attempts"][0]["delivery"]["usage"] = 10.into(); + assert!(parse_decision_bundle(&seal(valid)).is_ok()); + assert!(matches!(parse_decision_bundle(&seal(v)), Err(BundleError::InconsistentLedger))); + } +} +#[test] +fn report_v2_empty_unknowns_and_invalid_bundles_remain_bounded() { + let b = parsed(report_fixture()); + let report = serde_json::to_value(validate_recorded_bundle(&b)).unwrap(); + assert_eq!(report["unknown_accounting"], json!([])); + let mut bad = parsed(fixture()); + bad.attempts[0].accounting.charge = -1; + let report = serde_json::to_value(validate_recorded_bundle(&bad)).unwrap(); + assert_eq!(report["consistency"], "mismatch"); + assert_eq!(report["accounting"], json!([])); + assert_eq!(report["unknown_accounting"], json!([])); + assert_eq!(report["source_bundle_sha256"], Value::Null); + assert_eq!(report["attempts"], json!([])); +} #[test] fn missing_unknown_duplicate_and_wrong_schema_refuse() { assert!(matches!( @@ -184,8 +281,8 @@ fn unresolved_delivery_retains_hold_and_unknown_usage() { let r = validate_recorded_bundle(&b); assert_eq!(r.consistency, ReplayStatus::Matched); assert_eq!(r.validation, ReplayStatus::Incomplete); - assert_eq!(r.accounting[0].held, 6); - assert_eq!(r.accounting[0].charged, 0); + assert_eq!(r.unknown_accounting[0].held, 6); + assert_eq!(r.unknown_accounting[0].charged, 0); assert!(r .missing_requirements .contains(&MissingRequirement::MissingDelivery)); @@ -258,7 +355,7 @@ fn cancelled_unsent_recovery_releases_granted_hold() { a["recovery"] = json!({"recovery_id":1,"ordinal":0,"state":"cancelled","predecessor_id":null,"request":bytes(b"request")}); a["accounting"] = json!({"unit":"unknown","quote":{"input_tokens":2,"output_cap":4},"granted":true,"budget_reference":1,"charge":0,"hold_bound":0}); let r = validate_recorded_bundle(&parsed(v)); - assert_eq!(r.accounting[0].held, 0); + assert_eq!(r.unknown_accounting[0].held, 0); } #[test] fn shared_occurrence_requires_each_consuming_attempt_scope() { @@ -516,7 +613,7 @@ fn reserved_unsent_incident_reconstructs_recorded_preparation_without_claiming_d let r = validate_recorded_bundle(&parsed(v)); assert_eq!(r.reconstruction, ReplayStatus::Matched); assert_eq!(r.validation, ReplayStatus::Incomplete); - assert_eq!(r.accounting[0].held, 514); + assert_eq!(r.unknown_accounting[0].held, 514); } #[test] @@ -642,7 +739,7 @@ fn unsent_timed_refusal_keeps_uncovered_reservation_facts() { assert_eq!(report.reconstruction, ReplayStatus::Matched, "{state}"); assert_eq!(report.validation, ReplayStatus::Incomplete, "{state}"); assert_eq!( - report.accounting[0].held, + report.unknown_accounting[0].held, if state == "reserved" { 1538 } else { 0 } ); } From 0899ecab937f6813a5fa5fb354287d3a1386b020 Mon Sep 17 00:00:00 2001 From: Alejandro Castellanos <5405628+k2jac9@users.noreply.github.com> Date: Sun, 15 Nov 2026 23:19:44 -0500 Subject: [PATCH 45/51] Document V2 selection and replay semantics with final source evidence --- docs/decision-bundle-v1.md | 31 ++++++++++--- docs/decision-replay.md | 23 ++++++++++ docs/runtime-adoption.md | 91 ++++++++++++++++++++++++++++++++++++++ 3 files changed, 139 insertions(+), 6 deletions(-) diff --git a/docs/decision-bundle-v1.md b/docs/decision-bundle-v1.md index ecca1f1..39600eb 100644 --- a/docs/decision-bundle-v1.md +++ b/docs/decision-bundle-v1.md @@ -266,19 +266,38 @@ the incident manifest fingerprint and immutable runtime identity `canned-no-inference`. There is no allowlisted real-model token runtime in C1; legacy and other runtime accounting stays `unknown`. Do not infer units from sizes or arbitrary model contract strings. Budget references are opaque local -IDs, not global totals or authority. Report totals sum only exported attempts, -grouped by declared unit; `unknown` totals remain numerically recorded unknown -units and must not be interpreted as model-token or monetary totals. +IDs, not global totals or authority. + +The current reporting successor is `decision-replay-report/2`. Its `accounting` +array contains numeric `charged`/`held` totals only for established compatible +units (`logical_support`, `fixture_bytes`, `model_tokens`), over exported attempts. +It never contains an `unknown` numeric total. Instead, `unknown_accounting` is an +ordered list of `{attempt_id, charged, held}` observations in bundle attempt order. +Each observation preserves that attempt's original ledger numbers without addition +across attempts. A valid report without unknown observations has an empty list. +`unknown_accounting_unit` remains a missing requirement; unknown labels do not +establish a common dimension. Independent unknown charges or holds of `i64::MAX` +and 1 are permitted when each record is valid. Quote/charge/hold bounds, usage +consistency and all ledger checks still apply per attempt. Known-unit charge, +hold and combined charge-plus-hold totals still refuse overflow. + +The earlier unversioned replay report deliberately summed unknown values. That +contract is historical and superseded by this reporting decision; retained old +reports are not rewritten. DecisionBundleV1's wire schema, bytes, integrity digest, +occurrence identities, dispositions, admissions and ledger writes are unchanged. +There is no bundle migration. Archived pre-successor documentation is retained in +`final-fix/before-decision-bundle-v1.md` in the project evidence packet. ## Replay semantics and pure boundary -Reports contain independent `consistency`, `reconstruction`, `validation`, root +Reports declare `report_schema: "decision-replay-report/2"` and contain independent `consistency`, `reconstruction`, `validation`, root `recorded_disposition`, root `validator_provenance`, `missing_requirements`, -`accounting`, `source_bundle_sha256`, `reevaluated_validator` and per-attempt +`accounting`, `unknown_accounting`, `source_bundle_sha256`, `reevaluated_validator` and per-attempt `attempts` results. Status serialization is `matched`, `mismatch`, `incomplete`, `not_reproducible`. Aggregation prioritizes mismatch, then incomplete, then not reproducible. An invalid directly constructed value yields consistency mismatch -without cloning its private metadata into the report. +with empty accounting and unknown-observation lists, without cloning its private +metadata into the report. Missing requirements serialize as `validator_unavailable`, `renderer_unavailable`, `missing_input`, `missing_delivery`, `missing_dependency`, diff --git a/docs/decision-replay.md b/docs/decision-replay.md index cff5fd5..823abb0 100644 --- a/docs/decision-replay.md +++ b/docs/decision-replay.md @@ -45,6 +45,29 @@ child-process/network calls in one run is bounded observational evidence; it is not a universal proof for other paths or a detector of an in-memory Store. The source boundary supplies the separate evidence for Store initialization. +## Report version 2 + +The executable emits `report_schema: "decision-replay-report/2"`. `accounting` +contains totals only for known compatible units; `unknown_accounting` preserves +ordered `{attempt_id, charged, held}` observations without cross-attempt addition. +Empty unknown lists are explicit. Missing unit requirements remain visible. +Invalid bundles refuse in this CLI; the direct replay API returns a bounded +mismatch with empty accounting lists. Per-record ledger checks and known-unit +aggregate overflow checks remain enforced. + +This versions the report, not DecisionBundleV1. Existing bundle bytes and digests +remain valid; no ledger history or disposition is rewritten. Consumers must use +the report schema to distinguish these semantics from the historical unversioned +report, whose unknown totals were invalid dimensional aggregates. The repository +runner serializes the report generically; the Python reader consumes bundle bytes, +not this report. The local consumer audit found no other report-shape consumer +requiring migration. External consumers were not inventoried. + +The exact integrated-review bundles (charges 10/20 and `i64::MAX`/1) are retained +and replayed by the new S3 binary under `final-fix/replay-probes/`; their original +input hashes are checked before and after the separate processes. Old review +outputs remain historical evidence beside the original inputs. + ## Python reader and E1 helper boundary `scripts/decision_bundle_v1.py` uses only the Python standard library. It performs diff --git a/docs/runtime-adoption.md b/docs/runtime-adoption.md index f13f373..c2bba6f 100644 --- a/docs/runtime-adoption.md +++ b/docs/runtime-adoption.md @@ -590,3 +590,94 @@ categories remain deferred to integrated review. All 199 files protected by the fix intake, four owner policies, review/context identities and candidate selection bytes were checked unchanged. Hosted CI, other full S2 roles, real-model/ML consumers and activation remain unexecuted, as before. + +### Final integrated-review successor: selection V2 and replay report V2 + +The final correction uses executable source +`5e9c76e0b95fa2559fc594523fd9739263577a1b` (S3). The subsequent documentation-only +successor is mapped in `final-fix/docs-only-transition.json`; it changes only +`docs/runtime-adoption.md`, `docs/decision-bundle-v1.md`, and +`docs/decision-replay.md`. Historical G1/G2 qualification records, old source, +selections and accepted pointer remain intact. This section supersedes the earlier +runtime proposal for final acceptance; activation remains a root action after +independent scoped review. + +`kairotensor-project-runtime/2` retains the exact G1-qualified rustc, Cargo, +Python, native/helper, review and acceptance identities and requires exactly one +additional tool: the selected rustc path's sibling `rustdoc`. Its path and SHA-256 +are bound, its arguments are fixed to `--version`, and its version must match the +release/commit/date derived from the qualified rustc's first version line. The +launcher verifies the actual rustdoc hash and version, puts it on the private +PATH and sets `RUSTDOC`, alongside `RUSTC`. Invocation receipts include its actual +identity and explicitly identify it as supplementary selection-V2 evidence; +G1 never tested or recorded rustdoc. Missing or additional runtime tool keys, +wrong sibling/hash/version/arguments, and inherited `RUSTDOC`, `RUSTDOCFLAGS`, +`CARGO_ENCODED_RUSTDOCFLAGS` or `CARGO_BUILD_RUSTDOC` (including empty values) refuse. +Existing Cargo config/selector and native environment refusals remain in force. + +Five new candidate selections reside under +`/home/neo/.local/state/kairotensor/runtime/candidates/final-fix-20261116/`. +The original V1 accepted pointer is deliberately unchanged during review; the V2 +launcher explicitly refuses it. Use `--selection` with a new V2 candidate for +qualification. Retained old source/checkouts and their V1 launcher continue to +use the old pointer. There is no silent fallback, installation or workstation +runtime-default change. Root must bind reviewed S3 evidence and the candidate +selection before changing that pointer, then test a fresh database. + +The replay correction is independently versioned as `decision-replay-report/2`. +Known compatible units retain numeric totals; unknown units become ordered +per-attempt observations without addition. DecisionBundleV1 bytes, integrity, +per-attempt accounting, ledger writes and historical dispositions remain intact. +The two integrated-review input bundles are preserved and replayed by the new +source-bound binary under `final-fix/replay-probes/`. This is offline reconstruction +and contract checking, with no model calls or new semantic-quality claim. + +The project Cargo documentation-test invocation passed with **zero doctests**: +that is tooling coverage, not an invented test. A separate, exact artifact source +(`final-fix/rustdoc_fixture.rs`) ran **one real passing doctest** through the V2 +selected rustdoc with selected Rustc. Supplementary source, executable identities, +commands and results are retained in `final-fix/doc-probes/` and +`final-fix/rustdoc-fixture-final/`. The one-test artifact was repeated with +compiler temporary output explicitly declared under its evidence directory; both +passing results are retained, not counted as two distinct tests. All five qualified +runtime pairs also execute +actual V2 identity/child-Python probes and the focused 28-test launcher suite. + +The new Store health probe is built from S3 and the exact approved probe source; +`final-fix/activation-probe/build-identities.json` binds compiler, library, +dependencies, source and binary. Its fresh disposable create/reopen observations +retain exact current bytes, spent 7, reserved 12 and unresolved attempt 2, with no +second claim/cancel and zero backend calls. They are candidate checks; root's +post-review activation must use another fresh database. + +Two supplementary storage jobs initially refused because their retention lists +included existing regular files and the storage inspector requires directories. +Their manifests and failed outputs remain in `doc-probes/` and `activation-probe/`. +New manifests retain enclosing output directories under the same 0.05 GiB bounds; +exact file hashes remain separately bound. The admitted retry passed. No storage +guard, acceptance gate, application source or historical receipt was changed. + +M1 remains a minor shared-example warning cleanup follow-up; protected adapter +code and warnings are retained. Hosted CI, real-model/ML consumers, external report +consumer inventory, seven-day observation and shared publication remain outside +this candidate's evidence. The prior lost preflight display remains an explicit +historical preservation defect. This work neither extends the storage exception +(expiry `2026-11-17T18:04:38Z`) nor authorizes reservation release or asset retirement. + +#### S3 full runtime matrix + +`final-fix/verification-rollup.json` binds all five final-S3 runtime pairs to the +unchanged 40-command workload and passing independent receipt revalidation: +**200 commands**, with **362 Rust, 371 full Python, 18 reference and 8 oracle** +checks per role, zero skips/ignored/filtered checks. Explicit overlapping suites +remain 24/50/15/19/7; the five full roles total 4370 test executions including +those overlaps. The five separate focused 28-test launcher suites add 140 +executions. These are actual reruns at S3, not relabelled historical G1/G2 runs. + +Three fresh registered Rust checkouts distinguish source/compiler/native keys. +Python-varied rows rerun against the exactly bound stable Rust key. All use the +original storage/Cargo wrappers with 2 GiB admission per fresh key; supplemental +probes retain their separate 0.05 GiB jobs. The warning inventory contains 17 +lines per role (85 including repeated diagnostics and summaries across the full +matrix), all retained in `final-fix/compiler-warnings.json`. Passing matrix and +candidate health still require independent review and root activation. From 7c7ca36df6b71bfc0b9644d452e1092fd8f2d258 Mon Sep 17 00:00:00 2001 From: Alejandro Castellanos <5405628+k2jac9@users.noreply.github.com> Date: Sun, 15 Nov 2026 23:45:47 -0500 Subject: [PATCH 46/51] docs: remove unrelated integration references --- docs/decision-ablation.md | 3 +-- docs/runtime-adoption.md | 2 +- docs/superpowers/plans/2026-11-15-autonomous-roadmap.md | 9 ++++----- docs/superpowers/plans/2026-11-15-decision-replay.md | 2 +- .../plans/2026-11-15-evidence-reliability-p0.md | 2 +- .../plans/2026-11-15-incident-store-bridge.md | 2 +- docs/superpowers/plans/2026-11-15-offline-diagnostics.md | 8 ++++---- docs/superpowers/plans/2026-11-15-policy-activation.md | 2 +- docs/superpowers/plans/2026-11-15-sqlite-lifetimes.md | 2 +- .../specs/2026-11-15-evidence-reliability-review.md | 2 +- .../specs/2026-11-15-reliability-continuation.md | 4 ++-- 11 files changed, 18 insertions(+), 20 deletions(-) diff --git a/docs/decision-ablation.md b/docs/decision-ablation.md index 830ecfe..24edad3 100644 --- a/docs/decision-ablation.md +++ b/docs/decision-ablation.md @@ -3,8 +3,7 @@ `scripts/decision_ablation.py` plans hypothetical omissions and scores explicit recorded outcome attestations. It performs file input/output only. It has no run mode, backend imports, executable command output, process calls, model loading, -provider requests or network access. E2 portability remains Deferred; Rocket is -outside this work. +provider requests or network access. E2 portability remains Deferred. ```sh python3 scripts/decision_ablation.py plan complete-bundle.json > plan.json diff --git a/docs/runtime-adoption.md b/docs/runtime-adoption.md index c2bba6f..a41b542 100644 --- a/docs/runtime-adoption.md +++ b/docs/runtime-adoption.md @@ -455,7 +455,7 @@ measurement workflows (handoff, recovery-calibration, recovery-policy, token-cap retain Python `3.11` and have no real-model qualification under this core matrix. Accepted checkout v4.2.2 is intentionally retained; adopting latest v7.0.1 is a separate owned gap. Previous Rust upstream support and the complete separately -promised consumer inventory remain not established. No model, Rocket, deployment, +promised consumer inventory remain not established. No model, deployment, shared publication or bulk dependency change is covered here. Root, acting under Alejandro's autonomous project authority, owns review and diff --git a/docs/superpowers/plans/2026-11-15-autonomous-roadmap.md b/docs/superpowers/plans/2026-11-15-autonomous-roadmap.md index f15b170..6aedb27 100644 --- a/docs/superpowers/plans/2026-11-15-autonomous-roadmap.md +++ b/docs/superpowers/plans/2026-11-15-autonomous-roadmap.md @@ -1,10 +1,9 @@ # KairoTensor autonomous execution map Authority: on 15 November 2026 the owner authorized autonomous implementation, -research, documented decisions and continued execution. The owner explicitly -excluded any Rocket connection. Existing available credits may be used for -bounded, useful experiments; this is not authority for top-ups, subscription -changes or spending beyond verified existing balances. +research, documented decisions and continued execution. Existing available +credits may be used for bounded, useful experiments; this is not authority for +top-ups, subscription changes or spending beyond verified existing balances. This map supersedes the earlier review-only execution restriction. Historical reviews and receipts remain unchanged. Checkboxes in the component plans describe @@ -19,7 +18,7 @@ future work; completion is recorded with commits and evidence in the ledger. | C | [Decision replay](2026-11-15-decision-replay.md) | After B; authorized export, honest missing-data outcomes, pure replay. | | D | [Policy activation](2026-11-15-policy-activation.md) | After A-C; comparable offline evaluation and transactional activation. | | E1 | [Offline diagnostics](2026-11-15-offline-diagnostics.md) | After C; bounded plans and recorded-outcome scoring. | -| E2 | Same diagnostics plan, conditional task | Portable interchange only when a real second consumer requires it. No Rocket integration. | +| E2 | Same diagnostics plan, conditional task | Portable interchange only when a real second consumer requires it. | | F | Bounded experiments | After relevant offline gates; separate run manifests and exact credit bounds. | | G | [Runtime qualification](2026-11-15-runtime-qualification.md) | After integrated core; qualify isolated current/preview/previous lanes and exact project pins. | diff --git a/docs/superpowers/plans/2026-11-15-decision-replay.md b/docs/superpowers/plans/2026-11-15-decision-replay.md index 63cbe97..8a87ac4 100644 --- a/docs/superpowers/plans/2026-11-15-decision-replay.md +++ b/docs/superpowers/plans/2026-11-15-decision-replay.md @@ -20,7 +20,7 @@ section C. Prerequisites: A and B reviewed. - Preserve arbitrary bytes, repeated input positions, lineage and uncertain holds. - Do not export cross-scope/global budget totals or infer missing validators. - Pure replay cannot invoke tools, a model, network, Store initialization or - migrations. No Rocket integration; use normal storage admission. + migrations. Use normal storage admission. - Resource bounds apply before cloning public constructed values and before materializing oversized SQLite payload/metadata fields, not only JSON parsing. diff --git a/docs/superpowers/plans/2026-11-15-evidence-reliability-p0.md b/docs/superpowers/plans/2026-11-15-evidence-reliability-p0.md index 32e9f63..a629af9 100644 --- a/docs/superpowers/plans/2026-11-15-evidence-reliability-p0.md +++ b/docs/superpowers/plans/2026-11-15-evidence-reliability-p0.md @@ -270,4 +270,4 @@ and independently useful with the explanation API. No migration is needed. Next: follow the [execution map](2026-11-15-autonomous-roadmap.md) into the separate incident bridge, decision replay, policy activation and offline diagnostics plans. -No Rocket integration. Portable interchange retains its real-consumer prerequisite. +Portable interchange retains its real-consumer prerequisite. diff --git a/docs/superpowers/plans/2026-11-15-incident-store-bridge.md b/docs/superpowers/plans/2026-11-15-incident-store-bridge.md index 4f6e476..487d06c 100644 --- a/docs/superpowers/plans/2026-11-15-incident-store-bridge.md +++ b/docs/superpowers/plans/2026-11-15-incident-store-bridge.md @@ -22,7 +22,7 @@ section B. Prerequisite: P0 A complete and reviewed. - All required IDs come from the pinned workload definition. Caller flags and caller-supplied payload replacements cannot authorize preparation. - Fixture byte accounting is labelled explicitly; no live model calls. -- No Rocket connection. Do not change `scripts/incident_routing.py` or the +- Do not change `scripts/incident_routing.py` or the frozen `workloads/incident-routing-v1/` files. - Use admitted Cargo output and retain failed checks; honor the autonomous map. diff --git a/docs/superpowers/plans/2026-11-15-offline-diagnostics.md b/docs/superpowers/plans/2026-11-15-offline-diagnostics.md index 3eb4141..3ffe1aa 100644 --- a/docs/superpowers/plans/2026-11-15-offline-diagnostics.md +++ b/docs/superpowers/plans/2026-11-15-offline-diagnostics.md @@ -10,7 +10,7 @@ prepare portable evidence only when a real second consumer requires it. optional future Rust envelope codec that grants no admission. **Tech Stack:** Python standard library; optional Rust serde and B's SHA-256. **Spec:** [continuation specification](../specs/2026-11-15-reliability-continuation.md), -section E. No Rocket integration or cross-repository build dependency. +section E. No cross-repository build dependency. ## Global Constraints @@ -120,9 +120,9 @@ unknown measurements remain unknown plus bounds, never zero. ### Task 2: Conditional portable envelope **Entry gate:** name an actual second consumer, its owner identity mapping and -scope/purpose needs in a new intake receipt. The current roadmap names none and -explicitly excludes Rocket. Until this condition is met, record Deferred and -continue other work; do not invent a mock consumer as justification. +scope/purpose needs in a new intake receipt. The current roadmap names none. +Until this condition is met, record Deferred and continue other work; do not +invent a mock consumer as justification. **Allowed files after the gate:** create `src/evidence_envelope.rs`, `tests/evidence_envelope.rs`, `examples/export_evidence.rs`, diff --git a/docs/superpowers/plans/2026-11-15-policy-activation.md b/docs/superpowers/plans/2026-11-15-policy-activation.md index a01b851..eb20fd1 100644 --- a/docs/superpowers/plans/2026-11-15-policy-activation.md +++ b/docs/superpowers/plans/2026-11-15-policy-activation.md @@ -21,7 +21,7 @@ section D. Prerequisites: A-C reviewed. Only this stage introduces a migration. - Activation atomically changes policy admissions; never changes old send claims, outstanding holds, historical dispositions or original deadlines. - Rollback advances history and needs compatibility, not superiority. -- No Rocket integration, live model call, new service or live-database migration. +- No live model call, new service or live-database migration. - Next schema version is determined from integrated source; do not assume v10. ## Review Focus diff --git a/docs/superpowers/plans/2026-11-15-sqlite-lifetimes.md b/docs/superpowers/plans/2026-11-15-sqlite-lifetimes.md index 5513d7b..6386b85 100644 --- a/docs/superpowers/plans/2026-11-15-sqlite-lifetimes.md +++ b/docs/superpowers/plans/2026-11-15-sqlite-lifetimes.md @@ -13,7 +13,7 @@ ownership around them. No new dependency, service, schema or model call. - Preserve transaction commit/rollback behavior and original raised exceptions. - Never suppress ResourceWarning, weaken audit checks, modify retained bundles, - release uncertain holds, execute inference or connect to Rocket. + release uncertain holds or execute inference. - A connection returned successfully transfers ownership to the caller; a setup failure must close it. Source evidence and rejected fixture copies remain retained. - Use existing admitted fixture runner; no rebuild is needed for Python changes. diff --git a/docs/superpowers/specs/2026-11-15-evidence-reliability-review.md b/docs/superpowers/specs/2026-11-15-evidence-reliability-review.md index 0ffe6db..d26b3a1 100644 --- a/docs/superpowers/specs/2026-11-15-evidence-reliability-review.md +++ b/docs/superpowers/specs/2026-11-15-evidence-reliability-review.md @@ -132,7 +132,7 @@ The following points must be frozen before its implementation WorkOrder: | B | Store-backed incident fixture | Strict case-scoped mappings, captured bytes, byte-identical rendering, label-independent validation and existing timed gates. Separate implementation plan. | | C | Authorized decision export and deterministic replay | Freeze bundle schema, explicit error type, authorizer contract, redaction boundary and versioned validator registry first; no execution during replay. | | D | Evaluated policy activation | Freeze counters/denominators, authority, supported artifacts, exact comparison manifest and next free schema version; bootstrap/promote/rollback have separate gates. | -| E | Portable envelope and bounded diagnostics | Only after a named second consumer needs it; Rocket-owned adapter stays a separate task. No live ablation execution implied. | +| E | Portable envelope and bounded diagnostics | Only after a named second consumer needs it. No live ablation execution implied. | Do not put C-E in the same execution batch as A. A policy registry must not manufacture send authority; a stored evaluation remains an attestation. Preserve diff --git a/docs/superpowers/specs/2026-11-15-reliability-continuation.md b/docs/superpowers/specs/2026-11-15-reliability-continuation.md index 15e8105..a8d9f78 100644 --- a/docs/superpowers/specs/2026-11-15-reliability-continuation.md +++ b/docs/superpowers/specs/2026-11-15-reliability-continuation.md @@ -3,7 +3,7 @@ Date: 15 November 2026. Source inspected: main `20fab84a6baa525452191ecf80eeee339f0b5ac7`; preceding P0 plan commit `8d4cce8`. All new names below are proposed interfaces until their implementation commits. -The owner authorized autonomous execution and excluded Rocket integration. +The owner authorized autonomous execution. ## Evidence and known limits @@ -205,4 +205,4 @@ claim and no pruning of opaque dependency manifests. Portable envelopes remain conditional on an actual second consumer. Their parser and digest checker cannot grant admission; a receiver must authenticate owner mapping and recheck full reference, scope/purpose, context and current authority. -No cross-store atomicity, implicit TTL, Rocket connection or live ablation runner. +No cross-store atomicity, implicit TTL or live ablation runner. From 62787673a61d3f126464a5087036194726d67ec6 Mon Sep 17 00:00:00 2001 From: Alejandro Castellanos <5405628+k2jac9@users.noreply.github.com> Date: Sun, 15 Nov 2026 23:50:15 -0500 Subject: [PATCH 47/51] Separate prepared adapter operations and preserve existing verification logs --- docs/runtime-adoption.md | 3 + docs/verification-maintenance.md | 44 ++++++++++++ examples/budgeted_recovery.rs | 2 +- examples/corrected_handoff.rs | 1 + examples/corrected_handoff/adapter.rs | 67 ++--------------- .../corrected_handoff/prepared_adapter.rs | 72 +++++++++++++++++++ examples/recovery_policy_eval.rs | 5 +- examples/scoped_model_reuse.rs | 1 + examples/token_cap_eval.rs | 2 +- scripts/test_verify_runtime_matrix.py | 71 +++++++++++++++++- scripts/verify_runtime_matrix.py | 4 +- 11 files changed, 203 insertions(+), 69 deletions(-) create mode 100644 docs/verification-maintenance.md create mode 100644 examples/corrected_handoff/prepared_adapter.rs diff --git a/docs/runtime-adoption.md b/docs/runtime-adoption.md index a41b542..35b1f9b 100644 --- a/docs/runtime-adoption.md +++ b/docs/runtime-adoption.md @@ -1,5 +1,8 @@ # Runtime adoption and reproducible baseline +Maintenance successor: [example warnings and log preservation](verification-maintenance.md). +The dated qualification records below describe their original source revisions. + Checked on 2026-11-15. This change freezes a locally verified dependency resolution and requires Cargo to honor it in the existing CI commands. Runtime migration, hosted CI execution, release acceptance and rollback qualification diff --git a/docs/verification-maintenance.md b/docs/verification-maintenance.md new file mode 100644 index 0000000..5e4ff6c --- /dev/null +++ b/docs/verification-maintenance.md @@ -0,0 +1,44 @@ +# Example warnings and verification log preservation + +The simple handoff and scoped-reuse examples compile the basic adapter. Examples +that prepare a request before reserving usage opt into +`examples/corrected_handoff/prepared_adapter.rs`. Preparation, generation and +discard operations retain their original byte protocol and single-use handles. +Recovery-policy traces retain both adapter source files in their manifest. + +The two simple examples deny dead code during compilation, so their former +warning categories now fail the existing Cargo checks if reintroduced. This +cleanup adds no warning suppression and changes no Store or accounting contract. + +## Recording new verification runs + +`scripts/verify_runtime_matrix.py` requires a fresh directory for each runtime +lane. Its command runner creates stdout and stderr exclusively before starting +the command. An existing file or symlink, including a redirected stdout target, +refuses dispatch without replacing that evidence. Concurrent attempts to use the +same log paths cannot both dispatch. Receipts retain the actual process exit, +elapsed time and hashes of both streams, including failed commands. + +If creating the second stream fails, the newly created first stream can remain +empty. Preserve it as an incomplete attempt and use a fresh run directory. Do not +remove files to retry under an old name. A directory owned by one run may update +its aggregate progress receipt; this is not an append-only storage service. + +Outer controllers and shell redirection must also use unique run directories and +exclusive writes (`open('xb')` in Python or `set -o noclobber` in Bash). Keep +command output, process exit and digests together. The verifier cannot protect +output files independently opened by an outer shell or executed application. + +## Historical evidence gap + +One superseded standalone preflight display was overwritten during an earlier +controller run. A bounded search did not recover its bytes. Its original exit +status and SHA-256 remain recorded, as do the original inner command logs and +artifacts. The gap remains a historical preservation defect; later runs do not +reconstruct or replace it. + +The original incident and search records remain in the project artifact directory +under `reliability-implementation-20261115/task-g1/`, named +`controller-log-collision.json` and `old-preflight-recovery-search.json`. +Subsequent complete qualification packets are separate evidence. No historical +receipt, runtime selection or accepted rollback asset is changed by this cleanup. diff --git a/examples/budgeted_recovery.rs b/examples/budgeted_recovery.rs index 74143d2..adcac1d 100644 --- a/examples/budgeted_recovery.rs +++ b/examples/budgeted_recovery.rs @@ -1,6 +1,6 @@ //! Bounded recovery: deterministic byte-token fixture by default, --live for Qwen. #[allow(dead_code)] -#[path = "corrected_handoff/adapter.rs"] +#[path = "corrected_handoff/prepared_adapter.rs"] mod adapter; use kairotensor::{Disposition, RecoveryStart, Store}; use serde::{Deserialize, Serialize}; diff --git a/examples/corrected_handoff.rs b/examples/corrected_handoff.rs index 5a1e678..aed7f2a 100644 --- a/examples/corrected_handoff.rs +++ b/examples/corrected_handoff.rs @@ -1,5 +1,6 @@ //! Run with `cargo run --example corrected_handoff -- [--live] [--trace FILE]`. //! All release data is synthetic. The adapter only generates text. +#![deny(dead_code)] #[path = "corrected_handoff/adapter.rs"] mod adapter; diff --git a/examples/corrected_handoff/adapter.rs b/examples/corrected_handoff/adapter.rs index 829fdee..aeadc25 100644 --- a/examples/corrected_handoff/adapter.rs +++ b/examples/corrected_handoff/adapter.rs @@ -18,21 +18,8 @@ pub struct Reply { pub roundtrip_us: u64, } -/// Single-use quote issued by the live worker that owns the prepared tensors. -pub struct Prepared { - handle: String, - pub input_tokens: u64, - pub max_new_tokens: u64, -} - -pub struct PreparedReply { - pub reply: Reply, - pub hit_cap: bool, - pub ended_eos: bool, -} - pub struct Adapter { - worker: Option, + pub(super) worker: Option, } impl Adapter { @@ -63,52 +50,6 @@ impl Adapter { Ok(serde_json::to_vec(&request)?) } - fn live_worker(&mut self) -> Result<&mut Worker> { - self.worker.as_mut().ok_or_else(|| "exact preflight requires the live worker".into()) - } - - /// Tokenize without generating. This is the exact prompt ID count for the - /// pinned model and template. Keep the frozen bytes for dispatch accounting. - pub fn prepare(&mut self, request: &[u8]) -> Result { - let request_utf8 = std::str::from_utf8(request)?; - let command = json!({"op":"prepare", "request_utf8":request_utf8}); - let result = self.live_worker()?.exchange(&serde_json::to_vec(&command)?)?; - Ok(Prepared { - handle: result["handle"].as_str().ok_or("missing preparation handle")?.to_owned(), - input_tokens: result["input_tokens"].as_u64().ok_or("missing preflight input count")?, - max_new_tokens: result["max_new_tokens"].as_u64().ok_or("missing preflight output cap")?, - }) - } - - /// Use only after a successful reservation. The worker consumes the handle - /// before generating, including when a response is lost or generation fails. - pub fn generate_prepared(&mut self, prepared: Prepared) -> Result { - let start = Instant::now(); - let command = json!({"op":"generate", "handle":prepared.handle, - "input_tokens":prepared.input_tokens, "max_new_tokens":prepared.max_new_tokens}); - let response = self.live_worker()?.exchange(&serde_json::to_vec(&command)?)?; - let mut reply = Worker::reply(&response)?; - let output = reply.output_tokens.ok_or("missing output count")?; - let hit_cap = response["hit_cap"].as_bool().ok_or("missing output-cap indicator")?; - let ended_eos = response["ended_eos"].as_bool().ok_or("missing EOS indicator")?; - if reply.input_tokens != Some(prepared.input_tokens) || output > prepared.max_new_tokens - || hit_cap != (output == prepared.max_new_tokens) - { - return Err("generation did not match prepared quote".into()); - } - reply.roundtrip_us = start.elapsed().as_micros() as u64; - Ok(PreparedReply { reply, hit_cap, ended_eos }) - } - - pub fn discard(&mut self, prepared: Prepared) -> Result<()> { - let command = json!({"op":"discard", "handle":prepared.handle}); - let response = self.live_worker()?.exchange(&serde_json::to_vec(&command)?)?; - if response["discarded"].as_str() != Some(prepared.handle.as_str()) { - return Err("worker did not confirm discard".into()); - } - Ok(()) - } - pub fn dispatch(&mut self, request: &[u8]) -> Result { let start = Instant::now(); let mut reply = match self.worker.as_mut() { @@ -159,7 +100,7 @@ fn fixture_answer(data: &Value) -> Result { })) } -struct Worker { +pub(super) struct Worker { child: Child, stdin: ChildStdin, stdout: BufReader, @@ -191,7 +132,7 @@ impl Worker { Ok(serde_json::from_str(&line)?) } - fn exchange(&mut self, request: &[u8]) -> Result { + pub(super) fn exchange(&mut self, request: &[u8]) -> Result { if request.contains(&b'\n') { return Err("request must be one JSONL record".into()); } @@ -205,7 +146,7 @@ impl Worker { Ok(response) } - fn reply(response: &Value) -> Result { + pub(super) fn reply(response: &Value) -> Result { let count = |key: &str| -> Result { Ok(response[key].as_u64().ok_or("missing worker measurement")?) }; diff --git a/examples/corrected_handoff/prepared_adapter.rs b/examples/corrected_handoff/prepared_adapter.rs new file mode 100644 index 0000000..0eb5f0a --- /dev/null +++ b/examples/corrected_handoff/prepared_adapter.rs @@ -0,0 +1,72 @@ +//! Opt-in prepared-request operations for examples that reserve exact model usage. +#[path = "adapter.rs"] +mod basic; +pub use basic::Adapter; +use basic::{Reply, Worker}; +use serde_json::json; +use std::error::Error; +use std::time::Instant; + +type Result = std::result::Result>; + +/// Single-use quote issued by the live worker that owns the prepared tensors. +pub struct Prepared { + handle: String, + pub input_tokens: u64, + pub max_new_tokens: u64, +} + +pub struct PreparedReply { + pub reply: Reply, + pub hit_cap: bool, + pub ended_eos: bool, +} + +impl Adapter { + fn live_worker(&mut self) -> Result<&mut Worker> { + self.worker.as_mut().ok_or_else(|| "exact preflight requires the live worker".into()) + } + + /// Tokenize without generating. This is the exact prompt ID count for the + /// pinned model and template. Keep the frozen bytes for dispatch accounting. + pub fn prepare(&mut self, request: &[u8]) -> Result { + let request_utf8 = std::str::from_utf8(request)?; + let command = json!({"op":"prepare", "request_utf8":request_utf8}); + let result = self.live_worker()?.exchange(&serde_json::to_vec(&command)?)?; + Ok(Prepared { + handle: result["handle"].as_str().ok_or("missing preparation handle")?.to_owned(), + input_tokens: result["input_tokens"].as_u64().ok_or("missing preflight input count")?, + max_new_tokens: result["max_new_tokens"].as_u64().ok_or("missing preflight output cap")?, + }) + } + + /// Use only after a successful reservation. The worker consumes the handle + /// before generating, including when a response is lost or generation fails. + pub fn generate_prepared(&mut self, prepared: Prepared) -> Result { + let start = Instant::now(); + let command = json!({"op":"generate", "handle":prepared.handle, + "input_tokens":prepared.input_tokens, "max_new_tokens":prepared.max_new_tokens}); + let response = self.live_worker()?.exchange(&serde_json::to_vec(&command)?)?; + let mut reply = Worker::reply(&response)?; + let output = reply.output_tokens.ok_or("missing output count")?; + let hit_cap = response["hit_cap"].as_bool().ok_or("missing output-cap indicator")?; + let ended_eos = response["ended_eos"].as_bool().ok_or("missing EOS indicator")?; + if reply.input_tokens != Some(prepared.input_tokens) || output > prepared.max_new_tokens + || hit_cap != (output == prepared.max_new_tokens) + { + return Err("generation did not match prepared quote".into()); + } + reply.roundtrip_us = start.elapsed().as_micros() as u64; + Ok(PreparedReply { reply, hit_cap, ended_eos }) + } + + pub fn discard(&mut self, prepared: Prepared) -> Result<()> { + let command = json!({"op":"discard", "handle":prepared.handle}); + let response = self.live_worker()?.exchange(&serde_json::to_vec(&command)?)?; + if response["discarded"].as_str() != Some(prepared.handle.as_str()) { + return Err("worker did not confirm discard".into()); + } + Ok(()) + } + +} diff --git a/examples/recovery_policy_eval.rs b/examples/recovery_policy_eval.rs index b5874f5..7c6257d 100644 --- a/examples/recovery_policy_eval.rs +++ b/examples/recovery_policy_eval.rs @@ -3,7 +3,7 @@ //! Default output is synthetic byte-token mechanics, never model efficacy. //! --live opts into the existing pinned Qwen adapter; collection never fits a policy. #[allow(dead_code)] -#[path = "corrected_handoff/adapter.rs"] +#[path = "corrected_handoff/prepared_adapter.rs"] mod adapter; #[path = "recovery_policy_eval/tests.rs"] #[cfg(test)] @@ -825,7 +825,8 @@ fn make_journal( "fixture_vocabulary":if live {None} else {Some("UTF-8 bytes; not model tokens")}, "started_unix_ms":SystemTime::now().duration_since(UNIX_EPOCH)?.as_millis(), "working_directory_commit_best_effort":commit,"runner_source_utf8":include_str!("recovery_policy_eval.rs"), - "adapter_source_utf8":include_str!("corrected_handoff/adapter.rs"),"worker_source_utf8":include_str!("../scripts/handoff_model.py"), + "adapter_source_utf8":include_str!("corrected_handoff/adapter.rs"), + "prepared_adapter_source_utf8":include_str!("corrected_handoff/prepared_adapter.rs"),"worker_source_utf8":include_str!("../scripts/handoff_model.py"), "platform":{"os":std::env::consts::OS,"arch":std::env::consts::ARCH,"logical_parallelism":std::thread::available_parallelism().ok().map(|n|n.get()), "cpuinfo":fs::read_to_string("/proc/cpuinfo").ok(),"kernel":fs::read_to_string("/proc/version").ok()}, "database":database.to_string_lossy(), diff --git a/examples/scoped_model_reuse.rs b/examples/scoped_model_reuse.rs index 5c60108..3de6925 100644 --- a/examples/scoped_model_reuse.rs +++ b/examples/scoped_model_reuse.rs @@ -1,5 +1,6 @@ //! cargo run --example scoped_model_reuse -- [--live | --replay-invalid] [--trace FILE] //! Synthetic, single-writer fixture. Both model tasks only produce text. +#![deny(dead_code)] #[path = "corrected_handoff/adapter.rs"] mod adapter; diff --git a/examples/token_cap_eval.rs b/examples/token_cap_eval.rs index 98cce84..1172cba 100644 --- a/examples/token_cap_eval.rs +++ b/examples/token_cap_eval.rs @@ -1,7 +1,7 @@ //! Run without arguments for a zero-inference archived token-budget boundary check. //! Use --live --family NAME --trace FILE for actual independent capped generations. #[allow(dead_code)] -#[path = "corrected_handoff/adapter.rs"] +#[path = "corrected_handoff/prepared_adapter.rs"] mod adapter; #[path = "token_cap_eval/cases.rs"] mod cases; diff --git a/scripts/test_verify_runtime_matrix.py b/scripts/test_verify_runtime_matrix.py index 43219f6..7456bd3 100644 --- a/scripts/test_verify_runtime_matrix.py +++ b/scripts/test_verify_runtime_matrix.py @@ -1,5 +1,6 @@ """Behavioral checks use tiny executables, never qualification evidence.""" import copy +from concurrent.futures import ThreadPoolExecutor import hashlib import importlib.util import json @@ -8,6 +9,7 @@ import subprocess import sys import tempfile +import threading import unittest SPEC = importlib.util.spec_from_file_location('matrix', Path(__file__).with_name('verify_runtime_matrix.py')) @@ -38,7 +40,74 @@ def command(self, code, kind='python', contains=None): 'timeout': 5, 'artifacts': []} def run_command(self, command): - return self.m.execute_command(command, self.root, os.environ.copy(), self.root / 'logs') + logs = tempfile.mkdtemp(prefix='logs-', dir=self.root) + return self.m.execute_command(command, self.root, os.environ.copy(), logs) + + def test_existing_log_refuses_dispatch_and_preserves_bytes(self): + for name in ['test.stdout', 'test.stderr']: + with self.subTest(name=name): + logs = self.root / name + logs.mkdir() + existing = logs / name + existing.write_bytes(b'original evidence\x00\xff') + marker = self.root / ('sent-' + name) + command = self.command(f'from pathlib import Path; Path({str(marker)!r}).touch()', 'fixture') + with self.assertRaises(FileExistsError): + self.m.execute_command(command, self.root, os.environ.copy(), logs) + self.assertEqual(existing.read_bytes(), b'original evidence\x00\xff') + self.assertFalse(marker.exists()) + + def test_redirected_output_collision_preserves_target(self): + target = self.root / 'old-preflight.log' + target.write_bytes(b'original preflight') + marker = self.root / 'sent' + command = self.command(f'from pathlib import Path; Path({str(marker)!r}).touch()', 'fixture') + for symlink in [False, True]: + with self.subTest(symlink=symlink): + output = self.root / 'redirect' if symlink else target + if symlink: + output.symlink_to(target) + command['stdout_path'] = str(output) + with self.assertRaises(FileExistsError): + self.run_command(command) + self.assertEqual(target.read_bytes(), b'original preflight') + self.assertFalse(marker.exists()) + + def test_repeated_command_keeps_original_receipt_valid(self): + command = self.command('print("Ran 2 tests\\nOK")') + logs = self.root / 'logs' + record = self.m.execute_command(command, self.root, os.environ.copy(), logs) + with self.assertRaises(FileExistsError): + self.m.execute_command(command, self.root, os.environ.copy(), logs) + self.m.check_command_receipt(command, record) + + def test_concurrent_command_log_claim_has_one_winner(self): + marker = self.root / 'dispatches' + command = self.command( + f'from pathlib import Path\nwith Path({str(marker)!r}).open("ab") as f: f.write(b"sent\\n")\nprint("Ran 2 tests\\nOK")') + barrier = threading.Barrier(2) + def run(): + barrier.wait(timeout=5) + try: + return self.m.execute_command(command, self.root, os.environ.copy(), self.root / 'logs') + except FileExistsError: + return None + with ThreadPoolExecutor(max_workers=2) as pool: + results = list(pool.map(lambda _: run(), range(2))) + winners = [record for record in results if record is not None] + self.assertEqual(len(winners), 1) + self.assertEqual(marker.read_bytes(), b'sent\n') + self.m.check_command_receipt(command, winners[0]) + + def test_existing_lane_directory_preserves_receipt(self): + manifest = self.manifest() + out = self.root / 'result' + out.mkdir() + receipt = out / 'receipt.json' + receipt.write_bytes(b'prior lane receipt') + with self.assertRaises(FileExistsError): + self.m.run_lane(manifest, manifest['lanes'][0], out) + self.assertEqual(receipt.read_bytes(), b'prior lane receipt') def test_wrong_binary_digest_refused(self): self.identity['sha256'] = '0' * 64 diff --git a/scripts/verify_runtime_matrix.py b/scripts/verify_runtime_matrix.py index 89051ea..a6487c3 100644 --- a/scripts/verify_runtime_matrix.py +++ b/scripts/verify_runtime_matrix.py @@ -286,7 +286,9 @@ def execute_command(command, root, env, logs): 'started_at':datetime.datetime.now(datetime.timezone.utc).isoformat(), 'environment':{k:env[k] for k in [*PROFILE,'PATH','RUSTC','RUNNER_TEMP','KAIROTENSOR_RECOVERY_RUNNER','KAIROTENSOR_DECISION_REPLAY'] if k in env}, 'timeout':False,'exit':None} - with stdout.open('wb') as out,stderr.open('wb') as err: + # Claim both outputs before dispatch. A prior file or symlink is evidence, + # never permission to truncate it. Keep any partial claim after a refusal. + with stdout.open('xb') as out,stderr.open('xb') as err: try: p=subprocess.Popen(command['argv'],cwd=root,env=env,stdout=out,stderr=err,start_new_session=True) try: record['exit']=p.wait(timeout=command['timeout']) From c7501c4e9c29a5cd5432c44b4c075671bfa9ae84 Mon Sep 17 00:00:00 2001 From: Alejandro Castellanos <5405628+k2jac9@users.noreply.github.com> Date: Sun, 15 Nov 2026 23:53:52 -0500 Subject: [PATCH 48/51] Document warning cleanup and log preservation verification --- docs/verification-maintenance.md | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/docs/verification-maintenance.md b/docs/verification-maintenance.md index 5e4ff6c..7a25155 100644 --- a/docs/verification-maintenance.md +++ b/docs/verification-maintenance.md @@ -42,3 +42,23 @@ under `reliability-implementation-20261115/task-g1/`, named `controller-log-collision.json` and `old-preflight-recovery-search.json`. Subsequent complete qualification packets are separate evidence. No historical receipt, runtime selection or accepted rollback asset is changed by this cleanup. + +## Verification + +Executable source `62787673a61d3f126464a5087036194726d67ec6` was checked locally +on 16 November 2026 UTC using the existing selected runtimes: + +- Rust 1.100.0, 1.99.1 and the pinned 1.102.0 nightly each passed all 362 + `cargo test --locked --offline --all-targets` tests with zero compiler warnings. +- Python 3.15.2, 3.14.10 and 3.16.0a2 each passed all 376 script tests, including + the actual Rust fixture-runner integrations. No tests were skipped. +- The 18 reference and eight oracle tests passed. Seven affected offline example + runs passed, including invalid-output replay. A fresh recovery trace retained + the exact bytes of both adapter source files. +- The compile guards failed on the original warnings before the refactor. New + overwrite tests failed before exclusive creation was implemented and passed + afterward. Independent review found no actionable issues. + +Receipts, stream hashes, failures and review are retained in the project artifact +directory under `maintenance-warnings-9ug0kq4c/`. These results are local offline +regression evidence; hosted CI and live-model execution were not performed. From e43677d6129853d5ea93c0938f4cce9b0e7daea3 Mon Sep 17 00:00:00 2001 From: Alejandro Castellanos <5405628+k2jac9@users.noreply.github.com> Date: Mon, 16 Nov 2026 00:38:44 -0500 Subject: [PATCH 49/51] docs: map reliability commits and verification boundaries for review --- docs/reliability-review-guide.md | 101 +++++++++++++++++++++++++++++++ 1 file changed, 101 insertions(+) create mode 100644 docs/reliability-review-guide.md diff --git a/docs/reliability-review-guide.md b/docs/reliability-review-guide.md new file mode 100644 index 0000000..33e0c52 --- /dev/null +++ b/docs/reliability-review-guide.md @@ -0,0 +1,101 @@ +# Evidence reliability review guide + +This integration preserves the 48 local commits through `c7501c4e9c29a5cd5432c44b4c075671bfa9ae84`, based on `20fab84a6baa525452191ecf80eeee339f0b5ac7`. The groups below are review boundaries in the existing history, not independent releases. Later fixes must be read with their earlier implementations. + +## Review order + +| Area | Compare range | Commits | +| --- | --- | ---: | +| Eligibility and atomic correction | `20fab84a..1509c38` | 6 | +| Store-backed incident preparation | `1509c38..0dd48f8` | 4 | +| SQLite lifetime fixes and decision replay | `0dd48f8..b921743` | 7 | +| Policy registry, validation and activation | `b921743..a621ad8` | 10 | +| Bounded offline diagnostics | `a621ad8..794f64e` | 2 | +| Runtime qualification and local launcher | `794f64e..0899eca` | 16 | +| Documentation and warning/log maintenance | `0899eca..c7501c4` | 3 | + +Review the shared current-read transaction and absence of diagnostic payloads first; then batch admission ordering, rollback and expected-epoch conflicts. Incident preparation must render captured bytes and retain claim/publication checks. Recorded bundle replay must remain separate from execution and current access. Policy transitions must bind comparable evaluation evidence and active revision, preserve outstanding holds, and atomically revise admission. Offline ablations cannot authorize execution or narrow opaque dependencies. + +The schema migration is exercised on disposable populated databases. This PR does not migrate a live owner's database. Runtime qualification and local launcher code have a separate trust boundary: local acceptance artifacts bind actual executables; the launcher is not a sandbox for arbitrary workloads. + +## Verification evidence + +The latest executable source before this guide is `62787673a61d3f126464a5087036194726d67ec6`; `c7501c4` changes documentation only. [Maintenance verification](verification-maintenance.md) records 362 Rust tests on each of three compiler versions, 376 Python script tests on each of three interpreters with actual fixture integrations, 18 reference and eight oracle checks, and seven affected offline examples. These local runs had no compiler warnings or skipped tests. Earlier failed checks and the unrecoverable superseded preflight display remain historical evidence. + +The workflow defines five isolated combinations: stable, previous Rust, upcoming Rust, previous Python and upcoming Python. A configured matrix is not a passing hosted result. Consult the exact integration PR head and its Actions runs for hosted status; do not transfer a passing result from another commit. + +## Research and review limits + +All new semantic integration, replay, policy activation and ablation evidence here is offline or deterministic. It establishes tested contracts rather than improved model accuracy, independent execution authenticity, or typical savings. The sprint report in PR #19 retains its separately measured source. The hosted adapter in PR #18 is a separate change and is not included in this branch. Active external training is not a result of this PR. + +## Preserved commit sequence + +### Eligibility and atomic correction + +- `8d4cce8` docs: review evidence reliability and plan bounded P0 work +- `9141d8f` build: freeze the tested dependency resolution +- `0d7f61c` feat: explain current eligibility through the shared read gate +- `c322e57` docs: define autonomous reliability milestones and decision boundaries +- `cfa8d3c` fix: reject malformed accepted output in eligibility gate +- `1509c38` feat: atomically revise admissions at an expected epoch + +### Store-backed incident preparation + +- `261a863` feat: add pure versioned incident protocol and exact-byte digests +- `ef42bcf` fix: bound incident protocol allocation before size refusal +- `94a6d43` feat: exercise incident preparation through Store timed gates +- `0dd48f8` test: verify Store incident byte parity in CI + +### SQLite lifetime fixes and decision replay + +- `3b45721` docs: bind replay safeguards and SQLite lifetime repair +- `062725f` fix: close SQLite audit and controller connections +- `8a519e1` docs: define deterministic decision bundle digest encoding +- `43556ae` feat: define bounded pure decision bundle replay +- `b5756e0` fix: enforce recovery bundle ledger invariants +- `37f25c4` feat: authorize decision exports within one Store snapshot +- `b921743` test: exercise decision replay across process boundaries + +### Policy registry, validation and activation + +- `04cb3b6` feat: validate complete comparable policy evaluation records +- `62f5646` fix: validate partial policy facts and legacy attempt timing +- `0923e13` docs: bind policy migration and activation integration scopes +- `2cc5f9d` feat: persist immutable scoped policy versions and evaluations +- `f1a0af0` fix: prevent rowid replacement of immutable policy records +- `33b3303` docs: bind activation consistency and diagnostic runtime boundaries +- `ee5beb6` feat: atomically activate evaluated policies +- `a8f8c4b` test: demonstrate policy activation and rollback without model calls +- `9329950` fix: bound policy fixture input before deserialization +- `a621ad8` docs: retain policy activation fixture packet + +### Bounded offline diagnostics + +- `31e0dd1` feat: plan and score bounded offline ablations +- `794f64e` fix: preserve unspecified-unit accounting observations + +### Runtime qualification and local launcher + +- `5bf3c75` Add source-bound isolated runtime matrix verifier and refusal tests +- `26045ce` Bind runtime receipts to the complete qualification manifest +- `353ff2f` Retain empty log hashes alongside nonempty qualification results +- `ab8e162` Normalize runtime receipt counts across JSON persistence +- `dd52954` Bind persisted runtime command and child interpreter provenance +- `fce095d` docs: record isolated runtime qualification and rollback evidence +- `d60159d` fix: isolate unavailable runtime lanes at the matrix CLI +- `7684b4f` docs: record isolated lane fix and five-runtime requalification +- `ae4aa08` Pin qualified project runtimes and add verified local launcher +- `64e0b1b` Bind native helper resolution and reject Cargo selector bypasses +- `9d94dbf` Record final runtime launcher qualification and activation handoff +- `e32c968` Reject inherited native flag families and target-specific selectors +- `1870b60` Record native boundary correction verification and retained warnings +- `d370b41` Clarify units of the retained native wrapper audit +- `5e9c76e` Bind rustdoc selection and version unknown-unit replay observations +- `0899eca` Document V2 selection and replay semantics with final source evidence + +### Documentation and warning/log maintenance + +- `7c7ca36` docs: remove unrelated integration references +- `6278767` Separate prepared adapter operations and preserve existing verification logs +- `c7501c4` Document warning cleanup and log preservation verification + From c7703e15639eb0a2ee94aea48c7a3f3e844cde45 Mon Sep 17 00:00:00 2001 From: Alejandro Castellanos <5405628+k2jac9@users.noreply.github.com> Date: Mon, 16 Nov 2026 00:44:16 -0500 Subject: [PATCH 50/51] fix: isolate hosted test environments and pin current checkout action --- .github/workflows/check.yml | 17 ++++- .github/workflows/recovery-policy.yml | 2 +- docs/hosted-integration.md | 41 ++++++++++++ .../test_incident_calibration_controller.py | 64 +++++++++++++++++-- scripts/test_project_runtime.py | 16 ++++- 5 files changed, 130 insertions(+), 10 deletions(-) create mode 100644 docs/hosted-integration.md diff --git a/.github/workflows/check.yml b/.github/workflows/check.yml index 77103e9..5850930 100644 --- a/.github/workflows/check.yml +++ b/.github/workflows/check.yml @@ -59,7 +59,7 @@ jobs: EXPECTED_CARGO_VERSION: ${{ matrix.cargo-version }} EXPECTED_PYTHON_VERSION: ${{ matrix.python-version }} steps: - - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false - name: Select exact Rust lane @@ -158,3 +158,18 @@ jobs: - name: Exercise prepared backend with fake dependencies only run: python3 scripts/incident_transformers_controller.py fixture --out "$RUNNER_TEMP/incident-transformers-fixture" + + checkout-previous: + name: checkout previous stable (v6.1.0) + runs-on: ubuntu-latest + timeout-minutes: 5 + steps: + - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6.1.0 + with: + persist-credentials: false + - name: Verify event checkout and required source files + shell: bash + run: | + test "$(git rev-parse HEAD)" = "$GITHUB_SHA" + test -f Cargo.lock + test -f src/lib.rs diff --git a/.github/workflows/recovery-policy.yml b/.github/workflows/recovery-policy.yml index 4d2d222..f28f1c1 100644 --- a/.github/workflows/recovery-policy.yml +++ b/.github/workflows/recovery-policy.yml @@ -31,7 +31,7 @@ jobs: TOKENIZERS_PARALLELISM: 'false' OMP_NUM_THREADS: '2' steps: - - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 diff --git a/docs/hosted-integration.md b/docs/hosted-integration.md new file mode 100644 index 0000000..30ab56a --- /dev/null +++ b/docs/hosted-integration.md @@ -0,0 +1,41 @@ +# Hosted integration verification + +The first PR #20 run at `e43677d` reached the full Python scripts suite after +passing Rust and example execution. It exposed two test-environment assumptions: + +- The launcher fixture inherited `LD_LIBRARY_PATH` and `PKG_CONFIG_PATH` from + GitHub's Python setup. Its positive tests therefore stopped at the production + refusal boundary before reaching the behavior they intended to test. Fixture + setup now excludes those ambient values. An explicit subprocess regression + verifies the production launcher still refuses both and performs no workload + side effect. Production runtime selection is unchanged. +- A legacy late-receipt test depended on a 130 ms scheduling interval. The + deterministic-clock correction already present in PR #18 is included here, + retaining real child pipes, receipt parsing, settlement, and separate wall-clock + watchdog tests. A narrow procfs teardown-race fix is shared too. + +Before correction the inherited-path case failed locally with the same refusal. +Afterward, all 46 focused runtime/controller tests passed with those paths +present in the parent environment, and the full 377-test scripts suite passed +with the previously verified Rust fixture binaries. This is local evidence; +exact-head Actions results remain the hosted acceptance gate. + +The run also reported checkout v4's deprecated Node 20 runtime. On 16 November +2026, the authoritative actions/checkout release list identified v7.0.1 as +latest stable and v6.1.0 as the newest release of the previous major. The +maintained workflows now pin v7.0.1 to +`3d3c42e5aac5ba805825da76410c181273ba90b1`, preserving `contents: read` and +`persist-credentials: false`. A small separate job verifies the previous-major +checkout at `d23441a48e516b6c34aea4fa41551a30e30af803`. The published v6 beta is +older than both stable lines; no upcoming-release preview was available in the +58-entry release inventory. Its absence is recorded rather than substituting +an old beta. Upstream gives no formal support promise for the previous line. + +The exact prior checkout remains recoverable in `e43677d`; no history was +rewritten. The new action uses Node 24 and its safer fork rules do not require +an opt-out for these push, pull_request and workflow_dispatch triggers. No +live recovery workflow is dispatched by this maintenance change. The recovery +workflow's changed checkout still needs its own future execution evidence. + +Source-bound logs and upstream selection responses are retained under the +project artifact packet `submission-finalization-mkpl39bc/ci-fix/`. diff --git a/scripts/test_incident_calibration_controller.py b/scripts/test_incident_calibration_controller.py index 6afc6bb..0d5e841 100644 --- a/scripts/test_incident_calibration_controller.py +++ b/scripts/test_incident_calibration_controller.py @@ -105,10 +105,57 @@ def test_wrong_quote_runtime_and_malformed_usage_fail_closed(self): self.out=Path(self.tmp.name)/'bad-usage';report=self.execute('wrong_usage') self.assertEqual(report['claimed_slots'],1);self.assertEqual(report['unresolved_hold_units'],269) self.assertEqual(report['known_receipt_units'],0);self.assertFalse(report['complete']) + def execute_late_receipt_at_fixed_clock(self,fault): + # Capture a real successful fixture manifest, then run the unchanged + # child, worker pipes, PreparedBridge, receipt and settlement logic with + # deterministic clock positions. The old +20ms delayed reply had only + # 130ms before the read cutoff, so host scheduling could turn this + # accounting test into a timeout/unknown-hold test instead. + from unittest.mock import patch + import incident_calibration_controller as controller + destination=self.out + self.out=destination.with_name(destination.name+'-seed') + seed=self.execute();self.assertTrue(seed['complete'],self.diagnostics(seed)) + manifest=json.loads((self.out/'manifest.json').read_text()) + self.out=destination;self.out.mkdir() + manifest.update(start_ns=0,work_deadline_ns=100_000_000, + deadline_ns=1_000_000_000,fault=fault) + controller._durable_new(self.out/'manifest.json',manifest) + tick=[10_000_000] + workers=[];popen=controller.subprocess.Popen + def launch(*args,**kwargs): + worker=popen(*args,**kwargs);workers.append(worker);return worker + read=controller._Transport._line + def observed_reply(transport,deadline): + raw=read(transport,deadline) + if deadline==(manifest['deadline_ns']+manifest['work_deadline_ns'])//2: + # Generate was sent at tick10ms; its complete reply arrives at + # tick150ms, late for work but inside the550ms receipt window. + tick[0]=150_000_000 + return raw + with patch.object(controller,'now_ns',lambda mode:tick[0]), \ + patch.object(controller._Transport,'_line',observed_reply), \ + patch.object(controller,'arm_parent_death',lambda parent:None), \ + patch.object(controller.subprocess,'Popen',launch): + # Only the in-process test controller skips parent-death setup; + # its real spawned fixture worker retains the production safeguard. + try:controller._child(self.out,0) + finally: + # Production child exits after this path; the in-process test + # must explicitly close its inherited pipe handles instead. + for worker in workers: + for stream in (worker.stdin,worker.stdout,worker.stderr): + if stream is not None:stream.close() + report=audit_run(self.out) + self.assertEqual(report['claimed_slots'],1,self.diagnostics(report)) + self.assertEqual(json.loads((self.out/'receipt-0.json').read_text())['observed_ns'],150_000_000) + return report + def test_late_valid_reply_charged_and_audit_fsync_failure_not_zeroed(self): - report=self.execute('late_reply',1800) - self.assertFalse(report['complete']);self.assertEqual(report['ledger_charged_units'],180) - self.assertEqual(report['late_receipts'],1);self.assertEqual(report['unresolved_hold_units'],0) + report=self.execute_late_receipt_at_fixed_clock('late_reply') + diagnostic=self.diagnostics(report) + self.assertFalse(report['complete'],diagnostic);self.assertEqual(report['ledger_charged_units'],180,diagnostic) + self.assertEqual(report['late_receipts'],1,diagnostic);self.assertEqual(report['unresolved_hold_units'],0,diagnostic) self.out=Path(self.tmp.name)/'fsync';report=self.execute('receipt_fsync_failure') self.assertFalse(report['audit_complete']);self.assertEqual(report['ledger_charged_units'],180) self.assertEqual(report['unresolved_hold_units'],0) @@ -122,9 +169,10 @@ def test_failed_claim_never_sends_and_eof_retains_hold(self): self.assertEqual(report['claimed_slots'],1);self.assertEqual(report['unresolved_hold_units'],269) self.assertEqual(report['known_receipt_units'],0) def test_late_fallback_receipt_retains_usage_and_timing(self): - report=self.execute('late_receipt_failure',1800) - self.assertFalse(report['complete']);self.assertEqual(report['late_receipts'],1) - self.assertEqual(report['known_receipt_units'],180);self.assertEqual(report['unresolved_hold_units'],269) + report=self.execute_late_receipt_at_fixed_clock('late_receipt_failure') + diagnostic=self.diagnostics(report) + self.assertFalse(report['complete'],diagnostic);self.assertEqual(report['late_receipts'],1,diagnostic) + self.assertEqual(report['known_receipt_units'],180,diagnostic);self.assertEqual(report['unresolved_hold_units'],269,diagnostic) class AuditAndLifetimeTests(unittest.TestCase): def setUp(self): @@ -200,7 +248,9 @@ def running(pid): path=proc_for(pid) if path is None:return False try:return path.joinpath('stat').read_text().split()[2] not in ('Z','X') - except FileNotFoundError:return False + # procfs may report ENOENT or ESRCH when the process disappears + # between locating its entry and reading stat during teardown. + except (FileNotFoundError,ProcessLookupError):return False def children_of(pid): result=[] for path in Path('/proc').glob('[0-9]*'): diff --git a/scripts/test_project_runtime.py b/scripts/test_project_runtime.py index 2032d4e..c0bb3e0 100644 --- a/scripts/test_project_runtime.py +++ b/scripts/test_project_runtime.py @@ -34,7 +34,8 @@ def setUp(self): self.commit = subprocess.check_output(['git', '-C', str(self.repo), 'rev-parse', 'HEAD'], text=True).strip() self.state = self.root / 'state' self.env = {k: v for k, v in os.environ.items() if not k.startswith(('CARGO_', 'RUST', 'PYTHON', 'KAIROTENSOR_')) - and k not in {'CC', 'CXX', 'AR', 'LD', 'CFLAGS', 'CXXFLAGS', 'LDFLAGS', 'VIRTUAL_ENV', 'CONDA_PREFIX'}} + and k not in {'CC', 'CXX', 'AR', 'LD', 'CFLAGS', 'CXXFLAGS', 'LDFLAGS', 'VIRTUAL_ENV', 'CONDA_PREFIX', + 'LD_LIBRARY_PATH', 'PKG_CONFIG_PATH'}} self.env['PYTHONDONTWRITEBYTECODE'] = '1' # Isolated Cargo home avoids workstation config influencing the fixture. self.env['CARGO_HOME'] = str(self.root / 'cargo-home') @@ -248,6 +249,19 @@ def test_git_environment_source_override_refuses(self): r = self.run_launcher('identity', env=dict(self.env, GIT_DIR=str(self.repo / '.git'))) self.assertEqual(r.returncode, 2, r.stderr) + def test_hosted_setup_paths_still_refuse_at_real_launcher_boundary(self): + # Fixture setup excludes these ambient Actions values. Production must + # still refuse them when explicitly supplied to a real invocation. + for name in ('LD_LIBRARY_PATH', 'PKG_CONFIG_PATH'): + with self.subTest(name=name): + marker = self.root / name + result = self.run_launcher('python', '-c', + f'from pathlib import Path; Path({str(marker)!r}).touch()', + env=dict(self.env, **{name: '/unbound/fixture/path'})) + self.assertEqual(result.returncode, 2, result.stderr) + self.assertIn(name, result.stderr) + self.assertFalse(marker.exists()) + def assert_native_overrides_refused(self, cases): for index, (key, value) in enumerate(cases): with self.subTest(key=key, value=value): From afd8c5d9a65167abdc2a8c8c112416b9a3e91847 Mon Sep 17 00:00:00 2001 From: Alejandro Castellanos <5405628+k2jac9@users.noreply.github.com> Date: Mon, 16 Nov 2026 00:59:44 -0500 Subject: [PATCH 51/51] Make prepared worker hang test wait for a durable claim --- docs/hosted-integration.md | 21 +++++++++ .../test_incident_transformers_controller.py | 45 +++++++++++++++++-- 2 files changed, 62 insertions(+), 4 deletions(-) diff --git a/docs/hosted-integration.md b/docs/hosted-integration.md index 30ab56a..8ef29ad 100644 --- a/docs/hosted-integration.md +++ b/docs/hosted-integration.md @@ -39,3 +39,24 @@ workflow's changed checkout still needs its own future execution evidence. Source-bound logs and upstream selection responses are retained under the project artifact packet `submission-finalization-mkpl39bc/ci-fix/`. + +The combined PR #18 Python-previous lane at `10f618c` exposed another startup +assumption: the prepared-worker hang test spent its two-second global budget +before claiming a slot. The supervisor correctly killed the child with no +claimed work and no hold; the test incorrectly expected a post-claim hold. +The load-failure check is now separate. The hang test waits for a real durable +claim within the usual bounded fixture startup window, then advances only the +supervising test process's clock to the unchanged manifest deadline. The real +child, worker, pipes, ledger and process-group termination remain in use. It +requires exactly one claim, the 269-unit hold, no receipts or settlement, zero +model calls, and a timed-out, reaped child killed by the supervisor. + +A retained reproduction delays the real child's readiness by 2.1 seconds: +the original test fails with the hosted zero-claim result, and the corrected +test passes with that same delay. All 14 prepared-controller tests and all +438 Python scripts tests passed locally using the accepted Python runtime and +existing verified Rust fixture binaries. Production code and runtime guards +are unchanged. Evidence is retained in +`submission-finalization-mkpl39bc/pr18/integrated/fixture-fix/`; exact-head hosted +checks remain required. The injected supervisor clock is deterministic test +evidence, not a real-model timing measurement. diff --git a/scripts/test_incident_transformers_controller.py b/scripts/test_incident_transformers_controller.py index a5587f0..846c31a 100644 --- a/scripts/test_incident_transformers_controller.py +++ b/scripts/test_incident_transformers_controller.py @@ -137,13 +137,50 @@ def slow_persistence(path,payload,**kwargs): self.assertTrue(report['audit_complete'],report);self.assertFalse(report['complete'],report) self.assertEqual(report['ledger_charged_units'],180,report);self.assertEqual(report['late_receipts'],0,report) self.assertEqual(json.loads((self.out/'receipt-0.json').read_text())['observed_ns'],50) - def test_worker_load_failure_and_hang(self): + def test_worker_load_failure_never_claims(self): report=self.execute('load_failure') self.assertEqual(report['claimed_slots'],0,report);self.assertEqual(report['ledger_charged_units'],0,report) - self.out=Path(self.temp.name)/'hang' - report=run_prepared(self.out,approval=InvocationApproval('offline_fake_dependencies'),fault='hang_generation',budget_ms=2000,audit_reserve_ms=500) - self.assertFalse(report['complete'],report);self.assertEqual(report['unresolved_hold_units'],269,report) + def test_worker_hang_after_claim_retains_hold_and_is_reaped(self): + import signal,time + from contextlib import closing + from unittest.mock import patch + import incident_calibration_controller as ledger + original_clock=ledger.now_ns + expired_at=None;claim_observed_at=None + def expire_after_claim(mode): + nonlocal expired_at,claim_observed_at + if expired_at is not None:return expired_at + try: + with closing(sqlite3.connect(f'file:{self.out}/ledger.sqlite?mode=ro',uri=True,timeout=0)) as db: + claimed=db.execute("SELECT count(*) FROM events WHERE event='claimed'").fetchone()[0] + except sqlite3.OperationalError: + # The real child may not have created/initialized its ledger yet. + claimed=0 + if claimed: + claim_observed_at=time.monotonic() + expired_at=json.loads((self.out/'manifest.json').read_text())['deadline_ns'] + return expired_at + return original_clock(mode) + # Give startup the usual bounded fixture window. Only the supervising + # test process's clock advances after a real durable claim; the child, + # worker, pipes, ledger and process-group watchdog remain unchanged. + # A two-second global budget previously expired before any claim on CI. + with patch.object(ledger,'now_ns',expire_after_claim): + report=run_prepared(self.out,approval=InvocationApproval('offline_fake_dependencies'),fault='hang_generation',budget_ms=30000,audit_reserve_ms=1000) + self.assertIsNotNone(claim_observed_at,report) + self.assertLess(time.monotonic()-claim_observed_at,5,report) + self.assertFalse(report['complete'],report);self.assertTrue(report['audit_complete'],report) + self.assertEqual(report['claimed_slots'],1,report);self.assertEqual(report['settled_slots'],0,report) + self.assertEqual(report['unresolved_hold_units'],269,report) + self.assertEqual(report['ledger_charged_units'],0,report);self.assertEqual(report['known_receipt_units'],0,report) + self.assertEqual(report['known_observation_units'],0,report);self.assertEqual(report['new_model_calls'],0,report) + self.assertFalse(any(self.out.glob('backend-receipt-*.json'))) + self.assertFalse(any(self.out.glob('receipt-*.json'))) + self.assertTrue(report['supervisor']['timed_out'],report) + self.assertTrue(report['supervisor']['deadline_overrun'],report) self.assertTrue(report['supervisor']['child_reaped'],report) + self.assertTrue(report['supervisor']['group_terminated'],report) + self.assertEqual(report['supervisor']['exit_code'],-signal.SIGKILL,report) def test_no_live_cli_or_artifact_grant(self): import subprocess,sys script=Path(__file__).with_name('incident_transformers_controller.py')