From e6b2cb7de6f63a22892183e0007ad6994b36c0f3 Mon Sep 17 00:00:00 2001 From: Russ Palermo <175215383+palermo-git@users.noreply.github.com> Date: Wed, 19 Aug 2026 19:47:11 -0400 Subject: [PATCH 1/8] build(release): cut and prove the 1.2.0 release candidate (R06) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Settle the source to the 1.2.0 candidate and add the artifact-boundary proof tooling that R06's acceptance requires. Version 1.2.0 (SemVer minor: R04 added the backward-compatible public handle_slot_origin/2 callback). Cut the dated CHANGELOG [1.2.0] section and comparison links; README/ROADMAP describe 1.2.0 as a built, unpublished candidate (not published or tagged). Gates (red-before-green, mutation-proven): - test/release/release_contract_test.exs — version advances past the last published (1.1.0), matches the newest dated CHANGELOG section + comparison links, and pins docs source_ref. Watched RED at 1.1.0 before the bump. - scripts/release/verify_package.sh + package_files.manifest — the built tar's exact 51 regular files must equal the manifest (rejects any glob-matched ignored/untracked leak: README.secret / lib/.env / docs/adr/private-note.md all packaged while the old CI grep gate exited 0), no symlink/special entry, metadata name/version replicant/1.2.0. Red-proven on all four mutations. - test/release/public_surface_test.exs — semantic (not grep) R01-R05 surface: handle_slot_origin/2 + handle_session_identity/2 optional callbacks, the IDENTIFY_SYSTEM query, version-tiered slot-invalidation SQL, the value-free telemetry boundary (a leaked value REDs it — proven), and the R01 fault + absent-slot fail-closed :data_gap halt. Replaces the CI source greps. Build + consume the immutable artifact: - scripts/release/build_candidate.sh — builds ONE tar from `git archive HEAD` (not the checkout, so ignored/untracked files cannot contaminate it), runs the boundary gate, content-addresses + retains it read-only with a backup, and writes a gitignored receipt (version, source commit, mix.lock digest, size, SHA-256). Fail-closed identity-collision checks; never publishes/tags. - scripts/release/consume_candidate.sh (+ unpack_validated.exs, consumer_smoke.exs) — Hex-checksum-validated unpack of the exact retained tar (a tampered byte REDs it), a fresh external consumer compiled warnings-as-errors from only the extraction, provenance-checked to load Replicant from under scratch, then the R01-R05 public surface exercised from the artifact-derived modules. - scripts/release/upload_candidate.exs — guarded EXACT-BYTE uploader for R07 (hex_core :mix_hex_api_release.publish/3, replace:false), dry-run only here: verifies digest identity, Hex-validated metadata, source-commit, and no colliding tag/release, reads NO credential, uploads nothing. CI release-artifact job now runs the manifest gate, a build+consume, and the uploader dry-run in place of the required/forbidden-path greps. Ticket: baselabs/replicant#6. R06 never publishes, tags, or reads a credential; publication is R07 under separate human authorization naming version + digest. --- .github/workflows/ci.yml | 52 ++++--- CHANGELOG.md | 5 +- README.md | 6 +- docs/ROADMAP.md | 12 +- mix.exs | 2 +- scripts/release/build_candidate.sh | 133 ++++++++++++++++++ scripts/release/consume_candidate.sh | 80 +++++++++++ scripts/release/consumer_smoke.exs | 57 ++++++++ scripts/release/package_files.manifest | 51 +++++++ scripts/release/unpack_validated.exs | 23 ++++ scripts/release/upload_candidate.exs | 184 +++++++++++++++++++++++++ scripts/release/verify_package.sh | 121 ++++++++++++++++ test/release/public_surface_test.exs | 97 +++++++++++++ test/release/release_contract_test.exs | 58 ++++++++ 14 files changed, 846 insertions(+), 35 deletions(-) create mode 100755 scripts/release/build_candidate.sh create mode 100755 scripts/release/consume_candidate.sh create mode 100644 scripts/release/consumer_smoke.exs create mode 100644 scripts/release/package_files.manifest create mode 100644 scripts/release/unpack_validated.exs create mode 100644 scripts/release/upload_candidate.exs create mode 100755 scripts/release/verify_package.sh create mode 100644 test/release/public_surface_test.exs create mode 100644 test/release/release_contract_test.exs diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index eb25072..646ba64 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -146,33 +146,27 @@ jobs: - run: mix compile --warnings-as-errors - run: mix docs --warnings-as-errors - - name: Build and inspect unpacked Hex package + # Exact package-boundary gate: the built tar's regular-file set must equal the checked-in + # manifest (rejects any ignored/untracked glob-matched leak such as README.secret / lib/.env), + # with no symlink/special entry, and metadata name/version replicant/<@version>. This replaces + # the prior required/forbidden-path greps, which exited 0 while synthetic secret files packaged. + - name: Verify the package boundary against the exact manifest + run: bash scripts/release/verify_package.sh + + # Build ONE candidate tarball and prove it from a fresh external consumer that touches only + # the Hex-checksum-validated extraction — never this checkout or its _build. A missing/corrupt + # packaged file is invisible from the checkout; it reds here. The consumer smoke exercises the + # R01-R05 public surface semantically (no source greps). + - name: Build the candidate and consume it from a fresh project run: | - package_dir=$(mktemp -d) - trap 'rm -rf "$package_dir"' EXIT - mix hex.build --unpack --output "$package_dir" - - for required in lib/replicant/session_identity.ex docs/adr/0007-actual-replication-session-identity.md .formatter.exs mix.exs README.md LICENSE NOTICE CHANGELOG.md usage-rules.md; do - test -e "$package_dir/$required" || { - echo "::error::Missing package path: $required" - exit 1 - } - done - - test ! -e "$package_dir/AGENTS.md" || { - echo "::error::Package contains the private contributor-agent contract" - exit 1 - } - - if grep -nH 'AGENTS\.md' "$package_dir/README.md" "$package_dir/usage-rules.md" "$package_dir/CONTRIBUTING.md" "$package_dir/notebooks/getting_started.livemd"; then - echo "::error::Published documentation references the excluded AGENTS.md contract" - exit 1 - fi - - if find "$package_dir" -type d \( -name test -o -name .forge -o -name _build -o -name superpowers \) -print -quit | grep -q .; then - echo "::error::Package contains test, lifecycle, Forge, or build residue" - exit 1 - fi - - grep -q 'handle_session_identity' "$package_dir/lib/replicant/sink.ex" - grep -q 'IDENTIFY_SYSTEM' "$package_dir/lib/replicant/query_builder.ex" + mix hex.build --output "$RUNNER_TEMP/replicant.tar" + bash scripts/release/verify_package.sh "$RUNNER_TEMP/replicant.tar" + bash scripts/release/consume_candidate.sh "$RUNNER_TEMP/replicant.tar" + + # Dry-run the guarded exact-byte uploader: all publish preconditions (digest identity, + # Hex-validated metadata, source-commit existence, no colliding tag/release, replace:false) + # must pass with NO credential read and NO upload. Publication is R07, human-authorized. + - name: Dry-run the guarded release uploader (no publish, no credential) + run: | + BUILD_DATE=ci bash scripts/release/build_candidate.sh + mix run scripts/release/upload_candidate.exs diff --git a/CHANGELOG.md b/CHANGELOG.md index 751f93d..6ee5bc4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +## [1.2.0] - 2026-08-19 + ### Added - **Proven support for PostgreSQL 15, 16, 17, and 18, with version-gated capabilities.** The CI @@ -558,7 +560,8 @@ against a real-PG16 crash-injection suite (loss = 0, effect-dup = 0). **permanent** fail-closed halt (operator restart required), not auto-retry (spec §6 / §14.18). -[Unreleased]: https://github.com/baselabs/replicant/compare/v1.1.0...HEAD +[Unreleased]: https://github.com/baselabs/replicant/compare/v1.2.0...HEAD +[1.2.0]: https://github.com/baselabs/replicant/compare/v1.1.0...v1.2.0 [1.1.0]: https://github.com/baselabs/replicant/compare/v1.0.0...v1.1.0 [1.0.0]: https://github.com/baselabs/replicant/compare/v0.3.1...v1.0.0 [0.3.1]: https://github.com/baselabs/replicant/compare/v0.3.0...v0.3.1 diff --git a/README.md b/README.md index bfb6ea4..2cf2920 100644 --- a/README.md +++ b/README.md @@ -10,7 +10,11 @@ consumer sibling to [`arcadic`](https://github.com/baselabs/arcadic). Multitenancy, classification, and Ash resources live one layer up, in the [`ash_replicant`](https://hex.pm/packages/ash_replicant) sink adapter. -> **Status:** 1.1.0 is published on Hex (tagged `v1.1.0`). Replicant owns +> **Status:** 1.1.0 is the latest release published on Hex (tagged `v1.1.0`). 1.2.0 +> is the prepared release candidate — it adds proven PostgreSQL 15–18 support, the typed +> `handle_slot_origin/2` callback, typed-shape telemetry validation, and a value-safety +> regression suite (see CHANGELOG `[1.2.0]`) — and is **not yet published or tagged**. +> Replicant owns > the replication slot via `Postgrex.ReplicationConnection`, acks only after the > sink durably commits (ack-after-checkpoint), halts fail-closed on slot > invalidation, and is proven by a real-PG16 crash-injection suite diff --git a/docs/ROADMAP.md b/docs/ROADMAP.md index 218384e..b79fded 100644 --- a/docs/ROADMAP.md +++ b/docs/ROADMAP.md @@ -1,6 +1,6 @@ # Replicant — Feature Tracker -**Updated:** 2026-08-13 · **Release:** `v1.1.0` (tagged) · **Branch:** `main` +**Updated:** 2026-08-19 · **Latest published:** `v1.1.0` (tagged) · **Candidate:** `1.2.0` (built, unpublished) · **Branch:** `main` > **⚠ Commit-SHA note.** Git history was rewritten after most of this file was > written, so the historical commit SHAs cited in the slice rows below (e.g. @@ -15,8 +15,14 @@ The initial sequencing plan below is complete, and both packages have continued through later releases: -- **`replicant` 1.1.0 is published and tagged** `v1.1.0` (a post-1.0 hardening patch — - see CHANGELOG `[1.1.0]`; 1.0.0 shipped at `v1.0.0`). +- **`replicant` 1.1.0 is the latest published release**, tagged `v1.1.0` (a post-1.0 + hardening patch — see CHANGELOG `[1.1.0]`; 1.0.0 shipped at `v1.0.0`). +- **`replicant` 1.2.0 is a built, verified release candidate — not yet published or + tagged.** It carries the R01–R05 fixes (fail-closed unknown-checkpoint halt, typed + telemetry shapes, logical-message value-safety, the `handle_slot_origin/2` callback, + and proven PostgreSQL 15–18 support); see CHANGELOG `[1.2.0]`. Publication and tagging + require separate explicit human authorization naming the exact version and artifact + digest. - **`ash_replicant` 0.4.0 is published and tagged** `v0.4.0` at `d4e9457`. Its current `main` at `197fca18594a4dbb20e57bc065ef78f30133ae5c` now consumes Replicant `>= 1.0.0 and < 2.0.0-0`, locks 1.1.0, and independently tests exact diff --git a/mix.exs b/mix.exs index 02b9e54..0aec44d 100644 --- a/mix.exs +++ b/mix.exs @@ -1,7 +1,7 @@ defmodule Replicant.MixProject do use Mix.Project - @version "1.1.0" + @version "1.2.0" @source_url "https://github.com/baselabs/replicant" def project do diff --git a/scripts/release/build_candidate.sh b/scripts/release/build_candidate.sh new file mode 100755 index 0000000..350be6e --- /dev/null +++ b/scripts/release/build_candidate.sh @@ -0,0 +1,133 @@ +#!/usr/bin/env bash +# Build exactly ONE immutable Replicant release candidate from a clean, committed source tree +# and retain it — never publish, tag, or read a Hex credential (that is R07, human-authorized). +# +# The candidate is built from `git archive HEAD` into a throwaway tree, NOT from the working +# checkout: `mix hex.build` expands the configured `files:` globs from `File.cwd!/0`, so an +# ignored or untracked matching file (a `README.secret`, a `lib/.env`) would contaminate a +# checkout build. `git archive` contains only committed bytes, so the tarball's contents are +# exactly the reviewed commit. +# +# The result is content-addressed, made read-only, copied to a durable backup, and described in +# a receipt recording the exact source commit, mix.lock digest, size, and SHA-256 — the identity +# R07 verifies before uploading these exact bytes. +# +# Env: +# BUILD_DATE ISO date stamped into the receipt (default: `date -u`); pass for reproducibility. +set -euo pipefail + +repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" +cd "$repo_root" + +artifacts_dir="$repo_root/.kimosabe/artifacts" +mkdir -p "$artifacts_dir/by-digest" + +log() { echo "build_candidate: $*" >&2; } +die() { echo "::error::build_candidate: $*" >&2; exit 1; } + +sha256_of() { + if command -v sha256sum >/dev/null 2>&1; then sha256sum "$1" | awk '{print $1}'; + else shasum -a 256 "$1" | awk '{print $1}'; fi +} + +# --- 0. Source must be clean and committed (settle source before candidate bytes). ---------- +[[ -z "$(git status --porcelain)" ]] || die "working tree is dirty; commit the final source before minting the candidate" + +commit="$(git rev-parse HEAD)" +version="$(grep -oE '@version "[^"]+"' mix.exs | head -1 | sed -E 's/@version "([^"]+)"/\1/')" +[[ -n "$version" ]] || die "could not read @version from mix.exs" +tag="v$version" +log "candidate replicant $version from commit $commit" + +# --- 1. Identity collisions fail closed. ----------------------------------------------------- +# Local tag: a hard check (always available). Remote tag / GitHub release / Hex: best-effort — +# a DETECTED collision aborts; an unavailable check is logged, never a silent fail-open. The +# authoritative pre-upload identity gate is re-run in R07 before any bytes leave. +if git rev-parse -q --verify "refs/tags/$tag" >/dev/null; then + die "local tag $tag already exists — $version identity is taken" +fi +if remote_tag="$(git ls-remote --tags origin "refs/tags/$tag" 2>/dev/null)"; then + [[ -z "$remote_tag" ]] || die "remote tag $tag already exists on origin — $version identity is taken" +else + log "WARN: could not reach origin to check remote tag $tag (offline?); R07 will re-check" +fi +if command -v gh >/dev/null 2>&1; then + if gh release view "$tag" >/dev/null 2>&1; then + die "GitHub release $tag already exists — $version identity is taken" + else + log "GitHub release $tag not found (expected)" + fi +else + log "WARN: gh CLI absent; skipped GitHub release check for $tag" +fi +if hex_out="$(mix hex.info replicant "$version" 2>/dev/null)"; then + if echo "$hex_out" | grep -qiE 'Released|Config:'; then + die "Hex already has replicant $version — refusing to re-mint a published version" + fi +else + log "Hex reports no replicant $version (expected) or is unreachable" +fi + +# --- 2. Build ONCE from the archived commit into a throwaway tree. --------------------------- +build_tree="$(mktemp -d "${TMPDIR:-/tmp}/replicant-candidate.XXXXXX")" +cleanup() { rm -rf "$build_tree"; } +trap cleanup EXIT + +git archive "$commit" | tar -x -C "$build_tree" +staged_tar="$build_tree/replicant-$version.tar" +( cd "$build_tree" && MIX_ENV=dev MIX_BUILD_PATH="$build_tree/_build" mix hex.build --output "$staged_tar" >/dev/null ) +[[ -f "$staged_tar" ]] || die "hex.build produced no tarball" + +digest="$(sha256_of "$staged_tar")" +size="$(wc -c < "$staged_tar" | tr -d ' ')" +lock_digest="$(sha256_of "$repo_root/mix.lock")" +log "built replicant-$version.tar size=$size sha256=$digest" + +# --- 3. Verify the package boundary of the freshly minted bytes BEFORE retaining. ------------ +EXPECTED_VERSION="$version" bash "$repo_root/scripts/release/verify_package.sh" "$staged_tar" >&2 + +# --- 4. Content-address, retain read-only, no-clobber, plus a durable backup. ---------------- +primary="$artifacts_dir/replicant-$version.tar" +by_digest="$artifacts_dir/by-digest/$digest.tar" +backup="$artifacts_dir/replicant-$version.backup.tar" + +if [[ -f "$primary" ]]; then + existing="$(sha256_of "$primary")" + if [[ "$existing" == "$digest" ]]; then + log "candidate already retained with identical digest — idempotent re-run, keeping existing bytes" + else + die "a DIFFERENT replicant-$version.tar is already retained (digest $existing != $digest); a rebuilt tarball is NOT the same artifact — resolve the source drift" + fi +else + install -m 0444 "$staged_tar" "$primary" +fi +[[ -f "$by_digest" ]] || install -m 0444 "$staged_tar" "$by_digest" +[[ -f "$backup" ]] || install -m 0444 "$staged_tar" "$backup" +chmod 0444 "$primary" "$by_digest" "$backup" + +# --- 5. Durable receipt (gitignored). -------------------------------------------------------- +build_date="${BUILD_DATE:-$(date -u +%Y-%m-%dT%H:%M:%SZ)}" +receipt="$artifacts_dir/replicant-$version-receipt.txt" +# The receipt is retained read-only (0444); an idempotent re-run must be able to regenerate it, +# so drop any prior copy before writing rather than `cat >`-ing onto a read-only file. +rm -f "$receipt" +cat > "$receipt" </dev/null | tail -1) +built_at: $build_date +published: NO — R06 never publishes/tags/creates a GitHub release/reads a credential. +EOF +chmod 0444 "$receipt" + +log "receipt written: $receipt" +echo "$digest $primary" diff --git a/scripts/release/consume_candidate.sh b/scripts/release/consume_candidate.sh new file mode 100755 index 0000000..074a70e --- /dev/null +++ b/scripts/release/consume_candidate.sh @@ -0,0 +1,80 @@ +#!/usr/bin/env bash +# Prove the retained Replicant candidate from a FRESH, external consumer that touches only the +# extracted package bytes — not the repository checkout, not the repo's _build. +# +# Steps: Hex-checksum-validate + unpack the exact retained tar -> build a brand-new Mix consumer +# whose only Replicant source is the extraction -> clear MIX_PATH/ERL_LIBS -> deps.get -> compile +# with warnings-as-errors -> run a semantic R01-R05 smoke that asserts the loaded Replicant BEAM +# resolves under the scratch tree. A missing/corrupt packaged file is invisible from the checkout; +# it is caught here. +# +# Usage: consume_candidate.sh [TARBALL] (default: .kimosabe/artifacts/replicant-.tar) +set -euo pipefail + +repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" +log() { echo "consume_candidate: $*" >&2; } +die() { echo "::error::consume_candidate: $*" >&2; exit 1; } + +version="$(grep -oE '@version "[^"]+"' "$repo_root/mix.exs" | head -1 | sed -E 's/@version "([^"]+)"/\1/')" +tarball="${1:-$repo_root/.kimosabe/artifacts/replicant-$version.tar}" +[[ -f "$tarball" ]] || die "candidate tarball not found: $tarball (build it first with build_candidate.sh)" + +scratch="$(mktemp -d "${TMPDIR:-/tmp}/replicant-consume.XXXXXX")" +scratch="$(cd "$scratch" && pwd -P)" +cleanup() { rm -rf "$scratch"; } +trap cleanup EXIT +src="$scratch/replicant_src" +consumer="$scratch/consumer" + +# --- 1. Hex-checksum-validated unpack of the EXACT retained tar (no rebuild). ---------------- +( cd "$repo_root" && mix run --no-start scripts/release/unpack_validated.exs "$tarball" "$src" ) >&2 +[[ -f "$src/mix.exs" ]] || die "extraction produced no mix.exs at $src" + +# --- 2. A brand-new consumer that depends ONLY on the extracted source. ---------------------- +mkdir -p "$consumer/lib" +cat > "$consumer/mix.exs" < 1.20", deps: deps()] + end + + def application, do: [extra_applications: [:logger]] + + defp deps do + [{:replicant, path: "$src"}] + end +end +EOF + +cat > "$consumer/lib/replicant_consumer_smoke.ex" <<'EOF' +defmodule ReplicantConsumerSmoke do + @moduledoc "Compile-only proof that the extracted Replicant package's public API is usable." + def slot_query(v), do: Replicant.QueryBuilder.slot_invalidation_status("s", v) +end +EOF + +# --- 3. Fresh resolve + compile from a clean environment (no repo _build, no MIX_PATH). ------- +log "building fresh consumer against extracted source $src" +( + cd "$consumer" + env -u MIX_PATH -u ERL_LIBS -u REPLICANT_TEST_URL MIX_ENV=dev \ + mix deps.get >&2 + env -u MIX_PATH -u ERL_LIBS -u REPLICANT_TEST_URL MIX_ENV=dev \ + mix compile --warnings-as-errors >&2 + + # Provenance: the resolved dependency source must be the scratch extraction, never the repo. + dep_path="$(env -u MIX_PATH mix run --no-start -e 'IO.puts(Path.expand(Mix.Project.deps_paths()[:replicant]))')" + case "$dep_path" in + "$scratch"/*) : ;; + *) echo "::error::consume_candidate: replicant dep resolved to $dep_path, not under scratch $scratch" >&2; exit 1 ;; + esac + echo "consume_candidate: replicant dep resolves to $dep_path" >&2 + + # --- 4. Semantic R01-R05 smoke from the artifact-derived, freshly compiled modules. -------- + env -u MIX_PATH -u ERL_LIBS -u REPLICANT_TEST_URL MIX_ENV=dev SMOKE_SCRATCH="$scratch" \ + mix run --no-start "$repo_root/scripts/release/consumer_smoke.exs" >&2 +) + +log "OK — candidate $version consumed from a fresh external project ($tarball)" diff --git a/scripts/release/consumer_smoke.exs b/scripts/release/consumer_smoke.exs new file mode 100644 index 0000000..2b4dfe9 --- /dev/null +++ b/scripts/release/consumer_smoke.exs @@ -0,0 +1,57 @@ +# Exercise the R01-R05 public/runtime contracts from the ARTIFACT-DERIVED, freshly compiled +# Replicant — semantically, not by text grep. Run inside the scratch consumer project (which +# depends only on the extracted package source), so `:code.which/1` must resolve Replicant under +# the scratch tree: a leak of the repo checkout or its _build would fail the provenance guard. +# +# Usage (inside the consumer project): SMOKE_SCRATCH= mix run ../consumer_smoke.exs +scratch = System.get_env("SMOKE_SCRATCH") || raise "SMOKE_SCRATCH not set" + +path = Replicant |> :code.which() |> to_string() + +unless String.starts_with?(path, scratch) do + raise "provenance: Replicant loaded from #{path}, not under scratch #{scratch} — the consumer is not running the extracted artifact" +end + +# R04 + D2 — public Sink callbacks are present. +cbs = Replicant.Sink.behaviour_info(:callbacks) +unless {:handle_slot_origin, 2} in cbs, do: raise("R04: handle_slot_origin/2 missing from packaged Sink") +unless {:handle_session_identity, 2} in cbs, do: raise("D2: handle_session_identity/2 missing") + +# D2 — the identity query and struct ship. +unless Replicant.QueryBuilder.identify_system() == "IDENTIFY_SYSTEM", do: raise("D2: IDENTIFY_SYSTEM query missing") +_ = %Replicant.SessionIdentity{system_identifier: 1, timeline_id: 1, current_lsn: 0, database: "x"} + +# R02/R03 — the value-free telemetry boundary rejects a wrong-shape value without echoing it. +secret = "SECRET-ROW-VALUE-consumer" + +try do + Replicant.Telemetry.validate!(%{commit_lsn: secret}) + raise "R02: validate! accepted a string LSN" +rescue + e in ArgumentError -> + if String.contains?(Exception.message(e), secret), do: raise("R02/R03: telemetry leaked the value") +end + +# R05 — version-tiered slot-invalidation query. +{:ok, pg15} = Replicant.QueryBuilder.slot_invalidation_status("s", 150_000) +{:ok, pg17} = Replicant.QueryBuilder.slot_invalidation_status("s", 170_000) +if String.contains?(pg15, "conflicting"), do: raise("R05: PG15 query must not select `conflicting`") +unless String.contains?(pg17, "invalidation_reason"), do: raise("R05: PG17 query must select `invalidation_reason`") + +# R01 — unknown checkpoint + absent slot halts fail-closed, never creating a slot. +state = %Replicant.Connection{ + step: :invalidation_check, + slot_name: "audit_slot", + publication: ["audit_pub"], + snapshot: false, + checkpoint_lsn: 0, + checkpoint_state: :fault, + failover: false +} + +case Replicant.Connection.handle_result([%Postgrex.Result{rows: []}], state) do + {:disconnect, :data_gap} -> :ok + other -> raise "R01: fault+absent slot did not halt fail-closed; got #{inspect(other)}" +end + +IO.puts("consumer_smoke: OK — R01-R05 public surface exercised from #{path}") diff --git a/scripts/release/package_files.manifest b/scripts/release/package_files.manifest new file mode 100644 index 0000000..501e425 --- /dev/null +++ b/scripts/release/package_files.manifest @@ -0,0 +1,51 @@ +.formatter.exs +CHANGELOG.md +CONTRIBUTING.md +LICENSE +NOTICE +README.md +docs/INVARIANTS.md +docs/ROADMAP.md +docs/adr/0001-logical-decoding-messages-delivery-guarantees.md +docs/adr/0002-multi-publication-per-pipeline.md +docs/adr/0003-value-free-error-boundary.md +docs/adr/0004-commit-lsn-transaction-watermark.md +docs/adr/0005-spill-is-ephemeral-scratch.md +docs/adr/0006-fail-closed-supervision.md +docs/adr/0007-actual-replication-session-identity.md +docs/adr/README.md +lib/replicant.ex +lib/replicant/application.ex +lib/replicant/assembler.ex +lib/replicant/assembler/batch.ex +lib/replicant/assembler/streaming.ex +lib/replicant/assembler_server.ex +lib/replicant/casting/array_parser.ex +lib/replicant/casting/types.ex +lib/replicant/change.ex +lib/replicant/checkpoint_store.ex +lib/replicant/config.ex +lib/replicant/connection.ex +lib/replicant/decoder/decoder.ex +lib/replicant/decoder/messages.ex +lib/replicant/decoder/oid_database.ex +lib/replicant/error.ex +lib/replicant/identifier.ex +lib/replicant/pipeline.ex +lib/replicant/query_builder.ex +lib/replicant/schema_change.ex +lib/replicant/session_identity.ex +lib/replicant/sink.ex +lib/replicant/snapshot_progress.ex +lib/replicant/snapshot_window.ex +lib/replicant/snapshotter.ex +lib/replicant/snapshotter/incremental.ex +lib/replicant/spill.ex +lib/replicant/spill/error.ex +lib/replicant/spill/reader.ex +lib/replicant/supervisor.ex +lib/replicant/telemetry.ex +lib/replicant/transaction.ex +mix.exs +notebooks/getting_started.livemd +usage-rules.md diff --git a/scripts/release/unpack_validated.exs b/scripts/release/unpack_validated.exs new file mode 100644 index 0000000..aecdc6c --- /dev/null +++ b/scripts/release/unpack_validated.exs @@ -0,0 +1,23 @@ +# Unpack an EXISTING Hex tarball with Hex checksum validation — NOT a rebuild. +# +# `mix hex.build --unpack` rebuilds the package in memory and unpacks THAT, so it cannot inspect a +# retained artifact. `:mix_hex_tarball.unpack/2` reads the exact tar bytes, recomputes the inner +# checksum, and compares it to the tar's CHECKSUM member — a tampered payload fails closed here. +# +# Usage: mix run scripts/release/unpack_validated.exs +Mix.ensure_application!(:hex) + +[tar, dest] = System.argv() +bin = File.read!(tar) +File.rm_rf!(dest) +File.mkdir_p!(dest) + +case :mix_hex_tarball.unpack(bin, String.to_charlist(dest)) do + {:ok, meta} -> + checksum = meta |> Map.get(:outer_checksum, "") |> Base.encode16(case: :lower) + IO.puts("unpack_validated: OK — Hex checksum validated, extracted to #{dest} (outer_checksum #{checksum})") + + {:error, reason} -> + IO.puts(:stderr, "::error::unpack_validated: Hex checksum validation FAILED: #{inspect(reason)}") + System.halt(1) +end diff --git a/scripts/release/upload_candidate.exs b/scripts/release/upload_candidate.exs new file mode 100644 index 0000000..7016a14 --- /dev/null +++ b/scripts/release/upload_candidate.exs @@ -0,0 +1,184 @@ +# Guarded EXACT-BYTE uploader for the retained Replicant candidate — R07 machinery, landed and +# dry-run-verified in R06. R06 runs this in dry-run ONLY: it never uploads, tags, creates a +# GitHub release, or reads a Hex credential. +# +# Why not `mix hex.publish`: the standard task (`Mix.Tasks.Hex.Publish.create_release/3`) calls +# `Hex.Tar.create!/3` and publishes a freshly rebuilt in-memory tarball — it CANNOT upload the +# retained artifact. The exact-byte path is the pinned hex_core primitive +# `:mix_hex_api_release.publish(config, tarball_bytes, %{replace: false})`, which POSTs the raw +# tarball binary unchanged. (`Hex.API.Release.publish/5` delegates to this same primitive; the +# low-level call is used directly so the uploaded bytes are provably the retained bytes.) +# +# `replace: false` is hardcoded: an existing release is never overwritten. +# +# Usage: +# mix run scripts/release/upload_candidate.exs # dry-run (R06 default) +# mix run scripts/release/upload_candidate.exs --publish # R07 ONLY; also requires +# # REPLICANT_PUBLISH_AUTHORIZED=: +Mix.ensure_application!(:hex) + +defmodule UploadCandidate do + @repo_root Path.expand("../..", __DIR__) + + def run(argv) do + publish? = "--publish" in argv + version = read_version() + artifacts = Path.join([@repo_root, ".kimosabe", "artifacts"]) + tar = Path.join(artifacts, "replicant-#{version}.tar") + receipt = Path.join(artifacts, "replicant-#{version}-receipt.txt") + + checks = [ + check_exists(tar, "retained artifact"), + check_exists(receipt, "receipt"), + check_digest(tar, receipt), + check_metadata(tar, version), + check_source_commit(receipt), + check_identity_free(version) + ] + + Enum.each(checks, fn + {:ok, msg} -> IO.puts("upload_candidate: OK — #{msg}") + {:error, msg} -> abort(msg) + end) + + opts = %{replace: false} + bytes = File.stat!(tar).size + + if publish? do + do_publish(version, tar, opts) + else + IO.puts(""" + upload_candidate: DRY-RUN — all guards passed, nothing uploaded. + would call: :mix_hex_api_release.publish(config, <#{bytes} exact retained bytes>, #{inspect(opts)}) + credential: NOT read (dry-run reads no Hex API key) + R07 requires: --publish AND REPLICANT_PUBLISH_AUTHORIZED=#{version}: + """) + end + end + + defp read_version do + Path.join(@repo_root, "mix.exs") + |> File.read!() + |> then(&Regex.run(~r/@version "([^"]+)"/, &1)) + |> Enum.at(1) + end + + defp check_exists(path, label) do + if File.regular?(path), do: {:ok, "#{label} present (#{path})"}, else: {:error, "#{label} missing: #{path}"} + end + + defp check_digest(tar, receipt) do + actual = sha256(File.read!(tar)) + + recorded = + receipt |> File.read!() |> then(&Regex.run(~r/sha256:\s+([0-9a-f]{64})/, &1)) |> Enum.at(1) + + cond do + is_nil(recorded) -> {:error, "receipt has no sha256"} + actual == recorded -> {:ok, "artifact bytes match the receipt digest (#{String.slice(actual, 0, 12)}…)"} + true -> {:error, "artifact digest #{actual} != receipt #{recorded} — bytes are not the recorded candidate"} + end + end + + defp check_metadata(tar, version) do + dest = Path.join(System.tmp_dir!(), "replicant-upload-meta-#{:erlang.phash2(tar)}") + File.rm_rf!(dest) + File.mkdir_p!(dest) + + try do + case :mix_hex_tarball.unpack(File.read!(tar), String.to_charlist(dest)) do + {:ok, meta} -> + m = Map.new(meta[:metadata] || %{}) + name = m["name"] || m[:name] + ver = m["version"] || m[:version] + + cond do + name != "replicant" -> {:error, "package name #{inspect(name)} != replicant"} + ver != version -> {:error, "package version #{inspect(ver)} != #{version}"} + true -> {:ok, "Hex-validated metadata is replicant #{ver}"} + end + + {:error, reason} -> + {:error, "Hex checksum validation failed: #{inspect(reason)}"} + end + after + File.rm_rf!(dest) + end + end + + defp check_source_commit(receipt) do + commit = + receipt |> File.read!() |> then(&Regex.run(~r/source_commit:\s+([0-9a-f]{40})/, &1)) |> Enum.at(1) + + cond do + is_nil(commit) -> + {:error, "receipt has no 40-char source_commit"} + + System.cmd("git", ["cat-file", "-e", commit <> "^{commit}"], cd: @repo_root, stderr_to_stdout: true) + |> elem(1) != 0 -> + {:error, "recorded source_commit #{commit} is not a commit in this repo"} + + true -> + {:ok, "source_commit #{String.slice(commit, 0, 12)}… exists"} + end + end + + # Identity collisions fail closed: the target version must not already be tagged locally or + # released on GitHub. (Hex existence is re-checked live at publish time by the server via + # replace:false, which rejects a duplicate.) + defp check_identity_free(version) do + tag = "v#{version}" + + tag_taken? = + System.cmd("git", ["rev-parse", "-q", "--verify", "refs/tags/#{tag}"], cd: @repo_root, stderr_to_stdout: true) + |> elem(1) == 0 + + gh_taken? = + case System.find_executable("gh") do + nil -> + false + + gh -> + System.cmd(gh, ["release", "view", tag], cd: @repo_root, stderr_to_stdout: true) |> elem(1) == 0 + end + + cond do + tag_taken? -> {:error, "local tag #{tag} already exists — identity taken"} + gh_taken? -> {:error, "GitHub release #{tag} already exists — identity taken"} + true -> {:ok, "no #{tag} tag or GitHub release yet (identity free)"} + end + end + + defp do_publish(version, tar, opts) do + authorized = System.get_env("REPLICANT_PUBLISH_AUTHORIZED") + digest = sha256(File.read!(tar)) + expected = "#{version}:#{digest}" + + unless authorized == expected do + abort( + "--publish requires REPLICANT_PUBLISH_AUTHORIZED=#{version}: matching the retained " <> + "artifact; refusing to upload without explicit human authorization naming the exact version and digest" + ) + end + + key = System.get_env("HEX_API_KEY") || abort("HEX_API_KEY not set") + config = :mix_hex_core.default_config() |> Map.put(:api_key, key) + + case :mix_hex_api_release.publish(config, File.read!(tar), opts) do + {:ok, {code, _, _}} when code in [200, 201] -> + IO.puts("upload_candidate: published replicant #{version} (exact bytes, #{String.slice(digest, 0, 12)}…)") + + other -> + abort("publish failed: #{inspect(other)}") + end + end + + defp sha256(bin), do: :crypto.hash(:sha256, bin) |> Base.encode16(case: :lower) + + defp abort(msg) do + IO.puts(:stderr, "::error::upload_candidate: #{msg}") + System.halt(1) + end +end + +UploadCandidate.run(System.argv()) diff --git a/scripts/release/verify_package.sh b/scripts/release/verify_package.sh new file mode 100755 index 0000000..86e3a29 --- /dev/null +++ b/scripts/release/verify_package.sh @@ -0,0 +1,121 @@ +#!/usr/bin/env bash +# Package-boundary gate for the Replicant Hex package. +# +# WHY: `mix.exs` `files:` is a glob allowlist (`lib`, `README*`, `docs/adr`, ...). Any +# ignored/untracked file that matches a glob silently enters the tarball — a synthetic +# `README.secret`, `lib/.env`, and `docs/adr/private-note.md` all packaged while the old CI +# grep gate still exited 0. This gate closes that hole: it compares the EXACT set of regular +# files in the built/retained tarball against a checked-in manifest and rejects every missing +# path, every extra path, and every symlink or special entry. It also asserts the package +# metadata name/version. +# +# Text greps are not an acceptance gate: this operates on the real unpacked tar bytes. +# +# Usage: +# verify_package.sh [TARBALL] +# TARBALL omitted -> build a fresh temporary tarball from the working tree (pre-mint check). +# TARBALL given -> verify that exact retained artifact (post-mint / R07 check). +# +# Env: +# EXPECTED_VERSION override the expected package version (default: mix.exs @version). +set -euo pipefail + +repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" +manifest="$repo_root/scripts/release/package_files.manifest" + +if [[ ! -f "$manifest" ]]; then + echo "::error::package manifest missing: $manifest" >&2 + exit 1 +fi + +expected_name="replicant" +expected_version="${EXPECTED_VERSION:-}" +if [[ -z "$expected_version" ]]; then + expected_version="$(grep -oE '@version "[^"]+"' "$repo_root/mix.exs" | head -1 | sed -E 's/@version "([^"]+)"/\1/')" +fi +if [[ -z "$expected_version" ]]; then + echo "::error::could not determine expected version" >&2 + exit 1 +fi + +work="$(mktemp -d)" +trap 'rm -rf "$work"' EXIT + +tarball="${1:-}" +if [[ -z "$tarball" ]]; then + # Build a fresh tarball from the working tree into an isolated build path so this can run + # standalone or from inside another mix invocation without contending on _build locks. + echo "verify_package: building a fresh tarball from the working tree" >&2 + ( cd "$repo_root" && MIX_ENV=dev MIX_BUILD_PATH="$work/_build" mix hex.build --output "$work/pkg.tar" >/dev/null ) + tarball="$work/pkg.tar" +fi + +if [[ ! -f "$tarball" ]]; then + echo "::error::tarball not found: $tarball" >&2 + exit 1 +fi + +# --- 1. Outer members are exactly the Hex envelope. ------------------------------------- +members="$(tar -tf "$tarball" | LC_ALL=C sort | tr '\n' ' ')" +expected_members="CHECKSUM VERSION contents.tar.gz metadata.config " +if [[ "$members" != "$expected_members" ]]; then + echo "::error::unexpected outer tar members: [$members] expected [$expected_members]" >&2 + exit 1 +fi + +# --- 2. Metadata name/version. ----------------------------------------------------------- +tar -xOf "$tarball" metadata.config > "$work/metadata.config" +got_name="$(grep -aoE '\{<<"name">>,<<"[^"]+">>\}' "$work/metadata.config" | head -1 | sed -E 's/.*<<"([^"]+)">>\}/\1/')" +got_version="$(grep -aoE '\{<<"version">>,<<"[^"]+">>\}' "$work/metadata.config" | head -1 | sed -E 's/.*<<"([^"]+)">>\}/\1/')" +if [[ "$got_name" != "$expected_name" ]]; then + echo "::error::package name is '$got_name', expected '$expected_name'" >&2 + exit 1 +fi +if [[ "$got_version" != "$expected_version" ]]; then + echo "::error::package version is '$got_version', expected '$expected_version'" >&2 + exit 1 +fi + +# --- 3. Extract contents and reject symlinks / special entries. -------------------------- +contents="$work/contents" +mkdir -p "$contents" +tar -xOf "$tarball" contents.tar.gz | tar -xzf - -C "$contents" + +if find "$contents" -type l | grep -q .; then + echo "::error::package contains a symlink:" >&2 + find "$contents" -type l | sed 's|^| |' >&2 + exit 1 +fi +if find "$contents" ! -type f ! -type d | grep -q .; then + echo "::error::package contains a special (non-regular, non-directory) entry:" >&2 + find "$contents" ! -type f ! -type d | sed 's|^| |' >&2 + exit 1 +fi + +# --- 4. Exact regular-file set == manifest. ---------------------------------------------- +actual="$work/actual.manifest" +( cd "$contents" && find . -type f | sed 's|^\./||' | LC_ALL=C sort ) > "$actual" + +expected_sorted="$work/expected.manifest" +LC_ALL=C sort "$manifest" > "$expected_sorted" + +extra="$(LC_ALL=C comm -13 "$expected_sorted" "$actual" || true)" +missing="$(LC_ALL=C comm -23 "$expected_sorted" "$actual" || true)" + +status=0 +if [[ -n "$extra" ]]; then + echo "::error::package contains files NOT in the manifest (boundary leak):" >&2 + echo "$extra" | sed 's|^| + |' >&2 + status=1 +fi +if [[ -n "$missing" ]]; then + echo "::error::package is MISSING files listed in the manifest:" >&2 + echo "$missing" | sed 's|^| - |' >&2 + status=1 +fi +if [[ $status -ne 0 ]]; then + exit 1 +fi + +count="$(wc -l < "$actual" | tr -d ' ')" +echo "verify_package: OK — $expected_name $got_version, $count regular files match the manifest, no symlink/special entries" diff --git a/test/release/public_surface_test.exs b/test/release/public_surface_test.exs new file mode 100644 index 0000000..c7f6109 --- /dev/null +++ b/test/release/public_surface_test.exs @@ -0,0 +1,97 @@ +defmodule Replicant.ReleasePublicSurfaceTest do + # The 1.2.0 candidate must actually carry the R01-R05 fixed surfaces, proven by + # SEMANTIC assertions rather than source text greps: a comment or dead string can satisfy + # `grep 'handle_session_identity'`, but cannot satisfy "the behaviour lists the callback" + # or "the query builder emits version-tiered SQL". Each assertion reds if its surface is + # removed, and none touches a live database. The artifact-derived counterpart runs the same + # exercises against the extracted package bytes in scripts/release/consume_candidate.sh. + use ExUnit.Case, async: true + + describe "R04 — typed slot-origin callback + D2 session identity (public Sink surface)" do + test "handle_slot_origin/2 and handle_session_identity/2 are optional Sink callbacks" do + callbacks = Replicant.Sink.behaviour_info(:callbacks) + optional = Replicant.Sink.behaviour_info(:optional_callbacks) + + assert {:handle_slot_origin, 2} in callbacks + assert {:handle_session_identity, 2} in callbacks + assert {:handle_slot_origin, 2} in optional + assert {:handle_session_identity, 2} in optional + end + + test "SessionIdentity carries the four typed identity fields" do + identity = %Replicant.SessionIdentity{ + system_identifier: 1, + timeline_id: 1, + current_lsn: 0, + database: "postgres" + } + + assert Map.keys(identity) |> Enum.sort() == + [:__struct__, :current_lsn, :database, :system_identifier, :timeline_id] + end + + test "the identity query is IDENTIFY_SYSTEM (the actual replication-session identity)" do + assert Replicant.QueryBuilder.identify_system() == "IDENTIFY_SYSTEM" + end + end + + describe "R05 — version-gated slot-invalidation query" do + test "slot_invalidation_status/2 selects a version-tiered column set" do + {:ok, pg15} = Replicant.QueryBuilder.slot_invalidation_status("s", 150_019) + {:ok, pg16} = Replicant.QueryBuilder.slot_invalidation_status("s", 160_014) + {:ok, pg17} = Replicant.QueryBuilder.slot_invalidation_status("s", 170_011) + + # PG15 has no `conflicting` column (added PG16); PG17 adds `invalidation_reason`. + refute pg15 =~ "conflicting" + assert pg16 =~ "conflicting" + refute pg16 =~ "invalidation_reason" + assert pg17 =~ "invalidation_reason" + end + end + + describe "R02/R03 — value-free telemetry boundary (no row/secret bytes escape)" do + test "a wrong-shape value on an allowlisted key raises with the value elided" do + secret = "SECRET-ROW-VALUE-9f3a" + + err = + assert_raise ArgumentError, fn -> + # commit_lsn's contract is :lsn (non-neg integer or nil); a string smuggling a row + # value must be rejected, and the rejection must not echo the bytes. + Replicant.Telemetry.validate!(%{commit_lsn: secret}) + end + + refute Exception.message(err) =~ secret + end + + test "an off-allowlist key is rejected without echoing the arbitrary key/value" do + err = + assert_raise ArgumentError, fn -> + Replicant.Telemetry.validate!(%{"row_password" => "hunter2"}) + end + + refute Exception.message(err) =~ "hunter2" + refute Exception.message(err) =~ "row_password" + end + end + + describe "R01 — unknown checkpoint with absent slot halts fail-closed" do + test "a fault checkpoint with no slot rows never emits CREATE_REPLICATION_SLOT" do + state = %Replicant.Connection{ + step: :invalidation_check, + slot_name: "audit_slot", + publication: ["audit_pub"], + snapshot: false, + checkpoint_lsn: 0, + checkpoint_state: :fault, + failover: false + } + + result = Replicant.Connection.handle_result([%Postgrex.Result{rows: []}], state) + + # The fix: a fail-closed data-gap halt, not a slot creation that would skip WAL. + assert result == {:disconnect, :data_gap} + + refute match?({:query, "CREATE_REPLICATION_SLOT" <> _, _}, result) + end + end +end diff --git a/test/release/release_contract_test.exs b/test/release/release_contract_test.exs new file mode 100644 index 0000000..7d084dd --- /dev/null +++ b/test/release/release_contract_test.exs @@ -0,0 +1,58 @@ +defmodule Replicant.ReleaseContractTest do + # Release identity is a public contract: the package version, the CHANGELOG's cut + # release section, and the doc source-ref tag must agree, and the candidate must + # supersede the last published version. Drift here is quiet until publication or a + # downstream resolution, so it is gated in the cold suite (no live substrate needed). + use ExUnit.Case, async: true + + @changelog Path.expand("../../CHANGELOG.md", __DIR__) + + # The last version published to Hex that this candidate supersedes. Bumped as part of + # cutting each release; a candidate that fails to advance past it reds here rather than + # re-minting an already-published version. + @last_published "1.1.0" + + defp version, do: Mix.Project.config()[:version] + + test "candidate version advances past the last published release" do + assert Version.compare(version(), @last_published) == :gt, + "mix.exs version #{version()} must be strictly greater than the last published " <> + "#{@last_published}; a re-minted published version is a hard stop" + end + + test "CHANGELOG cuts a dated release section for the candidate as the newest release" do + body = File.read!(@changelog) + v = version() + + assert body =~ ~r/^## \[#{Regex.escape(v)}\] - \d{4}-\d{2}-\d{2}$/m, + "CHANGELOG.md has no dated `## [#{v}] - YYYY-MM-DD` release section" + + released = + Regex.scan(~r/^## \[(\d+\.\d+\.\d+)\]/m, body) + |> Enum.map(fn [_, ver] -> ver end) + + assert List.first(released) == v, + "newest released CHANGELOG section is #{inspect(List.first(released))}, expected #{v}" + + assert Version.compare(List.first(released), Enum.at(released, 1)) == :gt, + "the candidate section must be newer than the section beneath it" + end + + test "CHANGELOG comparison links bind the candidate to the last published tag" do + body = File.read!(@changelog) + v = version() + + assert body =~ + ~r{^\[#{Regex.escape(v)}\]: https://github.com/baselabs/replicant/compare/v#{Regex.escape(@last_published)}\.\.\.v#{Regex.escape(v)}$}m, + "missing/incorrect `[#{v}]:` comparison link to v#{@last_published}...v#{v}" + + assert body =~ + ~r{^\[Unreleased\]: https://github.com/baselabs/replicant/compare/v#{Regex.escape(v)}\.\.\.HEAD$}m, + "the [Unreleased] link must compare from the freshly cut v#{v} tag" + end + + test "docs source_ref pins the candidate version tag" do + assert Mix.Project.config()[:docs][:source_ref] == "v#{version()}", + "docs source_ref must be v#{version()} so HexDocs source links resolve to the release tag" + end +end From bc4ea247295d324110bb90699c1c86c116b24eeb Mon Sep 17 00:00:00 2001 From: Russ Palermo <175215383+palermo-git@users.noreply.github.com> Date: Wed, 19 Aug 2026 20:05:55 -0400 Subject: [PATCH 2/8] fix(package): close candidate identity and provenance gaps Fail closed on every remote identity check, cryptographically witness the artifact receipt to its source commit, rehash all retained copies, verify the package from extracted bytes, and keep CI proof builds throwaway-only. --- .github/workflows/ci.yml | 31 +--- scripts/release/build_candidate.sh | 189 +++++++++++----------- scripts/release/consume_candidate.sh | 54 ++++++- scripts/release/package_identity.exs | 151 ++++++++++++++++++ scripts/release/package_witness.exs | 208 +++++++++++++++++++++++++ scripts/release/upload_candidate.exs | 206 +++++++++--------------- scripts/release/verify_package.sh | 4 +- test/release/package_identity_test.exs | 76 +++++++++ test/release/package_witness_test.exs | 82 ++++++++++ test/release/release_contract_test.exs | 7 + 10 files changed, 751 insertions(+), 257 deletions(-) create mode 100644 scripts/release/package_identity.exs create mode 100644 scripts/release/package_witness.exs create mode 100644 test/release/package_identity_test.exs create mode 100644 test/release/package_witness_test.exs diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 646ba64..41abb0b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -146,27 +146,10 @@ jobs: - run: mix compile --warnings-as-errors - run: mix docs --warnings-as-errors - # Exact package-boundary gate: the built tar's regular-file set must equal the checked-in - # manifest (rejects any ignored/untracked glob-matched leak such as README.secret / lib/.env), - # with no symlink/special entry, and metadata name/version replicant/<@version>. This replaces - # the prior required/forbidden-path greps, which exited 0 while synthetic secret files packaged. - - name: Verify the package boundary against the exact manifest - run: bash scripts/release/verify_package.sh - - # Build ONE candidate tarball and prove it from a fresh external consumer that touches only - # the Hex-checksum-validated extraction — never this checkout or its _build. A missing/corrupt - # packaged file is invisible from the checkout; it reds here. The consumer smoke exercises the - # R01-R05 public surface semantically (no source greps). - - name: Build the candidate and consume it from a fresh project - run: | - mix hex.build --output "$RUNNER_TEMP/replicant.tar" - bash scripts/release/verify_package.sh "$RUNNER_TEMP/replicant.tar" - bash scripts/release/consume_candidate.sh "$RUNNER_TEMP/replicant.tar" - - # Dry-run the guarded exact-byte uploader: all publish preconditions (digest identity, - # Hex-validated metadata, source-commit existence, no colliding tag/release, replace:false) - # must pass with NO credential read and NO upload. Publication is R07, human-authorized. - - name: Dry-run the guarded release uploader (no publish, no credential) - run: | - BUILD_DATE=ci bash scripts/release/build_candidate.sh - mix run scripts/release/upload_candidate.exs + # Build throwaway bytes from the exact commit and run the complete package workflow without + # retaining a candidate: exact manifest, Hex validation, artifact-derived audit/compile/docs, + # fresh-consumer provenance + semantic smoke, immutable witness, strict collision checks, and + # credential-free exact-byte uploader dry-run. The real candidate is minted once, from merged + # main, after this review set and all matrix rows settle. + - name: Prove the complete package workflow without minting a candidate + run: BUILD_DATE=ci bash scripts/release/build_candidate.sh --check diff --git a/scripts/release/build_candidate.sh b/scripts/release/build_candidate.sh index 350be6e..dc9ea62 100755 --- a/scripts/release/build_candidate.sh +++ b/scripts/release/build_candidate.sh @@ -1,26 +1,17 @@ #!/usr/bin/env bash -# Build exactly ONE immutable Replicant release candidate from a clean, committed source tree -# and retain it — never publish, tag, or read a Hex credential (that is R07, human-authorized). -# -# The candidate is built from `git archive HEAD` into a throwaway tree, NOT from the working -# checkout: `mix hex.build` expands the configured `files:` globs from `File.cwd!/0`, so an -# ignored or untracked matching file (a `README.secret`, a `lib/.env`) would contaminate a -# checkout build. `git archive` contains only committed bytes, so the tarball's contents are -# exactly the reviewed commit. -# -# The result is content-addressed, made read-only, copied to a durable backup, and described in -# a receipt recording the exact source commit, mix.lock digest, size, and SHA-256 — the identity -# R07 verifies before uploading these exact bytes. -# -# Env: -# BUILD_DATE ISO date stamped into the receipt (default: `date -u`); pass for reproducibility. +# Build and prove package bytes from an exact commit. `--check` uses only throwaway bytes; +# the default mint mode is allowed only on clean main equal to live origin/main. set -euo pipefail repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" cd "$repo_root" -artifacts_dir="$repo_root/.kimosabe/artifacts" -mkdir -p "$artifacts_dir/by-digest" +mode="mint" +if [[ "${1:-}" == "--check" ]]; then + mode="check" + shift +fi +[[ $# -eq 0 ]] || { echo "usage: build_candidate.sh [--check]" >&2; exit 2; } log() { echo "build_candidate: $*" >&2; } die() { echo "::error::build_candidate: $*" >&2; exit 1; } @@ -30,104 +21,110 @@ sha256_of() { else shasum -a 256 "$1" | awk '{print $1}'; fi } -# --- 0. Source must be clean and committed (settle source before candidate bytes). ---------- -[[ -z "$(git status --porcelain)" ]] || die "working tree is dirty; commit the final source before minting the candidate" - +[[ -z "$(git status --porcelain)" ]] || die "working tree is dirty" commit="$(git rev-parse HEAD)" version="$(grep -oE '@version "[^"]+"' mix.exs | head -1 | sed -E 's/@version "([^"]+)"/\1/')" -[[ -n "$version" ]] || die "could not read @version from mix.exs" -tag="v$version" -log "candidate replicant $version from commit $commit" - -# --- 1. Identity collisions fail closed. ----------------------------------------------------- -# Local tag: a hard check (always available). Remote tag / GitHub release / Hex: best-effort — -# a DETECTED collision aborts; an unavailable check is logged, never a silent fail-open. The -# authoritative pre-upload identity gate is re-run in R07 before any bytes leave. -if git rev-parse -q --verify "refs/tags/$tag" >/dev/null; then - die "local tag $tag already exists — $version identity is taken" -fi -if remote_tag="$(git ls-remote --tags origin "refs/tags/$tag" 2>/dev/null)"; then - [[ -z "$remote_tag" ]] || die "remote tag $tag already exists on origin — $version identity is taken" -else - log "WARN: could not reach origin to check remote tag $tag (offline?); R07 will re-check" -fi -if command -v gh >/dev/null 2>&1; then - if gh release view "$tag" >/dev/null 2>&1; then - die "GitHub release $tag already exists — $version identity is taken" +[[ -n "$version" ]] || die "could not read package version" + +if [[ "$mode" == "mint" ]]; then + [[ "$(git branch --show-current)" == "main" ]] || die "candidate mint is allowed only on main" + + if remote_main="$(git ls-remote --exit-code origin refs/heads/main 2>/dev/null)"; then + remote_main="$(printf '%s' "$remote_main" | awk '{print $1}')" + [[ "$remote_main" == "$commit" ]] || die "HEAD $commit does not equal live origin/main $remote_main" else - log "GitHub release $tag not found (expected)" + die "could not prove live origin/main identity" fi -else - log "WARN: gh CLI absent; skipped GitHub release check for $tag" -fi -if hex_out="$(mix hex.info replicant "$version" 2>/dev/null)"; then - if echo "$hex_out" | grep -qiE 'Released|Config:'; then - die "Hex already has replicant $version — refusing to re-mint a published version" - fi -else - log "Hex reports no replicant $version (expected) or is unreachable" fi -# --- 2. Build ONCE from the archived commit into a throwaway tree. --------------------------- -build_tree="$(mktemp -d "${TMPDIR:-/tmp}/replicant-candidate.XXXXXX")" -cleanup() { rm -rf "$build_tree"; } +elixir -r "$repo_root/scripts/release/package_identity.exs" \ + -e 'Replicant.PackageIdentity.verify_candidate!(hd(System.argv()))' -- "$version" + +build_tree="$(mktemp -d "${TMPDIR:-/tmp}/replicant-package.XXXXXX")" +witness_ref="" +cleanup() { + if [[ "$mode" == "check" && -n "$witness_ref" ]]; then + git update-ref -d "$witness_ref" >/dev/null 2>&1 || true + fi + rm -rf "$build_tree" +} trap cleanup EXIT -git archive "$commit" | tar -x -C "$build_tree" +source_archive="$build_tree/source.tar" +git archive --format=tar --output="$source_archive" "$commit" +tar -xf "$source_archive" -C "$build_tree" staged_tar="$build_tree/replicant-$version.tar" ( cd "$build_tree" && MIX_ENV=dev MIX_BUILD_PATH="$build_tree/_build" mix hex.build --output "$staged_tar" >/dev/null ) -[[ -f "$staged_tar" ]] || die "hex.build produced no tarball" +[[ -f "$staged_tar" ]] || die "mix hex.build produced no tarball" digest="$(sha256_of "$staged_tar")" size="$(wc -c < "$staged_tar" | tr -d ' ')" -lock_digest="$(sha256_of "$repo_root/mix.lock")" -log "built replicant-$version.tar size=$size sha256=$digest" +lock_digest="$(sha256_of "$build_tree/mix.lock")" +verification="$build_tree/verification.txt" -# --- 3. Verify the package boundary of the freshly minted bytes BEFORE retaining. ------------ EXPECTED_VERSION="$version" bash "$repo_root/scripts/release/verify_package.sh" "$staged_tar" >&2 +bash "$repo_root/scripts/release/consume_candidate.sh" "$staged_tar" "$verification" >&2 + +if [[ "$mode" == "check" ]]; then + artifacts_dir="$build_tree/artifacts" + witness_ref="refs/attestations/checks/replicant/$version-$commit" +else + artifacts_dir="$repo_root/.kimosabe/artifacts" + witness_ref="refs/attestations/packages/replicant/$version" +fi -# --- 4. Content-address, retain read-only, no-clobber, plus a durable backup. ---------------- primary="$artifacts_dir/replicant-$version.tar" by_digest="$artifacts_dir/by-digest/$digest.tar" -backup="$artifacts_dir/replicant-$version.backup.tar" +backup="$artifacts_dir/backups/replicant-$version.tar" +receipt="$artifacts_dir/replicant-$version-receipt.txt" -if [[ -f "$primary" ]]; then - existing="$(sha256_of "$primary")" - if [[ "$existing" == "$digest" ]]; then - log "candidate already retained with identical digest — idempotent re-run, keeping existing bytes" - else - die "a DIFFERENT replicant-$version.tar is already retained (digest $existing != $digest); a rebuilt tarball is NOT the same artifact — resolve the source drift" - fi -else - install -m 0444 "$staged_tar" "$primary" +for target in "$primary" "$by_digest" "$backup" "$receipt"; do + [[ ! -e "$target" ]] || die "refusing to overwrite existing package evidence: $target" +done +if git show-ref --verify --quiet "$witness_ref"; then + die "package witness already exists: $witness_ref" fi -[[ -f "$by_digest" ]] || install -m 0444 "$staged_tar" "$by_digest" -[[ -f "$backup" ]] || install -m 0444 "$staged_tar" "$backup" -chmod 0444 "$primary" "$by_digest" "$backup" -# --- 5. Durable receipt (gitignored). -------------------------------------------------------- -build_date="${BUILD_DATE:-$(date -u +%Y-%m-%dT%H:%M:%SZ)}" -receipt="$artifacts_dir/replicant-$version-receipt.txt" -# The receipt is retained read-only (0444); an idempotent re-run must be able to regenerate it, -# so drop any prior copy before writing rather than `cat >`-ing onto a read-only file. -rm -f "$receipt" -cat > "$receipt" </dev/null | tail -1) -built_at: $build_date -published: NO — R06 never publishes/tags/creates a GitHub release/reads a credential. -EOF -chmod 0444 "$receipt" - -log "receipt written: $receipt" +elixir -r "$repo_root/scripts/release/package_witness.exs" \ + -e 'Replicant.PackageWitness.retain_copies!(hd(System.argv()), tl(System.argv()))' -- \ + "$staged_tar" "$primary" "$by_digest" "$backup" + +receipt_tmp="$build_tree/receipt.txt" +{ + echo "Replicant package candidate receipt" + echo "version: $version" + echo "source_commit: $commit" + echo "artifact: $primary" + echo "artifact_backup: $backup" + echo "artifact_by_digest: $by_digest" + echo "size_bytes: $size" + echo "sha256: $digest" + echo "source_mix_lock_sha256: $lock_digest" + echo "elixir: $(elixir --version 2>/dev/null | tail -1)" + echo "built_at: ${BUILD_DATE:-$(date -u +%Y-%m-%dT%H:%M:%SZ)}" + echo "published: NO" + echo "" + cat "$verification" +} > "$receipt_tmp" + +elixir -r "$repo_root/scripts/release/package_witness.exs" \ + -e 'Replicant.PackageWitness.retain_copies!(hd(System.argv()), tl(System.argv()))' -- \ + "$receipt_tmp" "$receipt" + +elixir -r "$repo_root/scripts/release/package_witness.exs" \ + -e 'Replicant.PackageWitness.verify_copies!(tl(System.argv()), hd(System.argv()))' -- \ + "$digest" "$primary" "$by_digest" "$backup" + +elixir -r "$repo_root/scripts/release/package_witness.exs" \ + -e 'Replicant.PackageWitness.create!(Enum.at(System.argv(), 0), Enum.at(System.argv(), 1), Enum.at(System.argv(), 2), Enum.at(System.argv(), 3))' -- \ + "$repo_root" "$witness_ref" "$commit" "$receipt" + +mix run --no-start "$repo_root/scripts/release/upload_candidate.exs" -- \ + --artifact "$primary" --receipt "$receipt" --witness-ref "$witness_ref" + +if [[ "$mode" == "check" ]]; then + log "CHECK PASS — throwaway package built, audited, documented, consumed, and uploader dry-run verified" +else + log "MINT PASS — immutable candidate retained and witnessed at $witness_ref" +fi echo "$digest $primary" diff --git a/scripts/release/consume_candidate.sh b/scripts/release/consume_candidate.sh index 074a70e..c136236 100755 --- a/scripts/release/consume_candidate.sh +++ b/scripts/release/consume_candidate.sh @@ -8,7 +8,7 @@ # resolves under the scratch tree. A missing/corrupt packaged file is invisible from the checkout; # it is caught here. # -# Usage: consume_candidate.sh [TARBALL] (default: .kimosabe/artifacts/replicant-.tar) +# Usage: consume_candidate.sh [TARBALL] [VERIFICATION_RECEIPT] set -euo pipefail repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" @@ -17,6 +17,7 @@ die() { echo "::error::consume_candidate: $*" >&2; exit 1; } version="$(grep -oE '@version "[^"]+"' "$repo_root/mix.exs" | head -1 | sed -E 's/@version "([^"]+)"/\1/')" tarball="${1:-$repo_root/.kimosabe/artifacts/replicant-$version.tar}" +verification_receipt="${2:-}" [[ -f "$tarball" ]] || die "candidate tarball not found: $tarball (build it first with build_candidate.sh)" scratch="$(mktemp -d "${TMPDIR:-/tmp}/replicant-consume.XXXXXX")" @@ -26,11 +27,30 @@ trap cleanup EXIT src="$scratch/replicant_src" consumer="$scratch/consumer" +sha256_of() { + if command -v sha256sum >/dev/null 2>&1; then sha256sum "$1" | awk '{print $1}'; + else shasum -a 256 "$1" | awk '{print $1}'; fi +} + # --- 1. Hex-checksum-validated unpack of the EXACT retained tar (no rebuild). ---------------- ( cd "$repo_root" && mix run --no-start scripts/release/unpack_validated.exs "$tarball" "$src" ) >&2 [[ -f "$src/mix.exs" ]] || die "extraction produced no mix.exs at $src" -# --- 2. A brand-new consumer that depends ONLY on the extracted source. ---------------------- +# --- 2. Audit, compile, and render docs from the exact extracted package source. --------------- +log "auditing and building the extracted package source" +( + cd "$src" + env -u MIX_PATH -u ERL_LIBS -u REPLICANT_TEST_URL MIX_ENV=dev mix deps.get >&2 + env -u MIX_PATH -u ERL_LIBS -u REPLICANT_TEST_URL MIX_ENV=dev mix audit >&2 + env -u MIX_PATH -u ERL_LIBS -u REPLICANT_TEST_URL MIX_ENV=dev mix compile --warnings-as-errors >&2 + env -u MIX_PATH -u ERL_LIBS -u REPLICANT_TEST_URL MIX_ENV=dev mix docs --warnings-as-errors >&2 + env -u MIX_PATH -u ERL_LIBS -u REPLICANT_TEST_URL MIX_ENV=dev mix deps.tree --format plain > "$scratch/package-deps.txt" +) + +package_lock_digest="$(sha256_of "$src/mix.lock")" +package_deps_digest="$(sha256_of "$scratch/package-deps.txt")" + +# --- 3. A brand-new consumer that depends ONLY on the extracted source. ---------------------- mkdir -p "$consumer/lib" cat > "$consumer/mix.exs" <&2 env -u MIX_PATH -u ERL_LIBS -u REPLICANT_TEST_URL MIX_ENV=dev \ mix compile --warnings-as-errors >&2 + env -u MIX_PATH -u ERL_LIBS -u REPLICANT_TEST_URL MIX_ENV=dev \ + mix deps.tree --format plain > "$scratch/consumer-deps.txt" # Provenance: the resolved dependency source must be the scratch extraction, never the repo. dep_path="$(env -u MIX_PATH mix run --no-start -e 'IO.puts(Path.expand(Mix.Project.deps_paths()[:replicant]))')" @@ -72,9 +94,33 @@ log "building fresh consumer against extracted source $src" esac echo "consume_candidate: replicant dep resolves to $dep_path" >&2 - # --- 4. Semantic R01-R05 smoke from the artifact-derived, freshly compiled modules. -------- + # --- 5. Semantic R01-R05 smoke from the artifact-derived, freshly compiled modules. -------- env -u MIX_PATH -u ERL_LIBS -u REPLICANT_TEST_URL MIX_ENV=dev SMOKE_SCRATCH="$scratch" \ mix run --no-start "$repo_root/scripts/release/consumer_smoke.exs" >&2 ) +consumer_lock_digest="$(sha256_of "$consumer/mix.lock")" +consumer_deps_digest="$(sha256_of "$scratch/consumer-deps.txt")" +artifact_digest="$(sha256_of "$tarball")" + +if [[ -n "$verification_receipt" ]]; then + { + echo "artifact_sha256: $artifact_digest" + echo "package_lock_sha256: $package_lock_digest" + echo "package_dependency_tree_sha256: $package_deps_digest" + echo "consumer_lock_sha256: $consumer_lock_digest" + echo "consumer_dependency_tree_sha256: $consumer_deps_digest" + echo "package_audit: PASS" + echo "package_compile: PASS" + echo "package_docs: PASS" + echo "consumer_compile: PASS" + echo "consumer_smoke: PASS" + echo "" + echo "package_dependency_tree:" + sed 's/^/ /' "$scratch/package-deps.txt" + echo "consumer_dependency_tree:" + sed 's/^/ /' "$scratch/consumer-deps.txt" + } > "$verification_receipt" +fi + log "OK — candidate $version consumed from a fresh external project ($tarball)" diff --git a/scripts/release/package_identity.exs b/scripts/release/package_identity.exs new file mode 100644 index 0000000..c236960 --- /dev/null +++ b/scripts/release/package_identity.exs @@ -0,0 +1,151 @@ +defmodule Replicant.PackageIdentity do + @moduledoc false + + @repo_root Path.expand("../..", __DIR__) + @github_release_url "https://api.github.com/repos/baselabs/replicant/releases/tags/" + @hex_release_url "https://hex.pm/api/packages/replicant/releases/" + + def check_candidate(version, runner \\ &run/2) do + tag = "v#{version}" + + with :ok <- absent_local_tag(tag, runner), + :ok <- absent_remote_tag(tag, runner), + :ok <- absent_http(@github_release_url <> tag, "GitHub release", runner), + :ok <- absent_http(@hex_release_url <> version, "Hex release", runner) do + :ok + end + end + + def check_publish(version, source_commit, runner \\ &run/2) do + tag = "v#{version}" + + with :ok <- matching_local_tag(tag, source_commit, runner), + :ok <- matching_remote_tag(tag, source_commit, runner), + :ok <- absent_http(@github_release_url <> tag, "GitHub release", runner), + :ok <- absent_http(@hex_release_url <> version, "Hex release", runner) do + :ok + end + end + + def verify_candidate!(version), do: check_candidate(version) |> unwrap!() + + def verify_publish!(version, source_commit), + do: check_publish(version, source_commit) |> unwrap!() + + defp absent_local_tag(tag, runner) do + case runner.("git", ["rev-parse", "-q", "--verify", "refs/tags/#{tag}"]) do + {_, 1} -> + :ok + + {_, 0} -> + {:error, "local tag #{tag} already exists"} + + {output, status} -> + {:error, "local tag check failed (exit #{status}): #{structural(output)}"} + end + end + + defp absent_remote_tag(tag, runner) do + case runner.("git", ["ls-remote", "--exit-code", "--tags", "origin", "refs/tags/#{tag}"]) do + {_, 2} -> + :ok + + {_, 0} -> + {:error, "remote tag #{tag} already exists on origin"} + + {output, status} -> + {:error, "remote tag check failed (exit #{status}): #{structural(output)}"} + end + end + + defp matching_local_tag(tag, commit, runner) do + case runner.("git", ["rev-parse", "refs/tags/#{tag}^{}"]) do + {output, 0} -> + compare_tag("local", tag, String.trim(output), commit) + + {output, status} -> + {:error, "local tag #{tag} is unavailable (exit #{status}): #{structural(output)}"} + end + end + + defp matching_remote_tag(tag, commit, runner) do + args = [ + "ls-remote", + "--exit-code", + "--tags", + "origin", + "refs/tags/#{tag}", + "refs/tags/#{tag}^{}" + ] + + case runner.("git", args) do + {output, 0} -> + resolved = remote_tag_commit(output, tag) + + if is_nil(resolved) do + {:error, "remote tag #{tag} returned no parseable commit"} + else + compare_tag("remote", tag, resolved, commit) + end + + {output, status} -> + {:error, "remote tag #{tag} is unavailable (exit #{status}): #{structural(output)}"} + end + end + + defp remote_tag_commit(output, tag) do + refs = + output + |> String.split("\n", trim: true) + |> Enum.map(&String.split(&1, "\t", parts: 2)) + |> Map.new(fn [sha, ref] -> {ref, sha} end) + + refs["refs/tags/#{tag}^{}"] || refs["refs/tags/#{tag}"] + end + + defp compare_tag(_location, _tag, commit, commit), do: :ok + + defp compare_tag(location, tag, actual, expected) do + {:error, "#{location} tag #{tag} resolves to #{actual}, expected recorded source #{expected}"} + end + + defp absent_http(url, label, runner) do + args = [ + "--silent", + "--show-error", + "--output", + "/dev/null", + "--write-out", + "%{http_code}", + url + ] + + case runner.("curl", args) do + {"404", 0} -> + :ok + + {"200", 0} -> + {:error, "#{label} already exists"} + + {status, 0} -> + {:error, "#{label} check returned HTTP #{String.trim(status)}"} + + {output, exit_status} -> + {:error, "#{label} check failed (exit #{exit_status}): #{structural(output)}"} + end + end + + defp run(command, args) do + case System.find_executable(command) do + nil -> {"required command unavailable", 127} + executable -> System.cmd(executable, args, cd: @repo_root, stderr_to_stdout: true) + end + end + + defp structural(output) do + if String.trim(output) == "", do: "no output", else: "command returned output" + end + + defp unwrap!(:ok), do: :ok + defp unwrap!({:error, message}), do: raise(message) +end diff --git a/scripts/release/package_witness.exs b/scripts/release/package_witness.exs new file mode 100644 index 0000000..8565ad9 --- /dev/null +++ b/scripts/release/package_witness.exs @@ -0,0 +1,208 @@ +defmodule Replicant.PackageWitness do + @moduledoc false + + @zero String.duplicate("0", 40) + + def create(repo, ref, source_commit, receipt_path) do + receipt = File.read!(receipt_path) + + with :ok <- receipt_commit_matches(receipt, source_commit), + :ok <- commit_exists(repo, source_commit), + {:ok, witness} <- create_commit(repo, source_commit, receipt_path, receipt), + :ok <- create_ref(repo, ref, witness), + :ok <- verify(repo, ref, nil, receipt_path) do + :ok + end + end + + def create!(repo, ref, source_commit, receipt_path) do + case create(repo, ref, source_commit, receipt_path) do + :ok -> :ok + {:error, message} -> raise message + end + end + + def verify(repo, ref, artifact_path, receipt_path) do + receipt = File.read!(receipt_path) + + with {:ok, witnessed} <- git_output(repo, ["show", "#{ref}:candidate-receipt.txt"]), + :ok <- exact_receipt(witnessed, receipt), + {:ok, parent} <- git_output(repo, ["rev-parse", "#{ref}^"]), + {:ok, commit} <- receipt_value(receipt, "source_commit", ~r/^[0-9a-f]{40}$/), + :ok <- exact_parent(String.trim(parent), commit), + :ok <- verify_artifact(artifact_path, receipt) do + :ok + end + end + + def verify!(repo, ref, artifact_path, receipt_path) do + case verify(repo, ref, artifact_path, receipt_path) do + :ok -> :ok + {:error, message} -> raise message + end + end + + def verify_copies(paths, expected_digest) do + Enum.reduce_while(paths, :ok, fn path, :ok -> + cond do + not File.regular?(path) -> + {:halt, {:error, "retained copy missing or non-regular: #{path}"}} + + sha256(File.read!(path)) != expected_digest -> + {:halt, {:error, "retained copy digest mismatch: #{path}"}} + + true -> + {:cont, :ok} + end + end) + end + + def verify_copies!(paths, expected_digest) do + case verify_copies(paths, expected_digest) do + :ok -> :ok + {:error, message} -> raise message + end + end + + def retain_copies!(source, destinations) do + bytes = File.read!(source) + + Enum.each(destinations, fn destination -> + File.mkdir_p!(Path.dirname(destination)) + + case File.open(destination, [:write, :binary, :exclusive]) do + {:ok, io} -> + try do + IO.binwrite(io, bytes) + after + File.close(io) + end + + File.chmod!(destination, 0o444) + + {:error, :eexist} -> + raise "refusing to overwrite retained package copy: #{destination}" + + {:error, reason} -> + raise "could not retain package copy #{destination}: #{inspect(reason)}" + end + end) + end + + def receipt_source_commit(receipt_path) do + receipt_path |> File.read!() |> receipt_value("source_commit", ~r/^[0-9a-f]{40}$/) + end + + defp create_commit(repo, source_commit, receipt_path, _receipt) do + scratch = + Path.join( + System.tmp_dir!(), + "replicant-package-witness-#{System.unique_integer([:positive])}" + ) + + File.mkdir_p!(scratch) + + try do + tree_input = Path.join(scratch, "tree") + message = Path.join(scratch, "message") + File.write!(message, "Replicant package witness\n") + + with {:ok, blob} <- git_output(repo, ["hash-object", "-w", receipt_path]), + :ok <- + File.write(tree_input, "100644 blob #{String.trim(blob)}\tcandidate-receipt.txt\n"), + {:ok, tree} <- git_mktree(repo, tree_input), + {:ok, witness} <- + git_output(repo, [ + "commit-tree", + String.trim(tree), + "-p", + source_commit, + "-F", + message + ]) do + {:ok, String.trim(witness)} + end + after + File.rm_rf!(scratch) + end + end + + defp create_ref(repo, ref, witness) do + case System.cmd("git", ["update-ref", ref, witness, @zero], cd: repo, stderr_to_stdout: true) do + {_, 0} -> :ok + {_, _} -> {:error, "package witness ref already exists or could not be created: #{ref}"} + end + end + + defp commit_exists(repo, commit) do + case System.cmd("git", ["cat-file", "-e", "#{commit}^{commit}"], + cd: repo, + stderr_to_stdout: true + ) do + {_, 0} -> :ok + {_, _} -> {:error, "recorded source commit is not present"} + end + end + + defp receipt_commit_matches(receipt, commit) do + case receipt_value(receipt, "source_commit", ~r/^[0-9a-f]{40}$/) do + {:ok, ^commit} -> :ok + {:ok, _} -> {:error, "receipt source commit does not match witness parent"} + error -> error + end + end + + defp exact_receipt(receipt, receipt), do: :ok + defp exact_receipt(_, _), do: {:error, "receipt does not match immutable witness"} + + defp exact_parent(commit, commit), do: :ok + defp exact_parent(_, _), do: {:error, "witness parent does not match receipt source commit"} + + defp verify_artifact(nil, _receipt), do: :ok + + defp verify_artifact(path, receipt) do + with {:ok, recorded} <- receipt_value(receipt, "sha256", ~r/^[0-9a-f]{64}$/) do + cond do + not File.regular?(path) -> {:error, "candidate artifact missing or non-regular"} + sha256(File.read!(path)) == recorded -> :ok + true -> {:error, "candidate artifact digest does not match witnessed receipt"} + end + end + end + + defp receipt_value(receipt, key, format) do + case Regex.run(~r/^#{Regex.escape(key)}:\s*(\S+)\s*$/m, receipt) do + [_, value] -> + if value =~ format, do: {:ok, value}, else: {:error, "receipt has malformed #{key}"} + + _ -> + {:error, "receipt has no #{key}"} + end + end + + defp git_output(repo, args) do + env = [ + {"GIT_AUTHOR_NAME", "Replicant Package Witness"}, + {"GIT_AUTHOR_EMAIL", "replicant@example.invalid"}, + {"GIT_COMMITTER_NAME", "Replicant Package Witness"}, + {"GIT_COMMITTER_EMAIL", "replicant@example.invalid"} + ] + + case System.cmd("git", args, cd: repo, env: env, stderr_to_stdout: true) do + {output, 0} -> {:ok, output} + {_, _} -> {:error, "package witness is missing or unreadable"} + end + end + + defp git_mktree(repo, input_path) do + case System.cmd("sh", ["-c", "git mktree < \"$1\"", "sh", input_path], + cd: repo, + stderr_to_stdout: true + ) do + {output, 0} -> {:ok, output} + {_, _} -> {:error, "could not create package witness tree"} + end + end + + defp sha256(bytes), do: :crypto.hash(:sha256, bytes) |> Base.encode16(case: :lower) +end diff --git a/scripts/release/upload_candidate.exs b/scripts/release/upload_candidate.exs index 7016a14..e9e2d92 100644 --- a/scripts/release/upload_candidate.exs +++ b/scripts/release/upload_candidate.exs @@ -1,58 +1,50 @@ -# Guarded EXACT-BYTE uploader for the retained Replicant candidate — R07 machinery, landed and -# dry-run-verified in R06. R06 runs this in dry-run ONLY: it never uploads, tags, creates a -# GitHub release, or reads a Hex credential. -# -# Why not `mix hex.publish`: the standard task (`Mix.Tasks.Hex.Publish.create_release/3`) calls -# `Hex.Tar.create!/3` and publishes a freshly rebuilt in-memory tarball — it CANNOT upload the -# retained artifact. The exact-byte path is the pinned hex_core primitive -# `:mix_hex_api_release.publish(config, tarball_bytes, %{replace: false})`, which POSTs the raw -# tarball binary unchanged. (`Hex.API.Release.publish/5` delegates to this same primitive; the -# low-level call is used directly so the uploaded bytes are provably the retained bytes.) -# -# `replace: false` is hardcoded: an existing release is never overwritten. -# -# Usage: -# mix run scripts/release/upload_candidate.exs # dry-run (R06 default) -# mix run scripts/release/upload_candidate.exs --publish # R07 ONLY; also requires -# # REPLICANT_PUBLISH_AUTHORIZED=: +# Guarded exact-byte uploader for a witnessed Replicant package candidate. +# Dry-run is the default and never reads a credential or sends package bytes. Mix.ensure_application!(:hex) -defmodule UploadCandidate do +Code.require_file("package_identity.exs", __DIR__) +Code.require_file("package_witness.exs", __DIR__) + +defmodule Replicant.UploadCandidate do + @moduledoc false + @repo_root Path.expand("../..", __DIR__) def run(argv) do - publish? = "--publish" in argv + {opts, rest, invalid} = + OptionParser.parse(argv, + strict: [publish: :boolean, artifact: :string, receipt: :string, witness_ref: :string] + ) + + if rest != [] or invalid != [], do: abort("invalid arguments") + version = read_version() artifacts = Path.join([@repo_root, ".kimosabe", "artifacts"]) - tar = Path.join(artifacts, "replicant-#{version}.tar") - receipt = Path.join(artifacts, "replicant-#{version}-receipt.txt") - - checks = [ - check_exists(tar, "retained artifact"), - check_exists(receipt, "receipt"), - check_digest(tar, receipt), - check_metadata(tar, version), - check_source_commit(receipt), - check_identity_free(version) - ] - - Enum.each(checks, fn - {:ok, msg} -> IO.puts("upload_candidate: OK — #{msg}") - {:error, msg} -> abort(msg) - end) - - opts = %{replace: false} - bytes = File.stat!(tar).size - - if publish? do - do_publish(version, tar, opts) + tar = opts[:artifact] || Path.join(artifacts, "replicant-#{version}.tar") + receipt = opts[:receipt] || Path.join(artifacts, "replicant-#{version}-receipt.txt") + witness_ref = opts[:witness_ref] || "refs/attestations/packages/replicant/#{version}" + publish? = opts[:publish] || false + + with :ok <- regular_file(tar), + :ok <- regular_file(receipt), + :ok <- Replicant.PackageWitness.verify(@repo_root, witness_ref, tar, receipt), + {:ok, source_commit} <- Replicant.PackageWitness.receipt_source_commit(receipt), + :ok <- check_metadata(tar, version), + :ok <- check_identity(version, source_commit, publish?) do + if publish? do + publish(version, tar) + else + bytes = File.stat!(tar).size + + IO.puts(""" + upload_candidate: DRY-RUN — all guards passed, nothing uploaded. + would call: Hex.API.Release.publish("hexpm", <#{bytes} exact witnessed bytes>, [key: ], , false) + credential: NOT read + publication additionally requires --publish and exact version:digest authorization + """) + end else - IO.puts(""" - upload_candidate: DRY-RUN — all guards passed, nothing uploaded. - would call: :mix_hex_api_release.publish(config, <#{bytes} exact retained bytes>, #{inspect(opts)}) - credential: NOT read (dry-run reads no Hex API key) - R07 requires: --publish AND REPLICANT_PUBLISH_AUTHORIZED=#{version}: - """) + {:error, message} -> abort(message) end end @@ -60,125 +52,77 @@ defmodule UploadCandidate do Path.join(@repo_root, "mix.exs") |> File.read!() |> then(&Regex.run(~r/@version "([^"]+)"/, &1)) - |> Enum.at(1) - end - - defp check_exists(path, label) do - if File.regular?(path), do: {:ok, "#{label} present (#{path})"}, else: {:error, "#{label} missing: #{path}"} + |> case do + [_, version] -> version + _ -> abort("could not read package version") + end end - defp check_digest(tar, receipt) do - actual = sha256(File.read!(tar)) - - recorded = - receipt |> File.read!() |> then(&Regex.run(~r/sha256:\s+([0-9a-f]{64})/, &1)) |> Enum.at(1) - - cond do - is_nil(recorded) -> {:error, "receipt has no sha256"} - actual == recorded -> {:ok, "artifact bytes match the receipt digest (#{String.slice(actual, 0, 12)}…)"} - true -> {:error, "artifact digest #{actual} != receipt #{recorded} — bytes are not the recorded candidate"} - end + defp regular_file(path) do + if File.regular?(path), do: :ok, else: {:error, "required regular file missing: #{path}"} end defp check_metadata(tar, version) do - dest = Path.join(System.tmp_dir!(), "replicant-upload-meta-#{:erlang.phash2(tar)}") - File.rm_rf!(dest) + dest = + Path.join(System.tmp_dir!(), "replicant-upload-meta-#{System.unique_integer([:positive])}") + File.mkdir_p!(dest) try do case :mix_hex_tarball.unpack(File.read!(tar), String.to_charlist(dest)) do {:ok, meta} -> - m = Map.new(meta[:metadata] || %{}) - name = m["name"] || m[:name] - ver = m["version"] || m[:version] + metadata = Map.new(meta[:metadata] || %{}) + name = metadata["name"] || metadata[:name] + found_version = metadata["version"] || metadata[:version] cond do - name != "replicant" -> {:error, "package name #{inspect(name)} != replicant"} - ver != version -> {:error, "package version #{inspect(ver)} != #{version}"} - true -> {:ok, "Hex-validated metadata is replicant #{ver}"} + name != "replicant" -> {:error, "package name is not replicant"} + found_version != version -> {:error, "package version does not match #{version}"} + true -> :ok end - {:error, reason} -> - {:error, "Hex checksum validation failed: #{inspect(reason)}"} + {:error, _reason} -> + {:error, "Hex checksum validation failed"} end after File.rm_rf!(dest) end end - defp check_source_commit(receipt) do - commit = - receipt |> File.read!() |> then(&Regex.run(~r/source_commit:\s+([0-9a-f]{40})/, &1)) |> Enum.at(1) - - cond do - is_nil(commit) -> - {:error, "receipt has no 40-char source_commit"} + defp check_identity(version, source_commit, true), + do: Replicant.PackageIdentity.check_publish(version, source_commit) - System.cmd("git", ["cat-file", "-e", commit <> "^{commit}"], cd: @repo_root, stderr_to_stdout: true) - |> elem(1) != 0 -> - {:error, "recorded source_commit #{commit} is not a commit in this repo"} + defp check_identity(version, _source_commit, false), + do: Replicant.PackageIdentity.check_candidate(version) - true -> - {:ok, "source_commit #{String.slice(commit, 0, 12)}… exists"} - end - end - - # Identity collisions fail closed: the target version must not already be tagged locally or - # released on GitHub. (Hex existence is re-checked live at publish time by the server via - # replace:false, which rejects a duplicate.) - defp check_identity_free(version) do - tag = "v#{version}" - - tag_taken? = - System.cmd("git", ["rev-parse", "-q", "--verify", "refs/tags/#{tag}"], cd: @repo_root, stderr_to_stdout: true) - |> elem(1) == 0 - - gh_taken? = - case System.find_executable("gh") do - nil -> - false - - gh -> - System.cmd(gh, ["release", "view", tag], cd: @repo_root, stderr_to_stdout: true) |> elem(1) == 0 - end - - cond do - tag_taken? -> {:error, "local tag #{tag} already exists — identity taken"} - gh_taken? -> {:error, "GitHub release #{tag} already exists — identity taken"} - true -> {:ok, "no #{tag} tag or GitHub release yet (identity free)"} - end - end - - defp do_publish(version, tar, opts) do - authorized = System.get_env("REPLICANT_PUBLISH_AUTHORIZED") + defp publish(version, tar) do digest = sha256(File.read!(tar)) expected = "#{version}:#{digest}" - unless authorized == expected do - abort( - "--publish requires REPLICANT_PUBLISH_AUTHORIZED=#{version}: matching the retained " <> - "artifact; refusing to upload without explicit human authorization naming the exact version and digest" - ) + unless System.get_env("REPLICANT_PUBLISH_AUTHORIZED") == expected do + abort("--publish requires exact version:digest authorization for the witnessed artifact") end key = System.get_env("HEX_API_KEY") || abort("HEX_API_KEY not set") - config = :mix_hex_core.default_config() |> Map.put(:api_key, key) - case :mix_hex_api_release.publish(config, File.read!(tar), opts) do - {:ok, {code, _, _}} when code in [200, 201] -> - IO.puts("upload_candidate: published replicant #{version} (exact bytes, #{String.slice(digest, 0, 12)}…)") + case Hex.API.Release.publish("hexpm", File.read!(tar), [key: key], fn _ -> nil end, false) do + {:ok, {status, _, _}} when status in 200..299 -> + IO.puts("upload_candidate: published replicant #{version} from exact witnessed bytes") + + {:ok, {status, _, _}} -> + abort("publish failed with HTTP #{status}") - other -> - abort("publish failed: #{inspect(other)}") + {:error, _reason} -> + abort("publish failed before a successful HTTP response") end end - defp sha256(bin), do: :crypto.hash(:sha256, bin) |> Base.encode16(case: :lower) + defp sha256(bytes), do: :crypto.hash(:sha256, bytes) |> Base.encode16(case: :lower) - defp abort(msg) do - IO.puts(:stderr, "::error::upload_candidate: #{msg}") + defp abort(message) do + IO.puts(:stderr, "::error::upload_candidate: #{message}") System.halt(1) end end -UploadCandidate.run(System.argv()) +Replicant.UploadCandidate.run(System.argv()) diff --git a/scripts/release/verify_package.sh b/scripts/release/verify_package.sh index 86e3a29..0bd34d5 100755 --- a/scripts/release/verify_package.sh +++ b/scripts/release/verify_package.sh @@ -105,12 +105,12 @@ missing="$(LC_ALL=C comm -23 "$expected_sorted" "$actual" || true)" status=0 if [[ -n "$extra" ]]; then echo "::error::package contains files NOT in the manifest (boundary leak):" >&2 - echo "$extra" | sed 's|^| + |' >&2 + printf ' + %s\n' "${extra//$'\n'/$'\n + '}" >&2 status=1 fi if [[ -n "$missing" ]]; then echo "::error::package is MISSING files listed in the manifest:" >&2 - echo "$missing" | sed 's|^| - |' >&2 + printf ' - %s\n' "${missing//$'\n'/$'\n - '}" >&2 status=1 fi if [[ $status -ne 0 ]]; then diff --git a/test/release/package_identity_test.exs b/test/release/package_identity_test.exs new file mode 100644 index 0000000..8eb03ea --- /dev/null +++ b/test/release/package_identity_test.exs @@ -0,0 +1,76 @@ +Code.require_file("../../scripts/release/package_identity.exs", __DIR__) + +defmodule Replicant.PackageIdentityTest do + use ExUnit.Case, async: true + + alias Replicant.PackageIdentity + + @version "1.2.0" + @commit String.duplicate("a", 40) + + test "candidate identity is free only when every authoritative check proves absence" do + runner = fn + "git", ["rev-parse", "-q", "--verify", "refs/tags/v1.2.0"] -> + {"", 1} + + "git", ["ls-remote", "--exit-code", "--tags", "origin", "refs/tags/v1.2.0"] -> + {"", 2} + + "curl", args when is_list(args) -> + assert List.last(args) in [ + "https://api.github.com/repos/baselabs/replicant/releases/tags/v1.2.0", + "https://hex.pm/api/packages/replicant/releases/1.2.0" + ] + + {"404", 0} + end + + assert :ok == PackageIdentity.check_candidate(@version, runner) + end + + test "remote tag lookup errors fail closed instead of becoming absence" do + runner = fn + "git", ["rev-parse", "-q", "--verify", "refs/tags/v1.2.0"] -> + {"", 1} + + "git", ["ls-remote", "--exit-code", "--tags", "origin", "refs/tags/v1.2.0"] -> + {"network unavailable", 128} + end + + assert {:error, message} = PackageIdentity.check_candidate(@version, runner) + assert message =~ "remote tag check failed" + end + + test "ambiguous GitHub and Hex responses fail closed" do + base = fn + "git", ["rev-parse", "-q", "--verify", "refs/tags/v1.2.0"] -> + {"", 1} + + "git", ["ls-remote", "--exit-code", "--tags", "origin", "refs/tags/v1.2.0"] -> + {"", 2} + + "curl", args -> + case List.last(args) do + "https://api.github.com/repos/baselabs/replicant/releases/tags/v1.2.0" -> {"503", 0} + "https://hex.pm/api/packages/replicant/releases/1.2.0" -> {"404", 0} + end + end + + assert {:error, message} = PackageIdentity.check_candidate(@version, base) + assert message =~ "GitHub release check returned HTTP 503" + end + + test "publish requires the local and remote tag to dereference to the recorded commit" do + runner = fn + "git", ["rev-parse", "refs/tags/v1.2.0^{}"] -> + {@commit <> "\n", 0} + + "git", + ["ls-remote", "--exit-code", "--tags", "origin", "refs/tags/v1.2.0", "refs/tags/v1.2.0^{}"] -> + {String.duplicate("b", 40) <> "\trefs/tags/v1.2.0\n", 0} + end + + assert {:error, message} = PackageIdentity.check_publish(@version, @commit, runner) + assert message =~ "remote tag v1.2.0 resolves to" + end +end diff --git a/test/release/package_witness_test.exs b/test/release/package_witness_test.exs new file mode 100644 index 0000000..e21e8d9 --- /dev/null +++ b/test/release/package_witness_test.exs @@ -0,0 +1,82 @@ +Code.require_file("../../scripts/release/package_witness.exs", __DIR__) + +defmodule Replicant.PackageWitnessTest do + use ExUnit.Case, async: false + + alias Replicant.PackageWitness + + setup do + root = + Path.join( + System.tmp_dir!(), + "replicant-package-witness-#{System.unique_integer([:positive])}" + ) + + File.mkdir_p!(root) + System.cmd("git", ["init", "-q"], cd: root) + System.cmd("git", ["config", "user.email", "test@example.invalid"], cd: root) + System.cmd("git", ["config", "user.name", "Replicant Test"], cd: root) + + File.write!(Path.join(root, "source"), "one\n") + System.cmd("git", ["add", "source"], cd: root) + System.cmd("git", ["commit", "-q", "-m", "one"], cd: root) + first = git!(root, ["rev-parse", "HEAD"]) + + File.write!(Path.join(root, "source"), "two\n") + System.cmd("git", ["commit", "-q", "-am", "two"], cd: root) + second = git!(root, ["rev-parse", "HEAD"]) + + artifact = Path.join(root, "artifact.tar") + File.write!(artifact, "synthetic artifact bytes") + digest = sha256(File.read!(artifact)) + receipt = Path.join(root, "receipt.txt") + File.write!(receipt, receipt_body(first, digest)) + + on_exit(fn -> File.rm_rf!(root) end) + + %{ + root: root, + first: first, + second: second, + artifact: artifact, + digest: digest, + receipt: receipt + } + end + + test "witness binds the exact receipt and source commit", ctx do + ref = "refs/attestations/packages/replicant/1.2.0" + assert :ok == PackageWitness.create(ctx.root, ref, ctx.first, ctx.receipt) + assert :ok == PackageWitness.verify(ctx.root, ref, ctx.artifact, ctx.receipt) + + File.write!(ctx.receipt, receipt_body(ctx.second, ctx.digest)) + + assert {:error, message} = PackageWitness.verify(ctx.root, ref, ctx.artifact, ctx.receipt) + assert message =~ "receipt does not match immutable witness" + end + + test "every retained copy must match the candidate digest", ctx do + backup = Path.join(ctx.root, "backup.tar") + by_digest = Path.join(ctx.root, "by-digest.tar") + File.cp!(ctx.artifact, backup) + File.cp!(ctx.artifact, by_digest) + File.write!(backup, "corrupt") + + assert {:error, message} = + PackageWitness.verify_copies([ctx.artifact, backup, by_digest], ctx.digest) + + assert message =~ "retained copy digest mismatch" + assert message =~ backup + end + + defp git!(root, args) do + {output, 0} = System.cmd("git", args, cd: root) + String.trim(output) + end + + defp receipt_body(commit, digest) do + "version: 1.2.0\nsource_commit: #{commit}\nsha256: #{digest}\n" + end + + defp sha256(bytes), do: :crypto.hash(:sha256, bytes) |> Base.encode16(case: :lower) +end diff --git a/test/release/release_contract_test.exs b/test/release/release_contract_test.exs index 7d084dd..94bb986 100644 --- a/test/release/release_contract_test.exs +++ b/test/release/release_contract_test.exs @@ -55,4 +55,11 @@ defmodule Replicant.ReleaseContractTest do assert Mix.Project.config()[:docs][:source_ref] == "v#{version()}", "docs source_ref must be v#{version()} so HexDocs source links resolve to the release tag" end + + test "exact-byte uploader uses Hex's authenticated API wrapper and never the raw client" do + body = File.read!(Path.expand("../../scripts/release/upload_candidate.exs", __DIR__)) + + assert body =~ "Hex.API.Release.publish" + refute body =~ ":mix_hex_api_release.publish" + end end From a8d82abed0ffcb63a64de12babcfec7d34bff490 Mon Sep 17 00:00:00 2001 From: Russ Palermo <175215383+palermo-git@users.noreply.github.com> Date: Wed, 19 Aug 2026 20:07:27 -0400 Subject: [PATCH 3/8] fix(package): pass uploader options through mix run Remove the file-script argument separator that Mix forwards verbatim, and pin the invocation shape with a regression assertion. --- scripts/release/build_candidate.sh | 2 +- test/release/release_contract_test.exs | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/scripts/release/build_candidate.sh b/scripts/release/build_candidate.sh index dc9ea62..ca71875 100755 --- a/scripts/release/build_candidate.sh +++ b/scripts/release/build_candidate.sh @@ -119,7 +119,7 @@ elixir -r "$repo_root/scripts/release/package_witness.exs" \ -e 'Replicant.PackageWitness.create!(Enum.at(System.argv(), 0), Enum.at(System.argv(), 1), Enum.at(System.argv(), 2), Enum.at(System.argv(), 3))' -- \ "$repo_root" "$witness_ref" "$commit" "$receipt" -mix run --no-start "$repo_root/scripts/release/upload_candidate.exs" -- \ +mix run --no-start "$repo_root/scripts/release/upload_candidate.exs" \ --artifact "$primary" --receipt "$receipt" --witness-ref "$witness_ref" if [[ "$mode" == "check" ]]; then diff --git a/test/release/release_contract_test.exs b/test/release/release_contract_test.exs index 94bb986..9eaf652 100644 --- a/test/release/release_contract_test.exs +++ b/test/release/release_contract_test.exs @@ -58,8 +58,10 @@ defmodule Replicant.ReleaseContractTest do test "exact-byte uploader uses Hex's authenticated API wrapper and never the raw client" do body = File.read!(Path.expand("../../scripts/release/upload_candidate.exs", __DIR__)) + builder = File.read!(Path.expand("../../scripts/release/build_candidate.sh", __DIR__)) assert body =~ "Hex.API.Release.publish" refute body =~ ":mix_hex_api_release.publish" + refute builder =~ ~r/upload_candidate\.exs" --\s/ end end From 7cdf97c94d9c105716a7adf68cae3da5cfc51f88 Mon Sep 17 00:00:00 2001 From: Russ Palermo <175215383+palermo-git@users.noreply.github.com> Date: Wed, 19 Aug 2026 20:10:04 -0400 Subject: [PATCH 4/8] fix(package): verify the published Hex checksum After an authorized exact-byte upload, refetch the release metadata and require its checksum to equal the witnessed artifact before reporting success. --- scripts/release/package_checksum.exs | 44 ++++++++++++++++++++++++++ scripts/release/upload_candidate.exs | 7 +++- test/release/package_checksum_test.exs | 28 ++++++++++++++++ 3 files changed, 78 insertions(+), 1 deletion(-) create mode 100644 scripts/release/package_checksum.exs create mode 100644 test/release/package_checksum_test.exs diff --git a/scripts/release/package_checksum.exs b/scripts/release/package_checksum.exs new file mode 100644 index 0000000..1adf0e8 --- /dev/null +++ b/scripts/release/package_checksum.exs @@ -0,0 +1,44 @@ +defmodule Replicant.PackageChecksum do + @moduledoc false + + def classify({:ok, {200, _headers, %{"checksum" => checksum}}}, checksum), do: :ok + + def classify({:ok, {200, _headers, %{"checksum" => actual}}}, expected) + when is_binary(actual) do + {:error, "published checksum #{actual} does not match uploaded artifact #{expected}"} + end + + def classify({:ok, {200, _headers, _body}}, _expected), + do: {:error, "published release response has no checksum"} + + def classify({:ok, {status, _headers, _body}}, _expected), + do: {:error, "published release checksum read returned HTTP #{status}"} + + def classify({:error, _reason}, _expected), + do: {:error, "published release checksum read failed"} + + def verify!(version, expected, key, attempts \\ 5) do + result = + 1..attempts + |> Enum.reduce_while(nil, fn attempt, _last -> + response = apply(Hex.API.Release, :get, ["hexpm", "replicant", version, [key: key]]) + + case classify(response, expected) do + :ok -> + {:halt, :ok} + + {:error, _} = error when attempt == attempts -> + {:halt, error} + + {:error, _} = error -> + Process.sleep(1_000) + {:cont, error} + end + end) + + case result do + :ok -> :ok + {:error, message} -> raise message + end + end +end diff --git a/scripts/release/upload_candidate.exs b/scripts/release/upload_candidate.exs index e9e2d92..8a7b190 100644 --- a/scripts/release/upload_candidate.exs +++ b/scripts/release/upload_candidate.exs @@ -4,6 +4,7 @@ Mix.ensure_application!(:hex) Code.require_file("package_identity.exs", __DIR__) Code.require_file("package_witness.exs", __DIR__) +Code.require_file("package_checksum.exs", __DIR__) defmodule Replicant.UploadCandidate do @moduledoc false @@ -107,7 +108,11 @@ defmodule Replicant.UploadCandidate do case Hex.API.Release.publish("hexpm", File.read!(tar), [key: key], fn _ -> nil end, false) do {:ok, {status, _, _}} when status in 200..299 -> - IO.puts("upload_candidate: published replicant #{version} from exact witnessed bytes") + Replicant.PackageChecksum.verify!(version, digest, key) + + IO.puts( + "upload_candidate: published replicant #{version}; Hex checksum matches exact witnessed bytes" + ) {:ok, {status, _, _}} -> abort("publish failed with HTTP #{status}") diff --git a/test/release/package_checksum_test.exs b/test/release/package_checksum_test.exs new file mode 100644 index 0000000..94728b3 --- /dev/null +++ b/test/release/package_checksum_test.exs @@ -0,0 +1,28 @@ +Code.require_file("../../scripts/release/package_checksum.exs", __DIR__) + +defmodule Replicant.PackageChecksumTest do + use ExUnit.Case, async: true + + alias Replicant.PackageChecksum + + @digest String.duplicate("a", 64) + + test "accepts only the exact checksum returned for the published release" do + assert :ok == + PackageChecksum.classify({:ok, {200, [], %{"checksum" => @digest}}}, @digest) + + assert {:error, message} = + PackageChecksum.classify( + {:ok, {200, [], %{"checksum" => String.duplicate("b", 64)}}}, + @digest + ) + + assert message =~ "does not match" + end + + test "missing, malformed, or failed release reads fail closed" do + assert {:error, _} = PackageChecksum.classify({:ok, {200, [], %{}}}, @digest) + assert {:error, _} = PackageChecksum.classify({:ok, {503, [], %{}}}, @digest) + assert {:error, _} = PackageChecksum.classify({:error, :timeout}, @digest) + end +end From 38f851ea8d5910b33adeb1c81254caa7b1615b0e Mon Sep 17 00:00:00 2001 From: Russ Palermo <175215383+palermo-git@users.noreply.github.com> Date: Wed, 19 Aug 2026 20:12:36 -0400 Subject: [PATCH 5/8] fix(package): hold witnessed bytes through upload Reject symlink and writable inputs, read candidate evidence once, and carry the same immutable bytes through validation, authorization, upload, and checksum. --- scripts/release/package_witness.exs | 40 +++++++++++++++++++++----- scripts/release/upload_candidate.exs | 35 +++++++++++----------- test/release/package_witness_test.exs | 14 +++++++++ test/release/release_contract_test.exs | 1 + 4 files changed, 67 insertions(+), 23 deletions(-) diff --git a/scripts/release/package_witness.exs b/scripts/release/package_witness.exs index 8565ad9..eb75748 100644 --- a/scripts/release/package_witness.exs +++ b/scripts/release/package_witness.exs @@ -1,6 +1,8 @@ defmodule Replicant.PackageWitness do @moduledoc false + import Bitwise, only: [band: 2] + @zero String.duplicate("0", 40) def create(repo, ref, source_commit, receipt_path) do @@ -24,13 +26,18 @@ defmodule Replicant.PackageWitness do def verify(repo, ref, artifact_path, receipt_path) do receipt = File.read!(receipt_path) + artifact = if artifact_path, do: File.read!(artifact_path), else: nil + + verify_content(repo, ref, artifact, receipt) + end + def verify_content(repo, ref, artifact, receipt) do with {:ok, witnessed} <- git_output(repo, ["show", "#{ref}:candidate-receipt.txt"]), :ok <- exact_receipt(witnessed, receipt), {:ok, parent} <- git_output(repo, ["rev-parse", "#{ref}^"]), {:ok, commit} <- receipt_value(receipt, "source_commit", ~r/^[0-9a-f]{40}$/), :ok <- exact_parent(String.trim(parent), commit), - :ok <- verify_artifact(artifact_path, receipt) do + :ok <- verify_artifact(artifact, receipt) do :ok end end @@ -93,6 +100,27 @@ defmodule Replicant.PackageWitness do receipt_path |> File.read!() |> receipt_value("source_commit", ~r/^[0-9a-f]{40}$/) end + def receipt_source_commit_content(receipt) do + receipt_value(receipt, "source_commit", ~r/^[0-9a-f]{40}$/) + end + + def read_immutable(path) do + case File.lstat(path) do + {:ok, %File.Stat{type: :regular, mode: mode}} -> + if band(mode, 0o222) == 0 do + File.read(path) + else + {:error, "uploader input must be read-only: #{path}"} + end + + {:ok, _stat} -> + {:error, "uploader input must be a regular file: #{path}"} + + {:error, _reason} -> + {:error, "uploader input is missing or unreadable: #{path}"} + end + end + defp create_commit(repo, source_commit, receipt_path, _receipt) do scratch = Path.join( @@ -160,13 +188,11 @@ defmodule Replicant.PackageWitness do defp verify_artifact(nil, _receipt), do: :ok - defp verify_artifact(path, receipt) do + defp verify_artifact(bytes, receipt) do with {:ok, recorded} <- receipt_value(receipt, "sha256", ~r/^[0-9a-f]{64}$/) do - cond do - not File.regular?(path) -> {:error, "candidate artifact missing or non-regular"} - sha256(File.read!(path)) == recorded -> :ok - true -> {:error, "candidate artifact digest does not match witnessed receipt"} - end + if sha256(bytes) == recorded, + do: :ok, + else: {:error, "candidate artifact digest does not match witnessed receipt"} end end diff --git a/scripts/release/upload_candidate.exs b/scripts/release/upload_candidate.exs index 8a7b190..54ad18a 100644 --- a/scripts/release/upload_candidate.exs +++ b/scripts/release/upload_candidate.exs @@ -26,16 +26,23 @@ defmodule Replicant.UploadCandidate do witness_ref = opts[:witness_ref] || "refs/attestations/packages/replicant/#{version}" publish? = opts[:publish] || false - with :ok <- regular_file(tar), - :ok <- regular_file(receipt), - :ok <- Replicant.PackageWitness.verify(@repo_root, witness_ref, tar, receipt), - {:ok, source_commit} <- Replicant.PackageWitness.receipt_source_commit(receipt), - :ok <- check_metadata(tar, version), + with {:ok, tar_bytes} <- Replicant.PackageWitness.read_immutable(tar), + {:ok, receipt_bytes} <- Replicant.PackageWitness.read_immutable(receipt), + :ok <- + Replicant.PackageWitness.verify_content( + @repo_root, + witness_ref, + tar_bytes, + receipt_bytes + ), + {:ok, source_commit} <- + Replicant.PackageWitness.receipt_source_commit_content(receipt_bytes), + :ok <- check_metadata(tar_bytes, version), :ok <- check_identity(version, source_commit, publish?) do if publish? do - publish(version, tar) + publish(version, tar_bytes) else - bytes = File.stat!(tar).size + bytes = byte_size(tar_bytes) IO.puts(""" upload_candidate: DRY-RUN — all guards passed, nothing uploaded. @@ -59,18 +66,14 @@ defmodule Replicant.UploadCandidate do end end - defp regular_file(path) do - if File.regular?(path), do: :ok, else: {:error, "required regular file missing: #{path}"} - end - - defp check_metadata(tar, version) do + defp check_metadata(tar_bytes, version) do dest = Path.join(System.tmp_dir!(), "replicant-upload-meta-#{System.unique_integer([:positive])}") File.mkdir_p!(dest) try do - case :mix_hex_tarball.unpack(File.read!(tar), String.to_charlist(dest)) do + case :mix_hex_tarball.unpack(tar_bytes, String.to_charlist(dest)) do {:ok, meta} -> metadata = Map.new(meta[:metadata] || %{}) name = metadata["name"] || metadata[:name] @@ -96,8 +99,8 @@ defmodule Replicant.UploadCandidate do defp check_identity(version, _source_commit, false), do: Replicant.PackageIdentity.check_candidate(version) - defp publish(version, tar) do - digest = sha256(File.read!(tar)) + defp publish(version, tar_bytes) do + digest = sha256(tar_bytes) expected = "#{version}:#{digest}" unless System.get_env("REPLICANT_PUBLISH_AUTHORIZED") == expected do @@ -106,7 +109,7 @@ defmodule Replicant.UploadCandidate do key = System.get_env("HEX_API_KEY") || abort("HEX_API_KEY not set") - case Hex.API.Release.publish("hexpm", File.read!(tar), [key: key], fn _ -> nil end, false) do + case Hex.API.Release.publish("hexpm", tar_bytes, [key: key], fn _ -> nil end, false) do {:ok, {status, _, _}} when status in 200..299 -> Replicant.PackageChecksum.verify!(version, digest, key) diff --git a/test/release/package_witness_test.exs b/test/release/package_witness_test.exs index e21e8d9..e75f1a7 100644 --- a/test/release/package_witness_test.exs +++ b/test/release/package_witness_test.exs @@ -69,6 +69,20 @@ defmodule Replicant.PackageWitnessTest do assert message =~ backup end + test "uploader inputs must be read-only regular files, never symlinks", ctx do + link = Path.join(ctx.root, "artifact-link.tar") + File.ln_s!(ctx.artifact, link) + + assert {:error, message} = PackageWitness.read_immutable(link) + assert message =~ "regular file" + + assert {:error, message} = PackageWitness.read_immutable(ctx.artifact) + assert message =~ "read-only" + + File.chmod!(ctx.artifact, 0o444) + assert {:ok, "synthetic artifact bytes"} = PackageWitness.read_immutable(ctx.artifact) + end + defp git!(root, args) do {output, 0} = System.cmd("git", args, cd: root) String.trim(output) diff --git a/test/release/release_contract_test.exs b/test/release/release_contract_test.exs index 9eaf652..2f6c957 100644 --- a/test/release/release_contract_test.exs +++ b/test/release/release_contract_test.exs @@ -62,6 +62,7 @@ defmodule Replicant.ReleaseContractTest do assert body =~ "Hex.API.Release.publish" refute body =~ ":mix_hex_api_release.publish" + refute body =~ "File.read!(tar)" refute builder =~ ~r/upload_candidate\.exs" --\s/ end end From 455bf862b362067e8ea9c3e353b377a55772d31e Mon Sep 17 00:00:00 2001 From: Russ Palermo <175215383+palermo-git@users.noreply.github.com> Date: Wed, 19 Aug 2026 20:33:19 -0400 Subject: [PATCH 6/8] fix(release): close candidate review gaps --- .gitignore | 1 + CHANGELOG.md | 38 ++++++++-------- scripts/release/build_candidate.sh | 42 ++++++++++++++++-- scripts/release/consumer_smoke.exs | 2 +- scripts/release/package_identity.exs | 26 +++++++++-- scripts/release/package_publisher.exs | 52 ++++++++++++++++++++++ scripts/release/package_witness.exs | 58 ++++++++++++++++++------- scripts/release/unpack_validated.exs | 19 ++++++-- scripts/release/upload_candidate.exs | 26 +++-------- test/release/package_identity_test.exs | 12 +++++ test/release/package_publisher_test.exs | 58 +++++++++++++++++++++++++ test/release/package_witness_test.exs | 37 +++++++++++++--- test/release/release_contract_test.exs | 22 +++++----- 13 files changed, 308 insertions(+), 85 deletions(-) create mode 100644 scripts/release/package_publisher.exs create mode 100644 test/release/package_publisher_test.exs diff --git a/.gitignore b/.gitignore index 092a6af..8a9d33b 100644 --- a/.gitignore +++ b/.gitignore @@ -36,6 +36,7 @@ priv/plts/ /.zcode/ /.agent/ /.aider/ +/.kimosabe/ # Superpowers lifecycle artifacts (specs, plans, handoffs, reviews) — local only, never tracked /docs/superpowers/ diff --git a/CHANGELOG.md b/CHANGELOG.md index 6ee5bc4..64b738c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -52,6 +52,24 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 (`wal_status = 'lost'` is PG15's sole invalidation signal). Proven red-first at the unit level and verified against live PostgreSQL 15/16/17/18. +- **An unknown checkpoint with an absent replication slot now halts fail-closed instead of + silently creating a fresh slot (data-integrity, fail-closed).** In sink-owned mode a + checkpoint read fault (`sink.checkpoint/0` raising or erroring) reads as `checkpoint_lsn 0` + with `checkpoint_state: :fault`. The §14.15 streaming fail-open (resume-from-0, the + idempotent sink dedups the re-stream) is safe only when the slot is **present** — a resume + clamps to the slot's server-side `confirmed_flush_lsn`, so nothing is skipped. With the slot + **absent** there is nothing to resume: the connect path previously treated the fault-as-0 as + a genuine empty first run and created a fresh `CREATE_REPLICATION_SLOT`, which begins + streaming at its own creation LSN and silently skips every transaction between the (unknown) + real checkpoint and now — an unrecoverable data gap. That path now halts fail-closed in the + `:data_gap` family with a distinct, value-free telemetry reason + (`[:replicant, :connection, :slot_invalidated]`, `reason: :checkpoint_unknown`) and never + emits `CREATE_REPLICATION_SLOT`, including in incremental-snapshot mode when its separate + progress token is empty. A genuinely **empty** checkpoint (`checkpoint_state: :empty` — a real + first activation / go-forward) still creates the slot as before. Covered by red-first + connect-decision unit tests across plain and incremental modes and a live PostgreSQL fault probe + (raising-checkpoint sink, absent slot → structural halt, zero slots created on the server). + ### Security - **Telemetry metadata and measurements are now validated by a closed key set AND a per-key @@ -98,26 +116,6 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 field and is absent here) so the replay is a durable no-op, agreeing with `README.md` and ADR-0001; the `handle_message/2` docstring states the same. -### Fixed - -- **An unknown checkpoint with an absent replication slot now halts fail-closed instead of - silently creating a fresh slot (data-integrity, fail-closed).** In sink-owned mode a - checkpoint read fault (`sink.checkpoint/0` raising or erroring) reads as `checkpoint_lsn 0` - with `checkpoint_state: :fault`. The §14.15 streaming fail-open (resume-from-0, the - idempotent sink dedups the re-stream) is safe only when the slot is **present** — a resume - clamps to the slot's server-side `confirmed_flush_lsn`, so nothing is skipped. With the slot - **absent** there is nothing to resume: the connect path previously treated the fault-as-0 as - a genuine empty first run and created a fresh `CREATE_REPLICATION_SLOT`, which begins - streaming at its own creation LSN and silently skips every transaction between the (unknown) - real checkpoint and now — an unrecoverable data gap. That path now halts fail-closed in the - `:data_gap` family with a distinct, value-free telemetry reason - (`[:replicant, :connection, :slot_invalidated]`, `reason: :checkpoint_unknown`) and never - emits `CREATE_REPLICATION_SLOT`, including in incremental-snapshot mode when its separate - progress token is empty. A genuinely **empty** checkpoint (`checkpoint_state: :empty` — a real - first activation / go-forward) still creates the slot as before. Covered by red-first - connect-decision unit tests across plain and incremental modes and a live PostgreSQL fault probe - (raising-checkpoint sink, absent slot → structural halt, zero slots created on the server). - ## [1.1.0] - 2026-08-13 ### Fixed diff --git a/scripts/release/build_candidate.sh b/scripts/release/build_candidate.sh index ca71875..d4cc956 100755 --- a/scripts/release/build_candidate.sh +++ b/scripts/release/build_candidate.sh @@ -42,11 +42,40 @@ elixir -r "$repo_root/scripts/release/package_identity.exs" \ build_tree="$(mktemp -d "${TMPDIR:-/tmp}/replicant-package.XXXXXX")" witness_ref="" +primary="" +by_digest="" +backup="" +receipt="" +artifacts_retained=0 +receipt_retained=0 +witness_owned=0 +mint_complete=0 cleanup() { + status=$? + trap - EXIT + if [[ "$mode" == "check" && -n "$witness_ref" ]]; then git update-ref -d "$witness_ref" >/dev/null 2>&1 || true + elif [[ "$mode" == "mint" && $mint_complete -eq 0 ]]; then + if [[ $witness_owned -eq 1 ]]; then + git update-ref -d "$witness_ref" >/dev/null 2>&1 || true + fi + + if [[ $receipt_retained -eq 1 ]]; then + chmod u+w "$receipt" >/dev/null 2>&1 || true + rm -f -- "$receipt" + fi + + if [[ $artifacts_retained -eq 1 ]]; then + for target in "$primary" "$by_digest" "$backup"; do + chmod u+w "$target" >/dev/null 2>&1 || true + rm -f -- "$target" + done + fi fi + rm -rf "$build_tree" + exit "$status" } trap cleanup EXIT @@ -85,10 +114,6 @@ if git show-ref --verify --quiet "$witness_ref"; then die "package witness already exists: $witness_ref" fi -elixir -r "$repo_root/scripts/release/package_witness.exs" \ - -e 'Replicant.PackageWitness.retain_copies!(hd(System.argv()), tl(System.argv()))' -- \ - "$staged_tar" "$primary" "$by_digest" "$backup" - receipt_tmp="$build_tree/receipt.txt" { echo "Replicant package candidate receipt" @@ -107,14 +132,21 @@ receipt_tmp="$build_tree/receipt.txt" cat "$verification" } > "$receipt_tmp" +elixir -r "$repo_root/scripts/release/package_witness.exs" \ + -e 'Replicant.PackageWitness.retain_copies!(hd(System.argv()), tl(System.argv()))' -- \ + "$staged_tar" "$primary" "$by_digest" "$backup" +artifacts_retained=1 + elixir -r "$repo_root/scripts/release/package_witness.exs" \ -e 'Replicant.PackageWitness.retain_copies!(hd(System.argv()), tl(System.argv()))' -- \ "$receipt_tmp" "$receipt" +receipt_retained=1 elixir -r "$repo_root/scripts/release/package_witness.exs" \ -e 'Replicant.PackageWitness.verify_copies!(tl(System.argv()), hd(System.argv()))' -- \ "$digest" "$primary" "$by_digest" "$backup" +witness_owned=1 elixir -r "$repo_root/scripts/release/package_witness.exs" \ -e 'Replicant.PackageWitness.create!(Enum.at(System.argv(), 0), Enum.at(System.argv(), 1), Enum.at(System.argv(), 2), Enum.at(System.argv(), 3))' -- \ "$repo_root" "$witness_ref" "$commit" "$receipt" @@ -122,6 +154,8 @@ elixir -r "$repo_root/scripts/release/package_witness.exs" \ mix run --no-start "$repo_root/scripts/release/upload_candidate.exs" \ --artifact "$primary" --receipt "$receipt" --witness-ref "$witness_ref" +mint_complete=1 + if [[ "$mode" == "check" ]]; then log "CHECK PASS — throwaway package built, audited, documented, consumed, and uploader dry-run verified" else diff --git a/scripts/release/consumer_smoke.exs b/scripts/release/consumer_smoke.exs index 2b4dfe9..3feabbb 100644 --- a/scripts/release/consumer_smoke.exs +++ b/scripts/release/consumer_smoke.exs @@ -8,7 +8,7 @@ scratch = System.get_env("SMOKE_SCRATCH") || raise "SMOKE_SCRATCH not set" path = Replicant |> :code.which() |> to_string() -unless String.starts_with?(path, scratch) do +unless String.starts_with?(path, scratch <> "/") do raise "provenance: Replicant loaded from #{path}, not under scratch #{scratch} — the consumer is not running the extracted artifact" end diff --git a/scripts/release/package_identity.exs b/scripts/release/package_identity.exs index c236960..f19f3e4 100644 --- a/scripts/release/package_identity.exs +++ b/scripts/release/package_identity.exs @@ -113,6 +113,10 @@ defmodule Replicant.PackageIdentity do args = [ "--silent", "--show-error", + "--connect-timeout", + "10", + "--max-time", + "30", "--output", "/dev/null", "--write-out", @@ -135,10 +139,26 @@ defmodule Replicant.PackageIdentity do end end - defp run(command, args) do + @doc false + def run(command, args, timeout_ms \\ 30_000) do case System.find_executable(command) do - nil -> {"required command unavailable", 127} - executable -> System.cmd(executable, args, cd: @repo_root, stderr_to_stdout: true) + nil -> + {"required command unavailable", 127} + + executable -> + task = + Task.async(fn -> + System.cmd(executable, args, cd: @repo_root, stderr_to_stdout: true) + end) + + case Task.yield(task, timeout_ms) do + {:ok, result} -> + result + + nil -> + Task.shutdown(task, :brutal_kill) + {"command timed out", 124} + end end end diff --git a/scripts/release/package_publisher.exs b/scripts/release/package_publisher.exs new file mode 100644 index 0000000..9a50e08 --- /dev/null +++ b/scripts/release/package_publisher.exs @@ -0,0 +1,52 @@ +unless Code.ensure_loaded?(Replicant.PackageChecksum) do + Code.require_file("package_checksum.exs", __DIR__) +end + +defmodule Replicant.PackagePublisher do + @moduledoc false + + def publish(version, tar_bytes, opts \\ []) do + digest = sha256(tar_bytes) + expected = "#{version}:#{digest}" + env = Keyword.get(opts, :env, &System.get_env/1) + release_api = Keyword.get(opts, :release_api, Hex.API.Release) + checksum = Keyword.get(opts, :checksum, Replicant.PackageChecksum) + authorization = env.("REPLICANT_PUBLISH_AUTHORIZED") + key = env.("HEX_API_KEY") + + cond do + authorization != expected -> + {:error, + "--publish requires exact version:digest authorization for the witnessed artifact"} + + key in [nil, ""] -> + {:error, "HEX_API_KEY not set"} + + true -> + publish_exact(release_api, checksum, version, tar_bytes, digest, key) + end + end + + defp publish_exact(release_api, checksum, version, tar_bytes, digest, key) do + response = + apply(release_api, :publish, ["hexpm", tar_bytes, [key: key], fn _ -> nil end, false]) + + case response do + {:ok, {status, _, _}} when status in 200..299 -> + case apply(checksum, :verify!, [version, digest, key]) do + :ok -> {:ok, digest} + _ -> {:error, "published release checksum verification failed"} + end + + {:ok, {status, _, _}} -> + {:error, "publish failed with HTTP #{status}"} + + {:error, _reason} -> + {:error, "publish failed before a successful HTTP response"} + end + rescue + _error -> {:error, "published release checksum verification failed"} + end + + defp sha256(bytes), do: :crypto.hash(:sha256, bytes) |> Base.encode16(case: :lower) +end diff --git a/scripts/release/package_witness.exs b/scripts/release/package_witness.exs index eb75748..1d9271c 100644 --- a/scripts/release/package_witness.exs +++ b/scripts/release/package_witness.exs @@ -74,26 +74,20 @@ defmodule Replicant.PackageWitness do def retain_copies!(source, destinations) do bytes = File.read!(source) - Enum.each(destinations, fn destination -> + Enum.reduce(destinations, [], fn destination, created -> File.mkdir_p!(Path.dirname(destination)) - case File.open(destination, [:write, :binary, :exclusive]) do - {:ok, io} -> - try do - IO.binwrite(io, bytes) - after - File.close(io) - end - - File.chmod!(destination, 0o444) - - {:error, :eexist} -> - raise "refusing to overwrite retained package copy: #{destination}" - - {:error, reason} -> - raise "could not retain package copy #{destination}: #{inspect(reason)}" + try do + retain_copy!(destination, bytes) + [destination | created] + rescue + error -> + remove_created(created) + reraise error, __STACKTRACE__ end end) + + :ok end def receipt_source_commit(receipt_path) do @@ -155,6 +149,38 @@ defmodule Replicant.PackageWitness do end end + defp remove_created(paths) do + Enum.each(paths, fn path -> + File.chmod(path, 0o600) + File.rm(path) + end) + end + + defp retain_copy!(destination, bytes) do + case File.open(destination, [:write, :binary, :exclusive]) do + {:ok, io} -> + try do + try do + :ok = IO.binwrite(io, bytes) + after + File.close(io) + end + + File.chmod!(destination, 0o444) + rescue + error -> + remove_created([destination]) + reraise error, __STACKTRACE__ + end + + {:error, :eexist} -> + raise "refusing to overwrite retained package copy: #{destination}" + + {:error, reason} -> + raise "could not retain package copy #{destination}: #{inspect(reason)}" + end + end + defp create_ref(repo, ref, witness) do case System.cmd("git", ["update-ref", ref, witness, @zero], cd: repo, stderr_to_stdout: true) do {_, 0} -> :ok diff --git a/scripts/release/unpack_validated.exs b/scripts/release/unpack_validated.exs index aecdc6c..a3237fc 100644 --- a/scripts/release/unpack_validated.exs +++ b/scripts/release/unpack_validated.exs @@ -13,11 +13,22 @@ File.rm_rf!(dest) File.mkdir_p!(dest) case :mix_hex_tarball.unpack(bin, String.to_charlist(dest)) do - {:ok, meta} -> - checksum = meta |> Map.get(:outer_checksum, "") |> Base.encode16(case: :lower) - IO.puts("unpack_validated: OK — Hex checksum validated, extracted to #{dest} (outer_checksum #{checksum})") + {:ok, %{outer_checksum: checksum}} when is_binary(checksum) and byte_size(checksum) > 0 -> + encoded = Base.encode16(checksum, case: :lower) + + IO.puts( + "unpack_validated: OK — Hex checksum validated, extracted to #{dest} (outer_checksum #{encoded})" + ) + + {:ok, _meta} -> + IO.puts(:stderr, "::error::unpack_validated: Hex validation returned no outer checksum") + System.halt(1) {:error, reason} -> - IO.puts(:stderr, "::error::unpack_validated: Hex checksum validation FAILED: #{inspect(reason)}") + IO.puts( + :stderr, + "::error::unpack_validated: Hex checksum validation FAILED: #{inspect(reason)}" + ) + System.halt(1) end diff --git a/scripts/release/upload_candidate.exs b/scripts/release/upload_candidate.exs index 54ad18a..892d6cc 100644 --- a/scripts/release/upload_candidate.exs +++ b/scripts/release/upload_candidate.exs @@ -4,7 +4,7 @@ Mix.ensure_application!(:hex) Code.require_file("package_identity.exs", __DIR__) Code.require_file("package_witness.exs", __DIR__) -Code.require_file("package_checksum.exs", __DIR__) +Code.require_file("package_publisher.exs", __DIR__) defmodule Replicant.UploadCandidate do @moduledoc false @@ -100,33 +100,17 @@ defmodule Replicant.UploadCandidate do do: Replicant.PackageIdentity.check_candidate(version) defp publish(version, tar_bytes) do - digest = sha256(tar_bytes) - expected = "#{version}:#{digest}" - - unless System.get_env("REPLICANT_PUBLISH_AUTHORIZED") == expected do - abort("--publish requires exact version:digest authorization for the witnessed artifact") - end - - key = System.get_env("HEX_API_KEY") || abort("HEX_API_KEY not set") - - case Hex.API.Release.publish("hexpm", tar_bytes, [key: key], fn _ -> nil end, false) do - {:ok, {status, _, _}} when status in 200..299 -> - Replicant.PackageChecksum.verify!(version, digest, key) - + case Replicant.PackagePublisher.publish(version, tar_bytes) do + {:ok, _digest} -> IO.puts( "upload_candidate: published replicant #{version}; Hex checksum matches exact witnessed bytes" ) - {:ok, {status, _, _}} -> - abort("publish failed with HTTP #{status}") - - {:error, _reason} -> - abort("publish failed before a successful HTTP response") + {:error, message} -> + abort(message) end end - defp sha256(bytes), do: :crypto.hash(:sha256, bytes) |> Base.encode16(case: :lower) - defp abort(message) do IO.puts(:stderr, "::error::upload_candidate: #{message}") System.halt(1) diff --git a/test/release/package_identity_test.exs b/test/release/package_identity_test.exs index 8eb03ea..add00b8 100644 --- a/test/release/package_identity_test.exs +++ b/test/release/package_identity_test.exs @@ -17,6 +17,9 @@ defmodule Replicant.PackageIdentityTest do {"", 2} "curl", args when is_list(args) -> + assert "--connect-timeout" in args + assert "--max-time" in args + assert List.last(args) in [ "https://api.github.com/repos/baselabs/replicant/releases/tags/v1.2.0", "https://hex.pm/api/packages/replicant/releases/1.2.0" @@ -73,4 +76,13 @@ defmodule Replicant.PackageIdentityTest do assert {:error, message} = PackageIdentity.check_publish(@version, @commit, runner) assert message =~ "remote tag v1.2.0 resolves to" end + + test "command execution has a hard deadline" do + started = System.monotonic_time(:millisecond) + + assert {"command timed out", 124} = + PackageIdentity.run("sh", ["-c", "sleep 1"], 25) + + assert System.monotonic_time(:millisecond) - started < 500 + end end diff --git a/test/release/package_publisher_test.exs b/test/release/package_publisher_test.exs new file mode 100644 index 0000000..8743211 --- /dev/null +++ b/test/release/package_publisher_test.exs @@ -0,0 +1,58 @@ +Code.require_file("../../scripts/release/package_publisher.exs", __DIR__) + +defmodule Replicant.PackagePublisherTest do + use ExUnit.Case, async: true + + alias Replicant.PackagePublisher + + defmodule ReleaseAPI do + def publish(repository, bytes, auth, progress, replace?) do + send(self(), {:publish, repository, bytes, auth, is_function(progress, 1), replace?}) + {:ok, {201, [], %{}}} + end + end + + defmodule Checksum do + def verify!(version, digest, key) do + send(self(), {:checksum, version, digest, key}) + :ok + end + end + + test "publishes the exact authorized bytes through the public Hex wrapper and verifies checksum" do + bytes = "exact witnessed package bytes" + digest = :crypto.hash(:sha256, bytes) |> Base.encode16(case: :lower) + + env = fn + "REPLICANT_PUBLISH_AUTHORIZED" -> "1.2.0:#{digest}" + "HEX_API_KEY" -> "test-key" + end + + assert {:ok, ^digest} = + PackagePublisher.publish("1.2.0", bytes, + env: env, + release_api: ReleaseAPI, + checksum: Checksum + ) + + assert_received {:publish, "hexpm", ^bytes, [key: "test-key"], true, false} + assert_received {:checksum, "1.2.0", ^digest, "test-key"} + end + + test "wrong authorization rejects before any publish call" do + env = fn + "REPLICANT_PUBLISH_AUTHORIZED" -> "1.2.0:wrong" + "HEX_API_KEY" -> "test-key" + end + + assert {:error, message} = + PackagePublisher.publish("1.2.0", "bytes", + env: env, + release_api: ReleaseAPI, + checksum: Checksum + ) + + assert message =~ "exact version:digest authorization" + refute_received {:publish, _, _, _, _, _} + end +end diff --git a/test/release/package_witness_test.exs b/test/release/package_witness_test.exs index e75f1a7..757b077 100644 --- a/test/release/package_witness_test.exs +++ b/test/release/package_witness_test.exs @@ -13,17 +13,17 @@ defmodule Replicant.PackageWitnessTest do ) File.mkdir_p!(root) - System.cmd("git", ["init", "-q"], cd: root) - System.cmd("git", ["config", "user.email", "test@example.invalid"], cd: root) - System.cmd("git", ["config", "user.name", "Replicant Test"], cd: root) + git!(root, ["init", "-q"]) + git!(root, ["config", "user.email", "test@example.invalid"]) + git!(root, ["config", "user.name", "Replicant Test"]) File.write!(Path.join(root, "source"), "one\n") - System.cmd("git", ["add", "source"], cd: root) - System.cmd("git", ["commit", "-q", "-m", "one"], cd: root) + git!(root, ["add", "source"]) + git!(root, ["commit", "-q", "-m", "one"]) first = git!(root, ["rev-parse", "HEAD"]) File.write!(Path.join(root, "source"), "two\n") - System.cmd("git", ["commit", "-q", "-am", "two"], cd: root) + git!(root, ["commit", "-q", "-am", "two"]) second = git!(root, ["rev-parse", "HEAD"]) artifact = Path.join(root, "artifact.tar") @@ -69,6 +69,31 @@ defmodule Replicant.PackageWitnessTest do assert message =~ backup end + test "retained copies refuse to overwrite an existing destination", ctx do + fresh = Path.join(ctx.root, "fresh.tar") + destination = Path.join(ctx.root, "retained.tar") + File.write!(destination, "existing") + + assert_raise RuntimeError, ~r/refusing to overwrite retained package copy/, fn -> + PackageWitness.retain_copies!(ctx.artifact, [fresh, destination]) + end + + refute File.exists?(fresh) + assert File.read!(destination) == "existing" + end + + test "witness creation rejects a receipt for a different source commit", ctx do + File.write!(ctx.receipt, receipt_body(ctx.second, ctx.digest)) + + assert {:error, "receipt source commit does not match witness parent"} = + PackageWitness.create( + ctx.root, + "refs/attestations/packages/replicant/1.2.0", + ctx.first, + ctx.receipt + ) + end + test "uploader inputs must be read-only regular files, never symlinks", ctx do link = Path.join(ctx.root, "artifact-link.tar") File.ln_s!(ctx.artifact, link) diff --git a/test/release/release_contract_test.exs b/test/release/release_contract_test.exs index 2f6c957..7a226f1 100644 --- a/test/release/release_contract_test.exs +++ b/test/release/release_contract_test.exs @@ -38,6 +38,18 @@ defmodule Replicant.ReleaseContractTest do "the candidate section must be newer than the section beneath it" end + test "candidate changelog has one group for each change type" do + [_, after_candidate] = + File.read!(@changelog) |> String.split("## [#{version()}]", parts: 2) + + [candidate | _] = String.split(after_candidate, ~r/^## \[/m) + + headings = Regex.scan(~r/^### (.+)$/m, candidate) |> Enum.map(fn [_, heading] -> heading end) + + assert headings == Enum.uniq(headings), + "the candidate changelog repeats a change-type heading: #{inspect(headings)}" + end + test "CHANGELOG comparison links bind the candidate to the last published tag" do body = File.read!(@changelog) v = version() @@ -55,14 +67,4 @@ defmodule Replicant.ReleaseContractTest do assert Mix.Project.config()[:docs][:source_ref] == "v#{version()}", "docs source_ref must be v#{version()} so HexDocs source links resolve to the release tag" end - - test "exact-byte uploader uses Hex's authenticated API wrapper and never the raw client" do - body = File.read!(Path.expand("../../scripts/release/upload_candidate.exs", __DIR__)) - builder = File.read!(Path.expand("../../scripts/release/build_candidate.sh", __DIR__)) - - assert body =~ "Hex.API.Release.publish" - refute body =~ ":mix_hex_api_release.publish" - refute body =~ "File.read!(tar)" - refute builder =~ ~r/upload_candidate\.exs" --\s/ - end end From 38b01a0752c978e419d721aaecdd9b7f19a5055d Mon Sep 17 00:00:00 2001 From: Russ Palermo <175215383+palermo-git@users.noreply.github.com> Date: Wed, 19 Aug 2026 20:43:05 -0400 Subject: [PATCH 7/8] fix(release): make candidate cleanup ownership-safe --- scripts/release/build_candidate.sh | 18 ++++++------- scripts/release/package_witness.exs | 35 ++++++++++++++++++++------ test/release/package_identity_test.exs | 4 +-- test/release/package_witness_test.exs | 30 +++++++++++++++++++++- 4 files changed, 68 insertions(+), 19 deletions(-) diff --git a/scripts/release/build_candidate.sh b/scripts/release/build_candidate.sh index d4cc956..7577689 100755 --- a/scripts/release/build_candidate.sh +++ b/scripts/release/build_candidate.sh @@ -42,6 +42,7 @@ elixir -r "$repo_root/scripts/release/package_identity.exs" \ build_tree="$(mktemp -d "${TMPDIR:-/tmp}/replicant-package.XXXXXX")" witness_ref="" +witness_oid="" primary="" by_digest="" backup="" @@ -54,13 +55,11 @@ cleanup() { status=$? trap - EXIT - if [[ "$mode" == "check" && -n "$witness_ref" ]]; then - git update-ref -d "$witness_ref" >/dev/null 2>&1 || true - elif [[ "$mode" == "mint" && $mint_complete -eq 0 ]]; then - if [[ $witness_owned -eq 1 ]]; then - git update-ref -d "$witness_ref" >/dev/null 2>&1 || true - fi + if [[ $witness_owned -eq 1 && ( "$mode" == "check" || $mint_complete -eq 0 ) ]]; then + git update-ref -d "$witness_ref" "$witness_oid" >/dev/null 2>&1 || true + fi + if [[ "$mode" == "mint" && $mint_complete -eq 0 ]]; then if [[ $receipt_retained -eq 1 ]]; then chmod u+w "$receipt" >/dev/null 2>&1 || true rm -f -- "$receipt" @@ -146,10 +145,11 @@ elixir -r "$repo_root/scripts/release/package_witness.exs" \ -e 'Replicant.PackageWitness.verify_copies!(tl(System.argv()), hd(System.argv()))' -- \ "$digest" "$primary" "$by_digest" "$backup" -witness_owned=1 -elixir -r "$repo_root/scripts/release/package_witness.exs" \ - -e 'Replicant.PackageWitness.create!(Enum.at(System.argv(), 0), Enum.at(System.argv(), 1), Enum.at(System.argv(), 2), Enum.at(System.argv(), 3))' -- \ +witness_oid="$(elixir -r "$repo_root/scripts/release/package_witness.exs" \ + -e 'IO.puts(Replicant.PackageWitness.create!(Enum.at(System.argv(), 0), Enum.at(System.argv(), 1), Enum.at(System.argv(), 2), Enum.at(System.argv(), 3)))' -- \ "$repo_root" "$witness_ref" "$commit" "$receipt" +)" +witness_owned=1 mix run --no-start "$repo_root/scripts/release/upload_candidate.exs" \ --artifact "$primary" --receipt "$receipt" --witness-ref "$witness_ref" diff --git a/scripts/release/package_witness.exs b/scripts/release/package_witness.exs index 1d9271c..32b2ada 100644 --- a/scripts/release/package_witness.exs +++ b/scripts/release/package_witness.exs @@ -10,20 +10,29 @@ defmodule Replicant.PackageWitness do with :ok <- receipt_commit_matches(receipt, source_commit), :ok <- commit_exists(repo, source_commit), - {:ok, witness} <- create_commit(repo, source_commit, receipt_path, receipt), - :ok <- create_ref(repo, ref, witness), - :ok <- verify(repo, ref, nil, receipt_path) do - :ok + {:ok, witness} <- create_commit(repo, source_commit, receipt_path, receipt) do + create_ref_and_verify(repo, ref, witness, receipt_path) end end def create!(repo, ref, source_commit, receipt_path) do case create(repo, ref, source_commit, receipt_path) do - :ok -> :ok + {:ok, witness} -> witness {:error, message} -> raise message end end + @doc false + def delete_ref(repo, ref, expected_witness) do + case System.cmd("git", ["update-ref", "-d", ref, expected_witness], + cd: repo, + stderr_to_stdout: true + ) do + {_, 0} -> :ok + {_, _} -> {:error, "package witness moved; refusing cleanup"} + end + end + def verify(repo, ref, artifact_path, receipt_path) do receipt = File.read!(receipt_path) artifact = if artifact_path, do: File.read!(artifact_path), else: nil @@ -75,9 +84,8 @@ defmodule Replicant.PackageWitness do bytes = File.read!(source) Enum.reduce(destinations, [], fn destination, created -> - File.mkdir_p!(Path.dirname(destination)) - try do + File.mkdir_p!(Path.dirname(destination)) retain_copy!(destination, bytes) [destination | created] rescue @@ -188,6 +196,19 @@ defmodule Replicant.PackageWitness do end end + defp create_ref_and_verify(repo, ref, witness, receipt_path) do + with :ok <- create_ref(repo, ref, witness) do + case verify(repo, ref, nil, receipt_path) do + :ok -> + {:ok, witness} + + {:error, _message} = error -> + delete_ref(repo, ref, witness) + error + end + end + end + defp commit_exists(repo, commit) do case System.cmd("git", ["cat-file", "-e", "#{commit}^{commit}"], cd: repo, diff --git a/test/release/package_identity_test.exs b/test/release/package_identity_test.exs index add00b8..af46b28 100644 --- a/test/release/package_identity_test.exs +++ b/test/release/package_identity_test.exs @@ -17,8 +17,8 @@ defmodule Replicant.PackageIdentityTest do {"", 2} "curl", args when is_list(args) -> - assert "--connect-timeout" in args - assert "--max-time" in args + assert Enum.chunk_every(args, 2, 1, :discard) |> Enum.member?(["--connect-timeout", "10"]) + assert Enum.chunk_every(args, 2, 1, :discard) |> Enum.member?(["--max-time", "30"]) assert List.last(args) in [ "https://api.github.com/repos/baselabs/replicant/releases/tags/v1.2.0", diff --git a/test/release/package_witness_test.exs b/test/release/package_witness_test.exs index 757b077..c68694f 100644 --- a/test/release/package_witness_test.exs +++ b/test/release/package_witness_test.exs @@ -46,7 +46,7 @@ defmodule Replicant.PackageWitnessTest do test "witness binds the exact receipt and source commit", ctx do ref = "refs/attestations/packages/replicant/1.2.0" - assert :ok == PackageWitness.create(ctx.root, ref, ctx.first, ctx.receipt) + assert {:ok, _witness} = PackageWitness.create(ctx.root, ref, ctx.first, ctx.receipt) assert :ok == PackageWitness.verify(ctx.root, ref, ctx.artifact, ctx.receipt) File.write!(ctx.receipt, receipt_body(ctx.second, ctx.digest)) @@ -82,6 +82,34 @@ defmodule Replicant.PackageWitnessTest do assert File.read!(destination) == "existing" end + test "a later destination directory failure removes earlier copies", ctx do + fresh = Path.join(ctx.root, "fresh.tar") + blocked_parent = Path.join(ctx.root, "blocked-parent") + File.write!(blocked_parent, "not a directory") + + assert_raise File.Error, fn -> + PackageWitness.retain_copies!(ctx.artifact, [ + fresh, + Path.join(blocked_parent, "retained.tar") + ]) + end + + refute File.exists?(fresh) + assert File.read!(blocked_parent) == "not a directory" + end + + test "witness deletion requires the exact object still at the ref", ctx do + ref = "refs/attestations/packages/replicant/1.2.0" + assert {:ok, first_witness} = PackageWitness.create(ctx.root, ref, ctx.first, ctx.receipt) + + git!(ctx.root, ["update-ref", ref, ctx.second, first_witness]) + + assert {:error, "package witness moved; refusing cleanup"} = + PackageWitness.delete_ref(ctx.root, ref, first_witness) + + assert git!(ctx.root, ["rev-parse", ref]) == ctx.second + end + test "witness creation rejects a receipt for a different source commit", ctx do File.write!(ctx.receipt, receipt_body(ctx.second, ctx.digest)) From 695f28e3284e66520c82abfa9aaff1331c7fc735 Mon Sep 17 00:00:00 2001 From: Russ Palermo <175215383+palermo-git@users.noreply.github.com> Date: Wed, 19 Aug 2026 20:48:53 -0400 Subject: [PATCH 8/8] fix(release): fail closed on witness cleanup errors --- scripts/release/build_candidate.sh | 9 ++++++++- scripts/release/package_witness.exs | 6 ++++-- 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/scripts/release/build_candidate.sh b/scripts/release/build_candidate.sh index 7577689..68b925c 100755 --- a/scripts/release/build_candidate.sh +++ b/scripts/release/build_candidate.sh @@ -53,10 +53,14 @@ witness_owned=0 mint_complete=0 cleanup() { status=$? + cleanup_failed=0 trap - EXIT if [[ $witness_owned -eq 1 && ( "$mode" == "check" || $mint_complete -eq 0 ) ]]; then - git update-ref -d "$witness_ref" "$witness_oid" >/dev/null 2>&1 || true + if ! git update-ref -d "$witness_ref" "$witness_oid" >/dev/null 2>&1; then + echo "::error::build_candidate: package witness cleanup failed" >&2 + cleanup_failed=1 + fi fi if [[ "$mode" == "mint" && $mint_complete -eq 0 ]]; then @@ -74,6 +78,9 @@ cleanup() { fi rm -rf "$build_tree" + if [[ $status -eq 0 && $cleanup_failed -eq 1 ]]; then + status=1 + fi exit "$status" } trap cleanup EXIT diff --git a/scripts/release/package_witness.exs b/scripts/release/package_witness.exs index 32b2ada..c864e82 100644 --- a/scripts/release/package_witness.exs +++ b/scripts/release/package_witness.exs @@ -203,8 +203,10 @@ defmodule Replicant.PackageWitness do {:ok, witness} {:error, _message} = error -> - delete_ref(repo, ref, witness) - error + case delete_ref(repo, ref, witness) do + :ok -> error + {:error, _cleanup} -> {:error, "package witness verification and cleanup failed"} + end end end end