From 04aecef698e888d8353864c90d023ba71e7c7cc4 Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Thu, 16 Apr 2026 20:44:29 -0700 Subject: [PATCH] misc cleanups - fix `nix run .` to execute `bin/zig` and set `mainProgram` - narrow package `meta.platforms` to the current system and require `sha256` for nightly packages too - add `coreutils` to the dev shell because the update script now computes hashes locally - rewrite `update` to use a temp dir, fail fast on fetch errors, retry downloads, verify release-only tarballs per platform, and derive `sha256` from verified artifacts - preserve historical nightly entries while replacing current release and master metadata so stale nested attrs cannot leak forward - expand CI to build and run Zig, smoke test both templates, and check that `./update` only changes `mirrors.json` and `sources.json` - tighten the auto-update workflow with scoped staging, no-op commit skipping, write permissions, and concurrency protection - update the init and compiler-dev templates to use newer `nixpkgs`, reuse the exported overlay, and refresh the compiler-dev lockfile --- .github/workflows/test.yml | 25 ++++- .github/workflows/update.yml | 23 ++-- default.nix | 8 +- flake.nix | 3 +- templates/compiler-dev/flake.lock | 8 +- templates/compiler-dev/flake.nix | 2 +- templates/init/flake.nix | 9 +- update | 171 +++++++++++++++++++----------- 8 files changed, 162 insertions(+), 87 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 24ef6a8e..56d625e2 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -9,7 +9,30 @@ jobs: with: nix_path: nixpkgs=channel:nixos-unstable - run: nix flake check + - run: nix build --no-link .#default .#master + - run: nix run . -- version + - run: | + repo="$PWD" + tmpdir=$(mktemp -d) + trap 'rm -rf "$tmpdir"' EXIT + cd "$tmpdir" + nix flake init -t "$repo#init" + nix flake show + - run: | + repo="$PWD" + tmpdir=$(mktemp -d) + trap 'rm -rf "$tmpdir"' EXIT + cd "$tmpdir" + nix flake init -t "$repo#compiler-dev" + nix flake show # Verify the update script WORKS but we don't actually commit anything # in the test job. - - run: ./update + - run: | + ./update + unexpected=$(git status --porcelain | sed 's/^.. //' | grep -vE '^(mirrors\.json|sources\.json)$' || true) + if [ -n "$unexpected" ]; then + echo "unexpected files changed:" >&2 + printf '%s\n' "$unexpected" >&2 + exit 1 + fi diff --git a/.github/workflows/update.yml b/.github/workflows/update.yml index a3e6eb3f..adc71dd7 100644 --- a/.github/workflows/update.yml +++ b/.github/workflows/update.yml @@ -3,6 +3,14 @@ on: schedule: - cron: '0 */12 * * *' workflow_dispatch: + +permissions: + contents: write + +concurrency: + group: update-sources + cancel-in-progress: true + jobs: update-sources: runs-on: ubuntu-latest @@ -12,9 +20,12 @@ jobs: with: nix_path: nixpkgs=channel:nixos-unstable - run: ./update - - run: "git config user.email mitchellh@users.noreply.github.com" - - run: "git config user.name zig-overlay" - - run: "git add -A" - - run: "git commit -m 'update sources.json' || true" - - run: "git push -u origin main" - + - run: git config user.email mitchellh@users.noreply.github.com + - run: git config user.name zig-overlay + - run: | + git add mirrors.json sources.json + if git diff --cached --quiet; then + exit 0 + fi + git commit -m 'update Zig release metadata' + git push origin main diff --git a/default.nix b/default.nix index c16c093a..7af39072 100644 --- a/default.nix +++ b/default.nix @@ -12,7 +12,6 @@ url, version, sha256, - platforms, }: let tarballName = lib.lists.last (lib.strings.split "/" url); srcIsFromZigLang = lib.strings.hasPrefix "https://ziglang.org/" url; @@ -67,7 +66,8 @@ meta = pkgs.zig.meta // { - inherit platforms; + mainProgram = "zig"; + platforms = [system]; }; }); @@ -77,7 +77,6 @@ (k: v: mkBinaryInstall { inherit (v.${system}) version url sha256; - platforms = builtins.attrNames v; }) (lib.attrsets.filterAttrs (k: v: (builtins.hasAttr system v) && (v.${system}.url != null) && (v.${system}.sha256 != null)) @@ -95,11 +94,10 @@ ) (mkBinaryInstall { inherit (v.${system}) version url sha256; - platforms = builtins.attrNames v; }) ) (lib.attrsets.filterAttrs - (k: v: (builtins.hasAttr system v) && (v.${system}.url != null)) + (k: v: (builtins.hasAttr system v) && (v.${system}.url != null) && (v.${system}.sha256 != null)) sources.master); # This determines the latest /released/ version. diff --git a/flake.nix b/flake.nix index a1bd7f0e..161b1946 100644 --- a/flake.nix +++ b/flake.nix @@ -46,7 +46,7 @@ default = self.apps.${system}.zig; zig = { type = "app"; - program = self.packages.${system}.default.outPath; + program = "${self.packages.${system}.default}/bin/zig"; }; }); @@ -57,6 +57,7 @@ lib.mapAttrs (system: pkgs: { default = pkgs.mkShell { nativeBuildInputs = with pkgs; [ + coreutils curl jq minisign diff --git a/templates/compiler-dev/flake.lock b/templates/compiler-dev/flake.lock index e8cfdd01..fbb98c44 100644 --- a/templates/compiler-dev/flake.lock +++ b/templates/compiler-dev/flake.lock @@ -36,16 +36,16 @@ }, "nixpkgs": { "locked": { - "lastModified": 1747958103, - "narHash": "sha256-qmmFCrfBwSHoWw7cVK4Aj+fns+c54EBP8cGqp/yK410=", + "lastModified": 1776221942, + "narHash": "sha256-FbQAeVNi7G4v3QCSThrSAAvzQTmrmyDLiHNPvTF2qFM=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "fe51d34885f7b5e3e7b59572796e1bcb427eccb1", + "rev": "1766437c5509f444c1b15331e82b8b6a9b967000", "type": "github" }, "original": { "owner": "NixOS", - "ref": "nixpkgs-unstable", + "ref": "nixos-25.11", "repo": "nixpkgs", "type": "github" } diff --git a/templates/compiler-dev/flake.nix b/templates/compiler-dev/flake.nix index cc72b5e5..21e1b02e 100644 --- a/templates/compiler-dev/flake.nix +++ b/templates/compiler-dev/flake.nix @@ -2,7 +2,7 @@ description = "Zig compiler development."; inputs = { - nixpkgs.url = "github:NixOS/nixpkgs/nixos-25.05"; + nixpkgs.url = "github:NixOS/nixpkgs/nixos-25.11"; flake-utils.url = "github:numtide/flake-utils"; # Used for shell.nix diff --git a/templates/init/flake.nix b/templates/init/flake.nix index 869a0f19..cdcda526 100644 --- a/templates/init/flake.nix +++ b/templates/init/flake.nix @@ -2,7 +2,7 @@ description = "An empty project that uses Zig."; inputs = { - nixpkgs.url = "github:nixos/nixpkgs/nixos-23.05"; + nixpkgs.url = "github:NixOS/nixpkgs/nixos-25.11"; flake-utils.url = "github:numtide/flake-utils"; zig.url = "github:mitchellh/zig-overlay"; @@ -19,12 +19,7 @@ flake-utils, ... } @ inputs: let - overlays = [ - # Other overlays - (final: prev: { - zigpkgs = inputs.zig.packages.${prev.system}; - }) - ]; + overlays = [inputs.zig.overlays.default]; # Our supported systems are the same supported systems as the Zig binaries systems = builtins.attrNames inputs.zig.packages; diff --git a/update b/update index b6b570d9..990a13fb 100755 --- a/update +++ b/update @@ -1,70 +1,119 @@ #!/usr/bin/env nix-shell -#! nix-shell -p curl jq minisign -i sh -set -e +#! nix-shell -p coreutils curl jq minisign -i sh +set -eu -curl -s 'https://ziglang.org/download/community-mirrors.txt' | jq -R '.' | jq -s . > mirrors.json +tmpdir=$(mktemp -d) +cleanup() { + rm -rf "$tmpdir" +} +trap cleanup EXIT INT TERM + +fetch() { + curl -fsSL --retry 3 --retry-delay 1 "$1" +} + +fetch 'https://ziglang.org/download/community-mirrors.txt' | jq -R '.' | jq -s . > "$tmpdir/mirrors.json" # The well known public key for Zig PUBLIC_KEY="RWSGOq2NVecA2UPNdBUZykf1CCb147pkmdtYxgb3Ti+JO/wCYvhbAb/U" # Grab the main index.json (contains all releases + master) -rm -rf index.json index.json.minisig -curl -s 'https://ziglang.org/download/index.json' > index.json -VERSION=$(cat index.json | jq -r '.master.version') +fetch 'https://ziglang.org/download/index.json' > "$tmpdir/index.json" +VERSION=$(jq -r '.master.version' "$tmpdir/index.json") echo "Parsing master version: ${VERSION}" # Download the versioned index.json and its signature, then verify. # The signature is generated against the versioned file, not the generic one. -curl -s "https://ziglang.org/builds/zig-${VERSION}-index.json" > versioned-index.json -curl -s "https://ziglang.org/builds/zig-${VERSION}-index.json.minisig" > index.json.minisig -minisign -V -P ${PUBLIC_KEY} -x index.json.minisig -m versioned-index.json +fetch "https://ziglang.org/builds/zig-${VERSION}-index.json" > "$tmpdir/versioned-index.json" +fetch "https://ziglang.org/builds/zig-${VERSION}-index.json.minisig" > "$tmpdir/versioned-index.json.minisig" +minisign -V -P "$PUBLIC_KEY" -x "$tmpdir/versioned-index.json.minisig" -m "$tmpdir/versioned-index.json" # The main index.json is not signed, but every individual tarball has a # .minisig signature. For release versions that only appear in the main -# index (not in the verified versioned index), verify that a valid -# signature exists for at least one tarball per version. -NEW_VERSIONS=$(jq -rs ' +# index (not in the verified versioned index), verify every supported +# tarball signature and compute sha256 from the verified artifact. +NEW_VERSIONS_JSON="$tmpdir/new-versions.json" +jq -rs ' . as [$main, $verified] | - [$main | keys[] | select(. != "master") | select(. as $k | ($verified | has($k)) | not)] | - .[] -' index.json versioned-index.json) - -FAILED="" -for v in $NEW_VERSIONS; do - # Get all tarball URLs for this version (only for our supported targets) - TARBALL_URLS=$(jq -r --arg v "$v" ' + [ + $main + | keys[] + | select(. != "master") + | select(. as $k | ($verified | has($k)) | not) + ] +' "$tmpdir/index.json" "$tmpdir/versioned-index.json" > "$NEW_VERSIONS_JSON" + +VERIFIED_RELEASES_JSON="$tmpdir/verified-releases.json" +printf '{}\n' > "$VERIFIED_RELEASES_JSON" + +jq -r '.[]' "$NEW_VERSIONS_JSON" > "$tmpdir/new-versions.txt" +while IFS= read -r version; do + [ -n "$version" ] || continue + + version_targets="$tmpdir/${version}-targets.jsonl" + version_platforms="$tmpdir/${version}-platforms.jsonl" + : > "$version_platforms" + + jq -c --arg version "$version" ' ["aarch64-linux", "x86_64-linux", "aarch64-macos", "x86_64-macos", "aarch64-windows", "x86_64-windows"] as $targets | - .[$v] | to_entries[] | select(.key as $k | any($targets[]; . == $k)) | .value.tarball - ' index.json) - - for TARBALL_URL in $TARBALL_URLS; do - TARBALL_FILE=$(basename "$TARBALL_URL") - echo "Verifying signature for ${v}: ${TARBALL_FILE}..." - - curl -s "$TARBALL_URL" > "verify-${TARBALL_FILE}" - curl -s "${TARBALL_URL}.minisig" > "verify-${TARBALL_FILE}.minisig" - if ! minisign -V -P ${PUBLIC_KEY} -x "verify-${TARBALL_FILE}.minisig" -m "verify-${TARBALL_FILE}"; then - echo "ERROR: signature verification failed for ${v} (${TARBALL_FILE})" - FAILED="${FAILED} ${v}" + .[$version] + | to_entries[] + | select(.key as $key | any($targets[]; . == $key)) + | select(.value.tarball != null) + | { + target: .key, + tarball: .value.tarball, + } + ' "$tmpdir/index.json" > "$version_targets" + + if [ ! -s "$version_targets" ]; then + echo "Skipping ${version}: no supported targets found." + continue + fi + + version_failed=0 + while IFS= read -r target_entry; do + [ -n "$target_entry" ] || continue + + target=$(printf '%s\n' "$target_entry" | jq -r '.target') + tarball_url=$(printf '%s\n' "$target_entry" | jq -r '.tarball') + tarball_file="$tmpdir/$(basename "$tarball_url")" + + echo "Verifying signature for ${version}: $(basename "$tarball_url")..." + fetch "$tarball_url" > "$tarball_file" + fetch "${tarball_url}.minisig" > "$tarball_file.minisig" + + if ! minisign -V -P "$PUBLIC_KEY" -x "$tarball_file.minisig" -m "$tarball_file"; then + echo "ERROR: signature verification failed for ${version} (${target})" >&2 + version_failed=1 + break fi - rm -f "verify-${TARBALL_FILE}" "verify-${TARBALL_FILE}.minisig" - done -done - -if [ -n "$FAILED" ]; then - echo "ERROR: Signature verification failed for:${FAILED}" - echo "Falling back to verified index only." - cp versioned-index.json merged-index.json -else - # Start from the verified index and add only the newly verified releases - VERIFIED_VERSIONS=$(echo "$NEW_VERSIONS" | jq -R '.' | jq -s '.') - jq -s --argjson versions "$VERIFIED_VERSIONS" ' - .[1] * (.[0] | to_entries | map(select(.key as $k | $versions | index($k))) | from_entries) - ' index.json versioned-index.json > merged-index.json -fi - -# Build our new sources.json from the merged index -cat merged-index.json | jq ' + + sha256=$(sha256sum "$tarball_file" | awk '{print $1}') + jq -n \ + --arg target "$target" \ + --arg tarball "$tarball_url" \ + --arg sha256 "$sha256" \ + --arg version "$version" \ + '{($target): {tarball: $tarball, shasum: $sha256, version: $version}}' \ + >> "$version_platforms" + done < "$version_targets" + + if [ "$version_failed" -ne 0 ]; then + continue + fi + + version_release="$tmpdir/${version}-release.json" + jq -s --arg version "$version" '{($version): (add // {})}' "$version_platforms" > "$version_release" + jq -s '.[0] * .[1]' "$VERIFIED_RELEASES_JSON" "$version_release" > "$tmpdir/verified-releases.next.json" + mv "$tmpdir/verified-releases.next.json" "$VERIFIED_RELEASES_JSON" +done < "$tmpdir/new-versions.txt" + +jq -s '.[0] * .[1]' "$tmpdir/versioned-index.json" "$VERIFIED_RELEASES_JSON" > "$tmpdir/combined-index.json" + +# Build our new sources.json from the verified index plus any individually +# verified release-only additions. +jq ' ["aarch64-linux", "x86_64-linux", "aarch64-macos", "x86_64-macos", "aarch64-windows", "x86_64-windows"] as $targets | def todarwin(x): x | gsub("macos"; "darwin"); def toentry(vsn; x): @@ -92,17 +141,15 @@ reduce to_entries[] as $entry ({}; . * ( ) } )) -' > sources.new.json - -# For debugging -# cat sources.new.json -# exit +' "$tmpdir/combined-index.json" > "$tmpdir/sources.new.json" -# Copy the old file since jq can't modify in-place. This is also a backup. -cp sources.json sources.old.json - -# Recursive merge -jq -s '.[0] * .[1]' sources.old.json sources.new.json > sources.json +# Preserve historical nightly entries while replacing current metadata. +jq -s ' + . as [$old, $new] | + $new + { + master: (($old.master // {}) + ($new.master // {})) + } +' sources.json "$tmpdir/sources.new.json" > "$tmpdir/sources.json" -# Clean up temp files -rm -f index.json versioned-index.json merged-index.json index.json.minisig sources.old.json sources.new.json +mv "$tmpdir/mirrors.json" mirrors.json +mv "$tmpdir/sources.json" sources.json