fix(ci): retry Nix shell and app dependency preparation

.agents/skills/watch-github-actions/SKILL.md

@@ -125,6 +125,13 @@ gh run list --json databaseId,status,headBranch,url --jq '.[] | {id: .databaseId
 
 ## View Job Logs
 
+`setup-nix` retries development-shell preparation once when `prepare-shell`
+is enabled. Inspect both attempts in the job log; `setup-rust` assumes the
+shell has already been prepared. Cargo, lint, and test commands are not retried.
+Direct Nix builds and app dependency preparation also retry once; apps run
+once after preparation succeeds. Skipped dependent E2E suites indicate blocked
+coverage.
+
 For `Trivy Changes`, inspect the `Resolve PR baseline` step for the base and head
 SHAs. PR runs compare the tested merge commit with its
 first parent; change detection and scans must use the same pair. On reruns, do

.github/actions/check-protobuf-compatibility/action.yml

@@ -19,4 +19,7 @@ runs:
       env:
         CHECK_REF: ${{ inputs.ref }}
       run: |
+        # Retry dependency preparation, then run the compatibility check once.
+        nix build --no-link --no-write-lock-file .#check-protobuf-compatibility ||
+          nix build --no-link --no-write-lock-file .#check-protobuf-compatibility
         nix run .#check-protobuf-compatibility --no-write-lock-file -- "$CHECK_REF"

.github/actions/setup-nix/action.yml

@@ -2,14 +2,23 @@
 # SPDX-License-Identifier: Apache-2.0
 
 name: Setup Nix
-description: Install Nix and configure the OpenShell Cachix cache
+description: Install Nix, configure Cachix, and optionally prepare the development shell
 
 inputs:
   cachix-auth-token:
     description: Token used to write build outputs to Cachix
     required: false
     default: ""
 
+  prepare-shell:
+    description: Prepare the development shell, retrying once on failure
+    required: false
+    default: "false"
+  shell-installable:
+    description: Development shell to prepare
+    required: false
+    default: "."
+
 runs:
   using: composite
   steps:
@@ -22,3 +31,16 @@ runs:
         name: openshell
         authToken: ${{ inputs.cachix-auth-token }}
         skipPush: ${{ inputs.cachix-auth-token == '' }}
+
+    - name: Prepare Nix development shell
+      if: inputs.prepare-shell == 'true'
+      shell: bash
+      env:
+        NIX_SHELL_INSTALLABLE: ${{ inputs.shell-installable }}
+      run: |
+        # HTTP 416 is not retried by Nix; a fresh invocation restarts downloads.
+        if nix develop "$NIX_SHELL_INSTALLABLE" -c true; then
+          exit 0
+        fi
+        echo "::warning::Nix shell preparation failed; retrying once."
+        nix develop "$NIX_SHELL_INSTALLABLE" -c true

.github/actions/setup-rust/action.yml

@@ -2,7 +2,7 @@
 # SPDX-License-Identifier: Apache-2.0
 
 name: Setup Rust
-description: Configure the Nix development shell and Rust caches
+description: Configure Rust caches after setup-nix has prepared the development shell
 
 inputs:
   cache-key:
@@ -23,10 +23,6 @@ runs:
         shell_drv=$(nix eval --raw --impure .#devShells --apply 'shells: shells.${builtins.currentSystem}.default.drvPath')
         echo "hash=$(nix hash file --type sha256 --base16 "$shell_drv")" >> "$GITHUB_OUTPUT"
 
-    - name: Realize Nix development shell
-      shell: bash
-      run: nix develop -c true
-
     - name: Cache Rust target and registry
       uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2
       with:

.github/workflows/branch-checks.yml

@@ -114,14 +114,11 @@ jobs:
     steps:
       - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
 
-      - uses: cachix/install-nix-action@13d8dd58da0234aa297dedd986986ccb8e7f3e24 # v31.11.1
-        with:
-          github_access_token: ${{ secrets.GITHUB_TOKEN }}
-
-      - uses: cachix/cachix-action@5f2d7c5294214f71b873db4b969586b980625e71 # v17
+      - uses: ./.github/actions/setup-nix
         with:
-          name: openshell
-          authToken: ${{ secrets.CACHIX_AUTH_TOKEN }}
+          cachix-auth-token: ${{ secrets.CACHIX_AUTH_TOKEN }}
+          prepare-shell: "true"
+          shell-installable: .#devShells.x86_64-linux.default
 
       - name: Check dependencies
         run: cargo deny check licenses bans sources
@@ -137,6 +134,7 @@ jobs:
 
       - uses: ./.github/actions/setup-nix
         with:
+          prepare-shell: "true"
           cachix-auth-token: ${{ secrets.CACHIX_AUTH_TOKEN }}
 
       - name: Format
@@ -158,6 +156,7 @@ jobs:
 
       - uses: ./.github/actions/setup-nix
         with:
+          prepare-shell: "true"
           cachix-auth-token: ${{ secrets.CACHIX_AUTH_TOKEN }}
 
       - name: Verify Cargo lockfiles
@@ -185,6 +184,7 @@ jobs:
 
       - uses: ./.github/actions/setup-nix
         with:
+          prepare-shell: "true"
           cachix-auth-token: ${{ secrets.CACHIX_AUTH_TOKEN }}
 
       - uses: ./.github/actions/setup-rust
@@ -221,6 +221,7 @@ jobs:
 
       - uses: ./.github/actions/setup-nix
         with:
+          prepare-shell: "true"
           cachix-auth-token: ${{ secrets.CACHIX_AUTH_TOKEN }}
 
       - uses: ./.github/actions/setup-rust
@@ -257,6 +258,7 @@ jobs:
 
       - uses: ./.github/actions/setup-nix
         with:
+          prepare-shell: "true"
           cachix-auth-token: ${{ secrets.CACHIX_AUTH_TOKEN }}
 
       - uses: ./.github/actions/setup-rust
@@ -306,6 +308,7 @@ jobs:
 
       - uses: ./.github/actions/setup-nix
         with:
+          prepare-shell: "true"
           cachix-auth-token: ${{ secrets.CACHIX_AUTH_TOKEN }}
 
       - uses: ./.github/actions/setup-rust
@@ -330,6 +333,7 @@ jobs:
 
       - uses: ./.github/actions/setup-nix
         with:
+          prepare-shell: "true"
           cachix-auth-token: ${{ secrets.CACHIX_AUTH_TOKEN }}
 
       - uses: ./.github/actions/setup-rust
@@ -356,6 +360,7 @@ jobs:
 
       - uses: ./.github/actions/setup-nix
         with:
+          prepare-shell: "true"
           cachix-auth-token: ${{ secrets.CACHIX_AUTH_TOKEN }}
 
       - uses: ./.github/actions/setup-rust

.github/workflows/branch-e2e.yml

@@ -144,6 +144,7 @@ jobs:
           ref: ${{ github.sha }}
       - uses: ./.github/actions/setup-nix
         with:
+          prepare-shell: "true"
           cachix-auth-token: ${{ secrets.CACHIX_AUTH_TOKEN }}
       - uses: ./.github/actions/build-rust-binary
         with:
@@ -184,6 +185,7 @@ jobs:
           ref: ${{ github.sha }}
       - uses: ./.github/actions/setup-nix
         with:
+          prepare-shell: "true"
           cachix-auth-token: ${{ secrets.CACHIX_AUTH_TOKEN }}
       - uses: ./.github/actions/build-rust-binary
         with:

.github/workflows/build-binaries.yml

@@ -51,6 +51,7 @@ jobs:
           ref: ${{ inputs.checkout-ref || github.sha }}
       - uses: ./.github/actions/setup-nix
         with:
+          prepare-shell: "true"
           cachix-auth-token: ${{ secrets.CACHIX_AUTH_TOKEN }}
       - uses: ./.github/actions/build-rust-binary
         with:

.github/workflows/build-vm-driver.yml

@@ -52,6 +52,7 @@ jobs:
 
       - uses: ./.github/actions/setup-nix
         with:
+          prepare-shell: "true"
           cachix-auth-token: ${{ secrets.CACHIX_AUTH_TOKEN }}
 
       - name: Download openshell-sandbox
@@ -81,7 +82,9 @@ jobs:
           path: vm-init
 
       - name: Build VM runtime
-        run: nix build .#vm-runtime
+        run: |
+          # HTTP 416 is not retried by Nix; restart the build once on failure.
+          nix build .#vm-runtime || nix build .#vm-runtime
 
       - name: Assemble compressed VM runtime
         run: |

.github/workflows/integration-runner.yml

@@ -85,4 +85,7 @@ jobs:
           ENVIRONMENT: ${{ matrix.environment }}
           INSTALLER: ${{ matrix.installer }}
           TESTSUITE: ${{ matrix.testsuite }}
-        run: nix run .#tmachine -- test "${ENVIRONMENT}" "${INSTALLER}" "${TESTSUITE}"
+        run: |
+          # Retry dependency preparation, then execute the test suite once.
+          nix build --no-link .#tmachine || nix build --no-link .#tmachine
+          nix run .#tmachine -- test "${ENVIRONMENT}" "${INSTALLER}" "${TESTSUITE}"

.github/workflows/prepare-integration-inputs.yml

@@ -152,13 +152,22 @@ jobs:
           docker save --output artifacts/images/openshell-supervisor-tmachine.tar openshell/supervisor:tmachine
 
       - name: Build test archives
-        run: nix run .#build-artifacts-test-archives
+        run: |
+          # Retry dependency preparation, then generate artifacts once.
+          nix build --no-link .#build-artifacts-test-archives || nix build --no-link .#build-artifacts-test-archives
+          nix run .#build-artifacts-test-archives
 
       - name: Build test workload images
-        run: nix run .#build-artifacts-test-images
+        run: |
+          # Retry dependency preparation, then generate artifacts once.
+          nix build --no-link .#build-artifacts-test-images || nix build --no-link .#build-artifacts-test-images
+          nix run .#build-artifacts-test-images
 
       - name: Package Helm chart
-        run: nix run .#build-artifacts-helm
+        run: |
+          # Retry dependency preparation, then generate artifacts once.
+          nix build --no-link .#build-artifacts-helm || nix build --no-link .#build-artifacts-helm
+          nix run .#build-artifacts-helm
 
       - name: Upload integration inputs
         id: upload-integration-inputs

.github/workflows/trivy-changes.yml

@@ -110,6 +110,8 @@ jobs:
 
       - name: Set up Nix
         uses: ./.github/actions/setup-nix
+        with:
+          prepare-shell: "true"
 
       - name: Test report comparison
         run: tasks/scripts/trivy-scan-test.sh

.github/workflows/trivy-scan.yml

@@ -105,6 +105,7 @@ jobs:
       - name: Set up Nix
         uses: ./.github/actions/setup-nix
         with:
+          prepare-shell: "true"
           cachix-auth-token: ${{ github.event_name != 'pull_request' && secrets.CACHIX_AUTH_TOKEN || '' }}
 
       - name: Test scan reporting

.github/workflows/workflow-security.yml

@@ -54,6 +54,7 @@ jobs:
       - name: Set up Nix
         uses: ./.github/actions/setup-nix
         with:
+          prepare-shell: "true"
           cachix-auth-token: ${{ github.event_name != 'pull_request' && secrets.CACHIX_AUTH_TOKEN || '' }}
 
       - name: Run Actionlint
@@ -141,6 +142,7 @@ jobs:
       - name: Set up Nix
         uses: ./.github/actions/setup-nix
         with:
+          prepare-shell: "true"
           cachix-auth-token: ${{ github.event_name != 'pull_request' && secrets.CACHIX_AUTH_TOKEN || '' }}
 
       - name: Run Zizmor

CI.md

@@ -511,3 +511,24 @@ merge.
 
 Do not add the informational Actionlint, Zizmor, Dependency Review, or CodeQL
 jobs to the required status list while they remain in observation mode.
+
+## Nix download recovery
+
+Jobs that enter the development shell enable `prepare-shell: "true"` on
+`setup-nix`. After configuring Cachix, the action prepares the shell with
+`nix develop -c true` and retries once on failure. Use `shell-installable`
+to select a different development shell. Rust setup assumes this preparation
+has completed. Jobs that only use Nix apps leave shell preparation disabled.
+
+Nix can report a transport error after receiving a complete cache download,
+then resume at EOF and receive HTTP 416. A fresh invocation restarts the
+operation. Both attempts appear in the job log; a second failure fails the
+step. Any preparation failure is retried once, including deterministic errors.
+Cargo, lint, and test commands are not retried.
+
+Direct `nix build` commands retry once. Before each `nix run`, CI builds the
+app's package with `nix build --no-link`, retrying preparation once, then runs
+the app once. The artifact and protobuf-check apps expose matching package
+outputs for this preparation. Runtime failures from tests, artifact generation,
+and compatibility checks are not retried. Downloads initiated inside an app
+are outside this preparation retry.

flake.nix

@@ -182,6 +182,11 @@
         };
 
         packages = {
+          # Expose app derivations so CI can prepare them before executing once.
+          check-protobuf-compatibility = checkProtobufCompatibility;
+          build-artifacts-test-archives = artifacts.testArchives;
+          build-artifacts-test-images = artifacts.testImages;
+          build-artifacts-helm = artifacts.helm;
           vm-runtime = vmRuntime;
           tmachine = testMachines.package;
           tmachine-config = testMachines.config;