chore(ci): retry transient module and Docker Hub fetches (PIPE-1058, PIPE-1092)

.github/scripts/retry.sh

@@ -0,0 +1,52 @@
+#!/usr/bin/env bash
+# retry.sh — run a command, retrying only on transient network/registry failures.
+#
+# Absorbs the CI flakes tracked in PIPE-1058 (Go module-proxy blips:
+# proxy.golang.org HTTP/2 stream errors, i/o timeouts) and PIPE-1092
+# (Docker Hub pull timeouts to registry-1.docker.io). Bounded, so a persistent
+# outage still fails the job after RETRY_MAX attempts.
+#
+# By default it retries ONLY when the command output matches a known transient
+# signature, so wrapping a build/test/install step never masks a real compile
+# or test failure. Set RETRY_ANY=1 to retry on any non-zero exit.
+#
+# Usage: bash .github/scripts/retry.sh <command> [args...]
+# Env:   RETRY_MAX (default 3), RETRY_DELAY base backoff seconds (default 10),
+#        RETRY_ANY (default 0).
+set -uo pipefail
+
+max="${RETRY_MAX:-3}"
+base_delay="${RETRY_DELAY:-10}"
+retry_any="${RETRY_ANY:-0}"
+
+# Signatures of transient network/registry failures that clear on a retry.
+transient_re='proxy\.golang\.org|sum\.golang\.org|registry-1\.docker\.io|registry\.docker\.io|i/o timeout|TLS handshake timeout|connection reset|connection refused|unexpected EOF|stream error|INTERNAL_ERROR|net/http: request canceled|Client\.Timeout exceeded|Service Unavailable|Bad Gateway|Gateway Time-?out|Too Many Requests|temporary failure|no such host'
+
+attempt=1
+while true; do
+  log="$(mktemp)"
+  "$@" 2>&1 | tee "$log"
+  status="${PIPESTATUS[0]}"
+  if [ "$status" -eq 0 ]; then
+    rm -f "$log"
+    exit 0
+  fi
+
+  if [ "$attempt" -ge "$max" ]; then
+    echo "retry: '$*' failed after ${max} attempt(s) (exit ${status}); giving up." >&2
+    rm -f "$log"
+    exit "$status"
+  fi
+
+  if [ "$retry_any" != "1" ] && ! grep -qiE "$transient_re" "$log"; then
+    echo "retry: '$*' failed (exit ${status}) with no transient-network signature; not retrying." >&2
+    rm -f "$log"
+    exit "$status"
+  fi
+  rm -f "$log"
+
+  delay=$(( base_delay * attempt ))
+  echo "retry: transient failure on attempt ${attempt}/${max} (exit ${status}); retrying '$*' in ${delay}s..." >&2
+  sleep "$delay"
+  attempt=$(( attempt + 1 ))
+done

.github/workflows/ci.yml

@@ -63,11 +63,11 @@ jobs:
       - name: Download dependencies
         run: |
           go env
-          go mod download
+          bash .github/scripts/retry.sh go mod download
 
       - name: Install tools
         run: |
-          make install-tools
+          bash .github/scripts/retry.sh make install-tools
 
   test:
     name: Test
@@ -159,12 +159,31 @@ jobs:
       - name: Checkout
         uses: actions/checkout@v7
 
+      # Only the linux legs build container images (the goreleaser dockers are
+      # linux/amd64 and linux/arm64), so QEMU/buildx are set up only there. The
+      # freebsd/darwin/windows legs cross-compile binaries only, so they no
+      # longer pull from Docker Hub at all, which removes the transient-pull
+      # failure they used to hit (PIPE-1092). On the linux legs the QEMU and
+      # buildkit images are pre-pulled with retry and pinned, so a transient
+      # Docker Hub blip during setup no longer fails the build.
+      - name: Pre-pull buildx base images (retry transient Docker Hub pulls)
+        if: matrix.goos == 'linux'
+        run: |
+          bash .github/scripts/retry.sh docker pull tonistiigi/binfmt:latest
+          bash .github/scripts/retry.sh docker pull moby/buildkit:buildx-stable-1
+
       - name: Set up Docker Buildx
+        if: matrix.goos == 'linux'
         id: buildx
         uses: docker/setup-buildx-action@v4
+        with:
+          driver-opts: image=moby/buildkit:buildx-stable-1
 
       - name: Set up QEMU
+        if: matrix.goos == 'linux'
         uses: docker/setup-qemu-action@v4
+        with:
+          image: tonistiigi/binfmt:latest
 
       - name: Setup Go
         uses: actions/setup-go@v7
@@ -212,12 +231,24 @@ jobs:
         run: |
           sudo apt-get install -y tree
 
+      # Pre-pull the QEMU/buildkit images with retry so a transient Docker Hub
+      # blip during buildx/QEMU setup doesn't fail the merge build (PIPE-1092).
+      # The images are pinned to match what the setup actions request below.
+      - name: Pre-pull buildx base images (retry transient Docker Hub pulls)
+        run: |
+          bash .github/scripts/retry.sh docker pull tonistiigi/binfmt:latest
+          bash .github/scripts/retry.sh docker pull moby/buildkit:buildx-stable-1
+
       - name: Set up Docker Buildx
         id: buildx
         uses: docker/setup-buildx-action@v4
+        with:
+          driver-opts: image=moby/buildkit:buildx-stable-1
 
       - name: Set up QEMU
         uses: docker/setup-qemu-action@v4
+        with:
+          image: tonistiigi/binfmt:latest
 
       - name: Setup Go
         uses: actions/setup-go@v7

.goreleaser.yaml

@@ -6,7 +6,9 @@ partial:
 
 before:
   hooks:
-    - make install-tools
+    # Retry the tool install so a transient Go module-proxy blip during the
+    # GoReleaser pre-build hook doesn't fail the build leg (PIPE-1058).
+    - bash .github/scripts/retry.sh make install-tools
     - make man
 
 builds: