diff --git a/.github/scripts/retry.sh b/.github/scripts/retry.sh new file mode 100755 index 0000000..928c176 --- /dev/null +++ b/.github/scripts/retry.sh @@ -0,0 +1,52 @@ +#!/usr/bin/env bash +# retry.sh — run a command, retrying only on transient network/registry failures. +# +# Absorbs the CI flakes tracked in PIPE-1058 (Go module-proxy blips: +# proxy.golang.org HTTP/2 stream errors, i/o timeouts) and PIPE-1092 +# (Docker Hub pull timeouts to registry-1.docker.io). Bounded, so a persistent +# outage still fails the job after RETRY_MAX attempts. +# +# By default it retries ONLY when the command output matches a known transient +# signature, so wrapping a build/test/install step never masks a real compile +# or test failure. Set RETRY_ANY=1 to retry on any non-zero exit. +# +# Usage: bash .github/scripts/retry.sh [args...] +# Env: RETRY_MAX (default 3), RETRY_DELAY base backoff seconds (default 10), +# RETRY_ANY (default 0). +set -uo pipefail + +max="${RETRY_MAX:-3}" +base_delay="${RETRY_DELAY:-10}" +retry_any="${RETRY_ANY:-0}" + +# Signatures of transient network/registry failures that clear on a retry. +transient_re='proxy\.golang\.org|sum\.golang\.org|registry-1\.docker\.io|registry\.docker\.io|i/o timeout|TLS handshake timeout|connection reset|connection refused|unexpected EOF|stream error|INTERNAL_ERROR|net/http: request canceled|Client\.Timeout exceeded|Service Unavailable|Bad Gateway|Gateway Time-?out|Too Many Requests|temporary failure|no such host' + +attempt=1 +while true; do + log="$(mktemp)" + "$@" 2>&1 | tee "$log" + status="${PIPESTATUS[0]}" + if [ "$status" -eq 0 ]; then + rm -f "$log" + exit 0 + fi + + if [ "$attempt" -ge "$max" ]; then + echo "retry: '$*' failed after ${max} attempt(s) (exit ${status}); giving up." >&2 + rm -f "$log" + exit "$status" + fi + + if [ "$retry_any" != "1" ] && ! grep -qiE "$transient_re" "$log"; then + echo "retry: '$*' failed (exit ${status}) with no transient-network signature; not retrying." >&2 + rm -f "$log" + exit "$status" + fi + rm -f "$log" + + delay=$(( base_delay * attempt )) + echo "retry: transient failure on attempt ${attempt}/${max} (exit ${status}); retrying '$*' in ${delay}s..." >&2 + sleep "$delay" + attempt=$(( attempt + 1 )) +done diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 2ef081b..107b0c3 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -63,11 +63,11 @@ jobs: - name: Download dependencies run: | go env - go mod download + bash .github/scripts/retry.sh go mod download - name: Install tools run: | - make install-tools + bash .github/scripts/retry.sh make install-tools test: name: Test @@ -159,12 +159,31 @@ jobs: - name: Checkout uses: actions/checkout@v7 + # Only the linux legs build container images (the goreleaser dockers are + # linux/amd64 and linux/arm64), so QEMU/buildx are set up only there. The + # freebsd/darwin/windows legs cross-compile binaries only, so they no + # longer pull from Docker Hub at all, which removes the transient-pull + # failure they used to hit (PIPE-1092). On the linux legs the QEMU and + # buildkit images are pre-pulled with retry and pinned, so a transient + # Docker Hub blip during setup no longer fails the build. + - name: Pre-pull buildx base images (retry transient Docker Hub pulls) + if: matrix.goos == 'linux' + run: | + bash .github/scripts/retry.sh docker pull tonistiigi/binfmt:latest + bash .github/scripts/retry.sh docker pull moby/buildkit:buildx-stable-1 + - name: Set up Docker Buildx + if: matrix.goos == 'linux' id: buildx uses: docker/setup-buildx-action@v4 + with: + driver-opts: image=moby/buildkit:buildx-stable-1 - name: Set up QEMU + if: matrix.goos == 'linux' uses: docker/setup-qemu-action@v4 + with: + image: tonistiigi/binfmt:latest - name: Setup Go uses: actions/setup-go@v7 @@ -212,12 +231,24 @@ jobs: run: | sudo apt-get install -y tree + # Pre-pull the QEMU/buildkit images with retry so a transient Docker Hub + # blip during buildx/QEMU setup doesn't fail the merge build (PIPE-1092). + # The images are pinned to match what the setup actions request below. + - name: Pre-pull buildx base images (retry transient Docker Hub pulls) + run: | + bash .github/scripts/retry.sh docker pull tonistiigi/binfmt:latest + bash .github/scripts/retry.sh docker pull moby/buildkit:buildx-stable-1 + - name: Set up Docker Buildx id: buildx uses: docker/setup-buildx-action@v4 + with: + driver-opts: image=moby/buildkit:buildx-stable-1 - name: Set up QEMU uses: docker/setup-qemu-action@v4 + with: + image: tonistiigi/binfmt:latest - name: Setup Go uses: actions/setup-go@v7 diff --git a/.goreleaser.yaml b/.goreleaser.yaml index 75a1eb4..62c19f6 100644 --- a/.goreleaser.yaml +++ b/.goreleaser.yaml @@ -6,7 +6,9 @@ partial: before: hooks: - - make install-tools + # Retry the tool install so a transient Go module-proxy blip during the + # GoReleaser pre-build hook doesn't fail the build leg (PIPE-1058). + - bash .github/scripts/retry.sh make install-tools - make man builds: