test: phase 2 CI exercise

.buildkite/perf.sh

@@ -19,6 +19,11 @@ if [[ ${BUILDKITE_MESSAGE:-} == *"[skip buildkite]"* ]] || [[ ${BUILDKITE_MESSAG
   exit 0
 fi
 
+# Buildkite holds no image-push credentials, so a changed container image
+# might still be waiting on image-push.yml's maintainer approval when this
+# run starts. Wait for the registry to catch up before pulling anything.
+WAIT_FOR_IMAGES_BRANCH="$BUILDKITE_BRANCH" "$(dirname "$0")/../containers/wait-for-images.sh"
+
 os=$(go env GOOS)
 
 # shellcheck source=lib-provider.sh

.buildkite/test.sh

@@ -25,6 +25,11 @@ while IFS= read -r varname; do
 done < <(MSYS_NO_PATHCONV=1 git ls-tree --name-only refs/public-variables-tmp:.github/public-variables/)
 git update-ref -d refs/public-variables-tmp
 
+# Buildkite holds no image-push credentials, so a changed container image
+# might still be waiting on image-push.yml's maintainer approval when this
+# run starts. Wait for the registry to catch up before pulling anything.
+WAIT_FOR_IMAGES_BRANCH="$BUILDKITE_BRANCH" "$(dirname "$0")/../containers/wait-for-images.sh"
+
 export PATH=$PATH:/home/linuxbrew/.linuxbrew/bin
 os=$(go env GOOS)
 

.github/workflows/container-tests.yml

@@ -47,6 +47,12 @@ jobs:
       - uses: actions/checkout@v7
       - name: Run containers/autotag_test.sh
         run: containers/autotag_test.sh
+      - name: Run containers/registry_tag_exists_test.sh
+        run: containers/registry_tag_exists_test.sh
+      - name: Run containers/validate_image_tag_test.sh
+        run: containers/validate_image_tag_test.sh
+      - name: Run containers/wait_for_images_test.sh
+        run: containers/wait_for_images_test.sh
 
   container-build-and-test:
     name: ${{ matrix.os }} - Test container ${{ matrix.containers }}

.github/workflows/image-build-push.yml

@@ -3,13 +3,18 @@ defaults:
   run:
     shell: bash
 
-# Placeholder for #8609 phase 2 (fork-safe automatic image build/push). This
-# do-nothing stub exists only so the workflow name/triggers are registered on
-# the default branch before the real detect/approval/build logic lands - see
-# the phase 2 PR, which will replace this file's contents via rebase.
-# image-push.yml's `workflow_run` trigger only fires for a listener that
-# exists on the default branch, so this stub unblocks testing that dependency
-# ahead of the full PR.
+# For a fork PR, this workflow may run a fork's own Dockerfile/build scripts,
+# so the `build` job never references a secret - the trusted side (loading
+# the artifact `build` produces and actually pushing it) lives in
+# image-push.yml, triggered via workflow_run once this workflow completes,
+# gated behind its own approval.
+#
+# For anything else (a push, or a same-repo PR - `detect`'s `is_fork` output,
+# using the same fork-check idiom as push-tagged-image.yml), there's nothing
+# a maintainer's own branch could smuggle into a job that every other
+# secret-using workflow here doesn't already run unguarded, so
+# `build-and-push` builds and pushes directly, with no environment/approval
+# gate at all - same trust level as `main-build.yml`.
 on:
   pull_request:
     branches: [main]
@@ -22,14 +27,260 @@ on:
       - "containers/**"
       - ".github/workflows/image-build-push.yml"
 
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+env:
+  DOCKER_ORG: "${{ vars.DOCKER_ORG }}"
+
 permissions:
   contents: read
 
 jobs:
-  placeholder:
-    name: "Placeholder (see #8609)"
+  detect:
+    name: Detect changed images
+    runs-on: ubuntu-24.04
+    outputs:
+      matrix: ${{ steps.detect.outputs.matrix }}
+      needs_build: ${{ steps.detect.outputs.needs_build }}
+      is_fork: ${{ steps.detect.outputs.is_fork }}
+    steps:
+      - uses: actions/checkout@v7
+      - name: Compute per-image build status
+        id: detect
+        run: |
+          set -eu -o pipefail
+          BRANCH="${{ github.head_ref || github.ref_name }}"
+          SANITIZED_BRANCH="$(echo "$BRANCH" | sed -E 's/[^A-Za-z0-9_.-]+/-/g')"
+
+          # repo_suffix|hash paths|make dir|make target|arch-suffixed target?|extra repo suffixes
+          # Keep in sync with Makefile's autotag-images target.
+          # TODO(#8609): only the default db variant (mariadb_11.8) is listed
+          # here - see the TODO on autotag-images in the top-level Makefile.
+          CONFIGS=(
+            'ddev-webserver|containers/ddev-webserver containers/containers_shared.mk|ddev-webserver|images|false|ddev-webserver-prod'
+            'ddev-traefik-router|containers/ddev-traefik-router containers/containers_shared.mk|ddev-traefik-router|container|false|'
+            'ddev-ssh-agent|containers/ddev-ssh-agent containers/containers_shared.mk|ddev-ssh-agent|container|false|'
+            'ddev-xhgui|containers/ddev-xhgui containers/containers_shared.mk|ddev-xhgui|container|false|'
+            'ddev-dbserver-mariadb-11.8|containers/ddev-dbserver containers/get_arch.sh|ddev-dbserver|mariadb_11.8|true|'
+          )
+
+          MATRIX_JSON="[]"
+          for entry in "${CONFIGS[@]}"; do
+            IFS='|' read -r repo_suffix hash_paths make_dir make_target arch_suffixed extra_repo_suffixes <<< "$entry"
+            hash="$(containers/hash-paths.sh $hash_paths)"
+            tag="${SANITIZED_BRANCH}-${hash}"
+            repo="${DOCKER_ORG}/${repo_suffix}"
+            if containers/registry-tag-exists.sh "$repo" "$tag"; then
+              echo "detect: ${repo}:${tag} already exists, nothing to build"
+              continue
+            fi
+            echo "detect: ${repo}:${tag} needs building"
+            MATRIX_JSON="$(echo "$MATRIX_JSON" | jq -c \
+              --arg repo "$repo" \
+              --arg tag "$tag" \
+              --arg make_dir "$make_dir" \
+              --arg make_target "$make_target" \
+              --arg arch_suffixed "$arch_suffixed" \
+              --arg extra_repo_suffixes "$extra_repo_suffixes" \
+              '. + [{"repo": $repo, "tag": $tag, "make_dir": $make_dir, "make_target": $make_target, "arch_suffixed": $arch_suffixed, "extra_repo_suffixes": $extra_repo_suffixes}]')"
+          done
+
+          echo "matrix=${MATRIX_JSON}" >> "$GITHUB_OUTPUT"
+          if [ "$(echo "$MATRIX_JSON" | jq 'length')" -gt 0 ]; then
+            echo "needs_build=true" >> "$GITHUB_OUTPUT"
+          else
+            echo "needs_build=false" >> "$GITHUB_OUTPUT"
+          fi
+
+          if [ "${{ github.event_name }}" = "pull_request" ] && \
+             [ "${{ github.event.pull_request.head.repo.owner.login }}" != "${{ github.repository_owner }}" ]; then
+            echo "is_fork=true" >> "$GITHUB_OUTPUT"
+          else
+            echo "is_fork=false" >> "$GITHUB_OUTPUT"
+          fi
+
+  # --- Fork PRs: build with no secrets (this job never has registry
+  # credentials, so there's nothing to gain by gating it - see #8609
+  # discussion), then hand off to image-push.yml for the trusted,
+  # approval-gated push. ---
+
+  build:
+    name: Build ${{ matrix.image.repo }} (${{ matrix.arch }})
+    needs: detect
+    if: needs.detect.outputs.needs_build == 'true' && needs.detect.outputs.is_fork == 'true'
+    strategy:
+      fail-fast: false
+      matrix:
+        image: ${{ fromJson(needs.detect.outputs.matrix) }}
+        arch: [amd64, arm64]
+    runs-on: ${{ matrix.arch == 'arm64' && 'ubuntu-24.04-arm' || 'ubuntu-24.04' }}
+    permissions:
+      contents: read
+    steps:
+      - uses: actions/checkout@v7
+      - name: Build ${{ matrix.image.repo }}:${{ matrix.image.tag }}-${{ matrix.arch }}
+        env:
+          DOCKER_ORG: "${{ vars.DOCKER_ORG }}"
+        run: |
+          set -eu -o pipefail
+          VERSION="${{ matrix.image.tag }}-${{ matrix.arch }}"
+          MAKE_TARGET="${{ matrix.image.make_target }}"
+          if [ "${{ matrix.image.arch_suffixed }}" = "true" ]; then
+            MAKE_TARGET="${MAKE_TARGET}_${{ matrix.arch }}"
+          fi
+          make -C "containers/${{ matrix.image.make_dir }}" "$MAKE_TARGET" VERSION="$VERSION"
+
+          REPOS="${{ matrix.image.repo }}"
+          for suffix in ${{ matrix.image.extra_repo_suffixes }}; do
+            REPOS="${REPOS} ${DOCKER_ORG}/${suffix}"
+          done
+
+          REFS=()
+          : > repos.txt
+          for repo in $REPOS; do
+            REFS+=("${repo}:${VERSION}")
+            echo "$repo" >> repos.txt
+          done
+          docker save "${REFS[@]}" -o image.tar
+          echo -n "${{ matrix.image.tag }}" > tag.txt
+          echo -n "${{ matrix.arch }}" > arch.txt
+      - uses: actions/upload-artifact@v7
+        with:
+          name: image-${{ matrix.image.make_dir }}-${{ matrix.arch }}
+          path: |
+            image.tar
+            repos.txt
+            tag.txt
+            arch.txt
+          retention-days: 1
+
+  # --- Pushes and same-repo PRs: no fork content ever runs here, so build
+  # and push directly with no environment/approval gate - same trust level
+  # as main-build.yml, which already uses this same secret unguarded. ---
+
+  build-and-push:
+    name: Build and push ${{ matrix.image.repo }} (${{ matrix.arch }})
+    needs: detect
+    if: needs.detect.outputs.needs_build == 'true' && needs.detect.outputs.is_fork == 'false'
+    strategy:
+      fail-fast: false
+      matrix:
+        image: ${{ fromJson(needs.detect.outputs.matrix) }}
+        arch: [amd64, arm64]
+    runs-on: ${{ matrix.arch == 'arm64' && 'ubuntu-24.04-arm' || 'ubuntu-24.04' }}
+    steps:
+      - uses: actions/checkout@v7
+
+      - name: Load 1password secret(s)
+        uses: 1password/load-secrets-action@v4
+        with:
+          export-env: true
+        env:
+          OP_SERVICE_ACCOUNT_TOKEN: "${{ secrets.PUSH_SERVICE_ACCOUNT_TOKEN }}"
+          DOCKERHUB_TOKEN: "op://push-secrets/DOCKERHUB_TOKEN/credential"
+
+      - name: Login to DockerHub
+        uses: docker/login-action@v4
+        with:
+          username: ${{ vars.DOCKERHUB_USERNAME }}
+          password: ${{ env.DOCKERHUB_TOKEN }}
+
+      - name: Build and push ${{ matrix.image.repo }}:${{ matrix.image.tag }}-${{ matrix.arch }}
+        env:
+          DOCKER_ORG: "${{ vars.DOCKER_ORG }}"
+        run: |
+          set -eu -o pipefail
+          VERSION="${{ matrix.image.tag }}-${{ matrix.arch }}"
+          MAKE_TARGET="${{ matrix.image.make_target }}"
+          if [ "${{ matrix.image.arch_suffixed }}" = "true" ]; then
+            MAKE_TARGET="${MAKE_TARGET}_${{ matrix.arch }}"
+          fi
+          make -C "containers/${{ matrix.image.make_dir }}" "$MAKE_TARGET" VERSION="$VERSION"
+
+          REPOS="${{ matrix.image.repo }}"
+          for suffix in ${{ matrix.image.extra_repo_suffixes }}; do
+            REPOS="${REPOS} ${DOCKER_ORG}/${suffix}"
+          done
+          for repo in $REPOS; do
+            docker push "${repo}:${VERSION}"
+          done
+
+  create-manifests:
+    name: Create manifest for ${{ matrix.image.repo }}
+    needs: [detect, build-and-push]
+    if: needs.detect.outputs.needs_build == 'true' && needs.detect.outputs.is_fork == 'false'
+    strategy:
+      fail-fast: false
+      matrix:
+        image: ${{ fromJson(needs.detect.outputs.matrix) }}
     runs-on: ubuntu-24.04
+    permissions:
+      contents: read
+      pull-requests: write
     steps:
-      - name: Do nothing
+      - uses: actions/checkout@v7
+
+      - name: Load 1password secret(s)
+        uses: 1password/load-secrets-action@v4
+        with:
+          export-env: true
+        env:
+          OP_SERVICE_ACCOUNT_TOKEN: "${{ secrets.PUSH_SERVICE_ACCOUNT_TOKEN }}"
+          DOCKERHUB_TOKEN: "op://push-secrets/DOCKERHUB_TOKEN/credential"
+
+      - name: Login to DockerHub
+        uses: docker/login-action@v4
+        with:
+          username: ${{ vars.DOCKERHUB_USERNAME }}
+          password: ${{ env.DOCKERHUB_TOKEN }}
+
+      - name: Create manifest and clean up per-arch tags
+        id: manifest
+        env:
+          DOCKER_ORG: "${{ vars.DOCKER_ORG }}"
         run: |
-          echo "Placeholder for #8609 phase 2 - no-op until the real workflow lands."
+          set -eu -o pipefail
+          TAG="${{ matrix.image.tag }}"
+
+          DOCKERHUB_JWT="$(curl -s -H "Content-Type: application/json" -X POST \
+            -d '{"username":"'"${{ vars.DOCKERHUB_USERNAME }}"'","password":"'"${DOCKERHUB_TOKEN}"'"}' \
+            https://hub.docker.com/v2/users/login/ | jq -r .token)"
+
+          REPOS="${{ matrix.image.repo }}"
+          for suffix in ${{ matrix.image.extra_repo_suffixes }}; do
+            REPOS="${REPOS} ${DOCKER_ORG}/${suffix}"
+          done
+
+          PUSHED_SUMMARY=""
+          for repo in $REPOS; do
+            docker buildx imagetools create -t "${repo}:${TAG}" "${repo}:${TAG}-amd64" "${repo}:${TAG}-arm64"
+            PUSHED_SUMMARY="${PUSHED_SUMMARY}- \`${repo}:${TAG}\`"$'\n'
+            for arch in amd64 arm64; do
+              echo "Removing intermediary tag ${repo}:${TAG}-${arch}"
+              curl -s -X DELETE -H "Authorization: JWT ${DOCKERHUB_JWT}" \
+                "https://hub.docker.com/v2/repositories/${repo}/tags/${TAG}-${arch}/" >/dev/null || true
+            done
+          done
+
+          {
+            echo "summary<<IMAGE_PUSH_EOF"
+            echo "$PUSHED_SUMMARY"
+            echo "IMAGE_PUSH_EOF"
+          } >> "$GITHUB_OUTPUT"
+
+      - name: Comment on the pull request
+        if: github.event_name == 'pull_request'
+        uses: actions/github-script@v9
+        env:
+          IMAGE_PUSH_SUMMARY: ${{ steps.manifest.outputs.summary }}
+        with:
+          script: |
+            const summary = (process.env.IMAGE_PUSH_SUMMARY || "").trim();
+            await github.rest.issues.createComment({
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+              issue_number: context.issue.number,
+              body: `Pushed updated container image(s) for this PR:\n\n${summary}`,
+            });