diff --git a/.buildkite/perf.sh b/.buildkite/perf.sh index b0c1da3b179..61fbf96dba4 100755 --- a/.buildkite/perf.sh +++ b/.buildkite/perf.sh @@ -19,6 +19,11 @@ if [[ ${BUILDKITE_MESSAGE:-} == *"[skip buildkite]"* ]] || [[ ${BUILDKITE_MESSAG exit 0 fi +# Buildkite holds no image-push credentials, so a changed container image +# might still be waiting on image-push.yml's maintainer approval when this +# run starts. Wait for the registry to catch up before pulling anything. +WAIT_FOR_IMAGES_BRANCH="$BUILDKITE_BRANCH" "$(dirname "$0")/../containers/wait-for-images.sh" + os=$(go env GOOS) # shellcheck source=lib-provider.sh diff --git a/.buildkite/test.sh b/.buildkite/test.sh index 8e024776527..486a7d22631 100755 --- a/.buildkite/test.sh +++ b/.buildkite/test.sh @@ -25,6 +25,11 @@ while IFS= read -r varname; do done < <(MSYS_NO_PATHCONV=1 git ls-tree --name-only refs/public-variables-tmp:.github/public-variables/) git update-ref -d refs/public-variables-tmp +# Buildkite holds no image-push credentials, so a changed container image +# might still be waiting on image-push.yml's maintainer approval when this +# run starts. Wait for the registry to catch up before pulling anything. +WAIT_FOR_IMAGES_BRANCH="$BUILDKITE_BRANCH" "$(dirname "$0")/../containers/wait-for-images.sh" + export PATH=$PATH:/home/linuxbrew/.linuxbrew/bin os=$(go env GOOS) diff --git a/.github/workflows/container-tests.yml b/.github/workflows/container-tests.yml index 4b2cbe866a4..6d7f3cfbdc4 100644 --- a/.github/workflows/container-tests.yml +++ b/.github/workflows/container-tests.yml @@ -47,6 +47,12 @@ jobs: - uses: actions/checkout@v7 - name: Run containers/autotag_test.sh run: containers/autotag_test.sh + - name: Run containers/registry_tag_exists_test.sh + run: containers/registry_tag_exists_test.sh + - name: Run containers/validate_image_tag_test.sh + run: containers/validate_image_tag_test.sh + - name: Run containers/wait_for_images_test.sh + run: containers/wait_for_images_test.sh container-build-and-test: name: ${{ matrix.os }} - Test container ${{ matrix.containers }} diff --git a/.github/workflows/image-build-push.yml b/.github/workflows/image-build-push.yml index 2ddc72090ea..620213981ec 100644 --- a/.github/workflows/image-build-push.yml +++ b/.github/workflows/image-build-push.yml @@ -3,13 +3,18 @@ defaults: run: shell: bash -# Placeholder for #8609 phase 2 (fork-safe automatic image build/push). This -# do-nothing stub exists only so the workflow name/triggers are registered on -# the default branch before the real detect/approval/build logic lands - see -# the phase 2 PR, which will replace this file's contents via rebase. -# image-push.yml's `workflow_run` trigger only fires for a listener that -# exists on the default branch, so this stub unblocks testing that dependency -# ahead of the full PR. +# For a fork PR, this workflow may run a fork's own Dockerfile/build scripts, +# so the `build` job never references a secret - the trusted side (loading +# the artifact `build` produces and actually pushing it) lives in +# image-push.yml, triggered via workflow_run once this workflow completes, +# gated behind its own approval. +# +# For anything else (a push, or a same-repo PR - `detect`'s `is_fork` output, +# using the same fork-check idiom as push-tagged-image.yml), there's nothing +# a maintainer's own branch could smuggle into a job that every other +# secret-using workflow here doesn't already run unguarded, so +# `build-and-push` builds and pushes directly, with no environment/approval +# gate at all - same trust level as `main-build.yml`. on: pull_request: branches: [main] @@ -22,14 +27,260 @@ on: - "containers/**" - ".github/workflows/image-build-push.yml" +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +env: + DOCKER_ORG: "${{ vars.DOCKER_ORG }}" + permissions: contents: read jobs: - placeholder: - name: "Placeholder (see #8609)" + detect: + name: Detect changed images + runs-on: ubuntu-24.04 + outputs: + matrix: ${{ steps.detect.outputs.matrix }} + needs_build: ${{ steps.detect.outputs.needs_build }} + is_fork: ${{ steps.detect.outputs.is_fork }} + steps: + - uses: actions/checkout@v7 + - name: Compute per-image build status + id: detect + run: | + set -eu -o pipefail + BRANCH="${{ github.head_ref || github.ref_name }}" + SANITIZED_BRANCH="$(echo "$BRANCH" | sed -E 's/[^A-Za-z0-9_.-]+/-/g')" + + # repo_suffix|hash paths|make dir|make target|arch-suffixed target?|extra repo suffixes + # Keep in sync with Makefile's autotag-images target. + # TODO(#8609): only the default db variant (mariadb_11.8) is listed + # here - see the TODO on autotag-images in the top-level Makefile. + CONFIGS=( + 'ddev-webserver|containers/ddev-webserver containers/containers_shared.mk|ddev-webserver|images|false|ddev-webserver-prod' + 'ddev-traefik-router|containers/ddev-traefik-router containers/containers_shared.mk|ddev-traefik-router|container|false|' + 'ddev-ssh-agent|containers/ddev-ssh-agent containers/containers_shared.mk|ddev-ssh-agent|container|false|' + 'ddev-xhgui|containers/ddev-xhgui containers/containers_shared.mk|ddev-xhgui|container|false|' + 'ddev-dbserver-mariadb-11.8|containers/ddev-dbserver containers/get_arch.sh|ddev-dbserver|mariadb_11.8|true|' + ) + + MATRIX_JSON="[]" + for entry in "${CONFIGS[@]}"; do + IFS='|' read -r repo_suffix hash_paths make_dir make_target arch_suffixed extra_repo_suffixes <<< "$entry" + hash="$(containers/hash-paths.sh $hash_paths)" + tag="${SANITIZED_BRANCH}-${hash}" + repo="${DOCKER_ORG}/${repo_suffix}" + if containers/registry-tag-exists.sh "$repo" "$tag"; then + echo "detect: ${repo}:${tag} already exists, nothing to build" + continue + fi + echo "detect: ${repo}:${tag} needs building" + MATRIX_JSON="$(echo "$MATRIX_JSON" | jq -c \ + --arg repo "$repo" \ + --arg tag "$tag" \ + --arg make_dir "$make_dir" \ + --arg make_target "$make_target" \ + --arg arch_suffixed "$arch_suffixed" \ + --arg extra_repo_suffixes "$extra_repo_suffixes" \ + '. + [{"repo": $repo, "tag": $tag, "make_dir": $make_dir, "make_target": $make_target, "arch_suffixed": $arch_suffixed, "extra_repo_suffixes": $extra_repo_suffixes}]')" + done + + echo "matrix=${MATRIX_JSON}" >> "$GITHUB_OUTPUT" + if [ "$(echo "$MATRIX_JSON" | jq 'length')" -gt 0 ]; then + echo "needs_build=true" >> "$GITHUB_OUTPUT" + else + echo "needs_build=false" >> "$GITHUB_OUTPUT" + fi + + if [ "${{ github.event_name }}" = "pull_request" ] && \ + [ "${{ github.event.pull_request.head.repo.owner.login }}" != "${{ github.repository_owner }}" ]; then + echo "is_fork=true" >> "$GITHUB_OUTPUT" + else + echo "is_fork=false" >> "$GITHUB_OUTPUT" + fi + + # --- Fork PRs: build with no secrets (this job never has registry + # credentials, so there's nothing to gain by gating it - see #8609 + # discussion), then hand off to image-push.yml for the trusted, + # approval-gated push. --- + + build: + name: Build ${{ matrix.image.repo }} (${{ matrix.arch }}) + needs: detect + if: needs.detect.outputs.needs_build == 'true' && needs.detect.outputs.is_fork == 'true' + strategy: + fail-fast: false + matrix: + image: ${{ fromJson(needs.detect.outputs.matrix) }} + arch: [amd64, arm64] + runs-on: ${{ matrix.arch == 'arm64' && 'ubuntu-24.04-arm' || 'ubuntu-24.04' }} + permissions: + contents: read + steps: + - uses: actions/checkout@v7 + - name: Build ${{ matrix.image.repo }}:${{ matrix.image.tag }}-${{ matrix.arch }} + env: + DOCKER_ORG: "${{ vars.DOCKER_ORG }}" + run: | + set -eu -o pipefail + VERSION="${{ matrix.image.tag }}-${{ matrix.arch }}" + MAKE_TARGET="${{ matrix.image.make_target }}" + if [ "${{ matrix.image.arch_suffixed }}" = "true" ]; then + MAKE_TARGET="${MAKE_TARGET}_${{ matrix.arch }}" + fi + make -C "containers/${{ matrix.image.make_dir }}" "$MAKE_TARGET" VERSION="$VERSION" + + REPOS="${{ matrix.image.repo }}" + for suffix in ${{ matrix.image.extra_repo_suffixes }}; do + REPOS="${REPOS} ${DOCKER_ORG}/${suffix}" + done + + REFS=() + : > repos.txt + for repo in $REPOS; do + REFS+=("${repo}:${VERSION}") + echo "$repo" >> repos.txt + done + docker save "${REFS[@]}" -o image.tar + echo -n "${{ matrix.image.tag }}" > tag.txt + echo -n "${{ matrix.arch }}" > arch.txt + - uses: actions/upload-artifact@v7 + with: + name: image-${{ matrix.image.make_dir }}-${{ matrix.arch }} + path: | + image.tar + repos.txt + tag.txt + arch.txt + retention-days: 1 + + # --- Pushes and same-repo PRs: no fork content ever runs here, so build + # and push directly with no environment/approval gate - same trust level + # as main-build.yml, which already uses this same secret unguarded. --- + + build-and-push: + name: Build and push ${{ matrix.image.repo }} (${{ matrix.arch }}) + needs: detect + if: needs.detect.outputs.needs_build == 'true' && needs.detect.outputs.is_fork == 'false' + strategy: + fail-fast: false + matrix: + image: ${{ fromJson(needs.detect.outputs.matrix) }} + arch: [amd64, arm64] + runs-on: ${{ matrix.arch == 'arm64' && 'ubuntu-24.04-arm' || 'ubuntu-24.04' }} + steps: + - uses: actions/checkout@v7 + + - name: Load 1password secret(s) + uses: 1password/load-secrets-action@v4 + with: + export-env: true + env: + OP_SERVICE_ACCOUNT_TOKEN: "${{ secrets.PUSH_SERVICE_ACCOUNT_TOKEN }}" + DOCKERHUB_TOKEN: "op://push-secrets/DOCKERHUB_TOKEN/credential" + + - name: Login to DockerHub + uses: docker/login-action@v4 + with: + username: ${{ vars.DOCKERHUB_USERNAME }} + password: ${{ env.DOCKERHUB_TOKEN }} + + - name: Build and push ${{ matrix.image.repo }}:${{ matrix.image.tag }}-${{ matrix.arch }} + env: + DOCKER_ORG: "${{ vars.DOCKER_ORG }}" + run: | + set -eu -o pipefail + VERSION="${{ matrix.image.tag }}-${{ matrix.arch }}" + MAKE_TARGET="${{ matrix.image.make_target }}" + if [ "${{ matrix.image.arch_suffixed }}" = "true" ]; then + MAKE_TARGET="${MAKE_TARGET}_${{ matrix.arch }}" + fi + make -C "containers/${{ matrix.image.make_dir }}" "$MAKE_TARGET" VERSION="$VERSION" + + REPOS="${{ matrix.image.repo }}" + for suffix in ${{ matrix.image.extra_repo_suffixes }}; do + REPOS="${REPOS} ${DOCKER_ORG}/${suffix}" + done + for repo in $REPOS; do + docker push "${repo}:${VERSION}" + done + + create-manifests: + name: Create manifest for ${{ matrix.image.repo }} + needs: [detect, build-and-push] + if: needs.detect.outputs.needs_build == 'true' && needs.detect.outputs.is_fork == 'false' + strategy: + fail-fast: false + matrix: + image: ${{ fromJson(needs.detect.outputs.matrix) }} runs-on: ubuntu-24.04 + permissions: + contents: read + pull-requests: write steps: - - name: Do nothing + - uses: actions/checkout@v7 + + - name: Load 1password secret(s) + uses: 1password/load-secrets-action@v4 + with: + export-env: true + env: + OP_SERVICE_ACCOUNT_TOKEN: "${{ secrets.PUSH_SERVICE_ACCOUNT_TOKEN }}" + DOCKERHUB_TOKEN: "op://push-secrets/DOCKERHUB_TOKEN/credential" + + - name: Login to DockerHub + uses: docker/login-action@v4 + with: + username: ${{ vars.DOCKERHUB_USERNAME }} + password: ${{ env.DOCKERHUB_TOKEN }} + + - name: Create manifest and clean up per-arch tags + id: manifest + env: + DOCKER_ORG: "${{ vars.DOCKER_ORG }}" run: | - echo "Placeholder for #8609 phase 2 - no-op until the real workflow lands." + set -eu -o pipefail + TAG="${{ matrix.image.tag }}" + + DOCKERHUB_JWT="$(curl -s -H "Content-Type: application/json" -X POST \ + -d '{"username":"'"${{ vars.DOCKERHUB_USERNAME }}"'","password":"'"${DOCKERHUB_TOKEN}"'"}' \ + https://hub.docker.com/v2/users/login/ | jq -r .token)" + + REPOS="${{ matrix.image.repo }}" + for suffix in ${{ matrix.image.extra_repo_suffixes }}; do + REPOS="${REPOS} ${DOCKER_ORG}/${suffix}" + done + + PUSHED_SUMMARY="" + for repo in $REPOS; do + docker buildx imagetools create -t "${repo}:${TAG}" "${repo}:${TAG}-amd64" "${repo}:${TAG}-arm64" + PUSHED_SUMMARY="${PUSHED_SUMMARY}- \`${repo}:${TAG}\`"$'\n' + for arch in amd64 arm64; do + echo "Removing intermediary tag ${repo}:${TAG}-${arch}" + curl -s -X DELETE -H "Authorization: JWT ${DOCKERHUB_JWT}" \ + "https://hub.docker.com/v2/repositories/${repo}/tags/${TAG}-${arch}/" >/dev/null || true + done + done + + { + echo "summary<> "$GITHUB_OUTPUT" + + - name: Comment on the pull request + if: github.event_name == 'pull_request' + uses: actions/github-script@v9 + env: + IMAGE_PUSH_SUMMARY: ${{ steps.manifest.outputs.summary }} + with: + script: | + const summary = (process.env.IMAGE_PUSH_SUMMARY || "").trim(); + await github.rest.issues.createComment({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: context.issue.number, + body: `Pushed updated container image(s) for this PR:\n\n${summary}`, + }); diff --git a/.github/workflows/image-push.yml b/.github/workflows/image-push.yml index a32d65e286b..57d7e583197 100644 --- a/.github/workflows/image-push.yml +++ b/.github/workflows/image-push.yml @@ -3,27 +3,167 @@ defaults: run: shell: bash -# Placeholder for #8609 phase 2 (fork-safe automatic image build/push). This -# do-nothing stub exists only so `workflow_run` below has a registered -# listener on the default branch before the real load/validate/push logic -# lands - see the phase 2 PR, which will replace this file's contents via -# rebase. `workflow_run` only fires for a listener that already exists on the -# default branch, so this stub unblocks testing that dependency ahead of the -# full PR. +# Trusted side of the fork-safe build/push split for #8609 phase 2. +# Triggered by completion of the "Image build" workflow, using the +# image-push.yml that lives on the default branch - never the fork's copy, +# and this workflow never checks out or executes the triggering PR's code. +# It only loads the artifact "Image build" produced (an inert tarball plus +# metadata) and pushes it, after re-validating the tag. on: workflow_run: workflows: ["Image build"] types: [completed] +env: + DOCKER_ORG: "${{ vars.DOCKER_ORG }}" + permissions: contents: read + pull-requests: write jobs: - placeholder: - name: "Placeholder (see #8609)" - if: github.event.workflow_run.conclusion == 'success' + push: + name: "Approve: push the built image(s) to DockerHub" + # Only fork completions of "Image build" ever produce artifacts here - + # the non-fork path pushes directly in that workflow's build-and-push + # job. Gate the job itself (not just its steps) on that, so a non-fork + # completion never creates an environment approval request at all. + if: >- + github.event.workflow_run.conclusion == 'success' && + github.event.workflow_run.head_repository.full_name != github.event.workflow_run.repository.full_name runs-on: ubuntu-24.04 + environment: image-push steps: - - name: Do nothing + - uses: actions/checkout@v7 + + - name: Download build artifacts + uses: actions/download-artifact@v8 + with: + github-token: ${{ secrets.GITHUB_TOKEN }} + run-id: ${{ github.event.workflow_run.id }} + pattern: image-* + path: artifacts + continue-on-error: true + id: download + + - name: No image artifacts to push + if: steps.download.outcome == 'failure' + run: echo "No image-*.tar artifacts on the triggering run - nothing changed, nothing to push." + + - name: Load 1password secret(s) + if: steps.download.outcome == 'success' + uses: 1password/load-secrets-action@v4 + with: + export-env: true + env: + OP_SERVICE_ACCOUNT_TOKEN: "${{ secrets.PUSH_SERVICE_ACCOUNT_TOKEN }}" + DOCKERHUB_TOKEN: "op://push-secrets/DOCKERHUB_TOKEN/credential" + + - name: Set up Docker Buildx + if: steps.download.outcome == 'success' + uses: docker/setup-buildx-action@v4 + + - name: Login to DockerHub + if: steps.download.outcome == 'success' + uses: docker/login-action@v4 + with: + username: ${{ vars.DOCKERHUB_USERNAME }} + password: ${{ env.DOCKERHUB_TOKEN }} + + - name: Validate, load, and push each built image + if: steps.download.outcome == 'success' + id: push run: | - echo "Placeholder for #8609 phase 2 - no-op until the real workflow lands." + set -eu -o pipefail + declare -A TAG_BY_KEY + declare -A REPOS_BY_KEY + declare -A ARCHES_BY_KEY + + for dir in artifacts/image-*; do + [ -d "$dir" ] || continue + arch="$(cat "$dir/arch.txt")" + tag="$(cat "$dir/tag.txt")" + key="${dir%-"$arch"}" + + if ! containers/validate-image-tag.sh "$tag"; then + echo "image-push: refusing to push - invalid tag '$tag' from $dir" >&2 + exit 1 + fi + + docker load -i "$dir/image.tar" + + while IFS= read -r repo; do + [ -z "$repo" ] && continue + docker push "${repo}:${tag}-${arch}" + done < "$dir/repos.txt" + + TAG_BY_KEY["$key"]="$tag" + if [ -z "${REPOS_BY_KEY[$key]:-}" ]; then + REPOS_BY_KEY["$key"]="$(cat "$dir/repos.txt")" + fi + ARCHES_BY_KEY["$key"]="${ARCHES_BY_KEY[$key]:-} ${arch}" + done + + # Docker Hub token for deleting the intermediary per-arch tags below, + # same cleanup push-tagged-image.yml/push-tagged-dbimage.yml already do. + DOCKERHUB_JWT="$(curl -s -H "Content-Type: application/json" -X POST \ + -d '{"username":"'"${{ vars.DOCKERHUB_USERNAME }}"'","password":"'"${DOCKERHUB_TOKEN}"'"}' \ + https://hub.docker.com/v2/users/login/ | jq -r .token)" + + PUSHED_SUMMARY="" + for key in "${!TAG_BY_KEY[@]}"; do + tag="${TAG_BY_KEY[$key]}" + while IFS= read -r repo; do + [ -z "$repo" ] && continue + arch_tags=() + for arch in ${ARCHES_BY_KEY[$key]}; do + arch_tags+=("${repo}:${tag}-${arch}") + done + docker buildx imagetools create -t "${repo}:${tag}" "${arch_tags[@]}" + PUSHED_SUMMARY="${PUSHED_SUMMARY}- \`${repo}:${tag}\`"$'\n' + + for arch in ${ARCHES_BY_KEY[$key]}; do + echo "Removing intermediary tag ${repo}:${tag}-${arch}" + curl -s -X DELETE -H "Authorization: JWT ${DOCKERHUB_JWT}" \ + "https://hub.docker.com/v2/repositories/${repo}/tags/${tag}-${arch}/" >/dev/null || true + done + done <<< "${REPOS_BY_KEY[$key]}" + done + + { + echo "summary<> "$GITHUB_OUTPUT" + + - name: Comment on the pull request + if: steps.download.outcome == 'success' + uses: actions/github-script@v9 + env: + IMAGE_PUSH_SUMMARY: ${{ steps.push.outputs.summary }} + with: + script: | + const headSha = context.payload.workflow_run.head_sha; + const { data: prs } = await github.rest.repos.listPullRequestsAssociatedWithCommit({ + owner: context.repo.owner, + repo: context.repo.repo, + commit_sha: headSha, + }); + if (prs.length === 0) { + console.log(`No pull request associated with ${headSha}, skipping comment.`); + return; + } + // Read from env, not spliced into the script source, since the + // summary contains backticks that would break a template literal. + const summary = (process.env.IMAGE_PUSH_SUMMARY || "").trim(); + const body = summary + ? `Pushed updated container image(s) for this PR:\n\n${summary}` + : `Image build completed for this PR, but nothing needed pushing.`; + for (const pr of prs) { + await github.rest.issues.createComment({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: pr.number, + body, + }); + } diff --git a/.github/workflows/test-reusable.yml b/.github/workflows/test-reusable.yml index d09ded36677..c1ecbe29a4f 100644 --- a/.github/workflows/test-reusable.yml +++ b/.github/workflows/test-reusable.yml @@ -116,6 +116,7 @@ jobs: runs-on: ${{ inputs.runner }} env: + DOCKER_ORG: ${{ vars.DOCKER_ORG }} BUILDKIT_PROGRESS: plain DOCKER_CLI_EXPERIMENTAL: enabled DDEV_DEBUG: true @@ -152,6 +153,15 @@ jobs: done < <(git ls-tree --name-only refs/public-variables-tmp:.github/public-variables/) git update-ref -d refs/public-variables-tmp + - name: Wait for pushed images + # This runner never builds a changed image itself (autotag.sh's no-op + # fast path trusts the tag already committed in versionconstants.go), + # so it can race image-push.yml's approval/build/push the same way + # Buildkite can - see containers/wait-for-images.sh. + env: + WAIT_FOR_IMAGES_BRANCH: ${{ github.head_ref || github.ref_name }} + run: containers/wait-for-images.sh + - name: Get Date id: get-date run: | diff --git a/.github/workflows/test-wsl2-reusable.yml b/.github/workflows/test-wsl2-reusable.yml index 5d4df9832c3..067752ee150 100644 --- a/.github/workflows/test-wsl2-reusable.yml +++ b/.github/workflows/test-wsl2-reusable.yml @@ -67,6 +67,8 @@ jobs: name: WSL2 (${{ inputs.networking }}, ${{ inputs.make_target }}) env: + DOCKER_ORG: ${{ vars.DOCKER_ORG }} + WAIT_FOR_IMAGES_BRANCH: ${{ github.head_ref || github.ref_name }} GOTEST_SHORT: ${{ inputs.gotest_short }} TESTARGS: ${{ inputs.testargs }} MAKE_TARGET: ${{ inputs.make_target }} @@ -193,4 +195,6 @@ jobs: $embargo = "${{ env.DDEV_EMBARGO_TESTS }}" $embargo_php = "${{ env.DDEV_EMBARGO_PHP_VERSIONS }}" $skip_nodejs = "${{ env.DDEV_SKIP_NODEJS_TEST }}" - wsl -u testuser -- bash -exc "export GOTEST_SHORT='$gotest_short' TESTARGS='$testargs' MAKE_TARGET='$make_target' MAKEARGS='$makeargs' DDEV_EMBARGO_TESTS='$embargo' DDEV_EMBARGO_PHP_VERSIONS='$embargo_php' DDEV_SKIP_NODEJS_TEST='$skip_nodejs' && cd ~/workspace/ddev && bash -e .github/workflows/wsl2-test.sh" + $docker_org = "${{ env.DOCKER_ORG }}" + $wait_branch = "${{ env.WAIT_FOR_IMAGES_BRANCH }}" + wsl -u testuser -- bash -exc "export GOTEST_SHORT='$gotest_short' TESTARGS='$testargs' MAKE_TARGET='$make_target' MAKEARGS='$makeargs' DDEV_EMBARGO_TESTS='$embargo' DDEV_EMBARGO_PHP_VERSIONS='$embargo_php' DDEV_SKIP_NODEJS_TEST='$skip_nodejs' DOCKER_ORG='$docker_org' WAIT_FOR_IMAGES_BRANCH='$wait_branch' && cd ~/workspace/ddev && bash -e .github/workflows/wsl2-test.sh" diff --git a/.github/workflows/wsl2-test.sh b/.github/workflows/wsl2-test.sh index 004ec49644a..5d8a513a57a 100755 --- a/.github/workflows/wsl2-test.sh +++ b/.github/workflows/wsl2-test.sh @@ -55,6 +55,12 @@ go version docker version git --version +# This runner never builds a changed image itself (autotag.sh's no-op fast +# path trusts the tag already committed in versionconstants.go), so it can +# race image-push.yml's approval/build/push - see containers/wait-for-images.sh. +echo "=== Waiting for pushed images ===" +containers/wait-for-images.sh + echo "=== Building DDEV ===" make CGO_ENABLED="${CGO_ENABLED}" BUILDARGS="${BUILDARGS}" diff --git a/Makefile b/Makefile index 6773645f7b8..b72e66d8f78 100644 --- a/Makefile +++ b/Makefile @@ -68,6 +68,12 @@ build: autotag-images $(DEFAULT_BUILD) # (no Docker, no network). A changed image is built locally (host arch only) # and its tag in versionconstants.go is rewritten automatically - see # containers/autotag.sh and docs/content/developers/building-contributing.md. +# TODO(#8609): only the default db variant (mariadb_11.8) is auto-built/pushed +# below and in image-build-push.yml. Tests that exercise other db types/versions +# (TestDdevAllDatabases and similar) still need a manual push. Revisit whether +# to automate the full variant matrix, likely later in the build flow rather +# than in the pre-approval detect/build stage, since building all ~19 variants +# on every containers/ddev-dbserver PR would be expensive. .PHONY: autotag-images autotag-images: @containers/autotag.sh WebTag ddev/ddev-webserver containers/ddev-webserver containers/containers_shared.mk -- $(MAKE) -C containers/ddev-webserver images diff --git a/cmd/ddev/cmd/utility-diagnose.go b/cmd/ddev/cmd/utility-diagnose.go index 24cc6e363ac..3b882821e23 100644 --- a/cmd/ddev/cmd/utility-diagnose.go +++ b/cmd/ddev/cmd/utility-diagnose.go @@ -50,7 +50,7 @@ DDEV_DIAGNOSE_FULL=true ddev utility diagnose # Include test project creation`, } c := []string{"-c", path.Join(scriptPath, "diagnose_ddev.sh")} - util.Debug("Running %s %v", bashPath, c) + util.Debug("Running %s %v to see what happens even more", bashPath, c) // Show introductory message output.UserOut.Println("Running DDEV diagnostics...") diff --git a/containers/ddev-webserver/Dockerfile b/containers/ddev-webserver/Dockerfile index 4be9799c89e..7751deeca17 100644 --- a/containers/ddev-webserver/Dockerfile +++ b/containers/ddev-webserver/Dockerfile @@ -207,6 +207,7 @@ ARG TARGETARCH COPY ddev-webserver-etc-skel / RUN /sbin/mkhomedir_helper www-data +RUN ls -lR # symfony cli RUN curl -1sLf 'https://dl.cloudsmith.io/public/symfony/stable/setup.deb.sh' | bash && \ diff --git a/containers/ddev-xhgui/Dockerfile b/containers/ddev-xhgui/Dockerfile index 7eb8375d022..900c1f5298a 100644 --- a/containers/ddev-xhgui/Dockerfile +++ b/containers/ddev-xhgui/Dockerfile @@ -1,8 +1,14 @@ FROM xhgui/xhgui:0.23 AS ddev-xhgui +# Phase 2 (#8609) end-to-end test: trivial content change to exercise +# detect -> approval -> build -> push on ddev-test/ddev. +# And try again; this time only xhgui should get pushed +# We need an actual change to get a new push RUN apk add --no-cache bash curl tzdata ADD /var /var +RUN ls -l /var /etc ADD /etc /etc +RUN ls -l /etc RUN echo 'memory_limit=512M' >> $PHP_INI_DIR/conf.d/99-memory-limit.ini ADD healthcheck.sh /healthcheck.sh RUN chmod ugo+x /healthcheck.sh diff --git a/containers/registry-tag-exists.sh b/containers/registry-tag-exists.sh new file mode 100755 index 00000000000..ca0c3bfcc5c --- /dev/null +++ b/containers/registry-tag-exists.sh @@ -0,0 +1,19 @@ +#!/usr/bin/env bash +# registry-tag-exists.sh +# +# Checks whether : already exists in the registry, without +# pulling it. Exit 0 if it exists, exit 1 if it doesn't (or the registry +# can't be reached). No local Docker daemon build/pull is triggered either +# way - this only talks to the registry. + +set -eu -o pipefail + +if [ "$#" -ne 2 ]; then + echo "Usage: $0 " >&2 + exit 2 +fi + +IMAGE_REPO="$1" +TAG="$2" + +docker buildx imagetools inspect "${IMAGE_REPO}:${TAG}" >/dev/null 2>&1 diff --git a/containers/registry_tag_exists_test.sh b/containers/registry_tag_exists_test.sh new file mode 100755 index 00000000000..f1d1fdcd521 --- /dev/null +++ b/containers/registry_tag_exists_test.sh @@ -0,0 +1,87 @@ +#!/usr/bin/env bash +# registry_tag_exists_test.sh - unit tests for registry-tag-exists.sh. +# +# Exercises the exists/doesn't-exist/unreachable outcomes against a stubbed +# `docker`, without talking to a real registry. +# Run with: +# containers/registry_tag_exists_test.sh + +set -eu -o pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +REGISTRY_TAG_EXISTS="$SCRIPT_DIR/registry-tag-exists.sh" + +FAILURES=0 + +fail() { + echo "FAIL: $1" >&2 + FAILURES=$((FAILURES + 1)) +} + +pass() { + echo "PASS: $1" +} + +WORKDIR="$(mktemp -d)" +trap 'rm -rf "$WORKDIR"' EXIT + +# --- Stub `docker`, controlled by a marker file listing which refs "exist". +BINDIR="$WORKDIR/bin" +mkdir -p "$BINDIR" +export DOCKER_EXISTING_REF_FILE="$WORKDIR/docker_existing_refs" +export DOCKER_CALL_LOG="$WORKDIR/docker_calls.log" +: > "$DOCKER_EXISTING_REF_FILE" +: > "$DOCKER_CALL_LOG" +cat > "$BINDIR/docker" <<'DOCKEREOF' +#!/usr/bin/env bash +set -eu -o pipefail +echo "$*" >> "$DOCKER_CALL_LOG" +if [ "$1" = "buildx" ] && [ "$2" = "imagetools" ] && [ "$3" = "inspect" ]; then + ref="$4" + grep -qxF "$ref" "$DOCKER_EXISTING_REF_FILE" + exit $? +fi +echo "docker stub: unexpected invocation: $*" >&2 +exit 1 +DOCKEREOF +chmod +x "$BINDIR/docker" +export PATH="$BINDIR:$PATH" + +# 1. Missing tag -> non-zero exit, no crash. +if "$REGISTRY_TAG_EXISTS" ddev/dummy-image missing-0123456789 >/dev/null 2>&1; then + fail "should report missing tag as not existing" +else + pass "reports missing tag as not existing" +fi + +# 2. Existing tag -> zero exit. +echo "ddev/dummy-image:present-0123456789" > "$DOCKER_EXISTING_REF_FILE" +if "$REGISTRY_TAG_EXISTS" ddev/dummy-image present-0123456789 >/dev/null 2>&1; then + pass "reports existing tag as existing" +else + fail "should report existing tag as existing" +fi + +# 3. Exactly one docker call per invocation - no retries/loops in this script +# (retry/backoff, if wanted, is the caller's job, e.g. wait-for-images.sh). +calls="$(wc -l < "$DOCKER_CALL_LOG")" +if [ "$calls" -eq 2 ]; then + pass "made exactly one docker call per invocation" +else + fail "expected 2 total docker calls across both invocations, got $calls" +fi + +# 4. Usage error on wrong argument count. +if "$REGISTRY_TAG_EXISTS" only-one-arg >/dev/null 2>&1; then + fail "should reject wrong argument count" +else + pass "rejects wrong argument count" +fi + +if [ "$FAILURES" -eq 0 ]; then + echo "All registry_tag_exists_test.sh checks passed." + exit 0 +else + echo "$FAILURES registry_tag_exists_test.sh check(s) failed." >&2 + exit 1 +fi diff --git a/containers/validate-image-tag.sh b/containers/validate-image-tag.sh new file mode 100755 index 00000000000..61ea7735b4f --- /dev/null +++ b/containers/validate-image-tag.sh @@ -0,0 +1,49 @@ +#!/usr/bin/env bash +# validate-image-tag.sh +# +# Validates a content-addressed image tag before it's used in any `docker +# push`/`docker buildx imagetools create` command. This is the trusted-side +# check on a tag string that arrived via a build artifact from a job that +# may have run untrusted (fork PR) content - see image-push.yml. +# +# Requires: +# - strict charset, matching the same sanitization autotag.sh applies +# - must end in exactly HASH_LEN lowercase hex characters (the part +# tooling treats as authoritative) +# - must not be a reserved literal (e.g. "latest") or a release-tag +# shape (vX.Y.Z), so a forged tag can never collide with a real one +# +# Env: +# HASH_LEN - hash length in hex chars (default 10, must match hash-paths.sh) + +set -eu -o pipefail + +HASH_LEN="${HASH_LEN:-10}" + +RESERVED_TAGS=(latest stable edge) + +if [ "$#" -ne 1 ]; then + echo "Usage: $0 " >&2 + exit 2 +fi + +TAG="$1" + +if ! [[ "$TAG" =~ ^[A-Za-z0-9_.-]+-[0-9a-f]{${HASH_LEN}}$ ]]; then + echo "validate-image-tag.sh: '${TAG}' does not match -<${HASH_LEN}-hex-char-hash>" >&2 + exit 1 +fi + +for reserved in "${RESERVED_TAGS[@]}"; do + if [ "$TAG" = "$reserved" ]; then + echo "validate-image-tag.sh: '${TAG}' is a reserved tag" >&2 + exit 1 + fi +done + +if [[ "$TAG" =~ ^v[0-9]+\.[0-9]+\.[0-9]+$ ]]; then + echo "validate-image-tag.sh: '${TAG}' looks like a release tag, not a content-hash tag" >&2 + exit 1 +fi + +exit 0 diff --git a/containers/validate_image_tag_test.sh b/containers/validate_image_tag_test.sh new file mode 100755 index 00000000000..021e9571154 --- /dev/null +++ b/containers/validate_image_tag_test.sh @@ -0,0 +1,60 @@ +#!/usr/bin/env bash +# validate_image_tag_test.sh - unit tests for validate-image-tag.sh. +# +# Pure string-format checks, no external stubs needed. +# Run with: +# containers/validate_image_tag_test.sh + +set -eu -o pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +VALIDATE="$SCRIPT_DIR/validate-image-tag.sh" + +FAILURES=0 + +fail() { + echo "FAIL: $1" >&2 + FAILURES=$((FAILURES + 1)) +} + +pass() { + echo "PASS: $1" +} + +assert_valid() { + local tag="$1" + if "$VALIDATE" "$tag" >/dev/null 2>&1; then + pass "accepts valid tag '$tag'" + else + fail "should have accepted valid tag '$tag'" + fi +} + +assert_invalid() { + local tag="$1" desc="$2" + if "$VALIDATE" "$tag" >/dev/null 2>&1; then + fail "should have rejected $desc ('$tag')" + else + pass "rejects $desc ('$tag')" + fi +} + +assert_valid "20260721_rfay_content_addressed_image_tags-36bceca65e" +assert_valid "main-0123456789" + +assert_invalid "latest" "the reserved literal 'latest'" +assert_invalid "stable" "the reserved literal 'stable'" +assert_invalid "v1.2.3" "a bare release tag" +assert_invalid "latest-0123456789a" "a fake tag with an 11-char hash suffix" +assert_invalid "latest-012345678" "a fake tag with a 9-char hash suffix" +assert_invalid "no-hash-suffix" "a tag without a hex hash suffix" +assert_invalid "bad chars!-0123456789" "a tag with disallowed characters" +assert_invalid "UPPERHASH-0123456789AB" "a tag with an uppercase hash suffix" + +if [ "$FAILURES" -eq 0 ]; then + echo "All validate_image_tag_test.sh checks passed." + exit 0 +else + echo "$FAILURES validate_image_tag_test.sh check(s) failed." >&2 + exit 1 +fi diff --git a/containers/wait-for-images.sh b/containers/wait-for-images.sh new file mode 100755 index 00000000000..892633134d3 --- /dev/null +++ b/containers/wait-for-images.sh @@ -0,0 +1,81 @@ +#!/usr/bin/env bash +# wait-for-images.sh +# +# Neither Buildkite nor the GitHub-hosted test-reusable.yml/ +# test-wsl2-reusable.yml runners hold image-push credentials, and none of +# them rebuild a changed image locally (autotag.sh's no-op fast path trusts +# the tag already committed in versionconstants.go, so a fresh runner with an +# empty Docker cache won't build it) - so any of them can race the +# image-push.yml GitHub Actions workflow: if this commit's containers/ +# changed, the image it needs might still be waiting on a maintainer's +# approval when this test run starts. Before running anything that pulls a +# DDEV image, poll the registry for the tags this checkout actually needs +# and wait for them to land. +# +# The tag is recomputed from real content (branch + hash-paths.sh), the same +# way image-build-push.yml's detect job does it - never read from +# versionconstants.go. That file's committed tag only has its hash kept +# current locally (autotag.sh skips rewriting the branch prefix when the hash +# hasn't changed), so it can carry a stale branch name from whatever branch +# last touched that image, while the registry holds the tag under *this* +# branch's name. Trusting the committed string would then wait forever for a +# tag nothing ever pushed. +# +# Fast path (the common case - nothing changed): one registry check per +# image, no wait. +# +# Env: +# WAIT_FOR_IMAGES_BRANCH - branch name to compute tags for (required) - +# pass the same value detect uses: for GitHub +# Actions that's head_ref || ref_name, for +# Buildkite it's $BUILDKITE_BRANCH. +# WAIT_FOR_IMAGES_ATTEMPTS - poll attempts before giving up (default 40) +# WAIT_FOR_IMAGES_SLEEP - seconds between attempts (default 30) +# +# Defaults give ~20 minutes - ddev-webserver alone takes ~6-8 minutes to build. + +set -eu -o pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +REPO_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)" +REGISTRY_TAG_EXISTS="$REPO_ROOT/containers/registry-tag-exists.sh" +HASH_PATHS_SH="$REPO_ROOT/containers/hash-paths.sh" +DOCKER_ORG="${DOCKER_ORG:-ddev}" + +ATTEMPTS="${WAIT_FOR_IMAGES_ATTEMPTS:-40}" +SLEEP_SECONDS="${WAIT_FOR_IMAGES_SLEEP:-30}" + +BRANCH="${WAIT_FOR_IMAGES_BRANCH:?wait-for-images.sh: WAIT_FOR_IMAGES_BRANCH must be set}" +SANITIZED_BRANCH="$(echo "$BRANCH" | sed -E 's/[^A-Za-z0-9_.-]+/-/g')" + +# repo_suffix|hash paths - keep in sync with image-build-push.yml's detect +# job and the Makefile's autotag-images target. +CONFIGS=( + 'ddev-webserver|containers/ddev-webserver containers/containers_shared.mk' + 'ddev-traefik-router|containers/ddev-traefik-router containers/containers_shared.mk' + 'ddev-ssh-agent|containers/ddev-ssh-agent containers/containers_shared.mk' + 'ddev-xhgui|containers/ddev-xhgui containers/containers_shared.mk' + 'ddev-dbserver-mariadb-11.8|containers/ddev-dbserver containers/get_arch.sh' +) + +for entry in "${CONFIGS[@]}"; do + IFS='|' read -r repo_suffix hash_paths <<< "$entry" + hash="$("$HASH_PATHS_SH" $hash_paths)" + tag="${SANITIZED_BRANCH}-${hash}" + image_repo="${DOCKER_ORG}/${repo_suffix}" + + attempt=1 + while true; do + if "$REGISTRY_TAG_EXISTS" "$image_repo" "$tag"; then + break + fi + if [ "$attempt" -ge "$ATTEMPTS" ]; then + echo "wait-for-images.sh: gave up waiting for ${image_repo}:${tag} after ${ATTEMPTS} attempts." >&2 + echo "wait-for-images.sh: has the maintainer approved the image-push run for this PR yet?" >&2 + exit 1 + fi + echo "wait-for-images.sh: ${image_repo}:${tag} not yet available, waiting... (attempt ${attempt}/${ATTEMPTS})" + sleep "$SLEEP_SECONDS" + attempt=$((attempt + 1)) + done +done diff --git a/containers/wait_for_images_test.sh b/containers/wait_for_images_test.sh new file mode 100755 index 00000000000..6737aa73f60 --- /dev/null +++ b/containers/wait_for_images_test.sh @@ -0,0 +1,177 @@ +#!/usr/bin/env bash +# wait_for_images_test.sh - unit tests for wait-for-images.sh. +# +# Exercises the fast-path/retry/give-up logic against a stubbed `docker` and +# the real hash-paths.sh (run against this checkout's actual content, so the +# expected tags are computed the same way wait-for-images.sh computes them - +# never read from versionconstants.go). No real registry or real sleeps. +# Run with: +# containers/wait_for_images_test.sh + +set -eu -o pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +WAIT_FOR_IMAGES="$SCRIPT_DIR/wait-for-images.sh" +HASH_PATHS="$SCRIPT_DIR/hash-paths.sh" + +FAILURES=0 + +fail() { + echo "FAIL: $1" >&2 + FAILURES=$((FAILURES + 1)) +} + +pass() { + echo "PASS: $1" +} + +assert_eq() { + local expected="$1" actual="$2" desc="$3" + if [ "$expected" = "$actual" ]; then + pass "$desc" + else + fail "$desc (expected '$expected', got '$actual')" + fi +} + +WORKDIR="$(mktemp -d)" +trap 'rm -rf "$WORKDIR"' EXIT + +# --- Stub `docker`: exists-by-default, except a ref can be configured to +# only start "existing" after N calls (via a per-ref counter file), so the +# eventually-recovers scenario is deterministic - no real sleeps or +# background processes needed. +BINDIR="$WORKDIR/bin" +mkdir -p "$BINDIR" +export DOCKER_EXISTING_REF_FILE="$WORKDIR/docker_existing_refs" +export DOCKER_DELAYED_REF_FILE="$WORKDIR/docker_delayed_ref" +export DOCKER_DELAYED_COUNTER_DIR="$WORKDIR/docker_delayed_counters" +export DOCKER_CALL_LOG="$WORKDIR/docker_calls.log" +mkdir -p "$DOCKER_DELAYED_COUNTER_DIR" +: > "$DOCKER_EXISTING_REF_FILE" +: > "$DOCKER_DELAYED_REF_FILE" +: > "$DOCKER_CALL_LOG" +cat > "$BINDIR/docker" <<'DOCKEREOF' +#!/usr/bin/env bash +set -eu -o pipefail +echo "$*" >> "$DOCKER_CALL_LOG" +if [ "$1" = "buildx" ] && [ "$2" = "imagetools" ] && [ "$3" = "inspect" ]; then + ref="$4" + if grep -qxF "$ref" "$DOCKER_EXISTING_REF_FILE"; then + exit 0 + fi + delayed_ref="$(cat "$DOCKER_DELAYED_REF_FILE" 2>/dev/null || true)" + if [ -n "$delayed_ref" ] && [ "$ref" = "$delayed_ref" ]; then + counter_file="$DOCKER_DELAYED_COUNTER_DIR/count" + count="$(cat "$counter_file" 2>/dev/null || echo 0)" + count=$((count + 1)) + echo "$count" > "$counter_file" + [ "$count" -ge 3 ] && exit 0 || exit 1 + fi + exit 1 +fi +echo "docker stub: unexpected invocation: $*" >&2 +exit 1 +DOCKEREOF +chmod +x "$BINDIR/docker" + +# --- Stub `sleep` so retry-budget tests run instantly and we can count waits. +export SLEEP_CALL_LOG="$WORKDIR/sleep_calls.log" +: > "$SLEEP_CALL_LOG" +cat > "$BINDIR/sleep" <<'SLEEPEOF' +#!/usr/bin/env bash +echo "$*" >> "$SLEEP_CALL_LOG" +SLEEPEOF +chmod +x "$BINDIR/sleep" + +export PATH="$BINDIR:$PATH" + +export DOCKER_ORG=ddevhq +BRANCH="test-branch" +export WAIT_FOR_IMAGES_BRANCH="$BRANCH" + +# Same repo_suffix|hash-paths list wait-for-images.sh uses - real hashes of +# this checkout's actual content, computed the same way the script does. +CONFIGS=( + 'ddev-webserver|containers/ddev-webserver containers/containers_shared.mk' + 'ddev-traefik-router|containers/ddev-traefik-router containers/containers_shared.mk' + 'ddev-ssh-agent|containers/ddev-ssh-agent containers/containers_shared.mk' + 'ddev-xhgui|containers/ddev-xhgui containers/containers_shared.mk' + 'ddev-dbserver-mariadb-11.8|containers/ddev-dbserver containers/get_arch.sh' +) +REPOS=() +TAGS=() +for entry in "${CONFIGS[@]}"; do + IFS='|' read -r repo_suffix hash_paths <<< "$entry" + hash="$("$HASH_PATHS" $hash_paths)" + REPOS+=("ddevhq/${repo_suffix}") + TAGS+=("${BRANCH}-${hash}") +done + +# 1. Fast path: every tag already exists -> one docker call per image, no sleep. +: > "$DOCKER_EXISTING_REF_FILE" +for i in "${!REPOS[@]}"; do + echo "${REPOS[$i]}:${TAGS[$i]}" >> "$DOCKER_EXISTING_REF_FILE" +done +: > "$DOCKER_CALL_LOG" +: > "$SLEEP_CALL_LOG" +if "$WAIT_FOR_IMAGES" >/dev/null 2>&1; then + pass "fast path succeeds when every tag already exists" +else + fail "fast path should succeed when every tag already exists" +fi +assert_eq "5" "$(wc -l < "$DOCKER_CALL_LOG")" "fast path makes exactly one docker call per image" +assert_eq "0" "$(wc -l < "$SLEEP_CALL_LOG")" "fast path never sleeps" + +# 2. A tag that's initially missing but becomes available on the 3rd check. +: > "$DOCKER_EXISTING_REF_FILE" +for i in "${!REPOS[@]}"; do + [ "$i" -eq 4 ] && continue + echo "${REPOS[$i]}:${TAGS[$i]}" >> "$DOCKER_EXISTING_REF_FILE" +done +echo "${REPOS[4]}:${TAGS[4]}" > "$DOCKER_DELAYED_REF_FILE" +rm -f "$DOCKER_DELAYED_COUNTER_DIR/count" +: > "$SLEEP_CALL_LOG" +if WAIT_FOR_IMAGES_ATTEMPTS=5 WAIT_FOR_IMAGES_SLEEP=0 "$WAIT_FOR_IMAGES" >/dev/null 2>&1; then + pass "recovers once a previously-missing tag appears within the attempt budget" +else + fail "should recover once a previously-missing tag appears within the attempt budget" +fi +assert_eq "2" "$(wc -l < "$SLEEP_CALL_LOG")" "sleeps twice while waiting for the tag to become available on the 3rd check" +: > "$DOCKER_DELAYED_REF_FILE" + +# 3. Gives up cleanly after exhausting the attempt budget, with a clear message. +: > "$DOCKER_EXISTING_REF_FILE" +: > "$SLEEP_CALL_LOG" +OUTPUT="$(WAIT_FOR_IMAGES_ATTEMPTS=3 WAIT_FOR_IMAGES_SLEEP=0 "$WAIT_FOR_IMAGES" 2>&1)" && RC=0 || RC=$? +if [ "$RC" -ne 0 ]; then + pass "gives up (non-zero exit) once the attempt budget is exhausted" +else + fail "should give up (non-zero exit) once the attempt budget is exhausted" +fi +case "$OUTPUT" in + *"gave up waiting"*"has the maintainer approved"*) pass "give-up message is actionable" ;; + *) fail "give-up message should mention giving up and approval: $OUTPUT" ;; +esac +assert_eq "2" "$(wc -l < "$SLEEP_CALL_LOG")" "sleeps exactly (attempts - 1) times before giving up on the first (unavailable) image" + +# 4. WAIT_FOR_IMAGES_BRANCH is required - a clear, immediate error when unset. +: > "$DOCKER_EXISTING_REF_FILE" +OUTPUT="$(env -u WAIT_FOR_IMAGES_BRANCH "$WAIT_FOR_IMAGES" 2>&1)" && RC=0 || RC=$? +if [ "$RC" -ne 0 ]; then + pass "errors out when WAIT_FOR_IMAGES_BRANCH is unset" +else + fail "should error out when WAIT_FOR_IMAGES_BRANCH is unset" +fi +case "$OUTPUT" in + *"WAIT_FOR_IMAGES_BRANCH must be set"*) pass "missing-branch message names the required variable" ;; + *) fail "missing-branch message should name WAIT_FOR_IMAGES_BRANCH: $OUTPUT" ;; +esac + +if [ "$FAILURES" -eq 0 ]; then + echo "All wait_for_images_test.sh checks passed." + exit 0 +else + echo "$FAILURES wait_for_images_test.sh check(s) failed." >&2 + exit 1 +fi diff --git a/docs/content/developers/building-contributing.md b/docs/content/developers/building-contributing.md index 7ac7670ece0..f213b3ac84d 100644 --- a/docs/content/developers/building-contributing.md +++ b/docs/content/developers/building-contributing.md @@ -155,6 +155,8 @@ make push VERSION= DOCKER_REPO=your/dockerrepo ### Pushes Using GitHub Actions +The [Image build](https://github.com/ddev/ddev/actions/workflows/image-build-push.yml) workflow (see [Automatic Image Build and Push](#automatic-image-build-and-push)) handles pushing a changed image automatically for any pull request, including forks. The workflows below are for manually pushing a specific tag — a re-push, or one of the `ddev-dbserver` variants other than the default `mariadb_11.8` that `make` auto-builds. + To manually push using GitHub Actions, #### For Most Images @@ -308,7 +310,18 @@ The Docker images that DDEV uses are included in the `containers/` directory: * `containers/ddev-traefik-router` is the current Traefik-based router image. * `containers/ddev-xhgui` provides a web interface to analyze performance profiles generated by xhprof. -When you change an image, running `make` from the repository root builds it locally and computes/updates its tag in `pkg/versionconstants/versionconstants.go` automatically — no manual tag-inventing or file-editing needed, and this works for any contributor, including from a fork. Getting that image into CI (a multi-arch push to the registry) still requires registry credentials that forks don't have, so please ask a maintainer if you need a container pushed to support a pull request. +When you change an image, running `make` from the repository root builds it locally and computes/updates its tag in `pkg/versionconstants/versionconstants.go` automatically — no manual tag-inventing or file-editing needed, and this works for any contributor, including from a fork. Once you push that commit as a pull request, the [Image build](https://github.com/ddev/ddev/actions/workflows/image-build-push.yml) workflow detects the changed image and builds and pushes it to the registry automatically — no maintainer needs to run anything by hand. See [Automatic Image Build and Push](#automatic-image-build-and-push) below for when that requires a maintainer's approval and when it doesn't. + +### Automatic Image Build and Push + +Opening a pull request that touches `containers/` triggers the [Image build](https://github.com/ddev/ddev/actions/workflows/image-build-push.yml) workflow. A `detect` job always runs first: it computes the real content hash of each changed image and checks whether that tag already exists in the registry — this never trusts the tag string committed in `versionconstants.go`, so it works the same way whether or not you ran `make` locally first. + +What happens next depends on whether the PR is from a fork: + +* **Fork PRs** (a real security boundary — the PR could contain an arbitrary Dockerfile/build script): a `build` job builds the image(s) per architecture with no registry credentials at all — nothing in that job can reach `docker.io`, so there's nothing to gain by gating it before it runs. Once it finishes, a separate, trusted `image-push.yml` workflow — which never checks out or runs the pull request's code — loads what it produced and pushes it, gated behind a maintainer's approval on the `image-push` environment. A comment is posted on the PR once the push completes. +* **Everything else** (a push to `main`, or a pull request from a branch in the same repository — no fork content is ever involved): `build-and-push` builds and pushes directly in one step, with no approval gate at all — the same trust level `main-build.yml` already runs at unguarded. A `create-manifests` job then assembles the multi-arch manifest and comments on the PR, if there is one. + +So a maintainer only ever needs to click **Approve** once — for a fork PR's push step — and only when the PR actually changed a container image; everything else is fully automatic. ## Pull Requests diff --git a/docs/content/developers/release-management.md b/docs/content/developers/release-management.md index 5d08677508a..a06e610970a 100644 --- a/docs/content/developers/release-management.md +++ b/docs/content/developers/release-management.md @@ -74,9 +74,29 @@ The following “Repository secret” environment variables must be configured i 2. Make sure you're about to create the right release tag. 3. Use the “Auto-generate release notes” option to get the commit list, then edit to add all the other necessary info. +## Automatic Image Build and Push + +Any pull request that changes `containers/` — including from a fork — is built and pushed automatically by the [Image build](https://github.com/ddev/ddev/blob/main/.github/workflows/image-build-push.yml) / [Image push](https://github.com/ddev/ddev/blob/main/.github/workflows/image-push.yml) workflow pair. See [Automatic Image Build and Push](building-contributing.md#automatic-image-build-and-push) in the contributing guide for how the flow works and why it's safe to run on fork-authored Dockerfiles. + +The two workflows below (manual `workflow_dispatch`) remain for re-pushing a specific tag and for `ddev-dbserver` variants other than the default `mariadb_11.8` that the automatic flow doesn't build. + +### One-time setup: the `image-push` GitHub Environment + +Fork PRs build with no registry credentials at all (nothing to gain by gating that step), then go through a single approval before the built image is actually pushed, gated by the `image-push` GitHub Environment (Settings → Environments): + +1. Create the environment `image-push`. +2. Add required reviewers (the maintainers/dev team). +3. Add `PUSH_SERVICE_ACCOUNT_TOKEN` as a secret **on this environment** (Settings → Environments → `image-push` → Secrets), using the same 1Password service-account token value already used elsewhere in this doc. It currently exists only as a repository secret; duplicating (or moving) it onto the `image-push` environment is what scopes `DOCKERHUB_TOKEN` access to only the approved `image-push.yml` job. + +This approval only applies to fork PRs. A push to `main` or a same-repo PR builds and pushes without any approval at all, using the repository-level `PUSH_SERVICE_ACCOUNT_TOKEN` secret directly (that path never declares `environment:` on its jobs, so this environment's protection rules don't apply to it). + +When testing this on `ddev-test/ddev`, do the same steps there first, and confirm `vars.DOCKER_ORG` on that repository points at the DockerHub org used for testing. + +Since a job referencing an environment that doesn't exist yet gets auto-created with no protection rules (silently *not* gating), verify the environment actually has a `required_reviewers` rule before relying on it, e.g. `gh api repos///environments/image-push`. + ## Pushing Docker Images with the GitHub Actions Workflow -The easiest way to push Docker images is to use the GitHub Actions workflow, especially if the code for the image is already in the [ddev/ddev](https://github.com/ddev/ddev) repository. +The easiest way to push Docker images is to use the GitHub Actions workflow, especially if the code for the image is already in the [ddev/ddev](https://github.com/ddev/ddev) repository. For a normal container change on a pull request, you shouldn't need this — see [Automatic Image Build and Push](#automatic-image-build-and-push) above. ### Actual release creation @@ -88,7 +108,7 @@ You can push all images besides `ddev-dbserver` at -If you need to push from a forked PR, you’ll have to do this from your fork (for example, `https://github.com/rfay/ddev/actions/workflows/push-tagged-image.yml`), and you’ll have to specify the branch on the fork. This requires setting the `DOCKERHUB_TOKEN` and `DOCKERHUB_USERNAME` secrets on the forked PR, for example `https://github.com/rfay/ddev/settings/secrets/actions`. You can do the same with `ddev-dbserver` at `https://github.com/rfay/ddev/actions/workflows/push-tagged-dbimage.yml` for example. +A forked PR that changes a container image no longer needs any of this — see [Automatic Image Build and Push](#automatic-image-build-and-push) above. The fork-your-own-secrets workaround described in earlier versions of this doc is superseded by that flow. * Visit `https://github.com/ddev/ddev/actions/workflows/push-tagged-image.yml`. * Click the “Push tagged image” workflow on the left side of the page. diff --git a/pkg/ddevapp/utils.go b/pkg/ddevapp/utils.go index 59249d78013..5ae29661721 100644 --- a/pkg/ddevapp/utils.go +++ b/pkg/ddevapp/utils.go @@ -119,11 +119,11 @@ func Cleanup(app *DdevApp) error { Profiles: []string{`*`}, }) if loadErr != nil { - util.Warning("Failed to load compose project for down: %v", loadErr) + util.Warning("Failed to load compose project for down just dumyy: %v", loadErr) } else { downCtx, svc, svcErr := dockerutil.NewComposeService() if svcErr != nil { - util.Warning("Failed to create compose service: %v", svcErr) + util.Warning("Failed to create compose service but I really tried: %v", svcErr) } else if downErr := svc.Down(downCtx, project.Name, api.DownOptions{Project: project, RemoveOrphans: true}); downErr != nil { util.Warning("Failed to docker-compose down: %v", downErr) } diff --git a/pkg/versionconstants/versionconstants.go b/pkg/versionconstants/versionconstants.go index 69f3d6d4869..b7d7a89fb8f 100644 --- a/pkg/versionconstants/versionconstants.go +++ b/pkg/versionconstants/versionconstants.go @@ -20,7 +20,7 @@ var AmplitudeAPIKey = "" var WebImg = "ddev/ddev-webserver" // WebTag defines the default web image tag -var WebTag = "20260721_rfay_content_addressed_image_tags-36bceca65e" // Note that this can be overridden by make +var WebTag = "20260814_rfay_test_phase2_xhgui_change-af66a92c42" // Note that this can be overridden by make // DBImg defines the default db image used for applications. var DBImg = "ddev/ddev-dbserver" @@ -44,7 +44,7 @@ var SSHAuthTag = "20260721_rfay_content_addressed_image_tags-8e8bf1217c" var XhguiImage = "ddev/ddev-xhgui" // XhguiTag is xhgui tag -var XhguiTag = "20260721_rfay_content_addressed_image_tags-f046b66382" +var XhguiTag = "20260814_rfay_test_phase2_xhgui_change-feb879f19d" // UtilitiesImage is used in bash scripts var UtilitiesImage = "ddev/ddev-utilities:latest"