diff --git a/.buildkite/perf.sh b/.buildkite/perf.sh index b0c1da3b179..d624a13f463 100755 --- a/.buildkite/perf.sh +++ b/.buildkite/perf.sh @@ -19,6 +19,9 @@ if [[ ${BUILDKITE_MESSAGE:-} == *"[skip buildkite]"* ]] || [[ ${BUILDKITE_MESSAG exit 0 fi +# A changed image may still be waiting on image-push.yml's approval +"$(dirname "$0")/../containers/wait-for-images.sh" + os=$(go env GOOS) # shellcheck source=lib-provider.sh diff --git a/.buildkite/test.sh b/.buildkite/test.sh index 8e024776527..3adbc442073 100755 --- a/.buildkite/test.sh +++ b/.buildkite/test.sh @@ -25,6 +25,9 @@ while IFS= read -r varname; do done < <(MSYS_NO_PATHCONV=1 git ls-tree --name-only refs/public-variables-tmp:.github/public-variables/) git update-ref -d refs/public-variables-tmp +# A changed image may still be waiting on image-push.yml's approval +"$(dirname "$0")/../containers/wait-for-images.sh" + export PATH=$PATH:/home/linuxbrew/.linuxbrew/bin os=$(go env GOOS) diff --git a/.github/workflows/container-tests.yml b/.github/workflows/container-tests.yml index 4b2cbe866a4..88a78074717 100644 --- a/.github/workflows/container-tests.yml +++ b/.github/workflows/container-tests.yml @@ -47,6 +47,18 @@ jobs: - uses: actions/checkout@v7 - name: Run containers/autotag_test.sh run: containers/autotag_test.sh + - name: Run containers/db_variants_test.sh + run: containers/db_variants_test.sh + - name: Run containers/required_image_tag_test.sh + run: containers/required_image_tag_test.sh + - name: Run containers/registry_tag_exists_test.sh + run: containers/registry_tag_exists_test.sh + - name: Run containers/validate_image_tag_test.sh + run: containers/validate_image_tag_test.sh + - name: Run containers/validate_image_repo_test.sh + run: containers/validate_image_repo_test.sh + - name: Run containers/wait_for_images_test.sh + run: containers/wait_for_images_test.sh container-build-and-test: name: ${{ matrix.os }} - Test container ${{ matrix.containers }} diff --git a/.github/workflows/image-build-push.yml b/.github/workflows/image-build-push.yml index 2ddc72090ea..f8248013e81 100644 --- a/.github/workflows/image-build-push.yml +++ b/.github/workflows/image-build-push.yml @@ -3,13 +3,18 @@ defaults: run: shell: bash -# Placeholder for #8609 phase 2 (fork-safe automatic image build/push). This -# do-nothing stub exists only so the workflow name/triggers are registered on -# the default branch before the real detect/approval/build logic lands - see -# the phase 2 PR, which will replace this file's contents via rebase. -# image-push.yml's `workflow_run` trigger only fires for a listener that -# exists on the default branch, so this stub unblocks testing that dependency -# ahead of the full PR. +# In a forked PR, this workflow may run a fork's own Dockerfile/build scripts, +# so the `build` job never references a secret - the trusted side (loading +# the artifact `build` produces and actually pushing it) lives in +# image-push.yml, triggered via workflow_run once this workflow completes, +# gated behind its own approval. +# +# For anything else (a push, or a same-repo PR - `detect`'s `is_fork` output, +# using the same fork-check idiom as push-tagged-image.yml), there's nothing +# a maintainer's own branch could smuggle into a job that every other +# secret-using workflow here doesn't already run unguarded, so +# `build-and-push` builds and pushes directly, with no environment/approval +# gate at all - same trust level as `main-build.yml`. on: pull_request: branches: [main] @@ -22,14 +27,305 @@ on: - "containers/**" - ".github/workflows/image-build-push.yml" +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +env: + DOCKER_ORG: "${{ vars.DOCKER_ORG || 'ddev' }}" + permissions: contents: read jobs: - placeholder: - name: "Placeholder (see #8609)" + detect: + name: Detect changed images + runs-on: ubuntu-24.04 + outputs: + build_matrix: ${{ steps.detect.outputs.build_matrix }} + manifest_matrix: ${{ steps.detect.outputs.manifest_matrix }} + needs_build: ${{ steps.detect.outputs.needs_build }} + is_fork: ${{ steps.fork.outputs.is_fork }} + steps: + - uses: actions/checkout@v7 + - name: Compute per-image build status + id: detect + # github.head_ref is attacker-controlled (a fork may name its branch + # anything git accepts, quotes and backticks included), so it reaches + # the script as data rather than as spliced-in source. + env: + REQUIRED_IMAGE_TAG_BRANCH: ${{ github.head_ref || github.ref_name }} + run: | + set -eu -o pipefail + source containers/image-configs.sh + + # build_matrix is one entry per (image, arch) rather than a cross + # product, because the older db variants are amd64-only. + BUILD_JSON="[]" + MANIFEST_JSON="[]" + for entry in "${DDEV_IMAGE_CONFIGS[@]}"; do + IFS='|' read -r repo_suffix tag_var hash_paths make_dir make_target arch_suffixed arches _ extra_repo_suffixes <<< "$entry" + # shellcheck disable=SC2086 # hash_paths is a space-separated path list + read -r state tag <<< "$(containers/required-image-tag.sh "$tag_var" $hash_paths)" + repo="${DOCKER_ORG}/${repo_suffix}" + if containers/registry-tag-exists.sh "$repo" "$tag"; then + echo "detect: ${repo}:${tag} already exists (${state}), nothing to build" + continue + fi + echo "detect: ${repo}:${tag} needs building (${state}) for arches: ${arches}" + for arch in $arches; do + BUILD_JSON="$(echo "$BUILD_JSON" | jq -c \ + --arg repo_suffix "$repo_suffix" \ + --arg repo "$repo" \ + --arg tag "$tag" \ + --arg arch "$arch" \ + --arg make_dir "$make_dir" \ + --arg make_target "$make_target" \ + --arg arch_suffixed "$arch_suffixed" \ + --arg extra_repo_suffixes "$extra_repo_suffixes" \ + '. + [{"repo_suffix": $repo_suffix, "repo": $repo, "tag": $tag, "arch": $arch, "make_dir": $make_dir, "make_target": $make_target, "arch_suffixed": $arch_suffixed, "extra_repo_suffixes": $extra_repo_suffixes}]')" + done + MANIFEST_JSON="$(echo "$MANIFEST_JSON" | jq -c \ + --arg repo "$repo" \ + --arg tag "$tag" \ + --arg arches "$arches" \ + --arg extra_repo_suffixes "$extra_repo_suffixes" \ + '. + [{"repo": $repo, "tag": $tag, "arches": $arches, "extra_repo_suffixes": $extra_repo_suffixes}]')" + done + + echo "build_matrix=${BUILD_JSON}" >> "$GITHUB_OUTPUT" + echo "manifest_matrix=${MANIFEST_JSON}" >> "$GITHUB_OUTPUT" + echo "detect: $(echo "$MANIFEST_JSON" | jq 'length') image(s) to build, $(echo "$BUILD_JSON" | jq 'length') build job(s)" + if [ "$(echo "$BUILD_JSON" | jq 'length')" -gt 0 ]; then + echo "needs_build=true" >> "$GITHUB_OUTPUT" + else + echo "needs_build=false" >> "$GITHUB_OUTPUT" + fi + + # Separate from the step above so nothing the per-image loop does can + # reach the output that decides whether the push secret gets loaded. + - name: Determine whether this is a fork + id: fork + env: + EVENT_NAME: ${{ github.event_name }} + HEAD_OWNER: ${{ github.event.pull_request.head.repo.owner.login }} + BASE_OWNER: ${{ github.repository_owner }} + run: | + set -eu -o pipefail + if [ "$EVENT_NAME" = "pull_request" ] && [ "$HEAD_OWNER" != "$BASE_OWNER" ]; then + echo "is_fork=true" >> "$GITHUB_OUTPUT" + else + echo "is_fork=false" >> "$GITHUB_OUTPUT" + fi + + # --- Fork PRs: build with no secrets (this job never has registry + # credentials, so there's nothing to gain by gating it - see #8609 + # discussion), then hand off to image-push.yml for the trusted, + # approval-gated push. --- + + build: + name: Build ${{ matrix.build.repo }} (${{ matrix.build.arch }}) + needs: detect + if: needs.detect.outputs.needs_build == 'true' && needs.detect.outputs.is_fork == 'true' + strategy: + fail-fast: false + matrix: + build: ${{ fromJson(needs.detect.outputs.build_matrix) }} + runs-on: ${{ matrix.build.arch == 'arm64' && 'ubuntu-24.04-arm' || 'ubuntu-24.04' }} + permissions: + contents: read + steps: + - uses: actions/checkout@v7 + - name: Build ${{ matrix.build.repo }}:${{ matrix.build.tag }}-${{ matrix.build.arch }} + run: | + set -eu -o pipefail + VERSION="${{ matrix.build.tag }}-${{ matrix.build.arch }}" + MAKE_TARGET="${{ matrix.build.make_target }}" + if [ "${{ matrix.build.arch_suffixed }}" = "true" ]; then + MAKE_TARGET="${MAKE_TARGET}_${{ matrix.build.arch }}" + fi + # DDEV_IMAGE_TAG is the tag people actually pull, recorded in the + # com.ddev.image-tag label; VERSION is the throwaway per-arch tag. + make -C "containers/${{ matrix.build.make_dir }}" "$MAKE_TARGET" \ + VERSION="$VERSION" DDEV_IMAGE_TAG="${{ matrix.build.tag }}" + + REPOS="${{ matrix.build.repo }}" + for suffix in ${{ matrix.build.extra_repo_suffixes }}; do + REPOS="${REPOS} ${DOCKER_ORG}/${suffix}" + done + + REFS=() + : > repos.txt + for repo in $REPOS; do + REFS+=("${repo}:${VERSION}") + echo "$repo" >> repos.txt + done + docker save "${REFS[@]}" -o image.tar + echo -n "${{ matrix.build.tag }}" > tag.txt + echo -n "${{ matrix.build.arch }}" > arch.txt + - uses: actions/upload-artifact@v7 + with: + # repo_suffix, not make_dir: all 20 db variants share one make_dir. + name: image-${{ matrix.build.repo_suffix }}-${{ matrix.build.arch }} + path: | + image.tar + repos.txt + tag.txt + arch.txt + # image-push.yml can't download an expired artifact, and the approval + # it waits on is a human one that may not come the same day. + retention-days: 7 + + # --- Pushes and same-repo PRs: no fork content ever runs here, so build + # and push directly with no environment/approval gate - same trust level + # as main-build.yml, which already uses this same secret unguarded. --- + + build-and-push: + name: Build and push ${{ matrix.build.repo }} (${{ matrix.build.arch }}) + needs: detect + if: needs.detect.outputs.needs_build == 'true' && needs.detect.outputs.is_fork == 'false' + strategy: + fail-fast: false + matrix: + build: ${{ fromJson(needs.detect.outputs.build_matrix) }} + runs-on: ${{ matrix.build.arch == 'arm64' && 'ubuntu-24.04-arm' || 'ubuntu-24.04' }} + steps: + - uses: actions/checkout@v7 + + - name: Load 1password secret(s) + uses: 1password/load-secrets-action@v4 + with: + export-env: true + env: + OP_SERVICE_ACCOUNT_TOKEN: "${{ secrets.PUSH_SERVICE_ACCOUNT_TOKEN }}" + DOCKERHUB_TOKEN: "op://push-secrets/DOCKERHUB_TOKEN/credential" + + - name: Login to DockerHub + uses: docker/login-action@v4 + with: + username: ${{ vars.DOCKERHUB_USERNAME }} + password: ${{ env.DOCKERHUB_TOKEN }} + + - name: Build and push ${{ matrix.build.repo }}:${{ matrix.build.tag }}-${{ matrix.build.arch }} + run: | + set -eu -o pipefail + VERSION="${{ matrix.build.tag }}-${{ matrix.build.arch }}" + MAKE_TARGET="${{ matrix.build.make_target }}" + if [ "${{ matrix.build.arch_suffixed }}" = "true" ]; then + MAKE_TARGET="${MAKE_TARGET}_${{ matrix.build.arch }}" + fi + # See the DDEV_IMAGE_TAG note in the fork-side `build` job above. + make -C "containers/${{ matrix.build.make_dir }}" "$MAKE_TARGET" \ + VERSION="$VERSION" DDEV_IMAGE_TAG="${{ matrix.build.tag }}" + + REPOS="${{ matrix.build.repo }}" + for suffix in ${{ matrix.build.extra_repo_suffixes }}; do + REPOS="${REPOS} ${DOCKER_ORG}/${suffix}" + done + for repo in $REPOS; do + docker push "${repo}:${VERSION}" + done + + create-manifests: + name: Create manifest for ${{ matrix.manifest.repo }} + needs: [detect, build-and-push] + if: needs.detect.outputs.needs_build == 'true' && needs.detect.outputs.is_fork == 'false' + strategy: + fail-fast: false + matrix: + manifest: ${{ fromJson(needs.detect.outputs.manifest_matrix) }} runs-on: ubuntu-24.04 + permissions: + contents: read + pull-requests: write steps: - - name: Do nothing + - uses: actions/checkout@v7 + + - name: Load 1password secret(s) + uses: 1password/load-secrets-action@v4 + with: + export-env: true + env: + OP_SERVICE_ACCOUNT_TOKEN: "${{ secrets.PUSH_SERVICE_ACCOUNT_TOKEN }}" + DOCKERHUB_TOKEN: "op://push-secrets/DOCKERHUB_TOKEN/credential" + + - name: Login to DockerHub + uses: docker/login-action@v4 + with: + username: ${{ vars.DOCKERHUB_USERNAME }} + password: ${{ env.DOCKERHUB_TOKEN }} + + - name: Create manifest and clean up per-arch tags + id: manifest + env: + # Trusted context, not the matrix: this is the readable alias + # published next to the hash tag, so it must not be fork-controlled + # beyond the branch name itself, which is sanitized below. + ALIAS_BRANCH: ${{ github.head_ref || github.ref_name }} run: | - echo "Placeholder for #8609 phase 2 - no-op until the real workflow lands." + set -eu -o pipefail + TAG="${{ matrix.manifest.tag }}" + # The oldest db variants are amd64-only, so the arch list comes from + # detect rather than being assumed to be both. + ARCHES="${{ matrix.manifest.arches }}" + ALIAS="$(echo "$ALIAS_BRANCH" | sed -E 's/[^A-Za-z0-9_.-]+/-/g')-${TAG}" + if ! containers/validate-image-tag.sh "$ALIAS"; then + echo "Alias '${ALIAS}' rejected; publishing only ${TAG}" >&2 + ALIAS="" + fi + # Descriptive metadata on the index the tag points at - the standard + # equivalent of a comment on a tag. See containers/image-metadata.sh. + read -r -a ANNOTATIONS <<< "$(containers/image-metadata.sh annotations "$TAG")" + + DOCKERHUB_JWT="$(curl -s -H "Content-Type: application/json" -X POST \ + -d '{"username":"'"${{ vars.DOCKERHUB_USERNAME }}"'","password":"'"${DOCKERHUB_TOKEN}"'"}' \ + https://hub.docker.com/v2/users/login/ | jq -r .token)" + + REPOS="${{ matrix.manifest.repo }}" + for suffix in ${{ matrix.manifest.extra_repo_suffixes }}; do + REPOS="${REPOS} ${DOCKER_ORG}/${suffix}" + done + + PUSHED_SUMMARY="" + for repo in $REPOS; do + ARCH_TAGS=() + for arch in $ARCHES; do + ARCH_TAGS+=("${repo}:${TAG}-${arch}") + done + NAMES=(-t "${repo}:${TAG}") + [ -n "$ALIAS" ] && NAMES+=(-t "${repo}:${ALIAS}") + docker buildx imagetools create "${ANNOTATIONS[@]}" "${NAMES[@]}" "${ARCH_TAGS[@]}" + # imagetools inherits the source media type and a Docker manifest + # list has nowhere to put annotations, so say when they were + # dropped rather than leaving it to be discovered later. + if ! docker buildx imagetools inspect --raw "${repo}:${TAG}" | jq -e '.annotations' >/dev/null 2>&1; then + echo "Note: ${repo}:${TAG} is not an OCI index, so index annotations were not stored." >&2 + fi + PUSHED_SUMMARY="${PUSHED_SUMMARY}- \`${repo}:${TAG}\`"$'\n' + for arch in $ARCHES; do + echo "Removing intermediary tag ${repo}:${TAG}-${arch}" + curl -s -X DELETE -H "Authorization: JWT ${DOCKERHUB_JWT}" \ + "https://hub.docker.com/v2/repositories/${repo}/tags/${TAG}-${arch}/" >/dev/null || true + done + done + + { + echo "summary<> "$GITHUB_OUTPUT" + + - name: Comment on the pull request + if: github.event_name == 'pull_request' + uses: actions/github-script@v9 + env: + IMAGE_PUSH_SUMMARY: ${{ steps.manifest.outputs.summary }} + with: + script: | + const summary = (process.env.IMAGE_PUSH_SUMMARY || "").trim(); + await github.rest.issues.createComment({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: context.issue.number, + body: `Pushed updated container image(s) for this PR:\n\n${summary}`, + }); diff --git a/.github/workflows/image-push.yml b/.github/workflows/image-push.yml index a32d65e286b..65594b749eb 100644 --- a/.github/workflows/image-push.yml +++ b/.github/workflows/image-push.yml @@ -3,27 +3,221 @@ defaults: run: shell: bash -# Placeholder for #8609 phase 2 (fork-safe automatic image build/push). This -# do-nothing stub exists only so `workflow_run` below has a registered -# listener on the default branch before the real load/validate/push logic -# lands - see the phase 2 PR, which will replace this file's contents via -# rebase. `workflow_run` only fires for a listener that already exists on the -# default branch, so this stub unblocks testing that dependency ahead of the -# full PR. +# Trusted side of the fork-safe build/push split for #8609 phase 2. +# Triggered by completion of the "Image build" workflow, using the +# image-push.yml that lives on the default branch - never the fork's copy, +# and this workflow never checks out or executes the triggering PR's code. +# It only loads the artifact "Image build" produced (an inert tarball plus +# metadata) and pushes it, after re-validating the tag. on: workflow_run: workflows: ["Image build"] types: [completed] +env: + DOCKER_ORG: "${{ vars.DOCKER_ORG || 'ddev' }}" + permissions: contents: read jobs: - placeholder: - name: "Placeholder (see #8609)" - if: github.event.workflow_run.conclusion == 'success' + # Ungated on purpose: asking a maintainer to approve a push only to discover + # the run built nothing trains people to click Approve without looking. + check-artifacts: + name: Check for built images + # Only fork completions of "Image build" ever produce artifacts here - + # the non-fork path pushes directly in that workflow's build-and-push job. + if: >- + github.event.workflow_run.conclusion == 'success' && + github.event.workflow_run.head_repository.full_name != github.event.workflow_run.repository.full_name runs-on: ubuntu-24.04 + permissions: + actions: read + outputs: + has_artifacts: ${{ steps.check.outputs.has_artifacts }} steps: - - name: Do nothing + - name: List artifacts on the triggering run + id: check + uses: actions/github-script@v9 + with: + script: | + const artifacts = await github.paginate( + github.rest.actions.listWorkflowRunArtifacts, + { owner: context.repo.owner, repo: context.repo.repo, + run_id: context.payload.workflow_run.id }); + const images = artifacts.filter((a) => a.name.startsWith("image-")); + console.log(`Found ${images.length} image-* artifact(s): ${images.map((a) => a.name).join(", ")}`); + core.setOutput("has_artifacts", images.length > 0 ? "true" : "false"); + + push: + name: "Approve: push the built image(s) to DockerHub" + needs: check-artifacts + if: needs.check-artifacts.outputs.has_artifacts == 'true' + runs-on: ubuntu-24.04 + environment: image-push + permissions: + # download-artifact needs actions:read to reach another run's artifacts. + actions: read + contents: read + pull-requests: write + steps: + - uses: actions/checkout@v7 + + # No continue-on-error: check-artifacts already established that these + # exist, so a failure here is a real one (an expired artifact, say) and + # must not be reported as "nothing needed pushing". + - name: Download build artifacts + uses: actions/download-artifact@v8 + with: + github-token: ${{ secrets.GITHUB_TOKEN }} + run-id: ${{ github.event.workflow_run.id }} + pattern: image-* + path: artifacts + + - name: Load 1password secret(s) + uses: 1password/load-secrets-action@v4 + with: + export-env: true + env: + OP_SERVICE_ACCOUNT_TOKEN: "${{ secrets.PUSH_SERVICE_ACCOUNT_TOKEN }}" + DOCKERHUB_TOKEN: "op://push-secrets/DOCKERHUB_TOKEN/credential" + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v4 + + - name: Login to DockerHub + uses: docker/login-action@v4 + with: + username: ${{ vars.DOCKERHUB_USERNAME }} + password: ${{ env.DOCKERHUB_TOKEN }} + + - name: Validate, load, and push each built image + id: push + env: + # From the workflow_run payload, not the artifact - the alias is + # published in the DDEV org, so its branch name has to come from a + # trusted context. Sanitized and validated below regardless. + ALIAS_BRANCH: ${{ github.event.workflow_run.head_branch }} run: | - echo "Placeholder for #8609 phase 2 - no-op until the real workflow lands." + set -eu -o pipefail + SANITIZED_BRANCH="$(echo "$ALIAS_BRANCH" | sed -E 's/[^A-Za-z0-9_.-]+/-/g')" + # The revision is this workflow's own checkout (the default branch), + # not the fork's, so record the commit the artifact was built from. + export DDEV_GIT_REVISION="${{ github.event.workflow_run.head_sha }}" + declare -A TAG_BY_KEY + declare -A REPOS_BY_KEY + declare -A ARCHES_BY_KEY + + # Everything read out of artifacts/ was written by a job that may + # have run fork-authored content, so the tag and every repository + # name is validated before it reaches a `docker push`. + for dir in artifacts/image-*; do + [ -d "$dir" ] || continue + arch="$(cat "$dir/arch.txt")" + tag="$(cat "$dir/tag.txt")" + key="${dir%-"$arch"}" + + if ! containers/validate-image-tag.sh "$tag"; then + echo "image-push: refusing to push - invalid tag '$tag' from $dir" >&2 + exit 1 + fi + + while IFS= read -r repo; do + [ -z "$repo" ] && continue + if ! containers/validate-image-repo.sh "$repo"; then + echo "image-push: refusing to push - disallowed repository '$repo' from $dir" >&2 + exit 1 + fi + done < "$dir/repos.txt" + + docker load -i "$dir/image.tar" + + while IFS= read -r repo; do + [ -z "$repo" ] && continue + docker push "${repo}:${tag}-${arch}" + done < "$dir/repos.txt" + + TAG_BY_KEY["$key"]="$tag" + if [ -z "${REPOS_BY_KEY[$key]:-}" ]; then + REPOS_BY_KEY["$key"]="$(cat "$dir/repos.txt")" + fi + ARCHES_BY_KEY["$key"]="${ARCHES_BY_KEY[$key]:-} ${arch}" + done + + # Docker Hub token for deleting the intermediary per-arch tags below, + # same cleanup push-tagged-image.yml/push-tagged-dbimage.yml already do. + DOCKERHUB_JWT="$(curl -s -H "Content-Type: application/json" -X POST \ + -d '{"username":"'"${{ vars.DOCKERHUB_USERNAME }}"'","password":"'"${DOCKERHUB_TOKEN}"'"}' \ + https://hub.docker.com/v2/users/login/ | jq -r .token)" + + PUSHED_SUMMARY="" + for key in "${!TAG_BY_KEY[@]}"; do + tag="${TAG_BY_KEY[$key]}" + while IFS= read -r repo; do + [ -z "$repo" ] && continue + arch_tags=() + for arch in ${ARCHES_BY_KEY[$key]}; do + arch_tags+=("${repo}:${tag}-${arch}") + done + names=(-t "${repo}:${tag}") + alias_tag="${SANITIZED_BRANCH}-${tag}" + if containers/validate-image-tag.sh "$alias_tag"; then + names+=(-t "${repo}:${alias_tag}") + else + echo "Alias '${alias_tag}' rejected; publishing only ${tag}" >&2 + fi + read -r -a annotations <<< "$(containers/image-metadata.sh annotations "$tag")" + docker buildx imagetools create "${annotations[@]}" "${names[@]}" "${arch_tags[@]}" + # See the same note in image-build-push.yml: a Docker manifest + # list has nowhere to store annotations. + if ! docker buildx imagetools inspect --raw "${repo}:${tag}" | jq -e '.annotations' >/dev/null 2>&1; then + echo "Note: ${repo}:${tag} is not an OCI index, so index annotations were not stored." >&2 + fi + PUSHED_SUMMARY="${PUSHED_SUMMARY}- \`${repo}:${tag}\`"$'\n' + + for arch in ${ARCHES_BY_KEY[$key]}; do + echo "Removing intermediary tag ${repo}:${tag}-${arch}" + curl -s -X DELETE -H "Authorization: JWT ${DOCKERHUB_JWT}" \ + "https://hub.docker.com/v2/repositories/${repo}/tags/${tag}-${arch}/" >/dev/null || true + done + done <<< "${REPOS_BY_KEY[$key]}" + done + + { + echo "summary<> "$GITHUB_OUTPUT" + + - name: Comment on the pull request + uses: actions/github-script@v9 + env: + IMAGE_PUSH_SUMMARY: ${{ steps.push.outputs.summary }} + with: + script: | + const headSha = context.payload.workflow_run.head_sha; + const { data: prs } = await github.rest.repos.listPullRequestsAssociatedWithCommit({ + owner: context.repo.owner, + repo: context.repo.repo, + commit_sha: headSha, + }); + if (prs.length === 0) { + console.log(`No pull request associated with ${headSha}, skipping comment.`); + return; + } + // Read from env, not spliced into the script source, since the + // summary contains backticks that would break a template literal. + const summary = (process.env.IMAGE_PUSH_SUMMARY || "").trim(); + if (!summary) { + core.setFailed("Push step produced no summary - nothing was pushed."); + return; + } + const body = `Pushed updated container image(s) for this PR:\n\n${summary}`; + for (const pr of prs) { + await github.rest.issues.createComment({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: pr.number, + body, + }); + } diff --git a/.github/workflows/push-tagged-dbimage.yml b/.github/workflows/push-tagged-dbimage.yml index ef0211c6ecf..2dfea5b2cc3 100644 --- a/.github/workflows/push-tagged-dbimage.yml +++ b/.github/workflows/push-tagged-dbimage.yml @@ -7,8 +7,8 @@ on: workflow_dispatch: inputs: tag: - description: Base tag for pushed dbimage (v1.25.0 for example)' - required: true + description: 'Tag to push (v1.25.0 for a release). Leave empty to use the tag this branch needs.' + required: false default: "" debug_enabled: description: 'Enable debug mode' @@ -18,39 +18,92 @@ on: env: REGISTRY: docker.io DOCKER_ORG: "${{ vars.DOCKER_ORG }}" - TAG: "${{ github.event.inputs.tag }}" - MULTI_ARCH_IMAGES: "mariadb_10.1 mariadb_10.2 mariadb_10.3 mariadb_10.4 mariadb_10.5 mariadb_10.6 mariadb_10.7 mariadb_10.8 mariadb_10.11 mariadb_11.4 mariadb_11.8 mariadb_12.3 mysql_5.7 mysql_8.0 mysql_8.4 mysql_9.7" permissions: contents: read jobs: + # The variant matrix and which of them are multi-arch both come from + # containers/ddev-dbserver/variants.txt, shared with that directory's + # Makefile and with image-configs.sh. + variants: + name: Read the db variant matrix + runs-on: ubuntu-24.04 + outputs: + matrix: ${{ steps.variants.outputs.matrix }} + multi_arch_images: ${{ steps.variants.outputs.multi_arch_images }} + all_repos: ${{ steps.variants.outputs.all_repos }} + tag: ${{ steps.tag.outputs.tag }} + alias: ${{ steps.tag.outputs.alias }} + steps: + - uses: actions/checkout@v7 + - id: variants + run: | + set -eu -o pipefail + V=containers/ddev-dbserver/variants.sh + { + echo "matrix=$($V json)" + echo "multi_arch_images=$($V multi-arch-variants)" + echo "all_repos=$($V repos | tr '\n' ' ')" + } >> "$GITHUB_OUTPUT" + # An empty tag input means "publish what this checkout needs", which is + # derivable; retyping the content hash by hand is how it ends up not + # matching versionconstants.go. + - id: tag + env: + INPUT_TAG: ${{ github.event.inputs.tag }} + ALIAS_BRANCH: ${{ github.ref_name }} + run: | + set -eu -o pipefail + HASH_LEN="${HASH_LEN:-10}" + if [ -n "$INPUT_TAG" ]; then + TAG="$INPUT_TAG" + echo "Using the tag supplied on the workflow input: ${TAG}" + else + TAG="$(containers/image-tag-for.sh ddev-dbserver-mariadb-11.8)" + echo "No tag supplied; using the tag this checkout needs: ${TAG}" + fi + containers/validate-image-tag.sh "$TAG" || [[ "$TAG" =~ ^v[0-9]+\.[0-9]+\.[0-9]+$ ]] + echo "tag=${TAG}" >> "$GITHUB_OUTPUT" + + # A readable neighbour for a bare hash, so the registry tag list + # still says which branch produced it. A release tag is already + # readable and gets none. + ALIAS="" + if [[ "$TAG" =~ ^[0-9a-f]{${HASH_LEN}}$ ]]; then + ALIAS="$(echo "$ALIAS_BRANCH" | sed -E 's/[^A-Za-z0-9_.-]+/-/g')-${TAG}" + if containers/validate-image-tag.sh "$ALIAS"; then + echo "Publishing the alias ${ALIAS} alongside ${TAG}" + else + echo "Alias '${ALIAS}' rejected; publishing only ${TAG}" >&2 + ALIAS="" + fi + fi + echo "alias=${ALIAS}" >> "$GITHUB_OUTPUT" + build-db-arch: - name: build ${{ matrix.arch }} ${{ matrix.dbtype }} + name: build ${{ matrix.build.arch }} ${{ matrix.build.dbtype }} + needs: variants + env: + TAG: ${{ needs.variants.outputs.tag }} strategy: fail-fast: false matrix: - arch: [amd64, arm64] - dbtype: [mariadb_5.5, mariadb_10.0, mariadb_10.1, mariadb_10.2, mariadb_10.3, mariadb_10.4, mariadb_10.5, mariadb_10.6, mariadb_10.7, mariadb_10.8, mariadb_10.11, mariadb_11.4, mariadb_11.8, mariadb_12.3, mysql_5.5, mysql_5.6, mysql_5.7, mysql_8.0, mysql_8.4, mysql_9.7] - # update 'meta' step below if you change this: - exclude: - - arch: arm64 - dbtype: mariadb_5.5 - - arch: arm64 - dbtype: mariadb_10.0 - - arch: arm64 - dbtype: mysql_5.5 - - arch: arm64 - dbtype: mysql_5.6 - runs-on: ${{ matrix.arch == 'arm64' && 'ubuntu-24.04-arm' || 'ubuntu-24.04' }} + build: ${{ fromJson(needs.variants.outputs.matrix) }} + runs-on: ${{ matrix.build.arch == 'arm64' && 'ubuntu-24.04-arm' || 'ubuntu-24.04' }} steps: - name: Determine if multi-arch build id: meta + env: + MULTI_ARCH_IMAGES: ${{ needs.variants.outputs.multi_arch_images }} + DBTYPE: ${{ matrix.build.dbtype }} run: | - case "${{ matrix.dbtype }}" in - mariadb_5.5|mariadb_10.0|mysql_5.5|mysql_5.6) echo "multi_arch=false" >> $GITHUB_OUTPUT ;; - *) echo "multi_arch=true" >> $GITHUB_OUTPUT ;; - esac + set -eu -o pipefail + if [[ " ${MULTI_ARCH_IMAGES} " == *" ${DBTYPE} "* ]]; then + echo "multi_arch=true" >> $GITHUB_OUTPUT + else + echo "multi_arch=false" >> $GITHUB_OUTPUT + fi - name: Load 1password secret(s) uses: 1password/load-secrets-action@v5 if: ${{ github.event_name != 'pull_request' || github.event.pull_request.head.repo.owner.login == github.repository_owner }} @@ -78,33 +131,33 @@ jobs: - name: Clean up stale arch tag before push run: | TOKEN=$(curl -s -H "Content-Type: application/json" -X POST -d '{"username":"${{ vars.DOCKERHUB_USERNAME }}","password":"${{ env.DOCKERHUB_TOKEN }}"}' https://hub.docker.com/v2/users/login/ | jq -r .token) - ORG_IMAGE=${DOCKER_ORG}/ddev-dbserver-$(echo "${{ matrix.dbtype }}" | tr '_' '-') - echo "Cleaning up stale tag for ${ORG_IMAGE}:${TAG}-${{ matrix.arch }}" - curl -s -X DELETE -H "Authorization: JWT $TOKEN" "https://hub.docker.com/v2/repositories/${ORG_IMAGE}/tags/${TAG}-${{ matrix.arch }}/" >/dev/null || true + ORG_IMAGE=${DOCKER_ORG}/ddev-dbserver-$(echo "${{ matrix.build.dbtype }}" | tr '_' '-') + echo "Cleaning up stale tag for ${ORG_IMAGE}:${TAG}-${{ matrix.build.arch }}" + curl -s -X DELETE -H "Authorization: JWT $TOKEN" "https://hub.docker.com/v2/repositories/${ORG_IMAGE}/tags/${TAG}-${{ matrix.build.arch }}/" >/dev/null || true - name: Setup tmate session uses: mxschmitt/action-tmate@v3 with: limit-access-to-actor: true github-token: ${{ secrets.GITHUB_TOKEN }} if: ${{ github.event_name == 'workflow_dispatch' && inputs.debug_enabled }} - - name: Build and push ${{ env.DOCKER_ORG }}/ddev-dbserver-${{ matrix.dbtype }}:${{ env.TAG }} ${{ matrix.arch }} image + - name: Build and push ${{ env.DOCKER_ORG }}/ddev-dbserver-${{ matrix.build.dbtype }}:${{ env.TAG }} ${{ matrix.build.arch }} image run: | set -eu cd containers/ddev-dbserver - target="${{ matrix.dbtype }}_${{ matrix.arch }}" - echo "Building target $target for arch ${{ matrix.arch }}" + target="${{ matrix.build.dbtype }}_${{ matrix.build.arch }}" + echo "Building target $target for arch ${{ matrix.build.arch }}" version="${TAG}" if [ "${{ steps.meta.outputs.multi_arch }}" = "true" ]; then - version="${version}-${{ matrix.arch }}" + version="${version}-${{ matrix.build.arch }}" fi make $target PUSH=true VERSION="${version}" DDEV_IMAGE_TAG="${TAG}" - name: Record image information id: image-info run: | set -eu - INSPECT_IMAGE=${DOCKER_ORG}/ddev-dbserver-$(echo "${{ matrix.dbtype }}" | tr '_' '-'):${TAG} + INSPECT_IMAGE=${DOCKER_ORG}/ddev-dbserver-$(echo "${{ matrix.build.dbtype }}" | tr '_' '-'):${TAG} if [ "${{ steps.meta.outputs.multi_arch }}" = "true" ]; then - INSPECT_IMAGE=${INSPECT_IMAGE}-${{ matrix.arch }} + INSPECT_IMAGE=${INSPECT_IMAGE}-${{ matrix.build.arch }} fi INSPECT_OUTPUT="" # Wait for image to be available with retry logic @@ -124,14 +177,21 @@ jobs: - name: Upload image info artifact uses: actions/upload-artifact@v7 with: - name: image-info-ddev-dbserver-${{ matrix.dbtype }}-${{ matrix.arch }} + name: image-info-ddev-dbserver-${{ matrix.build.dbtype }}-${{ matrix.build.arch }} path: image-info.txt retention-days: 1 create-manifests: name: create multi-arch db manifests - needs: build-db-arch + needs: [variants, build-db-arch] runs-on: ubuntu-24.04 + env: + MULTI_ARCH_IMAGES: ${{ needs.variants.outputs.multi_arch_images }} + ALL_REPOS: ${{ needs.variants.outputs.all_repos }} + TAG: ${{ needs.variants.outputs.tag }} + ALIAS: ${{ needs.variants.outputs.alias }} steps: + # Needed for containers/image-metadata.sh below. + - uses: actions/checkout@v7 - name: Load 1password secret(s) uses: 1password/load-secrets-action@v5 with: @@ -150,15 +210,31 @@ jobs: # Get Docker Hub token for cleanup TOKEN=$(curl -s -H "Content-Type: application/json" -X POST -d '{"username":"${{ vars.DOCKERHUB_USERNAME }}","password":"${{ env.DOCKERHUB_TOKEN }}"}' https://hub.docker.com/v2/users/login/ | jq -r .token) - # Create and push multi-arch manifests + # Create and push multi-arch manifests. ALIAS, when set, is a second + # name on the same manifest - see the variants job. for variant in ${MULTI_ARCH_IMAGES}; do ORG_IMAGE=${DOCKER_ORG}/ddev-dbserver-$(echo "${variant}" | tr '_' '-') - docker buildx imagetools create -t ${ORG_IMAGE}:${TAG} ${ORG_IMAGE}:${TAG}-amd64 ${ORG_IMAGE}:${TAG}-arm64 + NAMES=(-t "${ORG_IMAGE}:${TAG}") + [ -n "${ALIAS}" ] && NAMES+=(-t "${ORG_IMAGE}:${ALIAS}") + read -r -a ANNOTATIONS <<< "$(containers/image-metadata.sh annotations "${TAG}" "${ORG_IMAGE##*/}")" + docker buildx imagetools create "${ANNOTATIONS[@]}" "${NAMES[@]}" ${ORG_IMAGE}:${TAG}-amd64 ${ORG_IMAGE}:${TAG}-arm64 if [[ "$TAG" =~ ^v[0-9]+\.[0-9]+\.[0-9]+$ ]]; then docker buildx imagetools create -t ${ORG_IMAGE}:latest ${ORG_IMAGE}:${TAG} fi done + # The amd64-only variants are pushed straight to ${TAG} and never + # reach the loop above, so they need the alias adding separately. + if [ -n "${ALIAS}" ]; then + for repo in ${ALL_REPOS}; do + variant="$(echo "${repo#ddev-dbserver-}" | tr '-' '_')" + case " ${MULTI_ARCH_IMAGES} " in *" ${variant} "*) continue ;; esac + ORG_IMAGE=${DOCKER_ORG}/${repo} + echo "Aliasing ${ORG_IMAGE}:${TAG} as ${ALIAS}" + docker buildx imagetools create -t "${ORG_IMAGE}:${ALIAS}" "${ORG_IMAGE}:${TAG}" + done + fi + # Clean up intermediary single-arch tags from remote registry for variant in ${MULTI_ARCH_IMAGES}; do ORG_IMAGE=${DOCKER_ORG}/ddev-dbserver-$(echo "${variant}" | tr '_' '-') diff --git a/.github/workflows/push-tagged-image.yml b/.github/workflows/push-tagged-image.yml index b284511b23e..17977abb596 100644 --- a/.github/workflows/push-tagged-image.yml +++ b/.github/workflows/push-tagged-image.yml @@ -18,8 +18,8 @@ on: - ddev-xhgui - test-ssh-server tag: - description: Tag for pushed image (v1.25.0 for example) - required: true + description: 'Tag to push (v1.25.0 for a release). Leave empty to use the tag this branch needs.' + required: false default: "" debug_enabled: description: 'Enable debug mode' @@ -29,14 +29,60 @@ on: env: REGISTRY: docker.io DOCKER_ORG: "${{ vars.DOCKER_ORG }}" - TAG: "${{ github.event.inputs.tag }}" permissions: contents: read jobs: + # An empty tag input means "publish what this checkout needs", which is + # derivable; retyping the content hash by hand is how it ends up not + # matching versionconstants.go. + resolve-tag: + name: Resolve the tag to push + runs-on: ubuntu-24.04 + outputs: + tag: ${{ steps.tag.outputs.tag }} + alias: ${{ steps.tag.outputs.alias }} + steps: + - uses: actions/checkout@v7 + - id: tag + env: + INPUT_TAG: ${{ github.event.inputs.tag }} + IMAGE: ${{ github.event.inputs.image }} + ALIAS_BRANCH: ${{ github.ref_name }} + run: | + set -eu -o pipefail + HASH_LEN="${HASH_LEN:-10}" + if [ -n "$INPUT_TAG" ]; then + TAG="$INPUT_TAG" + echo "Using the tag supplied on the workflow input: ${TAG}" + else + TAG="$(containers/image-tag-for.sh "$IMAGE")" + echo "No tag supplied; using the tag this checkout needs: ${TAG}" + fi + containers/validate-image-tag.sh "$TAG" || [[ "$TAG" =~ ^v[0-9]+\.[0-9]+\.[0-9]+$ ]] + echo "tag=${TAG}" >> "$GITHUB_OUTPUT" + + # A readable neighbour for a bare hash, so the registry tag list + # still says which branch produced it. A release tag is already + # readable and gets none. + ALIAS="" + if [[ "$TAG" =~ ^[0-9a-f]{${HASH_LEN}}$ ]]; then + ALIAS="$(echo "$ALIAS_BRANCH" | sed -E 's/[^A-Za-z0-9_.-]+/-/g')-${TAG}" + if containers/validate-image-tag.sh "$ALIAS"; then + echo "Publishing the alias ${ALIAS} alongside ${TAG}" + else + echo "Alias '${ALIAS}' rejected; publishing only ${TAG}" >&2 + ALIAS="" + fi + fi + echo "alias=${ALIAS}" >> "$GITHUB_OUTPUT" + build-and-push: name: build ${{ matrix.arch }} ${{ github.event.inputs.image }} + needs: resolve-tag + env: + TAG: ${{ needs.resolve-tag.outputs.tag }} strategy: matrix: arch: [amd64, arm64] @@ -131,10 +177,15 @@ jobs: retention-days: 1 create-manifests: name: create multi-arch manifest - needs: build-and-push + needs: [resolve-tag, build-and-push] runs-on: ubuntu-24.04 if: ${{ needs.build-and-push.outputs.multi_arch == 'true' }} + env: + TAG: ${{ needs.resolve-tag.outputs.tag }} + ALIAS: ${{ needs.resolve-tag.outputs.alias }} steps: + # Needed for containers/image-metadata.sh below. + - uses: actions/checkout@v7 - name: Load 1password secret(s) uses: 1password/load-secrets-action@v5 with: @@ -159,9 +210,13 @@ jobs: MULTI_ARCH_IMAGES="${MULTI_ARCH_IMAGES} ${DOCKER_ORG}/ddev-webserver-prod" fi - # Create and push multi-arch manifests + # Create and push multi-arch manifests. ALIAS, when set, is a second + # name on the same manifest - see the resolve-tag job. for ORG_IMAGE in ${MULTI_ARCH_IMAGES}; do - docker buildx imagetools create -t ${ORG_IMAGE}:${TAG} ${ORG_IMAGE}:${TAG}-amd64 ${ORG_IMAGE}:${TAG}-arm64 + NAMES=(-t "${ORG_IMAGE}:${TAG}") + [ -n "${ALIAS}" ] && NAMES+=(-t "${ORG_IMAGE}:${ALIAS}") + read -r -a ANNOTATIONS <<< "$(containers/image-metadata.sh annotations "${TAG}" "${ORG_IMAGE##*/}")" + docker buildx imagetools create "${ANNOTATIONS[@]}" "${NAMES[@]}" ${ORG_IMAGE}:${TAG}-amd64 ${ORG_IMAGE}:${TAG}-arm64 if [[ "${TAG}" =~ ^v[0-9]+\.[0-9]+\.[0-9]+$ ]]; then docker buildx imagetools create -t ${ORG_IMAGE}:latest ${ORG_IMAGE}:${TAG} fi diff --git a/.github/workflows/test-reusable.yml b/.github/workflows/test-reusable.yml index 3052183d6ca..e5820fb0532 100644 --- a/.github/workflows/test-reusable.yml +++ b/.github/workflows/test-reusable.yml @@ -116,6 +116,7 @@ jobs: runs-on: ${{ inputs.runner }} env: + DOCKER_ORG: ${{ vars.DOCKER_ORG }} BUILDKIT_PROGRESS: plain DOCKER_CLI_EXPERIMENTAL: enabled DDEV_DEBUG: true @@ -152,6 +153,11 @@ jobs: done < <(git ls-tree --name-only refs/public-variables-tmp:.github/public-variables/) git update-ref -d refs/public-variables-tmp + - name: Wait for pushed images + # This runner holds no push credentials, so it can race + # image-push.yml's approval/build/push - see containers/wait-for-images.sh. + run: containers/wait-for-images.sh + - name: Get Date id: get-date run: | diff --git a/.github/workflows/test-wsl2-reusable.yml b/.github/workflows/test-wsl2-reusable.yml index 5d4df9832c3..698a73560d3 100644 --- a/.github/workflows/test-wsl2-reusable.yml +++ b/.github/workflows/test-wsl2-reusable.yml @@ -67,6 +67,7 @@ jobs: name: WSL2 (${{ inputs.networking }}, ${{ inputs.make_target }}) env: + DOCKER_ORG: ${{ vars.DOCKER_ORG }} GOTEST_SHORT: ${{ inputs.gotest_short }} TESTARGS: ${{ inputs.testargs }} MAKE_TARGET: ${{ inputs.make_target }} @@ -193,4 +194,5 @@ jobs: $embargo = "${{ env.DDEV_EMBARGO_TESTS }}" $embargo_php = "${{ env.DDEV_EMBARGO_PHP_VERSIONS }}" $skip_nodejs = "${{ env.DDEV_SKIP_NODEJS_TEST }}" - wsl -u testuser -- bash -exc "export GOTEST_SHORT='$gotest_short' TESTARGS='$testargs' MAKE_TARGET='$make_target' MAKEARGS='$makeargs' DDEV_EMBARGO_TESTS='$embargo' DDEV_EMBARGO_PHP_VERSIONS='$embargo_php' DDEV_SKIP_NODEJS_TEST='$skip_nodejs' && cd ~/workspace/ddev && bash -e .github/workflows/wsl2-test.sh" + $docker_org = "${{ env.DOCKER_ORG }}" + wsl -u testuser -- bash -exc "export GOTEST_SHORT='$gotest_short' TESTARGS='$testargs' MAKE_TARGET='$make_target' MAKEARGS='$makeargs' DDEV_EMBARGO_TESTS='$embargo' DDEV_EMBARGO_PHP_VERSIONS='$embargo_php' DDEV_SKIP_NODEJS_TEST='$skip_nodejs' DOCKER_ORG='$docker_org' && cd ~/workspace/ddev && bash -e .github/workflows/wsl2-test.sh" diff --git a/.github/workflows/wsl2-test.sh b/.github/workflows/wsl2-test.sh index 004ec49644a..c40bc4b3dcd 100755 --- a/.github/workflows/wsl2-test.sh +++ b/.github/workflows/wsl2-test.sh @@ -55,6 +55,11 @@ go version docker version git --version +# This runner holds no push credentials, so it can race image-push.yml's +# approval/build/push - see containers/wait-for-images.sh. +echo "=== Waiting for pushed images ===" +containers/wait-for-images.sh + echo "=== Building DDEV ===" make CGO_ENABLED="${CGO_ENABLED}" BUILDARGS="${BUILDARGS}" diff --git a/HANDOFF.md b/HANDOFF.md new file mode 100644 index 00000000000..cbd7ccf814a --- /dev/null +++ b/HANDOFF.md @@ -0,0 +1,278 @@ +# HANDOFF — PR #8707 review fixes (#8609 phase 2) + +Temporary file. Delete before merging. + +Branch `20260814_rfay_docker_update_phase_2`. Nothing has been pushed. + +## MUST TEST BEFORE MERGE + +### 0. Every image has to be republished at its bare-hash tag + +Tags are now bare content hashes (`ddev/ddev-webserver:36bceca65e`), which +invalidates every previously published content-hash tag. Nothing in either +organization carries the new tags yet — confirmed missing for webserver, +ssh-agent, and the db variants. **Integration tests cannot pull until the +first CI run publishes them**, which is 44 build jobs across 24 images. +Locally, `go test -run TestCmdVersion ./cmd/ddev/cmd/...` currently fails with +`manifest for ddev/ddev-ssh-agent:8e8bf1217c not found`, which is this and +nothing else. + +This is a one-time migration cost of the scheme change; after that first push +the tags stop depending on branch or organization, which is the whole point. + +Two further things have never run in CI and no test harness covers. Both are +cheap to get wrong and expensive to discover after merge. + +### 1. A `containers/ddev-dbserver` change must build and push all 20 variants + +This is the bug that motivated the second round of fixes, and this PR is +itself the first live test of the fix — adding `variants.txt` under +`containers/ddev-dbserver/` changed that directory's hash, so `BaseDBTag` is +now `20260814_rfay_docker_update_phase_2-1dc90407ef` and **CI must build and +push 36 jobs across 20 repositories before any test using a non-default +database can pass.** + +Watch for, on this PR's first real CI run: + +- `detect` reports 36 build jobs / 20 images, not 1. +- `create-manifests` (or `image-push`) produces a multi-arch manifest for + every one of the 20 `ddev/ddev-dbserver-*` repos at that tag, with the four + oldest (`mariadb-5.5`, `mariadb-10.0`, `mysql-5.5`, `mysql-5.6`) amd64-only. +- `TestDdevAllDatabases` passes, along with the tests in `db_test.go`, + `snapshot_test.go`, `config_test.go`, and `debug-migrate-database_test.go` + that pin a non-default database. + +Verify independently of CI's own reporting: + +```bash +TAG=$(grep -E '^var BaseDBTag' pkg/versionconstants/versionconstants.go | sed -E 's/.*"([^"]*)".*/\1/') +for r in $(containers/ddev-dbserver/variants.sh repos); do + printf '%-34s ' "$r"; containers/registry-tag-exists.sh "ddev/$r" "$TAG" && echo EXISTS || echo MISSING +done +``` + +Every line must say EXISTS. Any MISSING means the matrix regressed and +non-default database tests will fail. + +Note that item 1 below was written before the tag scheme changed: the tag to +check is now the bare `BaseDBTag` hash, not a branch-prefixed string, and the +`variants.sh repos` loop still works unchanged. + +### 2. `push-tagged-dbimage.yml` still works after the DRY refactor + +Its matrix, its `MULTI_ARCH_IMAGES` list, and its multi-arch/single-arch +decision were three separate hardcoded copies of the variant list; all three +now come from `variants.sh`. This is the release-time push path, so a mistake +here surfaces during a release. + +Its `tag` input is now optional — left empty it derives the tag the checkout +needs, which is the fix for having to transcribe `main-1dc90407ef` by hand. +Test **both** paths: empty (derives the bare hash) and an explicit `vX.Y.Z` +(the release path). + +Run it manually on `ddev-test/ddev` and confirm: + +- The `variants` job runs first and its matrix expands to **36** `build-db-arch` + jobs — the same count as before (20 variants × 2 arches, minus 4 arm64 + exclusions). +- With an empty tag input, the resolve step logs the derived tag and it matches + `BaseDBTag` in `versionconstants.go`. +- The four amd64-only variants get `multi_arch=false` in their `meta` step and + push an unsuffixed tag; the other 16 get `multi_arch=true` and push + `-amd64`/`-arm64`. +- `create-manifests` combines exactly the 16 multi-arch variants and deletes + the intermediary per-arch tags. +- All 20 repos carry the throwaway tag at the end. + +Locally I confirmed the generated lists are byte-identical to the ones they +replaced (`build-targets` for both host arches, `single-arch-targets`, +`test-targets` for both, and the `MULTI_ARCH_IMAGES` set), and that +`make -n` still resolves a target from each list including the amd64-only +ones under `CURRENT_ARCH=amd64`. That is static equivalence, not a live run. + +## Round 1 fixes — review findings + +### The tag-resolution bug (blocking) + +`wait-for-images.sh` computed `-`, but the tag ddev +pulls is the one committed in `versionconstants.go`, and `autotag.sh` rewrites +that line (branch prefix included) only when the hash changes. The two agree +only on a branch that changed the image, so any PR not touching `containers/` +would poll 20 minutes and fail every Buildkite and GitHub test job. `detect` +had the mirror-image bug: it re-pushed all images under a fresh branch prefix +on any `containers/` change, no-op or not. + +`containers/required-image-tag.sh` now resolves the tag once, for both +callers, reporting `committed` (hash matches — wait for that exact tag) or +`recomputed` (content changed — `make` builds it locally). This also removed +the `WAIT_FOR_IMAGES_BRANCH` plumbing from four callers, since the branch name +is no longer needed. + +### Security + +- `github.head_ref` reached a `run:` block spliced into the script. Git ref + names permit quotes and backticks, and `detect` emits `is_fork`, so injected + code could set `is_fork=false` and route fork content into `build-and-push`, + the job that loads `PUSH_SERVICE_ACCOUNT_TOKEN`. It now arrives via `env:`, + and `is_fork` moved to its own step. +- `image-push.yml` validated the tag but pushed to whatever repository names + the fork-produced artifact listed. `validate-image-repo.sh` constrains them + to `$DOCKER_ORG` plus an exact allowlist. + +### Silent failures + +- `image-push.yml` lacked `actions: read` for a cross-run artifact download, + and `continue-on-error` turned that into a "nothing needed pushing" comment. + A new ungated `check-artifacts` job gates the environment job instead, so a + fork build with nothing to push no longer asks for approval, and a failed + download is now fatal. +- Artifact retention 1 → 7 days; the gate is a human approval. + +### Smaller + +- `DDEV_IMAGE_TAG` was not passed, so `com.ddev.image-tag` recorded + `-` rather than the tag people pull, which + `imageVersionMismatch()` compares against. +- `DOCKER_ORG` falls back to `ddev` instead of producing `/ddev-webserver`. +- `validate-image-tag.sh`'s reserved-literal and `vX.Y.Z` checks were + unreachable behind the format check; they now test the part before the hash. +- BSD `wc -l` padding failed 5 checks on macOS. + +## Round 2 — the db variant matrix + +`GetDBImage()` in `pkg/docker/images.go` builds every variant's reference from +one shared `BaseDBTag`, so a dbserver change moves the tag for all 20 while +`make` built only `mariadb_11.8` and CI pushed only `ddev-dbserver-mariadb-11.8`. +The other 19 were referenced at a tag that existed nowhere. Introduced by +phase 1 (#8612): before that, `BaseDBTag` was hand-bumped after someone ran +`push-tagged-dbimage.yml` for all 20. + +Fixed by making `detect`'s matrix cover every variant. The build matrix is now +one entry per (image, arch) rather than a cross product, because the oldest +variants are amd64-only, and `create-manifests` takes its arch list from +`detect` instead of assuming both. Artifact names key on `repo_suffix`, not +`make_dir` — all 20 db variants share a `make_dir` and would have collided. + +`wait-for-images.sh` now also fails fast, with the command to run, when a +non-locally-built image is out of date. There is no local fallback for those +19 variants, and the tag `make` would invent depends on the runner's branch +name (detached HEAD on a PR checkout), so it may not match what CI pushed. + +### DRY + +The variant list was duplicated in four places. It now lives in +`containers/ddev-dbserver/variants.txt`, read through `variants.sh`, which +renders each consumer's view: + +| Consumer | View | +| --- | --- | +| `containers/ddev-dbserver/Makefile` | `build-targets`, `single-arch-targets`, `test-targets` | +| `containers/image-configs.sh` | `list` | +| `containers/validate-image-repo.sh` | `repos` | +| `.github/workflows/push-tagged-dbimage.yml` | `json`, `multi-arch-variants` | + +`variants.txt` sits inside the hashed dbserver directory on purpose: adding a +database version has to change the content hash, or `detect` would decide the +tag already exists and never build the new variant. + +## Test status + +127 checks across seven harnesses, all passing locally (macOS), all wired into +`container-tests.yml`: + +| Harness | Checks | +| --- | --- | +| `containers/autotag_test.sh` | 18 | +| `containers/db_variants_test.sh` | 15 (new) | +| `containers/registry_tag_exists_test.sh` | 4 | +| `containers/required_image_tag_test.sh` | 12 (new) | +| `containers/validate_image_repo_test.sh` | 37 (new) | +| `containers/validate_image_tag_test.sh` | 20 | +| `containers/wait_for_images_test.sh` | 21 | + +`shellcheck -x` clean on all new and changed scripts. `actionlint` clean on +all changed workflows apart from pre-existing SC2086/SC2046 notes in untouched +parts of `test-reusable.yml`. + +## Verification Claude ran + +Read-only against the real registry, plus this checkout. Nothing pushed. + +1. All seven harnesses. +2. `wait-for-images.sh` finds the four non-db images and correctly waits on the + db variants at the new `BaseDBTag`. +3. `detect` dry run: `matrix=[]` when nothing changed; 36 build jobs / 20 + manifests after a dbserver change; only xhgui after an xhgui change. +4. Hostile branch name `evil"; id; #` sanitizes to `evil-id-`, no execution. +5. Generated db lists byte-identical to the four hardcoded copies they replace; + `make -n` resolves a target from each list on both host arches. +6. `make autotag-images` built `mariadb_11.8` locally and rewrote `BaseDBTag`. +7. `docker buildx imagetools inspect` works with no Docker daemon running, + so the early placement of the wait step in the Buildkite scripts is fine. + +Not run: the artifact round-trip against a local `registry:2` (worth doing — +it needs no secrets and would exercise the multi-arch `imagetools create` +grouping that no unit test covers), and a full `make` build. + +## Other human verification + +On `ddev-test/ddev`, with the `image-push` environment configured. Confirm the +rule actually took: `gh api repos///environments/image-push` — an +environment referenced before it exists is auto-created with no protection. + +1. **A PR touching no `containers/` file.** Five `found …` lines within seconds, + then the tests run. This is the round-1 blocking fix and has never run in CI. +2. **A `containers/` file that isn't in any hash path.** `detect` reports + `already exists (committed)` for everything and builds nothing. +3. **An `ddev-xhgui` change on a same-repo branch.** Only xhgui builds; no + approval; `com.ddev.image-tag` on the result is the final tag, not + `-amd64`. +4. **The same from a fork.** No secret-loading step in `build`; exactly one + approval; the download succeeds rather than silently commenting + "nothing needed pushing". +5. **A fork PR touching `containers/` with no image change.** No approval + request at all. +6. **Adversarial artifact.** Overwrite `repos.txt` with `ddev/ddev-webserver` + before upload; the push must fail at `validate-image-repo.sh`. +7. **Hostile branch name** ``test`touch /tmp/pwned` ``; check the `detect` log. +8. **Expired artifact.** Approve after `retention-days`; must fail loudly. + +## Round 3 — hash-only tags + +The branch prefix in a tag carried no information but forced every consumer to +agree on a string. Tags are now the bare content hash; the branch survives as a +trailing comment in `versionconstants.go`, a `TagBranch` variable shown by +`ddev version` (`image-tag-branches`, collapsed when all images share a +branch), and a `-` alias published off the same manifest. + +`validate-image-tag.sh` accepts both forms, and the alias goes through it, so a +fork branch named `v1.25.0` can't publish something that reads as a release — +which finally makes the reserved/release checks load-bearing rather than dead. + +Comparison against `versionconstants.go` is exact now, so a line still in the +old form counts as stale and `make` migrates it. `wait-for-images.sh` lost its +fail-fast branch: the tag a non-locally-built image needs is branch-independent, +so waiting for it is well defined. + +The manual push workflows' `tag` input is optional; empty derives the tag via +`containers/image-tag-for.sh`. + +## Still open + +- **Fork artifact volume.** A fork changing `ddev-dbserver` now produces 36 + `docker save` tarballs. Local db images are 550–730MB, so that is roughly + 20GB of artifacts for one PR, compressed by `upload-artifact` but still + large, and now retained 7 days. This is the cost of building the full matrix + on the untrusted path; if it proves impractical, the fallback is to keep the + full matrix on the non-fork path and have forks use + `push-tagged-dbimage.yml`. Watch the first fork-side dbserver PR. +- **`actions: read` on `download-artifact@v8`** is added on the documented + requirement; not yet observed on a live run. +- **Registry pollution has no cleanup path.** Each image change adds tags that + are never removed — now 20 at a time for a dbserver change. +- **`registry-tag-exists.sh` cannot distinguish "missing" from "registry + unreachable."** A DockerHub blip costs a redundant rebuild in `detect` or 20 + minutes and a red build in `wait-for-images.sh`. It now makes 24 registry + calls per test run instead of 5, so the odds are higher. +- **The PR description** was rewritten for round 1 but does not yet mention the + db variant matrix. diff --git a/Makefile b/Makefile index 6773645f7b8..f3e8a3f71ad 100644 --- a/Makefile +++ b/Makefile @@ -68,6 +68,11 @@ build: autotag-images $(DEFAULT_BUILD) # (no Docker, no network). A changed image is built locally (host arch only) # and its tag in versionconstants.go is rewritten automatically - see # containers/autotag.sh and docs/content/developers/building-contributing.md. +# Only the default db variant (mariadb_11.8) is built here, to keep an +# unrelated rebuild cheap. Every ddev-dbserver variant shares BaseDBTag, so a +# dbserver change moves the tag for all of them at once; CI builds and pushes +# the whole matrix (containers/ddev-dbserver/variants.txt), and the rest are +# pulled from the registry. .PHONY: autotag-images autotag-images: @containers/autotag.sh WebTag ddev/ddev-webserver containers/ddev-webserver containers/containers_shared.mk -- $(MAKE) -C containers/ddev-webserver images @@ -225,7 +230,14 @@ setup: @mkdir -p $(TESTTMP) # Required static analysis targets for pre-push. -staticrequired: setup golangci-lint markdownlint zensical +staticrequired: setup golangci-lint markdownlint zensical check-image-tags + +# A changed image whose tag never made it into versionconstants.go ships a +# binary pulling tags nothing builds. Checked rather than fixed here, because +# fixing means a Docker build - run `make` for that. +.PHONY: check-image-tags +check-image-tags: + @containers/check-image-tags.sh # Fail rather than skip when a required tool is absent. These targets used to # print a note and exit 0, so `make staticrequired` reported success while diff --git a/containers/autotag.sh b/containers/autotag.sh index 9d118202783..7efc80afe04 100755 --- a/containers/autotag.sh +++ b/containers/autotag.sh @@ -2,14 +2,20 @@ # autotag.sh [--print-only] [ ...] [-- ] # # Detects whether an image's content hash (see hash-paths.sh) differs from -# the hash embedded in the tag currently committed in -# pkg/versionconstants/versionconstants.go. If unchanged, does nothing (no -# Docker, no network). If changed, builds the image locally at the new -# - tag (unless already built) using the given build command, -# then rewrites just that tag's line in versionconstants.go in place. +# the tag currently committed in pkg/versionconstants/versionconstants.go. If +# unchanged, does nothing (no Docker, no network). If changed, builds the +# image locally at the new tag (unless already built) using the given build +# command, then rewrites just that tag's line in versionconstants.go in place. # -# --print-only prints the candidate - tag and exits, without -# touching Docker or versionconstants.go. +# The tag is the bare content hash, with no branch prefix: the hash is the +# content address, and a prefix would only make the same image resolve to +# different strings depending on which branch or repository published it. +# The branch is recorded in a trailing comment instead, so the line still says +# where the image came from, and pushes publish a - alias +# pointing at the same manifest for anyone browsing the registry. +# +# --print-only prints the candidate tag and exits, without touching Docker or +# versionconstants.go. # # Env: # HASH_LEN - hash length in hex chars (default 10, must match hash-paths.sh) @@ -60,7 +66,8 @@ CURRENT_HASH="$(HASH_LEN="$HASH_LEN" "$SCRIPT_DIR/hash-paths.sh" "${HASH_PATHS[@ BRANCH="$(git -C "$REPO_ROOT" rev-parse --abbrev-ref HEAD 2>/dev/null || echo detached)" SANITIZED_BRANCH="$(echo "$BRANCH" | sed -E 's/[^A-Za-z0-9_.-]+/-/g')" -CANDIDATE_TAG="${SANITIZED_BRANCH}-${CURRENT_HASH}" +CANDIDATE_TAG="${CURRENT_HASH}" +ALIAS_TAG="${SANITIZED_BRANCH}-${CURRENT_HASH}" if [ "$PRINT_ONLY" = true ]; then echo "$CANDIDATE_TAG" @@ -73,9 +80,9 @@ if [ -z "$EXISTING_TAG" ]; then exit 1 fi -EXISTING_HASH="${EXISTING_TAG: -${HASH_LEN}}" - -if [ "$EXISTING_HASH" = "$CURRENT_HASH" ]; then +# Exact, not a trailing-hash match, so a line still in the old - +# form is rewritten to the bare hash the first time make runs. +if [ "$EXISTING_TAG" = "$CURRENT_HASH" ]; then # Unchanged - nothing to do. exit 0 fi @@ -89,10 +96,16 @@ else echo "autotag.sh: no build command given, cannot build ${IMAGE_REPO}:${CANDIDATE_TAG}" >&2 exit 1 fi - "${BUILD_CMD[@]}" "VERSION=${CANDIDATE_TAG}" + "${BUILD_CMD[@]}" "VERSION=${CANDIDATE_TAG}" "DDEV_IMAGE_TAG=${CANDIDATE_TAG}" fi -sed -i.bak -E "s/^var ${TAG_VAR} = \"[^\"]*\"/var ${TAG_VAR} = \"${CANDIDATE_TAG}\"/" "$VERSIONCONSTANTS_FILE" +# The trailing comment is rewritten wholesale, not preserved: it exists to say +# which branch produced this hash, which is the readability a bare hash costs. +# Branch carries the same thing as data, for `ddev version`. +sed -i.bak -E \ + -e "s|^var ${TAG_VAR} = \"[^\"]*\".*|var ${TAG_VAR} = \"${CANDIDATE_TAG}\" // ${ALIAS_TAG}|" \ + -e "s|^var ${TAG_VAR}Branch = \"[^\"]*\"|var ${TAG_VAR}Branch = \"${SANITIZED_BRANCH}\"|" \ + "$VERSIONCONSTANTS_FILE" rm -f "${VERSIONCONSTANTS_FILE}.bak" -echo "autotag.sh: updated ${TAG_VAR} to ${CANDIDATE_TAG} in $VERSIONCONSTANTS_FILE" +echo "autotag.sh: updated ${TAG_VAR} to ${CANDIDATE_TAG} (${ALIAS_TAG}) in $VERSIONCONSTANTS_FILE" diff --git a/containers/autotag_test.sh b/containers/autotag_test.sh index 19f0e020fe8..96ed74fa401 100755 --- a/containers/autotag_test.sh +++ b/containers/autotag_test.sh @@ -32,6 +32,11 @@ assert_eq() { fi } +# BSD wc pads its output with spaces; GNU wc does not. +count_lines() { + wc -l < "$1" | tr -d '[:space:]' +} + WORKDIR="$(mktemp -d)" trap 'rm -rf "$WORKDIR"' EXIT @@ -79,7 +84,10 @@ cat > "$VERSIONCONSTANTS" <<'EOF' package versionconstants // WebTag defines the default web image tag -var WebTag = "v1.0.0" // Note that this can be overridden by make +var WebTag = "v1.0.0" // some-old-branch-v1.0.0 + +// WebTagBranch is the branch WebTag's content was built from. +var WebTagBranch = "some-old-branch" EOF export VERSIONCONSTANTS_FILE="$VERSIONCONSTANTS" @@ -131,11 +139,8 @@ candidate="$("$AUTOTAG" --print-only WebTag ddev/dummy-image imgdir)" after="$(cat "$VERSIONCONSTANTS")" assert_eq "$before" "$after" "--print-only does not modify the versionconstants file" current_hash="$("$HASH_PATHS" imgdir)" -case "$candidate" in - *-"$current_hash") pass "--print-only candidate tag ends with the current hash" ;; - *) fail "--print-only candidate tag '$candidate' doesn't end with hash '$current_hash'" ;; -esac -calls="$(wc -l < "$DOCKER_CALL_LOG")" +assert_eq "$current_hash" "$candidate" "--print-only candidate tag is the bare content hash" +calls="$(count_lines "$DOCKER_CALL_LOG")" assert_eq "0" "$calls" "--print-only makes no docker calls" # 6. Change detected (today's fixture tag is "v1.0.0", never a real hash), @@ -149,24 +154,21 @@ else fail "build command should have run when the tag changed and no local image existed" fi +current_branch="$(git rev-parse --abbrev-ref HEAD | sed -E 's/[^A-Za-z0-9_.-]+/-/g')" new_tag_line="$(grep '^var WebTag = ' "$VERSIONCONSTANTS")" -case "$new_tag_line" in - *"-${current_hash}\""*) pass "versionconstants file rewritten with the new hash-suffixed tag" ;; - *) fail "versionconstants file not rewritten as expected: $new_tag_line" ;; -esac -case "$new_tag_line" in - *"// Note that this can be overridden by make"*) pass "trailing comment on the tag line is preserved" ;; - *) fail "trailing comment on the tag line was lost: $new_tag_line" ;; -esac +assert_eq "var WebTag = \"${current_hash}\" // ${current_branch}-${current_hash}" "$new_tag_line" \ + "the tag becomes the bare hash, with the branch alias as a trailing comment" +assert_eq "var WebTagBranch = \"${current_branch}\"" "$(grep '^var WebTagBranch = ' "$VERSIONCONSTANTS")" \ + "the companion Branch variable is updated for ddev version" # 7. Idempotency: re-run with no further changes -> no-op. No build, no file # rewrite, and (the key design property) no docker call at all. rm -f "$BUILD_MARKER" before2="$(cat "$VERSIONCONSTANTS")" -calls_before="$(wc -l < "$DOCKER_CALL_LOG")" +calls_before="$(count_lines "$DOCKER_CALL_LOG")" "$AUTOTAG" WebTag ddev/dummy-image imgdir -- bash -c "touch '$BUILD_MARKER'" after2="$(cat "$VERSIONCONSTANTS")" -calls_after="$(wc -l < "$DOCKER_CALL_LOG")" +calls_after="$(count_lines "$DOCKER_CALL_LOG")" if [ -f "$BUILD_MARKER" ]; then fail "unexpected rebuild on unchanged content" else @@ -179,8 +181,7 @@ assert_eq "$calls_before" "$calls_after" "no docker calls at all on the unchange # tag -> build is skipped, but the file is still rewritten. echo "changed again" > imgdir/Dockerfile new_hash="$("$HASH_PATHS" imgdir)" -branch="$(git rev-parse --abbrev-ref HEAD | sed -E 's/[^A-Za-z0-9_.-]+/-/g')" -echo "ddev/dummy-image:${branch}-${new_hash}" > "$DOCKER_EXISTING_REF_FILE" +echo "ddev/dummy-image:${new_hash}" > "$DOCKER_EXISTING_REF_FILE" rm -f "$BUILD_MARKER" "$AUTOTAG" WebTag ddev/dummy-image imgdir -- bash -c "touch '$BUILD_MARKER'" if [ -f "$BUILD_MARKER" ]; then @@ -188,12 +189,28 @@ if [ -f "$BUILD_MARKER" ]; then else pass "build skipped when a local image already exists at the computed tag" fi -if grep -q "\"${branch}-${new_hash}\"" "$VERSIONCONSTANTS"; then +if grep -q "^var WebTag = \"${new_hash}\"" "$VERSIONCONSTANTS"; then pass "versionconstants file rewritten even when the build was skipped" else fail "versionconstants file should still be rewritten when the build is skipped" fi +# 9. A line still carrying the old - form is migrated to the bare +# hash on the next run, even though its trailing hash already matches. +cat > "$VERSIONCONSTANTS" < ${tag}") +done + +if [ "${#STALE[@]}" -eq 0 ]; then + echo "check-image-tags.sh: versionconstants.go is up to date with containers/" + exit 0 +fi + +echo "check-image-tags.sh: containers/ changed but versionconstants.go wasn't updated:" >&2 +printf ' %s\n' "${STALE[@]}" >&2 +echo "check-image-tags.sh: run 'make' to rebuild the changed image(s) and rewrite those tags, then commit the result." >&2 +exit 1 diff --git a/containers/containers_shared.mk b/containers/containers_shared.mk index 4d5c3d4c070..030f7696d6e 100644 --- a/containers/containers_shared.mk +++ b/containers/containers_shared.mk @@ -13,10 +13,14 @@ DDEV_IMAGE_TAG ?= $(VERSION) DOTFILE_IMAGE = $(subst /,_,$(IMAGE))-$(VERSION) +# Standard OCI descriptive metadata (source, revision, created, ...) shared with +# every other image and with the push workflows - see containers/image-metadata.sh. +DDEV_IMAGE_LABELS = $(shell ../image-metadata.sh labels $(DDEV_IMAGE_TAG) $(notdir $(DOCKER_REPO))) + .PHONY: container push container: container-name - docker build -t $(DOCKER_REPO):$(VERSION) $(DOCKER_ARGS) --label "build-info=$(DOCKER_REPO):$(VERSION) commit=$(shell git describe --tags --always)" --label "com.ddev.image-tag=$(DDEV_IMAGE_TAG)" . + docker build -t $(DOCKER_REPO):$(VERSION) $(DOCKER_ARGS) --label "build-info=$(DOCKER_REPO):$(VERSION) commit=$(shell git describe --tags --always)" $(DDEV_IMAGE_LABELS) . container-name: @echo "container: $(DOCKER_REPO):$(VERSION)" @@ -35,6 +39,6 @@ push: $${tags} \ --label "build-info=$(DOCKER_ORG)/$${item}:$(VERSION) commit=$(shell git describe --tags --always) built $$(date) by $$(id -un) on $$(hostname)" \ --label "maintainer=DDEV " \ - --label "com.ddev.image-tag=$(DDEV_IMAGE_TAG)" \ + $$(../image-metadata.sh labels "$(DDEV_IMAGE_TAG)" "$${item}") \ $(DOCKER_ARGS) . ; \ done diff --git a/containers/db_variants_test.sh b/containers/db_variants_test.sh new file mode 100755 index 00000000000..e75a89f4c92 --- /dev/null +++ b/containers/db_variants_test.sh @@ -0,0 +1,110 @@ +#!/usr/bin/env bash +# db_variants_test.sh - unit tests for ddev-dbserver/variants.sh. +# +# Pure text transforms, no Docker or network. The interesting property is that +# every consumer's view stays mutually consistent, since the whole point of +# variants.txt is that the four of them can't drift apart. +# Run with: +# containers/db_variants_test.sh + +set -eu -o pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +VARIANTS="$SCRIPT_DIR/ddev-dbserver/variants.sh" + +FAILURES=0 + +fail() { + echo "FAIL: $1" >&2 + FAILURES=$((FAILURES + 1)) +} + +pass() { + echo "PASS: $1" +} + +assert_eq() { + local expected="$1" actual="$2" desc="$3" + if [ "$expected" = "$actual" ]; then + pass "$desc" + else + fail "$desc (expected '$expected', got '$actual')" + fi +} + +WORKDIR="$(mktemp -d)" +trap 'rm -rf "$WORKDIR"' EXIT + +# --- A fixture matrix, so these assertions don't move every time a database +# version is added or dropped. +export VARIANTS_FILE="$WORKDIR/variants.txt" +cat > "$VARIANTS_FILE" <<'EOF' +# comment line, and a blank line, both ignored + +mysql_9.7 amd64 arm64 +mysql_5.6 amd64 +mariadb_11.8 amd64 arm64 +mariadb_5.5 amd64 +EOF + +assert_eq "mysql_9.7_both mysql_5.6_amd64 mariadb_11.8_both mariadb_5.5_amd64" \ + "$("$VARIANTS" build-targets amd64)" "build-targets on amd64 covers every variant" +assert_eq "mysql_9.7_both mariadb_11.8_both" \ + "$("$VARIANTS" build-targets arm64)" "build-targets on arm64 omits the amd64-only variants" +assert_eq "mysql_9.7_amd64 mysql_9.7_arm64 mariadb_11.8_amd64 mariadb_11.8_arm64" \ + "$("$VARIANTS" single-arch-targets)" "single-arch-targets covers only multi-arch variants, both ways" +assert_eq "mysql_9.7 mariadb_11.8" \ + "$("$VARIANTS" multi-arch-variants)" "multi-arch-variants is what gets a combined manifest" +assert_eq "mysql_9.7_test mysql_5.6_test mariadb_11.8_test mariadb_5.5_test" \ + "$("$VARIANTS" test-targets amd64)" "test-targets on amd64 covers every variant" +assert_eq "mysql_9.7_test mariadb_11.8_test" \ + "$("$VARIANTS" test-targets arm64)" "test-targets on arm64 omits the amd64-only variants" +assert_eq "ddev-dbserver-mysql-9.7 ddev-dbserver-mysql-5.6 ddev-dbserver-mariadb-11.8 ddev-dbserver-mariadb-5.5" \ + "$("$VARIANTS" repos | tr '\n' ' ' | sed 's/ $//')" "repos names the Docker Hub repositories" +assert_eq "6" "$("$VARIANTS" json | jq 'length')" "json emits one entry per (variant, arch)" +assert_eq "amd64" "$("$VARIANTS" json | jq -r '[.[] | select(.dbtype=="mysql_5.6")] | .[].arch')" \ + "json gives an amd64-only variant exactly one entry" + +if "$VARIANTS" bogus-view >/dev/null 2>&1; then + fail "should reject an unknown view" +else + pass "rejects an unknown view" +fi +if "$VARIANTS" build-targets >/dev/null 2>&1; then + fail "should require a host arch for build-targets" +else + pass "requires a host arch for build-targets" +fi + +# --- Against the real variants.txt: the views have to agree with each other, +# which is what stops the Makefile, image-configs.sh, and the push workflow +# from disagreeing about what exists. +unset VARIANTS_FILE + +real_build_amd64="$("$VARIANTS" build-targets amd64 | wc -w | tr -d '[:space:]')" +real_repos="$("$VARIANTS" repos | wc -l | tr -d '[:space:]')" +assert_eq "$real_build_amd64" "$real_repos" "every variant has a repo and an amd64 build target" + +real_json="$("$VARIANTS" json | jq 'length')" +multi="$("$VARIANTS" multi-arch-variants | wc -w | tr -d '[:space:]')" +single_arch="$(( real_repos - multi ))" +assert_eq "$(( multi * 2 + single_arch ))" "$real_json" "the json matrix is 2 jobs per multi-arch variant plus 1 each for the rest" + +assert_eq "$(( multi * 2 ))" "$("$VARIANTS" single-arch-targets | wc -w | tr -d '[:space:]')" \ + "single-arch-targets is two per multi-arch variant" + +# The default variant `make` builds locally must be in the matrix, or +# image-configs.sh would mark a nonexistent variant as locally built. +if "$VARIANTS" repos | grep -qx "ddev-dbserver-mariadb-11.8"; then + pass "the default variant image-configs.sh builds locally is in the matrix" +else + fail "mariadb_11.8 (image-configs.sh's DDEV_DBSERVER_DEFAULT_TARGET) is missing from variants.txt" +fi + +if [ "$FAILURES" -eq 0 ]; then + echo "All db_variants_test.sh checks passed." + exit 0 +else + echo "$FAILURES db_variants_test.sh check(s) failed." >&2 + exit 1 +fi diff --git a/containers/ddev-dbserver/Makefile b/containers/ddev-dbserver/Makefile index c648aac64f4..74ed2d4f6e8 100644 --- a/containers/ddev-dbserver/Makefile +++ b/containers/ddev-dbserver/Makefile @@ -19,17 +19,11 @@ CURRENT_ARCH=$(shell ../get_arch.sh) # So has to explicitly declare anything it might need from there (like SHELL) SHELL = /bin/bash -BUILD_TARGETS=$(shell if [ "$(CURRENT_ARCH)" = "amd64" ] ; then \ - echo "mysql_9.7_both mysql_8.4_both mysql_8.0_both mysql_5.7_both mysql_5.6_amd64 mysql_5.5_amd64 mariadb_12.3_both mariadb_11.8_both mariadb_11.4_both mariadb_10.11_both mariadb_10.8_both mariadb_10.7_both mariadb_10.6_both mariadb_10.5_both mariadb_10.4_both mariadb_10.3_both mariadb_10.2_both mariadb_10.1_both mariadb_10.0_amd64 mariadb_5.5_amd64"; \ -else \ - echo "mysql_9.7_both mysql_8.4_both mysql_8.0_both mysql_5.7_both mariadb_12.3_both mariadb_11.8_both mariadb_11.4_both mariadb_10.11_both mariadb_10.8_both mariadb_10.7_both mariadb_10.6_both mariadb_10.5_both mariadb_10.4_both mariadb_10.3_both mariadb_10.2_both mariadb_10.1_both"; \ -fi ) -SINGLE_ARCH_TARGETS=mysql_9.7_amd64 mysql_9.7_arm64 mysql_8.4_amd64 mysql_8.4_arm64 mysql_8.0_amd64 mysql_8.0_arm64 mysql_5.7_amd64 mysql_5.7_arm64 mariadb_12.3_amd64 mariadb_12.3_arm64 mariadb_11.8_amd64 mariadb_11.8_arm64 mariadb_11.4_amd64 mariadb_11.4_arm64 mariadb_10.11_amd64 mariadb_10.11_arm64 mariadb_10.8_amd64 mariadb_10.8_arm64 mariadb_10.7_amd64 mariadb_10.7_arm64 mariadb_10.6_amd64 mariadb_10.6_arm64 mariadb_10.5_amd64 mariadb_10.5_arm64 mariadb_10.4_amd64 mariadb_10.4_arm64 mariadb_10.3_amd64 mariadb_10.3_arm64 mariadb_10.2_amd64 mariadb_10.2_arm64 mariadb_10.1_amd64 mariadb_10.1_arm64 -TEST_TARGETS=$(shell if [ "$(CURRENT_ARCH)" = "amd64" ] ; then \ - echo "mysql_9.7_test mysql_8.4_test mysql_8.0_test mysql_5.7_test mysql_5.6_test mysql_5.5_test mariadb_12.3_test mariadb_11.8_test mariadb_11.4_test mariadb_10.11_test mariadb_10.8_test mariadb_10.7_test mariadb_10.6_test mariadb_10.5_test mariadb_10.4_test mariadb_10.3_test mariadb_10.2_test mariadb_10.1_test mariadb_10.0_test mariadb_5.5_test"; \ -else \ - echo "mysql_9.7_test mysql_8.4_test mysql_8.0_test mysql_5.7_test mariadb_12.3_test mariadb_11.8_test mariadb_11.4_test mariadb_10.11_test mariadb_10.8_test mariadb_10.7_test mariadb_10.6_test mariadb_10.5_test mariadb_10.4_test mariadb_10.3_test mariadb_10.2_test mariadb_10.1_test"; \ -fi ) +# The variant matrix lives in variants.txt - see variants.sh for the views. +VARIANTS_SH=./variants.sh +BUILD_TARGETS=$(shell $(VARIANTS_SH) build-targets $(CURRENT_ARCH)) +SINGLE_ARCH_TARGETS=$(shell $(VARIANTS_SH) single-arch-targets) +TEST_TARGETS=$(shell $(VARIANTS_SH) test-targets $(CURRENT_ARCH)) container: build diff --git a/containers/ddev-dbserver/build_image.sh b/containers/ddev-dbserver/build_image.sh index 25e1486734d..9d86b436bd1 100755 --- a/containers/ddev-dbserver/build_image.sh +++ b/containers/ddev-dbserver/build_image.sh @@ -140,10 +140,15 @@ tag_directive="-t ${DOCKER_ORG}/ddev-dbserver-${DB_TYPE}-${DB_MAJOR_VERSION}:${I if [[ ${IMAGE_TAG} =~ ^v[0-9]+\.[0-9]+\.[0-9]+$ ]]; then tag_directive="$tag_directive -t ${DOCKER_ORG}/ddev-dbserver-${DB_TYPE}-${DB_MAJOR_VERSION}:latest" fi + +# Same OCI metadata every other image carries; this directory deliberately does +# not include containers_shared.mk, so it calls the generator itself. +label_directive="$("$(dirname "$0")/../image-metadata.sh" labels "${DDEV_IMAGE_TAG}" "ddev-dbserver-${DB_TYPE}-${DB_MAJOR_VERSION}")" + if [ ! -z ${PUSH:-} ]; then echo "building/pushing ddev/ddev-dbserver-${DB_TYPE}-${DB_MAJOR_VERSION}:${IMAGE_TAG}" set -x - docker buildx build --push --platform ${ARCHS} ${DOCKER_ARGS} --build-arg="BASE_IMAGE=${BASE_IMAGE}" --build-arg="DB_PINNED_VERSION=${DB_PINNED_VERSION}" --build-arg="DB_MAJOR_VERSION=${DB_MAJOR_VERSION}" --build-arg="DDEV_IMAGE_TAG=${DDEV_IMAGE_TAG}" ${tag_directive} . + docker buildx build --push --platform ${ARCHS} ${DOCKER_ARGS} --build-arg="BASE_IMAGE=${BASE_IMAGE}" --build-arg="DB_PINNED_VERSION=${DB_PINNED_VERSION}" --build-arg="DB_MAJOR_VERSION=${DB_MAJOR_VERSION}" --build-arg="DDEV_IMAGE_TAG=${DDEV_IMAGE_TAG}" ${label_directive} ${tag_directive} . set +x fi @@ -151,5 +156,5 @@ fi set -x if [ -z "${PUSH:-}" ]; then echo "Loading to local docker ddev/ddev-dbserver-${DB_TYPE}-${DB_MAJOR_VERSION}:${IMAGE_TAG}" - docker buildx build --load ${DOCKER_ARGS} --build-arg="DB_TYPE=${DB_TYPE}" --build-arg="DB_MAJOR_VERSION=${DB_MAJOR_VERSION}" --build-arg="BASE_IMAGE=${BASE_IMAGE}" --build-arg="DB_PINNED_VERSION=${DB_PINNED_VERSION}" --build-arg="DDEV_IMAGE_TAG=${DDEV_IMAGE_TAG}" ${tag_directive} . + docker buildx build --load ${DOCKER_ARGS} --build-arg="DB_TYPE=${DB_TYPE}" --build-arg="DB_MAJOR_VERSION=${DB_MAJOR_VERSION}" --build-arg="BASE_IMAGE=${BASE_IMAGE}" --build-arg="DB_PINNED_VERSION=${DB_PINNED_VERSION}" --build-arg="DDEV_IMAGE_TAG=${DDEV_IMAGE_TAG}" ${label_directive} ${tag_directive} . fi diff --git a/containers/ddev-dbserver/variants.sh b/containers/ddev-dbserver/variants.sh new file mode 100755 index 00000000000..f32c27622b6 --- /dev/null +++ b/containers/ddev-dbserver/variants.sh @@ -0,0 +1,122 @@ +#!/usr/bin/env bash +# variants.sh [] +# +# Renders the variant matrix in variants.txt for each of its consumers, so the +# list lives in exactly one place. Views: +# +# build-targets make targets for a `make build` on that host: +# _both, or _ for a +# variant that only builds on one. Variants the +# host can't build are omitted. +# single-arch-targets _amd64 _arm64 for every +# multi-arch variant, for one-arch-at-a-time +# builds (a multi-arch push builds each side +# separately, then combines them). +# multi-arch-variants the variants that get a combined manifest +# test-targets _test for what the host can build +# list | for shell consumers +# repos ddev-dbserver-- repository names +# json [{"dbtype":…,"arch":…}, …] for a GitHub matrix + +set -eu -o pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +VARIANTS_FILE="${VARIANTS_FILE:-$SCRIPT_DIR/variants.txt}" + +if [ "$#" -lt 1 ]; then + echo "Usage: $0 [host-arch]" >&2 + exit 2 +fi + +VIEW="$1" +HOST_ARCH="${2:-}" + +VARIANTS=() +ARCHES=() +while read -r variant arches; do + [ -z "$variant" ] && continue + case "$variant" in \#*) continue ;; esac + VARIANTS+=("$variant") + ARCHES+=("$arches") +done < "$VARIANTS_FILE" + +if [ "${#VARIANTS[@]}" -eq 0 ]; then + echo "$0: no variants found in $VARIANTS_FILE" >&2 + exit 1 +fi + +host_can_build() { + local arches="$1" + [ -z "$HOST_ARCH" ] && return 0 + [[ " $arches " == *" $HOST_ARCH "* ]] +} + +require_host_arch() { + if [ -z "$HOST_ARCH" ]; then + echo "$0: $VIEW needs a host arch argument" >&2 + exit 2 + fi +} + +OUT=() +for i in "${!VARIANTS[@]}"; do + variant="${VARIANTS[$i]}" + arches="${ARCHES[$i]}" + # shellcheck disable=SC2086 # arches is a space-separated list + set -- $arches + arch_count="$#" + + case "$VIEW" in + build-targets) + require_host_arch + host_can_build "$arches" || continue + if [ "$arch_count" -gt 1 ]; then + OUT+=("${variant}_both") + else + OUT+=("${variant}_$1") + fi + ;; + single-arch-targets) + [ "$arch_count" -gt 1 ] || continue + for arch in $arches; do + OUT+=("${variant}_${arch}") + done + ;; + multi-arch-variants) + [ "$arch_count" -gt 1 ] || continue + OUT+=("$variant") + ;; + test-targets) + require_host_arch + host_can_build "$arches" || continue + OUT+=("${variant}_test") + ;; + list) + OUT+=("${variant}|${arches}") + ;; + repos) + OUT+=("ddev-dbserver-${variant/_/-}") + ;; + json) + for arch in $arches; do + OUT+=("$(printf '{"dbtype":"%s","arch":"%s"}' "$variant" "$arch")") + done + ;; + *) + echo "$0: unknown view '$VIEW'" >&2 + exit 2 + ;; + esac +done + +case "$VIEW" in + json) + printf '[%s]\n' "$(IFS=,; echo "${OUT[*]}")" + ;; + list|repos) + printf '%s\n' "${OUT[@]}" + ;; + *) + echo "${OUT[*]}" + ;; +esac diff --git a/containers/ddev-dbserver/variants.txt b/containers/ddev-dbserver/variants.txt new file mode 100644 index 00000000000..7e845692b32 --- /dev/null +++ b/containers/ddev-dbserver/variants.txt @@ -0,0 +1,36 @@ +# The ddev-dbserver variant matrix - one line per variant: +# +# _ [ ...] +# +# Single source of truth, read through variants.sh in this directory. Order is +# significant: it's the order build targets are emitted in. +# +# Consumers: +# containers/ddev-dbserver/Makefile BUILD/SINGLE_ARCH/TEST targets +# containers/image-configs.sh automatic build/push flow +# containers/validate-image-repo.sh push allowlist +# .github/workflows/push-tagged-dbimage.yml manual push matrix +# +# Adding or removing a line changes this directory's content hash, so every +# variant is rebuilt and re-pushed under a new tag - which is what has to +# happen, since they all share BaseDBTag. +mysql_9.7 amd64 arm64 +mysql_8.4 amd64 arm64 +mysql_8.0 amd64 arm64 +mysql_5.7 amd64 arm64 +mysql_5.6 amd64 +mysql_5.5 amd64 +mariadb_12.3 amd64 arm64 +mariadb_11.8 amd64 arm64 +mariadb_11.4 amd64 arm64 +mariadb_10.11 amd64 arm64 +mariadb_10.8 amd64 arm64 +mariadb_10.7 amd64 arm64 +mariadb_10.6 amd64 arm64 +mariadb_10.5 amd64 arm64 +mariadb_10.4 amd64 arm64 +mariadb_10.3 amd64 arm64 +mariadb_10.2 amd64 arm64 +mariadb_10.1 amd64 arm64 +mariadb_10.0 amd64 +mariadb_5.5 amd64 diff --git a/containers/ddev-webserver/Makefile b/containers/ddev-webserver/Makefile index 0fdd0e56fd8..59d32075812 100644 --- a/containers/ddev-webserver/Makefile +++ b/containers/ddev-webserver/Makefile @@ -16,7 +16,7 @@ DOCKER_REPO ?= $(DOCKER_ORG)/ddev-webserver images: $(DEFAULT_IMAGES) $(DEFAULT_IMAGES): - docker build --label com.ddev.buildhost=${shell hostname} --label com.ddev.image-tag=$(DDEV_IMAGE_TAG) --target=$@ -t $(DOCKER_ORG)/$@:$(VERSION) $(DOCKER_ARGS) . + docker build --label com.ddev.buildhost=${shell hostname} $(shell ../image-metadata.sh labels $(DDEV_IMAGE_TAG) $@) --target=$@ -t $(DOCKER_ORG)/$@:$(VERSION) $(DOCKER_ARGS) . test: images diff --git a/containers/image-configs.sh b/containers/image-configs.sh new file mode 100644 index 00000000000..d08b6d70ea7 --- /dev/null +++ b/containers/image-configs.sh @@ -0,0 +1,50 @@ +#!/usr/bin/env bash +# image-configs.sh - the set of images the automatic build/push flow covers. +# +# Sourced (not executed) by wait-for-images.sh and by image-build-push.yml's +# detect job so the two can't drift apart. Keep in sync with the Makefile's +# autotag-images target. +# +# Fields, pipe-separated: +# repo_suffix Docker Hub repository under $DOCKER_ORG +# tag_var variable name in pkg/versionconstants/versionconstants.go +# hash_paths space-separated paths fed to hash-paths.sh +# make_dir directory under containers/ to run make in +# make_target make target that builds it +# arch_suffixed "true" if the target name takes an _ suffix +# arches space-separated architectures to build +# built_by_make "true" if `make` at the repo root builds it locally; +# a "false" image can only come from the registry +# extra_repo_suffixes further repositories the same target produces + +# shellcheck disable=SC2034 # consumed by whatever sources this file +DDEV_IMAGE_CONFIGS=( + 'ddev-webserver|WebTag|containers/ddev-webserver containers/containers_shared.mk|ddev-webserver|images|false|amd64 arm64|true|ddev-webserver-prod' + 'ddev-traefik-router|TraefikRouterTag|containers/ddev-traefik-router containers/containers_shared.mk|ddev-traefik-router|container|false|amd64 arm64|true|' + 'ddev-ssh-agent|SSHAuthTag|containers/ddev-ssh-agent containers/containers_shared.mk|ddev-ssh-agent|container|false|amd64 arm64|true|' + 'ddev-xhgui|XhguiTag|containers/ddev-xhgui containers/containers_shared.mk|ddev-xhgui|container|false|amd64 arm64|true|' +) + +# Every ddev-dbserver variant shares one BaseDBTag (see GetDBImage() in +# pkg/docker/images.go), so a change under containers/ddev-dbserver moves the +# tag for all of them at once and every variant has to be rebuilt and pushed +# together - otherwise TestDdevAllDatabases and every test pinning a +# non-default database pulls a tag nobody published. `make` builds only the +# default variant locally, to keep an unrelated rebuild cheap; the rest exist +# only in the registry. +# +# The variant list comes from containers/ddev-dbserver/variants.txt via +# variants.sh, shared with that directory's Makefile and with +# push-tagged-dbimage.yml. +DDEV_DBSERVER_HASH_PATHS='containers/ddev-dbserver containers/get_arch.sh' +DDEV_DBSERVER_DEFAULT_TARGET='mariadb_11.8' + +while IFS='|' read -r _target _arches; do + [ -z "$_target" ] && continue + _built_by_make=false + [ "$_target" = "$DDEV_DBSERVER_DEFAULT_TARGET" ] && _built_by_make=true + DDEV_IMAGE_CONFIGS+=( + "ddev-dbserver-${_target/_/-}|BaseDBTag|${DDEV_DBSERVER_HASH_PATHS}|ddev-dbserver|${_target}|true|${_arches}|${_built_by_make}|" + ) +done < <("$(dirname "${BASH_SOURCE[0]}")/ddev-dbserver/variants.sh" list) +unset _target _arches _built_by_make diff --git a/containers/image-metadata.sh b/containers/image-metadata.sh new file mode 100755 index 00000000000..cb8eac92b75 --- /dev/null +++ b/containers/image-metadata.sh @@ -0,0 +1,73 @@ +#!/usr/bin/env bash +# image-metadata.sh [] +# +# One definition of the descriptive metadata every DDEV image carries, in +# whichever form the caller needs: +# +# pairs key=value lines +# labels --label "key=value" ... for a docker/buildx build +# annotations --annotation "index:key=value" ... for `imagetools create` +# +# Labels and annotations are not interchangeable, which is why both exist: +# +# - Labels live in the image config, so they travel with a `docker pull` and +# `docker inspect` shows them offline. That is what a support report needs. +# - Annotations live on the manifest index, which is what a tag points at, so +# they describe the tag as a whole rather than one platform, and a registry +# can show them without pulling. A tag has no comment field; an index +# annotation is the closest standard equivalent. +# +# Env: +# DDEV_GIT_REVISION - commit to record (default: current HEAD) +# SOURCE_DATE_EPOCH - build timestamp override, for reproducibility + +set -eu -o pipefail + +if [ "$#" -lt 2 ]; then + echo "Usage: $0 <labels|annotations|pairs> <image-tag> [<title>]" >&2 + exit 2 +fi + +FORM="$1" +IMAGE_TAG="$2" +TITLE="${3:-}" + +REVISION="${DDEV_GIT_REVISION:-$(git rev-parse HEAD 2>/dev/null || echo unknown)}" +if [ -n "${SOURCE_DATE_EPOCH:-}" ]; then + CREATED="$(date -u -r "$SOURCE_DATE_EPOCH" +%Y-%m-%dT%H:%M:%SZ 2>/dev/null || date -u -d "@$SOURCE_DATE_EPOCH" +%Y-%m-%dT%H:%M:%SZ)" +else + CREATED="$(date -u +%Y-%m-%dT%H:%M:%SZ)" +fi + +PAIRS=( + "org.opencontainers.image.source=https://github.com/ddev/ddev" + "org.opencontainers.image.url=https://ddev.com" + "org.opencontainers.image.documentation=https://docs.ddev.com" + "org.opencontainers.image.vendor=DDEV" + "org.opencontainers.image.licenses=Apache-2.0" + "org.opencontainers.image.revision=${REVISION}" + "org.opencontainers.image.version=${IMAGE_TAG}" + "org.opencontainers.image.created=${CREATED}" + # The tag is a bare content hash, so this is the one field that says what + # the image is for without cross-referencing versionconstants.go. + "com.ddev.image-tag=${IMAGE_TAG}" +) +[ -n "$TITLE" ] && PAIRS+=("org.opencontainers.image.title=${TITLE}") + +case "$FORM" in + pairs) + printf '%s\n' "${PAIRS[@]}" + ;; + labels) + for p in "${PAIRS[@]}"; do printf -- '--label %q ' "$p"; done + echo + ;; + annotations) + for p in "${PAIRS[@]}"; do printf -- '--annotation %q ' "index:$p"; done + echo + ;; + *) + echo "$0: unknown form '$FORM'" >&2 + exit 2 + ;; +esac diff --git a/containers/image-tag-for.sh b/containers/image-tag-for.sh new file mode 100755 index 00000000000..0c5abb60077 --- /dev/null +++ b/containers/image-tag-for.sh @@ -0,0 +1,38 @@ +#!/usr/bin/env bash +# image-tag-for.sh <repo-suffix> +# +# Prints the tag this checkout needs for one image, by repository suffix +# (ddev-webserver, ddev-dbserver-mysql-8.0, ...). A thin lookup over +# image-configs.sh so the manual push workflows can default their tag input +# instead of asking a human to retype a content hash - the mismatch that +# causes is the whole reason wait-for-images.sh has to fail loudly. +# +# Exits non-zero for a repository the automatic flow doesn't cover +# (test-ssh-server, say), where there is no content hash to derive and the +# caller has to supply a tag. + +set -eu -o pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + +if [ "$#" -ne 1 ]; then + echo "Usage: $0 <repo-suffix>" >&2 + exit 2 +fi + +WANTED="$1" + +# shellcheck source=containers/image-configs.sh +source "$SCRIPT_DIR/image-configs.sh" + +for entry in "${DDEV_IMAGE_CONFIGS[@]}"; do + IFS='|' read -r repo_suffix tag_var hash_paths _ <<< "$entry" + [ "$repo_suffix" = "$WANTED" ] || continue + # shellcheck disable=SC2086 # hash_paths is a space-separated path list + read -r _ tag <<< "$("$SCRIPT_DIR/required-image-tag.sh" "$tag_var" $hash_paths)" + echo "$tag" + exit 0 +done + +echo "image-tag-for.sh: '${WANTED}' is not one of the content-addressed images; pass a tag explicitly" >&2 +exit 1 diff --git a/containers/image_readme_template.md b/containers/image_readme_template.md index 0c94919c9d9..6ce663c70b9 100644 --- a/containers/image_readme_template.md +++ b/containers/image_readme_template.md @@ -60,3 +60,6 @@ View [license information](https://github.com/ddev/ddev/blob/main/LICENSE) for t As with all Docker images, these likely also contain other software which may be under other licenses (such as Bash, etc from the base distribution, along with any direct or indirect dependencies of the primary software being contained). As for any pre-built image usage, it is the image user's responsibility to ensure that any use of this image complies with any relevant licenses for all software contained within. + +<!-- test: trivial fork containers/ change outside hash paths for #8609 phase 2 manual test 8 --> + diff --git a/containers/registry-tag-exists.sh b/containers/registry-tag-exists.sh new file mode 100755 index 00000000000..ca0c3bfcc5c --- /dev/null +++ b/containers/registry-tag-exists.sh @@ -0,0 +1,19 @@ +#!/usr/bin/env bash +# registry-tag-exists.sh <image-repo> <tag> +# +# Checks whether <image-repo>:<tag> already exists in the registry, without +# pulling it. Exit 0 if it exists, exit 1 if it doesn't (or the registry +# can't be reached). No local Docker daemon build/pull is triggered either +# way - this only talks to the registry. + +set -eu -o pipefail + +if [ "$#" -ne 2 ]; then + echo "Usage: $0 <image-repo> <tag>" >&2 + exit 2 +fi + +IMAGE_REPO="$1" +TAG="$2" + +docker buildx imagetools inspect "${IMAGE_REPO}:${TAG}" >/dev/null 2>&1 diff --git a/containers/registry_tag_exists_test.sh b/containers/registry_tag_exists_test.sh new file mode 100755 index 00000000000..dab6b7c28b4 --- /dev/null +++ b/containers/registry_tag_exists_test.sh @@ -0,0 +1,88 @@ +#!/usr/bin/env bash +# registry_tag_exists_test.sh - unit tests for registry-tag-exists.sh. +# +# Exercises the exists/doesn't-exist/unreachable outcomes against a stubbed +# `docker`, without talking to a real registry. +# Run with: +# containers/registry_tag_exists_test.sh + +set -eu -o pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +REGISTRY_TAG_EXISTS="$SCRIPT_DIR/registry-tag-exists.sh" + +FAILURES=0 + +fail() { + echo "FAIL: $1" >&2 + FAILURES=$((FAILURES + 1)) +} + +pass() { + echo "PASS: $1" +} + +WORKDIR="$(mktemp -d)" +trap 'rm -rf "$WORKDIR"' EXIT + +# --- Stub `docker`, controlled by a marker file listing which refs "exist". +BINDIR="$WORKDIR/bin" +mkdir -p "$BINDIR" +export DOCKER_EXISTING_REF_FILE="$WORKDIR/docker_existing_refs" +export DOCKER_CALL_LOG="$WORKDIR/docker_calls.log" +: > "$DOCKER_EXISTING_REF_FILE" +: > "$DOCKER_CALL_LOG" +cat > "$BINDIR/docker" <<'DOCKEREOF' +#!/usr/bin/env bash +set -eu -o pipefail +echo "$*" >> "$DOCKER_CALL_LOG" +if [ "$1" = "buildx" ] && [ "$2" = "imagetools" ] && [ "$3" = "inspect" ]; then + ref="$4" + grep -qxF "$ref" "$DOCKER_EXISTING_REF_FILE" + exit $? +fi +echo "docker stub: unexpected invocation: $*" >&2 +exit 1 +DOCKEREOF +chmod +x "$BINDIR/docker" +export PATH="$BINDIR:$PATH" + +# 1. Missing tag -> non-zero exit, no crash. +if "$REGISTRY_TAG_EXISTS" ddev/dummy-image missing-0123456789 >/dev/null 2>&1; then + fail "should report missing tag as not existing" +else + pass "reports missing tag as not existing" +fi + +# 2. Existing tag -> zero exit. +echo "ddev/dummy-image:present-0123456789" > "$DOCKER_EXISTING_REF_FILE" +if "$REGISTRY_TAG_EXISTS" ddev/dummy-image present-0123456789 >/dev/null 2>&1; then + pass "reports existing tag as existing" +else + fail "should report existing tag as existing" +fi + +# 3. Exactly one docker call per invocation - no retries/loops in this script +# (retry/backoff, if wanted, is the caller's job, e.g. wait-for-images.sh). +# BSD wc pads its output with spaces; GNU wc does not. +calls="$(wc -l < "$DOCKER_CALL_LOG" | tr -d '[:space:]')" +if [ "$calls" -eq 2 ]; then + pass "made exactly one docker call per invocation" +else + fail "expected 2 total docker calls across both invocations, got $calls" +fi + +# 4. Usage error on wrong argument count. +if "$REGISTRY_TAG_EXISTS" only-one-arg >/dev/null 2>&1; then + fail "should reject wrong argument count" +else + pass "rejects wrong argument count" +fi + +if [ "$FAILURES" -eq 0 ]; then + echo "All registry_tag_exists_test.sh checks passed." + exit 0 +else + echo "$FAILURES registry_tag_exists_test.sh check(s) failed." >&2 + exit 1 +fi diff --git a/containers/required-image-tag.sh b/containers/required-image-tag.sh new file mode 100755 index 00000000000..eab82c3d546 --- /dev/null +++ b/containers/required-image-tag.sh @@ -0,0 +1,49 @@ +#!/usr/bin/env bash +# required-image-tag.sh <TagVarName> <hash-path> [<hash-path> ...] +# +# Prints "<state> <tag>" for the image tag this checkout resolves to. The tag +# is the bare content hash either way - it does not depend on the branch, the +# fork, or the registry it was published to - so every caller agrees on it. +# The state says where that leaves the checkout: +# +# committed <tag> versionconstants.go already names this tag, so it is +# what ddev pulls and it has to exist in the registry. +# recomputed <tag> the content changed, so autotag.sh rewrites +# versionconstants.go to <tag> and, for an image `make` +# builds, produces it locally. +# +# Env: +# HASH_LEN - hash length in hex chars (default 10, must match +# hash-paths.sh) +# VERSIONCONSTANTS_FILE - path to versionconstants.go + +set -eu -o pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +REPO_ROOT="$(git rev-parse --show-toplevel)" +HASH_LEN="${HASH_LEN:-10}" +VERSIONCONSTANTS_FILE="${VERSIONCONSTANTS_FILE:-$REPO_ROOT/pkg/versionconstants/versionconstants.go}" + +if [ "$#" -lt 2 ]; then + echo "Usage: $0 <TagVarName> <hash-path> [<hash-path> ...]" >&2 + exit 2 +fi + +TAG_VAR="$1"; shift + +CURRENT_HASH="$(HASH_LEN="$HASH_LEN" "$SCRIPT_DIR/hash-paths.sh" "$@")" + +EXISTING_TAG="$(grep -E "^var ${TAG_VAR} = " "$VERSIONCONSTANTS_FILE" 2>/dev/null | sed -E "s/^var ${TAG_VAR} = \"([^\"]*)\".*/\\1/" || true)" +if [ -z "$EXISTING_TAG" ]; then + echo "required-image-tag.sh: could not find 'var ${TAG_VAR} = \"...\"' in $VERSIONCONSTANTS_FILE" >&2 + exit 1 +fi + +# Exact, not a trailing-hash match: a value still in the old <branch>-<hash> +# form is what ddev would pull, and it isn't this tag, so it counts as stale +# and `make` migrates the line. +if [ "$EXISTING_TAG" = "$CURRENT_HASH" ]; then + echo "committed ${CURRENT_HASH}" +else + echo "recomputed ${CURRENT_HASH}" +fi diff --git a/containers/required_image_tag_test.sh b/containers/required_image_tag_test.sh new file mode 100755 index 00000000000..50de5d875ca --- /dev/null +++ b/containers/required_image_tag_test.sh @@ -0,0 +1,168 @@ +#!/usr/bin/env bash +# required_image_tag_test.sh - unit tests for required-image-tag.sh. +# +# Runs against a throwaway versionconstants.go and this checkout's real +# content hashes. No Docker, no network. +# Run with: +# containers/required_image_tag_test.sh + +set -eu -o pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +REQUIRED_IMAGE_TAG="$SCRIPT_DIR/required-image-tag.sh" +HASH_PATHS="$SCRIPT_DIR/hash-paths.sh" + +FAILURES=0 + +fail() { + echo "FAIL: $1" >&2 + FAILURES=$((FAILURES + 1)) +} + +pass() { + echo "PASS: $1" +} + +assert_eq() { + local expected="$1" actual="$2" desc="$3" + if [ "$expected" = "$actual" ]; then + pass "$desc" + else + fail "$desc (expected '$expected', got '$actual')" + fi +} + +WORKDIR="$(mktemp -d)" +trap 'rm -rf "$WORKDIR"' EXIT + +export VERSIONCONSTANTS_FILE="$WORKDIR/versionconstants.go" +HASH_PATH_ARGS=(containers/ddev-xhgui containers/containers_shared.mk) +CURRENT_HASH="$("$HASH_PATHS" "${HASH_PATH_ARGS[@]}")" + +# 1. Committed value already the current hash -> committed, and the tag is +# that hash. This is the case that makes wait-for-images.sh wait. +echo "var XhguiTag = \"${CURRENT_HASH}\" // some_branch-${CURRENT_HASH}" > "$VERSIONCONSTANTS_FILE" +OUTPUT="$("$REQUIRED_IMAGE_TAG" XhguiTag "${HASH_PATH_ARGS[@]}")" +assert_eq "committed ${CURRENT_HASH}" "$OUTPUT" "reports committed when versionconstants.go already names this hash" + +# 2. Content no longer matches -> recomputed, same tag. The tag never depends +# on the branch, which is what lets detect, the runner, and `make` agree. +echo "var XhguiTag = \"0000000000\"" > "$VERSIONCONSTANTS_FILE" +OUTPUT="$("$REQUIRED_IMAGE_TAG" XhguiTag "${HASH_PATH_ARGS[@]}")" +assert_eq "recomputed ${CURRENT_HASH}" "$OUTPUT" "reports recomputed, still with the bare content hash" + +# 3. A value still in the old <branch>-<hash> form is what ddev would pull and +# it isn't this tag, so it counts as stale and `make` migrates the line. +echo "var XhguiTag = \"an_old_branch-${CURRENT_HASH}\"" > "$VERSIONCONSTANTS_FILE" +OUTPUT="$("$REQUIRED_IMAGE_TAG" XhguiTag "${HASH_PATH_ARGS[@]}")" +assert_eq "recomputed ${CURRENT_HASH}" "$OUTPUT" "treats a legacy branch-prefixed value as stale" + +# 4. The branch the caller happens to be on cannot change the answer. +echo "var XhguiTag = \"${CURRENT_HASH}\"" > "$VERSIONCONSTANTS_FILE" +assert_eq "$("$REQUIRED_IMAGE_TAG" XhguiTag "${HASH_PATH_ARGS[@]}")" \ + "$(REQUIRED_IMAGE_TAG_BRANCH='some/other branch' "$REQUIRED_IMAGE_TAG" XhguiTag "${HASH_PATH_ARGS[@]}")" \ + "the result is independent of the branch" + +if "$SCRIPT_DIR/validate-image-tag.sh" "$CURRENT_HASH" >/dev/null 2>&1; then + pass "the bare hash tag is one validate-image-tag.sh accepts" +else + fail "validate-image-tag.sh should accept the bare hash '$CURRENT_HASH'" +fi + +# 5. A missing tag variable is a hard error, not an empty tag. +echo "var SomethingElse = \"whatever\"" > "$VERSIONCONSTANTS_FILE" +OUTPUT="$("$REQUIRED_IMAGE_TAG" XhguiTag "${HASH_PATH_ARGS[@]}" 2>&1)" && RC=0 || RC=$? +if [ "$RC" -ne 0 ]; then + pass "errors out when the tag variable isn't in versionconstants.go" +else + fail "should error out when the tag variable isn't in versionconstants.go" +fi +case "$OUTPUT" in + *"could not find"*XhguiTag*) pass "missing-variable message names the variable" ;; + *) fail "missing-variable message should name the variable: $OUTPUT" ;; +esac + +# 6. Usage error on too few arguments. +if "$REQUIRED_IMAGE_TAG" XhguiTag >/dev/null 2>&1; then + fail "should reject a missing hash path" +else + pass "rejects a missing hash path" +fi + +# --- image-tag-for.sh, the lookup the manual push workflows use to default +# their tag input. Against the real image-configs.sh and this checkout. +unset VERSIONCONSTANTS_FILE +IMAGE_TAG_FOR="$SCRIPT_DIR/image-tag-for.sh" + +web_tag="$("$IMAGE_TAG_FOR" ddev-webserver)" +web_hash="$("$HASH_PATHS" containers/ddev-webserver containers/containers_shared.mk)" +case "$web_tag" in + *"$web_hash") pass "image-tag-for.sh resolves ddev-webserver to a tag with the current hash" ;; + *) fail "ddev-webserver tag '$web_tag' should end in '$web_hash'" ;; +esac + +# Every db variant shares BaseDBTag, so the lookup has to agree across them. +assert_eq "$("$IMAGE_TAG_FOR" ddev-dbserver-mariadb-11.8)" "$("$IMAGE_TAG_FOR" ddev-dbserver-mysql-8.0)" \ + "image-tag-for.sh gives every db variant the same tag" + +# An image the automatic flow doesn't cover has no hash to derive, so the +# caller has to be told rather than handed a wrong tag. +if "$IMAGE_TAG_FOR" test-ssh-server >/dev/null 2>&1; then + fail "should refuse an image that isn't content-addressed" +else + pass "refuses an image that isn't content-addressed" +fi +if "$IMAGE_TAG_FOR" no-such-image >/dev/null 2>&1; then + fail "should refuse an unknown image" +else + pass "refuses an unknown image" +fi + +# --- check-image-tags.sh, the `make staticrequired` gate. Driven with a +# throwaway versionconstants.go so it can be failed on purpose. +CHECK_IMAGE_TAGS="$SCRIPT_DIR/check-image-tags.sh" + +# shellcheck source=containers/image-configs.sh +source "$SCRIPT_DIR/image-configs.sh" + +CURRENT_FILE="$WORKDIR/current_versionconstants.go" +: > "$CURRENT_FILE" +seen="" +for entry in "${DDEV_IMAGE_CONFIGS[@]}"; do + IFS='|' read -r _ tag_var hash_paths _ <<< "$entry" + case " $seen " in *" $tag_var "*) continue ;; esac + seen="$seen $tag_var" + # shellcheck disable=SC2086 # hash_paths is a space-separated path list + echo "var ${tag_var} = \"$("$HASH_PATHS" $hash_paths)\"" >> "$CURRENT_FILE" +done + +if VERSIONCONSTANTS_FILE="$CURRENT_FILE" "$CHECK_IMAGE_TAGS" >/dev/null 2>&1; then + pass "check-image-tags.sh passes when every tag matches the content" +else + fail "check-image-tags.sh should pass when every tag matches the content" +fi + +STALE_FILE="$WORKDIR/stale_versionconstants.go" +sed 's/= "[0-9a-f]*"/= "0000000000"/' "$CURRENT_FILE" > "$STALE_FILE" +OUTPUT="$(VERSIONCONSTANTS_FILE="$STALE_FILE" "$CHECK_IMAGE_TAGS" 2>&1)" && RC=0 || RC=$? +if [ "$RC" -ne 0 ]; then + pass "check-image-tags.sh fails when versionconstants.go is stale" +else + fail "check-image-tags.sh should fail when versionconstants.go is stale" +fi +case "$OUTPUT" in + *"run 'make'"*) pass "the failure says how to fix it" ;; + *) fail "the failure should tell the contributor to run make: $OUTPUT" ;; +esac +case "$OUTPUT" in + *WebTag*BaseDBTag*|*BaseDBTag*WebTag*) pass "the failure names which tags are stale" ;; + *) fail "the failure should name the stale tags: $OUTPUT" ;; +esac + +if [ "$FAILURES" -eq 0 ]; then + echo "All required_image_tag_test.sh checks passed." + exit 0 +else + echo "$FAILURES required_image_tag_test.sh check(s) failed." >&2 + exit 1 +fi diff --git a/containers/validate-image-repo.sh b/containers/validate-image-repo.sh new file mode 100755 index 00000000000..a17ab7c4a35 --- /dev/null +++ b/containers/validate-image-repo.sh @@ -0,0 +1,55 @@ +#!/usr/bin/env bash +# validate-image-repo.sh <image-repo> +# +# Validates a `<org>/<name>` repository string before it's used in any `docker +# push`/`docker buildx imagetools create` command. The companion to +# validate-image-tag.sh: image-push.yml reads the repository list out of a +# build artifact produced by a job that may have run untrusted (fork PR) +# content, so without this a fork could name any repository the push +# credential can write to. +# +# Env: +# DOCKER_ORG - the only organization a push is allowed to target (required) + +set -eu -o pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + +ALLOWED_SUFFIXES=( + ddev-webserver + ddev-webserver-prod + ddev-traefik-router + ddev-ssh-agent + ddev-xhgui +) + +# The db repositories are exactly the variants in +# containers/ddev-dbserver/variants.txt - no pattern match, so a repository +# this project doesn't publish can't slip through on shape alone. +while IFS= read -r _repo; do + [ -n "$_repo" ] && ALLOWED_SUFFIXES+=("$_repo") +done < <("$SCRIPT_DIR/ddev-dbserver/variants.sh" repos) + +if [ "$#" -ne 1 ]; then + echo "Usage: $0 <image-repo>" >&2 + exit 2 +fi + +REPO="$1" +DOCKER_ORG="${DOCKER_ORG:?validate-image-repo.sh: DOCKER_ORG must be set}" + +if [ "${REPO%%/*}" != "$DOCKER_ORG" ] || [ "$REPO" = "${REPO#*/}" ]; then + echo "validate-image-repo.sh: '${REPO}' is not under the '${DOCKER_ORG}/' organization" >&2 + exit 1 +fi + +SUFFIX="${REPO#*/}" + +for allowed in "${ALLOWED_SUFFIXES[@]}"; do + if [ "$SUFFIX" = "$allowed" ]; then + exit 0 + fi +done + +echo "validate-image-repo.sh: '${REPO}' is not one of the repositories this flow may push" >&2 +exit 1 diff --git a/containers/validate-image-tag.sh b/containers/validate-image-tag.sh new file mode 100755 index 00000000000..40daac3cb40 --- /dev/null +++ b/containers/validate-image-tag.sh @@ -0,0 +1,69 @@ +#!/usr/bin/env bash +# validate-image-tag.sh <tag> +# +# Validates a content-addressed image tag before it's used in any `docker +# push`/`docker buildx imagetools create` command. This is the trusted-side +# check on a tag string that arrived via a build artifact from a job that +# may have run untrusted (fork PR) content - see image-push.yml. +# +# Two forms are accepted: +# <hash> the canonical tag, a bare content hash - what +# versionconstants.go holds and what ddev pulls +# <name>-<hash> the human-readable alias published alongside it, whose +# prefix is a sanitized branch name +# +# Requires: +# - strict charset, matching the same sanitization autotag.sh applies, and +# a leading character Docker actually accepts in a tag +# - must end in exactly HASH_LEN lowercase hex characters (the part +# tooling treats as authoritative) +# - neither the whole tag nor an alias prefix may be a reserved literal +# ("latest") or a release-tag shape (vX.Y.Z), so a fork that names its +# branch v1.25.0 can't publish something that reads as a release +# +# Env: +# HASH_LEN - hash length in hex chars (default 10, must match hash-paths.sh) + +set -eu -o pipefail + +HASH_LEN="${HASH_LEN:-10}" + +RESERVED_TAGS=(latest stable edge) + +if [ "$#" -ne 1 ]; then + echo "Usage: $0 <tag>" >&2 + exit 2 +fi + +TAG="$1" + +reject_reserved() { + local candidate="$1" what="$2" + for reserved in "${RESERVED_TAGS[@]}"; do + if [ "$candidate" = "$reserved" ]; then + echo "validate-image-tag.sh: '${TAG}' ${what} is the reserved tag '${reserved}'" >&2 + exit 1 + fi + done + if [[ "$candidate" =~ ^v[0-9]+\.[0-9]+\.[0-9]+$ ]]; then + echo "validate-image-tag.sh: '${TAG}' ${what} looks like a release tag, not a content-hash tag" >&2 + exit 1 + fi +} + +reject_reserved "$TAG" "is" + +if [[ "$TAG" =~ ^[0-9a-f]{${HASH_LEN}}$ ]]; then + exit 0 +fi + +if ! [[ "$TAG" =~ ^[A-Za-z0-9_][A-Za-z0-9_.-]*-[0-9a-f]{${HASH_LEN}}$ ]]; then + echo "validate-image-tag.sh: '${TAG}' is neither a ${HASH_LEN}-hex-char hash nor <name>-<hash>" >&2 + exit 1 +fi + +# Without this, "latest-0123456789" or "v1.2.3-0123456789" would sail through +# the format check above and land next to the real tags in the registry. +reject_reserved "${TAG%-*}" "starts with what" + +exit 0 diff --git a/containers/validate_image_repo_test.sh b/containers/validate_image_repo_test.sh new file mode 100755 index 00000000000..33e88a3b6b8 --- /dev/null +++ b/containers/validate_image_repo_test.sh @@ -0,0 +1,91 @@ +#!/usr/bin/env bash +# validate_image_repo_test.sh - unit tests for validate-image-repo.sh. +# +# Pure string checks, no external stubs needed. +# Run with: +# containers/validate_image_repo_test.sh + +set -eu -o pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +VALIDATE="$SCRIPT_DIR/validate-image-repo.sh" + +FAILURES=0 + +fail() { + echo "FAIL: $1" >&2 + FAILURES=$((FAILURES + 1)) +} + +pass() { + echo "PASS: $1" +} + +export DOCKER_ORG=ddev + +assert_valid() { + local repo="$1" + if "$VALIDATE" "$repo" >/dev/null 2>&1; then + pass "accepts '$repo'" + else + fail "should have accepted '$repo'" + fi +} + +assert_invalid() { + local repo="$1" desc="$2" + if "$VALIDATE" "$repo" >/dev/null 2>&1; then + fail "should have rejected $desc ('$repo')" + else + pass "rejects $desc ('$repo')" + fi +} + +assert_valid "ddev/ddev-webserver" +assert_valid "ddev/ddev-webserver-prod" +assert_valid "ddev/ddev-traefik-router" +assert_valid "ddev/ddev-ssh-agent" +assert_valid "ddev/ddev-xhgui" +assert_valid "ddev/ddev-dbserver-mariadb-11.8" +assert_valid "ddev/ddev-dbserver-mysql-8.0" + +assert_invalid "someoneelse/ddev-webserver" "another organization" +assert_invalid "ddev-webserver" "a bare name with no organization" +assert_invalid "ddev/ddev-webserver-evil" "an unknown repository in the right org" +assert_invalid "ddev/ddev-dbserver-postgres-16" "a db engine the flow doesn't build" +assert_invalid "ddev/ddev-dbserver-mariadb-99.9" "a correctly-shaped db variant that isn't in variants.txt" +assert_invalid "ddev/../../etc/passwd" "a path-traversal attempt" +assert_invalid "ddevhq/ddev-webserver" "an org that merely starts the same" + +# The db repositories are the variants themselves, so the allowlist tracks +# containers/ddev-dbserver/variants.txt rather than a name pattern. +while IFS= read -r suffix; do + [ -n "$suffix" ] && assert_valid "ddev/${suffix}" +done < <("$SCRIPT_DIR/ddev-dbserver/variants.sh" repos) + +# The org comes from a trusted workflow variable, so a differing DOCKER_ORG +# must move the whole allowlist rather than widen it. +if DOCKER_ORG=ddevhq "$VALIDATE" "ddevhq/ddev-webserver" >/dev/null 2>&1; then + pass "accepts the configured org when DOCKER_ORG differs" +else + fail "should accept the configured org when DOCKER_ORG differs" +fi +if DOCKER_ORG=ddevhq "$VALIDATE" "ddev/ddev-webserver" >/dev/null 2>&1; then + fail "should reject the default org when DOCKER_ORG points elsewhere" +else + pass "rejects the default org when DOCKER_ORG points elsewhere" +fi + +if env -u DOCKER_ORG "$VALIDATE" "ddev/ddev-webserver" >/dev/null 2>&1; then + fail "should refuse to run with DOCKER_ORG unset" +else + pass "refuses to run with DOCKER_ORG unset" +fi + +if [ "$FAILURES" -eq 0 ]; then + echo "All validate_image_repo_test.sh checks passed." + exit 0 +else + echo "$FAILURES validate_image_repo_test.sh check(s) failed." >&2 + exit 1 +fi diff --git a/containers/validate_image_tag_test.sh b/containers/validate_image_tag_test.sh new file mode 100755 index 00000000000..6311ce1a35f --- /dev/null +++ b/containers/validate_image_tag_test.sh @@ -0,0 +1,89 @@ +#!/usr/bin/env bash +# validate_image_tag_test.sh - unit tests for validate-image-tag.sh. +# +# Pure string-format checks, no external stubs needed. +# Run with: +# containers/validate_image_tag_test.sh + +set -eu -o pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +VALIDATE="$SCRIPT_DIR/validate-image-tag.sh" + +FAILURES=0 + +fail() { + echo "FAIL: $1" >&2 + FAILURES=$((FAILURES + 1)) +} + +pass() { + echo "PASS: $1" +} + +assert_valid() { + local tag="$1" + if "$VALIDATE" "$tag" >/dev/null 2>&1; then + pass "accepts valid tag '$tag'" + else + fail "should have accepted valid tag '$tag'" + fi +} + +assert_invalid() { + local tag="$1" desc="$2" + if "$VALIDATE" "$tag" >/dev/null 2>&1; then + fail "should have rejected $desc ('$tag')" + else + pass "rejects $desc ('$tag')" + fi +} + +assert_rejected_because() { + local tag="$1" needle="$2" desc="$3" + local output + if output="$("$VALIDATE" "$tag" 2>&1)"; then + fail "should have rejected $desc ('$tag')" + elif [[ "$output" == *"$needle"* ]]; then + pass "rejects $desc ('$tag') as expected" + else + fail "rejected $desc ('$tag') for the wrong reason: $output" + fi +} + +# The canonical form: a bare content hash, what versionconstants.go holds. +assert_valid "0123456789" +assert_valid "36bceca65e" + +# The readable alias published alongside it. +assert_valid "20260721_rfay_content_addressed_image_tags-36bceca65e" +assert_valid "main-0123456789" +assert_valid "v1.2.3-rc1-0123456789" + +assert_invalid "0123456789a" "an 11-hex-char bare tag" +assert_invalid "012345678" "a 9-hex-char bare tag" +assert_invalid "0123456789A" "a bare tag with an uppercase hex digit" + +assert_invalid "latest" "the reserved literal 'latest'" +assert_invalid "stable" "the reserved literal 'stable'" +assert_invalid "v1.2.3" "a bare release tag" +assert_invalid "latest-0123456789a" "a fake tag with an 11-char hash suffix" +assert_invalid "latest-012345678" "a fake tag with a 9-char hash suffix" +assert_invalid "no-hash-suffix" "a tag without a hex hash suffix" +assert_invalid "bad chars!-0123456789" "a tag with disallowed characters" +assert_invalid "UPPERHASH-0123456789AB" "a tag with an uppercase hash suffix" +assert_invalid "-leading-dash-0123456789" "a tag Docker would reject for its leading dash" +assert_invalid ".leading-dot-0123456789" "a tag Docker would reject for its leading dot" + +# A well-formed hash suffix must not be a way to smuggle a tag that reads as +# an official one; these are the checks a format-only validator would miss. +assert_rejected_because "latest-0123456789" "reserved tag" "'latest' dressed up with a hash suffix" +assert_rejected_because "v1.2.3-0123456789" "release tag" "a release tag dressed up with a hash suffix" + +if [ "$FAILURES" -eq 0 ]; then + echo "All validate_image_tag_test.sh checks passed." + exit 0 +else + echo "$FAILURES validate_image_tag_test.sh check(s) failed." >&2 + exit 1 +fi diff --git a/containers/wait-for-images.sh b/containers/wait-for-images.sh new file mode 100755 index 00000000000..15bff87b4cc --- /dev/null +++ b/containers/wait-for-images.sh @@ -0,0 +1,73 @@ +#!/usr/bin/env bash +# wait-for-images.sh +# +# Neither Buildkite nor the GitHub-hosted test-reusable.yml/ +# test-wsl2-reusable.yml runners hold image-push credentials, so any of them +# can race the image-push.yml GitHub Actions workflow: if this commit needs an +# image tag that only just got built, that tag might still be waiting on a +# maintainer's approval when this test run starts. Before running anything +# that pulls a DDEV image, poll the registry for the tags this checkout needs. +# +# Only the tags this checkout will actually *pull* are waited for, which +# required-image-tag.sh resolves the same way autotag.sh does: +# +# - hash unchanged -> versionconstants.go's committed tag is what gets +# pulled, so wait for exactly that, branch prefix and all. Recomputing a +# <this-branch>-<hash> tag here would wait forever on most pull requests, +# since nothing pushes a tag under a branch that didn't change the image. +# - hash changed -> `make` rebuilds the image locally on this runner, so +# there is nothing to wait for. +# +# Fast path (the common case - nothing changed): one registry check per image, +# no wait. +# +# Env: +# WAIT_FOR_IMAGES_ATTEMPTS - poll attempts before giving up (default 40) +# WAIT_FOR_IMAGES_SLEEP - seconds between attempts (default 30) +# +# Defaults give ~20 minutes - ddev-webserver alone takes ~6-8 minutes to build. + +set -eu -o pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +REGISTRY_TAG_EXISTS="$SCRIPT_DIR/registry-tag-exists.sh" +REQUIRED_IMAGE_TAG="$SCRIPT_DIR/required-image-tag.sh" +DOCKER_ORG="${DOCKER_ORG:-ddev}" + +ATTEMPTS="${WAIT_FOR_IMAGES_ATTEMPTS:-40}" +SLEEP_SECONDS="${WAIT_FOR_IMAGES_SLEEP:-30}" + +# shellcheck source=containers/image-configs.sh +source "$SCRIPT_DIR/image-configs.sh" + +for entry in "${DDEV_IMAGE_CONFIGS[@]}"; do + IFS='|' read -r repo_suffix tag_var hash_paths _ _ _ _ built_by_make _ <<< "$entry" + # shellcheck disable=SC2086 # hash_paths is a space-separated path list + read -r state tag <<< "$("$REQUIRED_IMAGE_TAG" "$tag_var" $hash_paths)" + image_repo="${DOCKER_ORG}/${repo_suffix}" + + # A changed image that `make` builds here needs no registry round trip. One + # it doesn't build - every ddev-dbserver variant but the default - still + # does, and the tag is the bare content hash, so it is the same string + # image-build-push.yml pushed regardless of branch. Wait for it. + if [ "$state" != "committed" ] && [ "$built_by_make" = "true" ]; then + echo "wait-for-images.sh: ${image_repo} content differs from versionconstants.go; make builds ${tag} locally, not waiting" + continue + fi + + attempt=1 + while true; do + if "$REGISTRY_TAG_EXISTS" "$image_repo" "$tag"; then + echo "wait-for-images.sh: found ${image_repo}:${tag}" + break + fi + if [ "$attempt" -ge "$ATTEMPTS" ]; then + echo "wait-for-images.sh: gave up waiting for ${image_repo}:${tag} after ${ATTEMPTS} attempts." >&2 + echo "wait-for-images.sh: has the maintainer approved the image-push run for this PR yet?" >&2 + exit 1 + fi + echo "wait-for-images.sh: ${image_repo}:${tag} not yet available, waiting... (attempt ${attempt}/${ATTEMPTS})" + sleep "$SLEEP_SECONDS" + attempt=$((attempt + 1)) + done +done diff --git a/containers/wait_for_images_test.sh b/containers/wait_for_images_test.sh new file mode 100755 index 00000000000..a313b6dd049 --- /dev/null +++ b/containers/wait_for_images_test.sh @@ -0,0 +1,266 @@ +#!/usr/bin/env bash +# wait_for_images_test.sh - unit tests for wait-for-images.sh. +# +# Exercises the fast-path/retry/give-up logic against a stubbed `docker` and a +# throwaway versionconstants.go, so the committed-vs-recomputed decision can be +# driven both ways without touching this checkout. No real registry, no real +# sleeps. +# Run with: +# containers/wait_for_images_test.sh + +set -eu -o pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +WAIT_FOR_IMAGES="$SCRIPT_DIR/wait-for-images.sh" +HASH_PATHS="$SCRIPT_DIR/hash-paths.sh" + +FAILURES=0 + +fail() { + echo "FAIL: $1" >&2 + FAILURES=$((FAILURES + 1)) +} + +pass() { + echo "PASS: $1" +} + +assert_eq() { + local expected="$1" actual="$2" desc="$3" + if [ "$expected" = "$actual" ]; then + pass "$desc" + else + fail "$desc (expected '$expected', got '$actual')" + fi +} + +# BSD wc pads its output with spaces; GNU wc does not. +count_lines() { + wc -l < "$1" | tr -d '[:space:]' +} + +WORKDIR="$(mktemp -d)" +trap 'rm -rf "$WORKDIR"' EXIT + +# --- Stub `docker`: a ref can be configured to "exist" outright, or to only +# start existing after N calls (via a counter file), so the eventually-recovers +# scenario is deterministic - no real sleeps or background processes needed. +BINDIR="$WORKDIR/bin" +mkdir -p "$BINDIR" +export DOCKER_EXISTING_REF_FILE="$WORKDIR/docker_existing_refs" +export DOCKER_DELAYED_REF_FILE="$WORKDIR/docker_delayed_ref" +export DOCKER_DELAYED_COUNTER_DIR="$WORKDIR/docker_delayed_counters" +export DOCKER_CALL_LOG="$WORKDIR/docker_calls.log" +mkdir -p "$DOCKER_DELAYED_COUNTER_DIR" +: > "$DOCKER_EXISTING_REF_FILE" +: > "$DOCKER_DELAYED_REF_FILE" +: > "$DOCKER_CALL_LOG" +cat > "$BINDIR/docker" <<'DOCKEREOF' +#!/usr/bin/env bash +set -eu -o pipefail +echo "$*" >> "$DOCKER_CALL_LOG" +if [ "$1" = "buildx" ] && [ "$2" = "imagetools" ] && [ "$3" = "inspect" ]; then + ref="$4" + if grep -qxF "$ref" "$DOCKER_EXISTING_REF_FILE"; then + exit 0 + fi + delayed_ref="$(cat "$DOCKER_DELAYED_REF_FILE" 2>/dev/null || true)" + if [ -n "$delayed_ref" ] && [ "$ref" = "$delayed_ref" ]; then + counter_file="$DOCKER_DELAYED_COUNTER_DIR/count" + count="$(cat "$counter_file" 2>/dev/null || echo 0)" + count=$((count + 1)) + echo "$count" > "$counter_file" + [ "$count" -ge 3 ] && exit 0 || exit 1 + fi + exit 1 +fi +echo "docker stub: unexpected invocation: $*" >&2 +exit 1 +DOCKEREOF +chmod +x "$BINDIR/docker" + +# --- Stub `sleep` so retry-budget tests run instantly and we can count waits. +export SLEEP_CALL_LOG="$WORKDIR/sleep_calls.log" +: > "$SLEEP_CALL_LOG" +cat > "$BINDIR/sleep" <<'SLEEPEOF' +#!/usr/bin/env bash +echo "$*" >> "$SLEEP_CALL_LOG" +SLEEPEOF +chmod +x "$BINDIR/sleep" + +export PATH="$BINDIR:$PATH" + +export DOCKER_ORG=ddevhq + +# shellcheck source=containers/image-configs.sh +source "$SCRIPT_DIR/image-configs.sh" + +# --- A throwaway versionconstants.go, so the tags waited for are whatever this +# test says they are rather than whatever the checkout happens to carry. +export VERSIONCONSTANTS_FILE="$WORKDIR/versionconstants.go" + +REPOS=() +TAG_VARS=() +declare -A HASH_BY_VAR +declare -A TAG_BY_VAR +for entry in "${DDEV_IMAGE_CONFIGS[@]}"; do + IFS='|' read -r repo_suffix tag_var hash_paths _ <<< "$entry" + if [ -z "${HASH_BY_VAR[$tag_var]:-}" ]; then + # shellcheck disable=SC2086 # hash_paths is a space-separated path list + HASH_BY_VAR["$tag_var"]="$("$HASH_PATHS" $hash_paths)" + TAG_BY_VAR["$tag_var"]="${HASH_BY_VAR[$tag_var]}" + fi + REPOS+=("ddevhq/${repo_suffix}") + TAG_VARS+=("$tag_var") +done +IMAGE_COUNT="${#REPOS[@]}" + +# All 20 ddev-dbserver variants share BaseDBTag, so the file has one line per +# distinct tag variable, not one per image. +write_versionconstants() { + : > "$VERSIONCONSTANTS_FILE" + for var in "${!TAG_BY_VAR[@]}"; do + echo "var ${var} = \"${TAG_BY_VAR[$var]}\"" >> "$VERSIONCONSTANTS_FILE" + done +} + +mark_all_existing() { + : > "$DOCKER_EXISTING_REF_FILE" + for i in "${!REPOS[@]}"; do + echo "${REPOS[$i]}:${TAG_BY_VAR[${TAG_VARS[$i]}]}" >> "$DOCKER_EXISTING_REF_FILE" + done +} + +write_versionconstants + +# 1. Fast path: every committed tag already exists -> one docker call per +# image, no sleep. This is the shape of every pull request that doesn't +# change a container image. +mark_all_existing +: > "$DOCKER_CALL_LOG" +: > "$SLEEP_CALL_LOG" +OUTPUT="$("$WAIT_FOR_IMAGES" 2>&1)" && RC=0 || RC=$? +if [ "$RC" -eq 0 ]; then + pass "fast path succeeds when every committed tag already exists" +else + fail "fast path should succeed when every committed tag already exists: $OUTPUT" +fi +assert_eq "$IMAGE_COUNT" "$(count_lines "$DOCKER_CALL_LOG")" "fast path makes exactly one docker call per image" +assert_eq "0" "$(count_lines "$SLEEP_CALL_LOG")" "fast path never sleeps" +case "$OUTPUT" in + *"found ${REPOS[0]}:${TAG_BY_VAR[WebTag]}"*) pass "prints confirmation for each found tag" ;; + *) fail "should print confirmation for each found tag: $OUTPUT" ;; +esac + +# 2. The regression that made every non-containers pull request hang was a tag +# that depended on the current branch. The tag is now the bare content hash, +# so the same checkout resolves to the same tag on any branch, in any fork. +case "$OUTPUT" in + *"found ${REPOS[0]}:${HASH_BY_VAR[WebTag]}"*) pass "waits for the bare content hash, with no branch prefix" ;; + *) fail "should wait for the bare hash '${HASH_BY_VAR[WebTag]}': $OUTPUT" ;; +esac +OUTPUT_OTHER_BRANCH="$(REQUIRED_IMAGE_TAG_BRANCH='some/other branch' "$WAIT_FOR_IMAGES" 2>&1)" +assert_eq "$OUTPUT" "$OUTPUT_OTHER_BRANCH" "the tags waited for don't depend on the branch" + +# 2b. Every ddev-dbserver variant is waited for, not just the default one that +# `make` builds locally - they all share BaseDBTag, so a dbserver change +# invalidates all of them at once. +for variant_repo in ddevhq/ddev-dbserver-mysql-8.0 ddevhq/ddev-dbserver-mariadb-10.11 ddevhq/ddev-dbserver-mariadb-5.5; do + case "$OUTPUT" in + *"found ${variant_repo}:"*) pass "waits for the non-default variant ${variant_repo}" ;; + *) fail "should wait for the non-default variant ${variant_repo}: $OUTPUT" ;; + esac +done + +# 3. Content that no longer matches versionconstants.go is built locally by +# make, so there is nothing to wait for and no registry call at all. +TAG_BY_VAR[WebTag]="0000000000" +write_versionconstants +mark_all_existing +: > "$DOCKER_CALL_LOG" +: > "$SLEEP_CALL_LOG" +OUTPUT="$("$WAIT_FOR_IMAGES" 2>&1)" && RC=0 || RC=$? +if [ "$RC" -eq 0 ]; then + pass "succeeds without waiting when content differs from versionconstants.go" +else + fail "should succeed when content differs from versionconstants.go: $OUTPUT" +fi +assert_eq "$(( IMAGE_COUNT - 1 ))" "$(count_lines "$DOCKER_CALL_LOG")" "skips the registry check for the locally-built image" +assert_eq "0" "$(count_lines "$SLEEP_CALL_LOG")" "never sleeps for a locally-built image" +case "$OUTPUT" in + *"not waiting"*) pass "says why it isn't waiting for the changed image" ;; + *) fail "should explain why it isn't waiting: $OUTPUT" ;; +esac +TAG_BY_VAR[WebTag]="${HASH_BY_VAR[WebTag]}" +write_versionconstants + +# 3b. A changed dbserver still has to wait for the 19 variants `make` doesn't +# build locally. The tag is the bare content hash, so it's the same string +# image-build-push.yml pushes and waiting for it is well-defined - only the +# default variant is skipped. +TAG_BY_VAR[BaseDBTag]="0000000000" +write_versionconstants +: > "$DOCKER_EXISTING_REF_FILE" +for i in "${!REPOS[@]}"; do + tag="${TAG_BY_VAR[${TAG_VARS[$i]}]}" + [ "${TAG_VARS[$i]}" = "BaseDBTag" ] && tag="${HASH_BY_VAR[BaseDBTag]}" + echo "${REPOS[$i]}:${tag}" >> "$DOCKER_EXISTING_REF_FILE" +done +: > "$DOCKER_CALL_LOG" +OUTPUT="$("$WAIT_FOR_IMAGES" 2>&1)" && RC=0 || RC=$? +if [ "$RC" -eq 0 ]; then + pass "waits for the db variants make doesn't build, rather than failing" +else + fail "should wait for the non-locally-built db variants: $OUTPUT" +fi +case "$OUTPUT" in + *"ddevhq/ddev-dbserver-mariadb-11.8 content differs"*) pass "skips only the default variant make builds locally" ;; + *) fail "should skip the locally-built default variant: $OUTPUT" ;; +esac +case "$OUTPUT" in + *"found ddevhq/ddev-dbserver-mysql-8.0:${HASH_BY_VAR[BaseDBTag]}"*) pass "waits for a non-default variant at the bare hash tag" ;; + *) fail "should wait for the non-default variants at the recomputed hash: $OUTPUT" ;; +esac +TAG_BY_VAR[BaseDBTag]="${HASH_BY_VAR[BaseDBTag]}" +write_versionconstants + +# 4. A tag that's initially missing but becomes available on the 3rd check. +LAST=$(( IMAGE_COUNT - 1 )) +: > "$DOCKER_EXISTING_REF_FILE" +for i in "${!REPOS[@]}"; do + [ "$i" -eq "$LAST" ] && continue + echo "${REPOS[$i]}:${TAG_BY_VAR[${TAG_VARS[$i]}]}" >> "$DOCKER_EXISTING_REF_FILE" +done +echo "${REPOS[$LAST]}:${TAG_BY_VAR[${TAG_VARS[$LAST]}]}" > "$DOCKER_DELAYED_REF_FILE" +rm -f "$DOCKER_DELAYED_COUNTER_DIR/count" +: > "$SLEEP_CALL_LOG" +if WAIT_FOR_IMAGES_ATTEMPTS=5 WAIT_FOR_IMAGES_SLEEP=0 "$WAIT_FOR_IMAGES" >/dev/null 2>&1; then + pass "recovers once a previously-missing tag appears within the attempt budget" +else + fail "should recover once a previously-missing tag appears within the attempt budget" +fi +assert_eq "2" "$(count_lines "$SLEEP_CALL_LOG")" "sleeps twice while waiting for the tag to become available on the 3rd check" +: > "$DOCKER_DELAYED_REF_FILE" + +# 5. Gives up cleanly after exhausting the attempt budget, with a clear message. +: > "$DOCKER_EXISTING_REF_FILE" +: > "$SLEEP_CALL_LOG" +OUTPUT="$(WAIT_FOR_IMAGES_ATTEMPTS=3 WAIT_FOR_IMAGES_SLEEP=0 "$WAIT_FOR_IMAGES" 2>&1)" && RC=0 || RC=$? +if [ "$RC" -ne 0 ]; then + pass "gives up (non-zero exit) once the attempt budget is exhausted" +else + fail "should give up (non-zero exit) once the attempt budget is exhausted" +fi +case "$OUTPUT" in + *"gave up waiting"*"has the maintainer approved"*) pass "give-up message is actionable" ;; + *) fail "give-up message should mention giving up and approval: $OUTPUT" ;; +esac +assert_eq "2" "$(count_lines "$SLEEP_CALL_LOG")" "sleeps exactly (attempts - 1) times before giving up on the first (unavailable) image" + +if [ "$FAILURES" -eq 0 ]; then + echo "All wait_for_images_test.sh checks passed." + exit 0 +else + echo "$FAILURES wait_for_images_test.sh check(s) failed." >&2 + exit 1 +fi diff --git a/docs/content/developers/building-contributing.md b/docs/content/developers/building-contributing.md index 7ac7670ece0..a0c4be5d75a 100644 --- a/docs/content/developers/building-contributing.md +++ b/docs/content/developers/building-contributing.md @@ -155,6 +155,8 @@ make push VERSION=<tag> DOCKER_REPO=your/dockerrepo ### Pushes Using GitHub Actions +Normally the [Image build](https://github.com/ddev/ddev/actions/workflows/image-build-push.yml) workflow (see [Automatic Image Build and Push](#automatic-image-build-and-push)) handles pushing a changed image automatically for any pull request, including forks. The workflows below are for manually pushing a specific tag — a re-push, or one of the `ddev-dbserver` variants other than the default `mariadb_11.8` that `make` auto-builds. + To manually push using GitHub Actions, #### For Most Images @@ -308,7 +310,24 @@ The Docker images that DDEV uses are included in the `containers/` directory: * `containers/ddev-traefik-router` is the current Traefik-based router image. * `containers/ddev-xhgui` provides a web interface to analyze performance profiles generated by xhprof. -When you change an image, running `make` from the repository root builds it locally and computes/updates its tag in `pkg/versionconstants/versionconstants.go` automatically — no manual tag-inventing or file-editing needed, and this works for any contributor, including from a fork. Getting that image into CI (a multi-arch push to the registry) still requires registry credentials that forks don't have, so please ask a maintainer if you need a container pushed to support a pull request. +When you change an image, running `make` from the repository root builds it locally and computes/updates its tag in `pkg/versionconstants/versionconstants.go` automatically — no manual tag-inventing or file-editing needed, and this works for any contributor, including from a fork. Once you push that commit as a pull request, the [Image build](https://github.com/ddev/ddev/actions/workflows/image-build-push.yml) workflow detects the changed image and builds and pushes it to the registry automatically — no maintainer needs to run anything by hand. See [Automatic Image Build and Push](#automatic-image-build-and-push) below for when that requires a maintainer's approval and when it doesn't. + +### Automatic Image Build and Push + +An image's tag is the bare content hash of the files it's built from — `ddev/ddev-webserver:36bceca65e`, with no branch prefix. The same content therefore resolves to the same tag no matter which branch, fork, or Docker Hub organization published it, which is what lets `make`, the CI detector, and every test runner agree without coordinating. `versionconstants.go` records the branch alongside it, both as a trailing comment and as a `WebTagBranch`-style variable that `ddev version` shows, and each push also publishes a readable `<branch>-<hash>` alias pointing at the same manifest. + +Opening a pull request that touches `containers/` triggers the [Image build](https://github.com/ddev/ddev/actions/workflows/image-build-push.yml) workflow. A `detect` job always runs first: it recomputes each image's content hash and checks whether that tag is already in the registry. If it is, there's nothing to build — so a pull request that touches `containers/` without changing an image costs one registry lookup per image and no build. + +The same resolution drives `containers/wait-for-images.sh`, which every test runner calls before pulling anything: it waits only for tags this commit will genuinely pull, and doesn't wait at all for an image whose content changed, since `make` builds that one locally on the runner. + +Changing `containers/ddev-dbserver` is the one case where `make` alone isn't enough. Every database variant shares a single `BaseDBTag`, so a dbserver change moves the tag for all 20 of them at once, while `make` only builds the default `mariadb_11.8` locally — the other 19 can only come from the registry. CI builds and pushes the whole matrix (36 jobs, from `containers/ddev-dbserver/variants.txt`), so **run `make` and commit the `versionconstants.go` change**; `wait-for-images.sh` stops the test run with that instruction if you don't. + +What happens next depends on whether the PR is from a fork: + +* **Fork PRs** (security boundary — the PR could contain an arbitrary Dockerfile/build script): a `build` job builds the image(s) per architecture with no registry credentials at all — nothing in that job can reach `docker.io`, so there's nothing to gain by gating it before it runs. Once it finishes, a separate, trusted `image-push.yml` workflow — which never checks out or runs the pull request's code — loads what it produced and pushes it, gated behind a maintainer's approval on the `image-push` environment. Both the tag and every repository name in that artifact are re-validated first (`containers/validate-image-tag.sh`, `containers/validate-image-repo.sh`), so an approval can only ever publish a hash-shaped tag under a known DDEV repository. A comment is posted on the PR once the push completes. If the build produced nothing to push, no approval is requested at all. +* **Everything else** (a push to `main`, or a pull request from a branch in the same repository — no fork content is ever involved): `build-and-push` builds and pushes directly in one step, with no approval gate at all — the same trust level `main-build.yml` already runs at unguarded. A `create-manifests` job then assembles the multi-arch manifest and comments on the PR, if there is one. + +So a maintainer only ever needs to click **Approve** once — for a fork PR's push step — and only when the PR actually changed a container image; everything else is fully automatic. ## Pull Requests diff --git a/docs/content/developers/release-management.md b/docs/content/developers/release-management.md index 5d08677508a..4c96adcc0de 100644 --- a/docs/content/developers/release-management.md +++ b/docs/content/developers/release-management.md @@ -74,9 +74,33 @@ The following “Repository secret” environment variables must be configured i 2. Make sure you're about to create the right release tag. 3. Use the “Auto-generate release notes” option to get the commit list, then edit to add all the other necessary info. +## Automatic Image Build and Push + +Any pull request that changes `containers/` — including from a fork — is built and pushed automatically by the [Image build](https://github.com/ddev/ddev/blob/main/.github/workflows/image-build-push.yml) / [Image push](https://github.com/ddev/ddev/blob/main/.github/workflows/image-push.yml) workflow pair. See [Automatic Image Build and Push](building-contributing.md#automatic-image-build-and-push) in the contributing guide for how the flow works and why it's safe to run on fork-authored Dockerfiles. + +The two workflows below (manual `workflow_dispatch`) remain for re-pushing a specific tag — for instance at release time, when every image is re-pushed under a `vX.Y.Z` tag. Their `tag` input is optional: leave it empty and the workflow uses the tag the checkout actually needs, which is safer than retyping a content hash. Supply it only for a release. + +When the tag is a content hash, these workflows also publish the `<branch>-<hash>` alias next to it — the same second name the automatic flow creates — so a manual push doesn't leave a bare hash with nothing readable beside it in the registry. A `vX.Y.Z` tag is already readable and gets no alias. + +A `containers/ddev-dbserver` change builds and pushes all 20 database variants (36 jobs), because they all share a single `BaseDBTag`. That variant matrix lives in `containers/ddev-dbserver/variants.txt` and is read by `variants.sh`, which also generates that directory's make targets, the automatic flow's image list, and `push-tagged-dbimage.yml`'s matrix — add a database version there and every consumer picks it up. + +### One-time setup: the `image-push` GitHub Environment + +Fork PRs build with no registry credentials at all (nothing to gain by gating that step), then go through a single approval before the built image is actually pushed, gated by the `image-push` GitHub Environment (Settings → Environments): + +1. Create the environment `image-push`. +2. Add required reviewers (the maintainers/dev team). +3. Add `PUSH_SERVICE_ACCOUNT_TOKEN` as a secret **on this environment** (Settings → Environments → `image-push` → Secrets), using the same 1Password service-account token value already used elsewhere in this doc. It currently exists only as a repository secret; duplicating (or moving) it onto the `image-push` environment is what scopes `DOCKERHUB_TOKEN` access to only the approved `image-push.yml` job. + +This approval only applies to fork PRs. A push to `main` or a same-repo PR builds and pushes without any approval at all, using the repository-level `PUSH_SERVICE_ACCOUNT_TOKEN` secret directly (that path never declares `environment:` on its jobs, so this environment's protection rules don't apply to it). + +When testing this on `ddev-test/ddev`, do the same steps there first, and confirm `vars.DOCKER_ORG` on that repository points at the DockerHub org used for testing. + +Since a job referencing an environment that doesn't exist yet gets auto-created with no protection rules (silently *not* gating), verify the environment actually has a `required_reviewers` rule before relying on it, e.g. `gh api repos/<owner>/<repo>/environments/image-push`. + ## Pushing Docker Images with the GitHub Actions Workflow -The easiest way to push Docker images is to use the GitHub Actions workflow, especially if the code for the image is already in the [ddev/ddev](https://github.com/ddev/ddev) repository. +The easiest way to push Docker images is to use the GitHub Actions workflow, especially if the code for the image is already in the [ddev/ddev](https://github.com/ddev/ddev) repository. For a normal container change on a pull request, you shouldn't need this — see [Automatic Image Build and Push](#automatic-image-build-and-push) above. ### Actual release creation @@ -88,7 +112,7 @@ You can push all images besides `ddev-dbserver` at <https://github.com/ddev/ddev You can push `ddev-dbserver` images at <https://github.com/ddev/ddev/actions/workflows/push-tagged-dbimage.yml> -If you need to push from a forked PR, you’ll have to do this from your fork (for example, `https://github.com/rfay/ddev/actions/workflows/push-tagged-image.yml`), and you’ll have to specify the branch on the fork. This requires setting the `DOCKERHUB_TOKEN` and `DOCKERHUB_USERNAME` secrets on the forked PR, for example `https://github.com/rfay/ddev/settings/secrets/actions`. You can do the same with `ddev-dbserver` at `https://github.com/rfay/ddev/actions/workflows/push-tagged-dbimage.yml` for example. +A forked PR that changes a container image no longer needs any of this — see [Automatic Image Build and Push](#automatic-image-build-and-push) above. The fork-your-own-secrets workaround described in earlier versions of this doc is superseded by that flow. * Visit `https://github.com/ddev/ddev/actions/workflows/push-tagged-image.yml`. * Click the “Push tagged image” workflow on the left side of the page. diff --git a/pkg/version/version.go b/pkg/version/version.go index d174f563a29..2494cc0b1ff 100644 --- a/pkg/version/version.go +++ b/pkg/version/version.go @@ -66,10 +66,38 @@ func GetVersionInfo() (map[string]string, error) { } versionInfo["mutagen"] = versionconstants.RequiredMutagenVersion versionInfo["xhgui-image"] = docker.GetXhguiImage() + versionInfo["image-tag-branches"] = imageTagBranches() return versionInfo, retErr } +// imageTagBranches describes which branch each image tag was built from. The +// tags themselves are bare content hashes, so without this there is nothing in +// `ddev version` to say where an image came from. Collapses to a single branch +// name in the usual case where every image was built from the same one. +func imageTagBranches() string { + branches := []struct{ image, branch string }{ + {"web", versionconstants.WebTagBranch}, + {"db", versionconstants.BaseDBTagBranch}, + {"router", versionconstants.TraefikRouterTagBranch}, + {"ddev-ssh-agent", versionconstants.SSHAuthTagBranch}, + {"xhgui", versionconstants.XhguiTagBranch}, + } + + parts := make([]string, 0, len(branches)) + allSame := true + for _, b := range branches { + if b.branch != branches[0].branch { + allSame = false + } + parts = append(parts, b.image+"="+b.branch) + } + if allSame { + return branches[0].branch + } + return strings.Join(parts, " ") +} + // GetDockerPlatform gets the platform used for Docker engine func GetDockerPlatform() (string, error) { info, err := dockerutil.GetDockerClientInfo() diff --git a/pkg/versionconstants/versionconstants.go b/pkg/versionconstants/versionconstants.go index 69f3d6d4869..b08ff2907ef 100644 --- a/pkg/versionconstants/versionconstants.go +++ b/pkg/versionconstants/versionconstants.go @@ -16,35 +16,58 @@ var DdevVersion = "" // Note that this is overridden by make // Compiled with link-time variables var AmplitudeAPIKey = "" +// Image tags are bare content hashes (see containers/hash-paths.sh), so the +// same image resolves to the same tag no matter which branch or repository +// published it. Each has a companion <Name>Branch naming the branch its +// content was built from - the readable hint a bare hash costs, shown by +// `ddev version` and republished as a <branch>-<hash> alias in the registry. +// All of these lines are maintained by containers/autotag.sh; running `make` +// updates them. + // WebImg defines the default web image used for applications. var WebImg = "ddev/ddev-webserver" // WebTag defines the default web image tag -var WebTag = "20260721_rfay_content_addressed_image_tags-36bceca65e" // Note that this can be overridden by make +var WebTag = "c202e92108" // 20260814_rfay_docker_update_phase_2-c202e92108 + +// WebTagBranch is the branch WebTag's content was built from. +var WebTagBranch = "20260814_rfay_docker_update_phase_2" // DBImg defines the default db image used for applications. var DBImg = "ddev/ddev-dbserver" // BaseDBTag is the main tag, DBTag is constructed from it -var BaseDBTag = "20260721_rfay_content_addressed_image_tags-278125d33a" +var BaseDBTag = "5a7c45ddbf" // 20260814_rfay_docker_update_phase_2-5a7c45ddbf + +// BaseDBTagBranch is the branch BaseDBTag's content was built from. +var BaseDBTagBranch = "20260814_rfay_docker_update_phase_2" // TraefikRouterImage is image for router var TraefikRouterImage = "ddev/ddev-traefik-router" // TraefikRouterTag is traefik router tag -var TraefikRouterTag = "20260721_rfay_content_addressed_image_tags-c96123b524" +var TraefikRouterTag = "bffcda31c5" // 20260814_rfay_docker_update_phase_2-bffcda31c5 + +// TraefikRouterTagBranch is the branch TraefikRouterTag's content was built from. +var TraefikRouterTagBranch = "20260814_rfay_docker_update_phase_2" // SSHAuthImage is image for agent var SSHAuthImage = "ddev/ddev-ssh-agent" // SSHAuthTag is ssh-agent auth tag -var SSHAuthTag = "20260721_rfay_content_addressed_image_tags-8e8bf1217c" +var SSHAuthTag = "bb5e9f0003" // 20260814_rfay_docker_update_phase_2-bb5e9f0003 + +// SSHAuthTagBranch is the branch SSHAuthTag's content was built from. +var SSHAuthTagBranch = "20260814_rfay_docker_update_phase_2" // XhguiImage is image for xhgui var XhguiImage = "ddev/ddev-xhgui" // XhguiTag is xhgui tag -var XhguiTag = "20260721_rfay_content_addressed_image_tags-f046b66382" +var XhguiTag = "8757c1e92a" // 20260814_rfay_docker_update_phase_2-8757c1e92a + +// XhguiTagBranch is the branch XhguiTag's content was built from. +var XhguiTagBranch = "20260814_rfay_docker_update_phase_2" // UtilitiesImage is used in bash scripts var UtilitiesImage = "ddev/ddev-utilities:latest"