diff --git a/.buildkite/perf.sh b/.buildkite/perf.sh index b0c1da3b179..d624a13f463 100755 --- a/.buildkite/perf.sh +++ b/.buildkite/perf.sh @@ -19,6 +19,9 @@ if [[ ${BUILDKITE_MESSAGE:-} == *"[skip buildkite]"* ]] || [[ ${BUILDKITE_MESSAG exit 0 fi +# A changed image may still be waiting on image-push.yml's approval +"$(dirname "$0")/../containers/wait-for-images.sh" + os=$(go env GOOS) # shellcheck source=lib-provider.sh diff --git a/.buildkite/test.sh b/.buildkite/test.sh index 8e024776527..3adbc442073 100755 --- a/.buildkite/test.sh +++ b/.buildkite/test.sh @@ -25,6 +25,9 @@ while IFS= read -r varname; do done < <(MSYS_NO_PATHCONV=1 git ls-tree --name-only refs/public-variables-tmp:.github/public-variables/) git update-ref -d refs/public-variables-tmp +# A changed image may still be waiting on image-push.yml's approval +"$(dirname "$0")/../containers/wait-for-images.sh" + export PATH=$PATH:/home/linuxbrew/.linuxbrew/bin os=$(go env GOOS) diff --git a/.github/workflows/container-tests.yml b/.github/workflows/container-tests.yml index 4b2cbe866a4..88a78074717 100644 --- a/.github/workflows/container-tests.yml +++ b/.github/workflows/container-tests.yml @@ -47,6 +47,18 @@ jobs: - uses: actions/checkout@v7 - name: Run containers/autotag_test.sh run: containers/autotag_test.sh + - name: Run containers/db_variants_test.sh + run: containers/db_variants_test.sh + - name: Run containers/required_image_tag_test.sh + run: containers/required_image_tag_test.sh + - name: Run containers/registry_tag_exists_test.sh + run: containers/registry_tag_exists_test.sh + - name: Run containers/validate_image_tag_test.sh + run: containers/validate_image_tag_test.sh + - name: Run containers/validate_image_repo_test.sh + run: containers/validate_image_repo_test.sh + - name: Run containers/wait_for_images_test.sh + run: containers/wait_for_images_test.sh container-build-and-test: name: ${{ matrix.os }} - Test container ${{ matrix.containers }} diff --git a/.github/workflows/image-build-push.yml b/.github/workflows/image-build-push.yml index 2ddc72090ea..f8248013e81 100644 --- a/.github/workflows/image-build-push.yml +++ b/.github/workflows/image-build-push.yml @@ -3,13 +3,18 @@ defaults: run: shell: bash -# Placeholder for #8609 phase 2 (fork-safe automatic image build/push). This -# do-nothing stub exists only so the workflow name/triggers are registered on -# the default branch before the real detect/approval/build logic lands - see -# the phase 2 PR, which will replace this file's contents via rebase. -# image-push.yml's `workflow_run` trigger only fires for a listener that -# exists on the default branch, so this stub unblocks testing that dependency -# ahead of the full PR. +# In a forked PR, this workflow may run a fork's own Dockerfile/build scripts, +# so the `build` job never references a secret - the trusted side (loading +# the artifact `build` produces and actually pushing it) lives in +# image-push.yml, triggered via workflow_run once this workflow completes, +# gated behind its own approval. +# +# For anything else (a push, or a same-repo PR - `detect`'s `is_fork` output, +# using the same fork-check idiom as push-tagged-image.yml), there's nothing +# a maintainer's own branch could smuggle into a job that every other +# secret-using workflow here doesn't already run unguarded, so +# `build-and-push` builds and pushes directly, with no environment/approval +# gate at all - same trust level as `main-build.yml`. on: pull_request: branches: [main] @@ -22,14 +27,305 @@ on: - "containers/**" - ".github/workflows/image-build-push.yml" +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +env: + DOCKER_ORG: "${{ vars.DOCKER_ORG || 'ddev' }}" + permissions: contents: read jobs: - placeholder: - name: "Placeholder (see #8609)" + detect: + name: Detect changed images + runs-on: ubuntu-24.04 + outputs: + build_matrix: ${{ steps.detect.outputs.build_matrix }} + manifest_matrix: ${{ steps.detect.outputs.manifest_matrix }} + needs_build: ${{ steps.detect.outputs.needs_build }} + is_fork: ${{ steps.fork.outputs.is_fork }} + steps: + - uses: actions/checkout@v7 + - name: Compute per-image build status + id: detect + # github.head_ref is attacker-controlled (a fork may name its branch + # anything git accepts, quotes and backticks included), so it reaches + # the script as data rather than as spliced-in source. + env: + REQUIRED_IMAGE_TAG_BRANCH: ${{ github.head_ref || github.ref_name }} + run: | + set -eu -o pipefail + source containers/image-configs.sh + + # build_matrix is one entry per (image, arch) rather than a cross + # product, because the older db variants are amd64-only. + BUILD_JSON="[]" + MANIFEST_JSON="[]" + for entry in "${DDEV_IMAGE_CONFIGS[@]}"; do + IFS='|' read -r repo_suffix tag_var hash_paths make_dir make_target arch_suffixed arches _ extra_repo_suffixes <<< "$entry" + # shellcheck disable=SC2086 # hash_paths is a space-separated path list + read -r state tag <<< "$(containers/required-image-tag.sh "$tag_var" $hash_paths)" + repo="${DOCKER_ORG}/${repo_suffix}" + if containers/registry-tag-exists.sh "$repo" "$tag"; then + echo "detect: ${repo}:${tag} already exists (${state}), nothing to build" + continue + fi + echo "detect: ${repo}:${tag} needs building (${state}) for arches: ${arches}" + for arch in $arches; do + BUILD_JSON="$(echo "$BUILD_JSON" | jq -c \ + --arg repo_suffix "$repo_suffix" \ + --arg repo "$repo" \ + --arg tag "$tag" \ + --arg arch "$arch" \ + --arg make_dir "$make_dir" \ + --arg make_target "$make_target" \ + --arg arch_suffixed "$arch_suffixed" \ + --arg extra_repo_suffixes "$extra_repo_suffixes" \ + '. + [{"repo_suffix": $repo_suffix, "repo": $repo, "tag": $tag, "arch": $arch, "make_dir": $make_dir, "make_target": $make_target, "arch_suffixed": $arch_suffixed, "extra_repo_suffixes": $extra_repo_suffixes}]')" + done + MANIFEST_JSON="$(echo "$MANIFEST_JSON" | jq -c \ + --arg repo "$repo" \ + --arg tag "$tag" \ + --arg arches "$arches" \ + --arg extra_repo_suffixes "$extra_repo_suffixes" \ + '. + [{"repo": $repo, "tag": $tag, "arches": $arches, "extra_repo_suffixes": $extra_repo_suffixes}]')" + done + + echo "build_matrix=${BUILD_JSON}" >> "$GITHUB_OUTPUT" + echo "manifest_matrix=${MANIFEST_JSON}" >> "$GITHUB_OUTPUT" + echo "detect: $(echo "$MANIFEST_JSON" | jq 'length') image(s) to build, $(echo "$BUILD_JSON" | jq 'length') build job(s)" + if [ "$(echo "$BUILD_JSON" | jq 'length')" -gt 0 ]; then + echo "needs_build=true" >> "$GITHUB_OUTPUT" + else + echo "needs_build=false" >> "$GITHUB_OUTPUT" + fi + + # Separate from the step above so nothing the per-image loop does can + # reach the output that decides whether the push secret gets loaded. + - name: Determine whether this is a fork + id: fork + env: + EVENT_NAME: ${{ github.event_name }} + HEAD_OWNER: ${{ github.event.pull_request.head.repo.owner.login }} + BASE_OWNER: ${{ github.repository_owner }} + run: | + set -eu -o pipefail + if [ "$EVENT_NAME" = "pull_request" ] && [ "$HEAD_OWNER" != "$BASE_OWNER" ]; then + echo "is_fork=true" >> "$GITHUB_OUTPUT" + else + echo "is_fork=false" >> "$GITHUB_OUTPUT" + fi + + # --- Fork PRs: build with no secrets (this job never has registry + # credentials, so there's nothing to gain by gating it - see #8609 + # discussion), then hand off to image-push.yml for the trusted, + # approval-gated push. --- + + build: + name: Build ${{ matrix.build.repo }} (${{ matrix.build.arch }}) + needs: detect + if: needs.detect.outputs.needs_build == 'true' && needs.detect.outputs.is_fork == 'true' + strategy: + fail-fast: false + matrix: + build: ${{ fromJson(needs.detect.outputs.build_matrix) }} + runs-on: ${{ matrix.build.arch == 'arm64' && 'ubuntu-24.04-arm' || 'ubuntu-24.04' }} + permissions: + contents: read + steps: + - uses: actions/checkout@v7 + - name: Build ${{ matrix.build.repo }}:${{ matrix.build.tag }}-${{ matrix.build.arch }} + run: | + set -eu -o pipefail + VERSION="${{ matrix.build.tag }}-${{ matrix.build.arch }}" + MAKE_TARGET="${{ matrix.build.make_target }}" + if [ "${{ matrix.build.arch_suffixed }}" = "true" ]; then + MAKE_TARGET="${MAKE_TARGET}_${{ matrix.build.arch }}" + fi + # DDEV_IMAGE_TAG is the tag people actually pull, recorded in the + # com.ddev.image-tag label; VERSION is the throwaway per-arch tag. + make -C "containers/${{ matrix.build.make_dir }}" "$MAKE_TARGET" \ + VERSION="$VERSION" DDEV_IMAGE_TAG="${{ matrix.build.tag }}" + + REPOS="${{ matrix.build.repo }}" + for suffix in ${{ matrix.build.extra_repo_suffixes }}; do + REPOS="${REPOS} ${DOCKER_ORG}/${suffix}" + done + + REFS=() + : > repos.txt + for repo in $REPOS; do + REFS+=("${repo}:${VERSION}") + echo "$repo" >> repos.txt + done + docker save "${REFS[@]}" -o image.tar + echo -n "${{ matrix.build.tag }}" > tag.txt + echo -n "${{ matrix.build.arch }}" > arch.txt + - uses: actions/upload-artifact@v7 + with: + # repo_suffix, not make_dir: all 20 db variants share one make_dir. + name: image-${{ matrix.build.repo_suffix }}-${{ matrix.build.arch }} + path: | + image.tar + repos.txt + tag.txt + arch.txt + # image-push.yml can't download an expired artifact, and the approval + # it waits on is a human one that may not come the same day. + retention-days: 7 + + # --- Pushes and same-repo PRs: no fork content ever runs here, so build + # and push directly with no environment/approval gate - same trust level + # as main-build.yml, which already uses this same secret unguarded. --- + + build-and-push: + name: Build and push ${{ matrix.build.repo }} (${{ matrix.build.arch }}) + needs: detect + if: needs.detect.outputs.needs_build == 'true' && needs.detect.outputs.is_fork == 'false' + strategy: + fail-fast: false + matrix: + build: ${{ fromJson(needs.detect.outputs.build_matrix) }} + runs-on: ${{ matrix.build.arch == 'arm64' && 'ubuntu-24.04-arm' || 'ubuntu-24.04' }} + steps: + - uses: actions/checkout@v7 + + - name: Load 1password secret(s) + uses: 1password/load-secrets-action@v4 + with: + export-env: true + env: + OP_SERVICE_ACCOUNT_TOKEN: "${{ secrets.PUSH_SERVICE_ACCOUNT_TOKEN }}" + DOCKERHUB_TOKEN: "op://push-secrets/DOCKERHUB_TOKEN/credential" + + - name: Login to DockerHub + uses: docker/login-action@v4 + with: + username: ${{ vars.DOCKERHUB_USERNAME }} + password: ${{ env.DOCKERHUB_TOKEN }} + + - name: Build and push ${{ matrix.build.repo }}:${{ matrix.build.tag }}-${{ matrix.build.arch }} + run: | + set -eu -o pipefail + VERSION="${{ matrix.build.tag }}-${{ matrix.build.arch }}" + MAKE_TARGET="${{ matrix.build.make_target }}" + if [ "${{ matrix.build.arch_suffixed }}" = "true" ]; then + MAKE_TARGET="${MAKE_TARGET}_${{ matrix.build.arch }}" + fi + # See the DDEV_IMAGE_TAG note in the fork-side `build` job above. + make -C "containers/${{ matrix.build.make_dir }}" "$MAKE_TARGET" \ + VERSION="$VERSION" DDEV_IMAGE_TAG="${{ matrix.build.tag }}" + + REPOS="${{ matrix.build.repo }}" + for suffix in ${{ matrix.build.extra_repo_suffixes }}; do + REPOS="${REPOS} ${DOCKER_ORG}/${suffix}" + done + for repo in $REPOS; do + docker push "${repo}:${VERSION}" + done + + create-manifests: + name: Create manifest for ${{ matrix.manifest.repo }} + needs: [detect, build-and-push] + if: needs.detect.outputs.needs_build == 'true' && needs.detect.outputs.is_fork == 'false' + strategy: + fail-fast: false + matrix: + manifest: ${{ fromJson(needs.detect.outputs.manifest_matrix) }} runs-on: ubuntu-24.04 + permissions: + contents: read + pull-requests: write steps: - - name: Do nothing + - uses: actions/checkout@v7 + + - name: Load 1password secret(s) + uses: 1password/load-secrets-action@v4 + with: + export-env: true + env: + OP_SERVICE_ACCOUNT_TOKEN: "${{ secrets.PUSH_SERVICE_ACCOUNT_TOKEN }}" + DOCKERHUB_TOKEN: "op://push-secrets/DOCKERHUB_TOKEN/credential" + + - name: Login to DockerHub + uses: docker/login-action@v4 + with: + username: ${{ vars.DOCKERHUB_USERNAME }} + password: ${{ env.DOCKERHUB_TOKEN }} + + - name: Create manifest and clean up per-arch tags + id: manifest + env: + # Trusted context, not the matrix: this is the readable alias + # published next to the hash tag, so it must not be fork-controlled + # beyond the branch name itself, which is sanitized below. + ALIAS_BRANCH: ${{ github.head_ref || github.ref_name }} run: | - echo "Placeholder for #8609 phase 2 - no-op until the real workflow lands." + set -eu -o pipefail + TAG="${{ matrix.manifest.tag }}" + # The oldest db variants are amd64-only, so the arch list comes from + # detect rather than being assumed to be both. + ARCHES="${{ matrix.manifest.arches }}" + ALIAS="$(echo "$ALIAS_BRANCH" | sed -E 's/[^A-Za-z0-9_.-]+/-/g')-${TAG}" + if ! containers/validate-image-tag.sh "$ALIAS"; then + echo "Alias '${ALIAS}' rejected; publishing only ${TAG}" >&2 + ALIAS="" + fi + # Descriptive metadata on the index the tag points at - the standard + # equivalent of a comment on a tag. See containers/image-metadata.sh. + read -r -a ANNOTATIONS <<< "$(containers/image-metadata.sh annotations "$TAG")" + + DOCKERHUB_JWT="$(curl -s -H "Content-Type: application/json" -X POST \ + -d '{"username":"'"${{ vars.DOCKERHUB_USERNAME }}"'","password":"'"${DOCKERHUB_TOKEN}"'"}' \ + https://hub.docker.com/v2/users/login/ | jq -r .token)" + + REPOS="${{ matrix.manifest.repo }}" + for suffix in ${{ matrix.manifest.extra_repo_suffixes }}; do + REPOS="${REPOS} ${DOCKER_ORG}/${suffix}" + done + + PUSHED_SUMMARY="" + for repo in $REPOS; do + ARCH_TAGS=() + for arch in $ARCHES; do + ARCH_TAGS+=("${repo}:${TAG}-${arch}") + done + NAMES=(-t "${repo}:${TAG}") + [ -n "$ALIAS" ] && NAMES+=(-t "${repo}:${ALIAS}") + docker buildx imagetools create "${ANNOTATIONS[@]}" "${NAMES[@]}" "${ARCH_TAGS[@]}" + # imagetools inherits the source media type and a Docker manifest + # list has nowhere to put annotations, so say when they were + # dropped rather than leaving it to be discovered later. + if ! docker buildx imagetools inspect --raw "${repo}:${TAG}" | jq -e '.annotations' >/dev/null 2>&1; then + echo "Note: ${repo}:${TAG} is not an OCI index, so index annotations were not stored." >&2 + fi + PUSHED_SUMMARY="${PUSHED_SUMMARY}- \`${repo}:${TAG}\`"$'\n' + for arch in $ARCHES; do + echo "Removing intermediary tag ${repo}:${TAG}-${arch}" + curl -s -X DELETE -H "Authorization: JWT ${DOCKERHUB_JWT}" \ + "https://hub.docker.com/v2/repositories/${repo}/tags/${TAG}-${arch}/" >/dev/null || true + done + done + + { + echo "summary<> "$GITHUB_OUTPUT" + + - name: Comment on the pull request + if: github.event_name == 'pull_request' + uses: actions/github-script@v9 + env: + IMAGE_PUSH_SUMMARY: ${{ steps.manifest.outputs.summary }} + with: + script: | + const summary = (process.env.IMAGE_PUSH_SUMMARY || "").trim(); + await github.rest.issues.createComment({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: context.issue.number, + body: `Pushed updated container image(s) for this PR:\n\n${summary}`, + }); diff --git a/.github/workflows/image-push.yml b/.github/workflows/image-push.yml index a32d65e286b..65594b749eb 100644 --- a/.github/workflows/image-push.yml +++ b/.github/workflows/image-push.yml @@ -3,27 +3,221 @@ defaults: run: shell: bash -# Placeholder for #8609 phase 2 (fork-safe automatic image build/push). This -# do-nothing stub exists only so `workflow_run` below has a registered -# listener on the default branch before the real load/validate/push logic -# lands - see the phase 2 PR, which will replace this file's contents via -# rebase. `workflow_run` only fires for a listener that already exists on the -# default branch, so this stub unblocks testing that dependency ahead of the -# full PR. +# Trusted side of the fork-safe build/push split for #8609 phase 2. +# Triggered by completion of the "Image build" workflow, using the +# image-push.yml that lives on the default branch - never the fork's copy, +# and this workflow never checks out or executes the triggering PR's code. +# It only loads the artifact "Image build" produced (an inert tarball plus +# metadata) and pushes it, after re-validating the tag. on: workflow_run: workflows: ["Image build"] types: [completed] +env: + DOCKER_ORG: "${{ vars.DOCKER_ORG || 'ddev' }}" + permissions: contents: read jobs: - placeholder: - name: "Placeholder (see #8609)" - if: github.event.workflow_run.conclusion == 'success' + # Ungated on purpose: asking a maintainer to approve a push only to discover + # the run built nothing trains people to click Approve without looking. + check-artifacts: + name: Check for built images + # Only fork completions of "Image build" ever produce artifacts here - + # the non-fork path pushes directly in that workflow's build-and-push job. + if: >- + github.event.workflow_run.conclusion == 'success' && + github.event.workflow_run.head_repository.full_name != github.event.workflow_run.repository.full_name runs-on: ubuntu-24.04 + permissions: + actions: read + outputs: + has_artifacts: ${{ steps.check.outputs.has_artifacts }} steps: - - name: Do nothing + - name: List artifacts on the triggering run + id: check + uses: actions/github-script@v9 + with: + script: | + const artifacts = await github.paginate( + github.rest.actions.listWorkflowRunArtifacts, + { owner: context.repo.owner, repo: context.repo.repo, + run_id: context.payload.workflow_run.id }); + const images = artifacts.filter((a) => a.name.startsWith("image-")); + console.log(`Found ${images.length} image-* artifact(s): ${images.map((a) => a.name).join(", ")}`); + core.setOutput("has_artifacts", images.length > 0 ? "true" : "false"); + + push: + name: "Approve: push the built image(s) to DockerHub" + needs: check-artifacts + if: needs.check-artifacts.outputs.has_artifacts == 'true' + runs-on: ubuntu-24.04 + environment: image-push + permissions: + # download-artifact needs actions:read to reach another run's artifacts. + actions: read + contents: read + pull-requests: write + steps: + - uses: actions/checkout@v7 + + # No continue-on-error: check-artifacts already established that these + # exist, so a failure here is a real one (an expired artifact, say) and + # must not be reported as "nothing needed pushing". + - name: Download build artifacts + uses: actions/download-artifact@v8 + with: + github-token: ${{ secrets.GITHUB_TOKEN }} + run-id: ${{ github.event.workflow_run.id }} + pattern: image-* + path: artifacts + + - name: Load 1password secret(s) + uses: 1password/load-secrets-action@v4 + with: + export-env: true + env: + OP_SERVICE_ACCOUNT_TOKEN: "${{ secrets.PUSH_SERVICE_ACCOUNT_TOKEN }}" + DOCKERHUB_TOKEN: "op://push-secrets/DOCKERHUB_TOKEN/credential" + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v4 + + - name: Login to DockerHub + uses: docker/login-action@v4 + with: + username: ${{ vars.DOCKERHUB_USERNAME }} + password: ${{ env.DOCKERHUB_TOKEN }} + + - name: Validate, load, and push each built image + id: push + env: + # From the workflow_run payload, not the artifact - the alias is + # published in the DDEV org, so its branch name has to come from a + # trusted context. Sanitized and validated below regardless. + ALIAS_BRANCH: ${{ github.event.workflow_run.head_branch }} run: | - echo "Placeholder for #8609 phase 2 - no-op until the real workflow lands." + set -eu -o pipefail + SANITIZED_BRANCH="$(echo "$ALIAS_BRANCH" | sed -E 's/[^A-Za-z0-9_.-]+/-/g')" + # The revision is this workflow's own checkout (the default branch), + # not the fork's, so record the commit the artifact was built from. + export DDEV_GIT_REVISION="${{ github.event.workflow_run.head_sha }}" + declare -A TAG_BY_KEY + declare -A REPOS_BY_KEY + declare -A ARCHES_BY_KEY + + # Everything read out of artifacts/ was written by a job that may + # have run fork-authored content, so the tag and every repository + # name is validated before it reaches a `docker push`. + for dir in artifacts/image-*; do + [ -d "$dir" ] || continue + arch="$(cat "$dir/arch.txt")" + tag="$(cat "$dir/tag.txt")" + key="${dir%-"$arch"}" + + if ! containers/validate-image-tag.sh "$tag"; then + echo "image-push: refusing to push - invalid tag '$tag' from $dir" >&2 + exit 1 + fi + + while IFS= read -r repo; do + [ -z "$repo" ] && continue + if ! containers/validate-image-repo.sh "$repo"; then + echo "image-push: refusing to push - disallowed repository '$repo' from $dir" >&2 + exit 1 + fi + done < "$dir/repos.txt" + + docker load -i "$dir/image.tar" + + while IFS= read -r repo; do + [ -z "$repo" ] && continue + docker push "${repo}:${tag}-${arch}" + done < "$dir/repos.txt" + + TAG_BY_KEY["$key"]="$tag" + if [ -z "${REPOS_BY_KEY[$key]:-}" ]; then + REPOS_BY_KEY["$key"]="$(cat "$dir/repos.txt")" + fi + ARCHES_BY_KEY["$key"]="${ARCHES_BY_KEY[$key]:-} ${arch}" + done + + # Docker Hub token for deleting the intermediary per-arch tags below, + # same cleanup push-tagged-image.yml/push-tagged-dbimage.yml already do. + DOCKERHUB_JWT="$(curl -s -H "Content-Type: application/json" -X POST \ + -d '{"username":"'"${{ vars.DOCKERHUB_USERNAME }}"'","password":"'"${DOCKERHUB_TOKEN}"'"}' \ + https://hub.docker.com/v2/users/login/ | jq -r .token)" + + PUSHED_SUMMARY="" + for key in "${!TAG_BY_KEY[@]}"; do + tag="${TAG_BY_KEY[$key]}" + while IFS= read -r repo; do + [ -z "$repo" ] && continue + arch_tags=() + for arch in ${ARCHES_BY_KEY[$key]}; do + arch_tags+=("${repo}:${tag}-${arch}") + done + names=(-t "${repo}:${tag}") + alias_tag="${SANITIZED_BRANCH}-${tag}" + if containers/validate-image-tag.sh "$alias_tag"; then + names+=(-t "${repo}:${alias_tag}") + else + echo "Alias '${alias_tag}' rejected; publishing only ${tag}" >&2 + fi + read -r -a annotations <<< "$(containers/image-metadata.sh annotations "$tag")" + docker buildx imagetools create "${annotations[@]}" "${names[@]}" "${arch_tags[@]}" + # See the same note in image-build-push.yml: a Docker manifest + # list has nowhere to store annotations. + if ! docker buildx imagetools inspect --raw "${repo}:${tag}" | jq -e '.annotations' >/dev/null 2>&1; then + echo "Note: ${repo}:${tag} is not an OCI index, so index annotations were not stored." >&2 + fi + PUSHED_SUMMARY="${PUSHED_SUMMARY}- \`${repo}:${tag}\`"$'\n' + + for arch in ${ARCHES_BY_KEY[$key]}; do + echo "Removing intermediary tag ${repo}:${tag}-${arch}" + curl -s -X DELETE -H "Authorization: JWT ${DOCKERHUB_JWT}" \ + "https://hub.docker.com/v2/repositories/${repo}/tags/${tag}-${arch}/" >/dev/null || true + done + done <<< "${REPOS_BY_KEY[$key]}" + done + + { + echo "summary<> "$GITHUB_OUTPUT" + + - name: Comment on the pull request + uses: actions/github-script@v9 + env: + IMAGE_PUSH_SUMMARY: ${{ steps.push.outputs.summary }} + with: + script: | + const headSha = context.payload.workflow_run.head_sha; + const { data: prs } = await github.rest.repos.listPullRequestsAssociatedWithCommit({ + owner: context.repo.owner, + repo: context.repo.repo, + commit_sha: headSha, + }); + if (prs.length === 0) { + console.log(`No pull request associated with ${headSha}, skipping comment.`); + return; + } + // Read from env, not spliced into the script source, since the + // summary contains backticks that would break a template literal. + const summary = (process.env.IMAGE_PUSH_SUMMARY || "").trim(); + if (!summary) { + core.setFailed("Push step produced no summary - nothing was pushed."); + return; + } + const body = `Pushed updated container image(s) for this PR:\n\n${summary}`; + for (const pr of prs) { + await github.rest.issues.createComment({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: pr.number, + body, + }); + } diff --git a/.github/workflows/push-tagged-dbimage.yml b/.github/workflows/push-tagged-dbimage.yml index ef0211c6ecf..2dfea5b2cc3 100644 --- a/.github/workflows/push-tagged-dbimage.yml +++ b/.github/workflows/push-tagged-dbimage.yml @@ -7,8 +7,8 @@ on: workflow_dispatch: inputs: tag: - description: Base tag for pushed dbimage (v1.25.0 for example)' - required: true + description: 'Tag to push (v1.25.0 for a release). Leave empty to use the tag this branch needs.' + required: false default: "" debug_enabled: description: 'Enable debug mode' @@ -18,39 +18,92 @@ on: env: REGISTRY: docker.io DOCKER_ORG: "${{ vars.DOCKER_ORG }}" - TAG: "${{ github.event.inputs.tag }}" - MULTI_ARCH_IMAGES: "mariadb_10.1 mariadb_10.2 mariadb_10.3 mariadb_10.4 mariadb_10.5 mariadb_10.6 mariadb_10.7 mariadb_10.8 mariadb_10.11 mariadb_11.4 mariadb_11.8 mariadb_12.3 mysql_5.7 mysql_8.0 mysql_8.4 mysql_9.7" permissions: contents: read jobs: + # The variant matrix and which of them are multi-arch both come from + # containers/ddev-dbserver/variants.txt, shared with that directory's + # Makefile and with image-configs.sh. + variants: + name: Read the db variant matrix + runs-on: ubuntu-24.04 + outputs: + matrix: ${{ steps.variants.outputs.matrix }} + multi_arch_images: ${{ steps.variants.outputs.multi_arch_images }} + all_repos: ${{ steps.variants.outputs.all_repos }} + tag: ${{ steps.tag.outputs.tag }} + alias: ${{ steps.tag.outputs.alias }} + steps: + - uses: actions/checkout@v7 + - id: variants + run: | + set -eu -o pipefail + V=containers/ddev-dbserver/variants.sh + { + echo "matrix=$($V json)" + echo "multi_arch_images=$($V multi-arch-variants)" + echo "all_repos=$($V repos | tr '\n' ' ')" + } >> "$GITHUB_OUTPUT" + # An empty tag input means "publish what this checkout needs", which is + # derivable; retyping the content hash by hand is how it ends up not + # matching versionconstants.go. + - id: tag + env: + INPUT_TAG: ${{ github.event.inputs.tag }} + ALIAS_BRANCH: ${{ github.ref_name }} + run: | + set -eu -o pipefail + HASH_LEN="${HASH_LEN:-10}" + if [ -n "$INPUT_TAG" ]; then + TAG="$INPUT_TAG" + echo "Using the tag supplied on the workflow input: ${TAG}" + else + TAG="$(containers/image-tag-for.sh ddev-dbserver-mariadb-11.8)" + echo "No tag supplied; using the tag this checkout needs: ${TAG}" + fi + containers/validate-image-tag.sh "$TAG" || [[ "$TAG" =~ ^v[0-9]+\.[0-9]+\.[0-9]+$ ]] + echo "tag=${TAG}" >> "$GITHUB_OUTPUT" + + # A readable neighbour for a bare hash, so the registry tag list + # still says which branch produced it. A release tag is already + # readable and gets none. + ALIAS="" + if [[ "$TAG" =~ ^[0-9a-f]{${HASH_LEN}}$ ]]; then + ALIAS="$(echo "$ALIAS_BRANCH" | sed -E 's/[^A-Za-z0-9_.-]+/-/g')-${TAG}" + if containers/validate-image-tag.sh "$ALIAS"; then + echo "Publishing the alias ${ALIAS} alongside ${TAG}" + else + echo "Alias '${ALIAS}' rejected; publishing only ${TAG}" >&2 + ALIAS="" + fi + fi + echo "alias=${ALIAS}" >> "$GITHUB_OUTPUT" + build-db-arch: - name: build ${{ matrix.arch }} ${{ matrix.dbtype }} + name: build ${{ matrix.build.arch }} ${{ matrix.build.dbtype }} + needs: variants + env: + TAG: ${{ needs.variants.outputs.tag }} strategy: fail-fast: false matrix: - arch: [amd64, arm64] - dbtype: [mariadb_5.5, mariadb_10.0, mariadb_10.1, mariadb_10.2, mariadb_10.3, mariadb_10.4, mariadb_10.5, mariadb_10.6, mariadb_10.7, mariadb_10.8, mariadb_10.11, mariadb_11.4, mariadb_11.8, mariadb_12.3, mysql_5.5, mysql_5.6, mysql_5.7, mysql_8.0, mysql_8.4, mysql_9.7] - # update 'meta' step below if you change this: - exclude: - - arch: arm64 - dbtype: mariadb_5.5 - - arch: arm64 - dbtype: mariadb_10.0 - - arch: arm64 - dbtype: mysql_5.5 - - arch: arm64 - dbtype: mysql_5.6 - runs-on: ${{ matrix.arch == 'arm64' && 'ubuntu-24.04-arm' || 'ubuntu-24.04' }} + build: ${{ fromJson(needs.variants.outputs.matrix) }} + runs-on: ${{ matrix.build.arch == 'arm64' && 'ubuntu-24.04-arm' || 'ubuntu-24.04' }} steps: - name: Determine if multi-arch build id: meta + env: + MULTI_ARCH_IMAGES: ${{ needs.variants.outputs.multi_arch_images }} + DBTYPE: ${{ matrix.build.dbtype }} run: | - case "${{ matrix.dbtype }}" in - mariadb_5.5|mariadb_10.0|mysql_5.5|mysql_5.6) echo "multi_arch=false" >> $GITHUB_OUTPUT ;; - *) echo "multi_arch=true" >> $GITHUB_OUTPUT ;; - esac + set -eu -o pipefail + if [[ " ${MULTI_ARCH_IMAGES} " == *" ${DBTYPE} "* ]]; then + echo "multi_arch=true" >> $GITHUB_OUTPUT + else + echo "multi_arch=false" >> $GITHUB_OUTPUT + fi - name: Load 1password secret(s) uses: 1password/load-secrets-action@v5 if: ${{ github.event_name != 'pull_request' || github.event.pull_request.head.repo.owner.login == github.repository_owner }} @@ -78,33 +131,33 @@ jobs: - name: Clean up stale arch tag before push run: | TOKEN=$(curl -s -H "Content-Type: application/json" -X POST -d '{"username":"${{ vars.DOCKERHUB_USERNAME }}","password":"${{ env.DOCKERHUB_TOKEN }}"}' https://hub.docker.com/v2/users/login/ | jq -r .token) - ORG_IMAGE=${DOCKER_ORG}/ddev-dbserver-$(echo "${{ matrix.dbtype }}" | tr '_' '-') - echo "Cleaning up stale tag for ${ORG_IMAGE}:${TAG}-${{ matrix.arch }}" - curl -s -X DELETE -H "Authorization: JWT $TOKEN" "https://hub.docker.com/v2/repositories/${ORG_IMAGE}/tags/${TAG}-${{ matrix.arch }}/" >/dev/null || true + ORG_IMAGE=${DOCKER_ORG}/ddev-dbserver-$(echo "${{ matrix.build.dbtype }}" | tr '_' '-') + echo "Cleaning up stale tag for ${ORG_IMAGE}:${TAG}-${{ matrix.build.arch }}" + curl -s -X DELETE -H "Authorization: JWT $TOKEN" "https://hub.docker.com/v2/repositories/${ORG_IMAGE}/tags/${TAG}-${{ matrix.build.arch }}/" >/dev/null || true - name: Setup tmate session uses: mxschmitt/action-tmate@v3 with: limit-access-to-actor: true github-token: ${{ secrets.GITHUB_TOKEN }} if: ${{ github.event_name == 'workflow_dispatch' && inputs.debug_enabled }} - - name: Build and push ${{ env.DOCKER_ORG }}/ddev-dbserver-${{ matrix.dbtype }}:${{ env.TAG }} ${{ matrix.arch }} image + - name: Build and push ${{ env.DOCKER_ORG }}/ddev-dbserver-${{ matrix.build.dbtype }}:${{ env.TAG }} ${{ matrix.build.arch }} image run: | set -eu cd containers/ddev-dbserver - target="${{ matrix.dbtype }}_${{ matrix.arch }}" - echo "Building target $target for arch ${{ matrix.arch }}" + target="${{ matrix.build.dbtype }}_${{ matrix.build.arch }}" + echo "Building target $target for arch ${{ matrix.build.arch }}" version="${TAG}" if [ "${{ steps.meta.outputs.multi_arch }}" = "true" ]; then - version="${version}-${{ matrix.arch }}" + version="${version}-${{ matrix.build.arch }}" fi make $target PUSH=true VERSION="${version}" DDEV_IMAGE_TAG="${TAG}" - name: Record image information id: image-info run: | set -eu - INSPECT_IMAGE=${DOCKER_ORG}/ddev-dbserver-$(echo "${{ matrix.dbtype }}" | tr '_' '-'):${TAG} + INSPECT_IMAGE=${DOCKER_ORG}/ddev-dbserver-$(echo "${{ matrix.build.dbtype }}" | tr '_' '-'):${TAG} if [ "${{ steps.meta.outputs.multi_arch }}" = "true" ]; then - INSPECT_IMAGE=${INSPECT_IMAGE}-${{ matrix.arch }} + INSPECT_IMAGE=${INSPECT_IMAGE}-${{ matrix.build.arch }} fi INSPECT_OUTPUT="" # Wait for image to be available with retry logic @@ -124,14 +177,21 @@ jobs: - name: Upload image info artifact uses: actions/upload-artifact@v7 with: - name: image-info-ddev-dbserver-${{ matrix.dbtype }}-${{ matrix.arch }} + name: image-info-ddev-dbserver-${{ matrix.build.dbtype }}-${{ matrix.build.arch }} path: image-info.txt retention-days: 1 create-manifests: name: create multi-arch db manifests - needs: build-db-arch + needs: [variants, build-db-arch] runs-on: ubuntu-24.04 + env: + MULTI_ARCH_IMAGES: ${{ needs.variants.outputs.multi_arch_images }} + ALL_REPOS: ${{ needs.variants.outputs.all_repos }} + TAG: ${{ needs.variants.outputs.tag }} + ALIAS: ${{ needs.variants.outputs.alias }} steps: + # Needed for containers/image-metadata.sh below. + - uses: actions/checkout@v7 - name: Load 1password secret(s) uses: 1password/load-secrets-action@v5 with: @@ -150,15 +210,31 @@ jobs: # Get Docker Hub token for cleanup TOKEN=$(curl -s -H "Content-Type: application/json" -X POST -d '{"username":"${{ vars.DOCKERHUB_USERNAME }}","password":"${{ env.DOCKERHUB_TOKEN }}"}' https://hub.docker.com/v2/users/login/ | jq -r .token) - # Create and push multi-arch manifests + # Create and push multi-arch manifests. ALIAS, when set, is a second + # name on the same manifest - see the variants job. for variant in ${MULTI_ARCH_IMAGES}; do ORG_IMAGE=${DOCKER_ORG}/ddev-dbserver-$(echo "${variant}" | tr '_' '-') - docker buildx imagetools create -t ${ORG_IMAGE}:${TAG} ${ORG_IMAGE}:${TAG}-amd64 ${ORG_IMAGE}:${TAG}-arm64 + NAMES=(-t "${ORG_IMAGE}:${TAG}") + [ -n "${ALIAS}" ] && NAMES+=(-t "${ORG_IMAGE}:${ALIAS}") + read -r -a ANNOTATIONS <<< "$(containers/image-metadata.sh annotations "${TAG}" "${ORG_IMAGE##*/}")" + docker buildx imagetools create "${ANNOTATIONS[@]}" "${NAMES[@]}" ${ORG_IMAGE}:${TAG}-amd64 ${ORG_IMAGE}:${TAG}-arm64 if [[ "$TAG" =~ ^v[0-9]+\.[0-9]+\.[0-9]+$ ]]; then docker buildx imagetools create -t ${ORG_IMAGE}:latest ${ORG_IMAGE}:${TAG} fi done + # The amd64-only variants are pushed straight to ${TAG} and never + # reach the loop above, so they need the alias adding separately. + if [ -n "${ALIAS}" ]; then + for repo in ${ALL_REPOS}; do + variant="$(echo "${repo#ddev-dbserver-}" | tr '-' '_')" + case " ${MULTI_ARCH_IMAGES} " in *" ${variant} "*) continue ;; esac + ORG_IMAGE=${DOCKER_ORG}/${repo} + echo "Aliasing ${ORG_IMAGE}:${TAG} as ${ALIAS}" + docker buildx imagetools create -t "${ORG_IMAGE}:${ALIAS}" "${ORG_IMAGE}:${TAG}" + done + fi + # Clean up intermediary single-arch tags from remote registry for variant in ${MULTI_ARCH_IMAGES}; do ORG_IMAGE=${DOCKER_ORG}/ddev-dbserver-$(echo "${variant}" | tr '_' '-') diff --git a/.github/workflows/push-tagged-image.yml b/.github/workflows/push-tagged-image.yml index b284511b23e..17977abb596 100644 --- a/.github/workflows/push-tagged-image.yml +++ b/.github/workflows/push-tagged-image.yml @@ -18,8 +18,8 @@ on: - ddev-xhgui - test-ssh-server tag: - description: Tag for pushed image (v1.25.0 for example) - required: true + description: 'Tag to push (v1.25.0 for a release). Leave empty to use the tag this branch needs.' + required: false default: "" debug_enabled: description: 'Enable debug mode' @@ -29,14 +29,60 @@ on: env: REGISTRY: docker.io DOCKER_ORG: "${{ vars.DOCKER_ORG }}" - TAG: "${{ github.event.inputs.tag }}" permissions: contents: read jobs: + # An empty tag input means "publish what this checkout needs", which is + # derivable; retyping the content hash by hand is how it ends up not + # matching versionconstants.go. + resolve-tag: + name: Resolve the tag to push + runs-on: ubuntu-24.04 + outputs: + tag: ${{ steps.tag.outputs.tag }} + alias: ${{ steps.tag.outputs.alias }} + steps: + - uses: actions/checkout@v7 + - id: tag + env: + INPUT_TAG: ${{ github.event.inputs.tag }} + IMAGE: ${{ github.event.inputs.image }} + ALIAS_BRANCH: ${{ github.ref_name }} + run: | + set -eu -o pipefail + HASH_LEN="${HASH_LEN:-10}" + if [ -n "$INPUT_TAG" ]; then + TAG="$INPUT_TAG" + echo "Using the tag supplied on the workflow input: ${TAG}" + else + TAG="$(containers/image-tag-for.sh "$IMAGE")" + echo "No tag supplied; using the tag this checkout needs: ${TAG}" + fi + containers/validate-image-tag.sh "$TAG" || [[ "$TAG" =~ ^v[0-9]+\.[0-9]+\.[0-9]+$ ]] + echo "tag=${TAG}" >> "$GITHUB_OUTPUT" + + # A readable neighbour for a bare hash, so the registry tag list + # still says which branch produced it. A release tag is already + # readable and gets none. + ALIAS="" + if [[ "$TAG" =~ ^[0-9a-f]{${HASH_LEN}}$ ]]; then + ALIAS="$(echo "$ALIAS_BRANCH" | sed -E 's/[^A-Za-z0-9_.-]+/-/g')-${TAG}" + if containers/validate-image-tag.sh "$ALIAS"; then + echo "Publishing the alias ${ALIAS} alongside ${TAG}" + else + echo "Alias '${ALIAS}' rejected; publishing only ${TAG}" >&2 + ALIAS="" + fi + fi + echo "alias=${ALIAS}" >> "$GITHUB_OUTPUT" + build-and-push: name: build ${{ matrix.arch }} ${{ github.event.inputs.image }} + needs: resolve-tag + env: + TAG: ${{ needs.resolve-tag.outputs.tag }} strategy: matrix: arch: [amd64, arm64] @@ -131,10 +177,15 @@ jobs: retention-days: 1 create-manifests: name: create multi-arch manifest - needs: build-and-push + needs: [resolve-tag, build-and-push] runs-on: ubuntu-24.04 if: ${{ needs.build-and-push.outputs.multi_arch == 'true' }} + env: + TAG: ${{ needs.resolve-tag.outputs.tag }} + ALIAS: ${{ needs.resolve-tag.outputs.alias }} steps: + # Needed for containers/image-metadata.sh below. + - uses: actions/checkout@v7 - name: Load 1password secret(s) uses: 1password/load-secrets-action@v5 with: @@ -159,9 +210,13 @@ jobs: MULTI_ARCH_IMAGES="${MULTI_ARCH_IMAGES} ${DOCKER_ORG}/ddev-webserver-prod" fi - # Create and push multi-arch manifests + # Create and push multi-arch manifests. ALIAS, when set, is a second + # name on the same manifest - see the resolve-tag job. for ORG_IMAGE in ${MULTI_ARCH_IMAGES}; do - docker buildx imagetools create -t ${ORG_IMAGE}:${TAG} ${ORG_IMAGE}:${TAG}-amd64 ${ORG_IMAGE}:${TAG}-arm64 + NAMES=(-t "${ORG_IMAGE}:${TAG}") + [ -n "${ALIAS}" ] && NAMES+=(-t "${ORG_IMAGE}:${ALIAS}") + read -r -a ANNOTATIONS <<< "$(containers/image-metadata.sh annotations "${TAG}" "${ORG_IMAGE##*/}")" + docker buildx imagetools create "${ANNOTATIONS[@]}" "${NAMES[@]}" ${ORG_IMAGE}:${TAG}-amd64 ${ORG_IMAGE}:${TAG}-arm64 if [[ "${TAG}" =~ ^v[0-9]+\.[0-9]+\.[0-9]+$ ]]; then docker buildx imagetools create -t ${ORG_IMAGE}:latest ${ORG_IMAGE}:${TAG} fi diff --git a/.github/workflows/test-reusable.yml b/.github/workflows/test-reusable.yml index 3052183d6ca..e5820fb0532 100644 --- a/.github/workflows/test-reusable.yml +++ b/.github/workflows/test-reusable.yml @@ -116,6 +116,7 @@ jobs: runs-on: ${{ inputs.runner }} env: + DOCKER_ORG: ${{ vars.DOCKER_ORG }} BUILDKIT_PROGRESS: plain DOCKER_CLI_EXPERIMENTAL: enabled DDEV_DEBUG: true @@ -152,6 +153,11 @@ jobs: done < <(git ls-tree --name-only refs/public-variables-tmp:.github/public-variables/) git update-ref -d refs/public-variables-tmp + - name: Wait for pushed images + # This runner holds no push credentials, so it can race + # image-push.yml's approval/build/push - see containers/wait-for-images.sh. + run: containers/wait-for-images.sh + - name: Get Date id: get-date run: | diff --git a/.github/workflows/test-wsl2-reusable.yml b/.github/workflows/test-wsl2-reusable.yml index 5d4df9832c3..698a73560d3 100644 --- a/.github/workflows/test-wsl2-reusable.yml +++ b/.github/workflows/test-wsl2-reusable.yml @@ -67,6 +67,7 @@ jobs: name: WSL2 (${{ inputs.networking }}, ${{ inputs.make_target }}) env: + DOCKER_ORG: ${{ vars.DOCKER_ORG }} GOTEST_SHORT: ${{ inputs.gotest_short }} TESTARGS: ${{ inputs.testargs }} MAKE_TARGET: ${{ inputs.make_target }} @@ -193,4 +194,5 @@ jobs: $embargo = "${{ env.DDEV_EMBARGO_TESTS }}" $embargo_php = "${{ env.DDEV_EMBARGO_PHP_VERSIONS }}" $skip_nodejs = "${{ env.DDEV_SKIP_NODEJS_TEST }}" - wsl -u testuser -- bash -exc "export GOTEST_SHORT='$gotest_short' TESTARGS='$testargs' MAKE_TARGET='$make_target' MAKEARGS='$makeargs' DDEV_EMBARGO_TESTS='$embargo' DDEV_EMBARGO_PHP_VERSIONS='$embargo_php' DDEV_SKIP_NODEJS_TEST='$skip_nodejs' && cd ~/workspace/ddev && bash -e .github/workflows/wsl2-test.sh" + $docker_org = "${{ env.DOCKER_ORG }}" + wsl -u testuser -- bash -exc "export GOTEST_SHORT='$gotest_short' TESTARGS='$testargs' MAKE_TARGET='$make_target' MAKEARGS='$makeargs' DDEV_EMBARGO_TESTS='$embargo' DDEV_EMBARGO_PHP_VERSIONS='$embargo_php' DDEV_SKIP_NODEJS_TEST='$skip_nodejs' DOCKER_ORG='$docker_org' && cd ~/workspace/ddev && bash -e .github/workflows/wsl2-test.sh" diff --git a/.github/workflows/wsl2-test.sh b/.github/workflows/wsl2-test.sh index 004ec49644a..c40bc4b3dcd 100755 --- a/.github/workflows/wsl2-test.sh +++ b/.github/workflows/wsl2-test.sh @@ -55,6 +55,11 @@ go version docker version git --version +# This runner holds no push credentials, so it can race image-push.yml's +# approval/build/push - see containers/wait-for-images.sh. +echo "=== Waiting for pushed images ===" +containers/wait-for-images.sh + echo "=== Building DDEV ===" make CGO_ENABLED="${CGO_ENABLED}" BUILDARGS="${BUILDARGS}" diff --git a/HANDOFF.md b/HANDOFF.md new file mode 100644 index 00000000000..cbd7ccf814a --- /dev/null +++ b/HANDOFF.md @@ -0,0 +1,278 @@ +# HANDOFF — PR #8707 review fixes (#8609 phase 2) + +Temporary file. Delete before merging. + +Branch `20260814_rfay_docker_update_phase_2`. Nothing has been pushed. + +## MUST TEST BEFORE MERGE + +### 0. Every image has to be republished at its bare-hash tag + +Tags are now bare content hashes (`ddev/ddev-webserver:36bceca65e`), which +invalidates every previously published content-hash tag. Nothing in either +organization carries the new tags yet — confirmed missing for webserver, +ssh-agent, and the db variants. **Integration tests cannot pull until the +first CI run publishes them**, which is 44 build jobs across 24 images. +Locally, `go test -run TestCmdVersion ./cmd/ddev/cmd/...` currently fails with +`manifest for ddev/ddev-ssh-agent:8e8bf1217c not found`, which is this and +nothing else. + +This is a one-time migration cost of the scheme change; after that first push +the tags stop depending on branch or organization, which is the whole point. + +Two further things have never run in CI and no test harness covers. Both are +cheap to get wrong and expensive to discover after merge. + +### 1. A `containers/ddev-dbserver` change must build and push all 20 variants + +This is the bug that motivated the second round of fixes, and this PR is +itself the first live test of the fix — adding `variants.txt` under +`containers/ddev-dbserver/` changed that directory's hash, so `BaseDBTag` is +now `20260814_rfay_docker_update_phase_2-1dc90407ef` and **CI must build and +push 36 jobs across 20 repositories before any test using a non-default +database can pass.** + +Watch for, on this PR's first real CI run: + +- `detect` reports 36 build jobs / 20 images, not 1. +- `create-manifests` (or `image-push`) produces a multi-arch manifest for + every one of the 20 `ddev/ddev-dbserver-*` repos at that tag, with the four + oldest (`mariadb-5.5`, `mariadb-10.0`, `mysql-5.5`, `mysql-5.6`) amd64-only. +- `TestDdevAllDatabases` passes, along with the tests in `db_test.go`, + `snapshot_test.go`, `config_test.go`, and `debug-migrate-database_test.go` + that pin a non-default database. + +Verify independently of CI's own reporting: + +```bash +TAG=$(grep -E '^var BaseDBTag' pkg/versionconstants/versionconstants.go | sed -E 's/.*"([^"]*)".*/\1/') +for r in $(containers/ddev-dbserver/variants.sh repos); do + printf '%-34s ' "$r"; containers/registry-tag-exists.sh "ddev/$r" "$TAG" && echo EXISTS || echo MISSING +done +``` + +Every line must say EXISTS. Any MISSING means the matrix regressed and +non-default database tests will fail. + +Note that item 1 below was written before the tag scheme changed: the tag to +check is now the bare `BaseDBTag` hash, not a branch-prefixed string, and the +`variants.sh repos` loop still works unchanged. + +### 2. `push-tagged-dbimage.yml` still works after the DRY refactor + +Its matrix, its `MULTI_ARCH_IMAGES` list, and its multi-arch/single-arch +decision were three separate hardcoded copies of the variant list; all three +now come from `variants.sh`. This is the release-time push path, so a mistake +here surfaces during a release. + +Its `tag` input is now optional — left empty it derives the tag the checkout +needs, which is the fix for having to transcribe `main-1dc90407ef` by hand. +Test **both** paths: empty (derives the bare hash) and an explicit `vX.Y.Z` +(the release path). + +Run it manually on `ddev-test/ddev` and confirm: + +- The `variants` job runs first and its matrix expands to **36** `build-db-arch` + jobs — the same count as before (20 variants × 2 arches, minus 4 arm64 + exclusions). +- With an empty tag input, the resolve step logs the derived tag and it matches + `BaseDBTag` in `versionconstants.go`. +- The four amd64-only variants get `multi_arch=false` in their `meta` step and + push an unsuffixed tag; the other 16 get `multi_arch=true` and push + `-amd64`/`-arm64`. +- `create-manifests` combines exactly the 16 multi-arch variants and deletes + the intermediary per-arch tags. +- All 20 repos carry the throwaway tag at the end. + +Locally I confirmed the generated lists are byte-identical to the ones they +replaced (`build-targets` for both host arches, `single-arch-targets`, +`test-targets` for both, and the `MULTI_ARCH_IMAGES` set), and that +`make -n` still resolves a target from each list including the amd64-only +ones under `CURRENT_ARCH=amd64`. That is static equivalence, not a live run. + +## Round 1 fixes — review findings + +### The tag-resolution bug (blocking) + +`wait-for-images.sh` computed `-`, but the tag ddev +pulls is the one committed in `versionconstants.go`, and `autotag.sh` rewrites +that line (branch prefix included) only when the hash changes. The two agree +only on a branch that changed the image, so any PR not touching `containers/` +would poll 20 minutes and fail every Buildkite and GitHub test job. `detect` +had the mirror-image bug: it re-pushed all images under a fresh branch prefix +on any `containers/` change, no-op or not. + +`containers/required-image-tag.sh` now resolves the tag once, for both +callers, reporting `committed` (hash matches — wait for that exact tag) or +`recomputed` (content changed — `make` builds it locally). This also removed +the `WAIT_FOR_IMAGES_BRANCH` plumbing from four callers, since the branch name +is no longer needed. + +### Security + +- `github.head_ref` reached a `run:` block spliced into the script. Git ref + names permit quotes and backticks, and `detect` emits `is_fork`, so injected + code could set `is_fork=false` and route fork content into `build-and-push`, + the job that loads `PUSH_SERVICE_ACCOUNT_TOKEN`. It now arrives via `env:`, + and `is_fork` moved to its own step. +- `image-push.yml` validated the tag but pushed to whatever repository names + the fork-produced artifact listed. `validate-image-repo.sh` constrains them + to `$DOCKER_ORG` plus an exact allowlist. + +### Silent failures + +- `image-push.yml` lacked `actions: read` for a cross-run artifact download, + and `continue-on-error` turned that into a "nothing needed pushing" comment. + A new ungated `check-artifacts` job gates the environment job instead, so a + fork build with nothing to push no longer asks for approval, and a failed + download is now fatal. +- Artifact retention 1 → 7 days; the gate is a human approval. + +### Smaller + +- `DDEV_IMAGE_TAG` was not passed, so `com.ddev.image-tag` recorded + `-` rather than the tag people pull, which + `imageVersionMismatch()` compares against. +- `DOCKER_ORG` falls back to `ddev` instead of producing `/ddev-webserver`. +- `validate-image-tag.sh`'s reserved-literal and `vX.Y.Z` checks were + unreachable behind the format check; they now test the part before the hash. +- BSD `wc -l` padding failed 5 checks on macOS. + +## Round 2 — the db variant matrix + +`GetDBImage()` in `pkg/docker/images.go` builds every variant's reference from +one shared `BaseDBTag`, so a dbserver change moves the tag for all 20 while +`make` built only `mariadb_11.8` and CI pushed only `ddev-dbserver-mariadb-11.8`. +The other 19 were referenced at a tag that existed nowhere. Introduced by +phase 1 (#8612): before that, `BaseDBTag` was hand-bumped after someone ran +`push-tagged-dbimage.yml` for all 20. + +Fixed by making `detect`'s matrix cover every variant. The build matrix is now +one entry per (image, arch) rather than a cross product, because the oldest +variants are amd64-only, and `create-manifests` takes its arch list from +`detect` instead of assuming both. Artifact names key on `repo_suffix`, not +`make_dir` — all 20 db variants share a `make_dir` and would have collided. + +`wait-for-images.sh` now also fails fast, with the command to run, when a +non-locally-built image is out of date. There is no local fallback for those +19 variants, and the tag `make` would invent depends on the runner's branch +name (detached HEAD on a PR checkout), so it may not match what CI pushed. + +### DRY + +The variant list was duplicated in four places. It now lives in +`containers/ddev-dbserver/variants.txt`, read through `variants.sh`, which +renders each consumer's view: + +| Consumer | View | +| --- | --- | +| `containers/ddev-dbserver/Makefile` | `build-targets`, `single-arch-targets`, `test-targets` | +| `containers/image-configs.sh` | `list` | +| `containers/validate-image-repo.sh` | `repos` | +| `.github/workflows/push-tagged-dbimage.yml` | `json`, `multi-arch-variants` | + +`variants.txt` sits inside the hashed dbserver directory on purpose: adding a +database version has to change the content hash, or `detect` would decide the +tag already exists and never build the new variant. + +## Test status + +127 checks across seven harnesses, all passing locally (macOS), all wired into +`container-tests.yml`: + +| Harness | Checks | +| --- | --- | +| `containers/autotag_test.sh` | 18 | +| `containers/db_variants_test.sh` | 15 (new) | +| `containers/registry_tag_exists_test.sh` | 4 | +| `containers/required_image_tag_test.sh` | 12 (new) | +| `containers/validate_image_repo_test.sh` | 37 (new) | +| `containers/validate_image_tag_test.sh` | 20 | +| `containers/wait_for_images_test.sh` | 21 | + +`shellcheck -x` clean on all new and changed scripts. `actionlint` clean on +all changed workflows apart from pre-existing SC2086/SC2046 notes in untouched +parts of `test-reusable.yml`. + +## Verification Claude ran + +Read-only against the real registry, plus this checkout. Nothing pushed. + +1. All seven harnesses. +2. `wait-for-images.sh` finds the four non-db images and correctly waits on the + db variants at the new `BaseDBTag`. +3. `detect` dry run: `matrix=[]` when nothing changed; 36 build jobs / 20 + manifests after a dbserver change; only xhgui after an xhgui change. +4. Hostile branch name `evil"; id; #` sanitizes to `evil-id-`, no execution. +5. Generated db lists byte-identical to the four hardcoded copies they replace; + `make -n` resolves a target from each list on both host arches. +6. `make autotag-images` built `mariadb_11.8` locally and rewrote `BaseDBTag`. +7. `docker buildx imagetools inspect` works with no Docker daemon running, + so the early placement of the wait step in the Buildkite scripts is fine. + +Not run: the artifact round-trip against a local `registry:2` (worth doing — +it needs no secrets and would exercise the multi-arch `imagetools create` +grouping that no unit test covers), and a full `make` build. + +## Other human verification + +On `ddev-test/ddev`, with the `image-push` environment configured. Confirm the +rule actually took: `gh api repos///environments/image-push` — an +environment referenced before it exists is auto-created with no protection. + +1. **A PR touching no `containers/` file.** Five `found …` lines within seconds, + then the tests run. This is the round-1 blocking fix and has never run in CI. +2. **A `containers/` file that isn't in any hash path.** `detect` reports + `already exists (committed)` for everything and builds nothing. +3. **An `ddev-xhgui` change on a same-repo branch.** Only xhgui builds; no + approval; `com.ddev.image-tag` on the result is the final tag, not + `-amd64`. +4. **The same from a fork.** No secret-loading step in `build`; exactly one + approval; the download succeeds rather than silently commenting + "nothing needed pushing". +5. **A fork PR touching `containers/` with no image change.** No approval + request at all. +6. **Adversarial artifact.** Overwrite `repos.txt` with `ddev/ddev-webserver` + before upload; the push must fail at `validate-image-repo.sh`. +7. **Hostile branch name** ``test`touch /tmp/pwned` ``; check the `detect` log. +8. **Expired artifact.** Approve after `retention-days`; must fail loudly. + +## Round 3 — hash-only tags + +The branch prefix in a tag carried no information but forced every consumer to +agree on a string. Tags are now the bare content hash; the branch survives as a +trailing comment in `versionconstants.go`, a `TagBranch` variable shown by +`ddev version` (`image-tag-branches`, collapsed when all images share a +branch), and a `-` alias published off the same manifest. + +`validate-image-tag.sh` accepts both forms, and the alias goes through it, so a +fork branch named `v1.25.0` can't publish something that reads as a release — +which finally makes the reserved/release checks load-bearing rather than dead. + +Comparison against `versionconstants.go` is exact now, so a line still in the +old form counts as stale and `make` migrates it. `wait-for-images.sh` lost its +fail-fast branch: the tag a non-locally-built image needs is branch-independent, +so waiting for it is well defined. + +The manual push workflows' `tag` input is optional; empty derives the tag via +`containers/image-tag-for.sh`. + +## Still open + +- **Fork artifact volume.** A fork changing `ddev-dbserver` now produces 36 + `docker save` tarballs. Local db images are 550–730MB, so that is roughly + 20GB of artifacts for one PR, compressed by `upload-artifact` but still + large, and now retained 7 days. This is the cost of building the full matrix + on the untrusted path; if it proves impractical, the fallback is to keep the + full matrix on the non-fork path and have forks use + `push-tagged-dbimage.yml`. Watch the first fork-side dbserver PR. +- **`actions: read` on `download-artifact@v8`** is added on the documented + requirement; not yet observed on a live run. +- **Registry pollution has no cleanup path.** Each image change adds tags that + are never removed — now 20 at a time for a dbserver change. +- **`registry-tag-exists.sh` cannot distinguish "missing" from "registry + unreachable."** A DockerHub blip costs a redundant rebuild in `detect` or 20 + minutes and a red build in `wait-for-images.sh`. It now makes 24 registry + calls per test run instead of 5, so the odds are higher. +- **The PR description** was rewritten for round 1 but does not yet mention the + db variant matrix. diff --git a/Makefile b/Makefile index 6773645f7b8..f3e8a3f71ad 100644 --- a/Makefile +++ b/Makefile @@ -68,6 +68,11 @@ build: autotag-images $(DEFAULT_BUILD) # (no Docker, no network). A changed image is built locally (host arch only) # and its tag in versionconstants.go is rewritten automatically - see # containers/autotag.sh and docs/content/developers/building-contributing.md. +# Only the default db variant (mariadb_11.8) is built here, to keep an +# unrelated rebuild cheap. Every ddev-dbserver variant shares BaseDBTag, so a +# dbserver change moves the tag for all of them at once; CI builds and pushes +# the whole matrix (containers/ddev-dbserver/variants.txt), and the rest are +# pulled from the registry. .PHONY: autotag-images autotag-images: @containers/autotag.sh WebTag ddev/ddev-webserver containers/ddev-webserver containers/containers_shared.mk -- $(MAKE) -C containers/ddev-webserver images @@ -225,7 +230,14 @@ setup: @mkdir -p $(TESTTMP) # Required static analysis targets for pre-push. -staticrequired: setup golangci-lint markdownlint zensical +staticrequired: setup golangci-lint markdownlint zensical check-image-tags + +# A changed image whose tag never made it into versionconstants.go ships a +# binary pulling tags nothing builds. Checked rather than fixed here, because +# fixing means a Docker build - run `make` for that. +.PHONY: check-image-tags +check-image-tags: + @containers/check-image-tags.sh # Fail rather than skip when a required tool is absent. These targets used to # print a note and exit 0, so `make staticrequired` reported success while diff --git a/containers/autotag.sh b/containers/autotag.sh index 9d118202783..7efc80afe04 100755 --- a/containers/autotag.sh +++ b/containers/autotag.sh @@ -2,14 +2,20 @@ # autotag.sh [--print-only] [ ...] [-- ] # # Detects whether an image's content hash (see hash-paths.sh) differs from -# the hash embedded in the tag currently committed in -# pkg/versionconstants/versionconstants.go. If unchanged, does nothing (no -# Docker, no network). If changed, builds the image locally at the new -# - tag (unless already built) using the given build command, -# then rewrites just that tag's line in versionconstants.go in place. +# the tag currently committed in pkg/versionconstants/versionconstants.go. If +# unchanged, does nothing (no Docker, no network). If changed, builds the +# image locally at the new tag (unless already built) using the given build +# command, then rewrites just that tag's line in versionconstants.go in place. # -# --print-only prints the candidate - tag and exits, without -# touching Docker or versionconstants.go. +# The tag is the bare content hash, with no branch prefix: the hash is the +# content address, and a prefix would only make the same image resolve to +# different strings depending on which branch or repository published it. +# The branch is recorded in a trailing comment instead, so the line still says +# where the image came from, and pushes publish a - alias +# pointing at the same manifest for anyone browsing the registry. +# +# --print-only prints the candidate tag and exits, without touching Docker or +# versionconstants.go. # # Env: # HASH_LEN - hash length in hex chars (default 10, must match hash-paths.sh) @@ -60,7 +66,8 @@ CURRENT_HASH="$(HASH_LEN="$HASH_LEN" "$SCRIPT_DIR/hash-paths.sh" "${HASH_PATHS[@ BRANCH="$(git -C "$REPO_ROOT" rev-parse --abbrev-ref HEAD 2>/dev/null || echo detached)" SANITIZED_BRANCH="$(echo "$BRANCH" | sed -E 's/[^A-Za-z0-9_.-]+/-/g')" -CANDIDATE_TAG="${SANITIZED_BRANCH}-${CURRENT_HASH}" +CANDIDATE_TAG="${CURRENT_HASH}" +ALIAS_TAG="${SANITIZED_BRANCH}-${CURRENT_HASH}" if [ "$PRINT_ONLY" = true ]; then echo "$CANDIDATE_TAG" @@ -73,9 +80,9 @@ if [ -z "$EXISTING_TAG" ]; then exit 1 fi -EXISTING_HASH="${EXISTING_TAG: -${HASH_LEN}}" - -if [ "$EXISTING_HASH" = "$CURRENT_HASH" ]; then +# Exact, not a trailing-hash match, so a line still in the old - +# form is rewritten to the bare hash the first time make runs. +if [ "$EXISTING_TAG" = "$CURRENT_HASH" ]; then # Unchanged - nothing to do. exit 0 fi @@ -89,10 +96,16 @@ else echo "autotag.sh: no build command given, cannot build ${IMAGE_REPO}:${CANDIDATE_TAG}" >&2 exit 1 fi - "${BUILD_CMD[@]}" "VERSION=${CANDIDATE_TAG}" + "${BUILD_CMD[@]}" "VERSION=${CANDIDATE_TAG}" "DDEV_IMAGE_TAG=${CANDIDATE_TAG}" fi -sed -i.bak -E "s/^var ${TAG_VAR} = \"[^\"]*\"/var ${TAG_VAR} = \"${CANDIDATE_TAG}\"/" "$VERSIONCONSTANTS_FILE" +# The trailing comment is rewritten wholesale, not preserved: it exists to say +# which branch produced this hash, which is the readability a bare hash costs. +# Branch carries the same thing as data, for `ddev version`. +sed -i.bak -E \ + -e "s|^var ${TAG_VAR} = \"[^\"]*\".*|var ${TAG_VAR} = \"${CANDIDATE_TAG}\" // ${ALIAS_TAG}|" \ + -e "s|^var ${TAG_VAR}Branch = \"[^\"]*\"|var ${TAG_VAR}Branch = \"${SANITIZED_BRANCH}\"|" \ + "$VERSIONCONSTANTS_FILE" rm -f "${VERSIONCONSTANTS_FILE}.bak" -echo "autotag.sh: updated ${TAG_VAR} to ${CANDIDATE_TAG} in $VERSIONCONSTANTS_FILE" +echo "autotag.sh: updated ${TAG_VAR} to ${CANDIDATE_TAG} (${ALIAS_TAG}) in $VERSIONCONSTANTS_FILE" diff --git a/containers/autotag_test.sh b/containers/autotag_test.sh index 19f0e020fe8..96ed74fa401 100755 --- a/containers/autotag_test.sh +++ b/containers/autotag_test.sh @@ -32,6 +32,11 @@ assert_eq() { fi } +# BSD wc pads its output with spaces; GNU wc does not. +count_lines() { + wc -l < "$1" | tr -d '[:space:]' +} + WORKDIR="$(mktemp -d)" trap 'rm -rf "$WORKDIR"' EXIT @@ -79,7 +84,10 @@ cat > "$VERSIONCONSTANTS" <<'EOF' package versionconstants // WebTag defines the default web image tag -var WebTag = "v1.0.0" // Note that this can be overridden by make +var WebTag = "v1.0.0" // some-old-branch-v1.0.0 + +// WebTagBranch is the branch WebTag's content was built from. +var WebTagBranch = "some-old-branch" EOF export VERSIONCONSTANTS_FILE="$VERSIONCONSTANTS" @@ -131,11 +139,8 @@ candidate="$("$AUTOTAG" --print-only WebTag ddev/dummy-image imgdir)" after="$(cat "$VERSIONCONSTANTS")" assert_eq "$before" "$after" "--print-only does not modify the versionconstants file" current_hash="$("$HASH_PATHS" imgdir)" -case "$candidate" in - *-"$current_hash") pass "--print-only candidate tag ends with the current hash" ;; - *) fail "--print-only candidate tag '$candidate' doesn't end with hash '$current_hash'" ;; -esac -calls="$(wc -l < "$DOCKER_CALL_LOG")" +assert_eq "$current_hash" "$candidate" "--print-only candidate tag is the bare content hash" +calls="$(count_lines "$DOCKER_CALL_LOG")" assert_eq "0" "$calls" "--print-only makes no docker calls" # 6. Change detected (today's fixture tag is "v1.0.0", never a real hash), @@ -149,24 +154,21 @@ else fail "build command should have run when the tag changed and no local image existed" fi +current_branch="$(git rev-parse --abbrev-ref HEAD | sed -E 's/[^A-Za-z0-9_.-]+/-/g')" new_tag_line="$(grep '^var WebTag = ' "$VERSIONCONSTANTS")" -case "$new_tag_line" in - *"-${current_hash}\""*) pass "versionconstants file rewritten with the new hash-suffixed tag" ;; - *) fail "versionconstants file not rewritten as expected: $new_tag_line" ;; -esac -case "$new_tag_line" in - *"// Note that this can be overridden by make"*) pass "trailing comment on the tag line is preserved" ;; - *) fail "trailing comment on the tag line was lost: $new_tag_line" ;; -esac +assert_eq "var WebTag = \"${current_hash}\" // ${current_branch}-${current_hash}" "$new_tag_line" \ + "the tag becomes the bare hash, with the branch alias as a trailing comment" +assert_eq "var WebTagBranch = \"${current_branch}\"" "$(grep '^var WebTagBranch = ' "$VERSIONCONSTANTS")" \ + "the companion Branch variable is updated for ddev version" # 7. Idempotency: re-run with no further changes -> no-op. No build, no file # rewrite, and (the key design property) no docker call at all. rm -f "$BUILD_MARKER" before2="$(cat "$VERSIONCONSTANTS")" -calls_before="$(wc -l < "$DOCKER_CALL_LOG")" +calls_before="$(count_lines "$DOCKER_CALL_LOG")" "$AUTOTAG" WebTag ddev/dummy-image imgdir -- bash -c "touch '$BUILD_MARKER'" after2="$(cat "$VERSIONCONSTANTS")" -calls_after="$(wc -l < "$DOCKER_CALL_LOG")" +calls_after="$(count_lines "$DOCKER_CALL_LOG")" if [ -f "$BUILD_MARKER" ]; then fail "unexpected rebuild on unchanged content" else @@ -179,8 +181,7 @@ assert_eq "$calls_before" "$calls_after" "no docker calls at all on the unchange # tag -> build is skipped, but the file is still rewritten. echo "changed again" > imgdir/Dockerfile new_hash="$("$HASH_PATHS" imgdir)" -branch="$(git rev-parse --abbrev-ref HEAD | sed -E 's/[^A-Za-z0-9_.-]+/-/g')" -echo "ddev/dummy-image:${branch}-${new_hash}" > "$DOCKER_EXISTING_REF_FILE" +echo "ddev/dummy-image:${new_hash}" > "$DOCKER_EXISTING_REF_FILE" rm -f "$BUILD_MARKER" "$AUTOTAG" WebTag ddev/dummy-image imgdir -- bash -c "touch '$BUILD_MARKER'" if [ -f "$BUILD_MARKER" ]; then @@ -188,12 +189,28 @@ if [ -f "$BUILD_MARKER" ]; then else pass "build skipped when a local image already exists at the computed tag" fi -if grep -q "\"${branch}-${new_hash}\"" "$VERSIONCONSTANTS"; then +if grep -q "^var WebTag = \"${new_hash}\"" "$VERSIONCONSTANTS"; then pass "versionconstants file rewritten even when the build was skipped" else fail "versionconstants file should still be rewritten when the build is skipped" fi +# 9. A line still carrying the old - form is migrated to the bare +# hash on the next run, even though its trailing hash already matches. +cat > "$VERSIONCONSTANTS" < ${tag}") +done + +if [ "${#STALE[@]}" -eq 0 ]; then + echo "check-image-tags.sh: versionconstants.go is up to date with containers/" + exit 0 +fi + +echo "check-image-tags.sh: containers/ changed but versionconstants.go wasn't updated:" >&2 +printf ' %s\n' "${STALE[@]}" >&2 +echo "check-image-tags.sh: run 'make' to rebuild the changed image(s) and rewrite those tags, then commit the result." >&2 +exit 1 diff --git a/containers/containers_shared.mk b/containers/containers_shared.mk index 4d5c3d4c070..030f7696d6e 100644 --- a/containers/containers_shared.mk +++ b/containers/containers_shared.mk @@ -13,10 +13,14 @@ DDEV_IMAGE_TAG ?= $(VERSION) DOTFILE_IMAGE = $(subst /,_,$(IMAGE))-$(VERSION) +# Standard OCI descriptive metadata (source, revision, created, ...) shared with +# every other image and with the push workflows - see containers/image-metadata.sh. +DDEV_IMAGE_LABELS = $(shell ../image-metadata.sh labels $(DDEV_IMAGE_TAG) $(notdir $(DOCKER_REPO))) + .PHONY: container push container: container-name - docker build -t $(DOCKER_REPO):$(VERSION) $(DOCKER_ARGS) --label "build-info=$(DOCKER_REPO):$(VERSION) commit=$(shell git describe --tags --always)" --label "com.ddev.image-tag=$(DDEV_IMAGE_TAG)" . + docker build -t $(DOCKER_REPO):$(VERSION) $(DOCKER_ARGS) --label "build-info=$(DOCKER_REPO):$(VERSION) commit=$(shell git describe --tags --always)" $(DDEV_IMAGE_LABELS) . container-name: @echo "container: $(DOCKER_REPO):$(VERSION)" @@ -35,6 +39,6 @@ push: $${tags} \ --label "build-info=$(DOCKER_ORG)/$${item}:$(VERSION) commit=$(shell git describe --tags --always) built $$(date) by $$(id -un) on $$(hostname)" \ --label "maintainer=DDEV " \ - --label "com.ddev.image-tag=$(DDEV_IMAGE_TAG)" \ + $$(../image-metadata.sh labels "$(DDEV_IMAGE_TAG)" "$${item}") \ $(DOCKER_ARGS) . ; \ done diff --git a/containers/db_variants_test.sh b/containers/db_variants_test.sh new file mode 100755 index 00000000000..e75a89f4c92 --- /dev/null +++ b/containers/db_variants_test.sh @@ -0,0 +1,110 @@ +#!/usr/bin/env bash +# db_variants_test.sh - unit tests for ddev-dbserver/variants.sh. +# +# Pure text transforms, no Docker or network. The interesting property is that +# every consumer's view stays mutually consistent, since the whole point of +# variants.txt is that the four of them can't drift apart. +# Run with: +# containers/db_variants_test.sh + +set -eu -o pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +VARIANTS="$SCRIPT_DIR/ddev-dbserver/variants.sh" + +FAILURES=0 + +fail() { + echo "FAIL: $1" >&2 + FAILURES=$((FAILURES + 1)) +} + +pass() { + echo "PASS: $1" +} + +assert_eq() { + local expected="$1" actual="$2" desc="$3" + if [ "$expected" = "$actual" ]; then + pass "$desc" + else + fail "$desc (expected '$expected', got '$actual')" + fi +} + +WORKDIR="$(mktemp -d)" +trap 'rm -rf "$WORKDIR"' EXIT + +# --- A fixture matrix, so these assertions don't move every time a database +# version is added or dropped. +export VARIANTS_FILE="$WORKDIR/variants.txt" +cat > "$VARIANTS_FILE" <<'EOF' +# comment line, and a blank line, both ignored + +mysql_9.7 amd64 arm64 +mysql_5.6 amd64 +mariadb_11.8 amd64 arm64 +mariadb_5.5 amd64 +EOF + +assert_eq "mysql_9.7_both mysql_5.6_amd64 mariadb_11.8_both mariadb_5.5_amd64" \ + "$("$VARIANTS" build-targets amd64)" "build-targets on amd64 covers every variant" +assert_eq "mysql_9.7_both mariadb_11.8_both" \ + "$("$VARIANTS" build-targets arm64)" "build-targets on arm64 omits the amd64-only variants" +assert_eq "mysql_9.7_amd64 mysql_9.7_arm64 mariadb_11.8_amd64 mariadb_11.8_arm64" \ + "$("$VARIANTS" single-arch-targets)" "single-arch-targets covers only multi-arch variants, both ways" +assert_eq "mysql_9.7 mariadb_11.8" \ + "$("$VARIANTS" multi-arch-variants)" "multi-arch-variants is what gets a combined manifest" +assert_eq "mysql_9.7_test mysql_5.6_test mariadb_11.8_test mariadb_5.5_test" \ + "$("$VARIANTS" test-targets amd64)" "test-targets on amd64 covers every variant" +assert_eq "mysql_9.7_test mariadb_11.8_test" \ + "$("$VARIANTS" test-targets arm64)" "test-targets on arm64 omits the amd64-only variants" +assert_eq "ddev-dbserver-mysql-9.7 ddev-dbserver-mysql-5.6 ddev-dbserver-mariadb-11.8 ddev-dbserver-mariadb-5.5" \ + "$("$VARIANTS" repos | tr '\n' ' ' | sed 's/ $//')" "repos names the Docker Hub repositories" +assert_eq "6" "$("$VARIANTS" json | jq 'length')" "json emits one entry per (variant, arch)" +assert_eq "amd64" "$("$VARIANTS" json | jq -r '[.[] | select(.dbtype=="mysql_5.6")] | .[].arch')" \ + "json gives an amd64-only variant exactly one entry" + +if "$VARIANTS" bogus-view >/dev/null 2>&1; then + fail "should reject an unknown view" +else + pass "rejects an unknown view" +fi +if "$VARIANTS" build-targets >/dev/null 2>&1; then + fail "should require a host arch for build-targets" +else + pass "requires a host arch for build-targets" +fi + +# --- Against the real variants.txt: the views have to agree with each other, +# which is what stops the Makefile, image-configs.sh, and the push workflow +# from disagreeing about what exists. +unset VARIANTS_FILE + +real_build_amd64="$("$VARIANTS" build-targets amd64 | wc -w | tr -d '[:space:]')" +real_repos="$("$VARIANTS" repos | wc -l | tr -d '[:space:]')" +assert_eq "$real_build_amd64" "$real_repos" "every variant has a repo and an amd64 build target" + +real_json="$("$VARIANTS" json | jq 'length')" +multi="$("$VARIANTS" multi-arch-variants | wc -w | tr -d '[:space:]')" +single_arch="$(( real_repos - multi ))" +assert_eq "$(( multi * 2 + single_arch ))" "$real_json" "the json matrix is 2 jobs per multi-arch variant plus 1 each for the rest" + +assert_eq "$(( multi * 2 ))" "$("$VARIANTS" single-arch-targets | wc -w | tr -d '[:space:]')" \ + "single-arch-targets is two per multi-arch variant" + +# The default variant `make` builds locally must be in the matrix, or +# image-configs.sh would mark a nonexistent variant as locally built. +if "$VARIANTS" repos | grep -qx "ddev-dbserver-mariadb-11.8"; then + pass "the default variant image-configs.sh builds locally is in the matrix" +else + fail "mariadb_11.8 (image-configs.sh's DDEV_DBSERVER_DEFAULT_TARGET) is missing from variants.txt" +fi + +if [ "$FAILURES" -eq 0 ]; then + echo "All db_variants_test.sh checks passed." + exit 0 +else + echo "$FAILURES db_variants_test.sh check(s) failed." >&2 + exit 1 +fi diff --git a/containers/ddev-dbserver/Makefile b/containers/ddev-dbserver/Makefile index c648aac64f4..74ed2d4f6e8 100644 --- a/containers/ddev-dbserver/Makefile +++ b/containers/ddev-dbserver/Makefile @@ -19,17 +19,11 @@ CURRENT_ARCH=$(shell ../get_arch.sh) # So has to explicitly declare anything it might need from there (like SHELL) SHELL = /bin/bash -BUILD_TARGETS=$(shell if [ "$(CURRENT_ARCH)" = "amd64" ] ; then \ - echo "mysql_9.7_both mysql_8.4_both mysql_8.0_both mysql_5.7_both mysql_5.6_amd64 mysql_5.5_amd64 mariadb_12.3_both mariadb_11.8_both mariadb_11.4_both mariadb_10.11_both mariadb_10.8_both mariadb_10.7_both mariadb_10.6_both mariadb_10.5_both mariadb_10.4_both mariadb_10.3_both mariadb_10.2_both mariadb_10.1_both mariadb_10.0_amd64 mariadb_5.5_amd64"; \ -else \ - echo "mysql_9.7_both mysql_8.4_both mysql_8.0_both mysql_5.7_both mariadb_12.3_both mariadb_11.8_both mariadb_11.4_both mariadb_10.11_both mariadb_10.8_both mariadb_10.7_both mariadb_10.6_both mariadb_10.5_both mariadb_10.4_both mariadb_10.3_both mariadb_10.2_both mariadb_10.1_both"; \ -fi ) -SINGLE_ARCH_TARGETS=mysql_9.7_amd64 mysql_9.7_arm64 mysql_8.4_amd64 mysql_8.4_arm64 mysql_8.0_amd64 mysql_8.0_arm64 mysql_5.7_amd64 mysql_5.7_arm64 mariadb_12.3_amd64 mariadb_12.3_arm64 mariadb_11.8_amd64 mariadb_11.8_arm64 mariadb_11.4_amd64 mariadb_11.4_arm64 mariadb_10.11_amd64 mariadb_10.11_arm64 mariadb_10.8_amd64 mariadb_10.8_arm64 mariadb_10.7_amd64 mariadb_10.7_arm64 mariadb_10.6_amd64 mariadb_10.6_arm64 mariadb_10.5_amd64 mariadb_10.5_arm64 mariadb_10.4_amd64 mariadb_10.4_arm64 mariadb_10.3_amd64 mariadb_10.3_arm64 mariadb_10.2_amd64 mariadb_10.2_arm64 mariadb_10.1_amd64 mariadb_10.1_arm64 -TEST_TARGETS=$(shell if [ "$(CURRENT_ARCH)" = "amd64" ] ; then \ - echo "mysql_9.7_test mysql_8.4_test mysql_8.0_test mysql_5.7_test mysql_5.6_test mysql_5.5_test mariadb_12.3_test mariadb_11.8_test mariadb_11.4_test mariadb_10.11_test mariadb_10.8_test mariadb_10.7_test mariadb_10.6_test mariadb_10.5_test mariadb_10.4_test mariadb_10.3_test mariadb_10.2_test mariadb_10.1_test mariadb_10.0_test mariadb_5.5_test"; \ -else \ - echo "mysql_9.7_test mysql_8.4_test mysql_8.0_test mysql_5.7_test mariadb_12.3_test mariadb_11.8_test mariadb_11.4_test mariadb_10.11_test mariadb_10.8_test mariadb_10.7_test mariadb_10.6_test mariadb_10.5_test mariadb_10.4_test mariadb_10.3_test mariadb_10.2_test mariadb_10.1_test"; \ -fi ) +# The variant matrix lives in variants.txt - see variants.sh for the views. +VARIANTS_SH=./variants.sh +BUILD_TARGETS=$(shell $(VARIANTS_SH) build-targets $(CURRENT_ARCH)) +SINGLE_ARCH_TARGETS=$(shell $(VARIANTS_SH) single-arch-targets) +TEST_TARGETS=$(shell $(VARIANTS_SH) test-targets $(CURRENT_ARCH)) container: build diff --git a/containers/ddev-dbserver/build_image.sh b/containers/ddev-dbserver/build_image.sh index 25e1486734d..9d86b436bd1 100755 --- a/containers/ddev-dbserver/build_image.sh +++ b/containers/ddev-dbserver/build_image.sh @@ -140,10 +140,15 @@ tag_directive="-t ${DOCKER_ORG}/ddev-dbserver-${DB_TYPE}-${DB_MAJOR_VERSION}:${I if [[ ${IMAGE_TAG} =~ ^v[0-9]+\.[0-9]+\.[0-9]+$ ]]; then tag_directive="$tag_directive -t ${DOCKER_ORG}/ddev-dbserver-${DB_TYPE}-${DB_MAJOR_VERSION}:latest" fi + +# Same OCI metadata every other image carries; this directory deliberately does +# not include containers_shared.mk, so it calls the generator itself. +label_directive="$("$(dirname "$0")/../image-metadata.sh" labels "${DDEV_IMAGE_TAG}" "ddev-dbserver-${DB_TYPE}-${DB_MAJOR_VERSION}")" + if [ ! -z ${PUSH:-} ]; then echo "building/pushing ddev/ddev-dbserver-${DB_TYPE}-${DB_MAJOR_VERSION}:${IMAGE_TAG}" set -x - docker buildx build --push --platform ${ARCHS} ${DOCKER_ARGS} --build-arg="BASE_IMAGE=${BASE_IMAGE}" --build-arg="DB_PINNED_VERSION=${DB_PINNED_VERSION}" --build-arg="DB_MAJOR_VERSION=${DB_MAJOR_VERSION}" --build-arg="DDEV_IMAGE_TAG=${DDEV_IMAGE_TAG}" ${tag_directive} . + docker buildx build --push --platform ${ARCHS} ${DOCKER_ARGS} --build-arg="BASE_IMAGE=${BASE_IMAGE}" --build-arg="DB_PINNED_VERSION=${DB_PINNED_VERSION}" --build-arg="DB_MAJOR_VERSION=${DB_MAJOR_VERSION}" --build-arg="DDEV_IMAGE_TAG=${DDEV_IMAGE_TAG}" ${label_directive} ${tag_directive} . set +x fi @@ -151,5 +156,5 @@ fi set -x if [ -z "${PUSH:-}" ]; then echo "Loading to local docker ddev/ddev-dbserver-${DB_TYPE}-${DB_MAJOR_VERSION}:${IMAGE_TAG}" - docker buildx build --load ${DOCKER_ARGS} --build-arg="DB_TYPE=${DB_TYPE}" --build-arg="DB_MAJOR_VERSION=${DB_MAJOR_VERSION}" --build-arg="BASE_IMAGE=${BASE_IMAGE}" --build-arg="DB_PINNED_VERSION=${DB_PINNED_VERSION}" --build-arg="DDEV_IMAGE_TAG=${DDEV_IMAGE_TAG}" ${tag_directive} . + docker buildx build --load ${DOCKER_ARGS} --build-arg="DB_TYPE=${DB_TYPE}" --build-arg="DB_MAJOR_VERSION=${DB_MAJOR_VERSION}" --build-arg="BASE_IMAGE=${BASE_IMAGE}" --build-arg="DB_PINNED_VERSION=${DB_PINNED_VERSION}" --build-arg="DDEV_IMAGE_TAG=${DDEV_IMAGE_TAG}" ${label_directive} ${tag_directive} . fi diff --git a/containers/ddev-dbserver/variants.sh b/containers/ddev-dbserver/variants.sh new file mode 100755 index 00000000000..f32c27622b6 --- /dev/null +++ b/containers/ddev-dbserver/variants.sh @@ -0,0 +1,122 @@ +#!/usr/bin/env bash +# variants.sh [] +# +# Renders the variant matrix in variants.txt for each of its consumers, so the +# list lives in exactly one place. Views: +# +# build-targets make targets for a `make build` on that host: +# _both, or _ for a +# variant that only builds on one. Variants the +# host can't build are omitted. +# single-arch-targets _amd64 _arm64 for every +# multi-arch variant, for one-arch-at-a-time +# builds (a multi-arch push builds each side +# separately, then combines them). +# multi-arch-variants the variants that get a combined manifest +# test-targets _test for what the host can build +# list | for shell consumers +# repos ddev-dbserver-- repository names +# json [{"dbtype":…,"arch":…}, …] for a GitHub matrix + +set -eu -o pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +VARIANTS_FILE="${VARIANTS_FILE:-$SCRIPT_DIR/variants.txt}" + +if [ "$#" -lt 1 ]; then + echo "Usage: $0 [host-arch]" >&2 + exit 2 +fi + +VIEW="$1" +HOST_ARCH="${2:-}" + +VARIANTS=() +ARCHES=() +while read -r variant arches; do + [ -z "$variant" ] && continue + case "$variant" in \#*) continue ;; esac + VARIANTS+=("$variant") + ARCHES+=("$arches") +done < "$VARIANTS_FILE" + +if [ "${#VARIANTS[@]}" -eq 0 ]; then + echo "$0: no variants found in $VARIANTS_FILE" >&2 + exit 1 +fi + +host_can_build() { + local arches="$1" + [ -z "$HOST_ARCH" ] && return 0 + [[ " $arches " == *" $HOST_ARCH "* ]] +} + +require_host_arch() { + if [ -z "$HOST_ARCH" ]; then + echo "$0: $VIEW needs a host arch argument" >&2 + exit 2 + fi +} + +OUT=() +for i in "${!VARIANTS[@]}"; do + variant="${VARIANTS[$i]}" + arches="${ARCHES[$i]}" + # shellcheck disable=SC2086 # arches is a space-separated list + set -- $arches + arch_count="$#" + + case "$VIEW" in + build-targets) + require_host_arch + host_can_build "$arches" || continue + if [ "$arch_count" -gt 1 ]; then + OUT+=("${variant}_both") + else + OUT+=("${variant}_$1") + fi + ;; + single-arch-targets) + [ "$arch_count" -gt 1 ] || continue + for arch in $arches; do + OUT+=("${variant}_${arch}") + done + ;; + multi-arch-variants) + [ "$arch_count" -gt 1 ] || continue + OUT+=("$variant") + ;; + test-targets) + require_host_arch + host_can_build "$arches" || continue + OUT+=("${variant}_test") + ;; + list) + OUT+=("${variant}|${arches}") + ;; + repos) + OUT+=("ddev-dbserver-${variant/_/-}") + ;; + json) + for arch in $arches; do + OUT+=("$(printf '{"dbtype":"%s","arch":"%s"}' "$variant" "$arch")") + done + ;; + *) + echo "$0: unknown view '$VIEW'" >&2 + exit 2 + ;; + esac +done + +case "$VIEW" in + json) + printf '[%s]\n' "$(IFS=,; echo "${OUT[*]}")" + ;; + list|repos) + printf '%s\n' "${OUT[@]}" + ;; + *) + echo "${OUT[*]}" + ;; +esac diff --git a/containers/ddev-dbserver/variants.txt b/containers/ddev-dbserver/variants.txt new file mode 100644 index 00000000000..7e845692b32 --- /dev/null +++ b/containers/ddev-dbserver/variants.txt @@ -0,0 +1,36 @@ +# The ddev-dbserver variant matrix - one line per variant: +# +# _ [ ...] +# +# Single source of truth, read through variants.sh in this directory. Order is +# significant: it's the order build targets are emitted in. +# +# Consumers: +# containers/ddev-dbserver/Makefile BUILD/SINGLE_ARCH/TEST targets +# containers/image-configs.sh automatic build/push flow +# containers/validate-image-repo.sh push allowlist +# .github/workflows/push-tagged-dbimage.yml manual push matrix +# +# Adding or removing a line changes this directory's content hash, so every +# variant is rebuilt and re-pushed under a new tag - which is what has to +# happen, since they all share BaseDBTag. +mysql_9.7 amd64 arm64 +mysql_8.4 amd64 arm64 +mysql_8.0 amd64 arm64 +mysql_5.7 amd64 arm64 +mysql_5.6 amd64 +mysql_5.5 amd64 +mariadb_12.3 amd64 arm64 +mariadb_11.8 amd64 arm64 +mariadb_11.4 amd64 arm64 +mariadb_10.11 amd64 arm64 +mariadb_10.8 amd64 arm64 +mariadb_10.7 amd64 arm64 +mariadb_10.6 amd64 arm64 +mariadb_10.5 amd64 arm64 +mariadb_10.4 amd64 arm64 +mariadb_10.3 amd64 arm64 +mariadb_10.2 amd64 arm64 +mariadb_10.1 amd64 arm64 +mariadb_10.0 amd64 +mariadb_5.5 amd64 diff --git a/containers/ddev-ssh-agent/Dockerfile b/containers/ddev-ssh-agent/Dockerfile index 81e3972dc8f..f1c9d451b9b 100644 --- a/containers/ddev-ssh-agent/Dockerfile +++ b/containers/ddev-ssh-agent/Dockerfile @@ -27,6 +27,7 @@ getent passwd "$(id -u)" >/dev/null 2>&1 || PS1='${debian_chroot:+($debian_chroo BASHRC EOF +# test: trivial fork PR change for #8609 phase 2 manual test 4 HEALTHCHECK --interval=1s --retries=5 --timeout=120s CMD ["/healthcheck.sh"] VOLUME ${SOCKET_DIR} diff --git a/containers/ddev-traefik-router/Dockerfile b/containers/ddev-traefik-router/Dockerfile index cb8e397111c..a697b244d54 100644 --- a/containers/ddev-traefik-router/Dockerfile +++ b/containers/ddev-traefik-router/Dockerfile @@ -10,5 +10,6 @@ RUN chmod ugo+rx /usr/local/bin/monitor-traefik-stderr.sh /usr/local/bin/docker- # Make Traefik commands work without --configFile by using default location # https://doc.traefik.io/traefik/getting-started/configuration-overview/#configuration-file RUN mkdir -p /etc/traefik && ln -s /mnt/ddev-global-cache/traefik/.static_config.yaml /etc/traefik/traefik.yaml +# test: trivial fork PR change for #8609 phase 2 manual test 4 HEALTHCHECK --interval=1s --timeout=120s --retries=1 --start-period=120s CMD /healthcheck.sh ENTRYPOINT ["/usr/local/bin/docker-entrypoint.sh", "/usr/local/bin/traefik"] diff --git a/containers/ddev-webserver/Dockerfile b/containers/ddev-webserver/Dockerfile index 4be9799c89e..133caa5f867 100644 --- a/containers/ddev-webserver/Dockerfile +++ b/containers/ddev-webserver/Dockerfile @@ -539,6 +539,7 @@ RUN chmod -f ugo+rx /usr/local/bin /usr/local/bin/* && \ RUN chmod ugo+w /etc/ssl/certs /usr/local/share/ca-certificates +# test: trivial fork PR change for #8609 phase 2 manual test 4 HEALTHCHECK --interval=1s --retries=120 --timeout=120s --start-period=120s CMD ["/healthcheck.sh"] CMD ["/start.sh"] RUN apt-get -qq clean -y && rm -rf /var/lib/apt/lists/* /tmp/* diff --git a/containers/ddev-webserver/Makefile b/containers/ddev-webserver/Makefile index 0fdd0e56fd8..59d32075812 100644 --- a/containers/ddev-webserver/Makefile +++ b/containers/ddev-webserver/Makefile @@ -16,7 +16,7 @@ DOCKER_REPO ?= $(DOCKER_ORG)/ddev-webserver images: $(DEFAULT_IMAGES) $(DEFAULT_IMAGES): - docker build --label com.ddev.buildhost=${shell hostname} --label com.ddev.image-tag=$(DDEV_IMAGE_TAG) --target=$@ -t $(DOCKER_ORG)/$@:$(VERSION) $(DOCKER_ARGS) . + docker build --label com.ddev.buildhost=${shell hostname} $(shell ../image-metadata.sh labels $(DDEV_IMAGE_TAG) $@) --target=$@ -t $(DOCKER_ORG)/$@:$(VERSION) $(DOCKER_ARGS) . test: images diff --git a/containers/image-configs.sh b/containers/image-configs.sh new file mode 100644 index 00000000000..d08b6d70ea7 --- /dev/null +++ b/containers/image-configs.sh @@ -0,0 +1,50 @@ +#!/usr/bin/env bash +# image-configs.sh - the set of images the automatic build/push flow covers. +# +# Sourced (not executed) by wait-for-images.sh and by image-build-push.yml's +# detect job so the two can't drift apart. Keep in sync with the Makefile's +# autotag-images target. +# +# Fields, pipe-separated: +# repo_suffix Docker Hub repository under $DOCKER_ORG +# tag_var variable name in pkg/versionconstants/versionconstants.go +# hash_paths space-separated paths fed to hash-paths.sh +# make_dir directory under containers/ to run make in +# make_target make target that builds it +# arch_suffixed "true" if the target name takes an _ suffix +# arches space-separated architectures to build +# built_by_make "true" if `make` at the repo root builds it locally; +# a "false" image can only come from the registry +# extra_repo_suffixes further repositories the same target produces + +# shellcheck disable=SC2034 # consumed by whatever sources this file +DDEV_IMAGE_CONFIGS=( + 'ddev-webserver|WebTag|containers/ddev-webserver containers/containers_shared.mk|ddev-webserver|images|false|amd64 arm64|true|ddev-webserver-prod' + 'ddev-traefik-router|TraefikRouterTag|containers/ddev-traefik-router containers/containers_shared.mk|ddev-traefik-router|container|false|amd64 arm64|true|' + 'ddev-ssh-agent|SSHAuthTag|containers/ddev-ssh-agent containers/containers_shared.mk|ddev-ssh-agent|container|false|amd64 arm64|true|' + 'ddev-xhgui|XhguiTag|containers/ddev-xhgui containers/containers_shared.mk|ddev-xhgui|container|false|amd64 arm64|true|' +) + +# Every ddev-dbserver variant shares one BaseDBTag (see GetDBImage() in +# pkg/docker/images.go), so a change under containers/ddev-dbserver moves the +# tag for all of them at once and every variant has to be rebuilt and pushed +# together - otherwise TestDdevAllDatabases and every test pinning a +# non-default database pulls a tag nobody published. `make` builds only the +# default variant locally, to keep an unrelated rebuild cheap; the rest exist +# only in the registry. +# +# The variant list comes from containers/ddev-dbserver/variants.txt via +# variants.sh, shared with that directory's Makefile and with +# push-tagged-dbimage.yml. +DDEV_DBSERVER_HASH_PATHS='containers/ddev-dbserver containers/get_arch.sh' +DDEV_DBSERVER_DEFAULT_TARGET='mariadb_11.8' + +while IFS='|' read -r _target _arches; do + [ -z "$_target" ] && continue + _built_by_make=false + [ "$_target" = "$DDEV_DBSERVER_DEFAULT_TARGET" ] && _built_by_make=true + DDEV_IMAGE_CONFIGS+=( + "ddev-dbserver-${_target/_/-}|BaseDBTag|${DDEV_DBSERVER_HASH_PATHS}|ddev-dbserver|${_target}|true|${_arches}|${_built_by_make}|" + ) +done < <("$(dirname "${BASH_SOURCE[0]}")/ddev-dbserver/variants.sh" list) +unset _target _arches _built_by_make diff --git a/containers/image-metadata.sh b/containers/image-metadata.sh new file mode 100755 index 00000000000..cb8eac92b75 --- /dev/null +++ b/containers/image-metadata.sh @@ -0,0 +1,73 @@ +#!/usr/bin/env bash +# image-metadata.sh [] +# +# One definition of the descriptive metadata every DDEV image carries, in +# whichever form the caller needs: +# +# pairs key=value lines +# labels --label "key=value" ... for a docker/buildx build +# annotations --annotation "index:key=value" ... for `imagetools create` +# +# Labels and annotations are not interchangeable, which is why both exist: +# +# - Labels live in the image config, so they travel with a `docker pull` and +# `docker inspect` shows them offline. That is what a support report needs. +# - Annotations live on the manifest index, which is what a tag points at, so +# they describe the tag as a whole rather than one platform, and a registry +# can show them without pulling. A tag has no comment field; an index +# annotation is the closest standard equivalent. +# +# Env: +# DDEV_GIT_REVISION - commit to record (default: current HEAD) +# SOURCE_DATE_EPOCH - build timestamp override, for reproducibility + +set -eu -o pipefail + +if [ "$#" -lt 2 ]; then + echo "Usage: $0 <labels|annotations|pairs> <image-tag> [<title>]" >&2 + exit 2 +fi + +FORM="$1" +IMAGE_TAG="$2" +TITLE="${3:-}" + +REVISION="${DDEV_GIT_REVISION:-$(git rev-parse HEAD 2>/dev/null || echo unknown)}" +if [ -n "${SOURCE_DATE_EPOCH:-}" ]; then + CREATED="$(date -u -r "$SOURCE_DATE_EPOCH" +%Y-%m-%dT%H:%M:%SZ 2>/dev/null || date -u -d "@$SOURCE_DATE_EPOCH" +%Y-%m-%dT%H:%M:%SZ)" +else + CREATED="$(date -u +%Y-%m-%dT%H:%M:%SZ)" +fi + +PAIRS=( + "org.opencontainers.image.source=https://github.com/ddev/ddev" + "org.opencontainers.image.url=https://ddev.com" + "org.opencontainers.image.documentation=https://docs.ddev.com" + "org.opencontainers.image.vendor=DDEV" + "org.opencontainers.image.licenses=Apache-2.0" + "org.opencontainers.image.revision=${REVISION}" + "org.opencontainers.image.version=${IMAGE_TAG}" + "org.opencontainers.image.created=${CREATED}" + # The tag is a bare content hash, so this is the one field that says what + # the image is for without cross-referencing versionconstants.go. + "com.ddev.image-tag=${IMAGE_TAG}" +) +[ -n "$TITLE" ] && PAIRS+=("org.opencontainers.image.title=${TITLE}") + +case "$FORM" in + pairs) + printf '%s\n' "${PAIRS[@]}" + ;; + labels) + for p in "${PAIRS[@]}"; do printf -- '--label %q ' "$p"; done + echo + ;; + annotations) + for p in "${PAIRS[@]}"; do printf -- '--annotation %q ' "index:$p"; done + echo + ;; + *) + echo "$0: unknown form '$FORM'" >&2 + exit 2 + ;; +esac diff --git a/containers/image-tag-for.sh b/containers/image-tag-for.sh new file mode 100755 index 00000000000..0c5abb60077 --- /dev/null +++ b/containers/image-tag-for.sh @@ -0,0 +1,38 @@ +#!/usr/bin/env bash +# image-tag-for.sh <repo-suffix> +# +# Prints the tag this checkout needs for one image, by repository suffix +# (ddev-webserver, ddev-dbserver-mysql-8.0, ...). A thin lookup over +# image-configs.sh so the manual push workflows can default their tag input +# instead of asking a human to retype a content hash - the mismatch that +# causes is the whole reason wait-for-images.sh has to fail loudly. +# +# Exits non-zero for a repository the automatic flow doesn't cover +# (test-ssh-server, say), where there is no content hash to derive and the +# caller has to supply a tag. + +set -eu -o pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + +if [ "$#" -ne 1 ]; then + echo "Usage: $0 <repo-suffix>" >&2 + exit 2 +fi + +WANTED="$1" + +# shellcheck source=containers/image-configs.sh +source "$SCRIPT_DIR/image-configs.sh" + +for entry in "${DDEV_IMAGE_CONFIGS[@]}"; do + IFS='|' read -r repo_suffix tag_var hash_paths _ <<< "$entry" + [ "$repo_suffix" = "$WANTED" ] || continue + # shellcheck disable=SC2086 # hash_paths is a space-separated path list + read -r _ tag <<< "$("$SCRIPT_DIR/required-image-tag.sh" "$tag_var" $hash_paths)" + echo "$tag" + exit 0 +done + +echo "image-tag-for.sh: '${WANTED}' is not one of the content-addressed images; pass a tag explicitly" >&2 +exit 1 diff --git a/containers/registry-tag-exists.sh b/containers/registry-tag-exists.sh new file mode 100755 index 00000000000..ca0c3bfcc5c --- /dev/null +++ b/containers/registry-tag-exists.sh @@ -0,0 +1,19 @@ +#!/usr/bin/env bash +# registry-tag-exists.sh <image-repo> <tag> +# +# Checks whether <image-repo>:<tag> already exists in the registry, without +# pulling it. Exit 0 if it exists, exit 1 if it doesn't (or the registry +# can't be reached). No local Docker daemon build/pull is triggered either +# way - this only talks to the registry. + +set -eu -o pipefail + +if [ "$#" -ne 2 ]; then + echo "Usage: $0 <image-repo> <tag>" >&2 + exit 2 +fi + +IMAGE_REPO="$1" +TAG="$2" + +docker buildx imagetools inspect "${IMAGE_REPO}:${TAG}" >/dev/null 2>&1 diff --git a/containers/registry_tag_exists_test.sh b/containers/registry_tag_exists_test.sh new file mode 100755 index 00000000000..dab6b7c28b4 --- /dev/null +++ b/containers/registry_tag_exists_test.sh @@ -0,0 +1,88 @@ +#!/usr/bin/env bash +# registry_tag_exists_test.sh - unit tests for registry-tag-exists.sh. +# +# Exercises the exists/doesn't-exist/unreachable outcomes against a stubbed +# `docker`, without talking to a real registry. +# Run with: +# containers/registry_tag_exists_test.sh + +set -eu -o pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +REGISTRY_TAG_EXISTS="$SCRIPT_DIR/registry-tag-exists.sh" + +FAILURES=0 + +fail() { + echo "FAIL: $1" >&2 + FAILURES=$((FAILURES + 1)) +} + +pass() { + echo "PASS: $1" +} + +WORKDIR="$(mktemp -d)" +trap 'rm -rf "$WORKDIR"' EXIT + +# --- Stub `docker`, controlled by a marker file listing which refs "exist". +BINDIR="$WORKDIR/bin" +mkdir -p "$BINDIR" +export DOCKER_EXISTING_REF_FILE="$WORKDIR/docker_existing_refs" +export DOCKER_CALL_LOG="$WORKDIR/docker_calls.log" +: > "$DOCKER_EXISTING_REF_FILE" +: > "$DOCKER_CALL_LOG" +cat > "$BINDIR/docker" <<'DOCKEREOF' +#!/usr/bin/env bash +set -eu -o pipefail +echo "$*" >> "$DOCKER_CALL_LOG" +if [ "$1" = "buildx" ] && [ "$2" = "imagetools" ] && [ "$3" = "inspect" ]; then + ref="$4" + grep -qxF "$ref" "$DOCKER_EXISTING_REF_FILE" + exit $? +fi +echo "docker stub: unexpected invocation: $*" >&2 +exit 1 +DOCKEREOF +chmod +x "$BINDIR/docker" +export PATH="$BINDIR:$PATH" + +# 1. Missing tag -> non-zero exit, no crash. +if "$REGISTRY_TAG_EXISTS" ddev/dummy-image missing-0123456789 >/dev/null 2>&1; then + fail "should report missing tag as not existing" +else + pass "reports missing tag as not existing" +fi + +# 2. Existing tag -> zero exit. +echo "ddev/dummy-image:present-0123456789" > "$DOCKER_EXISTING_REF_FILE" +if "$REGISTRY_TAG_EXISTS" ddev/dummy-image present-0123456789 >/dev/null 2>&1; then + pass "reports existing tag as existing" +else + fail "should report existing tag as existing" +fi + +# 3. Exactly one docker call per invocation - no retries/loops in this script +# (retry/backoff, if wanted, is the caller's job, e.g. wait-for-images.sh). +# BSD wc pads its output with spaces; GNU wc does not. +calls="$(wc -l < "$DOCKER_CALL_LOG" | tr -d '[:space:]')" +if [ "$calls" -eq 2 ]; then + pass "made exactly one docker call per invocation" +else + fail "expected 2 total docker calls across both invocations, got $calls" +fi + +# 4. Usage error on wrong argument count. +if "$REGISTRY_TAG_EXISTS" only-one-arg >/dev/null 2>&1; then + fail "should reject wrong argument count" +else + pass "rejects wrong argument count" +fi + +if [ "$FAILURES" -eq 0 ]; then + echo "All registry_tag_exists_test.sh checks passed." + exit 0 +else + echo "$FAILURES registry_tag_exists_test.sh check(s) failed." >&2 + exit 1 +fi diff --git a/containers/required-image-tag.sh b/containers/required-image-tag.sh new file mode 100755 index 00000000000..eab82c3d546 --- /dev/null +++ b/containers/required-image-tag.sh @@ -0,0 +1,49 @@ +#!/usr/bin/env bash +# required-image-tag.sh <TagVarName> <hash-path> [<hash-path> ...] +# +# Prints "<state> <tag>" for the image tag this checkout resolves to. The tag +# is the bare content hash either way - it does not depend on the branch, the +# fork, or the registry it was published to - so every caller agrees on it. +# The state says where that leaves the checkout: +# +# committed <tag> versionconstants.go already names this tag, so it is +# what ddev pulls and it has to exist in the registry. +# recomputed <tag> the content changed, so autotag.sh rewrites +# versionconstants.go to <tag> and, for an image `make` +# builds, produces it locally. +# +# Env: +# HASH_LEN - hash length in hex chars (default 10, must match +# hash-paths.sh) +# VERSIONCONSTANTS_FILE - path to versionconstants.go + +set -eu -o pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +REPO_ROOT="$(git rev-parse --show-toplevel)" +HASH_LEN="${HASH_LEN:-10}" +VERSIONCONSTANTS_FILE="${VERSIONCONSTANTS_FILE:-$REPO_ROOT/pkg/versionconstants/versionconstants.go}" + +if [ "$#" -lt 2 ]; then + echo "Usage: $0 <TagVarName> <hash-path> [<hash-path> ...]" >&2 + exit 2 +fi + +TAG_VAR="$1"; shift + +CURRENT_HASH="$(HASH_LEN="$HASH_LEN" "$SCRIPT_DIR/hash-paths.sh" "$@")" + +EXISTING_TAG="$(grep -E "^var ${TAG_VAR} = " "$VERSIONCONSTANTS_FILE" 2>/dev/null | sed -E "s/^var ${TAG_VAR} = \"([^\"]*)\".*/\\1/" || true)" +if [ -z "$EXISTING_TAG" ]; then + echo "required-image-tag.sh: could not find 'var ${TAG_VAR} = \"...\"' in $VERSIONCONSTANTS_FILE" >&2 + exit 1 +fi + +# Exact, not a trailing-hash match: a value still in the old <branch>-<hash> +# form is what ddev would pull, and it isn't this tag, so it counts as stale +# and `make` migrates the line. +if [ "$EXISTING_TAG" = "$CURRENT_HASH" ]; then + echo "committed ${CURRENT_HASH}" +else + echo "recomputed ${CURRENT_HASH}" +fi diff --git a/containers/required_image_tag_test.sh b/containers/required_image_tag_test.sh new file mode 100755 index 00000000000..50de5d875ca --- /dev/null +++ b/containers/required_image_tag_test.sh @@ -0,0 +1,168 @@ +#!/usr/bin/env bash +# required_image_tag_test.sh - unit tests for required-image-tag.sh. +# +# Runs against a throwaway versionconstants.go and this checkout's real +# content hashes. No Docker, no network. +# Run with: +# containers/required_image_tag_test.sh + +set -eu -o pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +REQUIRED_IMAGE_TAG="$SCRIPT_DIR/required-image-tag.sh" +HASH_PATHS="$SCRIPT_DIR/hash-paths.sh" + +FAILURES=0 + +fail() { + echo "FAIL: $1" >&2 + FAILURES=$((FAILURES + 1)) +} + +pass() { + echo "PASS: $1" +} + +assert_eq() { + local expected="$1" actual="$2" desc="$3" + if [ "$expected" = "$actual" ]; then + pass "$desc" + else + fail "$desc (expected '$expected', got '$actual')" + fi +} + +WORKDIR="$(mktemp -d)" +trap 'rm -rf "$WORKDIR"' EXIT + +export VERSIONCONSTANTS_FILE="$WORKDIR/versionconstants.go" +HASH_PATH_ARGS=(containers/ddev-xhgui containers/containers_shared.mk) +CURRENT_HASH="$("$HASH_PATHS" "${HASH_PATH_ARGS[@]}")" + +# 1. Committed value already the current hash -> committed, and the tag is +# that hash. This is the case that makes wait-for-images.sh wait. +echo "var XhguiTag = \"${CURRENT_HASH}\" // some_branch-${CURRENT_HASH}" > "$VERSIONCONSTANTS_FILE" +OUTPUT="$("$REQUIRED_IMAGE_TAG" XhguiTag "${HASH_PATH_ARGS[@]}")" +assert_eq "committed ${CURRENT_HASH}" "$OUTPUT" "reports committed when versionconstants.go already names this hash" + +# 2. Content no longer matches -> recomputed, same tag. The tag never depends +# on the branch, which is what lets detect, the runner, and `make` agree. +echo "var XhguiTag = \"0000000000\"" > "$VERSIONCONSTANTS_FILE" +OUTPUT="$("$REQUIRED_IMAGE_TAG" XhguiTag "${HASH_PATH_ARGS[@]}")" +assert_eq "recomputed ${CURRENT_HASH}" "$OUTPUT" "reports recomputed, still with the bare content hash" + +# 3. A value still in the old <branch>-<hash> form is what ddev would pull and +# it isn't this tag, so it counts as stale and `make` migrates the line. +echo "var XhguiTag = \"an_old_branch-${CURRENT_HASH}\"" > "$VERSIONCONSTANTS_FILE" +OUTPUT="$("$REQUIRED_IMAGE_TAG" XhguiTag "${HASH_PATH_ARGS[@]}")" +assert_eq "recomputed ${CURRENT_HASH}" "$OUTPUT" "treats a legacy branch-prefixed value as stale" + +# 4. The branch the caller happens to be on cannot change the answer. +echo "var XhguiTag = \"${CURRENT_HASH}\"" > "$VERSIONCONSTANTS_FILE" +assert_eq "$("$REQUIRED_IMAGE_TAG" XhguiTag "${HASH_PATH_ARGS[@]}")" \ + "$(REQUIRED_IMAGE_TAG_BRANCH='some/other branch' "$REQUIRED_IMAGE_TAG" XhguiTag "${HASH_PATH_ARGS[@]}")" \ + "the result is independent of the branch" + +if "$SCRIPT_DIR/validate-image-tag.sh" "$CURRENT_HASH" >/dev/null 2>&1; then + pass "the bare hash tag is one validate-image-tag.sh accepts" +else + fail "validate-image-tag.sh should accept the bare hash '$CURRENT_HASH'" +fi + +# 5. A missing tag variable is a hard error, not an empty tag. +echo "var SomethingElse = \"whatever\"" > "$VERSIONCONSTANTS_FILE" +OUTPUT="$("$REQUIRED_IMAGE_TAG" XhguiTag "${HASH_PATH_ARGS[@]}" 2>&1)" && RC=0 || RC=$? +if [ "$RC" -ne 0 ]; then + pass "errors out when the tag variable isn't in versionconstants.go" +else + fail "should error out when the tag variable isn't in versionconstants.go" +fi +case "$OUTPUT" in + *"could not find"*XhguiTag*) pass "missing-variable message names the variable" ;; + *) fail "missing-variable message should name the variable: $OUTPUT" ;; +esac + +# 6. Usage error on too few arguments. +if "$REQUIRED_IMAGE_TAG" XhguiTag >/dev/null 2>&1; then + fail "should reject a missing hash path" +else + pass "rejects a missing hash path" +fi + +# --- image-tag-for.sh, the lookup the manual push workflows use to default +# their tag input. Against the real image-configs.sh and this checkout. +unset VERSIONCONSTANTS_FILE +IMAGE_TAG_FOR="$SCRIPT_DIR/image-tag-for.sh" + +web_tag="$("$IMAGE_TAG_FOR" ddev-webserver)" +web_hash="$("$HASH_PATHS" containers/ddev-webserver containers/containers_shared.mk)" +case "$web_tag" in + *"$web_hash") pass "image-tag-for.sh resolves ddev-webserver to a tag with the current hash" ;; + *) fail "ddev-webserver tag '$web_tag' should end in '$web_hash'" ;; +esac + +# Every db variant shares BaseDBTag, so the lookup has to agree across them. +assert_eq "$("$IMAGE_TAG_FOR" ddev-dbserver-mariadb-11.8)" "$("$IMAGE_TAG_FOR" ddev-dbserver-mysql-8.0)" \ + "image-tag-for.sh gives every db variant the same tag" + +# An image the automatic flow doesn't cover has no hash to derive, so the +# caller has to be told rather than handed a wrong tag. +if "$IMAGE_TAG_FOR" test-ssh-server >/dev/null 2>&1; then + fail "should refuse an image that isn't content-addressed" +else + pass "refuses an image that isn't content-addressed" +fi +if "$IMAGE_TAG_FOR" no-such-image >/dev/null 2>&1; then + fail "should refuse an unknown image" +else + pass "refuses an unknown image" +fi + +# --- check-image-tags.sh, the `make staticrequired` gate. Driven with a +# throwaway versionconstants.go so it can be failed on purpose. +CHECK_IMAGE_TAGS="$SCRIPT_DIR/check-image-tags.sh" + +# shellcheck source=containers/image-configs.sh +source "$SCRIPT_DIR/image-configs.sh" + +CURRENT_FILE="$WORKDIR/current_versionconstants.go" +: > "$CURRENT_FILE" +seen="" +for entry in "${DDEV_IMAGE_CONFIGS[@]}"; do + IFS='|' read -r _ tag_var hash_paths _ <<< "$entry" + case " $seen " in *" $tag_var "*) continue ;; esac + seen="$seen $tag_var" + # shellcheck disable=SC2086 # hash_paths is a space-separated path list + echo "var ${tag_var} = \"$("$HASH_PATHS" $hash_paths)\"" >> "$CURRENT_FILE" +done + +if VERSIONCONSTANTS_FILE="$CURRENT_FILE" "$CHECK_IMAGE_TAGS" >/dev/null 2>&1; then + pass "check-image-tags.sh passes when every tag matches the content" +else + fail "check-image-tags.sh should pass when every tag matches the content" +fi + +STALE_FILE="$WORKDIR/stale_versionconstants.go" +sed 's/= "[0-9a-f]*"/= "0000000000"/' "$CURRENT_FILE" > "$STALE_FILE" +OUTPUT="$(VERSIONCONSTANTS_FILE="$STALE_FILE" "$CHECK_IMAGE_TAGS" 2>&1)" && RC=0 || RC=$? +if [ "$RC" -ne 0 ]; then + pass "check-image-tags.sh fails when versionconstants.go is stale" +else + fail "check-image-tags.sh should fail when versionconstants.go is stale" +fi +case "$OUTPUT" in + *"run 'make'"*) pass "the failure says how to fix it" ;; + *) fail "the failure should tell the contributor to run make: $OUTPUT" ;; +esac +case "$OUTPUT" in + *WebTag*BaseDBTag*|*BaseDBTag*WebTag*) pass "the failure names which tags are stale" ;; + *) fail "the failure should name the stale tags: $OUTPUT" ;; +esac + +if [ "$FAILURES" -eq 0 ]; then + echo "All required_image_tag_test.sh checks passed." + exit 0 +else + echo "$FAILURES required_image_tag_test.sh check(s) failed." >&2 + exit 1 +fi diff --git a/containers/validate-image-repo.sh b/containers/validate-image-repo.sh new file mode 100755 index 00000000000..a17ab7c4a35 --- /dev/null +++ b/containers/validate-image-repo.sh @@ -0,0 +1,55 @@ +#!/usr/bin/env bash +# validate-image-repo.sh <image-repo> +# +# Validates a `<org>/<name>` repository string before it's used in any `docker +# push`/`docker buildx imagetools create` command. The companion to +# validate-image-tag.sh: image-push.yml reads the repository list out of a +# build artifact produced by a job that may have run untrusted (fork PR) +# content, so without this a fork could name any repository the push +# credential can write to. +# +# Env: +# DOCKER_ORG - the only organization a push is allowed to target (required) + +set -eu -o pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + +ALLOWED_SUFFIXES=( + ddev-webserver + ddev-webserver-prod + ddev-traefik-router + ddev-ssh-agent + ddev-xhgui +) + +# The db repositories are exactly the variants in +# containers/ddev-dbserver/variants.txt - no pattern match, so a repository +# this project doesn't publish can't slip through on shape alone. +while IFS= read -r _repo; do + [ -n "$_repo" ] && ALLOWED_SUFFIXES+=("$_repo") +done < <("$SCRIPT_DIR/ddev-dbserver/variants.sh" repos) + +if [ "$#" -ne 1 ]; then + echo "Usage: $0 <image-repo>" >&2 + exit 2 +fi + +REPO="$1" +DOCKER_ORG="${DOCKER_ORG:?validate-image-repo.sh: DOCKER_ORG must be set}" + +if [ "${REPO%%/*}" != "$DOCKER_ORG" ] || [ "$REPO" = "${REPO#*/}" ]; then + echo "validate-image-repo.sh: '${REPO}' is not under the '${DOCKER_ORG}/' organization" >&2 + exit 1 +fi + +SUFFIX="${REPO#*/}" + +for allowed in "${ALLOWED_SUFFIXES[@]}"; do + if [ "$SUFFIX" = "$allowed" ]; then + exit 0 + fi +done + +echo "validate-image-repo.sh: '${REPO}' is not one of the repositories this flow may push" >&2 +exit 1 diff --git a/containers/validate-image-tag.sh b/containers/validate-image-tag.sh new file mode 100755 index 00000000000..40daac3cb40 --- /dev/null +++ b/containers/validate-image-tag.sh @@ -0,0 +1,69 @@ +#!/usr/bin/env bash +# validate-image-tag.sh <tag> +# +# Validates a content-addressed image tag before it's used in any `docker +# push`/`docker buildx imagetools create` command. This is the trusted-side +# check on a tag string that arrived via a build artifact from a job that +# may have run untrusted (fork PR) content - see image-push.yml. +# +# Two forms are accepted: +# <hash> the canonical tag, a bare content hash - what +# versionconstants.go holds and what ddev pulls +# <name>-<hash> the human-readable alias published alongside it, whose +# prefix is a sanitized branch name +# +# Requires: +# - strict charset, matching the same sanitization autotag.sh applies, and +# a leading character Docker actually accepts in a tag +# - must end in exactly HASH_LEN lowercase hex characters (the part +# tooling treats as authoritative) +# - neither the whole tag nor an alias prefix may be a reserved literal +# ("latest") or a release-tag shape (vX.Y.Z), so a fork that names its +# branch v1.25.0 can't publish something that reads as a release +# +# Env: +# HASH_LEN - hash length in hex chars (default 10, must match hash-paths.sh) + +set -eu -o pipefail + +HASH_LEN="${HASH_LEN:-10}" + +RESERVED_TAGS=(latest stable edge) + +if [ "$#" -ne 1 ]; then + echo "Usage: $0 <tag>" >&2 + exit 2 +fi + +TAG="$1" + +reject_reserved() { + local candidate="$1" what="$2" + for reserved in "${RESERVED_TAGS[@]}"; do + if [ "$candidate" = "$reserved" ]; then + echo "validate-image-tag.sh: '${TAG}' ${what} is the reserved tag '${reserved}'" >&2 + exit 1 + fi + done + if [[ "$candidate" =~ ^v[0-9]+\.[0-9]+\.[0-9]+$ ]]; then + echo "validate-image-tag.sh: '${TAG}' ${what} looks like a release tag, not a content-hash tag" >&2 + exit 1 + fi +} + +reject_reserved "$TAG" "is" + +if [[ "$TAG" =~ ^[0-9a-f]{${HASH_LEN}}$ ]]; then + exit 0 +fi + +if ! [[ "$TAG" =~ ^[A-Za-z0-9_][A-Za-z0-9_.-]*-[0-9a-f]{${HASH_LEN}}$ ]]; then + echo "validate-image-tag.sh: '${TAG}' is neither a ${HASH_LEN}-hex-char hash nor <name>-<hash>" >&2 + exit 1 +fi + +# Without this, "latest-0123456789" or "v1.2.3-0123456789" would sail through +# the format check above and land next to the real tags in the registry. +reject_reserved "${TAG%-*}" "starts with what" + +exit 0 diff --git a/containers/validate_image_repo_test.sh b/containers/validate_image_repo_test.sh new file mode 100755 index 00000000000..33e88a3b6b8 --- /dev/null +++ b/containers/validate_image_repo_test.sh @@ -0,0 +1,91 @@ +#!/usr/bin/env bash +# validate_image_repo_test.sh - unit tests for validate-image-repo.sh. +# +# Pure string checks, no external stubs needed. +# Run with: +# containers/validate_image_repo_test.sh + +set -eu -o pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +VALIDATE="$SCRIPT_DIR/validate-image-repo.sh" + +FAILURES=0 + +fail() { + echo "FAIL: $1" >&2 + FAILURES=$((FAILURES + 1)) +} + +pass() { + echo "PASS: $1" +} + +export DOCKER_ORG=ddev + +assert_valid() { + local repo="$1" + if "$VALIDATE" "$repo" >/dev/null 2>&1; then + pass "accepts '$repo'" + else + fail "should have accepted '$repo'" + fi +} + +assert_invalid() { + local repo="$1" desc="$2" + if "$VALIDATE" "$repo" >/dev/null 2>&1; then + fail "should have rejected $desc ('$repo')" + else + pass "rejects $desc ('$repo')" + fi +} + +assert_valid "ddev/ddev-webserver" +assert_valid "ddev/ddev-webserver-prod" +assert_valid "ddev/ddev-traefik-router" +assert_valid "ddev/ddev-ssh-agent" +assert_valid "ddev/ddev-xhgui" +assert_valid "ddev/ddev-dbserver-mariadb-11.8" +assert_valid "ddev/ddev-dbserver-mysql-8.0" + +assert_invalid "someoneelse/ddev-webserver" "another organization" +assert_invalid "ddev-webserver" "a bare name with no organization" +assert_invalid "ddev/ddev-webserver-evil" "an unknown repository in the right org" +assert_invalid "ddev/ddev-dbserver-postgres-16" "a db engine the flow doesn't build" +assert_invalid "ddev/ddev-dbserver-mariadb-99.9" "a correctly-shaped db variant that isn't in variants.txt" +assert_invalid "ddev/../../etc/passwd" "a path-traversal attempt" +assert_invalid "ddevhq/ddev-webserver" "an org that merely starts the same" + +# The db repositories are the variants themselves, so the allowlist tracks +# containers/ddev-dbserver/variants.txt rather than a name pattern. +while IFS= read -r suffix; do + [ -n "$suffix" ] && assert_valid "ddev/${suffix}" +done < <("$SCRIPT_DIR/ddev-dbserver/variants.sh" repos) + +# The org comes from a trusted workflow variable, so a differing DOCKER_ORG +# must move the whole allowlist rather than widen it. +if DOCKER_ORG=ddevhq "$VALIDATE" "ddevhq/ddev-webserver" >/dev/null 2>&1; then + pass "accepts the configured org when DOCKER_ORG differs" +else + fail "should accept the configured org when DOCKER_ORG differs" +fi +if DOCKER_ORG=ddevhq "$VALIDATE" "ddev/ddev-webserver" >/dev/null 2>&1; then + fail "should reject the default org when DOCKER_ORG points elsewhere" +else + pass "rejects the default org when DOCKER_ORG points elsewhere" +fi + +if env -u DOCKER_ORG "$VALIDATE" "ddev/ddev-webserver" >/dev/null 2>&1; then + fail "should refuse to run with DOCKER_ORG unset" +else + pass "refuses to run with DOCKER_ORG unset" +fi + +if [ "$FAILURES" -eq 0 ]; then + echo "All validate_image_repo_test.sh checks passed." + exit 0 +else + echo "$FAILURES validate_image_repo_test.sh check(s) failed." >&2 + exit 1 +fi diff --git a/containers/validate_image_tag_test.sh b/containers/validate_image_tag_test.sh new file mode 100755 index 00000000000..6311ce1a35f --- /dev/null +++ b/containers/validate_image_tag_test.sh @@ -0,0 +1,89 @@ +#!/usr/bin/env bash +# validate_image_tag_test.sh - unit tests for validate-image-tag.sh. +# +# Pure string-format checks, no external stubs needed. +# Run with: +# containers/validate_image_tag_test.sh + +set -eu -o pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +VALIDATE="$SCRIPT_DIR/validate-image-tag.sh" + +FAILURES=0 + +fail() { + echo "FAIL: $1" >&2 + FAILURES=$((FAILURES + 1)) +} + +pass() { + echo "PASS: $1" +} + +assert_valid() { + local tag="$1" + if "$VALIDATE" "$tag" >/dev/null 2>&1; then + pass "accepts valid tag '$tag'" + else + fail "should have accepted valid tag '$tag'" + fi +} + +assert_invalid() { + local tag="$1" desc="$2" + if "$VALIDATE" "$tag" >/dev/null 2>&1; then + fail "should have rejected $desc ('$tag')" + else + pass "rejects $desc ('$tag')" + fi +} + +assert_rejected_because() { + local tag="$1" needle="$2" desc="$3" + local output + if output="$("$VALIDATE" "$tag" 2>&1)"; then + fail "should have rejected $desc ('$tag')" + elif [[ "$output" == *"$needle"* ]]; then + pass "rejects $desc ('$tag') as expected" + else + fail "rejected $desc ('$tag') for the wrong reason: $output" + fi +} + +# The canonical form: a bare content hash, what versionconstants.go holds. +assert_valid "0123456789" +assert_valid "36bceca65e" + +# The readable alias published alongside it. +assert_valid "20260721_rfay_content_addressed_image_tags-36bceca65e" +assert_valid "main-0123456789" +assert_valid "v1.2.3-rc1-0123456789" + +assert_invalid "0123456789a" "an 11-hex-char bare tag" +assert_invalid "012345678" "a 9-hex-char bare tag" +assert_invalid "0123456789A" "a bare tag with an uppercase hex digit" + +assert_invalid "latest" "the reserved literal 'latest'" +assert_invalid "stable" "the reserved literal 'stable'" +assert_invalid "v1.2.3" "a bare release tag" +assert_invalid "latest-0123456789a" "a fake tag with an 11-char hash suffix" +assert_invalid "latest-012345678" "a fake tag with a 9-char hash suffix" +assert_invalid "no-hash-suffix" "a tag without a hex hash suffix" +assert_invalid "bad chars!-0123456789" "a tag with disallowed characters" +assert_invalid "UPPERHASH-0123456789AB" "a tag with an uppercase hash suffix" +assert_invalid "-leading-dash-0123456789" "a tag Docker would reject for its leading dash" +assert_invalid ".leading-dot-0123456789" "a tag Docker would reject for its leading dot" + +# A well-formed hash suffix must not be a way to smuggle a tag that reads as +# an official one; these are the checks a format-only validator would miss. +assert_rejected_because "latest-0123456789" "reserved tag" "'latest' dressed up with a hash suffix" +assert_rejected_because "v1.2.3-0123456789" "release tag" "a release tag dressed up with a hash suffix" + +if [ "$FAILURES" -eq 0 ]; then + echo "All validate_image_tag_test.sh checks passed." + exit 0 +else + echo "$FAILURES validate_image_tag_test.sh check(s) failed." >&2 + exit 1 +fi diff --git a/containers/wait-for-images.sh b/containers/wait-for-images.sh new file mode 100755 index 00000000000..15bff87b4cc --- /dev/null +++ b/containers/wait-for-images.sh @@ -0,0 +1,73 @@ +#!/usr/bin/env bash +# wait-for-images.sh +# +# Neither Buildkite nor the GitHub-hosted test-reusable.yml/ +# test-wsl2-reusable.yml runners hold image-push credentials, so any of them +# can race the image-push.yml GitHub Actions workflow: if this commit needs an +# image tag that only just got built, that tag might still be waiting on a +# maintainer's approval when this test run starts. Before running anything +# that pulls a DDEV image, poll the registry for the tags this checkout needs. +# +# Only the tags this checkout will actually *pull* are waited for, which +# required-image-tag.sh resolves the same way autotag.sh does: +# +# - hash unchanged -> versionconstants.go's committed tag is what gets +# pulled, so wait for exactly that, branch prefix and all. Recomputing a +# <this-branch>-<hash> tag here would wait forever on most pull requests, +# since nothing pushes a tag under a branch that didn't change the image. +# - hash changed -> `make` rebuilds the image locally on this runner, so +# there is nothing to wait for. +# +# Fast path (the common case - nothing changed): one registry check per image, +# no wait. +# +# Env: +# WAIT_FOR_IMAGES_ATTEMPTS - poll attempts before giving up (default 40) +# WAIT_FOR_IMAGES_SLEEP - seconds between attempts (default 30) +# +# Defaults give ~20 minutes - ddev-webserver alone takes ~6-8 minutes to build. + +set -eu -o pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +REGISTRY_TAG_EXISTS="$SCRIPT_DIR/registry-tag-exists.sh" +REQUIRED_IMAGE_TAG="$SCRIPT_DIR/required-image-tag.sh" +DOCKER_ORG="${DOCKER_ORG:-ddev}" + +ATTEMPTS="${WAIT_FOR_IMAGES_ATTEMPTS:-40}" +SLEEP_SECONDS="${WAIT_FOR_IMAGES_SLEEP:-30}" + +# shellcheck source=containers/image-configs.sh +source "$SCRIPT_DIR/image-configs.sh" + +for entry in "${DDEV_IMAGE_CONFIGS[@]}"; do + IFS='|' read -r repo_suffix tag_var hash_paths _ _ _ _ built_by_make _ <<< "$entry" + # shellcheck disable=SC2086 # hash_paths is a space-separated path list + read -r state tag <<< "$("$REQUIRED_IMAGE_TAG" "$tag_var" $hash_paths)" + image_repo="${DOCKER_ORG}/${repo_suffix}" + + # A changed image that `make` builds here needs no registry round trip. One + # it doesn't build - every ddev-dbserver variant but the default - still + # does, and the tag is the bare content hash, so it is the same string + # image-build-push.yml pushed regardless of branch. Wait for it. + if [ "$state" != "committed" ] && [ "$built_by_make" = "true" ]; then + echo "wait-for-images.sh: ${image_repo} content differs from versionconstants.go; make builds ${tag} locally, not waiting" + continue + fi + + attempt=1 + while true; do + if "$REGISTRY_TAG_EXISTS" "$image_repo" "$tag"; then + echo "wait-for-images.sh: found ${image_repo}:${tag}" + break + fi + if [ "$attempt" -ge "$ATTEMPTS" ]; then + echo "wait-for-images.sh: gave up waiting for ${image_repo}:${tag} after ${ATTEMPTS} attempts." >&2 + echo "wait-for-images.sh: has the maintainer approved the image-push run for this PR yet?" >&2 + exit 1 + fi + echo "wait-for-images.sh: ${image_repo}:${tag} not yet available, waiting... (attempt ${attempt}/${ATTEMPTS})" + sleep "$SLEEP_SECONDS" + attempt=$((attempt + 1)) + done +done diff --git a/containers/wait_for_images_test.sh b/containers/wait_for_images_test.sh new file mode 100755 index 00000000000..a313b6dd049 --- /dev/null +++ b/containers/wait_for_images_test.sh @@ -0,0 +1,266 @@ +#!/usr/bin/env bash +# wait_for_images_test.sh - unit tests for wait-for-images.sh. +# +# Exercises the fast-path/retry/give-up logic against a stubbed `docker` and a +# throwaway versionconstants.go, so the committed-vs-recomputed decision can be +# driven both ways without touching this checkout. No real registry, no real +# sleeps. +# Run with: +# containers/wait_for_images_test.sh + +set -eu -o pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +WAIT_FOR_IMAGES="$SCRIPT_DIR/wait-for-images.sh" +HASH_PATHS="$SCRIPT_DIR/hash-paths.sh" + +FAILURES=0 + +fail() { + echo "FAIL: $1" >&2 + FAILURES=$((FAILURES + 1)) +} + +pass() { + echo "PASS: $1" +} + +assert_eq() { + local expected="$1" actual="$2" desc="$3" + if [ "$expected" = "$actual" ]; then + pass "$desc" + else + fail "$desc (expected '$expected', got '$actual')" + fi +} + +# BSD wc pads its output with spaces; GNU wc does not. +count_lines() { + wc -l < "$1" | tr -d '[:space:]' +} + +WORKDIR="$(mktemp -d)" +trap 'rm -rf "$WORKDIR"' EXIT + +# --- Stub `docker`: a ref can be configured to "exist" outright, or to only +# start existing after N calls (via a counter file), so the eventually-recovers +# scenario is deterministic - no real sleeps or background processes needed. +BINDIR="$WORKDIR/bin" +mkdir -p "$BINDIR" +export DOCKER_EXISTING_REF_FILE="$WORKDIR/docker_existing_refs" +export DOCKER_DELAYED_REF_FILE="$WORKDIR/docker_delayed_ref" +export DOCKER_DELAYED_COUNTER_DIR="$WORKDIR/docker_delayed_counters" +export DOCKER_CALL_LOG="$WORKDIR/docker_calls.log" +mkdir -p "$DOCKER_DELAYED_COUNTER_DIR" +: > "$DOCKER_EXISTING_REF_FILE" +: > "$DOCKER_DELAYED_REF_FILE" +: > "$DOCKER_CALL_LOG" +cat > "$BINDIR/docker" <<'DOCKEREOF' +#!/usr/bin/env bash +set -eu -o pipefail +echo "$*" >> "$DOCKER_CALL_LOG" +if [ "$1" = "buildx" ] && [ "$2" = "imagetools" ] && [ "$3" = "inspect" ]; then + ref="$4" + if grep -qxF "$ref" "$DOCKER_EXISTING_REF_FILE"; then + exit 0 + fi + delayed_ref="$(cat "$DOCKER_DELAYED_REF_FILE" 2>/dev/null || true)" + if [ -n "$delayed_ref" ] && [ "$ref" = "$delayed_ref" ]; then + counter_file="$DOCKER_DELAYED_COUNTER_DIR/count" + count="$(cat "$counter_file" 2>/dev/null || echo 0)" + count=$((count + 1)) + echo "$count" > "$counter_file" + [ "$count" -ge 3 ] && exit 0 || exit 1 + fi + exit 1 +fi +echo "docker stub: unexpected invocation: $*" >&2 +exit 1 +DOCKEREOF +chmod +x "$BINDIR/docker" + +# --- Stub `sleep` so retry-budget tests run instantly and we can count waits. +export SLEEP_CALL_LOG="$WORKDIR/sleep_calls.log" +: > "$SLEEP_CALL_LOG" +cat > "$BINDIR/sleep" <<'SLEEPEOF' +#!/usr/bin/env bash +echo "$*" >> "$SLEEP_CALL_LOG" +SLEEPEOF +chmod +x "$BINDIR/sleep" + +export PATH="$BINDIR:$PATH" + +export DOCKER_ORG=ddevhq + +# shellcheck source=containers/image-configs.sh +source "$SCRIPT_DIR/image-configs.sh" + +# --- A throwaway versionconstants.go, so the tags waited for are whatever this +# test says they are rather than whatever the checkout happens to carry. +export VERSIONCONSTANTS_FILE="$WORKDIR/versionconstants.go" + +REPOS=() +TAG_VARS=() +declare -A HASH_BY_VAR +declare -A TAG_BY_VAR +for entry in "${DDEV_IMAGE_CONFIGS[@]}"; do + IFS='|' read -r repo_suffix tag_var hash_paths _ <<< "$entry" + if [ -z "${HASH_BY_VAR[$tag_var]:-}" ]; then + # shellcheck disable=SC2086 # hash_paths is a space-separated path list + HASH_BY_VAR["$tag_var"]="$("$HASH_PATHS" $hash_paths)" + TAG_BY_VAR["$tag_var"]="${HASH_BY_VAR[$tag_var]}" + fi + REPOS+=("ddevhq/${repo_suffix}") + TAG_VARS+=("$tag_var") +done +IMAGE_COUNT="${#REPOS[@]}" + +# All 20 ddev-dbserver variants share BaseDBTag, so the file has one line per +# distinct tag variable, not one per image. +write_versionconstants() { + : > "$VERSIONCONSTANTS_FILE" + for var in "${!TAG_BY_VAR[@]}"; do + echo "var ${var} = \"${TAG_BY_VAR[$var]}\"" >> "$VERSIONCONSTANTS_FILE" + done +} + +mark_all_existing() { + : > "$DOCKER_EXISTING_REF_FILE" + for i in "${!REPOS[@]}"; do + echo "${REPOS[$i]}:${TAG_BY_VAR[${TAG_VARS[$i]}]}" >> "$DOCKER_EXISTING_REF_FILE" + done +} + +write_versionconstants + +# 1. Fast path: every committed tag already exists -> one docker call per +# image, no sleep. This is the shape of every pull request that doesn't +# change a container image. +mark_all_existing +: > "$DOCKER_CALL_LOG" +: > "$SLEEP_CALL_LOG" +OUTPUT="$("$WAIT_FOR_IMAGES" 2>&1)" && RC=0 || RC=$? +if [ "$RC" -eq 0 ]; then + pass "fast path succeeds when every committed tag already exists" +else + fail "fast path should succeed when every committed tag already exists: $OUTPUT" +fi +assert_eq "$IMAGE_COUNT" "$(count_lines "$DOCKER_CALL_LOG")" "fast path makes exactly one docker call per image" +assert_eq "0" "$(count_lines "$SLEEP_CALL_LOG")" "fast path never sleeps" +case "$OUTPUT" in + *"found ${REPOS[0]}:${TAG_BY_VAR[WebTag]}"*) pass "prints confirmation for each found tag" ;; + *) fail "should print confirmation for each found tag: $OUTPUT" ;; +esac + +# 2. The regression that made every non-containers pull request hang was a tag +# that depended on the current branch. The tag is now the bare content hash, +# so the same checkout resolves to the same tag on any branch, in any fork. +case "$OUTPUT" in + *"found ${REPOS[0]}:${HASH_BY_VAR[WebTag]}"*) pass "waits for the bare content hash, with no branch prefix" ;; + *) fail "should wait for the bare hash '${HASH_BY_VAR[WebTag]}': $OUTPUT" ;; +esac +OUTPUT_OTHER_BRANCH="$(REQUIRED_IMAGE_TAG_BRANCH='some/other branch' "$WAIT_FOR_IMAGES" 2>&1)" +assert_eq "$OUTPUT" "$OUTPUT_OTHER_BRANCH" "the tags waited for don't depend on the branch" + +# 2b. Every ddev-dbserver variant is waited for, not just the default one that +# `make` builds locally - they all share BaseDBTag, so a dbserver change +# invalidates all of them at once. +for variant_repo in ddevhq/ddev-dbserver-mysql-8.0 ddevhq/ddev-dbserver-mariadb-10.11 ddevhq/ddev-dbserver-mariadb-5.5; do + case "$OUTPUT" in + *"found ${variant_repo}:"*) pass "waits for the non-default variant ${variant_repo}" ;; + *) fail "should wait for the non-default variant ${variant_repo}: $OUTPUT" ;; + esac +done + +# 3. Content that no longer matches versionconstants.go is built locally by +# make, so there is nothing to wait for and no registry call at all. +TAG_BY_VAR[WebTag]="0000000000" +write_versionconstants +mark_all_existing +: > "$DOCKER_CALL_LOG" +: > "$SLEEP_CALL_LOG" +OUTPUT="$("$WAIT_FOR_IMAGES" 2>&1)" && RC=0 || RC=$? +if [ "$RC" -eq 0 ]; then + pass "succeeds without waiting when content differs from versionconstants.go" +else + fail "should succeed when content differs from versionconstants.go: $OUTPUT" +fi +assert_eq "$(( IMAGE_COUNT - 1 ))" "$(count_lines "$DOCKER_CALL_LOG")" "skips the registry check for the locally-built image" +assert_eq "0" "$(count_lines "$SLEEP_CALL_LOG")" "never sleeps for a locally-built image" +case "$OUTPUT" in + *"not waiting"*) pass "says why it isn't waiting for the changed image" ;; + *) fail "should explain why it isn't waiting: $OUTPUT" ;; +esac +TAG_BY_VAR[WebTag]="${HASH_BY_VAR[WebTag]}" +write_versionconstants + +# 3b. A changed dbserver still has to wait for the 19 variants `make` doesn't +# build locally. The tag is the bare content hash, so it's the same string +# image-build-push.yml pushes and waiting for it is well-defined - only the +# default variant is skipped. +TAG_BY_VAR[BaseDBTag]="0000000000" +write_versionconstants +: > "$DOCKER_EXISTING_REF_FILE" +for i in "${!REPOS[@]}"; do + tag="${TAG_BY_VAR[${TAG_VARS[$i]}]}" + [ "${TAG_VARS[$i]}" = "BaseDBTag" ] && tag="${HASH_BY_VAR[BaseDBTag]}" + echo "${REPOS[$i]}:${tag}" >> "$DOCKER_EXISTING_REF_FILE" +done +: > "$DOCKER_CALL_LOG" +OUTPUT="$("$WAIT_FOR_IMAGES" 2>&1)" && RC=0 || RC=$? +if [ "$RC" -eq 0 ]; then + pass "waits for the db variants make doesn't build, rather than failing" +else + fail "should wait for the non-locally-built db variants: $OUTPUT" +fi +case "$OUTPUT" in + *"ddevhq/ddev-dbserver-mariadb-11.8 content differs"*) pass "skips only the default variant make builds locally" ;; + *) fail "should skip the locally-built default variant: $OUTPUT" ;; +esac +case "$OUTPUT" in + *"found ddevhq/ddev-dbserver-mysql-8.0:${HASH_BY_VAR[BaseDBTag]}"*) pass "waits for a non-default variant at the bare hash tag" ;; + *) fail "should wait for the non-default variants at the recomputed hash: $OUTPUT" ;; +esac +TAG_BY_VAR[BaseDBTag]="${HASH_BY_VAR[BaseDBTag]}" +write_versionconstants + +# 4. A tag that's initially missing but becomes available on the 3rd check. +LAST=$(( IMAGE_COUNT - 1 )) +: > "$DOCKER_EXISTING_REF_FILE" +for i in "${!REPOS[@]}"; do + [ "$i" -eq "$LAST" ] && continue + echo "${REPOS[$i]}:${TAG_BY_VAR[${TAG_VARS[$i]}]}" >> "$DOCKER_EXISTING_REF_FILE" +done +echo "${REPOS[$LAST]}:${TAG_BY_VAR[${TAG_VARS[$LAST]}]}" > "$DOCKER_DELAYED_REF_FILE" +rm -f "$DOCKER_DELAYED_COUNTER_DIR/count" +: > "$SLEEP_CALL_LOG" +if WAIT_FOR_IMAGES_ATTEMPTS=5 WAIT_FOR_IMAGES_SLEEP=0 "$WAIT_FOR_IMAGES" >/dev/null 2>&1; then + pass "recovers once a previously-missing tag appears within the attempt budget" +else + fail "should recover once a previously-missing tag appears within the attempt budget" +fi +assert_eq "2" "$(count_lines "$SLEEP_CALL_LOG")" "sleeps twice while waiting for the tag to become available on the 3rd check" +: > "$DOCKER_DELAYED_REF_FILE" + +# 5. Gives up cleanly after exhausting the attempt budget, with a clear message. +: > "$DOCKER_EXISTING_REF_FILE" +: > "$SLEEP_CALL_LOG" +OUTPUT="$(WAIT_FOR_IMAGES_ATTEMPTS=3 WAIT_FOR_IMAGES_SLEEP=0 "$WAIT_FOR_IMAGES" 2>&1)" && RC=0 || RC=$? +if [ "$RC" -ne 0 ]; then + pass "gives up (non-zero exit) once the attempt budget is exhausted" +else + fail "should give up (non-zero exit) once the attempt budget is exhausted" +fi +case "$OUTPUT" in + *"gave up waiting"*"has the maintainer approved"*) pass "give-up message is actionable" ;; + *) fail "give-up message should mention giving up and approval: $OUTPUT" ;; +esac +assert_eq "2" "$(count_lines "$SLEEP_CALL_LOG")" "sleeps exactly (attempts - 1) times before giving up on the first (unavailable) image" + +if [ "$FAILURES" -eq 0 ]; then + echo "All wait_for_images_test.sh checks passed." + exit 0 +else + echo "$FAILURES wait_for_images_test.sh check(s) failed." >&2 + exit 1 +fi diff --git a/docs/content/developers/building-contributing.md b/docs/content/developers/building-contributing.md index 7ac7670ece0..a0c4be5d75a 100644 --- a/docs/content/developers/building-contributing.md +++ b/docs/content/developers/building-contributing.md @@ -155,6 +155,8 @@ make push VERSION=<tag> DOCKER_REPO=your/dockerrepo ### Pushes Using GitHub Actions +Normally the [Image build](https://github.com/ddev/ddev/actions/workflows/image-build-push.yml) workflow (see [Automatic Image Build and Push](#automatic-image-build-and-push)) handles pushing a changed image automatically for any pull request, including forks. The workflows below are for manually pushing a specific tag — a re-push, or one of the `ddev-dbserver` variants other than the default `mariadb_11.8` that `make` auto-builds. + To manually push using GitHub Actions, #### For Most Images @@ -308,7 +310,24 @@ The Docker images that DDEV uses are included in the `containers/` directory: * `containers/ddev-traefik-router` is the current Traefik-based router image. * `containers/ddev-xhgui` provides a web interface to analyze performance profiles generated by xhprof. -When you change an image, running `make` from the repository root builds it locally and computes/updates its tag in `pkg/versionconstants/versionconstants.go` automatically — no manual tag-inventing or file-editing needed, and this works for any contributor, including from a fork. Getting that image into CI (a multi-arch push to the registry) still requires registry credentials that forks don't have, so please ask a maintainer if you need a container pushed to support a pull request. +When you change an image, running `make` from the repository root builds it locally and computes/updates its tag in `pkg/versionconstants/versionconstants.go` automatically — no manual tag-inventing or file-editing needed, and this works for any contributor, including from a fork. Once you push that commit as a pull request, the [Image build](https://github.com/ddev/ddev/actions/workflows/image-build-push.yml) workflow detects the changed image and builds and pushes it to the registry automatically — no maintainer needs to run anything by hand. See [Automatic Image Build and Push](#automatic-image-build-and-push) below for when that requires a maintainer's approval and when it doesn't. + +### Automatic Image Build and Push + +An image's tag is the bare content hash of the files it's built from — `ddev/ddev-webserver:36bceca65e`, with no branch prefix. The same content therefore resolves to the same tag no matter which branch, fork, or Docker Hub organization published it, which is what lets `make`, the CI detector, and every test runner agree without coordinating. `versionconstants.go` records the branch alongside it, both as a trailing comment and as a `WebTagBranch`-style variable that `ddev version` shows, and each push also publishes a readable `<branch>-<hash>` alias pointing at the same manifest. + +Opening a pull request that touches `containers/` triggers the [Image build](https://github.com/ddev/ddev/actions/workflows/image-build-push.yml) workflow. A `detect` job always runs first: it recomputes each image's content hash and checks whether that tag is already in the registry. If it is, there's nothing to build — so a pull request that touches `containers/` without changing an image costs one registry lookup per image and no build. + +The same resolution drives `containers/wait-for-images.sh`, which every test runner calls before pulling anything: it waits only for tags this commit will genuinely pull, and doesn't wait at all for an image whose content changed, since `make` builds that one locally on the runner. + +Changing `containers/ddev-dbserver` is the one case where `make` alone isn't enough. Every database variant shares a single `BaseDBTag`, so a dbserver change moves the tag for all 20 of them at once, while `make` only builds the default `mariadb_11.8` locally — the other 19 can only come from the registry. CI builds and pushes the whole matrix (36 jobs, from `containers/ddev-dbserver/variants.txt`), so **run `make` and commit the `versionconstants.go` change**; `wait-for-images.sh` stops the test run with that instruction if you don't. + +What happens next depends on whether the PR is from a fork: + +* **Fork PRs** (security boundary — the PR could contain an arbitrary Dockerfile/build script): a `build` job builds the image(s) per architecture with no registry credentials at all — nothing in that job can reach `docker.io`, so there's nothing to gain by gating it before it runs. Once it finishes, a separate, trusted `image-push.yml` workflow — which never checks out or runs the pull request's code — loads what it produced and pushes it, gated behind a maintainer's approval on the `image-push` environment. Both the tag and every repository name in that artifact are re-validated first (`containers/validate-image-tag.sh`, `containers/validate-image-repo.sh`), so an approval can only ever publish a hash-shaped tag under a known DDEV repository. A comment is posted on the PR once the push completes. If the build produced nothing to push, no approval is requested at all. +* **Everything else** (a push to `main`, or a pull request from a branch in the same repository — no fork content is ever involved): `build-and-push` builds and pushes directly in one step, with no approval gate at all — the same trust level `main-build.yml` already runs at unguarded. A `create-manifests` job then assembles the multi-arch manifest and comments on the PR, if there is one. + +So a maintainer only ever needs to click **Approve** once — for a fork PR's push step — and only when the PR actually changed a container image; everything else is fully automatic. ## Pull Requests diff --git a/docs/content/developers/release-management.md b/docs/content/developers/release-management.md index 5d08677508a..4c96adcc0de 100644 --- a/docs/content/developers/release-management.md +++ b/docs/content/developers/release-management.md @@ -74,9 +74,33 @@ The following “Repository secret” environment variables must be configured i 2. Make sure you're about to create the right release tag. 3. Use the “Auto-generate release notes” option to get the commit list, then edit to add all the other necessary info. +## Automatic Image Build and Push + +Any pull request that changes `containers/` — including from a fork — is built and pushed automatically by the [Image build](https://github.com/ddev/ddev/blob/main/.github/workflows/image-build-push.yml) / [Image push](https://github.com/ddev/ddev/blob/main/.github/workflows/image-push.yml) workflow pair. See [Automatic Image Build and Push](building-contributing.md#automatic-image-build-and-push) in the contributing guide for how the flow works and why it's safe to run on fork-authored Dockerfiles. + +The two workflows below (manual `workflow_dispatch`) remain for re-pushing a specific tag — for instance at release time, when every image is re-pushed under a `vX.Y.Z` tag. Their `tag` input is optional: leave it empty and the workflow uses the tag the checkout actually needs, which is safer than retyping a content hash. Supply it only for a release. + +When the tag is a content hash, these workflows also publish the `<branch>-<hash>` alias next to it — the same second name the automatic flow creates — so a manual push doesn't leave a bare hash with nothing readable beside it in the registry. A `vX.Y.Z` tag is already readable and gets no alias. + +A `containers/ddev-dbserver` change builds and pushes all 20 database variants (36 jobs), because they all share a single `BaseDBTag`. That variant matrix lives in `containers/ddev-dbserver/variants.txt` and is read by `variants.sh`, which also generates that directory's make targets, the automatic flow's image list, and `push-tagged-dbimage.yml`'s matrix — add a database version there and every consumer picks it up. + +### One-time setup: the `image-push` GitHub Environment + +Fork PRs build with no registry credentials at all (nothing to gain by gating that step), then go through a single approval before the built image is actually pushed, gated by the `image-push` GitHub Environment (Settings → Environments): + +1. Create the environment `image-push`. +2. Add required reviewers (the maintainers/dev team). +3. Add `PUSH_SERVICE_ACCOUNT_TOKEN` as a secret **on this environment** (Settings → Environments → `image-push` → Secrets), using the same 1Password service-account token value already used elsewhere in this doc. It currently exists only as a repository secret; duplicating (or moving) it onto the `image-push` environment is what scopes `DOCKERHUB_TOKEN` access to only the approved `image-push.yml` job. + +This approval only applies to fork PRs. A push to `main` or a same-repo PR builds and pushes without any approval at all, using the repository-level `PUSH_SERVICE_ACCOUNT_TOKEN` secret directly (that path never declares `environment:` on its jobs, so this environment's protection rules don't apply to it). + +When testing this on `ddev-test/ddev`, do the same steps there first, and confirm `vars.DOCKER_ORG` on that repository points at the DockerHub org used for testing. + +Since a job referencing an environment that doesn't exist yet gets auto-created with no protection rules (silently *not* gating), verify the environment actually has a `required_reviewers` rule before relying on it, e.g. `gh api repos/<owner>/<repo>/environments/image-push`. + ## Pushing Docker Images with the GitHub Actions Workflow -The easiest way to push Docker images is to use the GitHub Actions workflow, especially if the code for the image is already in the [ddev/ddev](https://github.com/ddev/ddev) repository. +The easiest way to push Docker images is to use the GitHub Actions workflow, especially if the code for the image is already in the [ddev/ddev](https://github.com/ddev/ddev) repository. For a normal container change on a pull request, you shouldn't need this — see [Automatic Image Build and Push](#automatic-image-build-and-push) above. ### Actual release creation @@ -88,7 +112,7 @@ You can push all images besides `ddev-dbserver` at <https://github.com/ddev/ddev You can push `ddev-dbserver` images at <https://github.com/ddev/ddev/actions/workflows/push-tagged-dbimage.yml> -If you need to push from a forked PR, you’ll have to do this from your fork (for example, `https://github.com/rfay/ddev/actions/workflows/push-tagged-image.yml`), and you’ll have to specify the branch on the fork. This requires setting the `DOCKERHUB_TOKEN` and `DOCKERHUB_USERNAME` secrets on the forked PR, for example `https://github.com/rfay/ddev/settings/secrets/actions`. You can do the same with `ddev-dbserver` at `https://github.com/rfay/ddev/actions/workflows/push-tagged-dbimage.yml` for example. +A forked PR that changes a container image no longer needs any of this — see [Automatic Image Build and Push](#automatic-image-build-and-push) above. The fork-your-own-secrets workaround described in earlier versions of this doc is superseded by that flow. * Visit `https://github.com/ddev/ddev/actions/workflows/push-tagged-image.yml`. * Click the “Push tagged image” workflow on the left side of the page. diff --git a/pkg/version/version.go b/pkg/version/version.go index d174f563a29..2494cc0b1ff 100644 --- a/pkg/version/version.go +++ b/pkg/version/version.go @@ -66,10 +66,38 @@ func GetVersionInfo() (map[string]string, error) { } versionInfo["mutagen"] = versionconstants.RequiredMutagenVersion versionInfo["xhgui-image"] = docker.GetXhguiImage() + versionInfo["image-tag-branches"] = imageTagBranches() return versionInfo, retErr } +// imageTagBranches describes which branch each image tag was built from. The +// tags themselves are bare content hashes, so without this there is nothing in +// `ddev version` to say where an image came from. Collapses to a single branch +// name in the usual case where every image was built from the same one. +func imageTagBranches() string { + branches := []struct{ image, branch string }{ + {"web", versionconstants.WebTagBranch}, + {"db", versionconstants.BaseDBTagBranch}, + {"router", versionconstants.TraefikRouterTagBranch}, + {"ddev-ssh-agent", versionconstants.SSHAuthTagBranch}, + {"xhgui", versionconstants.XhguiTagBranch}, + } + + parts := make([]string, 0, len(branches)) + allSame := true + for _, b := range branches { + if b.branch != branches[0].branch { + allSame = false + } + parts = append(parts, b.image+"="+b.branch) + } + if allSame { + return branches[0].branch + } + return strings.Join(parts, " ") +} + // GetDockerPlatform gets the platform used for Docker engine func GetDockerPlatform() (string, error) { info, err := dockerutil.GetDockerClientInfo() diff --git a/pkg/versionconstants/versionconstants.go b/pkg/versionconstants/versionconstants.go index 69f3d6d4869..858776e3649 100644 --- a/pkg/versionconstants/versionconstants.go +++ b/pkg/versionconstants/versionconstants.go @@ -16,35 +16,58 @@ var DdevVersion = "" // Note that this is overridden by make // Compiled with link-time variables var AmplitudeAPIKey = "" +// Image tags are bare content hashes (see containers/hash-paths.sh), so the +// same image resolves to the same tag no matter which branch or repository +// published it. Each has a companion <Name>Branch naming the branch its +// content was built from - the readable hint a bare hash costs, shown by +// `ddev version` and republished as a <branch>-<hash> alias in the registry. +// All of these lines are maintained by containers/autotag.sh; running `make` +// updates them. + // WebImg defines the default web image used for applications. var WebImg = "ddev/ddev-webserver" // WebTag defines the default web image tag -var WebTag = "20260721_rfay_content_addressed_image_tags-36bceca65e" // Note that this can be overridden by make +var WebTag = "370f47913b" // 20260816_rfay_test4_fork_multi_image-370f47913b + +// WebTagBranch is the branch WebTag's content was built from. +var WebTagBranch = "20260816_rfay_test4_fork_multi_image" // DBImg defines the default db image used for applications. var DBImg = "ddev/ddev-dbserver" // BaseDBTag is the main tag, DBTag is constructed from it -var BaseDBTag = "20260721_rfay_content_addressed_image_tags-278125d33a" +var BaseDBTag = "5a7c45ddbf" // 20260814_rfay_docker_update_phase_2-5a7c45ddbf + +// BaseDBTagBranch is the branch BaseDBTag's content was built from. +var BaseDBTagBranch = "20260814_rfay_docker_update_phase_2" // TraefikRouterImage is image for router var TraefikRouterImage = "ddev/ddev-traefik-router" // TraefikRouterTag is traefik router tag -var TraefikRouterTag = "20260721_rfay_content_addressed_image_tags-c96123b524" +var TraefikRouterTag = "1e79e9017f" // 20260816_rfay_test4_fork_multi_image-1e79e9017f + +// TraefikRouterTagBranch is the branch TraefikRouterTag's content was built from. +var TraefikRouterTagBranch = "20260816_rfay_test4_fork_multi_image" // SSHAuthImage is image for agent var SSHAuthImage = "ddev/ddev-ssh-agent" // SSHAuthTag is ssh-agent auth tag -var SSHAuthTag = "20260721_rfay_content_addressed_image_tags-8e8bf1217c" +var SSHAuthTag = "5b9c29ad26" // 20260816_rfay_test4_fork_multi_image-5b9c29ad26 + +// SSHAuthTagBranch is the branch SSHAuthTag's content was built from. +var SSHAuthTagBranch = "20260816_rfay_test4_fork_multi_image" // XhguiImage is image for xhgui var XhguiImage = "ddev/ddev-xhgui" // XhguiTag is xhgui tag -var XhguiTag = "20260721_rfay_content_addressed_image_tags-f046b66382" +var XhguiTag = "8757c1e92a" // 20260814_rfay_docker_update_phase_2-8757c1e92a + +// XhguiTagBranch is the branch XhguiTag's content was built from. +var XhguiTagBranch = "20260814_rfay_docker_update_phase_2" // UtilitiesImage is used in bash scripts var UtilitiesImage = "ddev/ddev-utilities:latest"