From b6b6a44abb0f91d8e94fdc9f26923afebda41aa9 Mon Sep 17 00:00:00 2001 From: HashWrangler <12961024+HashWrangler@users.noreply.github.com> Date: Wed, 17 Jun 2026 13:06:17 -0400 Subject: [PATCH 01/10] fix(build-push-docker-manifest): idempotent manifest + cosign reruns (RANE-4683) - Skip imagetools create when the tag already references the expected platform digests; fail if the tag exists with different digests - Skip cosign sign when verify already succeeds (idempotent rerun) - Retry cosign verify after sign for Sigstore propagation flakes Together these changes make build-publish manifest jobs safe to rerun without digest drift or redundant signing, while still failing loudly on real conflicts. --- .../rane-4683-manifest-cosign-hardening.md | 5 + actions/build-push-docker-manifest/action.yml | 130 ++++++++++++++++-- 2 files changed, 127 insertions(+), 8 deletions(-) create mode 100644 .changeset/rane-4683-manifest-cosign-hardening.md diff --git a/.changeset/rane-4683-manifest-cosign-hardening.md b/.changeset/rane-4683-manifest-cosign-hardening.md new file mode 100644 index 000000000..45ac67ae0 --- /dev/null +++ b/.changeset/rane-4683-manifest-cosign-hardening.md @@ -0,0 +1,5 @@ +--- +"build-push-docker-manifest": minor +--- + +Harden manifest create and cosign sign/verify for idempotent build-publish reruns (RANE-4683): skip imagetools create when the tag already points at the expected platform digests, skip cosign sign when a valid signature is already present, and retry cosign verify after signing to absorb Sigstore propagation lag diff --git a/actions/build-push-docker-manifest/action.yml b/actions/build-push-docker-manifest/action.yml index e4f002959..eb210b905 100644 --- a/actions/build-push-docker-manifest/action.yml +++ b/actions/build-push-docker-manifest/action.yml @@ -327,13 +327,72 @@ runs: ${{ steps.generate-annotations.outputs.annotation-flags }} TAG_FLAGS: ${{ steps.process-additional-tags.outputs.tag-flags }} run: | + set -euo pipefail + + normalize_digest_csv() { + local input="$1" + IFS=',' read -ra _digests <<< "$input" + local normalized=() + for digest in "${_digests[@]}"; do + digest=$(echo "$digest" | xargs) + if [[ -n "$digest" ]]; then + normalized+=("$digest") + fi + done + if [[ ${#normalized[@]} -eq 0 ]]; then + echo "" + return + fi + mapfile -t sorted < <(printf '%s\n' "${normalized[@]}" | sort) + local IFS=',' + echo "${sorted[*]}" + } + + get_existing_platform_digests() { + local tag="$1" + local inspect_json + if ! inspect_json=$(docker buildx imagetools inspect "${tag}" --format '{{json .}}' 2>/dev/null); then + return 1 + fi + echo "${inspect_json}" | jq -r ' + .manifest.manifests[]? + | select( + ((.platform.os // "") | ascii_downcase) != "unknown" + and ((.annotations."vnd.docker.reference.type" // "") != "attestation-manifest") + ) + | .digest + ' | sort | paste -sd, - + } + DOCKER_MANIFEST_NAME_WITH_TAG="${DOCKER_MANIFEST_NAME}:${DOCKER_MANIFEST_TAG}" + EXPECTED_DIGESTS=$(normalize_digest_csv "${DOCKER_IMAGE_NAME_DIGESTS}") + + if EXISTING_DIGESTS=$(get_existing_platform_digests "${DOCKER_MANIFEST_NAME_WITH_TAG}"); then + echo "Found existing manifest for ${DOCKER_MANIFEST_NAME_WITH_TAG}" + echo " Expected platform digests: ${EXPECTED_DIGESTS}" + echo " Existing platform digests: ${EXISTING_DIGESTS}" + + if [[ "${EXISTING_DIGESTS}" == "${EXPECTED_DIGESTS}" ]]; then + echo "✅ Manifest already exists with expected platform digests; skipping imagetools create (idempotent rerun)" + echo "manifest-create-skipped=true" | tee -a "${GITHUB_OUTPUT}" + exit 0 + fi + + echo "::error::Manifest tag ${DOCKER_MANIFEST_NAME_WITH_TAG} already exists with different platform digests" + echo "::error::Expected: ${EXPECTED_DIGESTS}" + echo "::error::Existing: ${EXISTING_DIGESTS}" + exit 1 + fi + + echo "manifest-create-skipped=false" | tee -a "${GITHUB_OUTPUT}" + # Convert comma-separated list into array and pass as separate arguments IFS=',' read -ra DIGESTS <<< "$DOCKER_IMAGE_NAME_DIGESTS" # Map each digest to include the manifest name PREFIXED_DIGESTS=() for digest in "${DIGESTS[@]}"; do - PREFIXED_DIGESTS+=("${DOCKER_MANIFEST_NAME}@${digest}") + digest=$(echo "$digest" | xargs) + [[ -n "$digest" ]] && PREFIXED_DIGESTS+=("${DOCKER_MANIFEST_NAME}@${digest}") done # Create Docker manifest @@ -412,17 +471,42 @@ runs: - name: Sign Docker Manifest using GH OIDC if: inputs.docker-manifest-sign == 'true' - shell: sh + id: sign-docker-manifest + shell: bash env: MANIFEST_NAME_WITH_DIGEST: ${{ steps.inspect-docker-manifest.outputs.manifest-name-with-digest }} - run: cosign sign "${MANIFEST_NAME_WITH_DIGEST}" --yes + GITHUB_WORKFLOW_REPOSITORY: ${{ inputs.github-workflow-repository }} + OIDC_ISSUER: ${{ inputs.cosign-oidc-issuer }} + OIDC_IDENTITY_REGEXP: ${{ inputs.cosign-oidc-identity-regexp }} + run: | + set -euo pipefail + + verify_manifest_signature() { + if [[ -n "${OIDC_IDENTITY_REGEXP}" ]]; then + cosign verify "${MANIFEST_NAME_WITH_DIGEST}" \ + --certificate-oidc-issuer "${OIDC_ISSUER}" \ + --certificate-identity-regexp "${OIDC_IDENTITY_REGEXP}" \ + --certificate-github-workflow-repository "${GITHUB_WORKFLOW_REPOSITORY}" + else + cosign verify "${MANIFEST_NAME_WITH_DIGEST}" + fi + } + + if verify_manifest_signature >/dev/null 2>&1; then + echo "✅ Manifest already signed; skipping cosign sign (idempotent rerun)" + echo "manifest-sign-skipped=true" | tee -a "${GITHUB_OUTPUT}" + exit 0 + fi + + echo "manifest-sign-skipped=false" | tee -a "${GITHUB_OUTPUT}" + cosign sign "${MANIFEST_NAME_WITH_DIGEST}" --yes - name: Verify Docker image signature if: inputs.docker-manifest-sign == 'true' && inputs.cosign-oidc-identity-regexp != '' - shell: sh + shell: bash env: MANIFEST_NAME_WITH_DIGEST: >- ${{ @@ -432,10 +516,30 @@ runs: OIDC_ISSUER: ${{ inputs.cosign-oidc-issuer }} OIDC_IDENTITY_REGEXP: ${{ inputs.cosign-oidc-identity-regexp }} run: | - cosign verify "${MANIFEST_NAME_WITH_DIGEST}" \ - --certificate-oidc-issuer "${OIDC_ISSUER}" \ - --certificate-identity-regexp "${OIDC_IDENTITY_REGEXP}" \ - --certificate-github-workflow-repository "${GITHUB_WORKFLOW_REPOSITORY}" + MAX_RETRIES=5 + RETRY_DELAY=10 + VERIFY_OK=false + + for i in $(seq 1 $MAX_RETRIES); do + echo "Attempt ${i}/${MAX_RETRIES}: Verifying cosign signature for ${MANIFEST_NAME_WITH_DIGEST}..." + + if cosign verify "${MANIFEST_NAME_WITH_DIGEST}" \ + --certificate-oidc-issuer "${OIDC_ISSUER}" \ + --certificate-identity-regexp "${OIDC_IDENTITY_REGEXP}" \ + --certificate-github-workflow-repository "${GITHUB_WORKFLOW_REPOSITORY}"; then + echo "Successfully verified signature on attempt ${i}" + VERIFY_OK=true + break + fi + + echo "Attempt ${i}/${MAX_RETRIES}: Signature not yet available, retrying in ${RETRY_DELAY}s..." + sleep $RETRY_DELAY + done + + if [[ "$VERIFY_OK" != "true" ]]; then + echo "::error::Failed to verify cosign signature for ${MANIFEST_NAME_WITH_DIGEST} after ${MAX_RETRIES} attempts" + exit 1 + fi - name: Summary output shell: bash @@ -456,6 +560,10 @@ runs: steps.inspect-docker-manifest.outputs.manifest-name-with-tag }} MANIFEST_TAG: ${{ inputs.docker-manifest-tag }} + MANIFEST_CREATE_SKIPPED: + ${{ steps.create-push-docker-manifest.outputs.manifest-create-skipped }} + MANIFEST_SIGN_SKIPPED: + ${{ steps.sign-docker-manifest.outputs.manifest-sign-skipped }} OIDC_ISSUER: ${{ inputs.cosign-oidc-issuer }} OIDC_IDENTITY_REGEXP: ${{ inputs.cosign-oidc-identity-regexp }} run: | @@ -467,10 +575,16 @@ runs: echo "Manifest additional tags: \`${MANIFEST_ADDITIONAL_TAGS:-None}\`" | tee -a "${GITHUB_STEP_SUMMARY}" echo "Manifest name with tag: \`${MANIFEST_NAME_WITH_TAG}\`" | tee -a "${GITHUB_STEP_SUMMARY}" echo "Manifest name with digest: \`${MANIFEST_NAME_WITH_DIGEST}\`" | tee -a "${GITHUB_STEP_SUMMARY}" + if [[ "${MANIFEST_CREATE_SKIPPED}" == "true" ]]; then + echo "Manifest create: skipped (existing tag already points at expected platform digests)" | tee -a "${GITHUB_STEP_SUMMARY}" + fi if [[ "${DOCKER_MANIFEST_SIGNED}" == 'true' ]]; then echo >> "${GITHUB_STEP_SUMMARY}" echo "#### Docker Manifest signed 📝" | tee -a "${GITHUB_STEP_SUMMARY}" echo "Manifest signed with cosign. To verify, run:" | tee -a "${GITHUB_STEP_SUMMARY}" + if [[ "${MANIFEST_SIGN_SKIPPED}" == "true" ]]; then + echo "Cosign sign: skipped (valid signature already present)" | tee -a "${GITHUB_STEP_SUMMARY}" + fi echo "\`\`\`shell" >> "${GITHUB_STEP_SUMMARY}" echo "cosign verify ${MANIFEST_NAME_WITH_DIGEST} --certificate-oidc-issuer ${OIDC_ISSUER} --certificate-identity-regexp '${OIDC_IDENTITY_REGEXP}' --certificate-github-workflow-repository ${GITHUB_WORKFLOW_REPOSITORY}" | tee -a "${GITHUB_STEP_SUMMARY}" echo "\`\`\`" >> "${GITHUB_STEP_SUMMARY}" From 49f5771e57a331eb095c8747fafb9fc28319a27a Mon Sep 17 00:00:00 2001 From: HashWrangler <12961024+HashWrangler@users.noreply.github.com> Date: Wed, 17 Jun 2026 13:14:07 -0400 Subject: [PATCH 02/10] fix(build-push-docker-manifest): address PR review feedback (RANE-4683) - Ensure jq is installed before digest comparison - Validate docker-image-name-digests is non-empty - Only skip cosign sign when OIDC identity constraints verify - Fail fast on unexpected imagetools inspect errors (not just missing tag) --- actions/build-push-docker-manifest/action.yml | 65 +++++++++++++++---- 1 file changed, 51 insertions(+), 14 deletions(-) diff --git a/actions/build-push-docker-manifest/action.yml b/actions/build-push-docker-manifest/action.yml index eb210b905..4fe4c8a7e 100644 --- a/actions/build-push-docker-manifest/action.yml +++ b/actions/build-push-docker-manifest/action.yml @@ -188,6 +188,18 @@ runs: }} registries: ${{ inputs.aws-account-number }} + - name: Ensure jq is installed + shell: bash + run: | + if command -v jq >/dev/null 2>&1; then + echo "jq already installed: $(jq --version)" + else + echo "jq not found; installing..." + sudo apt-get update -qq + sudo apt-get install -y jq + jq --version + fi + - name: Generate manifest annotations id: generate-annotations shell: bash @@ -350,10 +362,28 @@ runs: get_existing_platform_digests() { local tag="$1" - local inspect_json - if ! inspect_json=$(docker buildx imagetools inspect "${tag}" --format '{{json .}}' 2>/dev/null); then - return 1 + local inspect_json inspect_stderr inspect_status + + inspect_stderr=$(mktemp) + set +e + inspect_json=$(docker buildx imagetools inspect "${tag}" --format '{{json .}}' 2>"${inspect_stderr}") + inspect_status=$? + set -e + + if [[ "${inspect_status}" -ne 0 ]]; then + if grep -qiE 'not found|manifest unknown|name unknown|404|does not exist|no such manifest' "${inspect_stderr}"; then + rm -f "${inspect_stderr}" + return 1 + fi + + echo "::error::Failed to inspect manifest tag ${tag}" + cat "${inspect_stderr}" >&2 + rm -f "${inspect_stderr}" + exit 1 fi + + rm -f "${inspect_stderr}" + echo "${inspect_json}" | jq -r ' .manifest.manifests[]? | select( @@ -367,6 +397,11 @@ runs: DOCKER_MANIFEST_NAME_WITH_TAG="${DOCKER_MANIFEST_NAME}:${DOCKER_MANIFEST_TAG}" EXPECTED_DIGESTS=$(normalize_digest_csv "${DOCKER_IMAGE_NAME_DIGESTS}") + if [[ -z "${EXPECTED_DIGESTS}" ]]; then + echo "::error::docker-image-name-digests must contain at least one sha256 digest" + exit 1 + fi + if EXISTING_DIGESTS=$(get_existing_platform_digests "${DOCKER_MANIFEST_NAME_WITH_TAG}"); then echo "Found existing manifest for ${DOCKER_MANIFEST_NAME_WITH_TAG}" echo " Expected platform digests: ${EXPECTED_DIGESTS}" @@ -395,6 +430,11 @@ runs: [[ -n "$digest" ]] && PREFIXED_DIGESTS+=("${DOCKER_MANIFEST_NAME}@${digest}") done + if [[ ${#PREFIXED_DIGESTS[@]} -eq 0 ]]; then + echo "::error::docker-image-name-digests must contain at least one sha256 digest" + exit 1 + fi + # Create Docker manifest echo "Creating Docker manifest with tag: ${DOCKER_MANIFEST_TAG}" @@ -483,18 +523,14 @@ runs: set -euo pipefail verify_manifest_signature() { - if [[ -n "${OIDC_IDENTITY_REGEXP}" ]]; then - cosign verify "${MANIFEST_NAME_WITH_DIGEST}" \ - --certificate-oidc-issuer "${OIDC_ISSUER}" \ - --certificate-identity-regexp "${OIDC_IDENTITY_REGEXP}" \ - --certificate-github-workflow-repository "${GITHUB_WORKFLOW_REPOSITORY}" - else - cosign verify "${MANIFEST_NAME_WITH_DIGEST}" - fi + cosign verify "${MANIFEST_NAME_WITH_DIGEST}" \ + --certificate-oidc-issuer "${OIDC_ISSUER}" \ + --certificate-identity-regexp "${OIDC_IDENTITY_REGEXP}" \ + --certificate-github-workflow-repository "${GITHUB_WORKFLOW_REPOSITORY}" } - if verify_manifest_signature >/dev/null 2>&1; then - echo "✅ Manifest already signed; skipping cosign sign (idempotent rerun)" + if [[ -n "${OIDC_IDENTITY_REGEXP}" ]] && verify_manifest_signature >/dev/null 2>&1; then + echo "✅ Manifest already signed with expected identity; skipping cosign sign (idempotent rerun)" echo "manifest-sign-skipped=true" | tee -a "${GITHUB_OUTPUT}" exit 0 fi @@ -561,7 +597,8 @@ runs: }} MANIFEST_TAG: ${{ inputs.docker-manifest-tag }} MANIFEST_CREATE_SKIPPED: - ${{ steps.create-push-docker-manifest.outputs.manifest-create-skipped }} + ${{ steps.create-push-docker-manifest.outputs.manifest-create-skipped + }} MANIFEST_SIGN_SKIPPED: ${{ steps.sign-docker-manifest.outputs.manifest-sign-skipped }} OIDC_ISSUER: ${{ inputs.cosign-oidc-issuer }} From 372f0dbad78a477b5516b2c651df631eb7f618fc Mon Sep 17 00:00:00 2001 From: HashWrangler <12961024+HashWrangler@users.noreply.github.com> Date: Wed, 17 Jun 2026 13:24:44 -0400 Subject: [PATCH 03/10] chore(build-push-docker-manifest): address additional Copilot nits Keep MANIFEST_CREATE_SKIPPED expression on one line and only sleep between cosign verify retries, not after the final failed attempt. --- actions/build-push-docker-manifest/action.yml | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/actions/build-push-docker-manifest/action.yml b/actions/build-push-docker-manifest/action.yml index 4fe4c8a7e..7e7222a5c 100644 --- a/actions/build-push-docker-manifest/action.yml +++ b/actions/build-push-docker-manifest/action.yml @@ -568,8 +568,11 @@ runs: break fi - echo "Attempt ${i}/${MAX_RETRIES}: Signature not yet available, retrying in ${RETRY_DELAY}s..." - sleep $RETRY_DELAY + echo "Attempt ${i}/${MAX_RETRIES}: Signature not yet available..." + if [[ "${i}" -lt "${MAX_RETRIES}" ]]; then + echo "Retrying in ${RETRY_DELAY}s..." + sleep "${RETRY_DELAY}" + fi done if [[ "$VERIFY_OK" != "true" ]]; then From 16eec5a25adcdc6f5537ce7767c203fd5fe9cce3 Mon Sep 17 00:00:00 2001 From: HashWrangler <12961024+HashWrangler@users.noreply.github.com> Date: Wed, 17 Jun 2026 13:36:09 -0400 Subject: [PATCH 04/10] chore(build-push-docker-manifest): address latest Copilot nits - Fail fast when jq is missing and apt-get/sudo are unavailable - Shorten step/output ids so summary env expressions stay on one line under Prettier (avoids split ${{ }} expressions) --- actions/build-push-docker-manifest/action.yml | 44 +++++++++++-------- 1 file changed, 26 insertions(+), 18 deletions(-) diff --git a/actions/build-push-docker-manifest/action.yml b/actions/build-push-docker-manifest/action.yml index 7e7222a5c..0d92c0ac8 100644 --- a/actions/build-push-docker-manifest/action.yml +++ b/actions/build-push-docker-manifest/action.yml @@ -193,13 +193,24 @@ runs: run: | if command -v jq >/dev/null 2>&1; then echo "jq already installed: $(jq --version)" - else - echo "jq not found; installing..." - sudo apt-get update -qq - sudo apt-get install -y jq - jq --version + exit 0 + fi + + if ! command -v apt-get >/dev/null 2>&1; then + echo "::error::jq is required but not installed, and apt-get is unavailable on this runner. Install jq before running this action." + exit 1 fi + if ! command -v sudo >/dev/null 2>&1; then + echo "::error::jq is required but not installed, and sudo is unavailable on this runner. Install jq before running this action." + exit 1 + fi + + echo "jq not found; installing via apt-get..." + sudo apt-get update -qq + sudo apt-get install -y jq + jq --version + - name: Generate manifest annotations id: generate-annotations shell: bash @@ -327,7 +338,7 @@ runs: fi - name: Create and push Docker manifest - id: create-push-docker-manifest + id: manifest-create shell: bash env: DOCKER_MANIFEST_NAME: ${{ steps.manifest-name.outputs.name }} @@ -409,7 +420,7 @@ runs: if [[ "${EXISTING_DIGESTS}" == "${EXPECTED_DIGESTS}" ]]; then echo "✅ Manifest already exists with expected platform digests; skipping imagetools create (idempotent rerun)" - echo "manifest-create-skipped=true" | tee -a "${GITHUB_OUTPUT}" + echo "skipped=true" | tee -a "${GITHUB_OUTPUT}" exit 0 fi @@ -419,7 +430,7 @@ runs: exit 1 fi - echo "manifest-create-skipped=false" | tee -a "${GITHUB_OUTPUT}" + echo "skipped=false" | tee -a "${GITHUB_OUTPUT}" # Convert comma-separated list into array and pass as separate arguments IFS=',' read -ra DIGESTS <<< "$DOCKER_IMAGE_NAME_DIGESTS" @@ -511,7 +522,7 @@ runs: - name: Sign Docker Manifest using GH OIDC if: inputs.docker-manifest-sign == 'true' - id: sign-docker-manifest + id: sign-manifest shell: bash env: MANIFEST_NAME_WITH_DIGEST: @@ -531,11 +542,11 @@ runs: if [[ -n "${OIDC_IDENTITY_REGEXP}" ]] && verify_manifest_signature >/dev/null 2>&1; then echo "✅ Manifest already signed with expected identity; skipping cosign sign (idempotent rerun)" - echo "manifest-sign-skipped=true" | tee -a "${GITHUB_OUTPUT}" + echo "skipped=true" | tee -a "${GITHUB_OUTPUT}" exit 0 fi - echo "manifest-sign-skipped=false" | tee -a "${GITHUB_OUTPUT}" + echo "skipped=false" | tee -a "${GITHUB_OUTPUT}" cosign sign "${MANIFEST_NAME_WITH_DIGEST}" --yes - name: Verify Docker image signature @@ -599,11 +610,8 @@ runs: steps.inspect-docker-manifest.outputs.manifest-name-with-tag }} MANIFEST_TAG: ${{ inputs.docker-manifest-tag }} - MANIFEST_CREATE_SKIPPED: - ${{ steps.create-push-docker-manifest.outputs.manifest-create-skipped - }} - MANIFEST_SIGN_SKIPPED: - ${{ steps.sign-docker-manifest.outputs.manifest-sign-skipped }} + CREATE_SKIPPED: ${{ steps.manifest-create.outputs.skipped }} + SIGN_SKIPPED: ${{ steps.sign-manifest.outputs.skipped }} OIDC_ISSUER: ${{ inputs.cosign-oidc-issuer }} OIDC_IDENTITY_REGEXP: ${{ inputs.cosign-oidc-identity-regexp }} run: | @@ -615,14 +623,14 @@ runs: echo "Manifest additional tags: \`${MANIFEST_ADDITIONAL_TAGS:-None}\`" | tee -a "${GITHUB_STEP_SUMMARY}" echo "Manifest name with tag: \`${MANIFEST_NAME_WITH_TAG}\`" | tee -a "${GITHUB_STEP_SUMMARY}" echo "Manifest name with digest: \`${MANIFEST_NAME_WITH_DIGEST}\`" | tee -a "${GITHUB_STEP_SUMMARY}" - if [[ "${MANIFEST_CREATE_SKIPPED}" == "true" ]]; then + if [[ "${CREATE_SKIPPED}" == "true" ]]; then echo "Manifest create: skipped (existing tag already points at expected platform digests)" | tee -a "${GITHUB_STEP_SUMMARY}" fi if [[ "${DOCKER_MANIFEST_SIGNED}" == 'true' ]]; then echo >> "${GITHUB_STEP_SUMMARY}" echo "#### Docker Manifest signed 📝" | tee -a "${GITHUB_STEP_SUMMARY}" echo "Manifest signed with cosign. To verify, run:" | tee -a "${GITHUB_STEP_SUMMARY}" - if [[ "${MANIFEST_SIGN_SKIPPED}" == "true" ]]; then + if [[ "${SIGN_SKIPPED}" == "true" ]]; then echo "Cosign sign: skipped (valid signature already present)" | tee -a "${GITHUB_STEP_SUMMARY}" fi echo "\`\`\`shell" >> "${GITHUB_STEP_SUMMARY}" From f20a35e3d79976f6e0a0f757778bcfce5a1cc6e7 Mon Sep 17 00:00:00 2001 From: HashWrangler <12961024+HashWrangler@users.noreply.github.com> Date: Wed, 17 Jun 2026 13:43:54 -0400 Subject: [PATCH 05/10] fix(build-push-docker-manifest): de-duplicate platform digests for idempotency Treat digest lists as sets: sort -u in normalization and existing-manifest inspection, and reuse normalized digests when building imagetools create args. --- actions/build-push-docker-manifest/action.yml | 17 +++++------------ 1 file changed, 5 insertions(+), 12 deletions(-) diff --git a/actions/build-push-docker-manifest/action.yml b/actions/build-push-docker-manifest/action.yml index 0d92c0ac8..f016751eb 100644 --- a/actions/build-push-docker-manifest/action.yml +++ b/actions/build-push-docker-manifest/action.yml @@ -366,7 +366,7 @@ runs: echo "" return fi - mapfile -t sorted < <(printf '%s\n' "${normalized[@]}" | sort) + mapfile -t sorted < <(printf '%s\n' "${normalized[@]}" | sort -u) local IFS=',' echo "${sorted[*]}" } @@ -402,7 +402,7 @@ runs: and ((.annotations."vnd.docker.reference.type" // "") != "attestation-manifest") ) | .digest - ' | sort | paste -sd, - + ' | sort -u | paste -sd, - } DOCKER_MANIFEST_NAME_WITH_TAG="${DOCKER_MANIFEST_NAME}:${DOCKER_MANIFEST_TAG}" @@ -432,20 +432,13 @@ runs: echo "skipped=false" | tee -a "${GITHUB_OUTPUT}" - # Convert comma-separated list into array and pass as separate arguments - IFS=',' read -ra DIGESTS <<< "$DOCKER_IMAGE_NAME_DIGESTS" - # Map each digest to include the manifest name + # Use normalized, de-duplicated digests for manifest create + IFS=',' read -ra DIGESTS <<< "$EXPECTED_DIGESTS" PREFIXED_DIGESTS=() for digest in "${DIGESTS[@]}"; do - digest=$(echo "$digest" | xargs) - [[ -n "$digest" ]] && PREFIXED_DIGESTS+=("${DOCKER_MANIFEST_NAME}@${digest}") + PREFIXED_DIGESTS+=("${DOCKER_MANIFEST_NAME}@${digest}") done - if [[ ${#PREFIXED_DIGESTS[@]} -eq 0 ]]; then - echo "::error::docker-image-name-digests must contain at least one sha256 digest" - exit 1 - fi - # Create Docker manifest echo "Creating Docker manifest with tag: ${DOCKER_MANIFEST_TAG}" From 435c5a830d9d9c12bc65739cfabd615cb9858086 Mon Sep 17 00:00:00 2001 From: HashWrangler <12961024+HashWrangler@users.noreply.github.com> Date: Wed, 17 Jun 2026 13:50:32 -0400 Subject: [PATCH 06/10] fix(build-push-docker-manifest): address follow-up Copilot nits - Only sleep between manifest digest inspect retries, not after the final fail - Fail fast when an existing tag has no extractable platform digests instead of falling through to imagetools create (digest drift on rerun) --- actions/build-push-docker-manifest/action.yml | 20 ++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/actions/build-push-docker-manifest/action.yml b/actions/build-push-docker-manifest/action.yml index f016751eb..b294981bf 100644 --- a/actions/build-push-docker-manifest/action.yml +++ b/actions/build-push-docker-manifest/action.yml @@ -395,14 +395,22 @@ runs: rm -f "${inspect_stderr}" - echo "${inspect_json}" | jq -r ' + local existing_digests + existing_digests=$(echo "${inspect_json}" | jq -r ' .manifest.manifests[]? | select( ((.platform.os // "") | ascii_downcase) != "unknown" and ((.annotations."vnd.docker.reference.type" // "") != "attestation-manifest") ) | .digest - ' | sort -u | paste -sd, - + ' | sort -u | paste -sd, -) + + if [[ -z "${existing_digests}" ]]; then + echo "::error::Manifest tag ${tag} exists but no platform image digests could be extracted" + exit 1 + fi + + echo "${existing_digests}" } DOCKER_MANIFEST_NAME_WITH_TAG="${DOCKER_MANIFEST_NAME}:${DOCKER_MANIFEST_TAG}" @@ -490,9 +498,11 @@ runs: fi fi - echo "Attempt ${i}/${MAX_RETRIES}: Manifest not yet available (got: '${MANIFEST_DIGEST}'), retrying in ${RETRY_DELAY}s..." - - sleep $RETRY_DELAY + echo "Attempt ${i}/${MAX_RETRIES}: Manifest not yet available (got: '${MANIFEST_DIGEST}')..." + if [[ "${i}" -lt "${MAX_RETRIES}" ]]; then + echo "Retrying in ${RETRY_DELAY}s..." + sleep "${RETRY_DELAY}" + fi MANIFEST_DIGEST="" done From d67b417842711ff7fc255940b7ffab0f918bc826 Mon Sep 17 00:00:00 2001 From: HashWrangler <12961024+HashWrangler@users.noreply.github.com> Date: Wed, 17 Jun 2026 13:56:49 -0400 Subject: [PATCH 07/10] fix(build-push-docker-manifest): validate sha256 digest format early Reject non-sha256 tokens in normalize_digest_csv so bad docker-image-name-digests input fails fast with a clear error instead of deferring to imagetools create. --- actions/build-push-docker-manifest/action.yml | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/actions/build-push-docker-manifest/action.yml b/actions/build-push-docker-manifest/action.yml index b294981bf..e4cde4191 100644 --- a/actions/build-push-docker-manifest/action.yml +++ b/actions/build-push-docker-manifest/action.yml @@ -358,9 +358,14 @@ runs: local normalized=() for digest in "${_digests[@]}"; do digest=$(echo "$digest" | xargs) - if [[ -n "$digest" ]]; then - normalized+=("$digest") + if [[ -z "$digest" ]]; then + continue fi + if [[ ! "${digest}" =~ ^sha256:[a-f0-9]{64}$ ]]; then + echo "::error::docker-image-name-digests contains invalid digest '${digest}'; expected sha256:<64-hex>" + exit 1 + fi + normalized+=("$digest") done if [[ ${#normalized[@]} -eq 0 ]]; then echo "" From dae4b887eae40193ee17fb7cf6b626053771cee5 Mon Sep 17 00:00:00 2001 From: HashWrangler <12961024+HashWrangler@users.noreply.github.com> Date: Thu, 18 Jun 2026 14:49:00 -0400 Subject: [PATCH 08/10] chore(build-push-docker-manifest): replace jq install with availability guard jq is preinstalled on GitHub-hosted ubuntu-24.04 runners; fail fast with a clear error when missing instead of apt-get/sudo install logic. --- actions/build-push-docker-manifest/action.yml | 22 ++++--------------- 1 file changed, 4 insertions(+), 18 deletions(-) diff --git a/actions/build-push-docker-manifest/action.yml b/actions/build-push-docker-manifest/action.yml index e4cde4191..23ff340f2 100644 --- a/actions/build-push-docker-manifest/action.yml +++ b/actions/build-push-docker-manifest/action.yml @@ -188,28 +188,14 @@ runs: }} registries: ${{ inputs.aws-account-number }} - - name: Ensure jq is installed + - name: Ensure jq is available shell: bash run: | - if command -v jq >/dev/null 2>&1; then - echo "jq already installed: $(jq --version)" - exit 0 - fi - - if ! command -v apt-get >/dev/null 2>&1; then - echo "::error::jq is required but not installed, and apt-get is unavailable on this runner. Install jq before running this action." + if ! command -v jq >/dev/null 2>&1; then + echo "::error::jq is required but not installed on this runner (preinstalled on GitHub-hosted ubuntu-24.04). Install jq before running this action." exit 1 fi - - if ! command -v sudo >/dev/null 2>&1; then - echo "::error::jq is required but not installed, and sudo is unavailable on this runner. Install jq before running this action." - exit 1 - fi - - echo "jq not found; installing via apt-get..." - sudo apt-get update -qq - sudo apt-get install -y jq - jq --version + echo "jq available: $(jq --version)" - name: Generate manifest annotations id: generate-annotations From 18263d8ed9541942c5c02cb7170f83e4e1e0191c Mon Sep 17 00:00:00 2001 From: HashWrangler <12961024+HashWrangler@users.noreply.github.com> Date: Thu, 2 Jul 2026 13:45:20 -0400 Subject: [PATCH 09/10] fix(build-push-docker-manifest): ECR propagation guard + move verify to workflow MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Retry manifest inspect until platform digests match expected inputs (ECR tag propagation lag). Move the cosign verify gate with 5×10s retry from the composite action into reusable-docker-build-publish; keep idempotent create, sign-skip internal verify, and sign in the action (RANE-4683). --- .../rane-4683-manifest-cosign-hardening.md | 3 +- .../reusable-docker-build-publish.yml | 39 +++++ actions/build-push-docker-manifest/action.yml | 151 ++++++++++++------ .../reusable-docker-build-publish.yml | 39 +++++ 4 files changed, 179 insertions(+), 53 deletions(-) diff --git a/.changeset/rane-4683-manifest-cosign-hardening.md b/.changeset/rane-4683-manifest-cosign-hardening.md index 45ac67ae0..e3eccc422 100644 --- a/.changeset/rane-4683-manifest-cosign-hardening.md +++ b/.changeset/rane-4683-manifest-cosign-hardening.md @@ -1,5 +1,6 @@ --- "build-push-docker-manifest": minor +"reusable-docker-build-publish": patch --- -Harden manifest create and cosign sign/verify for idempotent build-publish reruns (RANE-4683): skip imagetools create when the tag already points at the expected platform digests, skip cosign sign when a valid signature is already present, and retry cosign verify after signing to absorb Sigstore propagation lag +Harden manifest create and cosign sign for idempotent build-publish reruns (RANE-4683): skip imagetools create when the tag already points at the expected platform digests, skip cosign sign when a valid signature is already present, retry manifest tag propagation after create to absorb ECR lag, and move the cosign verify gate (with 5×10s retry for Sigstore propagation) to reusable-docker-build-publish diff --git a/.github/workflows/reusable-docker-build-publish.yml b/.github/workflows/reusable-docker-build-publish.yml index ec416f586..45459b7b6 100644 --- a/.github/workflows/reusable-docker-build-publish.yml +++ b/.github/workflows/reusable-docker-build-publish.yml @@ -854,6 +854,45 @@ jobs: aws-role-arn: ${{ secrets.AWS_ROLE_PUBLISH_ARN }} aws-region: ${{ inputs.aws-region-ecr }} + - name: Verify Docker manifest signature + if: ${{ inputs.docker-manifest-sign == 'true' }} + shell: bash + env: + MANIFEST_NAME_WITH_DIGEST: ${{ steps.docker-manifest.outputs.manifest-name-with-digest }} + GITHUB_WORKFLOW_REPOSITORY: ${{ inputs.github-workflow-repository }} + OIDC_ISSUER: https://token.actions.githubusercontent.com + OIDC_IDENTITY_REGEXP: "^https://github.com/smartcontractkit/.*$" + run: | + set -euo pipefail + + MAX_RETRIES=5 + RETRY_DELAY=10 + VERIFY_OK=false + + for i in $(seq 1 $MAX_RETRIES); do + echo "Attempt ${i}/${MAX_RETRIES}: Verifying cosign signature for ${MANIFEST_NAME_WITH_DIGEST}..." + + if cosign verify "${MANIFEST_NAME_WITH_DIGEST}" \ + --certificate-oidc-issuer "${OIDC_ISSUER}" \ + --certificate-identity-regexp "${OIDC_IDENTITY_REGEXP}" \ + --certificate-github-workflow-repository "${GITHUB_WORKFLOW_REPOSITORY}"; then + echo "Successfully verified signature on attempt ${i}" + VERIFY_OK=true + break + fi + + echo "Attempt ${i}/${MAX_RETRIES}: Signature not yet available..." + if [[ "${i}" -lt "${MAX_RETRIES}" ]]; then + echo "Retrying in ${RETRY_DELAY}s..." + sleep "${RETRY_DELAY}" + fi + done + + if [[ "$VERIFY_OK" != "true" ]]; then + echo "::error::Failed to verify cosign signature for ${MANIFEST_NAME_WITH_DIGEST} after ${MAX_RETRIES} attempts" + exit 1 + fi + # Attest only the manifest index. Build-provenance attestations do not # transfer from the per-arch images to the index, so the index is # attested explicitly here. Requires the calling job to grant diff --git a/actions/build-push-docker-manifest/action.yml b/actions/build-push-docker-manifest/action.yml index 23ff340f2..c9bd59053 100644 --- a/actions/build-push-docker-manifest/action.yml +++ b/actions/build-push-docker-manifest/action.yml @@ -471,34 +471,123 @@ runs: env: DOCKER_MANIFEST_NAME: ${{ steps.manifest-name.outputs.name }} DOCKER_MANIFEST_TAG: ${{ inputs.docker-manifest-tag }} + DOCKER_IMAGE_NAME_DIGESTS: ${{ inputs.docker-image-name-digests }} run: | + set -euo pipefail + + normalize_digest_csv() { + local input="$1" + IFS=',' read -ra _digests <<< "$input" + local normalized=() + for digest in "${_digests[@]}"; do + digest=$(echo "$digest" | xargs) + if [[ -z "$digest" ]]; then + continue + fi + if [[ ! "${digest}" =~ ^sha256:[a-f0-9]{64}$ ]]; then + echo "::error::docker-image-name-digests contains invalid digest '${digest}'; expected sha256:<64-hex>" + exit 1 + fi + normalized+=("$digest") + done + if [[ ${#normalized[@]} -eq 0 ]]; then + echo "" + return + fi + mapfile -t sorted < <(printf '%s\n' "${normalized[@]}" | sort -u) + local IFS=',' + echo "${sorted[*]}" + } + + # Returns 0 when tag is readable and platform digests match expected; prints index digest to stdout. + # Returns 1 when tag is not found yet (ECR/tag propagation lag). + # Exits 1 on unexpected inspect errors. + wait_for_manifest_tag_propagation() { + local tag="$1" + local expected_digests="$2" + local inspect_json inspect_stderr inspect_status existing_digests index_digest + + inspect_stderr=$(mktemp) + set +e + inspect_json=$(docker buildx imagetools inspect "${tag}" --format '{{json .}}' 2>"${inspect_stderr}") + inspect_status=$? + set -e + + if [[ "${inspect_status}" -ne 0 ]]; then + if grep -qiE 'not found|manifest unknown|name unknown|404|does not exist|no such manifest' "${inspect_stderr}"; then + rm -f "${inspect_stderr}" + return 1 + fi + + echo "::error::Failed to inspect manifest tag ${tag}" + cat "${inspect_stderr}" >&2 + rm -f "${inspect_stderr}" + exit 1 + fi + + rm -f "${inspect_stderr}" + + existing_digests=$(echo "${inspect_json}" | jq -r ' + .manifest.manifests[]? + | select( + ((.platform.os // "") | ascii_downcase) != "unknown" + and ((.annotations."vnd.docker.reference.type" // "") != "attestation-manifest") + ) + | .digest + ' | sort -u | paste -sd, -) + + if [[ -z "${existing_digests}" ]]; then + echo "Manifest tag ${tag} is visible but platform digests are not yet available" >&2 + return 1 + fi + + if [[ "${existing_digests}" != "${expected_digests}" ]]; then + echo "Manifest tag ${tag} visible but platform digests not yet fully propagated" >&2 + echo " Expected: ${expected_digests}" >&2 + echo " Got: ${existing_digests}" >&2 + return 1 + fi + + index_digest=$(echo "${inspect_json}" | jq -r '.manifest.digest // .digest // empty') + if [[ ! "${index_digest}" =~ ^sha256:[a-f0-9]{64}$ ]]; then + echo "Manifest tag ${tag} matched platform digests but index digest not yet readable (got: '${index_digest}')" >&2 + return 1 + fi + + echo "${index_digest}" + return 0 + } + DOCKER_MANIFEST_NAME_WITH_TAG="${DOCKER_MANIFEST_NAME}:${DOCKER_MANIFEST_TAG}" + EXPECTED_DIGESTS=$(normalize_digest_csv "${DOCKER_IMAGE_NAME_DIGESTS}") + + if [[ -z "${EXPECTED_DIGESTS}" ]]; then + echo "::error::docker-image-name-digests must contain at least one sha256 digest" + exit 1 + fi MAX_RETRIES=5 RETRY_DELAY=10 MANIFEST_DIGEST="" for i in $(seq 1 $MAX_RETRIES); do - echo "Attempt ${i}/${MAX_RETRIES}: Inspecting manifest (${DOCKER_MANIFEST_NAME_WITH_TAG}) to retrieve digest..." + echo "Attempt ${i}/${MAX_RETRIES}: Waiting for manifest tag ${DOCKER_MANIFEST_NAME_WITH_TAG} to propagate in registry..." - if INSPECT_OUTPUT=$(docker buildx imagetools inspect "${DOCKER_MANIFEST_NAME_WITH_TAG}" 2>/dev/null); then - MANIFEST_DIGEST=$(echo "${INSPECT_OUTPUT}" | grep -m1 'Digest:' | awk '{print $2}') - if [[ "${MANIFEST_DIGEST}" =~ ^sha256:[a-f0-9]{64}$ ]]; then - echo "Successfully retrieved manifest digest on attempt ${i}: ${MANIFEST_DIGEST}" - break - fi + if MANIFEST_DIGEST=$(wait_for_manifest_tag_propagation "${DOCKER_MANIFEST_NAME_WITH_TAG}" "${EXPECTED_DIGESTS}"); then + echo "✅ Manifest tag propagated with expected platform digests on attempt ${i}: ${MANIFEST_DIGEST}" + break fi - echo "Attempt ${i}/${MAX_RETRIES}: Manifest not yet available (got: '${MANIFEST_DIGEST}')..." + MANIFEST_DIGEST="" if [[ "${i}" -lt "${MAX_RETRIES}" ]]; then echo "Retrying in ${RETRY_DELAY}s..." sleep "${RETRY_DELAY}" fi - MANIFEST_DIGEST="" done if [[ -z "${MANIFEST_DIGEST}" ]]; then - echo "::error::Failed to retrieve manifest digest for ${DOCKER_MANIFEST_NAME_WITH_TAG} after ${MAX_RETRIES} attempts" + echo "::error::Failed to verify manifest tag propagation for ${DOCKER_MANIFEST_NAME_WITH_TAG} after ${MAX_RETRIES} attempts" + echo "::error::Expected platform digests: ${EXPECTED_DIGESTS}" exit 1 fi @@ -543,48 +632,6 @@ runs: echo "skipped=false" | tee -a "${GITHUB_OUTPUT}" cosign sign "${MANIFEST_NAME_WITH_DIGEST}" --yes - - name: Verify Docker image signature - if: - inputs.docker-manifest-sign == 'true' && - inputs.cosign-oidc-identity-regexp != '' - shell: bash - env: - MANIFEST_NAME_WITH_DIGEST: >- - ${{ - steps.inspect-docker-manifest.outputs.manifest-name-with-digest - }} - GITHUB_WORKFLOW_REPOSITORY: ${{ inputs.github-workflow-repository }} - OIDC_ISSUER: ${{ inputs.cosign-oidc-issuer }} - OIDC_IDENTITY_REGEXP: ${{ inputs.cosign-oidc-identity-regexp }} - run: | - MAX_RETRIES=5 - RETRY_DELAY=10 - VERIFY_OK=false - - for i in $(seq 1 $MAX_RETRIES); do - echo "Attempt ${i}/${MAX_RETRIES}: Verifying cosign signature for ${MANIFEST_NAME_WITH_DIGEST}..." - - if cosign verify "${MANIFEST_NAME_WITH_DIGEST}" \ - --certificate-oidc-issuer "${OIDC_ISSUER}" \ - --certificate-identity-regexp "${OIDC_IDENTITY_REGEXP}" \ - --certificate-github-workflow-repository "${GITHUB_WORKFLOW_REPOSITORY}"; then - echo "Successfully verified signature on attempt ${i}" - VERIFY_OK=true - break - fi - - echo "Attempt ${i}/${MAX_RETRIES}: Signature not yet available..." - if [[ "${i}" -lt "${MAX_RETRIES}" ]]; then - echo "Retrying in ${RETRY_DELAY}s..." - sleep "${RETRY_DELAY}" - fi - done - - if [[ "$VERIFY_OK" != "true" ]]; then - echo "::error::Failed to verify cosign signature for ${MANIFEST_NAME_WITH_DIGEST} after ${MAX_RETRIES} attempts" - exit 1 - fi - - name: Summary output shell: bash env: diff --git a/workflows/reusable-docker-build-publish/reusable-docker-build-publish.yml b/workflows/reusable-docker-build-publish/reusable-docker-build-publish.yml index 1b581d404..c1d1d47fd 100644 --- a/workflows/reusable-docker-build-publish/reusable-docker-build-publish.yml +++ b/workflows/reusable-docker-build-publish/reusable-docker-build-publish.yml @@ -850,6 +850,45 @@ jobs: aws-role-arn: ${{ secrets.AWS_ROLE_PUBLISH_ARN }} aws-region: ${{ inputs.aws-region-ecr }} + - name: Verify Docker manifest signature + if: ${{ inputs.docker-manifest-sign == 'true' }} + shell: bash + env: + MANIFEST_NAME_WITH_DIGEST: ${{ steps.docker-manifest.outputs.manifest-name-with-digest }} + GITHUB_WORKFLOW_REPOSITORY: ${{ inputs.github-workflow-repository }} + OIDC_ISSUER: https://token.actions.githubusercontent.com + OIDC_IDENTITY_REGEXP: "^https://github.com/smartcontractkit/.*$" + run: | + set -euo pipefail + + MAX_RETRIES=5 + RETRY_DELAY=10 + VERIFY_OK=false + + for i in $(seq 1 $MAX_RETRIES); do + echo "Attempt ${i}/${MAX_RETRIES}: Verifying cosign signature for ${MANIFEST_NAME_WITH_DIGEST}..." + + if cosign verify "${MANIFEST_NAME_WITH_DIGEST}" \ + --certificate-oidc-issuer "${OIDC_ISSUER}" \ + --certificate-identity-regexp "${OIDC_IDENTITY_REGEXP}" \ + --certificate-github-workflow-repository "${GITHUB_WORKFLOW_REPOSITORY}"; then + echo "Successfully verified signature on attempt ${i}" + VERIFY_OK=true + break + fi + + echo "Attempt ${i}/${MAX_RETRIES}: Signature not yet available..." + if [[ "${i}" -lt "${MAX_RETRIES}" ]]; then + echo "Retrying in ${RETRY_DELAY}s..." + sleep "${RETRY_DELAY}" + fi + done + + if [[ "$VERIFY_OK" != "true" ]]; then + echo "::error::Failed to verify cosign signature for ${MANIFEST_NAME_WITH_DIGEST} after ${MAX_RETRIES} attempts" + exit 1 + fi + # Attest only the manifest index. Build-provenance attestations do not # transfer from the per-arch images to the index, so the index is # attested explicitly here. Requires the calling job to grant From 78f8f163f6c6b862f41acff5b963a303f5925802 Mon Sep 17 00:00:00 2001 From: HashWrangler <12961024+HashWrangler@users.noreply.github.com> Date: Thu, 2 Jul 2026 14:13:36 -0400 Subject: [PATCH 10/10] fix(build-push-docker-manifest): read additional tags from action input in summary The step summary referenced a manifest-additional-tags output that inspect-docker-manifest never sets; use the manifest-additional-tags input instead so additional tags display correctly. --- actions/build-push-docker-manifest/action.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/actions/build-push-docker-manifest/action.yml b/actions/build-push-docker-manifest/action.yml index c9bd59053..c083318ef 100644 --- a/actions/build-push-docker-manifest/action.yml +++ b/actions/build-push-docker-manifest/action.yml @@ -637,8 +637,7 @@ runs: env: DOCKER_MANIFEST_SIGNED: ${{ inputs.docker-manifest-sign }} GITHUB_WORKFLOW_REPOSITORY: ${{ inputs.github-workflow-repository }} - MANIFEST_ADDITIONAL_TAGS: - ${{ steps.inspect-docker-manifest.outputs.manifest-additional-tags }} + MANIFEST_ADDITIONAL_TAGS: ${{ inputs.manifest-additional-tags }} MANIFEST_DIGEST: ${{ steps.inspect-docker-manifest.outputs.manifest-digest }} MANIFEST_NAME: ${{ steps.inspect-docker-manifest.outputs.manifest-name}}