diff --git a/.changeset/rane-4683-manifest-cosign-hardening.md b/.changeset/rane-4683-manifest-cosign-hardening.md new file mode 100644 index 000000000..e3eccc422 --- /dev/null +++ b/.changeset/rane-4683-manifest-cosign-hardening.md @@ -0,0 +1,6 @@ +--- +"build-push-docker-manifest": minor +"reusable-docker-build-publish": patch +--- + +Harden manifest create and cosign sign for idempotent build-publish reruns (RANE-4683): skip imagetools create when the tag already points at the expected platform digests, skip cosign sign when a valid signature is already present, retry manifest tag propagation after create to absorb ECR lag, and move the cosign verify gate (with 5×10s retry for Sigstore propagation) to reusable-docker-build-publish diff --git a/.github/workflows/reusable-docker-build-publish.yml b/.github/workflows/reusable-docker-build-publish.yml index ec416f586..45459b7b6 100644 --- a/.github/workflows/reusable-docker-build-publish.yml +++ b/.github/workflows/reusable-docker-build-publish.yml @@ -854,6 +854,45 @@ jobs: aws-role-arn: ${{ secrets.AWS_ROLE_PUBLISH_ARN }} aws-region: ${{ inputs.aws-region-ecr }} + - name: Verify Docker manifest signature + if: ${{ inputs.docker-manifest-sign == 'true' }} + shell: bash + env: + MANIFEST_NAME_WITH_DIGEST: ${{ steps.docker-manifest.outputs.manifest-name-with-digest }} + GITHUB_WORKFLOW_REPOSITORY: ${{ inputs.github-workflow-repository }} + OIDC_ISSUER: https://token.actions.githubusercontent.com + OIDC_IDENTITY_REGEXP: "^https://github.com/smartcontractkit/.*$" + run: | + set -euo pipefail + + MAX_RETRIES=5 + RETRY_DELAY=10 + VERIFY_OK=false + + for i in $(seq 1 $MAX_RETRIES); do + echo "Attempt ${i}/${MAX_RETRIES}: Verifying cosign signature for ${MANIFEST_NAME_WITH_DIGEST}..." + + if cosign verify "${MANIFEST_NAME_WITH_DIGEST}" \ + --certificate-oidc-issuer "${OIDC_ISSUER}" \ + --certificate-identity-regexp "${OIDC_IDENTITY_REGEXP}" \ + --certificate-github-workflow-repository "${GITHUB_WORKFLOW_REPOSITORY}"; then + echo "Successfully verified signature on attempt ${i}" + VERIFY_OK=true + break + fi + + echo "Attempt ${i}/${MAX_RETRIES}: Signature not yet available..." + if [[ "${i}" -lt "${MAX_RETRIES}" ]]; then + echo "Retrying in ${RETRY_DELAY}s..." + sleep "${RETRY_DELAY}" + fi + done + + if [[ "$VERIFY_OK" != "true" ]]; then + echo "::error::Failed to verify cosign signature for ${MANIFEST_NAME_WITH_DIGEST} after ${MAX_RETRIES} attempts" + exit 1 + fi + # Attest only the manifest index. Build-provenance attestations do not # transfer from the per-arch images to the index, so the index is # attested explicitly here. Requires the calling job to grant diff --git a/actions/build-push-docker-manifest/action.yml b/actions/build-push-docker-manifest/action.yml index e4f002959..c083318ef 100644 --- a/actions/build-push-docker-manifest/action.yml +++ b/actions/build-push-docker-manifest/action.yml @@ -188,6 +188,15 @@ runs: }} registries: ${{ inputs.aws-account-number }} + - name: Ensure jq is available + shell: bash + run: | + if ! command -v jq >/dev/null 2>&1; then + echo "::error::jq is required but not installed on this runner (preinstalled on GitHub-hosted ubuntu-24.04). Install jq before running this action." + exit 1 + fi + echo "jq available: $(jq --version)" + - name: Generate manifest annotations id: generate-annotations shell: bash @@ -315,7 +324,7 @@ runs: fi - name: Create and push Docker manifest - id: create-push-docker-manifest + id: manifest-create shell: bash env: DOCKER_MANIFEST_NAME: ${{ steps.manifest-name.outputs.name }} @@ -327,10 +336,103 @@ runs: ${{ steps.generate-annotations.outputs.annotation-flags }} TAG_FLAGS: ${{ steps.process-additional-tags.outputs.tag-flags }} run: | + set -euo pipefail + + normalize_digest_csv() { + local input="$1" + IFS=',' read -ra _digests <<< "$input" + local normalized=() + for digest in "${_digests[@]}"; do + digest=$(echo "$digest" | xargs) + if [[ -z "$digest" ]]; then + continue + fi + if [[ ! "${digest}" =~ ^sha256:[a-f0-9]{64}$ ]]; then + echo "::error::docker-image-name-digests contains invalid digest '${digest}'; expected sha256:<64-hex>" + exit 1 + fi + normalized+=("$digest") + done + if [[ ${#normalized[@]} -eq 0 ]]; then + echo "" + return + fi + mapfile -t sorted < <(printf '%s\n' "${normalized[@]}" | sort -u) + local IFS=',' + echo "${sorted[*]}" + } + + get_existing_platform_digests() { + local tag="$1" + local inspect_json inspect_stderr inspect_status + + inspect_stderr=$(mktemp) + set +e + inspect_json=$(docker buildx imagetools inspect "${tag}" --format '{{json .}}' 2>"${inspect_stderr}") + inspect_status=$? + set -e + + if [[ "${inspect_status}" -ne 0 ]]; then + if grep -qiE 'not found|manifest unknown|name unknown|404|does not exist|no such manifest' "${inspect_stderr}"; then + rm -f "${inspect_stderr}" + return 1 + fi + + echo "::error::Failed to inspect manifest tag ${tag}" + cat "${inspect_stderr}" >&2 + rm -f "${inspect_stderr}" + exit 1 + fi + + rm -f "${inspect_stderr}" + + local existing_digests + existing_digests=$(echo "${inspect_json}" | jq -r ' + .manifest.manifests[]? + | select( + ((.platform.os // "") | ascii_downcase) != "unknown" + and ((.annotations."vnd.docker.reference.type" // "") != "attestation-manifest") + ) + | .digest + ' | sort -u | paste -sd, -) + + if [[ -z "${existing_digests}" ]]; then + echo "::error::Manifest tag ${tag} exists but no platform image digests could be extracted" + exit 1 + fi + + echo "${existing_digests}" + } + DOCKER_MANIFEST_NAME_WITH_TAG="${DOCKER_MANIFEST_NAME}:${DOCKER_MANIFEST_TAG}" - # Convert comma-separated list into array and pass as separate arguments - IFS=',' read -ra DIGESTS <<< "$DOCKER_IMAGE_NAME_DIGESTS" - # Map each digest to include the manifest name + EXPECTED_DIGESTS=$(normalize_digest_csv "${DOCKER_IMAGE_NAME_DIGESTS}") + + if [[ -z "${EXPECTED_DIGESTS}" ]]; then + echo "::error::docker-image-name-digests must contain at least one sha256 digest" + exit 1 + fi + + if EXISTING_DIGESTS=$(get_existing_platform_digests "${DOCKER_MANIFEST_NAME_WITH_TAG}"); then + echo "Found existing manifest for ${DOCKER_MANIFEST_NAME_WITH_TAG}" + echo " Expected platform digests: ${EXPECTED_DIGESTS}" + echo " Existing platform digests: ${EXISTING_DIGESTS}" + + if [[ "${EXISTING_DIGESTS}" == "${EXPECTED_DIGESTS}" ]]; then + echo "✅ Manifest already exists with expected platform digests; skipping imagetools create (idempotent rerun)" + echo "skipped=true" | tee -a "${GITHUB_OUTPUT}" + exit 0 + fi + + echo "::error::Manifest tag ${DOCKER_MANIFEST_NAME_WITH_TAG} already exists with different platform digests" + echo "::error::Expected: ${EXPECTED_DIGESTS}" + echo "::error::Existing: ${EXISTING_DIGESTS}" + exit 1 + fi + + echo "skipped=false" | tee -a "${GITHUB_OUTPUT}" + + # Use normalized, de-duplicated digests for manifest create + IFS=',' read -ra DIGESTS <<< "$EXPECTED_DIGESTS" PREFIXED_DIGESTS=() for digest in "${DIGESTS[@]}"; do PREFIXED_DIGESTS+=("${DOCKER_MANIFEST_NAME}@${digest}") @@ -369,32 +471,123 @@ runs: env: DOCKER_MANIFEST_NAME: ${{ steps.manifest-name.outputs.name }} DOCKER_MANIFEST_TAG: ${{ inputs.docker-manifest-tag }} + DOCKER_IMAGE_NAME_DIGESTS: ${{ inputs.docker-image-name-digests }} run: | + set -euo pipefail + + normalize_digest_csv() { + local input="$1" + IFS=',' read -ra _digests <<< "$input" + local normalized=() + for digest in "${_digests[@]}"; do + digest=$(echo "$digest" | xargs) + if [[ -z "$digest" ]]; then + continue + fi + if [[ ! "${digest}" =~ ^sha256:[a-f0-9]{64}$ ]]; then + echo "::error::docker-image-name-digests contains invalid digest '${digest}'; expected sha256:<64-hex>" + exit 1 + fi + normalized+=("$digest") + done + if [[ ${#normalized[@]} -eq 0 ]]; then + echo "" + return + fi + mapfile -t sorted < <(printf '%s\n' "${normalized[@]}" | sort -u) + local IFS=',' + echo "${sorted[*]}" + } + + # Returns 0 when tag is readable and platform digests match expected; prints index digest to stdout. + # Returns 1 when tag is not found yet (ECR/tag propagation lag). + # Exits 1 on unexpected inspect errors. + wait_for_manifest_tag_propagation() { + local tag="$1" + local expected_digests="$2" + local inspect_json inspect_stderr inspect_status existing_digests index_digest + + inspect_stderr=$(mktemp) + set +e + inspect_json=$(docker buildx imagetools inspect "${tag}" --format '{{json .}}' 2>"${inspect_stderr}") + inspect_status=$? + set -e + + if [[ "${inspect_status}" -ne 0 ]]; then + if grep -qiE 'not found|manifest unknown|name unknown|404|does not exist|no such manifest' "${inspect_stderr}"; then + rm -f "${inspect_stderr}" + return 1 + fi + + echo "::error::Failed to inspect manifest tag ${tag}" + cat "${inspect_stderr}" >&2 + rm -f "${inspect_stderr}" + exit 1 + fi + + rm -f "${inspect_stderr}" + + existing_digests=$(echo "${inspect_json}" | jq -r ' + .manifest.manifests[]? + | select( + ((.platform.os // "") | ascii_downcase) != "unknown" + and ((.annotations."vnd.docker.reference.type" // "") != "attestation-manifest") + ) + | .digest + ' | sort -u | paste -sd, -) + + if [[ -z "${existing_digests}" ]]; then + echo "Manifest tag ${tag} is visible but platform digests are not yet available" >&2 + return 1 + fi + + if [[ "${existing_digests}" != "${expected_digests}" ]]; then + echo "Manifest tag ${tag} visible but platform digests not yet fully propagated" >&2 + echo " Expected: ${expected_digests}" >&2 + echo " Got: ${existing_digests}" >&2 + return 1 + fi + + index_digest=$(echo "${inspect_json}" | jq -r '.manifest.digest // .digest // empty') + if [[ ! "${index_digest}" =~ ^sha256:[a-f0-9]{64}$ ]]; then + echo "Manifest tag ${tag} matched platform digests but index digest not yet readable (got: '${index_digest}')" >&2 + return 1 + fi + + echo "${index_digest}" + return 0 + } + DOCKER_MANIFEST_NAME_WITH_TAG="${DOCKER_MANIFEST_NAME}:${DOCKER_MANIFEST_TAG}" + EXPECTED_DIGESTS=$(normalize_digest_csv "${DOCKER_IMAGE_NAME_DIGESTS}") + + if [[ -z "${EXPECTED_DIGESTS}" ]]; then + echo "::error::docker-image-name-digests must contain at least one sha256 digest" + exit 1 + fi MAX_RETRIES=5 RETRY_DELAY=10 MANIFEST_DIGEST="" for i in $(seq 1 $MAX_RETRIES); do - echo "Attempt ${i}/${MAX_RETRIES}: Inspecting manifest (${DOCKER_MANIFEST_NAME_WITH_TAG}) to retrieve digest..." + echo "Attempt ${i}/${MAX_RETRIES}: Waiting for manifest tag ${DOCKER_MANIFEST_NAME_WITH_TAG} to propagate in registry..." - if INSPECT_OUTPUT=$(docker buildx imagetools inspect "${DOCKER_MANIFEST_NAME_WITH_TAG}" 2>/dev/null); then - MANIFEST_DIGEST=$(echo "${INSPECT_OUTPUT}" | grep -m1 'Digest:' | awk '{print $2}') - if [[ "${MANIFEST_DIGEST}" =~ ^sha256:[a-f0-9]{64}$ ]]; then - echo "Successfully retrieved manifest digest on attempt ${i}: ${MANIFEST_DIGEST}" - break - fi + if MANIFEST_DIGEST=$(wait_for_manifest_tag_propagation "${DOCKER_MANIFEST_NAME_WITH_TAG}" "${EXPECTED_DIGESTS}"); then + echo "✅ Manifest tag propagated with expected platform digests on attempt ${i}: ${MANIFEST_DIGEST}" + break fi - echo "Attempt ${i}/${MAX_RETRIES}: Manifest not yet available (got: '${MANIFEST_DIGEST}'), retrying in ${RETRY_DELAY}s..." - - sleep $RETRY_DELAY MANIFEST_DIGEST="" + if [[ "${i}" -lt "${MAX_RETRIES}" ]]; then + echo "Retrying in ${RETRY_DELAY}s..." + sleep "${RETRY_DELAY}" + fi done if [[ -z "${MANIFEST_DIGEST}" ]]; then - echo "::error::Failed to retrieve manifest digest for ${DOCKER_MANIFEST_NAME_WITH_TAG} after ${MAX_RETRIES} attempts" + echo "::error::Failed to verify manifest tag propagation for ${DOCKER_MANIFEST_NAME_WITH_TAG} after ${MAX_RETRIES} attempts" + echo "::error::Expected platform digests: ${EXPECTED_DIGESTS}" exit 1 fi @@ -412,38 +605,39 @@ runs: - name: Sign Docker Manifest using GH OIDC if: inputs.docker-manifest-sign == 'true' - shell: sh + id: sign-manifest + shell: bash env: MANIFEST_NAME_WITH_DIGEST: ${{ steps.inspect-docker-manifest.outputs.manifest-name-with-digest }} - run: cosign sign "${MANIFEST_NAME_WITH_DIGEST}" --yes - - - name: Verify Docker image signature - if: - inputs.docker-manifest-sign == 'true' && - inputs.cosign-oidc-identity-regexp != '' - shell: sh - env: - MANIFEST_NAME_WITH_DIGEST: >- - ${{ - steps.inspect-docker-manifest.outputs.manifest-name-with-digest - }} GITHUB_WORKFLOW_REPOSITORY: ${{ inputs.github-workflow-repository }} OIDC_ISSUER: ${{ inputs.cosign-oidc-issuer }} OIDC_IDENTITY_REGEXP: ${{ inputs.cosign-oidc-identity-regexp }} run: | - cosign verify "${MANIFEST_NAME_WITH_DIGEST}" \ - --certificate-oidc-issuer "${OIDC_ISSUER}" \ - --certificate-identity-regexp "${OIDC_IDENTITY_REGEXP}" \ - --certificate-github-workflow-repository "${GITHUB_WORKFLOW_REPOSITORY}" + set -euo pipefail + + verify_manifest_signature() { + cosign verify "${MANIFEST_NAME_WITH_DIGEST}" \ + --certificate-oidc-issuer "${OIDC_ISSUER}" \ + --certificate-identity-regexp "${OIDC_IDENTITY_REGEXP}" \ + --certificate-github-workflow-repository "${GITHUB_WORKFLOW_REPOSITORY}" + } + + if [[ -n "${OIDC_IDENTITY_REGEXP}" ]] && verify_manifest_signature >/dev/null 2>&1; then + echo "✅ Manifest already signed with expected identity; skipping cosign sign (idempotent rerun)" + echo "skipped=true" | tee -a "${GITHUB_OUTPUT}" + exit 0 + fi + + echo "skipped=false" | tee -a "${GITHUB_OUTPUT}" + cosign sign "${MANIFEST_NAME_WITH_DIGEST}" --yes - name: Summary output shell: bash env: DOCKER_MANIFEST_SIGNED: ${{ inputs.docker-manifest-sign }} GITHUB_WORKFLOW_REPOSITORY: ${{ inputs.github-workflow-repository }} - MANIFEST_ADDITIONAL_TAGS: - ${{ steps.inspect-docker-manifest.outputs.manifest-additional-tags }} + MANIFEST_ADDITIONAL_TAGS: ${{ inputs.manifest-additional-tags }} MANIFEST_DIGEST: ${{ steps.inspect-docker-manifest.outputs.manifest-digest }} MANIFEST_NAME: ${{ steps.inspect-docker-manifest.outputs.manifest-name}} @@ -456,6 +650,8 @@ runs: steps.inspect-docker-manifest.outputs.manifest-name-with-tag }} MANIFEST_TAG: ${{ inputs.docker-manifest-tag }} + CREATE_SKIPPED: ${{ steps.manifest-create.outputs.skipped }} + SIGN_SKIPPED: ${{ steps.sign-manifest.outputs.skipped }} OIDC_ISSUER: ${{ inputs.cosign-oidc-issuer }} OIDC_IDENTITY_REGEXP: ${{ inputs.cosign-oidc-identity-regexp }} run: | @@ -467,10 +663,16 @@ runs: echo "Manifest additional tags: \`${MANIFEST_ADDITIONAL_TAGS:-None}\`" | tee -a "${GITHUB_STEP_SUMMARY}" echo "Manifest name with tag: \`${MANIFEST_NAME_WITH_TAG}\`" | tee -a "${GITHUB_STEP_SUMMARY}" echo "Manifest name with digest: \`${MANIFEST_NAME_WITH_DIGEST}\`" | tee -a "${GITHUB_STEP_SUMMARY}" + if [[ "${CREATE_SKIPPED}" == "true" ]]; then + echo "Manifest create: skipped (existing tag already points at expected platform digests)" | tee -a "${GITHUB_STEP_SUMMARY}" + fi if [[ "${DOCKER_MANIFEST_SIGNED}" == 'true' ]]; then echo >> "${GITHUB_STEP_SUMMARY}" echo "#### Docker Manifest signed 📝" | tee -a "${GITHUB_STEP_SUMMARY}" echo "Manifest signed with cosign. To verify, run:" | tee -a "${GITHUB_STEP_SUMMARY}" + if [[ "${SIGN_SKIPPED}" == "true" ]]; then + echo "Cosign sign: skipped (valid signature already present)" | tee -a "${GITHUB_STEP_SUMMARY}" + fi echo "\`\`\`shell" >> "${GITHUB_STEP_SUMMARY}" echo "cosign verify ${MANIFEST_NAME_WITH_DIGEST} --certificate-oidc-issuer ${OIDC_ISSUER} --certificate-identity-regexp '${OIDC_IDENTITY_REGEXP}' --certificate-github-workflow-repository ${GITHUB_WORKFLOW_REPOSITORY}" | tee -a "${GITHUB_STEP_SUMMARY}" echo "\`\`\`" >> "${GITHUB_STEP_SUMMARY}" diff --git a/workflows/reusable-docker-build-publish/reusable-docker-build-publish.yml b/workflows/reusable-docker-build-publish/reusable-docker-build-publish.yml index 1b581d404..c1d1d47fd 100644 --- a/workflows/reusable-docker-build-publish/reusable-docker-build-publish.yml +++ b/workflows/reusable-docker-build-publish/reusable-docker-build-publish.yml @@ -850,6 +850,45 @@ jobs: aws-role-arn: ${{ secrets.AWS_ROLE_PUBLISH_ARN }} aws-region: ${{ inputs.aws-region-ecr }} + - name: Verify Docker manifest signature + if: ${{ inputs.docker-manifest-sign == 'true' }} + shell: bash + env: + MANIFEST_NAME_WITH_DIGEST: ${{ steps.docker-manifest.outputs.manifest-name-with-digest }} + GITHUB_WORKFLOW_REPOSITORY: ${{ inputs.github-workflow-repository }} + OIDC_ISSUER: https://token.actions.githubusercontent.com + OIDC_IDENTITY_REGEXP: "^https://github.com/smartcontractkit/.*$" + run: | + set -euo pipefail + + MAX_RETRIES=5 + RETRY_DELAY=10 + VERIFY_OK=false + + for i in $(seq 1 $MAX_RETRIES); do + echo "Attempt ${i}/${MAX_RETRIES}: Verifying cosign signature for ${MANIFEST_NAME_WITH_DIGEST}..." + + if cosign verify "${MANIFEST_NAME_WITH_DIGEST}" \ + --certificate-oidc-issuer "${OIDC_ISSUER}" \ + --certificate-identity-regexp "${OIDC_IDENTITY_REGEXP}" \ + --certificate-github-workflow-repository "${GITHUB_WORKFLOW_REPOSITORY}"; then + echo "Successfully verified signature on attempt ${i}" + VERIFY_OK=true + break + fi + + echo "Attempt ${i}/${MAX_RETRIES}: Signature not yet available..." + if [[ "${i}" -lt "${MAX_RETRIES}" ]]; then + echo "Retrying in ${RETRY_DELAY}s..." + sleep "${RETRY_DELAY}" + fi + done + + if [[ "$VERIFY_OK" != "true" ]]; then + echo "::error::Failed to verify cosign signature for ${MANIFEST_NAME_WITH_DIGEST} after ${MAX_RETRIES} attempts" + exit 1 + fi + # Attest only the manifest index. Build-provenance attestations do not # transfer from the per-arch images to the index, so the index is # attested explicitly here. Requires the calling job to grant