From 5839346df1dd9d8edaf4cf031e803f654f3026f1 Mon Sep 17 00:00:00 2001 From: trevorwilliams2025 Date: Wed, 15 Jul 2026 12:09:50 +1000 Subject: [PATCH 1/2] fix(pipelines): add retry wrappers to bare network calls in image mirror script (AROSLSRE-1490) The on-demand sync script (on-demand.sh) had several network-touching az/oras/oc calls without retry logic. A transient IMDS/ARM connection reset (BadStatusLine HTTP/1.1 000) on az keyvault secret download caused the mirror-oc-mirror-image Ev2 step to fail in uksouth (stg), even though the retry helper and pattern already existed in the script. Wrap all bare network calls with the existing retry helper: - az keyvault secret download (direct cause of the outage) - az cloud show (both registry and OCI layout paths, deduplicated) - oc registry login - oras login (both paths) - oras cp (both paths) Also fixes unquoted variables in the OCI-layout oras login call. --- pipelines/types/on-demand.sh | 51 ++++++++++++++++++++++++++---------- 1 file changed, 37 insertions(+), 14 deletions(-) diff --git a/pipelines/types/on-demand.sh b/pipelines/types/on-demand.sh index 5faeea90..86f5a39d 100644 --- a/pipelines/types/on-demand.sh +++ b/pipelines/types/on-demand.sh @@ -18,6 +18,10 @@ retry() { done } +get_acr_domain_suffix() { + az cloud show --query "suffixes.acrLoginServerEndpoint" --output tsv +} + copyImageFromRegistry() { # shortcut mirroring if the source registry is the same as the target ACR REQUIRED_REGISTRY_VARS=("TARGET_ACR" "SOURCE_REGISTRY") @@ -27,7 +31,7 @@ copyImageFromRegistry() { exit 1 fi done - ACR_DOMAIN_SUFFIX="$(az cloud show --query "suffixes.acrLoginServerEndpoint" --output tsv)" + ACR_DOMAIN_SUFFIX="$(retry 5 get_acr_domain_suffix)" if [[ "${SOURCE_REGISTRY}" == "${TARGET_ACR}${ACR_DOMAIN_SUFFIX}" ]]; then echo "Source and target registry are the same. No mirroring needed." exit 0 @@ -68,10 +72,17 @@ copyImageFromRegistry() { if [[ "${IS_CI_REGISTRY}" == "true" ]]; then echo "Setting up registry authentication for CI source registry." - oc registry login --to "${AUTH_JSON}" + oc_registry_login() { + oc registry login --to "${AUTH_JSON}" + } + retry 5 oc_registry_login else echo "Fetch pull secret for source registry ${SOURCE_REGISTRY} from ${PULL_SECRET_KV} KV." - az keyvault secret download --vault-name "${PULL_SECRET_KV}" --name "${PULL_SECRET}" -e base64 --file "${AUTH_JSON}" + fetch_pull_secret() { + az keyvault secret download --vault-name "${PULL_SECRET_KV}" \ + --name "${PULL_SECRET}" -e base64 --file "${AUTH_JSON}" + } + retry 5 fetch_pull_secret fi fi @@ -87,10 +98,13 @@ copyImageFromRegistry() { } retry 5 acr_login_target TARGET_ACR_LOGIN_SERVER="$(jq --raw-output .loginServer <<<"${RESPONSE}" )" - oras login --registry-config "${AUTH_JSON}" \ - --username 00000000-0000-0000-0000-000000000000 \ - --password-stdin \ - "${TARGET_ACR_LOGIN_SERVER}" <<<"$( jq --raw-output .accessToken <<<"${RESPONSE}" )" + oras_login_target() { + oras login --registry-config "${AUTH_JSON}" \ + --username 00000000-0000-0000-0000-000000000000 \ + --password-stdin \ + "${TARGET_ACR_LOGIN_SERVER}" <<<"$( jq --raw-output .accessToken <<<"${RESPONSE}" )" + } + retry 5 oras_login_target # at this point we have an auth config that can read from the source registry and # write to the target registry. @@ -111,7 +125,10 @@ copyImageFromRegistry() { TARGET_IMAGE="${TARGET_ACR_LOGIN_SERVER}/${REPOSITORY}:${DIGEST_NO_PREFIX}" echo "Mirroring image ${SRC_IMAGE} to ${TARGET_IMAGE}." echo "The image will still be available under it's original digest ${DIGEST} in the target registry." - oras cp "${SRC_IMAGE}" "${TARGET_IMAGE}" --from-registry-config "${AUTH_JSON}" --to-registry-config "${AUTH_JSON}" + oras_copy_registry() { + oras cp "${SRC_IMAGE}" "${TARGET_IMAGE}" --from-registry-config "${AUTH_JSON}" --to-registry-config "${AUTH_JSON}" + } + retry 5 oras_copy_registry } copyImageFromOciLayout() { @@ -151,9 +168,9 @@ copyImageFromOciLayout() { exit 1 fi - echo "✅ build_tag is: $BUILD_TAG" + echo "✅ build_tag is: $BUILD_TAG" - ACR_DOMAIN_SUFFIX="$(az cloud show --query "suffixes.acrLoginServerEndpoint" --output tsv)" + ACR_DOMAIN_SUFFIX="$(retry 5 get_acr_domain_suffix)" TARGET_ACR_LOGIN_SERVER="${TARGET_ACR}${ACR_DOMAIN_SUFFIX}" echo "Getting the ACR access token." @@ -167,8 +184,11 @@ copyImageFromOciLayout() { retry 5 acr_login_oci echo "Logging in with ORAS." - oras login $TARGET_ACR_LOGIN_SERVER --username $USERNAME --password-stdin <<< $PASSWORD - + oras_login_oci() { + oras login "$TARGET_ACR_LOGIN_SERVER" --username "$USERNAME" --password-stdin <<< "$PASSWORD" + } + retry 5 oras_login_oci + # Check for DRY_RUN if [ "${DRY_RUN:-false}" == "true" ]; then echo "DRY_RUN is enabled. Exiting without making changes." @@ -177,11 +197,14 @@ copyImageFromOciLayout() { # copy image from OCI layout to ACR TARGET_IMAGE="${TARGET_ACR_LOGIN_SERVER}/${REPOSITORY}:${BUILD_TAG}" - oras cp --from-oci-layout "${IMAGE_TAR_FILE}:${BUILD_TAG}" "${TARGET_IMAGE}" + oras_copy_oci() { + oras cp --from-oci-layout "${IMAGE_TAR_FILE}:${BUILD_TAG}" "${TARGET_IMAGE}" + } + retry 5 oras_copy_oci } if [[ -z "${IMAGE_TAR_FILE_NAME:-}" ]]; then copyImageFromRegistry else copyImageFromOciLayout -fi \ No newline at end of file +fi From 4e588281d534e44011d1531d3b3431afb5c04880 Mon Sep 17 00:00:00 2001 From: trevorwilliams2025 Date: Wed, 15 Jul 2026 12:41:13 +1000 Subject: [PATCH 2/2] fix(pipelines): harden retry wrappers per code review feedback --- pipelines/types/on-demand.sh | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/pipelines/types/on-demand.sh b/pipelines/types/on-demand.sh index 86f5a39d..9e383a26 100644 --- a/pipelines/types/on-demand.sh +++ b/pipelines/types/on-demand.sh @@ -19,7 +19,11 @@ retry() { } get_acr_domain_suffix() { - az cloud show --query "suffixes.acrLoginServerEndpoint" --output tsv + local suffix + if ! suffix="$(az cloud show --query "suffixes.acrLoginServerEndpoint" --output tsv)"; then + return 1 + fi + printf '%s' "${suffix}" } copyImageFromRegistry() { @@ -98,11 +102,12 @@ copyImageFromRegistry() { } retry 5 acr_login_target TARGET_ACR_LOGIN_SERVER="$(jq --raw-output .loginServer <<<"${RESPONSE}" )" + ACCESS_TOKEN="$(jq --raw-output .accessToken <<<"${RESPONSE}" )" oras_login_target() { oras login --registry-config "${AUTH_JSON}" \ --username 00000000-0000-0000-0000-000000000000 \ --password-stdin \ - "${TARGET_ACR_LOGIN_SERVER}" <<<"$( jq --raw-output .accessToken <<<"${RESPONSE}" )" + "${TARGET_ACR_LOGIN_SERVER}" <<<"${ACCESS_TOKEN}" } retry 5 oras_login_target