fix(pipelines): add retry wrappers to network calls in image mirror script (AROSLSRE-1490)

pipelines/types/on-demand.sh

@@ -18,6 +18,14 @@ retry() {
     done
 }
 
+get_acr_domain_suffix() {
+    local suffix
+    if ! suffix="$(az cloud show --query "suffixes.acrLoginServerEndpoint" --output tsv)"; then
+        return 1
+    fi
+    printf '%s' "${suffix}"
+}
+
 copyImageFromRegistry() {
     # shortcut mirroring if the source registry is the same as the target ACR
     REQUIRED_REGISTRY_VARS=("TARGET_ACR" "SOURCE_REGISTRY")
@@ -27,7 +35,7 @@ copyImageFromRegistry() {
             exit 1
         fi
     done
-    ACR_DOMAIN_SUFFIX="$(az cloud show --query "suffixes.acrLoginServerEndpoint" --output tsv)"
+    ACR_DOMAIN_SUFFIX="$(retry 5 get_acr_domain_suffix)"
     if [[ "${SOURCE_REGISTRY}" == "${TARGET_ACR}${ACR_DOMAIN_SUFFIX}" ]]; then
         echo "Source and target registry are the same. No mirroring needed."
         exit 0
@@ -68,10 +76,17 @@ copyImageFromRegistry() {
 
         if [[ "${IS_CI_REGISTRY}" == "true" ]]; then
             echo "Setting up registry authentication for CI source registry."
-            oc registry login --to "${AUTH_JSON}"
+            oc_registry_login() {
+              oc registry login --to "${AUTH_JSON}"
+            }
+            retry 5 oc_registry_login
         else
             echo "Fetch pull secret for source registry ${SOURCE_REGISTRY} from ${PULL_SECRET_KV} KV."
-            az keyvault secret download --vault-name "${PULL_SECRET_KV}" --name "${PULL_SECRET}" -e base64 --file "${AUTH_JSON}"
+            fetch_pull_secret() {
+              az keyvault secret download --vault-name "${PULL_SECRET_KV}" \
+                  --name "${PULL_SECRET}" -e base64 --file "${AUTH_JSON}"
+            }
+            retry 5 fetch_pull_secret
         fi
     fi
 
@@ -87,10 +102,14 @@ copyImageFromRegistry() {
     }
     retry 5 acr_login_target
     TARGET_ACR_LOGIN_SERVER="$(jq --raw-output .loginServer <<<"${RESPONSE}" )"
-    oras login --registry-config "${AUTH_JSON}" \
-               --username 00000000-0000-0000-0000-000000000000 \
-               --password-stdin \
-               "${TARGET_ACR_LOGIN_SERVER}" <<<"$( jq --raw-output .accessToken <<<"${RESPONSE}" )"
+    ACCESS_TOKEN="$(jq --raw-output .accessToken <<<"${RESPONSE}" )"
+    oras_login_target() {
+      oras login --registry-config "${AUTH_JSON}" \
+                 --username 00000000-0000-0000-0000-000000000000 \
+                 --password-stdin \
+                 "${TARGET_ACR_LOGIN_SERVER}" <<<"${ACCESS_TOKEN}"
+    }
+    retry 5 oras_login_target
 
     # at this point we have an auth config that can read from the source registry and
     # write to the target registry.
@@ -111,7 +130,10 @@ copyImageFromRegistry() {
     TARGET_IMAGE="${TARGET_ACR_LOGIN_SERVER}/${REPOSITORY}:${DIGEST_NO_PREFIX}"
     echo "Mirroring image ${SRC_IMAGE} to ${TARGET_IMAGE}."
     echo "The image will still be available under it's original digest ${DIGEST} in the target registry."
-    oras cp "${SRC_IMAGE}" "${TARGET_IMAGE}" --from-registry-config "${AUTH_JSON}" --to-registry-config "${AUTH_JSON}"
+    oras_copy_registry() {
+      oras cp "${SRC_IMAGE}" "${TARGET_IMAGE}" --from-registry-config "${AUTH_JSON}" --to-registry-config "${AUTH_JSON}"
+    }
+    retry 5 oras_copy_registry
 }
 
 copyImageFromOciLayout() {
@@ -151,9 +173,9 @@ copyImageFromOciLayout() {
     exit 1
     fi
 
-    echo "✅ build_tag is: $BUILD_TAG"    
+    echo "✅ build_tag is: $BUILD_TAG"
 
-    ACR_DOMAIN_SUFFIX="$(az cloud show --query "suffixes.acrLoginServerEndpoint" --output tsv)"
+    ACR_DOMAIN_SUFFIX="$(retry 5 get_acr_domain_suffix)"
     TARGET_ACR_LOGIN_SERVER="${TARGET_ACR}${ACR_DOMAIN_SUFFIX}"
 
     echo "Getting the ACR access token."
@@ -167,8 +189,11 @@ copyImageFromOciLayout() {
     retry 5 acr_login_oci
 
     echo "Logging in with ORAS."
-    oras login $TARGET_ACR_LOGIN_SERVER --username $USERNAME  --password-stdin <<< $PASSWORD
-
+    oras_login_oci() {
+      oras login "$TARGET_ACR_LOGIN_SERVER" --username "$USERNAME" --password-stdin <<< "$PASSWORD"
+    }
+    retry 5 oras_login_oci
+
     # Check for DRY_RUN
     if [ "${DRY_RUN:-false}" == "true" ]; then
         echo "DRY_RUN is enabled. Exiting without making changes."
@@ -177,11 +202,14 @@ copyImageFromOciLayout() {
 
     # copy image from OCI layout to ACR
     TARGET_IMAGE="${TARGET_ACR_LOGIN_SERVER}/${REPOSITORY}:${BUILD_TAG}"
-    oras cp --from-oci-layout "${IMAGE_TAR_FILE}:${BUILD_TAG}" "${TARGET_IMAGE}"
+    oras_copy_oci() {
+      oras cp --from-oci-layout "${IMAGE_TAR_FILE}:${BUILD_TAG}" "${TARGET_IMAGE}"
+    }
+    retry 5 oras_copy_oci
 }
 
 if [[ -z "${IMAGE_TAR_FILE_NAME:-}" ]]; then
     copyImageFromRegistry
 else
     copyImageFromOciLayout
-fi
+fi