From dbea7ea4a0daee08d0d3b0882af4b5a1626d1748 Mon Sep 17 00:00:00 2001 From: Wenyao Gao Date: Tue, 25 Aug 2026 21:15:49 -0700 Subject: [PATCH 1/2] fix(ci): import the DCGM exporter via enroot registry syntax --- runners/launch_gb200-nv.sh | 4 +++- runners/launch_gb300-nv.sh | 4 +++- runners/launch_h200-dgxc-slurm.sh | 4 +++- utils/test_gb200_power_official_contract.py | 6 +++++- 4 files changed, 14 insertions(+), 4 deletions(-) diff --git a/runners/launch_gb200-nv.sh b/runners/launch_gb200-nv.sh index 4d55473ce6..8af9ba8cea 100755 --- a/runners/launch_gb200-nv.sh +++ b/runners/launch_gb200-nv.sh @@ -313,8 +313,10 @@ fi if [[ "$USES_DCGM_POWER" == "1" ]]; then DCGM_EXPORTER_IMAGE="nvcr.io/nvidia/k8s/dcgm-exporter:4.6.0-4.8.3-distroless" + # enroot resolves bare paths against Docker Hub; nvcr.io pulls need the registry# form + DCGM_EXPORTER_ENROOT_REF="${DCGM_EXPORTER_IMAGE/nvcr.io\//nvcr.io#}" DCGM_EXPORTER_SQSH="${SQUASH_DIR}/$(echo "$DCGM_EXPORTER_IMAGE" | sed 's/[\/:@#]/_/g').sqsh" - import_squash "$DCGM_EXPORTER_SQSH" "$DCGM_EXPORTER_IMAGE" + import_squash "$DCGM_EXPORTER_SQSH" "$DCGM_EXPORTER_ENROOT_REF" test -r "$DCGM_EXPORTER_SQSH" || { echo "Error: DCGM exporter squash not readable: $DCGM_EXPORTER_SQSH" >&2; exit 1; } unsquashfs -l "$DCGM_EXPORTER_SQSH" > /dev/null || { echo "Error: DCGM exporter squash invalid: $DCGM_EXPORTER_SQSH" >&2; exit 1; } sha256sum "$DCGM_EXPORTER_SQSH" > "$GITHUB_WORKSPACE/exporter-image.sha256" diff --git a/runners/launch_gb300-nv.sh b/runners/launch_gb300-nv.sh index 6038c46819..3e0056d900 100644 --- a/runners/launch_gb300-nv.sh +++ b/runners/launch_gb300-nv.sh @@ -180,12 +180,14 @@ POWER_SRT_SLURM_PIN="6fc1bed01a0b82dae0088a105c03ce0cfb353443" if [[ "$USES_DCGM_POWER" == "1" ]]; then DCGM_EXPORTER_IMAGE="nvcr.io/nvidia/k8s/dcgm-exporter:4.6.0-4.8.3-distroless" + # enroot resolves bare paths against Docker Hub; nvcr.io pulls need the registry# form + DCGM_EXPORTER_ENROOT_REF="${DCGM_EXPORTER_IMAGE/nvcr.io\//nvcr.io#}" DCGM_EXPORTER_SQSH="/data/home/sa-shared/gharunners/squash/$(echo "$DCGM_EXPORTER_IMAGE" | sed 's/[\/:@#]/_/g').sqsh" # Note (wenyao): import_squash treats an existing unsquashfs-valid file # as a cache hit but does not re-validate a fresh import, so check # explicitly — on a compute node, like the import itself (login node is # x86, nodes aarch64). - import_squash "$DCGM_EXPORTER_SQSH" "$DCGM_EXPORTER_IMAGE" + import_squash "$DCGM_EXPORTER_SQSH" "$DCGM_EXPORTER_ENROOT_REF" test -r "$DCGM_EXPORTER_SQSH" || { echo "Error: DCGM exporter squash not readable: $DCGM_EXPORTER_SQSH" >&2; exit 1; } srun --account="$SLURM_ACCOUNT" --partition="$SLURM_PARTITION" --exclusive --time=30 bash -c "unsquashfs -l \"$DCGM_EXPORTER_SQSH\" > /dev/null" || { echo "Error: DCGM exporter squash invalid: $DCGM_EXPORTER_SQSH" >&2; exit 1; } sha256sum "$DCGM_EXPORTER_SQSH" > "$GITHUB_WORKSPACE/exporter-image.sha256" diff --git a/runners/launch_h200-dgxc-slurm.sh b/runners/launch_h200-dgxc-slurm.sh index da510b8177..6a91509ff9 100755 --- a/runners/launch_h200-dgxc-slurm.sh +++ b/runners/launch_h200-dgxc-slurm.sh @@ -207,6 +207,8 @@ if [[ "$IS_MULTINODE" == "true" ]]; then if [[ "$USES_DCGM_POWER" == "1" ]]; then DCGM_EXPORTER_IMAGE="nvcr.io/nvidia/k8s/dcgm-exporter:4.6.0-4.8.3-distroless" + # enroot resolves bare paths against Docker Hub; nvcr.io pulls need the registry# form + DCGM_EXPORTER_ENROOT_REF="${DCGM_EXPORTER_IMAGE/nvcr.io\//nvcr.io#}" DCGM_EXPORTER_SQSH="/data/gharunners/containers/$(echo "$DCGM_EXPORTER_IMAGE" | sed 's/[\/:@#]/_/g').sqsh" if ! unsquashfs -l "$DCGM_EXPORTER_SQSH" >/dev/null 2>&1; then DCGM_EXPORTER_LOCK="${DCGM_EXPORTER_SQSH}.lock" @@ -223,7 +225,7 @@ if [[ "$IS_MULTINODE" == "true" ]]; then rm -f \"$DCGM_EXPORTER_SQSH\" export ENROOT_CACHE_PATH=\${HOME}/.cache/enroot mkdir -p \"\$ENROOT_CACHE_PATH\" - enroot import -o \"$DCGM_EXPORTER_SQSH\" docker://$DCGM_EXPORTER_IMAGE + enroot import -o \"$DCGM_EXPORTER_SQSH\" \"docker://$DCGM_EXPORTER_ENROOT_REF\" " fi test -r "$DCGM_EXPORTER_SQSH" || { echo "Error: DCGM exporter squash is not readable: $DCGM_EXPORTER_SQSH" >&2; exit 1; } diff --git a/utils/test_gb200_power_official_contract.py b/utils/test_gb200_power_official_contract.py index 32bfddc4ed..08b4624579 100644 --- a/utils/test_gb200_power_official_contract.py +++ b/utils/test_gb200_power_official_contract.py @@ -87,7 +87,11 @@ def assert_pinned_clone_contract(launcher): def assert_exporter_provisioning(launcher): assert f'DCGM_EXPORTER_IMAGE="{EXPORTER_IMAGE}"' in launcher - assert 'import_squash "$DCGM_EXPORTER_SQSH" "$DCGM_EXPORTER_IMAGE"' in launcher + assert ( + 'DCGM_EXPORTER_ENROOT_REF="${DCGM_EXPORTER_IMAGE/nvcr.io\\//nvcr.io#}"' + in launcher + ) + assert 'import_squash "$DCGM_EXPORTER_SQSH" "$DCGM_EXPORTER_ENROOT_REF"' in launcher assert 'test -r "$DCGM_EXPORTER_SQSH"' in launcher assert ( 'sha256sum "$DCGM_EXPORTER_SQSH" > "$GITHUB_WORKSPACE/exporter-image.sha256"' From 800c8747263b1fd8eced2492a2eaadd92a226c38 Mon Sep 17 00:00:00 2001 From: Wenyao Gao Date: Tue, 25 Aug 2026 21:20:58 -0700 Subject: [PATCH 2/2] fix(ci): keep the plain exporter ref where enroot_uri_for_image converts --- runners/launch_gb200-nv.sh | 4 +--- utils/test_gb200_power_official_contract.py | 10 +++++----- utils/test_gb300_power_official_contract.py | 8 ++++++++ 3 files changed, 14 insertions(+), 8 deletions(-) diff --git a/runners/launch_gb200-nv.sh b/runners/launch_gb200-nv.sh index 8af9ba8cea..4d55473ce6 100755 --- a/runners/launch_gb200-nv.sh +++ b/runners/launch_gb200-nv.sh @@ -313,10 +313,8 @@ fi if [[ "$USES_DCGM_POWER" == "1" ]]; then DCGM_EXPORTER_IMAGE="nvcr.io/nvidia/k8s/dcgm-exporter:4.6.0-4.8.3-distroless" - # enroot resolves bare paths against Docker Hub; nvcr.io pulls need the registry# form - DCGM_EXPORTER_ENROOT_REF="${DCGM_EXPORTER_IMAGE/nvcr.io\//nvcr.io#}" DCGM_EXPORTER_SQSH="${SQUASH_DIR}/$(echo "$DCGM_EXPORTER_IMAGE" | sed 's/[\/:@#]/_/g').sqsh" - import_squash "$DCGM_EXPORTER_SQSH" "$DCGM_EXPORTER_ENROOT_REF" + import_squash "$DCGM_EXPORTER_SQSH" "$DCGM_EXPORTER_IMAGE" test -r "$DCGM_EXPORTER_SQSH" || { echo "Error: DCGM exporter squash not readable: $DCGM_EXPORTER_SQSH" >&2; exit 1; } unsquashfs -l "$DCGM_EXPORTER_SQSH" > /dev/null || { echo "Error: DCGM exporter squash invalid: $DCGM_EXPORTER_SQSH" >&2; exit 1; } sha256sum "$DCGM_EXPORTER_SQSH" > "$GITHUB_WORKSPACE/exporter-image.sha256" diff --git a/utils/test_gb200_power_official_contract.py b/utils/test_gb200_power_official_contract.py index 08b4624579..4b43eeb50b 100644 --- a/utils/test_gb200_power_official_contract.py +++ b/utils/test_gb200_power_official_contract.py @@ -87,11 +87,6 @@ def assert_pinned_clone_contract(launcher): def assert_exporter_provisioning(launcher): assert f'DCGM_EXPORTER_IMAGE="{EXPORTER_IMAGE}"' in launcher - assert ( - 'DCGM_EXPORTER_ENROOT_REF="${DCGM_EXPORTER_IMAGE/nvcr.io\\//nvcr.io#}"' - in launcher - ) - assert 'import_squash "$DCGM_EXPORTER_SQSH" "$DCGM_EXPORTER_ENROOT_REF"' in launcher assert 'test -r "$DCGM_EXPORTER_SQSH"' in launcher assert ( 'sha256sum "$DCGM_EXPORTER_SQSH" > "$GITHUB_WORKSPACE/exporter-image.sha256"' @@ -134,6 +129,11 @@ def test_launcher_detects_power_lane_from_recipe(): def test_launcher_provisions_exporter_through_squash_dir_cache(): launcher = GB200_LAUNCHER.read_text() assert_exporter_provisioning(launcher) + # import_squash routes through enroot_uri_for_image, which already emits + # the registry# form for nvcr.io refs — pass the plain image, not a + # pre-converted ref, or the helper double-parses it. + assert 'import_squash "$DCGM_EXPORTER_SQSH" "$DCGM_EXPORTER_IMAGE"' in launcher + assert 'enroot_uri=$(enroot_uri_for_image "$image")' in launcher assert 'DCGM_EXPORTER_SQSH="${SQUASH_DIR}/' in launcher assert 'unsquashfs -l "$DCGM_EXPORTER_SQSH"' in launcher diff --git a/utils/test_gb300_power_official_contract.py b/utils/test_gb300_power_official_contract.py index a645afca94..25bfff145a 100644 --- a/utils/test_gb300_power_official_contract.py +++ b/utils/test_gb300_power_official_contract.py @@ -187,6 +187,14 @@ def test_launcher_detects_power_lane_from_recipe(): def test_launcher_provisions_exporter_through_shared_squash_path(): launcher = LAUNCHER_PATH.read_text() assert_exporter_provisioning(launcher) + # This import_squash passes docker://$image straight to enroot, which + # resolves bare paths against Docker Hub — the nvcr.io pull needs the + # pre-converted registry# ref. + assert ( + 'DCGM_EXPORTER_ENROOT_REF="${DCGM_EXPORTER_IMAGE/nvcr.io\\//nvcr.io#}"' + in launcher + ) + assert 'import_squash "$DCGM_EXPORTER_SQSH" "$DCGM_EXPORTER_ENROOT_REF"' in launcher # No SQUASH_DIR var here; the /data/ mount avoids the /home NFS ELOOP bug. assert 'DCGM_EXPORTER_SQSH="/data/home/sa-shared/gharunners/squash/' in launcher assert 'srun --account="$SLURM_ACCOUNT" --partition="$SLURM_PARTITION" --exclusive --time=180' in launcher