diff --git a/runners/launch_gb300-nv.sh b/runners/launch_gb300-nv.sh index 6038c46819..3e0056d900 100644 --- a/runners/launch_gb300-nv.sh +++ b/runners/launch_gb300-nv.sh @@ -180,12 +180,14 @@ POWER_SRT_SLURM_PIN="6fc1bed01a0b82dae0088a105c03ce0cfb353443" if [[ "$USES_DCGM_POWER" == "1" ]]; then DCGM_EXPORTER_IMAGE="nvcr.io/nvidia/k8s/dcgm-exporter:4.6.0-4.8.3-distroless" + # enroot resolves bare paths against Docker Hub; nvcr.io pulls need the registry# form + DCGM_EXPORTER_ENROOT_REF="${DCGM_EXPORTER_IMAGE/nvcr.io\//nvcr.io#}" DCGM_EXPORTER_SQSH="/data/home/sa-shared/gharunners/squash/$(echo "$DCGM_EXPORTER_IMAGE" | sed 's/[\/:@#]/_/g').sqsh" # Note (wenyao): import_squash treats an existing unsquashfs-valid file # as a cache hit but does not re-validate a fresh import, so check # explicitly — on a compute node, like the import itself (login node is # x86, nodes aarch64). - import_squash "$DCGM_EXPORTER_SQSH" "$DCGM_EXPORTER_IMAGE" + import_squash "$DCGM_EXPORTER_SQSH" "$DCGM_EXPORTER_ENROOT_REF" test -r "$DCGM_EXPORTER_SQSH" || { echo "Error: DCGM exporter squash not readable: $DCGM_EXPORTER_SQSH" >&2; exit 1; } srun --account="$SLURM_ACCOUNT" --partition="$SLURM_PARTITION" --exclusive --time=30 bash -c "unsquashfs -l \"$DCGM_EXPORTER_SQSH\" > /dev/null" || { echo "Error: DCGM exporter squash invalid: $DCGM_EXPORTER_SQSH" >&2; exit 1; } sha256sum "$DCGM_EXPORTER_SQSH" > "$GITHUB_WORKSPACE/exporter-image.sha256" diff --git a/runners/launch_h200-dgxc-slurm.sh b/runners/launch_h200-dgxc-slurm.sh index da510b8177..6a91509ff9 100755 --- a/runners/launch_h200-dgxc-slurm.sh +++ b/runners/launch_h200-dgxc-slurm.sh @@ -207,6 +207,8 @@ if [[ "$IS_MULTINODE" == "true" ]]; then if [[ "$USES_DCGM_POWER" == "1" ]]; then DCGM_EXPORTER_IMAGE="nvcr.io/nvidia/k8s/dcgm-exporter:4.6.0-4.8.3-distroless" + # enroot resolves bare paths against Docker Hub; nvcr.io pulls need the registry# form + DCGM_EXPORTER_ENROOT_REF="${DCGM_EXPORTER_IMAGE/nvcr.io\//nvcr.io#}" DCGM_EXPORTER_SQSH="/data/gharunners/containers/$(echo "$DCGM_EXPORTER_IMAGE" | sed 's/[\/:@#]/_/g').sqsh" if ! unsquashfs -l "$DCGM_EXPORTER_SQSH" >/dev/null 2>&1; then DCGM_EXPORTER_LOCK="${DCGM_EXPORTER_SQSH}.lock" @@ -223,7 +225,7 @@ if [[ "$IS_MULTINODE" == "true" ]]; then rm -f \"$DCGM_EXPORTER_SQSH\" export ENROOT_CACHE_PATH=\${HOME}/.cache/enroot mkdir -p \"\$ENROOT_CACHE_PATH\" - enroot import -o \"$DCGM_EXPORTER_SQSH\" docker://$DCGM_EXPORTER_IMAGE + enroot import -o \"$DCGM_EXPORTER_SQSH\" \"docker://$DCGM_EXPORTER_ENROOT_REF\" " fi test -r "$DCGM_EXPORTER_SQSH" || { echo "Error: DCGM exporter squash is not readable: $DCGM_EXPORTER_SQSH" >&2; exit 1; } diff --git a/utils/test_gb200_power_official_contract.py b/utils/test_gb200_power_official_contract.py index 32bfddc4ed..4b43eeb50b 100644 --- a/utils/test_gb200_power_official_contract.py +++ b/utils/test_gb200_power_official_contract.py @@ -87,7 +87,6 @@ def assert_pinned_clone_contract(launcher): def assert_exporter_provisioning(launcher): assert f'DCGM_EXPORTER_IMAGE="{EXPORTER_IMAGE}"' in launcher - assert 'import_squash "$DCGM_EXPORTER_SQSH" "$DCGM_EXPORTER_IMAGE"' in launcher assert 'test -r "$DCGM_EXPORTER_SQSH"' in launcher assert ( 'sha256sum "$DCGM_EXPORTER_SQSH" > "$GITHUB_WORKSPACE/exporter-image.sha256"' @@ -130,6 +129,11 @@ def test_launcher_detects_power_lane_from_recipe(): def test_launcher_provisions_exporter_through_squash_dir_cache(): launcher = GB200_LAUNCHER.read_text() assert_exporter_provisioning(launcher) + # import_squash routes through enroot_uri_for_image, which already emits + # the registry# form for nvcr.io refs — pass the plain image, not a + # pre-converted ref, or the helper double-parses it. + assert 'import_squash "$DCGM_EXPORTER_SQSH" "$DCGM_EXPORTER_IMAGE"' in launcher + assert 'enroot_uri=$(enroot_uri_for_image "$image")' in launcher assert 'DCGM_EXPORTER_SQSH="${SQUASH_DIR}/' in launcher assert 'unsquashfs -l "$DCGM_EXPORTER_SQSH"' in launcher diff --git a/utils/test_gb300_power_official_contract.py b/utils/test_gb300_power_official_contract.py index a645afca94..25bfff145a 100644 --- a/utils/test_gb300_power_official_contract.py +++ b/utils/test_gb300_power_official_contract.py @@ -187,6 +187,14 @@ def test_launcher_detects_power_lane_from_recipe(): def test_launcher_provisions_exporter_through_shared_squash_path(): launcher = LAUNCHER_PATH.read_text() assert_exporter_provisioning(launcher) + # This import_squash passes docker://$image straight to enroot, which + # resolves bare paths against Docker Hub — the nvcr.io pull needs the + # pre-converted registry# ref. + assert ( + 'DCGM_EXPORTER_ENROOT_REF="${DCGM_EXPORTER_IMAGE/nvcr.io\\//nvcr.io#}"' + in launcher + ) + assert 'import_squash "$DCGM_EXPORTER_SQSH" "$DCGM_EXPORTER_ENROOT_REF"' in launcher # No SQUASH_DIR var here; the /data/ mount avoids the /home NFS ELOOP bug. assert 'DCGM_EXPORTER_SQSH="/data/home/sa-shared/gharunners/squash/' in launcher assert 'srun --account="$SLURM_ACCOUNT" --partition="$SLURM_PARTITION" --exclusive --time=180' in launcher