From e460a8c21576cd562295c0828bcf4cd658957c41 Mon Sep 17 00:00:00 2001 From: Stoo Davies <189670495+stoo-davies@users.noreply.github.com> Date: Mon, 20 Jul 2026 16:06:32 -0700 Subject: [PATCH 1/3] Fix: 3024, down any non-mlx interfaces Signed-off-by: Stoo Davies <189670495+stoo-davies@users.noreply.github.com> --- pxe/Makefile.toml | 4 +- pxe/common_files/forge-scout-network.sh | 62 +++++++++++++++++++ .../scout-oss-aarch64/mkosi.conf | 1 + .../mkosi.extra/opt/forge/forge-scout-pre.sh | 2 + .../scout-oss-aarch64/mkosi.postinst.chroot | 1 + .../scout-oss-x86_64/mkosi.conf | 1 + .../mkosi.extra/opt/forge/forge-scout-pre.sh | 4 +- .../scout-oss-x86_64/mkosi.postinst.chroot | 1 + 8 files changed, 74 insertions(+), 2 deletions(-) create mode 100755 pxe/common_files/forge-scout-network.sh diff --git a/pxe/Makefile.toml b/pxe/Makefile.toml index a81e03e594..8ad1f10dc1 100644 --- a/pxe/Makefile.toml +++ b/pxe/Makefile.toml @@ -130,13 +130,15 @@ script = ''' ''' [tasks.stage-scout-module-loader] -description = "Copies the scout module loader to build-output" +description = "Copies the Scout helper scripts to build-output" script = ''' mkdir -p ${REPO_ROOT}/pxe/mkosi.profiles/scout-oss-aarch64/mkosi.extra/build-output/ cp ${REPO_ROOT}/pxe/common_files/load_modules.sh ${REPO_ROOT}/pxe/mkosi.profiles/scout-oss-aarch64/mkosi.extra/build-output/ + cp ${REPO_ROOT}/pxe/common_files/forge-scout-network.sh ${REPO_ROOT}/pxe/mkosi.profiles/scout-oss-aarch64/mkosi.extra/build-output/ mkdir -p ${REPO_ROOT}/pxe/mkosi.profiles/scout-oss-x86_64/mkosi.extra/build-output/ cp ${REPO_ROOT}/pxe/common_files/load_modules.sh ${REPO_ROOT}/pxe/mkosi.profiles/scout-oss-x86_64/mkosi.extra/build-output/ + cp ${REPO_ROOT}/pxe/common_files/forge-scout-network.sh ${REPO_ROOT}/pxe/mkosi.profiles/scout-oss-x86_64/mkosi.extra/build-output/ ''' [tasks.build-boot-artifacts-x86-host] diff --git a/pxe/common_files/forge-scout-network.sh b/pxe/common_files/forge-scout-network.sh new file mode 100755 index 0000000000..241acb80d5 --- /dev/null +++ b/pxe/common_files/forge-scout-network.sh @@ -0,0 +1,62 @@ +#!/usr/bin/env sh +# +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +cmdline_file=${SCOUT_CMDLINE_FILE:-/proc/cmdline} +sys_class_net=${SCOUT_SYS_CLASS_NET:-/sys/class/net} +ip_command=${SCOUT_IP_COMMAND:-ip} + +# A bare anynic kernel option is an escape hatch for debugging unsupported NICs. +cmdline=$(cat "$cmdline_file") || exit 1 +case " $cmdline " in + *" anynic "*) + echo "Scout NIC filtering disabled: kernel_option=anynic" + exit 0 + ;; +esac + +failed=false +for net_path in "$sys_class_net"/* +do + [ -e "$net_path" ] || continue + + interface=${net_path##*/} + if [ "$interface" = "lo" ]; then + echo "Keeping network interface: interface=$interface reason=loopback" + continue + fi + + vendor=unknown + # USB and virtual interfaces do not expose this PCI vendor file and must + # therefore fall through to the disable path. + if [ -r "$net_path/device/vendor" ]; then + vendor=$(cat "$net_path/device/vendor") + fi + + if [ "$vendor" = "0x15b3" ]; then + echo "Keeping network interface: interface=$interface vendor=$vendor" + continue + fi + + echo "Disabling network interface: interface=$interface vendor=$vendor" + if ! "$ip_command" link set dev "$interface" down; then + echo "Failed to disable network interface: interface=$interface" >&2 + failed=true + fi +done + +[ "$failed" = false ] diff --git a/pxe/mkosi.profiles/scout-oss-aarch64/mkosi.conf b/pxe/mkosi.profiles/scout-oss-aarch64/mkosi.conf index f098285280..14d491c246 100644 --- a/pxe/mkosi.profiles/scout-oss-aarch64/mkosi.conf +++ b/pxe/mkosi.profiles/scout-oss-aarch64/mkosi.conf @@ -38,6 +38,7 @@ Packages= ibverbs-utils iperf3 ipmitool + iproute2 iputils-ping keyboard-configuration isc-dhcp-client diff --git a/pxe/mkosi.profiles/scout-oss-aarch64/mkosi.extra/opt/forge/forge-scout-pre.sh b/pxe/mkosi.profiles/scout-oss-aarch64/mkosi.extra/opt/forge/forge-scout-pre.sh index 4324d2a7a0..944667d880 100755 --- a/pxe/mkosi.profiles/scout-oss-aarch64/mkosi.extra/opt/forge/forge-scout-pre.sh +++ b/pxe/mkosi.profiles/scout-oss-aarch64/mkosi.extra/opt/forge/forge-scout-pre.sh @@ -38,6 +38,8 @@ do cli_cmd=`echo $line|cut -d'=' -f2` fi done +/opt/forge/forge-scout-network.sh || echo "Scout NIC filtering failed; continuing startup" >&2 + rm "/opt/forge/forge-scout.env" cat "/opt/forge/forge-scout.env.template" > "/opt/forge/forge-scout.env" echo server_uri=$server_uri >> "/opt/forge/forge-scout.env" diff --git a/pxe/mkosi.profiles/scout-oss-aarch64/mkosi.postinst.chroot b/pxe/mkosi.profiles/scout-oss-aarch64/mkosi.postinst.chroot index 73c0995ce4..4888752e13 100755 --- a/pxe/mkosi.profiles/scout-oss-aarch64/mkosi.postinst.chroot +++ b/pxe/mkosi.profiles/scout-oss-aarch64/mkosi.postinst.chroot @@ -54,6 +54,7 @@ ln -s /etc/systemd/system/check-scout-updates.timer /etc/systemd/system/timers.t # MODULE LOADER # ============================================================================ cp /build-output/load_modules.sh /opt/forge/ +cp /build-output/forge-scout-network.sh /opt/forge/ # Ensure key commands are present for bin in /opt/forge/forge-scout \ diff --git a/pxe/mkosi.profiles/scout-oss-x86_64/mkosi.conf b/pxe/mkosi.profiles/scout-oss-x86_64/mkosi.conf index 66335f2da2..13965a3338 100644 --- a/pxe/mkosi.profiles/scout-oss-x86_64/mkosi.conf +++ b/pxe/mkosi.profiles/scout-oss-x86_64/mkosi.conf @@ -37,6 +37,7 @@ Packages= ibverbs-utils iperf3 ipmitool + iproute2 iputils-ping keyboard-configuration isc-dhcp-client diff --git a/pxe/mkosi.profiles/scout-oss-x86_64/mkosi.extra/opt/forge/forge-scout-pre.sh b/pxe/mkosi.profiles/scout-oss-x86_64/mkosi.extra/opt/forge/forge-scout-pre.sh index 07e50f5b8d..944667d880 100755 --- a/pxe/mkosi.profiles/scout-oss-x86_64/mkosi.extra/opt/forge/forge-scout-pre.sh +++ b/pxe/mkosi.profiles/scout-oss-x86_64/mkosi.extra/opt/forge/forge-scout-pre.sh @@ -38,6 +38,8 @@ do cli_cmd=`echo $line|cut -d'=' -f2` fi done +/opt/forge/forge-scout-network.sh || echo "Scout NIC filtering failed; continuing startup" >&2 + rm "/opt/forge/forge-scout.env" cat "/opt/forge/forge-scout.env.template" > "/opt/forge/forge-scout.env" echo server_uri=$server_uri >> "/opt/forge/forge-scout.env" @@ -64,4 +66,4 @@ modprobe mlx5_ib || true udevadm settle chown nvidia-dcgm:nvidia-dcgm /var/log/nvidia-dcgm /opt/forge/load_modules.sh || true -udevadm settle \ No newline at end of file +udevadm settle diff --git a/pxe/mkosi.profiles/scout-oss-x86_64/mkosi.postinst.chroot b/pxe/mkosi.profiles/scout-oss-x86_64/mkosi.postinst.chroot index 73c0995ce4..4888752e13 100755 --- a/pxe/mkosi.profiles/scout-oss-x86_64/mkosi.postinst.chroot +++ b/pxe/mkosi.profiles/scout-oss-x86_64/mkosi.postinst.chroot @@ -54,6 +54,7 @@ ln -s /etc/systemd/system/check-scout-updates.timer /etc/systemd/system/timers.t # MODULE LOADER # ============================================================================ cp /build-output/load_modules.sh /opt/forge/ +cp /build-output/forge-scout-network.sh /opt/forge/ # Ensure key commands are present for bin in /opt/forge/forge-scout \ From 9ebbba5bc3e8b73be85e77ac75e9e3866cddca26 Mon Sep 17 00:00:00 2001 From: Stoo Davies <189670495+stoo-davies@users.noreply.github.com> Date: Fri, 24 Jul 2026 21:10:12 +0300 Subject: [PATCH 2/3] Addressing rabbit feedback. Signed-off-by: Stoo Davies <189670495+stoo-davies@users.noreply.github.com> --- .../scout-oss-aarch64/mkosi.extra/opt/forge/forge-scout-pre.sh | 2 +- .../scout-oss-x86_64/mkosi.extra/opt/forge/forge-scout-pre.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pxe/mkosi.profiles/scout-oss-aarch64/mkosi.extra/opt/forge/forge-scout-pre.sh b/pxe/mkosi.profiles/scout-oss-aarch64/mkosi.extra/opt/forge/forge-scout-pre.sh index 944667d880..5e8cd17973 100755 --- a/pxe/mkosi.profiles/scout-oss-aarch64/mkosi.extra/opt/forge/forge-scout-pre.sh +++ b/pxe/mkosi.profiles/scout-oss-aarch64/mkosi.extra/opt/forge/forge-scout-pre.sh @@ -38,7 +38,6 @@ do cli_cmd=`echo $line|cut -d'=' -f2` fi done -/opt/forge/forge-scout-network.sh || echo "Scout NIC filtering failed; continuing startup" >&2 rm "/opt/forge/forge-scout.env" cat "/opt/forge/forge-scout.env.template" > "/opt/forge/forge-scout.env" @@ -67,3 +66,4 @@ udevadm settle chown nvidia-dcgm:nvidia-dcgm /var/log/nvidia-dcgm /opt/forge/load_modules.sh || true udevadm settle +/opt/forge/forge-scout-network.sh || echo "Scout NIC filtering failed; continuing startup" >&2 diff --git a/pxe/mkosi.profiles/scout-oss-x86_64/mkosi.extra/opt/forge/forge-scout-pre.sh b/pxe/mkosi.profiles/scout-oss-x86_64/mkosi.extra/opt/forge/forge-scout-pre.sh index 944667d880..5e8cd17973 100755 --- a/pxe/mkosi.profiles/scout-oss-x86_64/mkosi.extra/opt/forge/forge-scout-pre.sh +++ b/pxe/mkosi.profiles/scout-oss-x86_64/mkosi.extra/opt/forge/forge-scout-pre.sh @@ -38,7 +38,6 @@ do cli_cmd=`echo $line|cut -d'=' -f2` fi done -/opt/forge/forge-scout-network.sh || echo "Scout NIC filtering failed; continuing startup" >&2 rm "/opt/forge/forge-scout.env" cat "/opt/forge/forge-scout.env.template" > "/opt/forge/forge-scout.env" @@ -67,3 +66,4 @@ udevadm settle chown nvidia-dcgm:nvidia-dcgm /var/log/nvidia-dcgm /opt/forge/load_modules.sh || true udevadm settle +/opt/forge/forge-scout-network.sh || echo "Scout NIC filtering failed; continuing startup" >&2 From d165852c715a26253fd991469e90ddea72391240 Mon Sep 17 00:00:00 2001 From: Stoo Davies <189670495+stoo-davies@users.noreply.github.com> Date: Tue, 28 Jul 2026 14:34:38 -0700 Subject: [PATCH 3/3] Switched to using mac= to id nic Signed-off-by: Stoo Davies <189670495+stoo-davies@users.noreply.github.com> --- pxe/common_files/forge-scout-network.sh | 254 ++++++++++++++++++++++-- 1 file changed, 232 insertions(+), 22 deletions(-) diff --git a/pxe/common_files/forge-scout-network.sh b/pxe/common_files/forge-scout-network.sh index 241acb80d5..9a57cf80d8 100755 --- a/pxe/common_files/forge-scout-network.sh +++ b/pxe/common_files/forge-scout-network.sh @@ -15,48 +15,258 @@ # See the License for the specific language governing permissions and # limitations under the License. # +set -u +set -f cmdline_file=${SCOUT_CMDLINE_FILE:-/proc/cmdline} sys_class_net=${SCOUT_SYS_CLASS_NET:-/sys/class/net} ip_command=${SCOUT_IP_COMMAND:-ip} +networkctl_command=${SCOUT_NETWORKCTL_COMMAND:-networkctl} +networkd_runtime_dir=${SCOUT_NETWORKD_RUNTIME_DIR:-/run/systemd/network} +networkd_dhcp_file=${SCOUT_NETWORKD_DHCP_FILE:-/etc/systemd/network/dhcp.network} -# A bare anynic kernel option is an escape hatch for debugging unsupported NICs. -cmdline=$(cat "$cmdline_file") || exit 1 -case " $cmdline " in - *" anynic "*) - echo "Scout NIC filtering disabled: kernel_option=anynic" +if ! cmdline=$(cat "$cmdline_file"); then + echo "Skipping Scout network configuration: reason=cmdline_unreadable" >&2 + exit 0 +fi + +preferred_mac= +mac_parameter_count=0 +for parameter in $cmdline +do + case "$parameter" in + mac=*) + mac_parameter_count=$((mac_parameter_count + 1)) + preferred_mac=${parameter#mac=} + ;; + esac +done +set +f + +if [ "$mac_parameter_count" -ne 1 ]; then + echo "Skipping Scout network configuration: reason=mac_parameter_count count=$mac_parameter_count" >&2 + exit 0 +fi + +case "$preferred_mac" in + [0-9A-Fa-f][0-9A-Fa-f]:[0-9A-Fa-f][0-9A-Fa-f]:[0-9A-Fa-f][0-9A-Fa-f]:[0-9A-Fa-f][0-9A-Fa-f]:[0-9A-Fa-f][0-9A-Fa-f]:[0-9A-Fa-f][0-9A-Fa-f]) + ;; + *) + echo "Skipping Scout network configuration: reason=invalid_mac_parameter" >&2 exit 0 ;; esac +preferred_mac=$(printf '%s\n' "$preferred_mac" | tr '[:upper:]' '[:lower:]') -failed=false +preferred_interface= +preferred_interface_count=0 for net_path in "$sys_class_net"/* do [ -e "$net_path" ] || continue + [ -r "$net_path/address" ] || continue - interface=${net_path##*/} - if [ "$interface" = "lo" ]; then - echo "Keeping network interface: interface=$interface reason=loopback" - continue + interface_mac=$(cat "$net_path/address") || continue + interface_mac=$(printf '%s\n' "$interface_mac" | tr '[:upper:]' '[:lower:]') + if [ "$interface_mac" = "$preferred_mac" ]; then + preferred_interface=${net_path##*/} + preferred_interface_count=$((preferred_interface_count + 1)) fi +done + +if [ "$preferred_interface_count" -ne 1 ]; then + echo "Skipping Scout network configuration: reason=preferred_interface_count count=$preferred_interface_count" >&2 + exit 0 +fi - vendor=unknown - # USB and virtual interfaces do not expose this PCI vendor file and must - # therefore fall through to the disable path. - if [ -r "$net_path/device/vendor" ]; then - vendor=$(cat "$net_path/device/vendor") +carrier_file="$sys_class_net/$preferred_interface/carrier" +if [ ! -r "$carrier_file" ] || [ "$(cat "$carrier_file")" != 1 ]; then + echo "Skipping Scout network configuration: interface=$preferred_interface reason=no_carrier" >&2 + exit 0 +fi + +if global_addresses=$("$ip_command" -o address show dev "$preferred_interface" scope global); then + if [ -z "$global_addresses" ]; then + echo "Skipping Scout network configuration: interface=$preferred_interface reason=no_global_address" >&2 + exit 0 fi +else + echo "Skipping Scout network configuration: interface=$preferred_interface reason=address_inspection_failed" >&2 + exit 0 +fi + +runtime_network_file="$networkd_runtime_dir/00-forge-scout-nonpreferred.network" + +# Build the target set from the configuration networkd selected for each link. +# Name= also matches alternative interface names, so matching only the current +# interface name here could miss links that the wildcard configuration affects. +set -- +for net_path in "$sys_class_net"/* +do + [ -e "$net_path" ] || continue + + interface=${net_path##*/} + [ "$interface" = "$preferred_interface" ] && continue + + if ! network_status=$( + "$networkctl_command" --json=short status "$interface" 2>/dev/null + ); then + # A link can disappear while interfaces are being enumerated. + [ -e "$net_path" ] || continue - if [ "$vendor" = "0x15b3" ]; then - echo "Keeping network interface: interface=$interface vendor=$vendor" - continue + echo "Skipping Scout network configuration: interface=$interface reason=network_status_inspection_failed" >&2 + exit 0 fi - echo "Disabling network interface: interface=$interface vendor=$vendor" - if ! "$ip_command" link set dev "$interface" down; then - echo "Failed to disable network interface: interface=$interface" >&2 - failed=true + case "$network_status" in + *"\"NetworkFile\":\"$networkd_dhcp_file\""*|*"\"NetworkFile\":\"$runtime_network_file\""*) + set -- "$@" "$interface" + ;; + esac +done + +if ! mkdir -p "$networkd_runtime_dir"; then + echo "Failed to create networkd runtime directory: directory=$networkd_runtime_dir" >&2 + exit 1 +fi + +if ! temporary_network_file=$(mktemp "$networkd_runtime_dir/.forge-scout-network.XXXXXX"); then + echo "Failed to create temporary networkd configuration: directory=$networkd_runtime_dir" >&2 + exit 1 +fi + +if ! { + printf '%s\n' \ + '[Match]' \ + 'Name=enx* enp* enP*' \ + "Property=!INTERFACE=$preferred_interface" \ + '' \ + '[Network]' \ + 'DHCP=no' \ + 'IPv6AcceptRA=no' \ + 'ConfigureWithoutCarrier=yes' +} >"$temporary_network_file"; then + echo "Failed to write networkd configuration: path=$temporary_network_file" >&2 + rm -f "$temporary_network_file" + exit 1 +fi + +if ! chmod 0644 "$temporary_network_file" || + ! mv -f "$temporary_network_file" "$runtime_network_file"; then + echo "Failed to install networkd configuration: path=$runtime_network_file" >&2 + rm -f "$temporary_network_file" + exit 1 +fi + +if ! "$networkctl_command" reload; then + echo "Failed to reload networkd configuration" >&2 + exit 1 +fi + +echo "Selected preferred network interface: interface=$preferred_interface mac=$preferred_mac" +failed=false + +wait_attempts=31 +wait_complete=false +wait_failed=false +while [ "$#" -gt 0 ] && [ "$wait_attempts" -gt 0 ] +do + all_interfaces_complete=true + incomplete_interfaces= + for interface + do + # A removed interface is safe; if it reappears, networkd will apply the + # persistent wildcard configuration. + if [ ! -e "$sys_class_net/$interface" ]; then + continue + fi + + incomplete_reason= + incomplete_detail= + if ! network_status=$( + "$networkctl_command" --json=short status "$interface" 2>/dev/null + ); then + incomplete_reason=network_status_inspection_failed + else + case "$network_status" in + *"\"NetworkFile\":\"$runtime_network_file\""*) + ;; + *) + incomplete_reason=unexpected_network_file + incomplete_detail="network_status=$network_status" + ;; + esac + + if [ -z "$incomplete_reason" ]; then + case "$network_status" in + *'"AdministrativeState":"failed"'*) + echo "Network interface reconfiguration failed: interface=$interface network_status=$network_status" >&2 + failed=true + wait_failed=true + break + ;; + *'"AdministrativeState":"configured"'*) + if ! global_addresses=$( + "$ip_command" -o address show dev "$interface" scope global + ); then + incomplete_reason=address_inspection_failed + elif ! ipv4_routes=$( + "$ip_command" -o -4 route show dev "$interface" scope global + ); then + incomplete_reason=ipv4_route_inspection_failed + elif ! ipv6_routes=$( + "$ip_command" -o -6 route show dev "$interface" scope global + ); then + incomplete_reason=ipv6_route_inspection_failed + elif ! non_link_local_ipv6_routes=$( + printf '%s\n' "$ipv6_routes" | + sed '/^[fF][eE][89aAbB][[:xdigit:]]:/d' + ); then + incomplete_reason=ipv6_route_filter_failed + elif [ -n "$global_addresses" ]; then + incomplete_reason=global_addresses_remain + incomplete_detail="global_addresses=$global_addresses" + elif [ -n "$ipv4_routes" ]; then + incomplete_reason=global_ipv4_routes_remain + incomplete_detail="ipv4_routes=$ipv4_routes" + elif [ -n "$non_link_local_ipv6_routes" ]; then + incomplete_reason=global_ipv6_routes_remain + incomplete_detail="ipv6_routes=$non_link_local_ipv6_routes" + else + continue + fi + ;; + *) + incomplete_reason=administrative_state_not_configured + incomplete_detail="network_status=$network_status" + ;; + esac + fi + fi + + all_interfaces_complete=false + incomplete_interfaces="${incomplete_interfaces}${incomplete_interfaces:+ }$interface" + if [ "$wait_attempts" -eq 1 ]; then + echo "Network interface still incomplete: interface=$interface reason=$incomplete_reason${incomplete_detail:+ $incomplete_detail}" >&2 + fi + done + + if [ "$wait_failed" = true ]; then + break + fi + if [ "$all_interfaces_complete" = true ]; then + wait_complete=true + break + fi + + wait_attempts=$((wait_attempts - 1)) + if [ "$wait_attempts" -gt 0 ]; then + sleep 1 fi done +if [ "$#" -gt 0 ] && [ "$wait_complete" = false ] && [ "$wait_failed" = false ]; then + echo "Timed out waiting for network interface reconfiguration: interfaces=${incomplete_interfaces:-unknown}" >&2 + failed=true +fi + [ "$failed" = false ]