fix: 3024, offline non-MLX interfaces so we skip onboard NICs

pxe/Makefile.toml

@@ -130,13 +130,15 @@ script = '''
 '''
 
 [tasks.stage-scout-module-loader]
-description = "Copies the scout module loader to build-output"
+description = "Copies the Scout helper scripts to build-output"
 script = '''
   mkdir -p ${REPO_ROOT}/pxe/mkosi.profiles/scout-oss-aarch64/mkosi.extra/build-output/
   cp ${REPO_ROOT}/pxe/common_files/load_modules.sh ${REPO_ROOT}/pxe/mkosi.profiles/scout-oss-aarch64/mkosi.extra/build-output/
+  cp ${REPO_ROOT}/pxe/common_files/forge-scout-network.sh ${REPO_ROOT}/pxe/mkosi.profiles/scout-oss-aarch64/mkosi.extra/build-output/
 
   mkdir -p ${REPO_ROOT}/pxe/mkosi.profiles/scout-oss-x86_64/mkosi.extra/build-output/
   cp ${REPO_ROOT}/pxe/common_files/load_modules.sh ${REPO_ROOT}/pxe/mkosi.profiles/scout-oss-x86_64/mkosi.extra/build-output/
+  cp ${REPO_ROOT}/pxe/common_files/forge-scout-network.sh ${REPO_ROOT}/pxe/mkosi.profiles/scout-oss-x86_64/mkosi.extra/build-output/
 '''
 
 [tasks.build-boot-artifacts-x86-host]

pxe/common_files/forge-scout-network.sh

@@ -0,0 +1,272 @@
+#!/usr/bin/env sh
+#
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+set -u
+set -f
+
+cmdline_file=${SCOUT_CMDLINE_FILE:-/proc/cmdline}
+sys_class_net=${SCOUT_SYS_CLASS_NET:-/sys/class/net}
+ip_command=${SCOUT_IP_COMMAND:-ip}
+networkctl_command=${SCOUT_NETWORKCTL_COMMAND:-networkctl}
+networkd_runtime_dir=${SCOUT_NETWORKD_RUNTIME_DIR:-/run/systemd/network}
+networkd_dhcp_file=${SCOUT_NETWORKD_DHCP_FILE:-/etc/systemd/network/dhcp.network}
+
+if ! cmdline=$(cat "$cmdline_file"); then
+	echo "Skipping Scout network configuration: reason=cmdline_unreadable" >&2
+	exit 0
+fi
+
+preferred_mac=
+mac_parameter_count=0
+for parameter in $cmdline
+do
+	case "$parameter" in
+		mac=*)
+			mac_parameter_count=$((mac_parameter_count + 1))
+			preferred_mac=${parameter#mac=}
+			;;
+	esac
+done
+set +f
+
+if [ "$mac_parameter_count" -ne 1 ]; then
+	echo "Skipping Scout network configuration: reason=mac_parameter_count count=$mac_parameter_count" >&2
+	exit 0
+fi
+
+case "$preferred_mac" in
+	[0-9A-Fa-f][0-9A-Fa-f]:[0-9A-Fa-f][0-9A-Fa-f]:[0-9A-Fa-f][0-9A-Fa-f]:[0-9A-Fa-f][0-9A-Fa-f]:[0-9A-Fa-f][0-9A-Fa-f]:[0-9A-Fa-f][0-9A-Fa-f])
+		;;
+	*)
+		echo "Skipping Scout network configuration: reason=invalid_mac_parameter" >&2
+		exit 0
+		;;
+esac
+preferred_mac=$(printf '%s\n' "$preferred_mac" | tr '[:upper:]' '[:lower:]')
+
+preferred_interface=
+preferred_interface_count=0
+for net_path in "$sys_class_net"/*
+do
+	[ -e "$net_path" ] || continue
+	[ -r "$net_path/address" ] || continue
+
+	interface_mac=$(cat "$net_path/address") || continue
+	interface_mac=$(printf '%s\n' "$interface_mac" | tr '[:upper:]' '[:lower:]')
+	if [ "$interface_mac" = "$preferred_mac" ]; then
+		preferred_interface=${net_path##*/}
+		preferred_interface_count=$((preferred_interface_count + 1))
+	fi
+done
+
+if [ "$preferred_interface_count" -ne 1 ]; then
+	echo "Skipping Scout network configuration: reason=preferred_interface_count count=$preferred_interface_count" >&2
+	exit 0
+fi
+
+carrier_file="$sys_class_net/$preferred_interface/carrier"
+if [ ! -r "$carrier_file" ] || [ "$(cat "$carrier_file")" != 1 ]; then
+	echo "Skipping Scout network configuration: interface=$preferred_interface reason=no_carrier" >&2
+	exit 0
+fi
+
+if global_addresses=$("$ip_command" -o address show dev "$preferred_interface" scope global); then
+	if [ -z "$global_addresses" ]; then
+		echo "Skipping Scout network configuration: interface=$preferred_interface reason=no_global_address" >&2
+		exit 0
+	fi
+else
+	echo "Skipping Scout network configuration: interface=$preferred_interface reason=address_inspection_failed" >&2
+	exit 0
+fi
+
+runtime_network_file="$networkd_runtime_dir/00-forge-scout-nonpreferred.network"
+
+# Build the target set from the configuration networkd selected for each link.
+# Name= also matches alternative interface names, so matching only the current
+# interface name here could miss links that the wildcard configuration affects.
+set --
+for net_path in "$sys_class_net"/*
+do
+	[ -e "$net_path" ] || continue
+
+	interface=${net_path##*/}
+	[ "$interface" = "$preferred_interface" ] && continue
+
+	if ! network_status=$(
+		"$networkctl_command" --json=short status "$interface" 2>/dev/null
+	); then
+		# A link can disappear while interfaces are being enumerated.
+		[ -e "$net_path" ] || continue
+
+		echo "Skipping Scout network configuration: interface=$interface reason=network_status_inspection_failed" >&2
+		exit 0
+	fi
+
+	case "$network_status" in
+		*"\"NetworkFile\":\"$networkd_dhcp_file\""*|*"\"NetworkFile\":\"$runtime_network_file\""*)
+			set -- "$@" "$interface"
+			;;
+	esac
+done
+
+if ! mkdir -p "$networkd_runtime_dir"; then
+	echo "Failed to create networkd runtime directory: directory=$networkd_runtime_dir" >&2
+	exit 1
+fi
+
+if ! temporary_network_file=$(mktemp "$networkd_runtime_dir/.forge-scout-network.XXXXXX"); then
+	echo "Failed to create temporary networkd configuration: directory=$networkd_runtime_dir" >&2
+	exit 1
+fi
+
+if ! {
+	printf '%s\n' \
+		'[Match]' \
+		'Name=enx* enp* enP*' \
+		"Property=!INTERFACE=$preferred_interface" \
+		'' \
+		'[Network]' \
+		'DHCP=no' \
+		'IPv6AcceptRA=no' \
+		'ConfigureWithoutCarrier=yes'
+} >"$temporary_network_file"; then
+	echo "Failed to write networkd configuration: path=$temporary_network_file" >&2
+	rm -f "$temporary_network_file"
+	exit 1
+fi
+
+if ! chmod 0644 "$temporary_network_file" ||
+	! mv -f "$temporary_network_file" "$runtime_network_file"; then
+	echo "Failed to install networkd configuration: path=$runtime_network_file" >&2
+	rm -f "$temporary_network_file"
+	exit 1
+fi
+
+if ! "$networkctl_command" reload; then
+	echo "Failed to reload networkd configuration" >&2
+	exit 1
+fi
+
+echo "Selected preferred network interface: interface=$preferred_interface mac=$preferred_mac"
+failed=false
+
+wait_attempts=31
+wait_complete=false
+wait_failed=false
+while [ "$#" -gt 0 ] && [ "$wait_attempts" -gt 0 ]
+do
+	all_interfaces_complete=true
+	incomplete_interfaces=
+	for interface
+	do
+		# A removed interface is safe; if it reappears, networkd will apply the
+		# persistent wildcard configuration.
+		if [ ! -e "$sys_class_net/$interface" ]; then
+			continue
+		fi
+
+		incomplete_reason=
+		incomplete_detail=
+		if ! network_status=$(
+			"$networkctl_command" --json=short status "$interface" 2>/dev/null
+		); then
+			incomplete_reason=network_status_inspection_failed
+		else
+			case "$network_status" in
+				*"\"NetworkFile\":\"$runtime_network_file\""*)
+					;;
+				*)
+					incomplete_reason=unexpected_network_file
+					incomplete_detail="network_status=$network_status"
+					;;
+			esac
+
+			if [ -z "$incomplete_reason" ]; then
+				case "$network_status" in
+					*'"AdministrativeState":"failed"'*)
+						echo "Network interface reconfiguration failed: interface=$interface network_status=$network_status" >&2
+						failed=true
+						wait_failed=true
+						break
+						;;
+					*'"AdministrativeState":"configured"'*)
+						if ! global_addresses=$(
+							"$ip_command" -o address show dev "$interface" scope global
+						); then
+							incomplete_reason=address_inspection_failed
+						elif ! ipv4_routes=$(
+							"$ip_command" -o -4 route show dev "$interface" scope global
+						); then
+							incomplete_reason=ipv4_route_inspection_failed
+						elif ! ipv6_routes=$(
+							"$ip_command" -o -6 route show dev "$interface" scope global
+						); then
+							incomplete_reason=ipv6_route_inspection_failed
+						elif ! non_link_local_ipv6_routes=$(
+							printf '%s\n' "$ipv6_routes" |
+								sed '/^[fF][eE][89aAbB][[:xdigit:]]:/d'
+						); then
+							incomplete_reason=ipv6_route_filter_failed
+						elif [ -n "$global_addresses" ]; then
+							incomplete_reason=global_addresses_remain
+							incomplete_detail="global_addresses=$global_addresses"
+						elif [ -n "$ipv4_routes" ]; then
+							incomplete_reason=global_ipv4_routes_remain
+							incomplete_detail="ipv4_routes=$ipv4_routes"
+						elif [ -n "$non_link_local_ipv6_routes" ]; then
+							incomplete_reason=global_ipv6_routes_remain
+							incomplete_detail="ipv6_routes=$non_link_local_ipv6_routes"
+						else
+							continue
+						fi
+						;;
+					*)
+						incomplete_reason=administrative_state_not_configured
+						incomplete_detail="network_status=$network_status"
+						;;
+				esac
+			fi
+		fi
+
+		all_interfaces_complete=false
+		incomplete_interfaces="${incomplete_interfaces}${incomplete_interfaces:+ }$interface"
+		if [ "$wait_attempts" -eq 1 ]; then
+			echo "Network interface still incomplete: interface=$interface reason=$incomplete_reason${incomplete_detail:+ $incomplete_detail}" >&2
+		fi
+	done
+
+	if [ "$wait_failed" = true ]; then
+		break
+	fi
+	if [ "$all_interfaces_complete" = true ]; then
+		wait_complete=true
+		break
+	fi
+
+	wait_attempts=$((wait_attempts - 1))
+	if [ "$wait_attempts" -gt 0 ]; then
+		sleep 1
+	fi
+done
+
+if [ "$#" -gt 0 ] && [ "$wait_complete" = false ] && [ "$wait_failed" = false ]; then
+	echo "Timed out waiting for network interface reconfiguration: interfaces=${incomplete_interfaces:-unknown}" >&2
+	failed=true
+fi
+
+[ "$failed" = false ]

pxe/mkosi.profiles/scout-oss-aarch64/mkosi.conf

@@ -38,6 +38,7 @@ Packages=
      ibverbs-utils
      iperf3
      ipmitool
+     iproute2
      iputils-ping
      keyboard-configuration
      isc-dhcp-client

pxe/mkosi.profiles/scout-oss-aarch64/mkosi.extra/opt/forge/forge-scout-pre.sh

@@ -38,6 +38,7 @@ do
                 cli_cmd=`echo $line|cut -d'=' -f2`
         fi
 done
+
 rm "/opt/forge/forge-scout.env"
 cat "/opt/forge/forge-scout.env.template" > "/opt/forge/forge-scout.env"
 echo server_uri=$server_uri >> "/opt/forge/forge-scout.env"
@@ -65,3 +66,4 @@ udevadm settle
 chown nvidia-dcgm:nvidia-dcgm /var/log/nvidia-dcgm
 /opt/forge/load_modules.sh || true
 udevadm settle
+/opt/forge/forge-scout-network.sh || echo "Scout NIC filtering failed; continuing startup" >&2

pxe/mkosi.profiles/scout-oss-aarch64/mkosi.postinst.chroot

@@ -59,6 +59,7 @@ ln -s /etc/systemd/system/check-scout-updates.timer /etc/systemd/system/timers.t
 # MODULE LOADER
 # ============================================================================
 cp /build-output/load_modules.sh /opt/forge/
+cp /build-output/forge-scout-network.sh /opt/forge/
 
 # Ensure key commands are present
 for bin in /opt/forge/forge-scout \

pxe/mkosi.profiles/scout-oss-x86_64/mkosi.conf

@@ -37,6 +37,7 @@ Packages=
      ibverbs-utils
      iperf3
      ipmitool
+     iproute2
      iputils-ping
      keyboard-configuration
      isc-dhcp-client

pxe/mkosi.profiles/scout-oss-x86_64/mkosi.extra/opt/forge/forge-scout-pre.sh

@@ -38,6 +38,7 @@ do
                 cli_cmd=`echo $line|cut -d'=' -f2`
         fi
 done
+
 rm "/opt/forge/forge-scout.env"
 cat "/opt/forge/forge-scout.env.template" > "/opt/forge/forge-scout.env"
 echo server_uri=$server_uri >> "/opt/forge/forge-scout.env"
@@ -64,4 +65,5 @@ modprobe mlx5_ib || true
 udevadm settle
 chown nvidia-dcgm:nvidia-dcgm /var/log/nvidia-dcgm
 /opt/forge/load_modules.sh || true
-udevadm settle
+udevadm settle
+/opt/forge/forge-scout-network.sh || echo "Scout NIC filtering failed; continuing startup" >&2

pxe/mkosi.profiles/scout-oss-x86_64/mkosi.postinst.chroot

@@ -59,6 +59,7 @@ ln -s /etc/systemd/system/check-scout-updates.timer /etc/systemd/system/timers.t
 # MODULE LOADER
 # ============================================================================
 cp /build-output/load_modules.sh /opt/forge/
+cp /build-output/forge-scout-network.sh /opt/forge/
 
 # Ensure key commands are present
 for bin in /opt/forge/forge-scout \