diff --git a/pxe/Makefile.toml b/pxe/Makefile.toml index 9a7b6c5c13..7513d1134c 100644 --- a/pxe/Makefile.toml +++ b/pxe/Makefile.toml @@ -130,13 +130,15 @@ script = ''' ''' [tasks.stage-scout-module-loader] -description = "Copies the scout module loader to build-output" +description = "Copies the Scout helper scripts to build-output" script = ''' mkdir -p ${REPO_ROOT}/pxe/mkosi.profiles/scout-oss-aarch64/mkosi.extra/build-output/ cp ${REPO_ROOT}/pxe/common_files/load_modules.sh ${REPO_ROOT}/pxe/mkosi.profiles/scout-oss-aarch64/mkosi.extra/build-output/ + cp ${REPO_ROOT}/pxe/common_files/forge-scout-network.sh ${REPO_ROOT}/pxe/mkosi.profiles/scout-oss-aarch64/mkosi.extra/build-output/ mkdir -p ${REPO_ROOT}/pxe/mkosi.profiles/scout-oss-x86_64/mkosi.extra/build-output/ cp ${REPO_ROOT}/pxe/common_files/load_modules.sh ${REPO_ROOT}/pxe/mkosi.profiles/scout-oss-x86_64/mkosi.extra/build-output/ + cp ${REPO_ROOT}/pxe/common_files/forge-scout-network.sh ${REPO_ROOT}/pxe/mkosi.profiles/scout-oss-x86_64/mkosi.extra/build-output/ ''' [tasks.build-boot-artifacts-x86-host] diff --git a/pxe/common_files/forge-scout-network.sh b/pxe/common_files/forge-scout-network.sh new file mode 100755 index 0000000000..9a57cf80d8 --- /dev/null +++ b/pxe/common_files/forge-scout-network.sh @@ -0,0 +1,272 @@ +#!/usr/bin/env sh +# +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +set -u +set -f + +cmdline_file=${SCOUT_CMDLINE_FILE:-/proc/cmdline} +sys_class_net=${SCOUT_SYS_CLASS_NET:-/sys/class/net} +ip_command=${SCOUT_IP_COMMAND:-ip} +networkctl_command=${SCOUT_NETWORKCTL_COMMAND:-networkctl} +networkd_runtime_dir=${SCOUT_NETWORKD_RUNTIME_DIR:-/run/systemd/network} +networkd_dhcp_file=${SCOUT_NETWORKD_DHCP_FILE:-/etc/systemd/network/dhcp.network} + +if ! cmdline=$(cat "$cmdline_file"); then + echo "Skipping Scout network configuration: reason=cmdline_unreadable" >&2 + exit 0 +fi + +preferred_mac= +mac_parameter_count=0 +for parameter in $cmdline +do + case "$parameter" in + mac=*) + mac_parameter_count=$((mac_parameter_count + 1)) + preferred_mac=${parameter#mac=} + ;; + esac +done +set +f + +if [ "$mac_parameter_count" -ne 1 ]; then + echo "Skipping Scout network configuration: reason=mac_parameter_count count=$mac_parameter_count" >&2 + exit 0 +fi + +case "$preferred_mac" in + [0-9A-Fa-f][0-9A-Fa-f]:[0-9A-Fa-f][0-9A-Fa-f]:[0-9A-Fa-f][0-9A-Fa-f]:[0-9A-Fa-f][0-9A-Fa-f]:[0-9A-Fa-f][0-9A-Fa-f]:[0-9A-Fa-f][0-9A-Fa-f]) + ;; + *) + echo "Skipping Scout network configuration: reason=invalid_mac_parameter" >&2 + exit 0 + ;; +esac +preferred_mac=$(printf '%s\n' "$preferred_mac" | tr '[:upper:]' '[:lower:]') + +preferred_interface= +preferred_interface_count=0 +for net_path in "$sys_class_net"/* +do + [ -e "$net_path" ] || continue + [ -r "$net_path/address" ] || continue + + interface_mac=$(cat "$net_path/address") || continue + interface_mac=$(printf '%s\n' "$interface_mac" | tr '[:upper:]' '[:lower:]') + if [ "$interface_mac" = "$preferred_mac" ]; then + preferred_interface=${net_path##*/} + preferred_interface_count=$((preferred_interface_count + 1)) + fi +done + +if [ "$preferred_interface_count" -ne 1 ]; then + echo "Skipping Scout network configuration: reason=preferred_interface_count count=$preferred_interface_count" >&2 + exit 0 +fi + +carrier_file="$sys_class_net/$preferred_interface/carrier" +if [ ! -r "$carrier_file" ] || [ "$(cat "$carrier_file")" != 1 ]; then + echo "Skipping Scout network configuration: interface=$preferred_interface reason=no_carrier" >&2 + exit 0 +fi + +if global_addresses=$("$ip_command" -o address show dev "$preferred_interface" scope global); then + if [ -z "$global_addresses" ]; then + echo "Skipping Scout network configuration: interface=$preferred_interface reason=no_global_address" >&2 + exit 0 + fi +else + echo "Skipping Scout network configuration: interface=$preferred_interface reason=address_inspection_failed" >&2 + exit 0 +fi + +runtime_network_file="$networkd_runtime_dir/00-forge-scout-nonpreferred.network" + +# Build the target set from the configuration networkd selected for each link. +# Name= also matches alternative interface names, so matching only the current +# interface name here could miss links that the wildcard configuration affects. +set -- +for net_path in "$sys_class_net"/* +do + [ -e "$net_path" ] || continue + + interface=${net_path##*/} + [ "$interface" = "$preferred_interface" ] && continue + + if ! network_status=$( + "$networkctl_command" --json=short status "$interface" 2>/dev/null + ); then + # A link can disappear while interfaces are being enumerated. + [ -e "$net_path" ] || continue + + echo "Skipping Scout network configuration: interface=$interface reason=network_status_inspection_failed" >&2 + exit 0 + fi + + case "$network_status" in + *"\"NetworkFile\":\"$networkd_dhcp_file\""*|*"\"NetworkFile\":\"$runtime_network_file\""*) + set -- "$@" "$interface" + ;; + esac +done + +if ! mkdir -p "$networkd_runtime_dir"; then + echo "Failed to create networkd runtime directory: directory=$networkd_runtime_dir" >&2 + exit 1 +fi + +if ! temporary_network_file=$(mktemp "$networkd_runtime_dir/.forge-scout-network.XXXXXX"); then + echo "Failed to create temporary networkd configuration: directory=$networkd_runtime_dir" >&2 + exit 1 +fi + +if ! { + printf '%s\n' \ + '[Match]' \ + 'Name=enx* enp* enP*' \ + "Property=!INTERFACE=$preferred_interface" \ + '' \ + '[Network]' \ + 'DHCP=no' \ + 'IPv6AcceptRA=no' \ + 'ConfigureWithoutCarrier=yes' +} >"$temporary_network_file"; then + echo "Failed to write networkd configuration: path=$temporary_network_file" >&2 + rm -f "$temporary_network_file" + exit 1 +fi + +if ! chmod 0644 "$temporary_network_file" || + ! mv -f "$temporary_network_file" "$runtime_network_file"; then + echo "Failed to install networkd configuration: path=$runtime_network_file" >&2 + rm -f "$temporary_network_file" + exit 1 +fi + +if ! "$networkctl_command" reload; then + echo "Failed to reload networkd configuration" >&2 + exit 1 +fi + +echo "Selected preferred network interface: interface=$preferred_interface mac=$preferred_mac" +failed=false + +wait_attempts=31 +wait_complete=false +wait_failed=false +while [ "$#" -gt 0 ] && [ "$wait_attempts" -gt 0 ] +do + all_interfaces_complete=true + incomplete_interfaces= + for interface + do + # A removed interface is safe; if it reappears, networkd will apply the + # persistent wildcard configuration. + if [ ! -e "$sys_class_net/$interface" ]; then + continue + fi + + incomplete_reason= + incomplete_detail= + if ! network_status=$( + "$networkctl_command" --json=short status "$interface" 2>/dev/null + ); then + incomplete_reason=network_status_inspection_failed + else + case "$network_status" in + *"\"NetworkFile\":\"$runtime_network_file\""*) + ;; + *) + incomplete_reason=unexpected_network_file + incomplete_detail="network_status=$network_status" + ;; + esac + + if [ -z "$incomplete_reason" ]; then + case "$network_status" in + *'"AdministrativeState":"failed"'*) + echo "Network interface reconfiguration failed: interface=$interface network_status=$network_status" >&2 + failed=true + wait_failed=true + break + ;; + *'"AdministrativeState":"configured"'*) + if ! global_addresses=$( + "$ip_command" -o address show dev "$interface" scope global + ); then + incomplete_reason=address_inspection_failed + elif ! ipv4_routes=$( + "$ip_command" -o -4 route show dev "$interface" scope global + ); then + incomplete_reason=ipv4_route_inspection_failed + elif ! ipv6_routes=$( + "$ip_command" -o -6 route show dev "$interface" scope global + ); then + incomplete_reason=ipv6_route_inspection_failed + elif ! non_link_local_ipv6_routes=$( + printf '%s\n' "$ipv6_routes" | + sed '/^[fF][eE][89aAbB][[:xdigit:]]:/d' + ); then + incomplete_reason=ipv6_route_filter_failed + elif [ -n "$global_addresses" ]; then + incomplete_reason=global_addresses_remain + incomplete_detail="global_addresses=$global_addresses" + elif [ -n "$ipv4_routes" ]; then + incomplete_reason=global_ipv4_routes_remain + incomplete_detail="ipv4_routes=$ipv4_routes" + elif [ -n "$non_link_local_ipv6_routes" ]; then + incomplete_reason=global_ipv6_routes_remain + incomplete_detail="ipv6_routes=$non_link_local_ipv6_routes" + else + continue + fi + ;; + *) + incomplete_reason=administrative_state_not_configured + incomplete_detail="network_status=$network_status" + ;; + esac + fi + fi + + all_interfaces_complete=false + incomplete_interfaces="${incomplete_interfaces}${incomplete_interfaces:+ }$interface" + if [ "$wait_attempts" -eq 1 ]; then + echo "Network interface still incomplete: interface=$interface reason=$incomplete_reason${incomplete_detail:+ $incomplete_detail}" >&2 + fi + done + + if [ "$wait_failed" = true ]; then + break + fi + if [ "$all_interfaces_complete" = true ]; then + wait_complete=true + break + fi + + wait_attempts=$((wait_attempts - 1)) + if [ "$wait_attempts" -gt 0 ]; then + sleep 1 + fi +done + +if [ "$#" -gt 0 ] && [ "$wait_complete" = false ] && [ "$wait_failed" = false ]; then + echo "Timed out waiting for network interface reconfiguration: interfaces=${incomplete_interfaces:-unknown}" >&2 + failed=true +fi + +[ "$failed" = false ] diff --git a/pxe/mkosi.profiles/scout-oss-aarch64/mkosi.conf b/pxe/mkosi.profiles/scout-oss-aarch64/mkosi.conf index b3d3f01044..7b66b2e2b7 100644 --- a/pxe/mkosi.profiles/scout-oss-aarch64/mkosi.conf +++ b/pxe/mkosi.profiles/scout-oss-aarch64/mkosi.conf @@ -38,6 +38,7 @@ Packages= ibverbs-utils iperf3 ipmitool + iproute2 iputils-ping keyboard-configuration isc-dhcp-client diff --git a/pxe/mkosi.profiles/scout-oss-aarch64/mkosi.extra/opt/forge/forge-scout-pre.sh b/pxe/mkosi.profiles/scout-oss-aarch64/mkosi.extra/opt/forge/forge-scout-pre.sh index 4324d2a7a0..5e8cd17973 100755 --- a/pxe/mkosi.profiles/scout-oss-aarch64/mkosi.extra/opt/forge/forge-scout-pre.sh +++ b/pxe/mkosi.profiles/scout-oss-aarch64/mkosi.extra/opt/forge/forge-scout-pre.sh @@ -38,6 +38,7 @@ do cli_cmd=`echo $line|cut -d'=' -f2` fi done + rm "/opt/forge/forge-scout.env" cat "/opt/forge/forge-scout.env.template" > "/opt/forge/forge-scout.env" echo server_uri=$server_uri >> "/opt/forge/forge-scout.env" @@ -65,3 +66,4 @@ udevadm settle chown nvidia-dcgm:nvidia-dcgm /var/log/nvidia-dcgm /opt/forge/load_modules.sh || true udevadm settle +/opt/forge/forge-scout-network.sh || echo "Scout NIC filtering failed; continuing startup" >&2 diff --git a/pxe/mkosi.profiles/scout-oss-aarch64/mkosi.postinst.chroot b/pxe/mkosi.profiles/scout-oss-aarch64/mkosi.postinst.chroot index 0e57825a71..04a0fd25b5 100755 --- a/pxe/mkosi.profiles/scout-oss-aarch64/mkosi.postinst.chroot +++ b/pxe/mkosi.profiles/scout-oss-aarch64/mkosi.postinst.chroot @@ -59,6 +59,7 @@ ln -s /etc/systemd/system/check-scout-updates.timer /etc/systemd/system/timers.t # MODULE LOADER # ============================================================================ cp /build-output/load_modules.sh /opt/forge/ +cp /build-output/forge-scout-network.sh /opt/forge/ # Ensure key commands are present for bin in /opt/forge/forge-scout \ diff --git a/pxe/mkosi.profiles/scout-oss-x86_64/mkosi.conf b/pxe/mkosi.profiles/scout-oss-x86_64/mkosi.conf index e5217c88fa..0fda93d7f9 100644 --- a/pxe/mkosi.profiles/scout-oss-x86_64/mkosi.conf +++ b/pxe/mkosi.profiles/scout-oss-x86_64/mkosi.conf @@ -37,6 +37,7 @@ Packages= ibverbs-utils iperf3 ipmitool + iproute2 iputils-ping keyboard-configuration isc-dhcp-client diff --git a/pxe/mkosi.profiles/scout-oss-x86_64/mkosi.extra/opt/forge/forge-scout-pre.sh b/pxe/mkosi.profiles/scout-oss-x86_64/mkosi.extra/opt/forge/forge-scout-pre.sh index 07e50f5b8d..5e8cd17973 100755 --- a/pxe/mkosi.profiles/scout-oss-x86_64/mkosi.extra/opt/forge/forge-scout-pre.sh +++ b/pxe/mkosi.profiles/scout-oss-x86_64/mkosi.extra/opt/forge/forge-scout-pre.sh @@ -38,6 +38,7 @@ do cli_cmd=`echo $line|cut -d'=' -f2` fi done + rm "/opt/forge/forge-scout.env" cat "/opt/forge/forge-scout.env.template" > "/opt/forge/forge-scout.env" echo server_uri=$server_uri >> "/opt/forge/forge-scout.env" @@ -64,4 +65,5 @@ modprobe mlx5_ib || true udevadm settle chown nvidia-dcgm:nvidia-dcgm /var/log/nvidia-dcgm /opt/forge/load_modules.sh || true -udevadm settle \ No newline at end of file +udevadm settle +/opt/forge/forge-scout-network.sh || echo "Scout NIC filtering failed; continuing startup" >&2 diff --git a/pxe/mkosi.profiles/scout-oss-x86_64/mkosi.postinst.chroot b/pxe/mkosi.profiles/scout-oss-x86_64/mkosi.postinst.chroot index 0e57825a71..04a0fd25b5 100755 --- a/pxe/mkosi.profiles/scout-oss-x86_64/mkosi.postinst.chroot +++ b/pxe/mkosi.profiles/scout-oss-x86_64/mkosi.postinst.chroot @@ -59,6 +59,7 @@ ln -s /etc/systemd/system/check-scout-updates.timer /etc/systemd/system/timers.t # MODULE LOADER # ============================================================================ cp /build-output/load_modules.sh /opt/forge/ +cp /build-output/forge-scout-network.sh /opt/forge/ # Ensure key commands are present for bin in /opt/forge/forge-scout \