diff --git a/Cargo.lock b/Cargo.lock index ce8febd5d5..5c928b19a3 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2860,6 +2860,7 @@ dependencies = [ "arc-swap", "async-trait", "bmc-explorer", + "bmc-mock", "bmc-vendor", "carbide-api-db", "carbide-api-model", diff --git a/crates/api-model/src/site_explorer/mod.rs b/crates/api-model/src/site_explorer/mod.rs index 0d67a314d7..384719bb5d 100644 --- a/crates/api-model/src/site_explorer/mod.rs +++ b/crates/api-model/src/site_explorer/mod.rs @@ -610,10 +610,8 @@ impl ExploredDpu { pub fn hardware_info(&self) -> ModelResult { let serial_number = self .report - .systems - .first() - .and_then(|system| system.serial_number.as_ref()) - .unwrap(); + .dpu_pairing_serial_number() + .ok_or(ModelError::MissingArgument("Missing DPU serial number"))?; let vendor = self .report .systems @@ -626,7 +624,7 @@ impl ExploredDpu { .and_then(|system| system.model.as_ref()); let dmi_data = self .report - .create_temporary_dmi_data(serial_number.as_str(), vendor, model); + .create_temporary_dmi_data(serial_number, vendor, model); let chassis_map = self .report @@ -1634,10 +1632,15 @@ pub fn is_bf2_dpu_part_number(part_number: &str) -> bool { // prefix matching for BlueField-2 DPU (https://docs.nvidia.com/nvidia-bluefield-2-ethernet-dpu-user-guide.pdf) normalized_part_number.starts_with("mbf2") } + +pub fn is_bf4_dpu_part_number(part_number: &str) -> bool { + let normalized_part_number = part_number.to_lowercase(); + normalized_part_number.starts_with("900-9d4b4") +} + // returns true if the passed in string is a BlueField part number pub fn is_bluefield_part_number(part_number: &str) -> bool { let normalized_part_number = part_number.trim().to_lowercase(); - normalized_part_number.contains("bluefield") || is_bf3_dpu_part_number(&normalized_part_number) // prefix matching for BlueField-3 SuperNICs (https://docs.nvidia.com/networking/display/bf3dpu) @@ -1645,6 +1648,7 @@ pub fn is_bluefield_part_number(part_number: &str) -> bool { // prefix matching for BlueField-2 DPU (https://docs.nvidia.com/nvidia-bluefield-2-ethernet-dpu-user-guide.pdf) // TODO (sp): should we be matching on all the individual models listed ("MBF2M516C-CECOT", .. etc) || is_bf2_dpu_part_number(&normalized_part_number) + || is_bf4_dpu_part_number(&normalized_part_number) } /// The kind of BlueField/Mellanox device, classified from its Redfish part number. @@ -1806,6 +1810,32 @@ impl EndpointExplorationReport { .map(str::to_string) .collect() } + + /// Serial key used to join a DPU BMC endpoint to the same DPU as reported by + /// its host BMC. + pub fn dpu_pairing_serial_number(&self) -> Option<&str> { + if !self.is_dpu() { + return None; + } + + self.systems + .first() + .and_then(|system| system.serial_number.as_deref()) + .map(str::trim) + .filter(|serial| !serial.is_empty()) + .or_else(|| { + // BF4 Redfish does not currently expose the product serial or + // DPU/NIC mode on the system object. The stable product serial + // lives on the Bluefield_BMC chassis and matches the serial the + // host BMC reports for the PCIe/network-adapter device. + self.chassis + .iter() + .find(|chassis| chassis.id == "Bluefield_BMC") + .and_then(|chassis| chassis.serial_number.as_deref()) + .map(str::trim) + .filter(|serial| !serial.is_empty()) + }) + } } /// Builds the [`ExploredMlxDevice`] view across a set of explored endpoints. @@ -1817,22 +1847,14 @@ impl EndpointExplorationReport { /// without those two fields. This is the same serial correlation site /// exploration already uses to attach DPUs to their hosts. pub fn collect_explored_mlx_devices(endpoints: &[ExploredEndpoint]) -> Vec { - // Index explored DPU endpoints by their (trimmed) system serial number, so a - // host-reported device can be matched to the DPU's own BMC endpoint -- the - // same key site exploration uses in `record_host_dpu_device`. Empty serials - // are skipped, and a serial reported by more than one DPU endpoint is dropped - // as ambiguous: better to attach nothing than to join to the wrong DPU. + // Index explored DPU endpoints by the serial that host BMCs report for the + // same device. Empty serials are skipped, and a serial reported by more than + // one DPU endpoint is dropped as ambiguous: better to attach nothing than to + // join to the wrong DPU. let mut dpu_by_serial: HashMap<&str, &ExploredEndpoint> = HashMap::new(); let mut ambiguous: HashSet<&str> = HashSet::new(); for ep in endpoints.iter().filter(|ep| ep.report.is_dpu()) { - let Some(serial) = ep - .report - .systems - .first() - .and_then(|system| system.serial_number.as_deref()) - .map(str::trim) - .filter(|serial| !serial.is_empty()) - else { + let Some(serial) = ep.report.dpu_pairing_serial_number() else { continue; }; if dpu_by_serial.insert(serial, ep).is_some() { @@ -2073,6 +2095,59 @@ mod explored_mlx_device_tests { assert_eq!(supernic.nic_mode, None); } + #[test] + fn projects_bf4_and_joins_dpu_by_bluefield_bmc_chassis_serial() { + const BF4_SERIAL: &str = "MT020000000003"; + let host = endpoint( + "192.0.2.20", + EndpointExplorationReport { + endpoint_type: EndpointType::Bmc, + systems: vec![ComputerSystem { + pcie_devices: vec![pcie( + "900-9D4B4-CWAA-TSA", + "82.48.0802", + BF4_SERIAL, + "mat_2", + )], + ..Default::default() + }], + ..Default::default() + }, + ); + let dpu = endpoint( + "192.0.2.50", + EndpointExplorationReport { + endpoint_type: EndpointType::Bmc, + systems: vec![ComputerSystem { + id: "Bluefield".to_string(), + // BF4 leaves the system serial unset; the stable product + // serial used for host pairing is on the Bluefield_BMC + // chassis below. + serial_number: None, + ..Default::default() + }], + chassis: vec![Chassis { + id: "Bluefield_BMC".to_string(), + serial_number: Some(BF4_SERIAL.to_string()), + ..Default::default() + }], + ..Default::default() + }, + ); + + let devices = collect_explored_mlx_devices(&[host, dpu]); + assert_eq!(devices.len(), 1); + + let bf4 = &devices[0]; + assert_eq!(bf4.device_kind, MlxDeviceKind::Unknown); + assert_eq!(bf4.part_number.as_deref(), Some("900-9D4B4-CWAA-TSA")); + assert_eq!(bf4.serial_number.as_deref(), Some(BF4_SERIAL)); + assert_eq!(bf4.dpu_bmc_ip, Some("192.0.2.50".parse().unwrap())); + // BF4 does not currently expose DPU/NIC mode through Redfish; missing + // mode must not prevent the serial join. + assert_eq!(bf4.nic_mode, None); + } + #[test] fn serial_join_skips_empty_and_ambiguous_serials() { let dpu = |addr: &str, serial: &str| { diff --git a/crates/bmc-explorer/src/error.rs b/crates/bmc-explorer/src/error.rs index 80617c7f16..b91d26f16f 100644 --- a/crates/bmc-explorer/src/error.rs +++ b/crates/bmc-explorer/src/error.rs @@ -15,6 +15,7 @@ * limitations under the License. */ +use std::error::Error as StdError; use std::fmt; use nv_redfish::{Bmc, Error as NvRedfishError}; @@ -54,6 +55,19 @@ impl fmt::Debug for Error { } } +impl StdError for Error +where + B: Bmc, + NvRedfishError: 'static, +{ + fn source(&self) -> Option<&(dyn StdError + 'static)> { + match self { + Self::NvRedfish { err, .. } => Some(err), + Self::BmcNotProvided(_) | Self::InvalidValue(_) => None, + } + } +} + impl Error { pub(crate) fn nv_redfish(context: &'static str) -> impl Fn(NvRedfishError) -> Self { move |err| Self::NvRedfish { context, err } diff --git a/crates/bmc-mock/src/hw/bluefield4.rs b/crates/bmc-mock/src/hw/bluefield4.rs index 46b7e53574..1009157497 100644 --- a/crates/bmc-mock/src/hw/bluefield4.rs +++ b/crates/bmc-mock/src/hw/bluefield4.rs @@ -18,8 +18,9 @@ use std::borrow::Cow; use std::sync::Arc; +use carbide_utils::arch::CpuArchitecture; use mac_address::MacAddress; -use rpc::machine_discovery::DiscoveryInfo; +use rpc::machine_discovery::{DiscoveryInfo, DmiData, DpuData}; use serde_json::json; use crate::{Callbacks, LogService, LogServices, hw, redfish}; @@ -122,7 +123,9 @@ impl Bluefield4<'_> { .build(), ), log_services: Some(Arc::new(Bf4LogServices { - event_log: DpuEventLog { entries: vec![] }, + event_log: DpuEventLog { + entries: vec!["DPU Warm Reset".to_string()], + }, })), storage: Some(vec![]), processors: Some(vec![]), @@ -170,7 +173,28 @@ impl Bluefield4<'_> { } pub fn discovery_info(&self) -> DiscoveryInfo { - DiscoveryInfo::default() + DiscoveryInfo { + machine_type: CpuArchitecture::Aarch64.to_string(), + machine_arch: Some(rpc::utils::cpu_architecture_to_rpc( + CpuArchitecture::Aarch64, + )), + dmi_data: Some(DmiData { + board_name: "BlueField-4 DPU".into(), + product_serial: self.product_serial_number.to_string(), + board_serial: carbide_utils::DEFAULT_DPU_DMI_BOARD_SERIAL_NUMBER.into(), + chassis_serial: carbide_utils::DEFAULT_DPU_DMI_CHASSIS_SERIAL_NUMBER.into(), + product_name: "BlueField-4 DPU".into(), + sys_vendor: "Nvidia".into(), + ..Default::default() + }), + dpu_info: Some(DpuData { + part_number: self.part_number().into(), + part_description: format!("NVIDIA BlueField-4 {}", self.part_number()), + factory_mac_address: self.host_mac_address.to_string(), + ..Default::default() + }), + ..Default::default() + } } fn part_number(&self) -> &'static str { diff --git a/crates/bmc-mock/src/test_support/mod.rs b/crates/bmc-mock/src/test_support/mod.rs index 6d61d972f3..dc6bb5a483 100644 --- a/crates/bmc-mock/src/test_support/mod.rs +++ b/crates/bmc-mock/src/test_support/mod.rs @@ -87,6 +87,20 @@ async fn test_bmc((router, state): (axum::Router, BmcState)) -> TestBmcHandle { } } +pub async fn bmc_for_machine(machine_info: MachineInfo) -> TestBmcHandle { + let machine_id = match &machine_info { + MachineInfo::Host(_) => "test-host-id", + MachineInfo::Dpu(_) => "test-dpu-id", + }; + test_bmc(machine_router( + &machine_info, + Arc::new(NoopCallbacks), + machine_id.to_string(), + false, + )) + .await +} + fn host_info(hw_type: HostHardwareType) -> MachineInfo { let ndpu = hw_type.fixed_number_of_dpu().unwrap_or(0); let mut pool = TEST_MAC_POOL.lock().unwrap(); diff --git a/crates/site-explorer/Cargo.toml b/crates/site-explorer/Cargo.toml index a3e2c77fb4..04649fc8ca 100644 --- a/crates/site-explorer/Cargo.toml +++ b/crates/site-explorer/Cargo.toml @@ -52,6 +52,7 @@ thiserror = { workspace = true } version-compare = { workspace = true } [dev-dependencies] +bmc-mock = { path = "../bmc-mock" } bmc-vendor = { path = "../bmc-vendor" } carbide-api-model = { path = "../api-model", default-features = false, features = [ "test-support", diff --git a/crates/site-explorer/src/lib.rs b/crates/site-explorer/src/lib.rs index f08fbbddf2..da1c552037 100644 --- a/crates/site-explorer/src/lib.rs +++ b/crates/site-explorer/src/lib.rs @@ -1186,17 +1186,14 @@ impl SiteExplorer { .map(|em| em.data.dpu_mode); DpuMode::resolve(declared, site_dpu_mode) }; - // Match HOST and DPU using SerialNumber. - // Compare DPU system.serial_number with HOST chassis.network_adapters[].serial_number + // Match HOST and DPU using the serial Redfish reports for the same + // physical card. BF4 does not expose that serial on the DPU system + // object, so this uses `EndpointExplorationReport::dpu_pairing_serial_number` + // rather than reading `systems[0].serial_number` directly. let mut dpu_sn_to_endpoint = HashMap::new(); for (_, ep) in explored_dpus { - if let Some(sn) = ep - .report - .systems - .first() - .and_then(|system| system.serial_number.as_ref()) - { - dpu_sn_to_endpoint.insert(sn.trim().to_string(), ep); + if let Some(sn) = ep.report.dpu_pairing_serial_number() { + dpu_sn_to_endpoint.insert(sn.to_string(), ep); } } @@ -2810,28 +2807,38 @@ impl SiteExplorer { return Ok(true); } - if let Some(nic_mode) = dpu_endpoint.report.nic_mode() { - // DPU's in NIC mode do not have full redfish functionality, - // for example, we will not be able to retrieve the base GUID - // from the redfish response. Skip the next check because the DPUs - // in NIC mode will not expose a pf0 interface to the host. - if nic_mode == NicMode::Nic { + match dpu_endpoint.report.nic_mode() { + Some(NicMode::Nic) => { + // DPU's in NIC mode do not have full redfish functionality, + // for example, we will not be able to retrieve the base GUID + // from the redfish response. Skip the next check because the DPUs + // in NIC mode will not expose a pf0 interface to the host. tracing::info!( "Site explorer found an uningested DPU (bmc ip: {}) in NIC mode", dpu_endpoint.address ); return Ok(true); } - } else { - tracing::error!( - "Site explorer found an uningested DPU (bmc ip: {}) without being able to determine if it is in NIC mode", - dpu_endpoint.address - ); - metrics.increment_host_dpu_pairing_blocker(PairingBlockerReason::DpuNicModeUnknown); - return Ok(false); + Some(NicMode::Dpu) => {} + None if dpu_endpoint.report.dpu_pairing_serial_number().is_some() => { + tracing::warn!( + "Site explorer found an uningested DPU (bmc ip: {}) without a Redfish DPU/NIC mode; continuing because it has a host-pairing serial", + dpu_endpoint.address + ); + } + None => { + tracing::error!( + "Site explorer found an uningested DPU (bmc ip: {}) without being able to determine if it is in NIC mode", + dpu_endpoint.address + ); + metrics.increment_host_dpu_pairing_blocker(PairingBlockerReason::DpuNicModeUnknown); + return Ok(false); + } } - // This is a bluefield in DPU mode + // This is a BlueField that should be pairable as a managed DPU. BF4 may + // not report mode, so host pairing and the PF MAC check decide whether + // it can continue. match find_host_pf_mac_address(dpu_endpoint) { Ok(_) => Ok(true), Err(error) => { @@ -3205,7 +3212,9 @@ impl SiteExplorer { // Preserve existing BF3 part-number heuristics when the operator // hasn't explicitly chosen a mode. Missing part numbers only // disable this heuristic fallback; explicit modes above do not - // require a part number. + // require a part number. BF4 does not currently + // expose a reliable DPU/NIC mode signal over Redfish, so the + // default path does not infer or reconfigure BF4 mode, dpu_part_number.and_then(|dpu_part_number| { if is_bf3_supernic_part_number(dpu_part_number) { Some(NicMode::Nic) diff --git a/crates/site-explorer/tests/site_explorer.rs b/crates/site-explorer/tests/site_explorer.rs index 565beec61b..5d2d98b7b7 100644 --- a/crates/site-explorer/tests/site_explorer.rs +++ b/crates/site-explorer/tests/site_explorer.rs @@ -19,7 +19,12 @@ use std::collections::HashMap; use std::net::IpAddr; use std::str::FromStr; use std::sync::Arc; +use std::time::Duration; +use bmc_explorer::ErrorClass; +use bmc_mock::mac_address_pool::PoolConfig as MacAddressPoolConfig; +use bmc_mock::test_support::{self as bmc_mock_support, TestBmc, axum_http_client}; +use bmc_mock::{DpuMachineInfo, DpuSettings, HostHardwareType, HostMachineInfo, MachineInfo}; use carbide_site_explorer::config::{SiteExplorerConfig, SiteExplorerExploreMode}; use carbide_test_harness::network::segment::TestNetworkSegment; use carbide_test_harness::prelude::*; @@ -3195,6 +3200,180 @@ async fn test_site_explorer_pairs_dpu_from_chassis_serial( Ok(()) } +/// Dell R760 BF4 host BMCs report the card as a PCIe device with a BF4 +/// `900-9D4B4...` part number, while the BF4 BMC does not expose the product +/// serial or DPU/NIC mode on its system object. Pairing must still treat the +/// host as a DPU host by using the DPU BMC's `Bluefield_BMC` chassis serial. +#[sqlx_test] +async fn test_site_explorer_pairs_bf4_dpu_from_bluefield_bmc_chassis_serial( + pool: PgPool, +) -> Result<(), Box> { + use model::expected_machine::{ExpectedMachine, ExpectedMachineData}; + + let env = Env::new(pool).await; + + const BF4_SERIAL: &str = "MT020000000003"; + let dpu_info = DpuMachineInfo { + hw_type: HostHardwareType::DellPowerEdgeR760Bf4, + bmc_mac_address: MacAddress::new([0x02, 0x00, 0x00, 0xbf, 0x04, 0x01]), + host_mac_address: MacAddress::new([0x02, 0x00, 0x00, 0xbf, 0x04, 0x02]), + oob_mac_address: MacAddress::new([0x02, 0x00, 0x00, 0xbf, 0x04, 0x03]), + serial: BF4_SERIAL.to_string(), + settings: DpuSettings::default(), + }; + let host_info = HostMachineInfo { + hw_type: HostHardwareType::DellPowerEdgeR760Bf4, + bmc_mac_address: MacAddress::new([0x02, 0x00, 0x00, 0x00, 0x00, 0x04]), + serial: "020000000004".to_string(), + dpus: vec![dpu_info.clone()], + non_dpu_mac_address: None, + nvos_mac_addresses: vec![], + switch_serial_number: None, + hw_mac_addr_pool: MacAddressPoolConfig::new( + MacAddress::new([0x02, 0x00, 0x00, 0x00, 0x00, 0x00]), + 16, + ) + .unwrap(), + }; + let host_bmc_mac = host_info.bmc_mac_address; + let host_serial = host_info.serial.clone(); + + let error_classifier = |err: &axum_http_client::Error| -> Option { + match err { + axum_http_client::Error::InvalidResponse { status, .. } => match *status { + http::StatusCode::NOT_FOUND => Some(ErrorClass::NotFound), + http::StatusCode::INTERNAL_SERVER_ERROR => Some(ErrorClass::InternalServerError), + _ => None, + }, + _ => None, + } + }; + let explorer_config: bmc_explorer::Config<'_, TestBmc> = bmc_explorer::Config { + boot_interface_mac: None, + error_classifier: &error_classifier, + retry_timeout: Duration::from_millis(0), + }; + let host_bmc = bmc_mock_support::bmc_for_machine(MachineInfo::Host(host_info)).await; + let dpu_bmc_report_source = + bmc_mock_support::bmc_for_machine(MachineInfo::Dpu(dpu_info.clone())).await; + let host_report = + bmc_explorer::nv_generate_exploration_report(host_bmc.service_root, &explorer_config) + .await?; + let mut dpu_report = bmc_explorer::nv_generate_exploration_report( + dpu_bmc_report_source.service_root, + &explorer_config, + ) + .await?; + + // bmc-mock still uses the generic BlueField OEM route for BF4 and + // synthesizes a `DpuMode` value there. The real R760/BF4 dump this + // regression is based on does not expose DPU/NIC mode, so clear only that + // synthetic field while keeping the rest of the Redfish-collected mock + // layout and serial data intact. + if let Some(system) = dpu_report.systems.first_mut() { + system.attributes.nic_mode = None; + } + + let bf4_device = host_report + .systems + .iter() + .flat_map(|system| system.pcie_devices.iter()) + .find(|device| { + device + .part_number + .as_deref() + .is_some_and(|part| part.starts_with("900-9D4B4")) + }) + .expect("R760 BF4 host report must include the BF4 PCIe device"); + assert_eq!(bf4_device.serial_number.as_deref(), Some(BF4_SERIAL)); + assert!( + dpu_report + .systems + .first() + .and_then(|system| system.serial_number.as_deref()) + .is_none(), + "BF4 DPU report should not expose a system serial" + ); + assert_eq!( + dpu_report + .chassis + .iter() + .find(|chassis| chassis.id == "Bluefield_BMC") + .and_then(|chassis| chassis.serial_number.as_deref()), + Some(BF4_SERIAL), + "BF4 DPU report should expose the pairing serial on Bluefield_BMC chassis" + ); + assert_eq!( + dpu_report.nic_mode(), + None, + "BF4 DPU report should not expose Redfish DPU/NIC mode" + ); + + let mut txn = env.pool.begin().await?; + db::expected_machine::create( + &mut txn, + ExpectedMachine { + id: None, + bmc_mac_address: host_bmc_mac, + data: ExpectedMachineData { + bmc_username: "ADMIN".to_string(), + bmc_password: "PASS".to_string(), + serial_number: host_serial, + metadata: model::metadata::Metadata::new_with_default_name(), + ..Default::default() + }, + }, + ) + .await?; + txn.commit().await?; + + let mut host_bmc = env.new_machine(&host_bmc_mac.to_string(), "Dell"); + let mut dpu_bmc = env.new_machine(&dpu_info.bmc_mac_address.to_string(), "NVIDIA/BF/BMC"); + host_bmc.discover_dhcp(env.api()).await?; + dpu_bmc.discover_dhcp(env.api()).await?; + + let explorer_config = SiteExplorerConfig { + enabled: Arc::new(true.into()), + retained_boot_interface_window: None, + explorations_per_run: 10, + concurrent_explorations: 1, + run_interval: std::time::Duration::from_secs(1), + create_machines: Arc::new(true.into()), + ..Default::default() + }; + let explorer = env.test_site_explorer(explorer_config); + explorer.insert_endpoint_results(vec![ + (dpu_bmc.ip.parse().unwrap(), Ok(dpu_report)), + (host_bmc.ip.parse().unwrap(), Ok(host_report)), + ]); + + explorer.run_single_iteration().await.unwrap(); + let mut txn = env.pool.begin().await?; + for ip in [host_bmc.ip.parse()?, dpu_bmc.ip.parse()?] { + db::explored_endpoints::set_preingestion_complete(ip, &mut txn).await?; + } + txn.commit().await?; + explorer.run_single_iteration().await.unwrap(); + + let explored_managed_hosts = db::explored_managed_host::find_all(&env.pool).await?; + assert_eq!( + explored_managed_hosts.len(), + 1, + "BF4 host should not fall through to the zero-DPU blocker path" + ); + assert_eq!( + explored_managed_hosts[0].dpus.len(), + 1, + "BF4 DPU should pair via the Bluefield_BMC chassis serial" + ); + assert_eq!( + explored_managed_hosts[0].dpus[0].bmc_ip, + dpu_bmc.ip.parse::()? + ); + + Ok(()) +} + /// A managed host's DPU-facing `machine_interface` is created (via DHCP) with /// just a MAC and no `boot_interface_id`. The exploration that ingests the host /// then backfills the vendor-specific Redfish interface id onto that row, matched