From b01c8730ec2ee2a7991d80daa8ed2b1c748df77f Mon Sep 17 00:00:00 2001 From: Dmitry Porokh Date: Thu, 9 Jul 2026 09:50:21 -0700 Subject: [PATCH] test(site-explorer): cover VR BF4 pairing from chassis inventory Add bmc-explorer test-support helpers for generating mock exploration reports, model the Vera Rubin BlueField chassis inventory in bmc-mock, and verify site-explorer pairs the DPU from the BlueField_0 chassis serial instead of Systems PCIe devices. Signed-off-by: Dmitry Porokh --- crates/bmc-explorer/Cargo.toml | 3 +- crates/bmc-explorer/src/test_support.rs | 90 +++++++++++++- .../tests/integration/bluefield4_explore.rs | 22 +++- .../tests/integration/vera_rubin_explore.rs | 28 +++++ crates/bmc-mock/src/hw/bluefield4.rs | 52 +++++++++ crates/bmc-mock/src/hw/dgx_vr_nvl.rs | 62 ++++++++++ crates/site-explorer/Cargo.toml | 1 + .../tests/integration/site_explorer.rs | 110 ++++++++++++++++++ 8 files changed, 364 insertions(+), 4 deletions(-) diff --git a/crates/bmc-explorer/Cargo.toml b/crates/bmc-explorer/Cargo.toml index 03aeaed238..a6eeebee23 100644 --- a/crates/bmc-explorer/Cargo.toml +++ b/crates/bmc-explorer/Cargo.toml @@ -30,6 +30,7 @@ path = "tests/integration/main.rs" # [local-dependencies] # DO NOT PUT DEPENDENCIES OTHER THAN LOCAL DEPS HERE, THEY SHOULD ALL HAVE 'path =' IN THEM. bmc-vendor = { path = "../bmc-vendor" } +bmc-mock = { path = "../bmc-mock", optional = true } carbide-network = { path = "../network", default-features = false } carbide-api-model = { path = "../api-model", default-features = false } # DO NOT PUT DEPENDENCIES OTHER THAN LOCAL DEPS HERE, THEY SHOULD ALL HAVE 'path =' IN THEM. @@ -70,7 +71,7 @@ tracing = { workspace = true } default = [] # Exposes src/test_support.rs (test-only helpers) without shipping them in a # production build. -test-support = [] +test-support = ["dep:bmc-mock"] [dev-dependencies] axum = { workspace = true } diff --git a/crates/bmc-explorer/src/test_support.rs b/crates/bmc-explorer/src/test_support.rs index 94e272f568..6a462fb670 100644 --- a/crates/bmc-explorer/src/test_support.rs +++ b/crates/bmc-explorer/src/test_support.rs @@ -19,12 +19,20 @@ //! compiles into a production build. use std::sync::Arc; +use std::time::Duration; +use bmc_mock::test_support::TestBmc; +use bmc_mock::test_support::axum_http_client::Error as TestBmcError; +use bmc_mock::{DpuMachineInfo, DpuSettings, HostHardwareType, HostMachineInfo, MachineInfo}; +use model::site_explorer::EndpointExplorationReport; use nv_redfish::{Bmc, Resource, ServiceRoot}; use crate::chassis::ExploredChassisCollection; use crate::computer_system::{self, ExploredComputerSystem}; -use crate::{Config, Error, build_chassis_explore_config, hw, hw_type, is_bluefield_system_id}; +use crate::{ + Config, Error, ErrorClass, build_chassis_explore_config, hw, hw_type, is_bluefield_system_id, + nv_generate_exploration_report, +}; /// Resolve the [`hw::HwType`] for an endpoint, running only the chassis + /// computer-system exploration that detection depends on. @@ -78,3 +86,83 @@ pub async fn detect_hw_type( Ok(hw_type(&root, &explored_system, &explored_chassis)) } + +pub type MockExplorerError = Error; + +#[derive(Clone, Debug)] +pub struct GeneratedEndpointReport { + pub machine_info: T, + pub report: EndpointExplorationReport, +} + +#[derive(Clone, Debug)] +pub struct GeneratedManagedHostReports { + pub host: GeneratedEndpointReport, + pub dpus: Vec>, +} + +pub async fn generate_report_for_machine( + machine_info: MachineInfo, +) -> Result { + let bmc = bmc_mock::test_support::bmc_for_machine(machine_info).await; + nv_generate_exploration_report(bmc.service_root, &explorer_config()).await +} + +pub async fn generate_managed_host_reports( + hw_type: HostHardwareType, +) -> Result { + generate_managed_host_reports_from_info(host_info_for_hw_type(hw_type)).await +} + +pub async fn generate_managed_host_reports_from_info( + host_info: HostMachineInfo, +) -> Result { + let host_report = generate_report_for_machine(MachineInfo::Host(host_info.clone())).await?; + let mut dpus = Vec::with_capacity(host_info.dpus.len()); + + for dpu_info in host_info.dpus.iter().cloned() { + let report = generate_report_for_machine(MachineInfo::Dpu(dpu_info.clone())).await?; + dpus.push(GeneratedEndpointReport { + machine_info: dpu_info, + report, + }); + } + + Ok(GeneratedManagedHostReports { + host: GeneratedEndpointReport { + machine_info: host_info, + report: host_report, + }, + dpus, + }) +} + +fn host_info_for_hw_type(hw_type: HostHardwareType) -> HostMachineInfo { + let dpu_count = hw_type.fixed_number_of_dpu().unwrap_or(0); + let mut pool = bmc_mock::test_support::TEST_MAC_POOL.lock().unwrap(); + let hw_mac_addr_pool = pool.allocate_range_config().unwrap(); + let dpus = (0..dpu_count) + .map(|_| DpuMachineInfo::new(hw_type, &mut pool, DpuSettings::default())) + .collect(); + + HostMachineInfo::new(hw_type, dpus, &mut pool, hw_mac_addr_pool) +} + +fn explorer_config() -> Config<'static, TestBmc> { + Config { + boot_interface_mac: None, + error_classifier: &error_classifier, + retry_timeout: Duration::from_millis(0), + } +} + +fn error_classifier(err: &TestBmcError) -> Option { + match err { + TestBmcError::InvalidResponse { status, .. } => match status.as_u16() { + 404 => Some(ErrorClass::NotFound), + 500 => Some(ErrorClass::InternalServerError), + _ => None, + }, + _ => None, + } +} diff --git a/crates/bmc-explorer/tests/integration/bluefield4_explore.rs b/crates/bmc-explorer/tests/integration/bluefield4_explore.rs index 73e1c11e37..b29e33ce2c 100644 --- a/crates/bmc-explorer/tests/integration/bluefield4_explore.rs +++ b/crates/bmc-explorer/tests/integration/bluefield4_explore.rs @@ -15,7 +15,7 @@ * limitations under the License. */ use bmc_explorer::nv_generate_exploration_report; -use bmc_mock::{DpuMachineInfo, DpuSettings, HostHardwareType, test_support}; +use bmc_mock::{DpuMachineInfo, DpuSettings, HostHardwareType, MachineInfo, test_support}; use mac_address::MacAddress; use model::site_explorer::EndpointType; use tokio::test; @@ -98,7 +98,16 @@ async fn explore_bluefield4_and_generate_machine_id_from_bluefield_bmc_chassis_s #[test] async fn explore_b4240v_and_generate_machine_id() { - let h = test_support::nvidia_dgx_vr_bluefield4_dpu_bmc(DpuSettings::default()).await; + let host_mac_address = MacAddress::new([0x02, 0x00, 0x00, 0xbf, 0x14, 0x02]); + let h = test_support::bmc_for_machine(MachineInfo::Dpu(DpuMachineInfo { + hw_type: HostHardwareType::NvidiaDgxVr, + bmc_mac_address: MacAddress::new([0x02, 0x00, 0x00, 0xbf, 0x14, 0x01]), + host_mac_address, + oob_mac_address: MacAddress::new([0x02, 0x00, 0x00, 0xbf, 0x14, 0x03]), + serial: "MT2610604VN5".to_string(), + settings: DpuSettings::default(), + })) + .await; let mut report = nv_generate_exploration_report(h.service_root, &common::explorer_config()) .await .expect("B4240V exploration should succeed"); @@ -108,6 +117,15 @@ async fn explore_b4240v_and_generate_machine_id() { assert!(report.chassis.iter().any(|chassis| { chassis.id == "BlueField_BMC_0" && chassis.model.as_deref() == Some("B4240V") })); + assert_eq!( + report + .systems + .first() + .expect("systems must be present") + .base_mac + .map(|mac| mac.to_mac()), + Some(host_mac_address) + ); let machine_id = report .generate_machine_id(false) diff --git a/crates/bmc-explorer/tests/integration/vera_rubin_explore.rs b/crates/bmc-explorer/tests/integration/vera_rubin_explore.rs index 31c925d803..98f37dfca8 100644 --- a/crates/bmc-explorer/tests/integration/vera_rubin_explore.rs +++ b/crates/bmc-explorer/tests/integration/vera_rubin_explore.rs @@ -36,6 +36,34 @@ async fn explore_nvidia_dgx_vr_and_generate_machine_id() { assert_eq!(report.endpoint_type, EndpointType::Bmc); assert!(!report.systems.is_empty(), "systems must be present"); assert!(!report.chassis.is_empty(), "chassis must be present"); + assert!( + report.systems[0].pcie_devices.is_empty(), + "VR host pairing should use the BlueField chassis inventory, not host PCIe devices" + ); + + let bluefield_chassis = report + .chassis + .iter() + .find(|chassis| chassis.id == "BlueField_0") + .expect("VR host report should expose the attached BF4 as BlueField_0 chassis"); + assert_eq!( + bluefield_chassis.part_number.as_deref(), + Some("900-9D4A4-00CB-TS4") + ); + assert!( + bluefield_chassis + .serial_number + .as_deref() + .is_some_and(|serial| !serial.is_empty()), + "BlueField_0 chassis should carry the DPU serial for host/DPU pairing" + ); + assert!( + bluefield_chassis + .network_adapters + .iter() + .any(|adapter| adapter.id == "BlueField_NIC_0"), + "BlueField_0 chassis should expose the real VR BlueField_NIC_0 adapter path" + ); let machine_id = report .generate_machine_id(true) diff --git a/crates/bmc-mock/src/hw/bluefield4.rs b/crates/bmc-mock/src/hw/bluefield4.rs index 99a69f47a0..31ab2d733b 100644 --- a/crates/bmc-mock/src/hw/bluefield4.rs +++ b/crates/bmc-mock/src/hw/bluefield4.rs @@ -44,6 +44,9 @@ impl Bluefield4<'_> { const SYSTEM_ID: &'static str = "BlueField_0"; const MANAGER_ID: &'static str = "BlueField_BMC_0"; const BMC_CHASSIS_ID: &'static str = "BlueField_BMC_0"; + const NETWORK_ADAPTER_ID: &'static str = "BlueField_NIC_0"; + const NETWORK_DEVICE_FUNCTION_ID: &'static str = "0"; + const NDF0_TO_BASE_MAC_OFFSET: u64 = 0x10; fn sensor_layout() -> redfish::sensor::Layout { // The older BF4 layout exposed these sensors below Card1. Newer @@ -70,6 +73,7 @@ impl Bluefield4<'_> { model: Some("NA".into()), part_number: Some(self.part_number().into()), serial_number: Some(self.product_serial_number.to_string().into()), + network_adapters: Some(self.network_adapters()), pcie_devices: Some(vec![]), sensors: Some(redfish::sensor::generate_chassis_sensors( "BlueField_0", @@ -113,6 +117,54 @@ impl Bluefield4<'_> { } } + fn network_adapters(&self) -> Vec { + let function = redfish::network_device_function::builder( + &redfish::network_device_function::chassis_resource( + Self::SYSTEM_ID, + Self::NETWORK_ADAPTER_ID, + Self::NETWORK_DEVICE_FUNCTION_ID, + ), + ) + .ethernet(json!({ + "PermanentMACAddress": Self::ndf0_permanent_mac(self.host_mac_address), + })) + .build(); + + vec![ + redfish::network_adapter::builder_from_nic( + &redfish::network_adapter::chassis_resource( + Self::SYSTEM_ID, + Self::NETWORK_ADAPTER_ID, + ), + &self.host_nic(), + ) + .network_device_functions( + &redfish::network_device_function::chassis_collection( + Self::SYSTEM_ID, + Self::NETWORK_ADAPTER_ID, + ), + vec![function], + ) + .status(redfish::resource::Status::Ok) + .build(), + ] + } + + fn ndf0_permanent_mac(host_mac_address: MacAddress) -> MacAddress { + Self::offset_mac(host_mac_address, Self::NDF0_TO_BASE_MAC_OFFSET) + } + + fn offset_mac(mac_address: MacAddress, offset: u64) -> MacAddress { + let bytes = mac_address.bytes(); + let value = u64::from_be_bytes([ + 0, 0, bytes[0], bytes[1], bytes[2], bytes[3], bytes[4], bytes[5], + ]) + .checked_add(offset) + .expect("BF4 NDF0 MAC offset must not overflow"); + let bytes = value.to_be_bytes(); + MacAddress::new([bytes[2], bytes[3], bytes[4], bytes[5], bytes[6], bytes[7]]) + } + pub fn system_config(&self, callbacks: Arc) -> redfish::computer_system::Config { let system_id = Self::SYSTEM_ID; redfish::computer_system::Config { diff --git a/crates/bmc-mock/src/hw/dgx_vr_nvl.rs b/crates/bmc-mock/src/hw/dgx_vr_nvl.rs index 993ba61519..40f3ee2c4d 100644 --- a/crates/bmc-mock/src/hw/dgx_vr_nvl.rs +++ b/crates/bmc-mock/src/hw/dgx_vr_nvl.rs @@ -34,6 +34,10 @@ pub struct DgxVrNvl<'a> { } impl DgxVrNvl<'_> { + const BLUEFIELD_CHASSIS_ID: &'static str = "BlueField_0"; + const BLUEFIELD_NIC_ID: &'static str = "BlueField_NIC_0"; + const BLUEFIELD_PCIE_DEVICE_ID: &'static str = "BlueField_0"; + pub fn manager_config(&self) -> redfish::manager::Config { let bmc_manager_id = "BMC_0"; let bmc_eth_builder = |eth| { @@ -154,6 +158,7 @@ impl DgxVrNvl<'_> { leak_detectors: None, ..redfish::chassis::SingleChassisConfig::defaults() }, + self.bluefield_chassis_config(), redfish::chassis::SingleChassisConfig { id: "HGX_Chassis_0".into(), chassis_type: "Zone".into(), @@ -169,6 +174,63 @@ impl DgxVrNvl<'_> { } } + fn bluefield_chassis_config(&self) -> redfish::chassis::SingleChassisConfig { + let bf4 = self.dpu.host_nic(); + redfish::chassis::SingleChassisConfig { + id: Self::BLUEFIELD_CHASSIS_ID.into(), + chassis_type: "Component".into(), + manufacturer: Some("NVIDIA".into()), + model: bf4.model.clone(), + part_number: bf4.part_number.clone(), + serial_number: bf4.serial_number.clone(), + network_adapters: Some(vec![self.bluefield_network_adapter()]), + pcie_devices: Some(vec![ + redfish::pcie_device::builder_from_nic( + &redfish::pcie_device::chassis_resource( + Self::BLUEFIELD_CHASSIS_ID, + Self::BLUEFIELD_PCIE_DEVICE_ID, + ), + &bf4, + ) + .status(redfish::resource::Status::Ok) + .build(), + ]), + sensors: None, + leak_detectors: None, + ..redfish::chassis::SingleChassisConfig::defaults() + } + } + + fn bluefield_network_adapter(&self) -> redfish::network_adapter::NetworkAdapter { + let network_device_functions = ["0", "1"] + .into_iter() + .map(|id| { + redfish::network_device_function::builder( + &redfish::network_device_function::chassis_resource( + Self::BLUEFIELD_CHASSIS_ID, + Self::BLUEFIELD_NIC_ID, + id, + ), + ) + .build() + }) + .collect(); + + redfish::network_adapter::builder(&redfish::network_adapter::chassis_resource( + Self::BLUEFIELD_CHASSIS_ID, + Self::BLUEFIELD_NIC_ID, + )) + .network_device_functions( + &redfish::network_device_function::chassis_collection( + Self::BLUEFIELD_CHASSIS_ID, + Self::BLUEFIELD_NIC_ID, + ), + network_device_functions, + ) + .status(redfish::resource::Status::Ok) + .build() + } + pub fn update_service_config(&self) -> redfish::update_service::UpdateServiceConfig { redfish::update_service::UpdateServiceConfig { firmware_inventory: vec![], diff --git a/crates/site-explorer/Cargo.toml b/crates/site-explorer/Cargo.toml index 11c476759d..cd20e0ff6f 100644 --- a/crates/site-explorer/Cargo.toml +++ b/crates/site-explorer/Cargo.toml @@ -59,6 +59,7 @@ thiserror = { workspace = true } version-compare = { workspace = true } [dev-dependencies] +bmc-explorer = { path = "../bmc-explorer", features = ["test-support"] } bmc-mock = { path = "../bmc-mock" } opentelemetry-prometheus = { workspace = true } prometheus = { workspace = true } diff --git a/crates/site-explorer/tests/integration/site_explorer.rs b/crates/site-explorer/tests/integration/site_explorer.rs index f08bee8f81..de95643058 100644 --- a/crates/site-explorer/tests/integration/site_explorer.rs +++ b/crates/site-explorer/tests/integration/site_explorer.rs @@ -20,6 +20,8 @@ use std::net::IpAddr; use std::str::FromStr; use std::sync::Arc; +use bmc_explorer::test_support::generate_managed_host_reports; +use bmc_mock::HostHardwareType; use carbide_site_explorer::config::{SiteExplorerConfig, SiteExplorerExploreMode}; use carbide_test_harness::network::segment::TestNetworkSegment; use carbide_test_harness::prelude::*; @@ -3199,6 +3201,114 @@ async fn test_site_explorer_pairs_dpu_from_chassis_serial( Ok(()) } +/// Vera Rubin host BMCs report the attached BF4 as its own `BlueField_0` +/// chassis, with the usable pairing serial on that chassis object and no DPU +/// under `Systems[].PCIeDevices`. This is the shape from the real VR Redfish +/// dump, and it must pair without falling through to the zero-DPU gate. +#[sqlx_test] +async fn test_site_explorer_pairs_vr_bf4_from_bluefield_chassis( + pool: PgPool, +) -> Result<(), Box> { + let env = Env::new(pool).await; + + let reports = generate_managed_host_reports(HostHardwareType::NvidiaDgxVr).await?; + let dpu = reports + .dpus + .first() + .expect("NvidiaDgxVr should generate one DPU"); + let host_bmc_mac = reports.host.machine_info.bmc_mac_address; + let dpu_bmc_mac = dpu.machine_info.bmc_mac_address; + let host_serial = reports + .host + .report + .systems + .first() + .and_then(|system| system.serial_number.clone()) + .expect("VR host report should include a system serial"); + let host_pf_mac = dpu + .report + .systems + .first() + .and_then(|system| system.base_mac) + .expect("DPU report should include host PF base MAC") + .to_mac(); + + let mut host_bmc = env.new_machine(&host_bmc_mac.to_string(), "NVIDIA"); + let mut dpu_bmc = env.new_machine(&dpu_bmc_mac.to_string(), "NVIDIA/BF/BMC"); + host_bmc.discover_dhcp(env.api()).await?; + dpu_bmc.discover_dhcp(env.api()).await?; + let host_bmc_ip: IpAddr = host_bmc.ip.parse()?; + let dpu_bmc_ip: IpAddr = dpu_bmc.ip.parse()?; + + let mut txn = env.pool.begin().await?; + db::expected_machine::create( + &mut txn, + ExpectedMachine { + id: None, + bmc_mac_address: host_bmc_mac, + data: ExpectedMachineData { + bmc_username: "ADMIN".to_string(), + bmc_password: "PASS".to_string(), + serial_number: host_serial, + metadata: Metadata::new_with_default_name(), + ..Default::default() + }, + }, + ) + .await?; + txn.commit().await?; + + let explorer_config = SiteExplorerConfig { + enabled: Arc::new(true.into()), + retained_boot_interface_window: None, + explorations_per_run: 10, + concurrent_explorations: 1, + run_interval: std::time::Duration::from_secs(1), + create_machines: Arc::new(true.into()), + ..Default::default() + }; + let explorer = env.test_site_explorer(explorer_config); + explorer.insert_endpoint_results(vec![ + (dpu_bmc_ip, Ok(dpu.report.clone())), + (host_bmc_ip, Ok(reports.host.report.clone())), + ]); + + explorer.run_single_iteration().await.unwrap(); + let mut txn = env.pool.begin().await?; + for ip in [host_bmc_ip, dpu_bmc_ip] { + db::explored_endpoints::set_preingestion_complete(ip, &mut txn).await?; + } + txn.commit().await?; + explorer.run_single_iteration().await.unwrap(); + + let explored_managed_hosts = db::explored_managed_host::find_all(&env.pool).await?; + assert_eq!( + explored_managed_hosts.len(), + 1, + "VR host should pair via BlueField_0 chassis serial" + ); + let managed_host = &explored_managed_hosts[0]; + assert_eq!(managed_host.dpus.len(), 1); + assert_eq!(managed_host.dpus[0].bmc_ip, dpu_bmc_ip); + assert_eq!( + managed_host.dpus[0].host_pf_mac_address, + Some(host_pf_mac), + "BF4 host-PF MAC should come from the DPU report base_mac" + ); + if let Some(blocker_metric) = env + .test_harness + .test_meter + .formatted_metric("carbide_host_dpu_pairing_blockers_count") + { + assert!( + !blocker_metric.contains("no_dpu_reported_by_host"), + "VR BF4 chassis pairing should not hit the zero-DPU blocker: {blocker_metric}" + ); + } + + Ok(()) +} + /// A managed host's DPU-facing `machine_interface` is created (via DHCP) with /// just a MAC and no `boot_interface_id`. The exploration that ingests the host /// then backfills the vendor-specific Redfish interface id onto that row, matched