diff --git a/Cargo.lock b/Cargo.lock index 7b2a6eb536..4505b68ab7 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2097,6 +2097,7 @@ dependencies = [ "tracing", "tracing-subscriber", "url", + "uuid", "x509-parser", ] diff --git a/crates/health/Cargo.toml b/crates/health/Cargo.toml index cc64fdfa2e..2e9b462818 100644 --- a/crates/health/Cargo.toml +++ b/crates/health/Cargo.toml @@ -89,6 +89,7 @@ nv-redfish = { workspace = true, features = [ ] } rand = { workspace = true } url = { workspace = true, features = ["serde"] } +uuid = { workspace = true } x509-parser = { workspace = true } # [local-dependencies] diff --git a/crates/health/benches/collector_pipeline.rs b/crates/health/benches/collector_pipeline.rs index 04511a8624..17e7ceb73c 100644 --- a/crates/health/benches/collector_pipeline.rs +++ b/crates/health/benches/collector_pipeline.rs @@ -21,7 +21,7 @@ use std::net::{IpAddr, Ipv4Addr}; use std::str::FromStr; use std::sync::Arc; -use carbide_health::endpoint::{BmcAddr, EndpointMetadata, MachineData}; +use carbide_health::endpoint::{BmcAddr, EndpointMetadata, MachineData, SharedSystemUuid}; use carbide_health::metrics::MetricsManager; use carbide_health::sink::{ CollectorEvent, CompositeDataSink, DataSink, EventContext, FirmwareInfo, LogRecord, @@ -59,9 +59,11 @@ fn event_context() -> EventContext { mac: MacAddress::from_str("42:9e:b1:bd:9d:dd").unwrap(), }, collector_type: "sensor_collector", + labels: Default::default(), metadata: Some(EndpointMetadata::Machine(MachineData { machine_id: Some(MACHINE_ID.parse().expect("valid machine id")), machine_serial: None, + system_uuid: SharedSystemUuid::default(), slot_number: None, tray_index: None, nvlink_domain_uuid: None, diff --git a/crates/health/benches/processor_pipeline.rs b/crates/health/benches/processor_pipeline.rs index 320828e3b2..ff2599ddfc 100644 --- a/crates/health/benches/processor_pipeline.rs +++ b/crates/health/benches/processor_pipeline.rs @@ -20,7 +20,7 @@ use std::net::{IpAddr, Ipv4Addr}; use std::str::FromStr; use std::sync::Arc; -use carbide_health::endpoint::{BmcAddr, EndpointMetadata, MachineData}; +use carbide_health::endpoint::{BmcAddr, EndpointMetadata, MachineData, SharedSystemUuid}; use carbide_health::metrics::MetricsManager; use carbide_health::processor::{ EventProcessingPipeline, EventProcessor, HealthReportProcessor, LeakEventProcessor, @@ -95,9 +95,11 @@ fn event_context() -> EventContext { mac: MacAddress::from_str("42:9e:b1:bd:9d:dd").unwrap(), }, collector_type: "sensor_collector", + labels: Default::default(), metadata: Some(EndpointMetadata::Machine(MachineData { machine_id: Some(MACHINE_ID.parse().expect("valid machine id")), machine_serial: None, + system_uuid: SharedSystemUuid::default(), slot_number: None, tray_index: None, nvlink_domain_uuid: None, @@ -273,9 +275,11 @@ fn rack_event_contexts(rack_id: &str, tray_count: usize) -> Vec { mac: MacAddress::from_str(&mac).unwrap(), }, collector_type: "sensor_collector", + labels: Default::default(), metadata: Some(EndpointMetadata::Machine(MachineData { machine_id: Some(MACHINE_ID.parse().expect("valid machine id")), machine_serial: None, + system_uuid: SharedSystemUuid::default(), slot_number: None, tray_index: None, nvlink_domain_uuid: None, diff --git a/crates/health/benches/sink_pipeline.rs b/crates/health/benches/sink_pipeline.rs index 49cd9bb5cd..b27541c9c9 100644 --- a/crates/health/benches/sink_pipeline.rs +++ b/crates/health/benches/sink_pipeline.rs @@ -21,7 +21,7 @@ use std::net::{IpAddr, Ipv4Addr}; use std::str::FromStr; use std::sync::Arc; -use carbide_health::endpoint::{BmcAddr, EndpointMetadata, MachineData}; +use carbide_health::endpoint::{BmcAddr, EndpointMetadata, MachineData, SharedSystemUuid}; use carbide_health::metrics::MetricsManager; use carbide_health::sink::{ Classification, CollectorEvent, CompositeDataSink, DataSink, EventContext, HealthReport, @@ -69,9 +69,11 @@ fn event_context_for_machine(machine_id: &str) -> EventContext { mac: MacAddress::from_str("42:9e:b1:bd:9d:dd").unwrap(), }, collector_type: "sensor_collector", + labels: Default::default(), metadata: Some(EndpointMetadata::Machine(MachineData { machine_id: Some(machine_id.parse().expect("valid machine id")), machine_serial: None, + system_uuid: SharedSystemUuid::default(), slot_number: None, tray_index: None, nvlink_domain_uuid: None, diff --git a/crates/health/example/config.example.toml b/crates/health/example/config.example.toml index 959e698334..436a464e3d 100644 --- a/crates/health/example/config.example.toml +++ b/crates/health/example/config.example.toml @@ -40,6 +40,7 @@ port = 443 mac = "aa:bb:cc:dd:ee:ff" username = "admin" password = "secret" +labels = { site = "dev", cluster = "cluster-01", environment = "development" } machine = { id = "fm100htjtiaehv1n5vh67tbmqq4eabcjdng40f7jupsadbedhruh6rag1l0", serial = "MN-001", slot_number = 15, tray_index = 5, nvlink_domain_uuid = "00000000-0000-0000-0000-000000000000" } [[endpoint_sources.static_bmc_endpoints]] diff --git a/crates/health/src/api_client.rs b/crates/health/src/api_client.rs index 19ee762043..c745e1bfbd 100644 --- a/crates/health/src/api_client.rs +++ b/crates/health/src/api_client.rs @@ -39,7 +39,7 @@ use crate::HealthError; use crate::bmc::{BmcClient, BoxFuture, CredentialProvider}; use crate::endpoint::{ BmcAddr, BmcCredentials, BmcEndpoint, EndpointMetadata, EndpointSource, MachineData, - PowerShelfData, SwitchData, SwitchEndpointRole, + PowerShelfData, SharedSystemUuid, SwitchData, SwitchEndpointRole, }; /// [`ApiEndpointSource`]. @@ -290,9 +290,11 @@ pub struct ApiEndpointSource { bmc_client_cache: Mutex>, } +#[derive(Clone)] struct CachedBmcClient { client: Arc, kind: ApiCredentialKind, + system_uuid: SharedSystemUuid, } impl ApiEndpointSource { @@ -484,6 +486,7 @@ impl ApiEndpointSource { .as_ref() .and_then(|info| info.dmi_data.as_ref()) .map(|dmi| dmi.chassis_serial.clone()), + system_uuid: SharedSystemUuid::default(), slot_number: machine .placement_in_rack .as_ref() @@ -588,7 +591,7 @@ impl ApiEndpointSource { rack_id: Option, credential_kind: ApiCredentialKind, ) -> Result, HealthError> { - let bmc = { + let cached = { let mut cache = self.bmc_client_cache.lock().expect("cache mutex poisoned"); cache_or_create_bmc_client(&mut cache, addr.mac, credential_kind, |kind| { let provider: Arc = Arc::new(ApiCredentialProvider { @@ -604,11 +607,16 @@ impl ApiEndpointSource { )?)) })? }; + let mut metadata = metadata; + if let Some(EndpointMetadata::Machine(machine)) = metadata.as_mut() { + machine.system_uuid = cached.system_uuid; + } Ok(Arc::new(BmcEndpoint { addr, metadata, rack_id, - bmc, + labels: Default::default(), + bmc: cached.client, })) } } @@ -618,7 +626,7 @@ fn cache_or_create_bmc_client( mac: MacAddress, credential_kind: ApiCredentialKind, make_client: impl FnOnce(ApiCredentialKind) -> Result, HealthError>, -) -> Result, HealthError> { +) -> Result { if let Some(existing) = cache.get(&mac) { if existing.kind != credential_kind { return Err(HealthError::GenericError(format!( @@ -629,18 +637,17 @@ fn cache_or_create_bmc_client( credential_kind.tag(), ))); } - return Ok(existing.client.clone()); + return Ok(existing.clone()); } let client = make_client(credential_kind.clone())?; - cache.insert( - mac, - CachedBmcClient { - client: client.clone(), - kind: credential_kind, - }, - ); - Ok(client) + let cached = CachedBmcClient { + client, + kind: credential_kind, + system_uuid: SharedSystemUuid::default(), + }; + cache.insert(mac, cached.clone()); + Ok(cached) } /// Returns the machine-level GPU driver version derived from discovery data. @@ -840,8 +847,8 @@ mod tests { ); } - #[test] - fn cache_returns_existing_client_on_matching_kind() { + #[tokio::test] + async fn cache_returns_existing_client_on_matching_kind() { let mut cache: HashMap = HashMap::new(); let factory_calls = AtomicUsize::new(0); @@ -860,10 +867,21 @@ mod tests { .expect("cache hit"); assert!( - Arc::ptr_eq(&first, &second), + Arc::ptr_eq(&first.client, &second.client), "cache hit must reuse the same BmcClient Arc — otherwise every \ iteration of discovery rebuilds the session and re-fetches creds" ); + let system_uuid = uuid::uuid!("4c4c4544-0044-4710-8052-cac04f4b4632"); + first + .system_uuid + .get_or_try_init(|| async { Ok::<_, std::convert::Infallible>(Some(system_uuid)) }) + .await + .expect("infallible UUID initialization"); + assert_eq!( + second.system_uuid.get(), + Some(system_uuid), + "cache hit must reuse machine UUID state across discovery iterations" + ); assert_eq!( factory_calls.load(Ordering::SeqCst), 1, diff --git a/crates/health/src/collectors/leak_detector.rs b/crates/health/src/collectors/leak_detector.rs index b9594193c1..f5fa2ce7ef 100644 --- a/crates/health/src/collectors/leak_detector.rs +++ b/crates/health/src/collectors/leak_detector.rs @@ -271,7 +271,7 @@ mod tests { use mac_address::MacAddress; use super::*; - use crate::endpoint::{BmcAddr, EndpointMetadata, MachineData}; + use crate::endpoint::{BmcAddr, EndpointMetadata, MachineData, SharedSystemUuid}; use crate::sink::HealthReportTarget; #[derive(Debug, Eq, PartialEq)] @@ -337,12 +337,14 @@ mod tests { metadata: Some(EndpointMetadata::Machine(MachineData { machine_id: None, machine_serial: None, + system_uuid: SharedSystemUuid::default(), slot_number: None, tray_index: None, nvlink_domain_uuid: None, driver_version: None, })), rack_id: None, + labels: Default::default(), } } diff --git a/crates/health/src/collectors/nvue/gnmi/on_change_processor.rs b/crates/health/src/collectors/nvue/gnmi/on_change_processor.rs index f8fd4786fa..647da3c540 100644 --- a/crates/health/src/collectors/nvue/gnmi/on_change_processor.rs +++ b/crates/health/src/collectors/nvue/gnmi/on_change_processor.rs @@ -383,6 +383,7 @@ mod tests { collector_type, metadata: None, rack_id: None, + labels: Default::default(), } } @@ -779,6 +780,7 @@ mod tests { mac: MacAddress::from_str("AA:BB:CC:DD:EE:FF").unwrap(), }, collector_type: ON_CHANGE_STREAM_ID_SYSTEM_EVENTS, + labels: Default::default(), metadata: Some(EndpointMetadata::Switch(SwitchData { id: Some(switch_id), serial: "SN-SWITCH-001".to_string(), diff --git a/crates/health/src/collectors/nvue/gnmi/sample_processor.rs b/crates/health/src/collectors/nvue/gnmi/sample_processor.rs index b140d6c564..1bb9fd2e02 100644 --- a/crates/health/src/collectors/nvue/gnmi/sample_processor.rs +++ b/crates/health/src/collectors/nvue/gnmi/sample_processor.rs @@ -1102,6 +1102,7 @@ mod tests { collector_type: NVUE_GNMI_SAMPLE_STREAM_ID, metadata: None, rack_id: None, + labels: Default::default(), }; GnmiSampleProcessor { data_sink: None, @@ -1167,6 +1168,7 @@ mod tests { mac: MacAddress::from_str("AA:BB:CC:DD:EE:FF").unwrap(), }, collector_type: NVUE_GNMI_SAMPLE_STREAM_ID, + labels: Default::default(), metadata: Some(EndpointMetadata::Switch(SwitchData { id: Some(switch_id), serial: "SN-SWITCH-001".to_string(), diff --git a/crates/health/src/collectors/nvue/rest/collector.rs b/crates/health/src/collectors/nvue/rest/collector.rs index 82ed5b3b6b..a4ec8d159d 100644 --- a/crates/health/src/collectors/nvue/rest/collector.rs +++ b/crates/health/src/collectors/nvue/rest/collector.rs @@ -1260,6 +1260,7 @@ mod tests { collector_type: COLLECTOR_NAME, metadata: None, rack_id: None, + labels: Default::default(), }; NvueRestCollector { diff --git a/crates/health/src/config.rs b/crates/health/src/config.rs index 617f7a092a..cb52afba8d 100644 --- a/crates/health/src/config.rs +++ b/crates/health/src/config.rs @@ -15,7 +15,7 @@ * limitations under the License. */ -use std::collections::HashSet; +use std::collections::{BTreeMap, HashSet}; use std::fmt::Debug; use std::net::{IpAddr, SocketAddr}; use std::path::{Path, PathBuf}; @@ -201,6 +201,9 @@ pub struct StaticBmcEndpoint { pub power_shelf: Option, pub switch: Option, pub rack_id: Option, + /// User-defined labels attached to telemetry emitted for this endpoint. + #[serde(default)] + pub labels: BTreeMap, } #[derive(Clone, Debug, serde::Deserialize, serde::Serialize)] @@ -272,6 +275,7 @@ impl Debug for StaticBmcEndpoint { .field("power_shelf", &self.power_shelf) .field("switch", &self.switch) .field("rack_id", &self.rack_id) + .field("labels", &self.labels) .finish() } } @@ -308,6 +312,52 @@ impl StaticBmcEndpoint { )); } + const RESERVED_LABELS: &[&str] = &[ + "collector_type", + "endpoint_ip", + "endpoint_key", + "endpoint_mac", + "machine_id", + "machine_slot_number", + "machine_tray_index", + "nvlink_domain_uuid", + "rack_id", + "serial_number", + "switch_id", + "switch_slot_number", + "switch_tray_index", + "system_uuid", + ]; + + if self.labels.len() > 32 { + return Err(format!( + "endpoint_sources.static_bmc_endpoints[{index}].labels supports at most 32 labels" + )); + } + + for (name, value) in &self.labels { + let mut chars = name.chars(); + let valid_start = chars + .next() + .is_some_and(|ch| ch.is_ascii_alphabetic() || ch == '_'); + let valid_rest = chars.all(|ch| ch.is_ascii_alphanumeric() || ch == '_'); + if !valid_start || !valid_rest { + return Err(format!( + "endpoint_sources.static_bmc_endpoints[{index}].labels key {name:?} must match [a-zA-Z_][a-zA-Z0-9_]*" + )); + } + if RESERVED_LABELS.contains(&name.as_str()) { + return Err(format!( + "endpoint_sources.static_bmc_endpoints[{index}].labels key {name:?} is reserved" + )); + } + if value.len() > 1024 { + return Err(format!( + "endpoint_sources.static_bmc_endpoints[{index}].labels value for {name:?} exceeds 1024 bytes" + )); + } + } + Ok(()) } } @@ -1847,6 +1897,7 @@ mod tests { power_shelf: None, switch: None, rack_id: None, + labels: BTreeMap::new(), } } @@ -3837,6 +3888,7 @@ ip = "10.0.1.2" mac = "11:22:33:44:55:11" username = "admin" password = "pass" +labels = { site = "rno-dev7", environment = "development" } machine = { id = "fm100htjtiaehv1n5vh67tbmqq4eabcjdng40f7jupsadbedhruh6rag1l0", serial = "MN-001", driver_version = "570.82", slot_number = 15, tray_index = 5, nvlink_domain_uuid = "00000000-0000-0000-0000-000000000000" } "#; @@ -3851,6 +3903,15 @@ machine = { id = "fm100htjtiaehv1n5vh67tbmqq4eabcjdng40f7jupsadbedhruh6rag1l0", .as_ref() .expect("machine metadata"); + config.validate().expect("valid custom labels"); + assert_eq!( + config.endpoint_sources.static_bmc_endpoints[0] + .labels + .get("site") + .map(String::as_str), + Some("rno-dev7") + ); + assert_eq!(machine.slot_number, Some(15)); assert_eq!(machine.tray_index, Some(5)); assert_eq!(machine.driver_version.as_deref(), Some("570.82")); @@ -3860,6 +3921,34 @@ machine = { id = "fm100htjtiaehv1n5vh67tbmqq4eabcjdng40f7jupsadbedhruh6rag1l0", ); } + #[test] + fn test_static_endpoint_rejects_invalid_or_reserved_label_names() { + for (name, expected) in [("bad-label", "must match"), ("system_uuid", "is reserved")] { + let toml_content = format!( + r#" +[endpoint_sources.carbide_api] +enabled = false + +[[endpoint_sources.static_bmc_endpoints]] +ip = "10.0.1.2" +mac = "11:22:33:44:55:11" +username = "admin" +password = "pass" +labels = {{ {name} = "value" }} +machine = {{ id = "fm100htjtiaehv1n5vh67tbmqq4eabcjdng40f7jupsadbedhruh6rag1l0" }} +"# + ); + let config: Config = Figment::new() + .merge(Serialized::defaults(Config::default())) + .merge(Toml::string(&toml_content)) + .extract() + .expect("label syntax should parse"); + + let error = config.validate().expect_err("label should be rejected"); + assert!(error.contains(expected), "unexpected error: {error}"); + } + } + #[test] fn test_static_endpoints_accept_position_field_aliases() { let toml_content = r#" diff --git a/crates/health/src/discovery/identity.rs b/crates/health/src/discovery/identity.rs new file mode 100644 index 0000000000..64c7235860 --- /dev/null +++ b/crates/health/src/discovery/identity.rs @@ -0,0 +1,169 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +use nv_redfish::ServiceRoot; + +use crate::HealthError; +use crate::endpoint::{BmcEndpoint, EndpointMetadata}; + +struct SystemIdentity { + id: String, + uuid: Option, + bios_version: Option, +} + +fn select_primary_system(systems: &[SystemIdentity]) -> Option<&SystemIdentity> { + systems + .iter() + .find(|system| { + system + .bios_version + .as_deref() + .is_some_and(|version| !version.trim().is_empty()) + }) + .or_else(|| systems.first()) +} + +/// Resolves the primary ComputerSystem UUID when it is not already known. +/// +/// A system with a non-empty BIOS version is preferred because BMCs may expose +/// auxiliary systems alongside the host. When no system has BIOS metadata, the +/// first collection member is used. +pub(super) async fn ensure_primary_system_uuid(endpoint: &BmcEndpoint) -> Result<(), HealthError> { + let Some(EndpointMetadata::Machine(machine)) = endpoint.metadata.as_ref() else { + return Ok(()); + }; + + machine + .system_uuid + .get_or_try_init(|| async { + let root = ServiceRoot::new(endpoint.bmc().clone()).await?; + let Some(systems) = root.systems().await? else { + tracing::warn!( + bmc_address = ?endpoint.addr, + "BMC does not expose a ComputerSystem collection" + ); + return Ok(None); + }; + let systems = systems.members().await?; + let identities: Vec = systems + .iter() + .map(|system| { + let raw = system.raw(); + SystemIdentity { + id: raw.base.id.clone(), + uuid: raw.uuid.flatten(), + bios_version: raw.bios_version.clone().flatten(), + } + }) + .collect(); + let Some(primary) = select_primary_system(&identities) else { + tracing::warn!( + bmc_address = ?endpoint.addr, + "BMC exposes an empty ComputerSystem collection" + ); + return Ok(None); + }; + if primary.uuid.is_none() { + tracing::warn!( + bmc_address = ?endpoint.addr, + system_id = %primary.id, + "Primary ComputerSystem does not expose a UUID" + ); + } + Ok::, HealthError>(primary.uuid) + }) + .await?; + + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + + const FIRST_UUID: uuid::Uuid = uuid::uuid!("11111111-1111-1111-1111-111111111111"); + const BIOS_UUID: uuid::Uuid = uuid::uuid!("22222222-2222-2222-2222-222222222222"); + + #[test] + fn primary_system_prefers_first_system_with_bios() { + let systems = [ + SystemIdentity { + id: "auxiliary".to_string(), + uuid: Some(FIRST_UUID), + bios_version: None, + }, + SystemIdentity { + id: "host".to_string(), + uuid: Some(BIOS_UUID), + bios_version: Some("1.2.3".to_string()), + }, + ]; + + let primary = select_primary_system(&systems).expect("primary system"); + + assert_eq!(primary.id, "host"); + assert_eq!(primary.uuid, Some(BIOS_UUID)); + } + + #[test] + fn primary_system_falls_back_to_first_member() { + let systems = [ + SystemIdentity { + id: "first".to_string(), + uuid: Some(FIRST_UUID), + bios_version: None, + }, + SystemIdentity { + id: "second".to_string(), + uuid: Some(BIOS_UUID), + bios_version: Some(" ".to_string()), + }, + ]; + + let primary = select_primary_system(&systems).expect("primary system"); + + assert_eq!(primary.id, "first"); + assert_eq!(primary.uuid, Some(FIRST_UUID)); + } + + #[test] + fn primary_system_prefers_host_bios_over_auxiliary_uuid() { + let systems = [ + SystemIdentity { + id: "HGX_Baseboard".to_string(), + uuid: Some(FIRST_UUID), + bios_version: None, + }, + SystemIdentity { + id: "host".to_string(), + uuid: None, + bios_version: Some("1.2.3".to_string()), + }, + ]; + + let primary = select_primary_system(&systems).expect("primary system"); + + assert_eq!(primary.id, "host"); + assert_eq!(primary.uuid, None); + } + + #[test] + fn primary_system_is_absent_for_empty_collection() { + assert!(select_primary_system(&[]).is_none()); + } +} diff --git a/crates/health/src/discovery/iteration.rs b/crates/health/src/discovery/iteration.rs index 47d22fc53c..78b32b8b4a 100644 --- a/crates/health/src/discovery/iteration.rs +++ b/crates/health/src/discovery/iteration.rs @@ -20,9 +20,12 @@ use std::collections::HashSet; use std::sync::Arc; use std::time::Instant; +use futures::{StreamExt, stream}; + use super::DiscoveryIterationStats; use super::cleanup::{stop_ineligible_nmxc_collectors, stop_removed_bmc_collectors}; use super::context::{CollectorKind, DiscoveryLoopContext}; +use super::identity::ensure_primary_system_uuid; use super::spawn::{spawn_collectors_for_endpoint, switch_supports_nmxc_subscription}; use crate::HealthError; use crate::config::Configurable; @@ -74,6 +77,24 @@ pub async fn run_discovery_iteration( .cloned() .collect(); + // Resolve machine identity before collectors start when possible. Shared + // write-once state propagates the result to running collectors and caches + // both present and absent UUIDs, preventing repeated successful BMC queries. + let identity_concurrency = ctx.discovery_config.discovery_concurrency.max(1); + stream::iter(sharded_endpoints.iter().cloned()) + .map(|endpoint| async move { + if let Err(error) = ensure_primary_system_uuid(&endpoint).await { + tracing::warn!( + ?error, + bmc_address = ?endpoint.addr, + "Could not resolve primary ComputerSystem UUID; continuing without it" + ); + } + }) + .buffer_unordered(identity_concurrency) + .collect::>() + .await; + if sharded_endpoints.is_empty() { tracing::warn!("No endpoints assigned to this shard"); } else { diff --git a/crates/health/src/discovery/mod.rs b/crates/health/src/discovery/mod.rs index d14d6c5af0..778f721c68 100644 --- a/crates/health/src/discovery/mod.rs +++ b/crates/health/src/discovery/mod.rs @@ -17,6 +17,7 @@ mod cleanup; mod context; +mod identity; mod iteration; mod spawn; diff --git a/crates/health/src/discovery/spawn.rs b/crates/health/src/discovery/spawn.rs index 157cc431d7..13364ee027 100644 --- a/crates/health/src/discovery/spawn.rs +++ b/crates/health/src/discovery/spawn.rs @@ -696,7 +696,8 @@ mod tests { }; use crate::endpoint::test_support::endpoint_with_creds; use crate::endpoint::{ - BmcAddr, BmcCredentials, EndpointMetadata, MachineData, SwitchData, SwitchEndpointRole, + BmcAddr, BmcCredentials, EndpointMetadata, MachineData, SharedSystemUuid, SwitchData, + SwitchEndpointRole, }; use crate::limiter::{NoopLimiter, RateLimiter}; use crate::metrics::MetricsManager; @@ -788,6 +789,7 @@ mod tests { .expect("valid machine id"), ), machine_serial: None, + system_uuid: SharedSystemUuid::default(), slot_number: None, tray_index: None, nvlink_domain_uuid: None, @@ -1218,6 +1220,7 @@ mod tests { "failing-switch-host", )), rack_id: None, + labels: Default::default(), bmc, }); diff --git a/crates/health/src/endpoint/cluster.rs b/crates/health/src/endpoint/cluster.rs index 73473b027d..5e1e6ffbe5 100644 --- a/crates/health/src/endpoint/cluster.rs +++ b/crates/health/src/endpoint/cluster.rs @@ -442,6 +442,7 @@ fn build_endpoints( addr, metadata: None, rack_id: node.rack.as_deref().map(RackId::new), + labels: Default::default(), bmc, })); } diff --git a/crates/health/src/endpoint/mod.rs b/crates/health/src/endpoint/mod.rs index 60106db4fc..d6564c8910 100644 --- a/crates/health/src/endpoint/mod.rs +++ b/crates/health/src/endpoint/mod.rs @@ -22,7 +22,7 @@ mod sources; pub use cluster::ClusterEndpointSource; pub use model::{ BmcAddr, BmcCredentials, BmcEndpoint, EndpointMetadata, EndpointSource, MachineData, - PowerShelfData, SwitchData, SwitchEndpointRole, + PowerShelfData, SharedSystemUuid, SwitchData, SwitchEndpointRole, }; pub use sources::{CompositeEndpointSource, StaticEndpointSource}; @@ -61,6 +61,7 @@ pub(crate) mod test_support { addr, metadata, rack_id, + labels: Default::default(), bmc, } } diff --git a/crates/health/src/endpoint/model.rs b/crates/health/src/endpoint/model.rs index a88b972f41..f6eb67e9a6 100644 --- a/crates/health/src/endpoint/model.rs +++ b/crates/health/src/endpoint/model.rs @@ -16,6 +16,8 @@ */ use std::borrow::Cow; +use std::collections::BTreeMap; +use std::future::Future; use std::net::IpAddr; use std::sync::Arc; @@ -25,16 +27,48 @@ use carbide_uuid::power_shelf::PowerShelfId; use carbide_uuid::rack::RackId; use carbide_uuid::switch::SwitchId; use mac_address::MacAddress; +use tokio::sync::OnceCell; use url::Url; use crate::HealthError; use crate::bmc::{BmcClient, BoxFuture}; +/// Shared, write-once UUID reported by a machine's primary ComputerSystem. +/// +/// Collectors clone endpoint metadata when they start, so this state must remain +/// shared for a UUID resolved after collector startup to reach emitted events. +#[derive(Clone, Debug, Default)] +pub struct SharedSystemUuid(Arc>>); + +impl SharedSystemUuid { + pub fn get(&self) -> Option { + self.0.get().copied().flatten() + } + + pub(crate) async fn get_or_try_init(&self, f: F) -> Result, E> + where + F: FnOnce() -> Fut, + Fut: Future, E>>, + { + self.0.get_or_try_init(f).await.copied() + } +} + +impl From> for SharedSystemUuid { + fn from(system_uuid: Option) -> Self { + match system_uuid { + Some(system_uuid) => Self(Arc::new(OnceCell::new_with(Some(Some(system_uuid))))), + None => Self::default(), + } + } +} + #[derive(Clone)] pub struct BmcEndpoint { pub addr: BmcAddr, pub metadata: Option, pub rack_id: Option, + pub labels: BTreeMap, pub bmc: Arc, } @@ -126,6 +160,14 @@ pub struct MachineData { /// Hardware chassis serial discovered from machine DMI data, when known. pub machine_serial: Option, + /// UUID reported by the primary Redfish ComputerSystem resource. + /// + /// Endpoint discovery resolves this on demand. The write-once shared state + /// propagates enrichment to collectors that already started and records + /// both present and absent UUID results so successful BMC queries happen + /// only once. + pub system_uuid: SharedSystemUuid, + /// Physical rack slot where the machine is installed, when known. pub slot_number: Option, @@ -225,12 +267,14 @@ pub trait EndpointSource: Send + Sync { #[cfg(test)] mod tests { + use std::convert::Infallible; use std::net::IpAddr; use std::str::FromStr; + use std::sync::atomic::{AtomicUsize, Ordering}; use mac_address::MacAddress; - use super::{BmcAddr, BmcCredentials}; + use super::{BmcAddr, BmcCredentials, SharedSystemUuid}; use crate::endpoint::test_support::endpoint_with_creds; fn addr(ip: &str, port: Option) -> BmcAddr { @@ -279,4 +323,25 @@ mod tests { assert_eq!(endpoint.switch_connect_host_for_uri(), "[2001:db8::1]"); } + + #[tokio::test] + async fn shared_system_uuid_caches_absent_result_across_clones() { + let state = SharedSystemUuid::default(); + let clone = state.clone(); + let query_count = AtomicUsize::new(0); + + for state in [&state, &clone] { + state + .get_or_try_init(|| async { + query_count.fetch_add(1, Ordering::SeqCst); + Ok::<_, Infallible>(None) + }) + .await + .expect("infallible UUID initialization"); + } + + assert_eq!(query_count.load(Ordering::SeqCst), 1); + assert_eq!(state.get(), None); + assert_eq!(clone.get(), None); + } } diff --git a/crates/health/src/endpoint/sources.rs b/crates/health/src/endpoint/sources.rs index 5e90aa3d57..85d1a1388c 100644 --- a/crates/health/src/endpoint/sources.rs +++ b/crates/health/src/endpoint/sources.rs @@ -30,7 +30,7 @@ use crate::bmc::{BmcClient, FixedCredentialProvider}; use crate::config::{StaticBmcEndpoint, StaticSwitchEndpointRole}; use crate::endpoint::{ BmcAddr, BmcCredentials, BmcEndpoint, BoxFuture, EndpointMetadata, EndpointSource, MachineData, - PowerShelfData, SwitchData, SwitchEndpointRole, + PowerShelfData, SharedSystemUuid, SwitchData, SwitchEndpointRole, }; pub struct StaticEndpointSource { @@ -152,6 +152,7 @@ impl StaticEndpointSource { Some(EndpointMetadata::Machine(MachineData { machine_id, machine_serial: machine.serial.clone(), + system_uuid: SharedSystemUuid::default(), slot_number: machine.slot_number, tray_index: machine.tray_index, nvlink_domain_uuid, @@ -192,6 +193,7 @@ impl StaticEndpointSource { addr, metadata, rack_id: cfg.rack_id.as_ref().map(|id| RackId::new(id.as_str())), + labels: cfg.labels.clone(), bmc, }; endpoints.push(Arc::new(endpoint)); @@ -288,6 +290,7 @@ mod tests { power_shelf: None, switch: None, rack_id: None, + labels: Default::default(), }, StaticBmcEndpoint { ip: ip("10.0.0.2"), @@ -299,6 +302,7 @@ mod tests { power_shelf: None, switch: None, rack_id: None, + labels: Default::default(), }, ]; @@ -334,6 +338,7 @@ mod tests { nmxt_enabled: None, }), rack_id: None, + labels: Default::default(), }]; let source = StaticEndpointSource::from_config(&configs, &reqwest(), None, 10); @@ -371,6 +376,7 @@ mod tests { }), switch: None, rack_id: None, + labels: Default::default(), }]; let source = StaticEndpointSource::from_config(&configs, &reqwest(), None, 10); @@ -411,6 +417,7 @@ mod tests { power_shelf: None, switch: None, rack_id: Some("RACK_1".to_string()), + labels: std::collections::BTreeMap::from([("site".to_string(), "dev".to_string())]), }]; let source = StaticEndpointSource::from_config(&configs, &reqwest(), None, 10); @@ -424,6 +431,10 @@ mod tests { .map(|rack_id| rack_id.as_str()), Some("RACK_1") ); + assert_eq!( + endpoints[0].labels.get("site").map(String::as_str), + Some("dev") + ); match &endpoints[0].metadata { Some(EndpointMetadata::Machine(machine)) => { assert_eq!(machine.machine_id, Some(machine_id)); @@ -456,6 +467,7 @@ mod tests { power_shelf: None, switch: None, rack_id: None, + labels: Default::default(), }]; let source = StaticEndpointSource::from_config(&configs, &reqwest(), None, 10); @@ -482,6 +494,7 @@ mod tests { power_shelf: None, switch: None, rack_id: None, + labels: Default::default(), }]; let source = StaticEndpointSource::from_config(&configs, &reqwest(), None, 10); diff --git a/crates/health/src/otlp/convert.rs b/crates/health/src/otlp/convert.rs index 93b8cd295c..0b18e3c6c2 100644 --- a/crates/health/src/otlp/convert.rs +++ b/crates/health/src/otlp/convert.rs @@ -88,6 +88,9 @@ fn resource_attributes(context: &EventContext) -> Vec { if let Some(machine_id) = context.machine_id() { attrs.push(kv("machine.id", machine_id.to_string())); } + if let Some(system_uuid) = context.system_uuid() { + attrs.push(kv("system.uuid", system_uuid.to_string())); + } if let Some(machine_serial) = context.machine_serial() { attrs.push(kv("machine.serial", machine_serial.to_string())); } @@ -136,6 +139,12 @@ fn resource_attributes(context: &EventContext) -> Vec { if let Some(tray) = context.switch_tray_index() { attrs.push(int_kv("switch.tray_index", i64::from(tray))); } + attrs.extend( + context + .labels() + .iter() + .map(|(name, value)| kv(name, value.clone())), + ); attrs } @@ -328,7 +337,8 @@ mod tests { use super::*; use crate::endpoint::{ - BmcAddr, EndpointMetadata, MachineData, PowerShelfData, SwitchData, SwitchEndpointRole, + BmcAddr, EndpointMetadata, MachineData, PowerShelfData, SharedSystemUuid, SwitchData, + SwitchEndpointRole, }; use crate::sink::{ Classification, HealthReport, HealthReportAlert, LogRecord, Probe, ReportSource, @@ -345,6 +355,7 @@ mod tests { collector_type: "test", metadata: None, rack_id: None, + labels: Default::default(), } } @@ -399,6 +410,10 @@ mod tests { mac: MacAddress::from_str("42:9e:b1:bd:9d:dd").expect("valid mac"), }, collector_type: "test", + labels: std::collections::BTreeMap::from([( + "site".to_string(), + "rno-dev7".to_string(), + )]), metadata: Some(EndpointMetadata::Machine(MachineData { machine_id: Some( "fm100htjtiaehv1n5vh67tbmqq4eabcjdng40f7jupsadbedhruh6rag1l0" @@ -406,6 +421,7 @@ mod tests { .expect("valid machine id"), ), machine_serial: Some("MN-001".to_string()), + system_uuid: Some(uuid::uuid!("4c4c4544-0044-4710-8052-cac04f4b4632")).into(), slot_number: Some(15), tray_index: Some(5), nvlink_domain_uuid: Some(domain_uuid), @@ -417,6 +433,11 @@ mod tests { let attrs = resource_attributes(&context); assert_eq!(attr_value(&attrs, "rack.id"), Some("RACK_1")); + assert_eq!(attr_value(&attrs, "site"), Some("rno-dev7")); + assert_eq!( + attr_value(&attrs, "system.uuid"), + Some("4c4c4544-0044-4710-8052-cac04f4b4632") + ); assert_eq!(attr_value(&attrs, "machine.serial"), Some("MN-001")); assert_eq!(attr_value(&attrs, "driver.version"), Some("570.82")); assert_eq!(attr_value(&attrs, "component.type"), Some("compute_node")); @@ -439,9 +460,11 @@ mod tests { mac: MacAddress::from_str("42:9e:b1:bd:9d:dd").expect("valid mac"), }, collector_type: "test", + labels: Default::default(), metadata: Some(EndpointMetadata::Machine(MachineData { machine_id: None, machine_serial: None, + system_uuid: SharedSystemUuid::default(), slot_number: None, tray_index: None, nvlink_domain_uuid: None, @@ -455,6 +478,7 @@ mod tests { assert_eq!(attr_value(&attrs, "machine.id"), None); assert_eq!(attr_value(&attrs, "component.type"), Some("compute_node")); assert_eq!(attr_value(&attrs, "machine.serial"), None); + assert_eq!(attr_value(&attrs, "system.uuid"), None); assert_eq!(attr_value(&attrs, "driver.version"), None); assert_eq!(attr_value(&attrs, "nvlink.domain.uuid"), None); } @@ -471,6 +495,7 @@ mod tests { mac: MacAddress::from_str("11:22:33:44:55:66").expect("valid mac"), }, collector_type: "test", + labels: Default::default(), metadata: Some(EndpointMetadata::Switch(SwitchData { id: Some(switch_id), serial: "SN-SWITCH-001".to_string(), @@ -508,6 +533,7 @@ mod tests { mac: MacAddress::from_str("11:22:33:44:55:66").expect("valid mac"), }, collector_type: "nvue_gnmi", + labels: Default::default(), metadata: Some(EndpointMetadata::Switch(SwitchData { id: Some(switch_id), serial: "SN-SWITCH-001".to_string(), @@ -560,6 +586,7 @@ mod tests { mac: MacAddress::from_str("22:33:44:55:66:77").expect("valid mac"), }, collector_type: "sensor_collector", + labels: Default::default(), metadata: Some(EndpointMetadata::Switch(SwitchData { id: Some(switch_id), serial: "SN-SWITCH-BMC-001".to_string(), @@ -609,6 +636,7 @@ mod tests { mac: MacAddress::from_str("33:44:55:66:77:88").expect("valid mac"), }, collector_type: "sensor_collector", + labels: Default::default(), metadata: Some(EndpointMetadata::PowerShelf(PowerShelfData { id: Some(power_shelf_id), serial: "SN-PS-001".to_string(), @@ -735,6 +763,7 @@ mod tests { collector_type: "test", metadata: None, rack_id: None, + labels: Default::default(), }; let ctx2 = EventContext { endpoint_key: "endpoint-b".to_string(), @@ -843,6 +872,7 @@ mod tests { mac: MacAddress::from_str("11:22:33:44:55:66").expect("valid mac"), }, collector_type: "nvue_gnmi", + labels: Default::default(), metadata: Some(EndpointMetadata::Switch(SwitchData { id: Some(switch_id), serial: "SN-SWITCH-001".to_string(), diff --git a/crates/health/src/processor/health_report.rs b/crates/health/src/processor/health_report.rs index cb4fa96ad3..046eeb6ee8 100644 --- a/crates/health/src/processor/health_report.rs +++ b/crates/health/src/processor/health_report.rs @@ -268,7 +268,7 @@ mod tests { use nv_redfish::resource::Health as BmcHealth; use super::*; - use crate::endpoint::{BmcAddr, EndpointMetadata, MachineData}; + use crate::endpoint::{BmcAddr, EndpointMetadata, MachineData, SharedSystemUuid}; use crate::sink::HealthReportTarget; fn test_context() -> EventContext { @@ -280,6 +280,7 @@ mod tests { mac: MacAddress::from_str("42:9e:b1:bd:9d:dd").expect("valid mac"), }, collector_type: "sensor_collector", + labels: Default::default(), metadata: Some(EndpointMetadata::Machine(MachineData { machine_id: Some( "fm100htjtiaehv1n5vh67tbmqq4eabcjdng40f7jupsadbedhruh6rag1l0" @@ -287,6 +288,7 @@ mod tests { .expect("valid machine id"), ), machine_serial: None, + system_uuid: SharedSystemUuid::default(), slot_number: None, tray_index: None, nvlink_domain_uuid: None, diff --git a/crates/health/src/processor/intrusion_events.rs b/crates/health/src/processor/intrusion_events.rs index f1c82afd07..f652b7ea78 100644 --- a/crates/health/src/processor/intrusion_events.rs +++ b/crates/health/src/processor/intrusion_events.rs @@ -189,6 +189,7 @@ mod tests { collector_type: "logs_collector", metadata: None, rack_id: None, + labels: Default::default(), } } diff --git a/crates/health/src/processor/leak_events.rs b/crates/health/src/processor/leak_events.rs index ca59005b24..5d8a67ddbe 100644 --- a/crates/health/src/processor/leak_events.rs +++ b/crates/health/src/processor/leak_events.rs @@ -161,6 +161,7 @@ mod tests { collector_type: "leak_detector_collector", metadata: None, rack_id: None, + labels: Default::default(), } } diff --git a/crates/health/src/processor/mod.rs b/crates/health/src/processor/mod.rs index 0560dd0d6b..f0f2877f5d 100644 --- a/crates/health/src/processor/mod.rs +++ b/crates/health/src/processor/mod.rs @@ -193,6 +193,7 @@ mod tests { collector_type: "test", metadata: None, rack_id: None, + labels: Default::default(), } } diff --git a/crates/health/src/processor/rack_leak.rs b/crates/health/src/processor/rack_leak.rs index f1a5e143ec..386fc2cbfa 100644 --- a/crates/health/src/processor/rack_leak.rs +++ b/crates/health/src/processor/rack_leak.rs @@ -152,6 +152,7 @@ mod tests { collector_type: "sensor_collector", metadata: None, rack_id: Some(RackId::new(rack)), + labels: Default::default(), } } @@ -166,6 +167,7 @@ mod tests { collector_type: "sensor_collector", metadata: None, rack_id: None, + labels: Default::default(), } } diff --git a/crates/health/src/sink/events.rs b/crates/health/src/sink/events.rs index 94bf3f0943..2abd643a8d 100644 --- a/crates/health/src/sink/events.rs +++ b/crates/health/src/sink/events.rs @@ -16,6 +16,7 @@ */ use std::borrow::Cow; +use std::collections::BTreeMap; use std::sync::Arc; use carbide_uuid::machine::MachineId; @@ -48,6 +49,7 @@ pub struct EventContext { pub collector_type: &'static str, pub metadata: Option, pub rack_id: Option, + pub labels: BTreeMap, } impl EventContext { @@ -58,6 +60,7 @@ impl EventContext { collector_type, metadata: endpoint.metadata.clone(), rack_id: endpoint.rack_id.clone(), + labels: endpoint.labels.clone(), } } @@ -65,6 +68,10 @@ impl EventContext { &self.endpoint_key } + pub fn labels(&self) -> &BTreeMap { + &self.labels + } + /// Returns machine metadata when this context belongs to a machine endpoint. fn machine_metadata(&self) -> Option<&MachineData> { let Some(EndpointMetadata::Machine(machine)) = self.metadata.as_ref() else { @@ -86,6 +93,12 @@ impl EventContext { .and_then(|machine| machine.machine_serial.as_deref()) } + /// Returns the UUID reported by the primary Redfish ComputerSystem. + pub fn system_uuid(&self) -> Option { + self.machine_metadata() + .and_then(|machine| machine.system_uuid.get()) + } + /// Returns the uniform GPU driver version when it is known for the machine. pub fn driver_version(&self) -> Option<&str> { self.machine_metadata() @@ -533,7 +546,7 @@ mod tests { use mac_address::MacAddress; use super::*; - use crate::endpoint::{MachineData, PowerShelfData, SwitchData}; + use crate::endpoint::{MachineData, PowerShelfData, SharedSystemUuid, SwitchData}; #[derive(Clone, Copy)] enum ContextKind { @@ -643,6 +656,7 @@ mod tests { ContextKind::Machine => Some(EndpointMetadata::Machine(MachineData { machine_id: Some(machine_id()), machine_serial: Some("MN-001".to_string()), + system_uuid: SharedSystemUuid::default(), slot_number: Some(7), tray_index: Some(3), nvlink_domain_uuid: Some(nvlink_domain_id()), @@ -670,9 +684,29 @@ mod tests { collector_type: "unit-test", metadata, rack_id: Some(RackId::new("rack-1")), + labels: Default::default(), } } + #[tokio::test] + async fn cloned_event_context_observes_later_system_uuid_resolution() { + let system_uuid = SharedSystemUuid::default(); + let mut context = context(ContextKind::Machine); + let Some(EndpointMetadata::Machine(machine)) = context.metadata.as_mut() else { + panic!("machine context"); + }; + machine.system_uuid = system_uuid.clone(); + let collector_context = context.clone(); + let expected = uuid::uuid!("4c4c4544-0044-4710-8052-cac04f4b4632"); + + system_uuid + .get_or_try_init(|| async { Ok::<_, std::convert::Infallible>(Some(expected)) }) + .await + .expect("infallible UUID initialization"); + + assert_eq!(collector_context.system_uuid(), Some(expected)); + } + fn summarize_context(context: EventContext) -> ContextSummary { ContextSummary { endpoint_key: context.endpoint_key().to_string(), diff --git a/crates/health/src/sink/health_report.rs b/crates/health/src/sink/health_report.rs index a0bb492a50..99c63458dd 100644 --- a/crates/health/src/sink/health_report.rs +++ b/crates/health/src/sink/health_report.rs @@ -250,7 +250,7 @@ mod tests { use mac_address::MacAddress; use super::*; - use crate::endpoint::{BmcAddr, EndpointMetadata, MachineData}; + use crate::endpoint::{BmcAddr, EndpointMetadata, MachineData, SharedSystemUuid}; use crate::sink::events::{Classification, HealthReportAlert, HealthReportSuccess, Probe}; fn machine_id(value: &str) -> MachineId { @@ -280,9 +280,11 @@ mod tests { mac: MacAddress::from_str("00:00:00:00:00:01").unwrap(), }, collector_type: "test", + labels: Default::default(), metadata: Some(EndpointMetadata::Machine(MachineData { machine_id: Some(id), machine_serial: None, + system_uuid: SharedSystemUuid::default(), slot_number: None, tray_index: None, nvlink_domain_uuid: None, diff --git a/crates/health/src/sink/log_file.rs b/crates/health/src/sink/log_file.rs index 246af6c212..8e40620b47 100644 --- a/crates/health/src/sink/log_file.rs +++ b/crates/health/src/sink/log_file.rs @@ -99,6 +99,8 @@ struct JsonLogRecord<'a> { #[serde(skip_serializing_if = "Option::is_none")] machine_id: Option, #[serde(skip_serializing_if = "Option::is_none")] + system_uuid: Option, + #[serde(skip_serializing_if = "Option::is_none")] machine_serial: Option<&'a str>, #[serde(skip_serializing_if = "Option::is_none")] driver_version: Option<&'a str>, @@ -106,6 +108,8 @@ struct JsonLogRecord<'a> { component_type: Option<&'static str>, #[serde(skip_serializing_if = "Option::is_none")] nvlink_domain_uuid: Option, + #[serde(skip_serializing_if = "std::collections::BTreeMap::is_empty")] + labels: &'a std::collections::BTreeMap, severity: &'a str, body: &'a str, #[serde(skip_serializing_if = "Vec::is_empty")] @@ -119,10 +123,12 @@ impl<'a> JsonLogRecord<'a> { endpoint: context.endpoint_key(), collector: context.collector_type, machine_id: context.machine_id().map(|id| id.to_string()), + system_uuid: context.system_uuid().map(|id| id.to_string()), machine_serial: context.machine_serial(), driver_version: context.driver_version(), component_type: context.component_type(), nvlink_domain_uuid: context.nvlink_domain_uuid().map(|id| id.to_string()), + labels: context.labels(), severity: &record.severity, body: &record.body, attributes: record @@ -290,12 +296,17 @@ mod tests { collector_type: "test", metadata: None, rack_id: None, + labels: Default::default(), } } /// Builds a log context with representative machine metadata. fn machine_context() -> EventContext { EventContext { + labels: std::collections::BTreeMap::from([( + "site".to_string(), + "rno-dev7".to_string(), + )]), metadata: Some(EndpointMetadata::Machine(MachineData { machine_id: Some( "fm100htjtiaehv1n5vh67tbmqq4eabcjdng40f7jupsadbedhruh6rag1l0" @@ -303,6 +314,7 @@ mod tests { .expect("valid machine id"), ), machine_serial: Some("MN-001".to_string()), + system_uuid: Some(uuid::uuid!("4c4c4544-0044-4710-8052-cac04f4b4632")).into(), slot_number: None, tray_index: None, nvlink_domain_uuid: Some(NvLinkDomainId::nil()), @@ -485,9 +497,14 @@ mod tests { parsed["machine_id"], "fm100htjtiaehv1n5vh67tbmqq4eabcjdng40f7jupsadbedhruh6rag1l0" ); + assert_eq!( + parsed["system_uuid"], + "4c4c4544-0044-4710-8052-cac04f4b4632" + ); assert_eq!(parsed["machine_serial"], "MN-001"); assert_eq!(parsed["driver_version"], "570.82"); assert_eq!(parsed["component_type"], "compute_node"); + assert_eq!(parsed["labels"]["site"], "rno-dev7"); assert_eq!( parsed["nvlink_domain_uuid"], "00000000-0000-0000-0000-000000000000" diff --git a/crates/health/src/sink/mod.rs b/crates/health/src/sink/mod.rs index a97633d85c..c6f390e394 100644 --- a/crates/health/src/sink/mod.rs +++ b/crates/health/src/sink/mod.rs @@ -132,7 +132,7 @@ mod tests { CollectorEvent, CompositeDataSink, DataSink, DiagnosticLogRecord, EventContext, LogRecord, MetricSample, PrometheusSink, }; - use crate::endpoint::{BmcAddr, EndpointMetadata, MachineData}; + use crate::endpoint::{BmcAddr, EndpointMetadata, MachineData, SharedSystemUuid}; use crate::metrics::MetricsManager; struct CountingSink { @@ -215,6 +215,7 @@ mod tests { collector_type: "test", metadata: None, rack_id: None, + labels: Default::default(), }; let event = CollectorEvent::Metric( @@ -263,6 +264,7 @@ mod tests { collector_type: "test", metadata: None, rack_id: None, + labels: Default::default(), }; let event = CollectorEvent::MetricCollectionStart; @@ -362,6 +364,10 @@ mod tests { mac: MacAddress::from_str("42:9e:b1:bd:9d:dd").unwrap(), }, collector_type: "test", + labels: std::collections::BTreeMap::from([( + "site".to_string(), + "rno-dev7".to_string(), + )]), metadata: Some(EndpointMetadata::Machine(MachineData { machine_id: Some( "fm100htjtiaehv1n5vh67tbmqq4eabcjdng40f7jupsadbedhruh6rag1l0" @@ -369,6 +375,7 @@ mod tests { .expect("valid machine id"), ), machine_serial: None, + system_uuid: SharedSystemUuid::default(), slot_number: None, tray_index: None, nvlink_domain_uuid: None, @@ -415,6 +422,7 @@ mod tests { .export_telemetry() .expect("telemetry export should work"); assert!(export_after_metric.contains("test_sink_hw_sensor_temperature_celsius")); + assert!(export_after_metric.contains("site=\"rno-dev7\"")); let service_metrics = metrics_manager .export_metrics() @@ -437,6 +445,7 @@ mod tests { mac: MacAddress::from_str("42:9e:b1:bd:9d:dd").unwrap(), }, collector_type: "sensor_collector", + labels: Default::default(), metadata: Some(EndpointMetadata::Machine(MachineData { machine_id: Some( "fm100htjtiaehv1n5vh67tbmqq4eabcjdng40f7jupsadbedhruh6rag1l0" @@ -444,6 +453,7 @@ mod tests { .expect("valid machine id"), ), machine_serial: None, + system_uuid: SharedSystemUuid::default(), slot_number: None, tray_index: None, nvlink_domain_uuid: None, @@ -495,6 +505,7 @@ mod tests { mac: MacAddress::from_str("42:9e:b1:bd:9d:dd").unwrap(), }, collector_type: "sensor_collector", + labels: Default::default(), metadata: Some(EndpointMetadata::Machine(MachineData { machine_id: Some( "fm100htjtiaehv1n5vh67tbmqq4eabcjdng40f7jupsadbedhruh6rag1l0" @@ -502,6 +513,7 @@ mod tests { .expect("valid machine id"), ), machine_serial: None, + system_uuid: SharedSystemUuid::default(), slot_number: None, tray_index: None, nvlink_domain_uuid: None, diff --git a/crates/health/src/sink/otlp.rs b/crates/health/src/sink/otlp.rs index 9fd37d0bbc..767130256b 100644 --- a/crates/health/src/sink/otlp.rs +++ b/crates/health/src/sink/otlp.rs @@ -292,6 +292,7 @@ mod tests { collector_type: "test", metadata: None, rack_id: None, + labels: Default::default(), } } diff --git a/crates/health/src/sink/prometheus.rs b/crates/health/src/sink/prometheus.rs index 01ead3a0fb..a3cfeba8eb 100644 --- a/crates/health/src/sink/prometheus.rs +++ b/crates/health/src/sink/prometheus.rs @@ -96,6 +96,9 @@ impl PrometheusSink { if let Some(machine_id) = context.machine_id() { labels.push((Cow::Borrowed("machine_id"), machine_id.to_string())); } + if let Some(system_uuid) = context.system_uuid() { + labels.push((Cow::Borrowed("system_uuid"), system_uuid.to_string())); + } if let Some(switch_id) = context.switch_id() { labels.push((Cow::Borrowed("switch_id"), switch_id.to_string())); } @@ -204,6 +207,12 @@ impl DataSink for PrometheusSink { } CollectorEvent::Metric(sample) => match self.get_or_create_stream_metrics(context) { Ok(stream_metrics) => { + let mut labels = sample.labels.clone(); + for (name, value) in context.labels() { + if labels.iter().all(|(existing, _)| existing.as_ref() != name) { + labels.push((Cow::Owned(name.clone()), value.clone())); + } + } stream_metrics.record( GaugeReading::new( Self::metric_reading_key(sample), @@ -212,7 +221,7 @@ impl DataSink for PrometheusSink { sample.unit.clone(), sample.value, ) - .with_labels(sample.labels.clone()), + .with_labels(labels), ); } Err(error) => { @@ -274,6 +283,7 @@ mod tests { mac: MacAddress::from_str("42:9e:b1:bd:9d:dd").unwrap(), }, collector_type: "sensor_collector", + labels: Default::default(), metadata: Some(EndpointMetadata::Machine(MachineData { machine_id: Some( "fm100htjtiaehv1n5vh67tbmqq4eabcjdng40f7jupsadbedhruh6rag1l0" @@ -281,6 +291,7 @@ mod tests { .expect("valid machine id"), ), machine_serial: Some("MN-001".to_string()), + system_uuid: Some(uuid::uuid!("4c4c4544-0044-4710-8052-cac04f4b4632")).into(), slot_number: Some(15), tray_index: Some(5), nvlink_domain_uuid: Some(NvLinkDomainId::nil()), @@ -301,6 +312,10 @@ mod tests { Some("fm100htjtiaehv1n5vh67tbmqq4eabcjdng40f7jupsadbedhruh6rag1l0") ); assert_eq!(label_value("serial_number"), Some("MN-001")); + assert_eq!( + label_value("system_uuid"), + Some("4c4c4544-0044-4710-8052-cac04f4b4632") + ); assert_eq!(label_value("rack_id"), Some("RACK_1")); assert_eq!(label_value("machine_slot_number"), Some("15")); assert_eq!(label_value("machine_tray_index"), Some("5")); @@ -322,6 +337,7 @@ mod tests { mac: MacAddress::from_str("11:22:33:44:55:66").unwrap(), }, collector_type: "switch_collector", + labels: Default::default(), metadata: Some(EndpointMetadata::Switch(SwitchData { id: Some(switch_id), serial: "SN-SWITCH-001".to_string(), diff --git a/crates/health/src/sink/tracing.rs b/crates/health/src/sink/tracing.rs index f412a07e68..18c1ed9e87 100644 --- a/crates/health/src/sink/tracing.rs +++ b/crates/health/src/sink/tracing.rs @@ -48,6 +48,8 @@ impl DataSink for TracingSink { tracing::info!( endpoint = %context.endpoint_key(), collector = %context.collector_type, + system_uuid = context.system_uuid().map(tracing::field::display), + labels = ?context.labels(), "Metric collection start" ); } @@ -55,6 +57,8 @@ impl DataSink for TracingSink { tracing::info!( endpoint = %context.endpoint_key(), collector = %context.collector_type, + system_uuid = context.system_uuid().map(tracing::field::display), + labels = ?context.labels(), metric = %metric.name, key = %metric.key, metric_type = %metric.metric_type, @@ -67,6 +71,8 @@ impl DataSink for TracingSink { tracing::info!( endpoint = %context.endpoint_key(), collector = %context.collector_type, + system_uuid = context.system_uuid().map(tracing::field::display), + labels = ?context.labels(), "Metric collection end" ); } @@ -74,6 +80,8 @@ impl DataSink for TracingSink { tracing::info!( endpoint = %context.endpoint_key(), collector = %context.collector_type, + system_uuid = context.system_uuid().map(tracing::field::display), + labels = ?context.labels(), "Collector removed" ); } @@ -84,10 +92,12 @@ impl DataSink for TracingSink { endpoint = %context.endpoint_key(), collector = %context.collector_type, machine_id = context.machine_id().map(tracing::field::display), + system_uuid = context.system_uuid().map(tracing::field::display), machine_serial = context.machine_serial(), driver_version = context.driver_version(), component_type = context.component_type(), nvlink_domain_uuid = context.nvlink_domain_uuid().map(tracing::field::display), + labels = ?context.labels(), severity = %record.severity, body = %record.body, attributes = ?record.attributes, @@ -98,6 +108,8 @@ impl DataSink for TracingSink { tracing::info!( endpoint = %context.endpoint_key(), collector = %context.collector_type, + system_uuid = context.system_uuid().map(tracing::field::display), + labels = ?context.labels(), firmware_name = %info.component, version = %info.version, "Firmware info event" @@ -108,6 +120,8 @@ impl DataSink for TracingSink { endpoint = %context.endpoint_key(), collector = %context.collector_type, machine_id = ?context.machine_id(), + system_uuid = context.system_uuid().map(tracing::field::display), + labels = ?context.labels(), success_count = report.successes.len(), alert_count = report.alerts.len(), alerts = ?report.alerts, diff --git a/docs/architecture/health_aggregation.md b/docs/architecture/health_aggregation.md index d2bfd071ee..5a7ed54e26 100644 --- a/docs/architecture/health_aggregation.md +++ b/docs/architecture/health_aggregation.md @@ -265,11 +265,23 @@ Finally, `nico-hw-health` also emits a health-rollup in `HealthReport` format to This assessed health status is built by comparing the metrics that are emitted from BMCs against well-defined ranges or by interpreting the `health_ok` values provided by BMCs. -For production deployments, `nico-hw-health` discovers machine, switch, and power-shelf BMC endpoints from NICo API via `[endpoint_sources.nico_api]`. Machine endpoints carry the inventory metadata needed to interpret hardware health in fleet context, including machine ID, serial number, rack ID, rack placement, and NVLink domain UUID when present. Switch endpoints carry switch ID, serial number, and rack placement when present. Local and test deployments can instead configure explicit machine, switch, or power-shelf identity with `[[endpoint_sources.static_bmc_endpoints]]`; static machine endpoints can include the same serial number, rack placement, and NVLink domain UUID metadata, static switch endpoints can include serial number and rack placement metadata, and all static endpoints can provide `rack_id` when rack-level rollups are needed. +**For production deployments**, `nico-hw-health` discovers machine, switch, and power-shelf BMC endpoints from NICo API via `[endpoint_sources.nico_api]`. + +Machine endpoints carry the inventory metadata needed to interpret hardware health in fleet context. This includes machine ID, primary Redfish system UUID, serial number, rack ID, rack placement, and NVLink domain UUID when present. + +Switch endpoints carry switch ID, serial number, and rack placement when present. + +**For local and test deployments**, you can configure explicit machine, switch, or power-shelf identity with `[[endpoint_sources.static_bmc_endpoints]]`. Note the following: + +- Static machine endpoints can include the same serial number, rack placement, and NVLink domain UUID metadata +- Static switch endpoints can include serial number and rack placement metadata +- All static endpoints can provide `rack_id` and validated custom telemetry `labels` +- The primary Redfish system UUID remains BMC-derived and cannot be overridden by a custom label The publishing sinks expose that inventory context using the conventions of the target backend: -- `[sinks.prometheus]` adds machine metadata as metric labels named `machine_id`, `serial_number`, `rack_id`, `machine_slot_number`, `machine_tray_index`, and `nvlink_domain_uuid`; switch metadata uses `switch_id`, `serial_number`, `rack_id`, `switch_slot_number`, and `switch_tray_index`. -- `[sinks.otlp]` adds machine metadata as OTLP resource attributes named `machine.id`, `rack.id`, integer `machine.slot_number`, integer `machine.tray_index`, and `nvlink.domain.uuid`; switch metadata uses `switch.id`, `rack.id`, integer `switch.slot_number`, and integer `switch.tray_index`. + +- `[sinks.prometheus]` adds _machine_ metadata as metric labels named `machine_id`, `system_uuid`, `serial_number`, `rack_id`, `machine_slot_number`, `machine_tray_index`, and `nvlink_domain_uuid` _Switch_ metadata labels are `switch_id`, `serial_number`, `rack_id`, `switch_slot_number`, and `switch_tray_index`. Static endpoint custom labels keep their configured names. +- `[sinks.otlp]` adds _machine_ metadata as OTLP resource attributes named `machine.id`, `system.uuid`, `rack.id`, integer `machine.slot_number`, integer `machine.tray_index`, and `nvlink.domain.uuid`. _Switch_ metadata labels are `switch.id`, `rack.id`, integer `switch.slot_number`, and integer `switch.tray_index`. Static endpoint custom labels keep their configured names. - `[sinks.health_report]`, `[sinks.rack_health_report]`, `[sinks.switch_health_report]`, and `[sinks.power_shelf_health_report]` use the same event context when submitting assessed health reports back to NICo API. The persisted `HealthReport` and `HealthProbeAlert` schemas remain the probe success/alert model described above. ### BMC inventory monitoring diff --git a/docs/operations/monitoring-health.md b/docs/operations/monitoring-health.md index 7b054b237f..658583d531 100644 --- a/docs/operations/monitoring-health.md +++ b/docs/operations/monitoring-health.md @@ -111,8 +111,15 @@ port = 443 mac = "aa:bb:cc:dd:ee:ff" username = "admin" password = "secret" +labels = { site = "rno-dev7", cluster = "cluster-01", environment = "development" } ``` +Static endpoints can define up to 32 custom telemetry labels. Label names must +match `[a-zA-Z_][a-zA-Z0-9_]*`. Names owned by the health service, such as +`machine_id`, `system_uuid`, `endpoint_ip`, and `collector_type`, are reserved. +Custom labels are exported as Prometheus labels and OTLP resource attributes, +as a `labels` object in JSONL logs, and as a structured map in tracing events. + Collector defaults from the example config: | Area | Parameter | Example value | Meaning |