From 966770de7224109f01ec0aaca926f40f117a9378 Mon Sep 17 00:00:00 2001 From: Jingxiang Zhang Date: Wed, 22 Jul 2026 11:31:23 -0700 Subject: [PATCH 1/6] feat(health): add primary system UUID to telemetry Signed-off-by: Jingxiang Zhang --- Cargo.lock | 1 + crates/health/Cargo.toml | 1 + crates/health/benches/collector_pipeline.rs | 1 + crates/health/benches/processor_pipeline.rs | 2 + crates/health/benches/sink_pipeline.rs | 1 + crates/health/src/api_client.rs | 1 + crates/health/src/discovery/identity.rs | 150 +++++++++++++++++++ crates/health/src/discovery/iteration.rs | 28 ++++ crates/health/src/discovery/mod.rs | 1 + crates/health/src/discovery/spawn.rs | 1 + crates/health/src/endpoint/model.rs | 6 + crates/health/src/endpoint/sources.rs | 1 + crates/health/src/otlp/convert.rs | 10 ++ crates/health/src/processor/health_report.rs | 1 + crates/health/src/sink/events.rs | 7 + crates/health/src/sink/health_report.rs | 1 + crates/health/src/sink/log_file.rs | 8 + crates/health/src/sink/mod.rs | 3 + crates/health/src/sink/prometheus.rs | 8 + crates/health/src/sink/tracing.rs | 7 + 20 files changed, 239 insertions(+) create mode 100644 crates/health/src/discovery/identity.rs diff --git a/Cargo.lock b/Cargo.lock index 7b2a6eb536..4505b68ab7 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2097,6 +2097,7 @@ dependencies = [ "tracing", "tracing-subscriber", "url", + "uuid", "x509-parser", ] diff --git a/crates/health/Cargo.toml b/crates/health/Cargo.toml index cc64fdfa2e..2e9b462818 100644 --- a/crates/health/Cargo.toml +++ b/crates/health/Cargo.toml @@ -89,6 +89,7 @@ nv-redfish = { workspace = true, features = [ ] } rand = { workspace = true } url = { workspace = true, features = ["serde"] } +uuid = { workspace = true } x509-parser = { workspace = true } # [local-dependencies] diff --git a/crates/health/benches/collector_pipeline.rs b/crates/health/benches/collector_pipeline.rs index 04511a8624..19065b916b 100644 --- a/crates/health/benches/collector_pipeline.rs +++ b/crates/health/benches/collector_pipeline.rs @@ -62,6 +62,7 @@ fn event_context() -> EventContext { metadata: Some(EndpointMetadata::Machine(MachineData { machine_id: Some(MACHINE_ID.parse().expect("valid machine id")), machine_serial: None, + machine_uuid: None, slot_number: None, tray_index: None, nvlink_domain_uuid: None, diff --git a/crates/health/benches/processor_pipeline.rs b/crates/health/benches/processor_pipeline.rs index 320828e3b2..8d126c30ee 100644 --- a/crates/health/benches/processor_pipeline.rs +++ b/crates/health/benches/processor_pipeline.rs @@ -98,6 +98,7 @@ fn event_context() -> EventContext { metadata: Some(EndpointMetadata::Machine(MachineData { machine_id: Some(MACHINE_ID.parse().expect("valid machine id")), machine_serial: None, + machine_uuid: None, slot_number: None, tray_index: None, nvlink_domain_uuid: None, @@ -276,6 +277,7 @@ fn rack_event_contexts(rack_id: &str, tray_count: usize) -> Vec { metadata: Some(EndpointMetadata::Machine(MachineData { machine_id: Some(MACHINE_ID.parse().expect("valid machine id")), machine_serial: None, + machine_uuid: None, slot_number: None, tray_index: None, nvlink_domain_uuid: None, diff --git a/crates/health/benches/sink_pipeline.rs b/crates/health/benches/sink_pipeline.rs index 49cd9bb5cd..64822691f9 100644 --- a/crates/health/benches/sink_pipeline.rs +++ b/crates/health/benches/sink_pipeline.rs @@ -72,6 +72,7 @@ fn event_context_for_machine(machine_id: &str) -> EventContext { metadata: Some(EndpointMetadata::Machine(MachineData { machine_id: Some(machine_id.parse().expect("valid machine id")), machine_serial: None, + machine_uuid: None, slot_number: None, tray_index: None, nvlink_domain_uuid: None, diff --git a/crates/health/src/api_client.rs b/crates/health/src/api_client.rs index 19ee762043..fc55d8902f 100644 --- a/crates/health/src/api_client.rs +++ b/crates/health/src/api_client.rs @@ -484,6 +484,7 @@ impl ApiEndpointSource { .as_ref() .and_then(|info| info.dmi_data.as_ref()) .map(|dmi| dmi.chassis_serial.clone()), + machine_uuid: None, slot_number: machine .placement_in_rack .as_ref() diff --git a/crates/health/src/discovery/identity.rs b/crates/health/src/discovery/identity.rs new file mode 100644 index 0000000000..fe0c46520d --- /dev/null +++ b/crates/health/src/discovery/identity.rs @@ -0,0 +1,150 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +use std::sync::Arc; + +use nv_redfish::ServiceRoot; + +use crate::HealthError; +use crate::endpoint::{BmcEndpoint, EndpointMetadata}; + +struct SystemIdentity { + id: String, + uuid: Option, + bios_version: Option, +} + +fn select_primary_system(systems: &[SystemIdentity]) -> Option<&SystemIdentity> { + systems + .iter() + .find(|system| { + system + .bios_version + .as_deref() + .is_some_and(|version| !version.trim().is_empty()) + }) + .or_else(|| systems.first()) +} + +/// Resolves the primary ComputerSystem UUID before collectors start. +/// +/// A system with a non-empty BIOS version is preferred because BMCs may expose +/// auxiliary systems alongside the host. This matches the primary-system rule +/// used by Fleet Intelligence inventory; when no system has BIOS metadata, the +/// first collection member is used. +pub(super) async fn with_primary_system_uuid( + endpoint: &Arc, +) -> Result, HealthError> { + if !matches!(endpoint.metadata, Some(EndpointMetadata::Machine(_))) { + return Ok(Arc::clone(endpoint)); + } + + let root = ServiceRoot::new(Arc::clone(endpoint.bmc())).await?; + let systems = root.systems().await?.ok_or_else(|| { + HealthError::GenericError(format!( + "BMC {:?} does not expose a ComputerSystem collection", + endpoint.addr + )) + })?; + let systems = systems.members().await?; + let identities: Vec = systems + .iter() + .map(|system| { + let raw = system.raw(); + SystemIdentity { + id: raw.base.id.clone(), + uuid: raw.uuid.flatten(), + bios_version: raw.bios_version.clone().flatten(), + } + }) + .collect(); + let primary = select_primary_system(&identities).ok_or_else(|| { + HealthError::GenericError(format!( + "BMC {:?} exposes an empty ComputerSystem collection", + endpoint.addr + )) + })?; + let machine_uuid = primary.uuid.ok_or_else(|| { + HealthError::GenericError(format!( + "primary ComputerSystem {} on BMC {:?} does not expose a UUID", + primary.id, endpoint.addr + )) + })?; + + let mut enriched = endpoint.as_ref().clone(); + let Some(EndpointMetadata::Machine(machine)) = enriched.metadata.as_mut() else { + unreachable!("machine endpoint checked above") + }; + machine.machine_uuid = Some(machine_uuid); + + Ok(Arc::new(enriched)) +} + +#[cfg(test)] +mod tests { + use super::*; + + const FIRST_UUID: uuid::Uuid = uuid::uuid!("11111111-1111-1111-1111-111111111111"); + const BIOS_UUID: uuid::Uuid = uuid::uuid!("22222222-2222-2222-2222-222222222222"); + + #[test] + fn primary_system_prefers_first_system_with_bios() { + let systems = [ + SystemIdentity { + id: "auxiliary".to_string(), + uuid: Some(FIRST_UUID), + bios_version: None, + }, + SystemIdentity { + id: "host".to_string(), + uuid: Some(BIOS_UUID), + bios_version: Some("1.2.3".to_string()), + }, + ]; + + let primary = select_primary_system(&systems).expect("primary system"); + + assert_eq!(primary.id, "host"); + assert_eq!(primary.uuid, Some(BIOS_UUID)); + } + + #[test] + fn primary_system_falls_back_to_first_member() { + let systems = [ + SystemIdentity { + id: "first".to_string(), + uuid: Some(FIRST_UUID), + bios_version: None, + }, + SystemIdentity { + id: "second".to_string(), + uuid: Some(BIOS_UUID), + bios_version: Some(" ".to_string()), + }, + ]; + + let primary = select_primary_system(&systems).expect("primary system"); + + assert_eq!(primary.id, "first"); + assert_eq!(primary.uuid, Some(FIRST_UUID)); + } + + #[test] + fn primary_system_is_absent_for_empty_collection() { + assert!(select_primary_system(&[]).is_none()); + } +} diff --git a/crates/health/src/discovery/iteration.rs b/crates/health/src/discovery/iteration.rs index 47d22fc53c..8f40cdcb24 100644 --- a/crates/health/src/discovery/iteration.rs +++ b/crates/health/src/discovery/iteration.rs @@ -20,9 +20,12 @@ use std::collections::HashSet; use std::sync::Arc; use std::time::Instant; +use futures::{StreamExt, stream}; + use super::DiscoveryIterationStats; use super::cleanup::{stop_ineligible_nmxc_collectors, stop_removed_bmc_collectors}; use super::context::{CollectorKind, DiscoveryLoopContext}; +use super::identity::with_primary_system_uuid; use super::spawn::{spawn_collectors_for_endpoint, switch_supports_nmxc_subscription}; use crate::HealthError; use crate::config::Configurable; @@ -74,6 +77,31 @@ pub async fn run_discovery_iteration( .cloned() .collect(); + // Resolve machine identity before any collector can emit. Failed endpoints + // are omitted from this pass and retried by the next discovery iteration, + // preserving the invariant that machine telemetry always carries the + // primary ComputerSystem UUID. + let identity_concurrency = ctx.discovery_config.discovery_concurrency.max(1); + let resolved_endpoints: Vec>> = stream::iter(sharded_endpoints) + .map(|endpoint| async move { + match with_primary_system_uuid(&endpoint).await { + Ok(endpoint) => Some(endpoint), + Err(error) => { + tracing::warn!( + ?error, + bmc_address = ?endpoint.addr, + "Could not resolve primary ComputerSystem UUID; skipping endpoint" + ); + None + } + } + }) + .buffer_unordered(identity_concurrency) + .collect() + .await; + let sharded_endpoints: Vec> = + resolved_endpoints.into_iter().flatten().collect(); + if sharded_endpoints.is_empty() { tracing::warn!("No endpoints assigned to this shard"); } else { diff --git a/crates/health/src/discovery/mod.rs b/crates/health/src/discovery/mod.rs index d14d6c5af0..778f721c68 100644 --- a/crates/health/src/discovery/mod.rs +++ b/crates/health/src/discovery/mod.rs @@ -17,6 +17,7 @@ mod cleanup; mod context; +mod identity; mod iteration; mod spawn; diff --git a/crates/health/src/discovery/spawn.rs b/crates/health/src/discovery/spawn.rs index 157cc431d7..4ed390ea00 100644 --- a/crates/health/src/discovery/spawn.rs +++ b/crates/health/src/discovery/spawn.rs @@ -788,6 +788,7 @@ mod tests { .expect("valid machine id"), ), machine_serial: None, + machine_uuid: None, slot_number: None, tray_index: None, nvlink_domain_uuid: None, diff --git a/crates/health/src/endpoint/model.rs b/crates/health/src/endpoint/model.rs index a88b972f41..9da7e45131 100644 --- a/crates/health/src/endpoint/model.rs +++ b/crates/health/src/endpoint/model.rs @@ -126,6 +126,12 @@ pub struct MachineData { /// Hardware chassis serial discovered from machine DMI data, when known. pub machine_serial: Option, + /// UUID reported by the primary Redfish ComputerSystem resource. + /// + /// Endpoint discovery resolves this before machine collectors start so every + /// telemetry event can carry the same hardware identity. + pub machine_uuid: Option, + /// Physical rack slot where the machine is installed, when known. pub slot_number: Option, diff --git a/crates/health/src/endpoint/sources.rs b/crates/health/src/endpoint/sources.rs index 5e90aa3d57..2006291a4b 100644 --- a/crates/health/src/endpoint/sources.rs +++ b/crates/health/src/endpoint/sources.rs @@ -152,6 +152,7 @@ impl StaticEndpointSource { Some(EndpointMetadata::Machine(MachineData { machine_id, machine_serial: machine.serial.clone(), + machine_uuid: None, slot_number: machine.slot_number, tray_index: machine.tray_index, nvlink_domain_uuid, diff --git a/crates/health/src/otlp/convert.rs b/crates/health/src/otlp/convert.rs index 93b8cd295c..f9c406be8d 100644 --- a/crates/health/src/otlp/convert.rs +++ b/crates/health/src/otlp/convert.rs @@ -88,6 +88,9 @@ fn resource_attributes(context: &EventContext) -> Vec { if let Some(machine_id) = context.machine_id() { attrs.push(kv("machine.id", machine_id.to_string())); } + if let Some(machine_uuid) = context.machine_uuid() { + attrs.push(kv("machine.uuid", machine_uuid.to_string())); + } if let Some(machine_serial) = context.machine_serial() { attrs.push(kv("machine.serial", machine_serial.to_string())); } @@ -406,6 +409,7 @@ mod tests { .expect("valid machine id"), ), machine_serial: Some("MN-001".to_string()), + machine_uuid: Some(uuid::uuid!("4c4c4544-0044-4710-8052-cac04f4b4632")), slot_number: Some(15), tray_index: Some(5), nvlink_domain_uuid: Some(domain_uuid), @@ -417,6 +421,10 @@ mod tests { let attrs = resource_attributes(&context); assert_eq!(attr_value(&attrs, "rack.id"), Some("RACK_1")); + assert_eq!( + attr_value(&attrs, "machine.uuid"), + Some("4c4c4544-0044-4710-8052-cac04f4b4632") + ); assert_eq!(attr_value(&attrs, "machine.serial"), Some("MN-001")); assert_eq!(attr_value(&attrs, "driver.version"), Some("570.82")); assert_eq!(attr_value(&attrs, "component.type"), Some("compute_node")); @@ -442,6 +450,7 @@ mod tests { metadata: Some(EndpointMetadata::Machine(MachineData { machine_id: None, machine_serial: None, + machine_uuid: None, slot_number: None, tray_index: None, nvlink_domain_uuid: None, @@ -455,6 +464,7 @@ mod tests { assert_eq!(attr_value(&attrs, "machine.id"), None); assert_eq!(attr_value(&attrs, "component.type"), Some("compute_node")); assert_eq!(attr_value(&attrs, "machine.serial"), None); + assert_eq!(attr_value(&attrs, "machine.uuid"), None); assert_eq!(attr_value(&attrs, "driver.version"), None); assert_eq!(attr_value(&attrs, "nvlink.domain.uuid"), None); } diff --git a/crates/health/src/processor/health_report.rs b/crates/health/src/processor/health_report.rs index cb4fa96ad3..c12abb0274 100644 --- a/crates/health/src/processor/health_report.rs +++ b/crates/health/src/processor/health_report.rs @@ -287,6 +287,7 @@ mod tests { .expect("valid machine id"), ), machine_serial: None, + machine_uuid: None, slot_number: None, tray_index: None, nvlink_domain_uuid: None, diff --git a/crates/health/src/sink/events.rs b/crates/health/src/sink/events.rs index 94bf3f0943..d7358f0550 100644 --- a/crates/health/src/sink/events.rs +++ b/crates/health/src/sink/events.rs @@ -86,6 +86,12 @@ impl EventContext { .and_then(|machine| machine.machine_serial.as_deref()) } + /// Returns the UUID reported by the primary Redfish ComputerSystem. + pub fn machine_uuid(&self) -> Option { + self.machine_metadata() + .and_then(|machine| machine.machine_uuid) + } + /// Returns the uniform GPU driver version when it is known for the machine. pub fn driver_version(&self) -> Option<&str> { self.machine_metadata() @@ -643,6 +649,7 @@ mod tests { ContextKind::Machine => Some(EndpointMetadata::Machine(MachineData { machine_id: Some(machine_id()), machine_serial: Some("MN-001".to_string()), + machine_uuid: None, slot_number: Some(7), tray_index: Some(3), nvlink_domain_uuid: Some(nvlink_domain_id()), diff --git a/crates/health/src/sink/health_report.rs b/crates/health/src/sink/health_report.rs index a0bb492a50..8098bfc921 100644 --- a/crates/health/src/sink/health_report.rs +++ b/crates/health/src/sink/health_report.rs @@ -283,6 +283,7 @@ mod tests { metadata: Some(EndpointMetadata::Machine(MachineData { machine_id: Some(id), machine_serial: None, + machine_uuid: None, slot_number: None, tray_index: None, nvlink_domain_uuid: None, diff --git a/crates/health/src/sink/log_file.rs b/crates/health/src/sink/log_file.rs index 246af6c212..73ae64beb6 100644 --- a/crates/health/src/sink/log_file.rs +++ b/crates/health/src/sink/log_file.rs @@ -99,6 +99,8 @@ struct JsonLogRecord<'a> { #[serde(skip_serializing_if = "Option::is_none")] machine_id: Option, #[serde(skip_serializing_if = "Option::is_none")] + machine_uuid: Option, + #[serde(skip_serializing_if = "Option::is_none")] machine_serial: Option<&'a str>, #[serde(skip_serializing_if = "Option::is_none")] driver_version: Option<&'a str>, @@ -119,6 +121,7 @@ impl<'a> JsonLogRecord<'a> { endpoint: context.endpoint_key(), collector: context.collector_type, machine_id: context.machine_id().map(|id| id.to_string()), + machine_uuid: context.machine_uuid().map(|id| id.to_string()), machine_serial: context.machine_serial(), driver_version: context.driver_version(), component_type: context.component_type(), @@ -303,6 +306,7 @@ mod tests { .expect("valid machine id"), ), machine_serial: Some("MN-001".to_string()), + machine_uuid: Some(uuid::uuid!("4c4c4544-0044-4710-8052-cac04f4b4632")), slot_number: None, tray_index: None, nvlink_domain_uuid: Some(NvLinkDomainId::nil()), @@ -485,6 +489,10 @@ mod tests { parsed["machine_id"], "fm100htjtiaehv1n5vh67tbmqq4eabcjdng40f7jupsadbedhruh6rag1l0" ); + assert_eq!( + parsed["machine_uuid"], + "4c4c4544-0044-4710-8052-cac04f4b4632" + ); assert_eq!(parsed["machine_serial"], "MN-001"); assert_eq!(parsed["driver_version"], "570.82"); assert_eq!(parsed["component_type"], "compute_node"); diff --git a/crates/health/src/sink/mod.rs b/crates/health/src/sink/mod.rs index a97633d85c..00387ad733 100644 --- a/crates/health/src/sink/mod.rs +++ b/crates/health/src/sink/mod.rs @@ -369,6 +369,7 @@ mod tests { .expect("valid machine id"), ), machine_serial: None, + machine_uuid: None, slot_number: None, tray_index: None, nvlink_domain_uuid: None, @@ -444,6 +445,7 @@ mod tests { .expect("valid machine id"), ), machine_serial: None, + machine_uuid: None, slot_number: None, tray_index: None, nvlink_domain_uuid: None, @@ -502,6 +504,7 @@ mod tests { .expect("valid machine id"), ), machine_serial: None, + machine_uuid: None, slot_number: None, tray_index: None, nvlink_domain_uuid: None, diff --git a/crates/health/src/sink/prometheus.rs b/crates/health/src/sink/prometheus.rs index 01ead3a0fb..b4fa35551d 100644 --- a/crates/health/src/sink/prometheus.rs +++ b/crates/health/src/sink/prometheus.rs @@ -96,6 +96,9 @@ impl PrometheusSink { if let Some(machine_id) = context.machine_id() { labels.push((Cow::Borrowed("machine_id"), machine_id.to_string())); } + if let Some(machine_uuid) = context.machine_uuid() { + labels.push((Cow::Borrowed("machine_uuid"), machine_uuid.to_string())); + } if let Some(switch_id) = context.switch_id() { labels.push((Cow::Borrowed("switch_id"), switch_id.to_string())); } @@ -281,6 +284,7 @@ mod tests { .expect("valid machine id"), ), machine_serial: Some("MN-001".to_string()), + machine_uuid: Some(uuid::uuid!("4c4c4544-0044-4710-8052-cac04f4b4632")), slot_number: Some(15), tray_index: Some(5), nvlink_domain_uuid: Some(NvLinkDomainId::nil()), @@ -301,6 +305,10 @@ mod tests { Some("fm100htjtiaehv1n5vh67tbmqq4eabcjdng40f7jupsadbedhruh6rag1l0") ); assert_eq!(label_value("serial_number"), Some("MN-001")); + assert_eq!( + label_value("machine_uuid"), + Some("4c4c4544-0044-4710-8052-cac04f4b4632") + ); assert_eq!(label_value("rack_id"), Some("RACK_1")); assert_eq!(label_value("machine_slot_number"), Some("15")); assert_eq!(label_value("machine_tray_index"), Some("5")); diff --git a/crates/health/src/sink/tracing.rs b/crates/health/src/sink/tracing.rs index f412a07e68..667d7ae7ef 100644 --- a/crates/health/src/sink/tracing.rs +++ b/crates/health/src/sink/tracing.rs @@ -48,6 +48,7 @@ impl DataSink for TracingSink { tracing::info!( endpoint = %context.endpoint_key(), collector = %context.collector_type, + machine_uuid = context.machine_uuid().map(tracing::field::display), "Metric collection start" ); } @@ -55,6 +56,7 @@ impl DataSink for TracingSink { tracing::info!( endpoint = %context.endpoint_key(), collector = %context.collector_type, + machine_uuid = context.machine_uuid().map(tracing::field::display), metric = %metric.name, key = %metric.key, metric_type = %metric.metric_type, @@ -67,6 +69,7 @@ impl DataSink for TracingSink { tracing::info!( endpoint = %context.endpoint_key(), collector = %context.collector_type, + machine_uuid = context.machine_uuid().map(tracing::field::display), "Metric collection end" ); } @@ -74,6 +77,7 @@ impl DataSink for TracingSink { tracing::info!( endpoint = %context.endpoint_key(), collector = %context.collector_type, + machine_uuid = context.machine_uuid().map(tracing::field::display), "Collector removed" ); } @@ -84,6 +88,7 @@ impl DataSink for TracingSink { endpoint = %context.endpoint_key(), collector = %context.collector_type, machine_id = context.machine_id().map(tracing::field::display), + machine_uuid = context.machine_uuid().map(tracing::field::display), machine_serial = context.machine_serial(), driver_version = context.driver_version(), component_type = context.component_type(), @@ -98,6 +103,7 @@ impl DataSink for TracingSink { tracing::info!( endpoint = %context.endpoint_key(), collector = %context.collector_type, + machine_uuid = context.machine_uuid().map(tracing::field::display), firmware_name = %info.component, version = %info.version, "Firmware info event" @@ -108,6 +114,7 @@ impl DataSink for TracingSink { endpoint = %context.endpoint_key(), collector = %context.collector_type, machine_id = ?context.machine_id(), + machine_uuid = context.machine_uuid().map(tracing::field::display), success_count = report.successes.len(), alert_count = report.alerts.len(), alerts = ?report.alerts, From 43adfca2c2dccd776e1e0ce715a3aa3ede551dc2 Mon Sep 17 00:00:00 2001 From: Jingxiang Zhang Date: Wed, 22 Jul 2026 11:41:19 -0700 Subject: [PATCH 2/6] refactor(health): name hardware identity system UUID Signed-off-by: Jingxiang Zhang --- crates/health/benches/collector_pipeline.rs | 2 +- crates/health/benches/processor_pipeline.rs | 4 ++-- crates/health/benches/sink_pipeline.rs | 2 +- crates/health/src/api_client.rs | 2 +- crates/health/src/discovery/identity.rs | 4 ++-- crates/health/src/discovery/spawn.rs | 2 +- crates/health/src/endpoint/model.rs | 2 +- crates/health/src/endpoint/sources.rs | 2 +- crates/health/src/otlp/convert.rs | 12 ++++++------ crates/health/src/processor/health_report.rs | 2 +- crates/health/src/sink/events.rs | 6 +++--- crates/health/src/sink/health_report.rs | 2 +- crates/health/src/sink/log_file.rs | 8 ++++---- crates/health/src/sink/mod.rs | 6 +++--- crates/health/src/sink/prometheus.rs | 8 ++++---- crates/health/src/sink/tracing.rs | 14 +++++++------- 16 files changed, 39 insertions(+), 39 deletions(-) diff --git a/crates/health/benches/collector_pipeline.rs b/crates/health/benches/collector_pipeline.rs index 19065b916b..4d7e7a2564 100644 --- a/crates/health/benches/collector_pipeline.rs +++ b/crates/health/benches/collector_pipeline.rs @@ -62,7 +62,7 @@ fn event_context() -> EventContext { metadata: Some(EndpointMetadata::Machine(MachineData { machine_id: Some(MACHINE_ID.parse().expect("valid machine id")), machine_serial: None, - machine_uuid: None, + system_uuid: None, slot_number: None, tray_index: None, nvlink_domain_uuid: None, diff --git a/crates/health/benches/processor_pipeline.rs b/crates/health/benches/processor_pipeline.rs index 8d126c30ee..0ec1b2215b 100644 --- a/crates/health/benches/processor_pipeline.rs +++ b/crates/health/benches/processor_pipeline.rs @@ -98,7 +98,7 @@ fn event_context() -> EventContext { metadata: Some(EndpointMetadata::Machine(MachineData { machine_id: Some(MACHINE_ID.parse().expect("valid machine id")), machine_serial: None, - machine_uuid: None, + system_uuid: None, slot_number: None, tray_index: None, nvlink_domain_uuid: None, @@ -277,7 +277,7 @@ fn rack_event_contexts(rack_id: &str, tray_count: usize) -> Vec { metadata: Some(EndpointMetadata::Machine(MachineData { machine_id: Some(MACHINE_ID.parse().expect("valid machine id")), machine_serial: None, - machine_uuid: None, + system_uuid: None, slot_number: None, tray_index: None, nvlink_domain_uuid: None, diff --git a/crates/health/benches/sink_pipeline.rs b/crates/health/benches/sink_pipeline.rs index 64822691f9..cb7655e97c 100644 --- a/crates/health/benches/sink_pipeline.rs +++ b/crates/health/benches/sink_pipeline.rs @@ -72,7 +72,7 @@ fn event_context_for_machine(machine_id: &str) -> EventContext { metadata: Some(EndpointMetadata::Machine(MachineData { machine_id: Some(machine_id.parse().expect("valid machine id")), machine_serial: None, - machine_uuid: None, + system_uuid: None, slot_number: None, tray_index: None, nvlink_domain_uuid: None, diff --git a/crates/health/src/api_client.rs b/crates/health/src/api_client.rs index fc55d8902f..d93dc55e00 100644 --- a/crates/health/src/api_client.rs +++ b/crates/health/src/api_client.rs @@ -484,7 +484,7 @@ impl ApiEndpointSource { .as_ref() .and_then(|info| info.dmi_data.as_ref()) .map(|dmi| dmi.chassis_serial.clone()), - machine_uuid: None, + system_uuid: None, slot_number: machine .placement_in_rack .as_ref() diff --git a/crates/health/src/discovery/identity.rs b/crates/health/src/discovery/identity.rs index fe0c46520d..726ca972ee 100644 --- a/crates/health/src/discovery/identity.rs +++ b/crates/health/src/discovery/identity.rs @@ -78,7 +78,7 @@ pub(super) async fn with_primary_system_uuid( endpoint.addr )) })?; - let machine_uuid = primary.uuid.ok_or_else(|| { + let system_uuid = primary.uuid.ok_or_else(|| { HealthError::GenericError(format!( "primary ComputerSystem {} on BMC {:?} does not expose a UUID", primary.id, endpoint.addr @@ -89,7 +89,7 @@ pub(super) async fn with_primary_system_uuid( let Some(EndpointMetadata::Machine(machine)) = enriched.metadata.as_mut() else { unreachable!("machine endpoint checked above") }; - machine.machine_uuid = Some(machine_uuid); + machine.system_uuid = Some(system_uuid); Ok(Arc::new(enriched)) } diff --git a/crates/health/src/discovery/spawn.rs b/crates/health/src/discovery/spawn.rs index 4ed390ea00..44438b2ebc 100644 --- a/crates/health/src/discovery/spawn.rs +++ b/crates/health/src/discovery/spawn.rs @@ -788,7 +788,7 @@ mod tests { .expect("valid machine id"), ), machine_serial: None, - machine_uuid: None, + system_uuid: None, slot_number: None, tray_index: None, nvlink_domain_uuid: None, diff --git a/crates/health/src/endpoint/model.rs b/crates/health/src/endpoint/model.rs index 9da7e45131..3e03d79519 100644 --- a/crates/health/src/endpoint/model.rs +++ b/crates/health/src/endpoint/model.rs @@ -130,7 +130,7 @@ pub struct MachineData { /// /// Endpoint discovery resolves this before machine collectors start so every /// telemetry event can carry the same hardware identity. - pub machine_uuid: Option, + pub system_uuid: Option, /// Physical rack slot where the machine is installed, when known. pub slot_number: Option, diff --git a/crates/health/src/endpoint/sources.rs b/crates/health/src/endpoint/sources.rs index 2006291a4b..2e085f48ac 100644 --- a/crates/health/src/endpoint/sources.rs +++ b/crates/health/src/endpoint/sources.rs @@ -152,7 +152,7 @@ impl StaticEndpointSource { Some(EndpointMetadata::Machine(MachineData { machine_id, machine_serial: machine.serial.clone(), - machine_uuid: None, + system_uuid: None, slot_number: machine.slot_number, tray_index: machine.tray_index, nvlink_domain_uuid, diff --git a/crates/health/src/otlp/convert.rs b/crates/health/src/otlp/convert.rs index f9c406be8d..ca1896b681 100644 --- a/crates/health/src/otlp/convert.rs +++ b/crates/health/src/otlp/convert.rs @@ -88,8 +88,8 @@ fn resource_attributes(context: &EventContext) -> Vec { if let Some(machine_id) = context.machine_id() { attrs.push(kv("machine.id", machine_id.to_string())); } - if let Some(machine_uuid) = context.machine_uuid() { - attrs.push(kv("machine.uuid", machine_uuid.to_string())); + if let Some(system_uuid) = context.system_uuid() { + attrs.push(kv("system.uuid", system_uuid.to_string())); } if let Some(machine_serial) = context.machine_serial() { attrs.push(kv("machine.serial", machine_serial.to_string())); @@ -409,7 +409,7 @@ mod tests { .expect("valid machine id"), ), machine_serial: Some("MN-001".to_string()), - machine_uuid: Some(uuid::uuid!("4c4c4544-0044-4710-8052-cac04f4b4632")), + system_uuid: Some(uuid::uuid!("4c4c4544-0044-4710-8052-cac04f4b4632")), slot_number: Some(15), tray_index: Some(5), nvlink_domain_uuid: Some(domain_uuid), @@ -422,7 +422,7 @@ mod tests { assert_eq!(attr_value(&attrs, "rack.id"), Some("RACK_1")); assert_eq!( - attr_value(&attrs, "machine.uuid"), + attr_value(&attrs, "system.uuid"), Some("4c4c4544-0044-4710-8052-cac04f4b4632") ); assert_eq!(attr_value(&attrs, "machine.serial"), Some("MN-001")); @@ -450,7 +450,7 @@ mod tests { metadata: Some(EndpointMetadata::Machine(MachineData { machine_id: None, machine_serial: None, - machine_uuid: None, + system_uuid: None, slot_number: None, tray_index: None, nvlink_domain_uuid: None, @@ -464,7 +464,7 @@ mod tests { assert_eq!(attr_value(&attrs, "machine.id"), None); assert_eq!(attr_value(&attrs, "component.type"), Some("compute_node")); assert_eq!(attr_value(&attrs, "machine.serial"), None); - assert_eq!(attr_value(&attrs, "machine.uuid"), None); + assert_eq!(attr_value(&attrs, "system.uuid"), None); assert_eq!(attr_value(&attrs, "driver.version"), None); assert_eq!(attr_value(&attrs, "nvlink.domain.uuid"), None); } diff --git a/crates/health/src/processor/health_report.rs b/crates/health/src/processor/health_report.rs index c12abb0274..19a6d13a03 100644 --- a/crates/health/src/processor/health_report.rs +++ b/crates/health/src/processor/health_report.rs @@ -287,7 +287,7 @@ mod tests { .expect("valid machine id"), ), machine_serial: None, - machine_uuid: None, + system_uuid: None, slot_number: None, tray_index: None, nvlink_domain_uuid: None, diff --git a/crates/health/src/sink/events.rs b/crates/health/src/sink/events.rs index d7358f0550..105ea27ae0 100644 --- a/crates/health/src/sink/events.rs +++ b/crates/health/src/sink/events.rs @@ -87,9 +87,9 @@ impl EventContext { } /// Returns the UUID reported by the primary Redfish ComputerSystem. - pub fn machine_uuid(&self) -> Option { + pub fn system_uuid(&self) -> Option { self.machine_metadata() - .and_then(|machine| machine.machine_uuid) + .and_then(|machine| machine.system_uuid) } /// Returns the uniform GPU driver version when it is known for the machine. @@ -649,7 +649,7 @@ mod tests { ContextKind::Machine => Some(EndpointMetadata::Machine(MachineData { machine_id: Some(machine_id()), machine_serial: Some("MN-001".to_string()), - machine_uuid: None, + system_uuid: None, slot_number: Some(7), tray_index: Some(3), nvlink_domain_uuid: Some(nvlink_domain_id()), diff --git a/crates/health/src/sink/health_report.rs b/crates/health/src/sink/health_report.rs index 8098bfc921..5665c915a9 100644 --- a/crates/health/src/sink/health_report.rs +++ b/crates/health/src/sink/health_report.rs @@ -283,7 +283,7 @@ mod tests { metadata: Some(EndpointMetadata::Machine(MachineData { machine_id: Some(id), machine_serial: None, - machine_uuid: None, + system_uuid: None, slot_number: None, tray_index: None, nvlink_domain_uuid: None, diff --git a/crates/health/src/sink/log_file.rs b/crates/health/src/sink/log_file.rs index 73ae64beb6..31c1591510 100644 --- a/crates/health/src/sink/log_file.rs +++ b/crates/health/src/sink/log_file.rs @@ -99,7 +99,7 @@ struct JsonLogRecord<'a> { #[serde(skip_serializing_if = "Option::is_none")] machine_id: Option, #[serde(skip_serializing_if = "Option::is_none")] - machine_uuid: Option, + system_uuid: Option, #[serde(skip_serializing_if = "Option::is_none")] machine_serial: Option<&'a str>, #[serde(skip_serializing_if = "Option::is_none")] @@ -121,7 +121,7 @@ impl<'a> JsonLogRecord<'a> { endpoint: context.endpoint_key(), collector: context.collector_type, machine_id: context.machine_id().map(|id| id.to_string()), - machine_uuid: context.machine_uuid().map(|id| id.to_string()), + system_uuid: context.system_uuid().map(|id| id.to_string()), machine_serial: context.machine_serial(), driver_version: context.driver_version(), component_type: context.component_type(), @@ -306,7 +306,7 @@ mod tests { .expect("valid machine id"), ), machine_serial: Some("MN-001".to_string()), - machine_uuid: Some(uuid::uuid!("4c4c4544-0044-4710-8052-cac04f4b4632")), + system_uuid: Some(uuid::uuid!("4c4c4544-0044-4710-8052-cac04f4b4632")), slot_number: None, tray_index: None, nvlink_domain_uuid: Some(NvLinkDomainId::nil()), @@ -490,7 +490,7 @@ mod tests { "fm100htjtiaehv1n5vh67tbmqq4eabcjdng40f7jupsadbedhruh6rag1l0" ); assert_eq!( - parsed["machine_uuid"], + parsed["system_uuid"], "4c4c4544-0044-4710-8052-cac04f4b4632" ); assert_eq!(parsed["machine_serial"], "MN-001"); diff --git a/crates/health/src/sink/mod.rs b/crates/health/src/sink/mod.rs index 00387ad733..1b82e35832 100644 --- a/crates/health/src/sink/mod.rs +++ b/crates/health/src/sink/mod.rs @@ -369,7 +369,7 @@ mod tests { .expect("valid machine id"), ), machine_serial: None, - machine_uuid: None, + system_uuid: None, slot_number: None, tray_index: None, nvlink_domain_uuid: None, @@ -445,7 +445,7 @@ mod tests { .expect("valid machine id"), ), machine_serial: None, - machine_uuid: None, + system_uuid: None, slot_number: None, tray_index: None, nvlink_domain_uuid: None, @@ -504,7 +504,7 @@ mod tests { .expect("valid machine id"), ), machine_serial: None, - machine_uuid: None, + system_uuid: None, slot_number: None, tray_index: None, nvlink_domain_uuid: None, diff --git a/crates/health/src/sink/prometheus.rs b/crates/health/src/sink/prometheus.rs index b4fa35551d..b5eff9a693 100644 --- a/crates/health/src/sink/prometheus.rs +++ b/crates/health/src/sink/prometheus.rs @@ -96,8 +96,8 @@ impl PrometheusSink { if let Some(machine_id) = context.machine_id() { labels.push((Cow::Borrowed("machine_id"), machine_id.to_string())); } - if let Some(machine_uuid) = context.machine_uuid() { - labels.push((Cow::Borrowed("machine_uuid"), machine_uuid.to_string())); + if let Some(system_uuid) = context.system_uuid() { + labels.push((Cow::Borrowed("system_uuid"), system_uuid.to_string())); } if let Some(switch_id) = context.switch_id() { labels.push((Cow::Borrowed("switch_id"), switch_id.to_string())); @@ -284,7 +284,7 @@ mod tests { .expect("valid machine id"), ), machine_serial: Some("MN-001".to_string()), - machine_uuid: Some(uuid::uuid!("4c4c4544-0044-4710-8052-cac04f4b4632")), + system_uuid: Some(uuid::uuid!("4c4c4544-0044-4710-8052-cac04f4b4632")), slot_number: Some(15), tray_index: Some(5), nvlink_domain_uuid: Some(NvLinkDomainId::nil()), @@ -306,7 +306,7 @@ mod tests { ); assert_eq!(label_value("serial_number"), Some("MN-001")); assert_eq!( - label_value("machine_uuid"), + label_value("system_uuid"), Some("4c4c4544-0044-4710-8052-cac04f4b4632") ); assert_eq!(label_value("rack_id"), Some("RACK_1")); diff --git a/crates/health/src/sink/tracing.rs b/crates/health/src/sink/tracing.rs index 667d7ae7ef..b45df6a656 100644 --- a/crates/health/src/sink/tracing.rs +++ b/crates/health/src/sink/tracing.rs @@ -48,7 +48,7 @@ impl DataSink for TracingSink { tracing::info!( endpoint = %context.endpoint_key(), collector = %context.collector_type, - machine_uuid = context.machine_uuid().map(tracing::field::display), + system_uuid = context.system_uuid().map(tracing::field::display), "Metric collection start" ); } @@ -56,7 +56,7 @@ impl DataSink for TracingSink { tracing::info!( endpoint = %context.endpoint_key(), collector = %context.collector_type, - machine_uuid = context.machine_uuid().map(tracing::field::display), + system_uuid = context.system_uuid().map(tracing::field::display), metric = %metric.name, key = %metric.key, metric_type = %metric.metric_type, @@ -69,7 +69,7 @@ impl DataSink for TracingSink { tracing::info!( endpoint = %context.endpoint_key(), collector = %context.collector_type, - machine_uuid = context.machine_uuid().map(tracing::field::display), + system_uuid = context.system_uuid().map(tracing::field::display), "Metric collection end" ); } @@ -77,7 +77,7 @@ impl DataSink for TracingSink { tracing::info!( endpoint = %context.endpoint_key(), collector = %context.collector_type, - machine_uuid = context.machine_uuid().map(tracing::field::display), + system_uuid = context.system_uuid().map(tracing::field::display), "Collector removed" ); } @@ -88,7 +88,7 @@ impl DataSink for TracingSink { endpoint = %context.endpoint_key(), collector = %context.collector_type, machine_id = context.machine_id().map(tracing::field::display), - machine_uuid = context.machine_uuid().map(tracing::field::display), + system_uuid = context.system_uuid().map(tracing::field::display), machine_serial = context.machine_serial(), driver_version = context.driver_version(), component_type = context.component_type(), @@ -103,7 +103,7 @@ impl DataSink for TracingSink { tracing::info!( endpoint = %context.endpoint_key(), collector = %context.collector_type, - machine_uuid = context.machine_uuid().map(tracing::field::display), + system_uuid = context.system_uuid().map(tracing::field::display), firmware_name = %info.component, version = %info.version, "Firmware info event" @@ -114,7 +114,7 @@ impl DataSink for TracingSink { endpoint = %context.endpoint_key(), collector = %context.collector_type, machine_id = ?context.machine_id(), - machine_uuid = context.machine_uuid().map(tracing::field::display), + system_uuid = context.system_uuid().map(tracing::field::display), success_count = report.successes.len(), alert_count = report.alerts.len(), alerts = ?report.alerts, From cbf10f062f1c08cf54c87daafc97fb45cefff13c Mon Sep 17 00:00:00 2001 From: Jingxiang Zhang Date: Wed, 22 Jul 2026 12:03:00 -0700 Subject: [PATCH 3/6] feat(health): support static endpoint telemetry labels Signed-off-by: Jingxiang Zhang --- crates/health/benches/collector_pipeline.rs | 1 + crates/health/benches/processor_pipeline.rs | 2 + crates/health/benches/sink_pipeline.rs | 1 + crates/health/example/config.example.toml | 1 + crates/health/src/api_client.rs | 1 + .../nvue/gnmi/on_change_processor.rs | 2 + .../collectors/nvue/gnmi/sample_processor.rs | 2 + .../src/collectors/nvue/rest/collector.rs | 1 + crates/health/src/config.rs | 91 ++++++++++++++++++- crates/health/src/discovery/spawn.rs | 1 + crates/health/src/endpoint/cluster.rs | 1 + crates/health/src/endpoint/mod.rs | 1 + crates/health/src/endpoint/model.rs | 2 + crates/health/src/endpoint/sources.rs | 12 +++ crates/health/src/otlp/convert.rs | 19 ++++ crates/health/src/processor/health_report.rs | 1 + .../health/src/processor/intrusion_events.rs | 1 + crates/health/src/processor/leak_events.rs | 1 + crates/health/src/processor/mod.rs | 1 + crates/health/src/processor/rack_leak.rs | 2 + crates/health/src/sink/events.rs | 8 ++ crates/health/src/sink/health_report.rs | 1 + crates/health/src/sink/log_file.rs | 9 ++ crates/health/src/sink/mod.rs | 9 ++ crates/health/src/sink/otlp.rs | 1 + crates/health/src/sink/prometheus.rs | 10 +- crates/health/src/sink/tracing.rs | 7 ++ docs/architecture/health_aggregation.md | 6 +- docs/operations/monitoring-health.md | 7 ++ 29 files changed, 197 insertions(+), 5 deletions(-) diff --git a/crates/health/benches/collector_pipeline.rs b/crates/health/benches/collector_pipeline.rs index 4d7e7a2564..6ec89c2d43 100644 --- a/crates/health/benches/collector_pipeline.rs +++ b/crates/health/benches/collector_pipeline.rs @@ -59,6 +59,7 @@ fn event_context() -> EventContext { mac: MacAddress::from_str("42:9e:b1:bd:9d:dd").unwrap(), }, collector_type: "sensor_collector", + labels: Default::default(), metadata: Some(EndpointMetadata::Machine(MachineData { machine_id: Some(MACHINE_ID.parse().expect("valid machine id")), machine_serial: None, diff --git a/crates/health/benches/processor_pipeline.rs b/crates/health/benches/processor_pipeline.rs index 0ec1b2215b..a7d6727c2d 100644 --- a/crates/health/benches/processor_pipeline.rs +++ b/crates/health/benches/processor_pipeline.rs @@ -95,6 +95,7 @@ fn event_context() -> EventContext { mac: MacAddress::from_str("42:9e:b1:bd:9d:dd").unwrap(), }, collector_type: "sensor_collector", + labels: Default::default(), metadata: Some(EndpointMetadata::Machine(MachineData { machine_id: Some(MACHINE_ID.parse().expect("valid machine id")), machine_serial: None, @@ -274,6 +275,7 @@ fn rack_event_contexts(rack_id: &str, tray_count: usize) -> Vec { mac: MacAddress::from_str(&mac).unwrap(), }, collector_type: "sensor_collector", + labels: Default::default(), metadata: Some(EndpointMetadata::Machine(MachineData { machine_id: Some(MACHINE_ID.parse().expect("valid machine id")), machine_serial: None, diff --git a/crates/health/benches/sink_pipeline.rs b/crates/health/benches/sink_pipeline.rs index cb7655e97c..a34d97d8b4 100644 --- a/crates/health/benches/sink_pipeline.rs +++ b/crates/health/benches/sink_pipeline.rs @@ -69,6 +69,7 @@ fn event_context_for_machine(machine_id: &str) -> EventContext { mac: MacAddress::from_str("42:9e:b1:bd:9d:dd").unwrap(), }, collector_type: "sensor_collector", + labels: Default::default(), metadata: Some(EndpointMetadata::Machine(MachineData { machine_id: Some(machine_id.parse().expect("valid machine id")), machine_serial: None, diff --git a/crates/health/example/config.example.toml b/crates/health/example/config.example.toml index 959e698334..436a464e3d 100644 --- a/crates/health/example/config.example.toml +++ b/crates/health/example/config.example.toml @@ -40,6 +40,7 @@ port = 443 mac = "aa:bb:cc:dd:ee:ff" username = "admin" password = "secret" +labels = { site = "dev", cluster = "cluster-01", environment = "development" } machine = { id = "fm100htjtiaehv1n5vh67tbmqq4eabcjdng40f7jupsadbedhruh6rag1l0", serial = "MN-001", slot_number = 15, tray_index = 5, nvlink_domain_uuid = "00000000-0000-0000-0000-000000000000" } [[endpoint_sources.static_bmc_endpoints]] diff --git a/crates/health/src/api_client.rs b/crates/health/src/api_client.rs index d93dc55e00..8006029de0 100644 --- a/crates/health/src/api_client.rs +++ b/crates/health/src/api_client.rs @@ -609,6 +609,7 @@ impl ApiEndpointSource { addr, metadata, rack_id, + labels: Default::default(), bmc, })) } diff --git a/crates/health/src/collectors/nvue/gnmi/on_change_processor.rs b/crates/health/src/collectors/nvue/gnmi/on_change_processor.rs index f8fd4786fa..647da3c540 100644 --- a/crates/health/src/collectors/nvue/gnmi/on_change_processor.rs +++ b/crates/health/src/collectors/nvue/gnmi/on_change_processor.rs @@ -383,6 +383,7 @@ mod tests { collector_type, metadata: None, rack_id: None, + labels: Default::default(), } } @@ -779,6 +780,7 @@ mod tests { mac: MacAddress::from_str("AA:BB:CC:DD:EE:FF").unwrap(), }, collector_type: ON_CHANGE_STREAM_ID_SYSTEM_EVENTS, + labels: Default::default(), metadata: Some(EndpointMetadata::Switch(SwitchData { id: Some(switch_id), serial: "SN-SWITCH-001".to_string(), diff --git a/crates/health/src/collectors/nvue/gnmi/sample_processor.rs b/crates/health/src/collectors/nvue/gnmi/sample_processor.rs index b140d6c564..1bb9fd2e02 100644 --- a/crates/health/src/collectors/nvue/gnmi/sample_processor.rs +++ b/crates/health/src/collectors/nvue/gnmi/sample_processor.rs @@ -1102,6 +1102,7 @@ mod tests { collector_type: NVUE_GNMI_SAMPLE_STREAM_ID, metadata: None, rack_id: None, + labels: Default::default(), }; GnmiSampleProcessor { data_sink: None, @@ -1167,6 +1168,7 @@ mod tests { mac: MacAddress::from_str("AA:BB:CC:DD:EE:FF").unwrap(), }, collector_type: NVUE_GNMI_SAMPLE_STREAM_ID, + labels: Default::default(), metadata: Some(EndpointMetadata::Switch(SwitchData { id: Some(switch_id), serial: "SN-SWITCH-001".to_string(), diff --git a/crates/health/src/collectors/nvue/rest/collector.rs b/crates/health/src/collectors/nvue/rest/collector.rs index 82ed5b3b6b..a4ec8d159d 100644 --- a/crates/health/src/collectors/nvue/rest/collector.rs +++ b/crates/health/src/collectors/nvue/rest/collector.rs @@ -1260,6 +1260,7 @@ mod tests { collector_type: COLLECTOR_NAME, metadata: None, rack_id: None, + labels: Default::default(), }; NvueRestCollector { diff --git a/crates/health/src/config.rs b/crates/health/src/config.rs index 617f7a092a..cb52afba8d 100644 --- a/crates/health/src/config.rs +++ b/crates/health/src/config.rs @@ -15,7 +15,7 @@ * limitations under the License. */ -use std::collections::HashSet; +use std::collections::{BTreeMap, HashSet}; use std::fmt::Debug; use std::net::{IpAddr, SocketAddr}; use std::path::{Path, PathBuf}; @@ -201,6 +201,9 @@ pub struct StaticBmcEndpoint { pub power_shelf: Option, pub switch: Option, pub rack_id: Option, + /// User-defined labels attached to telemetry emitted for this endpoint. + #[serde(default)] + pub labels: BTreeMap, } #[derive(Clone, Debug, serde::Deserialize, serde::Serialize)] @@ -272,6 +275,7 @@ impl Debug for StaticBmcEndpoint { .field("power_shelf", &self.power_shelf) .field("switch", &self.switch) .field("rack_id", &self.rack_id) + .field("labels", &self.labels) .finish() } } @@ -308,6 +312,52 @@ impl StaticBmcEndpoint { )); } + const RESERVED_LABELS: &[&str] = &[ + "collector_type", + "endpoint_ip", + "endpoint_key", + "endpoint_mac", + "machine_id", + "machine_slot_number", + "machine_tray_index", + "nvlink_domain_uuid", + "rack_id", + "serial_number", + "switch_id", + "switch_slot_number", + "switch_tray_index", + "system_uuid", + ]; + + if self.labels.len() > 32 { + return Err(format!( + "endpoint_sources.static_bmc_endpoints[{index}].labels supports at most 32 labels" + )); + } + + for (name, value) in &self.labels { + let mut chars = name.chars(); + let valid_start = chars + .next() + .is_some_and(|ch| ch.is_ascii_alphabetic() || ch == '_'); + let valid_rest = chars.all(|ch| ch.is_ascii_alphanumeric() || ch == '_'); + if !valid_start || !valid_rest { + return Err(format!( + "endpoint_sources.static_bmc_endpoints[{index}].labels key {name:?} must match [a-zA-Z_][a-zA-Z0-9_]*" + )); + } + if RESERVED_LABELS.contains(&name.as_str()) { + return Err(format!( + "endpoint_sources.static_bmc_endpoints[{index}].labels key {name:?} is reserved" + )); + } + if value.len() > 1024 { + return Err(format!( + "endpoint_sources.static_bmc_endpoints[{index}].labels value for {name:?} exceeds 1024 bytes" + )); + } + } + Ok(()) } } @@ -1847,6 +1897,7 @@ mod tests { power_shelf: None, switch: None, rack_id: None, + labels: BTreeMap::new(), } } @@ -3837,6 +3888,7 @@ ip = "10.0.1.2" mac = "11:22:33:44:55:11" username = "admin" password = "pass" +labels = { site = "rno-dev7", environment = "development" } machine = { id = "fm100htjtiaehv1n5vh67tbmqq4eabcjdng40f7jupsadbedhruh6rag1l0", serial = "MN-001", driver_version = "570.82", slot_number = 15, tray_index = 5, nvlink_domain_uuid = "00000000-0000-0000-0000-000000000000" } "#; @@ -3851,6 +3903,15 @@ machine = { id = "fm100htjtiaehv1n5vh67tbmqq4eabcjdng40f7jupsadbedhruh6rag1l0", .as_ref() .expect("machine metadata"); + config.validate().expect("valid custom labels"); + assert_eq!( + config.endpoint_sources.static_bmc_endpoints[0] + .labels + .get("site") + .map(String::as_str), + Some("rno-dev7") + ); + assert_eq!(machine.slot_number, Some(15)); assert_eq!(machine.tray_index, Some(5)); assert_eq!(machine.driver_version.as_deref(), Some("570.82")); @@ -3860,6 +3921,34 @@ machine = { id = "fm100htjtiaehv1n5vh67tbmqq4eabcjdng40f7jupsadbedhruh6rag1l0", ); } + #[test] + fn test_static_endpoint_rejects_invalid_or_reserved_label_names() { + for (name, expected) in [("bad-label", "must match"), ("system_uuid", "is reserved")] { + let toml_content = format!( + r#" +[endpoint_sources.carbide_api] +enabled = false + +[[endpoint_sources.static_bmc_endpoints]] +ip = "10.0.1.2" +mac = "11:22:33:44:55:11" +username = "admin" +password = "pass" +labels = {{ {name} = "value" }} +machine = {{ id = "fm100htjtiaehv1n5vh67tbmqq4eabcjdng40f7jupsadbedhruh6rag1l0" }} +"# + ); + let config: Config = Figment::new() + .merge(Serialized::defaults(Config::default())) + .merge(Toml::string(&toml_content)) + .extract() + .expect("label syntax should parse"); + + let error = config.validate().expect_err("label should be rejected"); + assert!(error.contains(expected), "unexpected error: {error}"); + } + } + #[test] fn test_static_endpoints_accept_position_field_aliases() { let toml_content = r#" diff --git a/crates/health/src/discovery/spawn.rs b/crates/health/src/discovery/spawn.rs index 44438b2ebc..2fcd1ac69d 100644 --- a/crates/health/src/discovery/spawn.rs +++ b/crates/health/src/discovery/spawn.rs @@ -1219,6 +1219,7 @@ mod tests { "failing-switch-host", )), rack_id: None, + labels: Default::default(), bmc, }); diff --git a/crates/health/src/endpoint/cluster.rs b/crates/health/src/endpoint/cluster.rs index 73473b027d..5e1e6ffbe5 100644 --- a/crates/health/src/endpoint/cluster.rs +++ b/crates/health/src/endpoint/cluster.rs @@ -442,6 +442,7 @@ fn build_endpoints( addr, metadata: None, rack_id: node.rack.as_deref().map(RackId::new), + labels: Default::default(), bmc, })); } diff --git a/crates/health/src/endpoint/mod.rs b/crates/health/src/endpoint/mod.rs index 60106db4fc..8ae1f1cd57 100644 --- a/crates/health/src/endpoint/mod.rs +++ b/crates/health/src/endpoint/mod.rs @@ -61,6 +61,7 @@ pub(crate) mod test_support { addr, metadata, rack_id, + labels: Default::default(), bmc, } } diff --git a/crates/health/src/endpoint/model.rs b/crates/health/src/endpoint/model.rs index 3e03d79519..42d5dd841d 100644 --- a/crates/health/src/endpoint/model.rs +++ b/crates/health/src/endpoint/model.rs @@ -16,6 +16,7 @@ */ use std::borrow::Cow; +use std::collections::BTreeMap; use std::net::IpAddr; use std::sync::Arc; @@ -35,6 +36,7 @@ pub struct BmcEndpoint { pub addr: BmcAddr, pub metadata: Option, pub rack_id: Option, + pub labels: BTreeMap, pub bmc: Arc, } diff --git a/crates/health/src/endpoint/sources.rs b/crates/health/src/endpoint/sources.rs index 2e085f48ac..d3adf0f7fa 100644 --- a/crates/health/src/endpoint/sources.rs +++ b/crates/health/src/endpoint/sources.rs @@ -193,6 +193,7 @@ impl StaticEndpointSource { addr, metadata, rack_id: cfg.rack_id.as_ref().map(|id| RackId::new(id.as_str())), + labels: cfg.labels.clone(), bmc, }; endpoints.push(Arc::new(endpoint)); @@ -289,6 +290,7 @@ mod tests { power_shelf: None, switch: None, rack_id: None, + labels: Default::default(), }, StaticBmcEndpoint { ip: ip("10.0.0.2"), @@ -300,6 +302,7 @@ mod tests { power_shelf: None, switch: None, rack_id: None, + labels: Default::default(), }, ]; @@ -335,6 +338,7 @@ mod tests { nmxt_enabled: None, }), rack_id: None, + labels: Default::default(), }]; let source = StaticEndpointSource::from_config(&configs, &reqwest(), None, 10); @@ -372,6 +376,7 @@ mod tests { }), switch: None, rack_id: None, + labels: Default::default(), }]; let source = StaticEndpointSource::from_config(&configs, &reqwest(), None, 10); @@ -412,6 +417,7 @@ mod tests { power_shelf: None, switch: None, rack_id: Some("RACK_1".to_string()), + labels: std::collections::BTreeMap::from([("site".to_string(), "dev".to_string())]), }]; let source = StaticEndpointSource::from_config(&configs, &reqwest(), None, 10); @@ -425,6 +431,10 @@ mod tests { .map(|rack_id| rack_id.as_str()), Some("RACK_1") ); + assert_eq!( + endpoints[0].labels.get("site").map(String::as_str), + Some("dev") + ); match &endpoints[0].metadata { Some(EndpointMetadata::Machine(machine)) => { assert_eq!(machine.machine_id, Some(machine_id)); @@ -457,6 +467,7 @@ mod tests { power_shelf: None, switch: None, rack_id: None, + labels: Default::default(), }]; let source = StaticEndpointSource::from_config(&configs, &reqwest(), None, 10); @@ -483,6 +494,7 @@ mod tests { power_shelf: None, switch: None, rack_id: None, + labels: Default::default(), }]; let source = StaticEndpointSource::from_config(&configs, &reqwest(), None, 10); diff --git a/crates/health/src/otlp/convert.rs b/crates/health/src/otlp/convert.rs index ca1896b681..d264e54e93 100644 --- a/crates/health/src/otlp/convert.rs +++ b/crates/health/src/otlp/convert.rs @@ -139,6 +139,12 @@ fn resource_attributes(context: &EventContext) -> Vec { if let Some(tray) = context.switch_tray_index() { attrs.push(int_kv("switch.tray_index", i64::from(tray))); } + attrs.extend( + context + .labels() + .iter() + .map(|(name, value)| kv(name, value.clone())), + ); attrs } @@ -348,6 +354,7 @@ mod tests { collector_type: "test", metadata: None, rack_id: None, + labels: Default::default(), } } @@ -402,6 +409,10 @@ mod tests { mac: MacAddress::from_str("42:9e:b1:bd:9d:dd").expect("valid mac"), }, collector_type: "test", + labels: std::collections::BTreeMap::from([( + "site".to_string(), + "rno-dev7".to_string(), + )]), metadata: Some(EndpointMetadata::Machine(MachineData { machine_id: Some( "fm100htjtiaehv1n5vh67tbmqq4eabcjdng40f7jupsadbedhruh6rag1l0" @@ -421,6 +432,7 @@ mod tests { let attrs = resource_attributes(&context); assert_eq!(attr_value(&attrs, "rack.id"), Some("RACK_1")); + assert_eq!(attr_value(&attrs, "site"), Some("rno-dev7")); assert_eq!( attr_value(&attrs, "system.uuid"), Some("4c4c4544-0044-4710-8052-cac04f4b4632") @@ -447,6 +459,7 @@ mod tests { mac: MacAddress::from_str("42:9e:b1:bd:9d:dd").expect("valid mac"), }, collector_type: "test", + labels: Default::default(), metadata: Some(EndpointMetadata::Machine(MachineData { machine_id: None, machine_serial: None, @@ -481,6 +494,7 @@ mod tests { mac: MacAddress::from_str("11:22:33:44:55:66").expect("valid mac"), }, collector_type: "test", + labels: Default::default(), metadata: Some(EndpointMetadata::Switch(SwitchData { id: Some(switch_id), serial: "SN-SWITCH-001".to_string(), @@ -518,6 +532,7 @@ mod tests { mac: MacAddress::from_str("11:22:33:44:55:66").expect("valid mac"), }, collector_type: "nvue_gnmi", + labels: Default::default(), metadata: Some(EndpointMetadata::Switch(SwitchData { id: Some(switch_id), serial: "SN-SWITCH-001".to_string(), @@ -570,6 +585,7 @@ mod tests { mac: MacAddress::from_str("22:33:44:55:66:77").expect("valid mac"), }, collector_type: "sensor_collector", + labels: Default::default(), metadata: Some(EndpointMetadata::Switch(SwitchData { id: Some(switch_id), serial: "SN-SWITCH-BMC-001".to_string(), @@ -619,6 +635,7 @@ mod tests { mac: MacAddress::from_str("33:44:55:66:77:88").expect("valid mac"), }, collector_type: "sensor_collector", + labels: Default::default(), metadata: Some(EndpointMetadata::PowerShelf(PowerShelfData { id: Some(power_shelf_id), serial: "SN-PS-001".to_string(), @@ -745,6 +762,7 @@ mod tests { collector_type: "test", metadata: None, rack_id: None, + labels: Default::default(), }; let ctx2 = EventContext { endpoint_key: "endpoint-b".to_string(), @@ -853,6 +871,7 @@ mod tests { mac: MacAddress::from_str("11:22:33:44:55:66").expect("valid mac"), }, collector_type: "nvue_gnmi", + labels: Default::default(), metadata: Some(EndpointMetadata::Switch(SwitchData { id: Some(switch_id), serial: "SN-SWITCH-001".to_string(), diff --git a/crates/health/src/processor/health_report.rs b/crates/health/src/processor/health_report.rs index 19a6d13a03..ce813db6db 100644 --- a/crates/health/src/processor/health_report.rs +++ b/crates/health/src/processor/health_report.rs @@ -280,6 +280,7 @@ mod tests { mac: MacAddress::from_str("42:9e:b1:bd:9d:dd").expect("valid mac"), }, collector_type: "sensor_collector", + labels: Default::default(), metadata: Some(EndpointMetadata::Machine(MachineData { machine_id: Some( "fm100htjtiaehv1n5vh67tbmqq4eabcjdng40f7jupsadbedhruh6rag1l0" diff --git a/crates/health/src/processor/intrusion_events.rs b/crates/health/src/processor/intrusion_events.rs index f1c82afd07..f652b7ea78 100644 --- a/crates/health/src/processor/intrusion_events.rs +++ b/crates/health/src/processor/intrusion_events.rs @@ -189,6 +189,7 @@ mod tests { collector_type: "logs_collector", metadata: None, rack_id: None, + labels: Default::default(), } } diff --git a/crates/health/src/processor/leak_events.rs b/crates/health/src/processor/leak_events.rs index ca59005b24..5d8a67ddbe 100644 --- a/crates/health/src/processor/leak_events.rs +++ b/crates/health/src/processor/leak_events.rs @@ -161,6 +161,7 @@ mod tests { collector_type: "leak_detector_collector", metadata: None, rack_id: None, + labels: Default::default(), } } diff --git a/crates/health/src/processor/mod.rs b/crates/health/src/processor/mod.rs index 0560dd0d6b..f0f2877f5d 100644 --- a/crates/health/src/processor/mod.rs +++ b/crates/health/src/processor/mod.rs @@ -193,6 +193,7 @@ mod tests { collector_type: "test", metadata: None, rack_id: None, + labels: Default::default(), } } diff --git a/crates/health/src/processor/rack_leak.rs b/crates/health/src/processor/rack_leak.rs index f1a5e143ec..386fc2cbfa 100644 --- a/crates/health/src/processor/rack_leak.rs +++ b/crates/health/src/processor/rack_leak.rs @@ -152,6 +152,7 @@ mod tests { collector_type: "sensor_collector", metadata: None, rack_id: Some(RackId::new(rack)), + labels: Default::default(), } } @@ -166,6 +167,7 @@ mod tests { collector_type: "sensor_collector", metadata: None, rack_id: None, + labels: Default::default(), } } diff --git a/crates/health/src/sink/events.rs b/crates/health/src/sink/events.rs index 105ea27ae0..07e491750f 100644 --- a/crates/health/src/sink/events.rs +++ b/crates/health/src/sink/events.rs @@ -16,6 +16,7 @@ */ use std::borrow::Cow; +use std::collections::BTreeMap; use std::sync::Arc; use carbide_uuid::machine::MachineId; @@ -48,6 +49,7 @@ pub struct EventContext { pub collector_type: &'static str, pub metadata: Option, pub rack_id: Option, + pub labels: BTreeMap, } impl EventContext { @@ -58,6 +60,7 @@ impl EventContext { collector_type, metadata: endpoint.metadata.clone(), rack_id: endpoint.rack_id.clone(), + labels: endpoint.labels.clone(), } } @@ -65,6 +68,10 @@ impl EventContext { &self.endpoint_key } + pub fn labels(&self) -> &BTreeMap { + &self.labels + } + /// Returns machine metadata when this context belongs to a machine endpoint. fn machine_metadata(&self) -> Option<&MachineData> { let Some(EndpointMetadata::Machine(machine)) = self.metadata.as_ref() else { @@ -677,6 +684,7 @@ mod tests { collector_type: "unit-test", metadata, rack_id: Some(RackId::new("rack-1")), + labels: Default::default(), } } diff --git a/crates/health/src/sink/health_report.rs b/crates/health/src/sink/health_report.rs index 5665c915a9..8cfbdf7903 100644 --- a/crates/health/src/sink/health_report.rs +++ b/crates/health/src/sink/health_report.rs @@ -280,6 +280,7 @@ mod tests { mac: MacAddress::from_str("00:00:00:00:00:01").unwrap(), }, collector_type: "test", + labels: Default::default(), metadata: Some(EndpointMetadata::Machine(MachineData { machine_id: Some(id), machine_serial: None, diff --git a/crates/health/src/sink/log_file.rs b/crates/health/src/sink/log_file.rs index 31c1591510..9fba62c623 100644 --- a/crates/health/src/sink/log_file.rs +++ b/crates/health/src/sink/log_file.rs @@ -108,6 +108,8 @@ struct JsonLogRecord<'a> { component_type: Option<&'static str>, #[serde(skip_serializing_if = "Option::is_none")] nvlink_domain_uuid: Option, + #[serde(skip_serializing_if = "std::collections::BTreeMap::is_empty")] + labels: &'a std::collections::BTreeMap, severity: &'a str, body: &'a str, #[serde(skip_serializing_if = "Vec::is_empty")] @@ -126,6 +128,7 @@ impl<'a> JsonLogRecord<'a> { driver_version: context.driver_version(), component_type: context.component_type(), nvlink_domain_uuid: context.nvlink_domain_uuid().map(|id| id.to_string()), + labels: context.labels(), severity: &record.severity, body: &record.body, attributes: record @@ -293,12 +296,17 @@ mod tests { collector_type: "test", metadata: None, rack_id: None, + labels: Default::default(), } } /// Builds a log context with representative machine metadata. fn machine_context() -> EventContext { EventContext { + labels: std::collections::BTreeMap::from([( + "site".to_string(), + "rno-dev7".to_string(), + )]), metadata: Some(EndpointMetadata::Machine(MachineData { machine_id: Some( "fm100htjtiaehv1n5vh67tbmqq4eabcjdng40f7jupsadbedhruh6rag1l0" @@ -496,6 +504,7 @@ mod tests { assert_eq!(parsed["machine_serial"], "MN-001"); assert_eq!(parsed["driver_version"], "570.82"); assert_eq!(parsed["component_type"], "compute_node"); + assert_eq!(parsed["labels"]["site"], "rno-dev7"); assert_eq!( parsed["nvlink_domain_uuid"], "00000000-0000-0000-0000-000000000000" diff --git a/crates/health/src/sink/mod.rs b/crates/health/src/sink/mod.rs index 1b82e35832..9dfbe54496 100644 --- a/crates/health/src/sink/mod.rs +++ b/crates/health/src/sink/mod.rs @@ -215,6 +215,7 @@ mod tests { collector_type: "test", metadata: None, rack_id: None, + labels: Default::default(), }; let event = CollectorEvent::Metric( @@ -263,6 +264,7 @@ mod tests { collector_type: "test", metadata: None, rack_id: None, + labels: Default::default(), }; let event = CollectorEvent::MetricCollectionStart; @@ -362,6 +364,10 @@ mod tests { mac: MacAddress::from_str("42:9e:b1:bd:9d:dd").unwrap(), }, collector_type: "test", + labels: std::collections::BTreeMap::from([( + "site".to_string(), + "rno-dev7".to_string(), + )]), metadata: Some(EndpointMetadata::Machine(MachineData { machine_id: Some( "fm100htjtiaehv1n5vh67tbmqq4eabcjdng40f7jupsadbedhruh6rag1l0" @@ -416,6 +422,7 @@ mod tests { .export_telemetry() .expect("telemetry export should work"); assert!(export_after_metric.contains("test_sink_hw_sensor_temperature_celsius")); + assert!(export_after_metric.contains("site=\"rno-dev7\"")); let service_metrics = metrics_manager .export_metrics() @@ -438,6 +445,7 @@ mod tests { mac: MacAddress::from_str("42:9e:b1:bd:9d:dd").unwrap(), }, collector_type: "sensor_collector", + labels: Default::default(), metadata: Some(EndpointMetadata::Machine(MachineData { machine_id: Some( "fm100htjtiaehv1n5vh67tbmqq4eabcjdng40f7jupsadbedhruh6rag1l0" @@ -497,6 +505,7 @@ mod tests { mac: MacAddress::from_str("42:9e:b1:bd:9d:dd").unwrap(), }, collector_type: "sensor_collector", + labels: Default::default(), metadata: Some(EndpointMetadata::Machine(MachineData { machine_id: Some( "fm100htjtiaehv1n5vh67tbmqq4eabcjdng40f7jupsadbedhruh6rag1l0" diff --git a/crates/health/src/sink/otlp.rs b/crates/health/src/sink/otlp.rs index 9fd37d0bbc..767130256b 100644 --- a/crates/health/src/sink/otlp.rs +++ b/crates/health/src/sink/otlp.rs @@ -292,6 +292,7 @@ mod tests { collector_type: "test", metadata: None, rack_id: None, + labels: Default::default(), } } diff --git a/crates/health/src/sink/prometheus.rs b/crates/health/src/sink/prometheus.rs index b5eff9a693..e99580f78c 100644 --- a/crates/health/src/sink/prometheus.rs +++ b/crates/health/src/sink/prometheus.rs @@ -207,6 +207,12 @@ impl DataSink for PrometheusSink { } CollectorEvent::Metric(sample) => match self.get_or_create_stream_metrics(context) { Ok(stream_metrics) => { + let mut labels = sample.labels.clone(); + for (name, value) in context.labels() { + if labels.iter().all(|(existing, _)| existing.as_ref() != name) { + labels.push((Cow::Owned(name.clone()), value.clone())); + } + } stream_metrics.record( GaugeReading::new( Self::metric_reading_key(sample), @@ -215,7 +221,7 @@ impl DataSink for PrometheusSink { sample.unit.clone(), sample.value, ) - .with_labels(sample.labels.clone()), + .with_labels(labels), ); } Err(error) => { @@ -277,6 +283,7 @@ mod tests { mac: MacAddress::from_str("42:9e:b1:bd:9d:dd").unwrap(), }, collector_type: "sensor_collector", + labels: Default::default(), metadata: Some(EndpointMetadata::Machine(MachineData { machine_id: Some( "fm100htjtiaehv1n5vh67tbmqq4eabcjdng40f7jupsadbedhruh6rag1l0" @@ -330,6 +337,7 @@ mod tests { mac: MacAddress::from_str("11:22:33:44:55:66").unwrap(), }, collector_type: "switch_collector", + labels: Default::default(), metadata: Some(EndpointMetadata::Switch(SwitchData { id: Some(switch_id), serial: "SN-SWITCH-001".to_string(), diff --git a/crates/health/src/sink/tracing.rs b/crates/health/src/sink/tracing.rs index b45df6a656..18c1ed9e87 100644 --- a/crates/health/src/sink/tracing.rs +++ b/crates/health/src/sink/tracing.rs @@ -49,6 +49,7 @@ impl DataSink for TracingSink { endpoint = %context.endpoint_key(), collector = %context.collector_type, system_uuid = context.system_uuid().map(tracing::field::display), + labels = ?context.labels(), "Metric collection start" ); } @@ -57,6 +58,7 @@ impl DataSink for TracingSink { endpoint = %context.endpoint_key(), collector = %context.collector_type, system_uuid = context.system_uuid().map(tracing::field::display), + labels = ?context.labels(), metric = %metric.name, key = %metric.key, metric_type = %metric.metric_type, @@ -70,6 +72,7 @@ impl DataSink for TracingSink { endpoint = %context.endpoint_key(), collector = %context.collector_type, system_uuid = context.system_uuid().map(tracing::field::display), + labels = ?context.labels(), "Metric collection end" ); } @@ -78,6 +81,7 @@ impl DataSink for TracingSink { endpoint = %context.endpoint_key(), collector = %context.collector_type, system_uuid = context.system_uuid().map(tracing::field::display), + labels = ?context.labels(), "Collector removed" ); } @@ -93,6 +97,7 @@ impl DataSink for TracingSink { driver_version = context.driver_version(), component_type = context.component_type(), nvlink_domain_uuid = context.nvlink_domain_uuid().map(tracing::field::display), + labels = ?context.labels(), severity = %record.severity, body = %record.body, attributes = ?record.attributes, @@ -104,6 +109,7 @@ impl DataSink for TracingSink { endpoint = %context.endpoint_key(), collector = %context.collector_type, system_uuid = context.system_uuid().map(tracing::field::display), + labels = ?context.labels(), firmware_name = %info.component, version = %info.version, "Firmware info event" @@ -115,6 +121,7 @@ impl DataSink for TracingSink { collector = %context.collector_type, machine_id = ?context.machine_id(), system_uuid = context.system_uuid().map(tracing::field::display), + labels = ?context.labels(), success_count = report.successes.len(), alert_count = report.alerts.len(), alerts = ?report.alerts, diff --git a/docs/architecture/health_aggregation.md b/docs/architecture/health_aggregation.md index d2bfd071ee..32de21b43c 100644 --- a/docs/architecture/health_aggregation.md +++ b/docs/architecture/health_aggregation.md @@ -265,11 +265,11 @@ Finally, `nico-hw-health` also emits a health-rollup in `HealthReport` format to This assessed health status is built by comparing the metrics that are emitted from BMCs against well-defined ranges or by interpreting the `health_ok` values provided by BMCs. -For production deployments, `nico-hw-health` discovers machine, switch, and power-shelf BMC endpoints from NICo API via `[endpoint_sources.nico_api]`. Machine endpoints carry the inventory metadata needed to interpret hardware health in fleet context, including machine ID, serial number, rack ID, rack placement, and NVLink domain UUID when present. Switch endpoints carry switch ID, serial number, and rack placement when present. Local and test deployments can instead configure explicit machine, switch, or power-shelf identity with `[[endpoint_sources.static_bmc_endpoints]]`; static machine endpoints can include the same serial number, rack placement, and NVLink domain UUID metadata, static switch endpoints can include serial number and rack placement metadata, and all static endpoints can provide `rack_id` when rack-level rollups are needed. +For production deployments, `nico-hw-health` discovers machine, switch, and power-shelf BMC endpoints from NICo API via `[endpoint_sources.nico_api]`. Machine endpoints carry the inventory metadata needed to interpret hardware health in fleet context, including machine ID, primary Redfish system UUID, serial number, rack ID, rack placement, and NVLink domain UUID when present. Switch endpoints carry switch ID, serial number, and rack placement when present. Local and test deployments can instead configure explicit machine, switch, or power-shelf identity with `[[endpoint_sources.static_bmc_endpoints]]`; static machine endpoints can include the same serial number, rack placement, and NVLink domain UUID metadata, static switch endpoints can include serial number and rack placement metadata, and all static endpoints can provide `rack_id` and validated custom telemetry `labels`. The primary Redfish system UUID remains BMC-derived and cannot be overridden by a custom label. The publishing sinks expose that inventory context using the conventions of the target backend: -- `[sinks.prometheus]` adds machine metadata as metric labels named `machine_id`, `serial_number`, `rack_id`, `machine_slot_number`, `machine_tray_index`, and `nvlink_domain_uuid`; switch metadata uses `switch_id`, `serial_number`, `rack_id`, `switch_slot_number`, and `switch_tray_index`. -- `[sinks.otlp]` adds machine metadata as OTLP resource attributes named `machine.id`, `rack.id`, integer `machine.slot_number`, integer `machine.tray_index`, and `nvlink.domain.uuid`; switch metadata uses `switch.id`, `rack.id`, integer `switch.slot_number`, and integer `switch.tray_index`. +- `[sinks.prometheus]` adds machine metadata as metric labels named `machine_id`, `system_uuid`, `serial_number`, `rack_id`, `machine_slot_number`, `machine_tray_index`, and `nvlink_domain_uuid`; switch metadata uses `switch_id`, `serial_number`, `rack_id`, `switch_slot_number`, and `switch_tray_index`. Static endpoint custom labels keep their configured names. +- `[sinks.otlp]` adds machine metadata as OTLP resource attributes named `machine.id`, `system.uuid`, `rack.id`, integer `machine.slot_number`, integer `machine.tray_index`, and `nvlink.domain.uuid`; switch metadata uses `switch.id`, `rack.id`, integer `switch.slot_number`, and integer `switch.tray_index`. Static endpoint custom labels keep their configured names. - `[sinks.health_report]`, `[sinks.rack_health_report]`, `[sinks.switch_health_report]`, and `[sinks.power_shelf_health_report]` use the same event context when submitting assessed health reports back to NICo API. The persisted `HealthReport` and `HealthProbeAlert` schemas remain the probe success/alert model described above. ### BMC inventory monitoring diff --git a/docs/operations/monitoring-health.md b/docs/operations/monitoring-health.md index 7b054b237f..b67c261767 100644 --- a/docs/operations/monitoring-health.md +++ b/docs/operations/monitoring-health.md @@ -111,8 +111,15 @@ port = 443 mac = "aa:bb:cc:dd:ee:ff" username = "admin" password = "secret" +labels = { site = "rno-dev7", cluster = "cluster-01", environment = "development" } ``` +Static endpoints may define up to 32 custom telemetry labels. Label names must +match `[a-zA-Z_][a-zA-Z0-9_]*`; names owned by the health service, such as +`machine_id`, `system_uuid`, `endpoint_ip`, and `collector_type`, are reserved. +Custom labels are exported as Prometheus labels and OTLP resource attributes, +as a `labels` object in JSONL logs, and as a structured map in tracing events. + Collector defaults from the example config: | Area | Parameter | Example value | Meaning | From fee711b66fdacd1a204dc39ef2dc5da0fd9ec72a Mon Sep 17 00:00:00 2001 From: Jingxiang Zhang Date: Wed, 22 Jul 2026 12:53:12 -0700 Subject: [PATCH 4/6] fix(health): keep telemetry when system UUID is unavailable Signed-off-by: Jingxiang Zhang --- crates/health/src/discovery/iteration.rs | 18 ++++++++---------- crates/health/src/endpoint/model.rs | 5 +++-- 2 files changed, 11 insertions(+), 12 deletions(-) diff --git a/crates/health/src/discovery/iteration.rs b/crates/health/src/discovery/iteration.rs index 8f40cdcb24..f86deea050 100644 --- a/crates/health/src/discovery/iteration.rs +++ b/crates/health/src/discovery/iteration.rs @@ -77,30 +77,28 @@ pub async fn run_discovery_iteration( .cloned() .collect(); - // Resolve machine identity before any collector can emit. Failed endpoints - // are omitted from this pass and retried by the next discovery iteration, - // preserving the invariant that machine telemetry always carries the - // primary ComputerSystem UUID. + // Resolve machine identity before collectors start when possible. UUID + // enrichment is best-effort: a BMC connection or data error must not + // suppress all telemetry for the endpoint. Resolution is attempted again + // on the next discovery iteration. let identity_concurrency = ctx.discovery_config.discovery_concurrency.max(1); - let resolved_endpoints: Vec>> = stream::iter(sharded_endpoints) + let sharded_endpoints: Vec> = stream::iter(sharded_endpoints) .map(|endpoint| async move { match with_primary_system_uuid(&endpoint).await { - Ok(endpoint) => Some(endpoint), + Ok(endpoint) => endpoint, Err(error) => { tracing::warn!( ?error, bmc_address = ?endpoint.addr, - "Could not resolve primary ComputerSystem UUID; skipping endpoint" + "Could not resolve primary ComputerSystem UUID; continuing without it" ); - None + endpoint } } }) .buffer_unordered(identity_concurrency) .collect() .await; - let sharded_endpoints: Vec> = - resolved_endpoints.into_iter().flatten().collect(); if sharded_endpoints.is_empty() { tracing::warn!("No endpoints assigned to this shard"); diff --git a/crates/health/src/endpoint/model.rs b/crates/health/src/endpoint/model.rs index 42d5dd841d..65dbe85083 100644 --- a/crates/health/src/endpoint/model.rs +++ b/crates/health/src/endpoint/model.rs @@ -130,8 +130,9 @@ pub struct MachineData { /// UUID reported by the primary Redfish ComputerSystem resource. /// - /// Endpoint discovery resolves this before machine collectors start so every - /// telemetry event can carry the same hardware identity. + /// Endpoint discovery attempts to resolve this before machine collectors + /// start. It remains absent when the BMC is unavailable or does not expose + /// a usable UUID, allowing telemetry collection to continue. pub system_uuid: Option, /// Physical rack slot where the machine is installed, when known. From bae7cd2b8616945691e9dcf84924edbe116b360d Mon Sep 17 00:00:00 2001 From: Jingxiang Zhang Date: Fri, 24 Jul 2026 10:38:37 -0700 Subject: [PATCH 5/6] fix(health): cache primary system UUID resolution Signed-off-by: Jingxiang Zhang --- crates/health/benches/collector_pipeline.rs | 4 +- crates/health/benches/processor_pipeline.rs | 6 +- crates/health/benches/sink_pipeline.rs | 4 +- crates/health/src/api_client.rs | 50 +++++--- crates/health/src/collectors/leak_detector.rs | 4 +- crates/health/src/discovery/identity.rs | 113 ++++++++++-------- crates/health/src/discovery/iteration.rs | 29 ++--- crates/health/src/discovery/spawn.rs | 5 +- crates/health/src/endpoint/mod.rs | 2 +- crates/health/src/endpoint/model.rs | 66 +++++++++- crates/health/src/endpoint/sources.rs | 4 +- crates/health/src/otlp/convert.rs | 7 +- crates/health/src/processor/health_report.rs | 4 +- crates/health/src/sink/events.rs | 25 +++- crates/health/src/sink/health_report.rs | 4 +- crates/health/src/sink/log_file.rs | 2 +- crates/health/src/sink/mod.rs | 8 +- crates/health/src/sink/prometheus.rs | 2 +- 18 files changed, 224 insertions(+), 115 deletions(-) diff --git a/crates/health/benches/collector_pipeline.rs b/crates/health/benches/collector_pipeline.rs index 6ec89c2d43..17e7ceb73c 100644 --- a/crates/health/benches/collector_pipeline.rs +++ b/crates/health/benches/collector_pipeline.rs @@ -21,7 +21,7 @@ use std::net::{IpAddr, Ipv4Addr}; use std::str::FromStr; use std::sync::Arc; -use carbide_health::endpoint::{BmcAddr, EndpointMetadata, MachineData}; +use carbide_health::endpoint::{BmcAddr, EndpointMetadata, MachineData, SharedSystemUuid}; use carbide_health::metrics::MetricsManager; use carbide_health::sink::{ CollectorEvent, CompositeDataSink, DataSink, EventContext, FirmwareInfo, LogRecord, @@ -63,7 +63,7 @@ fn event_context() -> EventContext { metadata: Some(EndpointMetadata::Machine(MachineData { machine_id: Some(MACHINE_ID.parse().expect("valid machine id")), machine_serial: None, - system_uuid: None, + system_uuid: SharedSystemUuid::default(), slot_number: None, tray_index: None, nvlink_domain_uuid: None, diff --git a/crates/health/benches/processor_pipeline.rs b/crates/health/benches/processor_pipeline.rs index a7d6727c2d..ff2599ddfc 100644 --- a/crates/health/benches/processor_pipeline.rs +++ b/crates/health/benches/processor_pipeline.rs @@ -20,7 +20,7 @@ use std::net::{IpAddr, Ipv4Addr}; use std::str::FromStr; use std::sync::Arc; -use carbide_health::endpoint::{BmcAddr, EndpointMetadata, MachineData}; +use carbide_health::endpoint::{BmcAddr, EndpointMetadata, MachineData, SharedSystemUuid}; use carbide_health::metrics::MetricsManager; use carbide_health::processor::{ EventProcessingPipeline, EventProcessor, HealthReportProcessor, LeakEventProcessor, @@ -99,7 +99,7 @@ fn event_context() -> EventContext { metadata: Some(EndpointMetadata::Machine(MachineData { machine_id: Some(MACHINE_ID.parse().expect("valid machine id")), machine_serial: None, - system_uuid: None, + system_uuid: SharedSystemUuid::default(), slot_number: None, tray_index: None, nvlink_domain_uuid: None, @@ -279,7 +279,7 @@ fn rack_event_contexts(rack_id: &str, tray_count: usize) -> Vec { metadata: Some(EndpointMetadata::Machine(MachineData { machine_id: Some(MACHINE_ID.parse().expect("valid machine id")), machine_serial: None, - system_uuid: None, + system_uuid: SharedSystemUuid::default(), slot_number: None, tray_index: None, nvlink_domain_uuid: None, diff --git a/crates/health/benches/sink_pipeline.rs b/crates/health/benches/sink_pipeline.rs index a34d97d8b4..b27541c9c9 100644 --- a/crates/health/benches/sink_pipeline.rs +++ b/crates/health/benches/sink_pipeline.rs @@ -21,7 +21,7 @@ use std::net::{IpAddr, Ipv4Addr}; use std::str::FromStr; use std::sync::Arc; -use carbide_health::endpoint::{BmcAddr, EndpointMetadata, MachineData}; +use carbide_health::endpoint::{BmcAddr, EndpointMetadata, MachineData, SharedSystemUuid}; use carbide_health::metrics::MetricsManager; use carbide_health::sink::{ Classification, CollectorEvent, CompositeDataSink, DataSink, EventContext, HealthReport, @@ -73,7 +73,7 @@ fn event_context_for_machine(machine_id: &str) -> EventContext { metadata: Some(EndpointMetadata::Machine(MachineData { machine_id: Some(machine_id.parse().expect("valid machine id")), machine_serial: None, - system_uuid: None, + system_uuid: SharedSystemUuid::default(), slot_number: None, tray_index: None, nvlink_domain_uuid: None, diff --git a/crates/health/src/api_client.rs b/crates/health/src/api_client.rs index 8006029de0..c745e1bfbd 100644 --- a/crates/health/src/api_client.rs +++ b/crates/health/src/api_client.rs @@ -39,7 +39,7 @@ use crate::HealthError; use crate::bmc::{BmcClient, BoxFuture, CredentialProvider}; use crate::endpoint::{ BmcAddr, BmcCredentials, BmcEndpoint, EndpointMetadata, EndpointSource, MachineData, - PowerShelfData, SwitchData, SwitchEndpointRole, + PowerShelfData, SharedSystemUuid, SwitchData, SwitchEndpointRole, }; /// [`ApiEndpointSource`]. @@ -290,9 +290,11 @@ pub struct ApiEndpointSource { bmc_client_cache: Mutex>, } +#[derive(Clone)] struct CachedBmcClient { client: Arc, kind: ApiCredentialKind, + system_uuid: SharedSystemUuid, } impl ApiEndpointSource { @@ -484,7 +486,7 @@ impl ApiEndpointSource { .as_ref() .and_then(|info| info.dmi_data.as_ref()) .map(|dmi| dmi.chassis_serial.clone()), - system_uuid: None, + system_uuid: SharedSystemUuid::default(), slot_number: machine .placement_in_rack .as_ref() @@ -589,7 +591,7 @@ impl ApiEndpointSource { rack_id: Option, credential_kind: ApiCredentialKind, ) -> Result, HealthError> { - let bmc = { + let cached = { let mut cache = self.bmc_client_cache.lock().expect("cache mutex poisoned"); cache_or_create_bmc_client(&mut cache, addr.mac, credential_kind, |kind| { let provider: Arc = Arc::new(ApiCredentialProvider { @@ -605,12 +607,16 @@ impl ApiEndpointSource { )?)) })? }; + let mut metadata = metadata; + if let Some(EndpointMetadata::Machine(machine)) = metadata.as_mut() { + machine.system_uuid = cached.system_uuid; + } Ok(Arc::new(BmcEndpoint { addr, metadata, rack_id, labels: Default::default(), - bmc, + bmc: cached.client, })) } } @@ -620,7 +626,7 @@ fn cache_or_create_bmc_client( mac: MacAddress, credential_kind: ApiCredentialKind, make_client: impl FnOnce(ApiCredentialKind) -> Result, HealthError>, -) -> Result, HealthError> { +) -> Result { if let Some(existing) = cache.get(&mac) { if existing.kind != credential_kind { return Err(HealthError::GenericError(format!( @@ -631,18 +637,17 @@ fn cache_or_create_bmc_client( credential_kind.tag(), ))); } - return Ok(existing.client.clone()); + return Ok(existing.clone()); } let client = make_client(credential_kind.clone())?; - cache.insert( - mac, - CachedBmcClient { - client: client.clone(), - kind: credential_kind, - }, - ); - Ok(client) + let cached = CachedBmcClient { + client, + kind: credential_kind, + system_uuid: SharedSystemUuid::default(), + }; + cache.insert(mac, cached.clone()); + Ok(cached) } /// Returns the machine-level GPU driver version derived from discovery data. @@ -842,8 +847,8 @@ mod tests { ); } - #[test] - fn cache_returns_existing_client_on_matching_kind() { + #[tokio::test] + async fn cache_returns_existing_client_on_matching_kind() { let mut cache: HashMap = HashMap::new(); let factory_calls = AtomicUsize::new(0); @@ -862,10 +867,21 @@ mod tests { .expect("cache hit"); assert!( - Arc::ptr_eq(&first, &second), + Arc::ptr_eq(&first.client, &second.client), "cache hit must reuse the same BmcClient Arc — otherwise every \ iteration of discovery rebuilds the session and re-fetches creds" ); + let system_uuid = uuid::uuid!("4c4c4544-0044-4710-8052-cac04f4b4632"); + first + .system_uuid + .get_or_try_init(|| async { Ok::<_, std::convert::Infallible>(Some(system_uuid)) }) + .await + .expect("infallible UUID initialization"); + assert_eq!( + second.system_uuid.get(), + Some(system_uuid), + "cache hit must reuse machine UUID state across discovery iterations" + ); assert_eq!( factory_calls.load(Ordering::SeqCst), 1, diff --git a/crates/health/src/collectors/leak_detector.rs b/crates/health/src/collectors/leak_detector.rs index b9594193c1..f5fa2ce7ef 100644 --- a/crates/health/src/collectors/leak_detector.rs +++ b/crates/health/src/collectors/leak_detector.rs @@ -271,7 +271,7 @@ mod tests { use mac_address::MacAddress; use super::*; - use crate::endpoint::{BmcAddr, EndpointMetadata, MachineData}; + use crate::endpoint::{BmcAddr, EndpointMetadata, MachineData, SharedSystemUuid}; use crate::sink::HealthReportTarget; #[derive(Debug, Eq, PartialEq)] @@ -337,12 +337,14 @@ mod tests { metadata: Some(EndpointMetadata::Machine(MachineData { machine_id: None, machine_serial: None, + system_uuid: SharedSystemUuid::default(), slot_number: None, tray_index: None, nvlink_domain_uuid: None, driver_version: None, })), rack_id: None, + labels: Default::default(), } } diff --git a/crates/health/src/discovery/identity.rs b/crates/health/src/discovery/identity.rs index 726ca972ee..64c7235860 100644 --- a/crates/health/src/discovery/identity.rs +++ b/crates/health/src/discovery/identity.rs @@ -15,8 +15,6 @@ * limitations under the License. */ -use std::sync::Arc; - use nv_redfish::ServiceRoot; use crate::HealthError; @@ -40,58 +38,58 @@ fn select_primary_system(systems: &[SystemIdentity]) -> Option<&SystemIdentity> .or_else(|| systems.first()) } -/// Resolves the primary ComputerSystem UUID before collectors start. +/// Resolves the primary ComputerSystem UUID when it is not already known. /// /// A system with a non-empty BIOS version is preferred because BMCs may expose -/// auxiliary systems alongside the host. This matches the primary-system rule -/// used by Fleet Intelligence inventory; when no system has BIOS metadata, the +/// auxiliary systems alongside the host. When no system has BIOS metadata, the /// first collection member is used. -pub(super) async fn with_primary_system_uuid( - endpoint: &Arc, -) -> Result, HealthError> { - if !matches!(endpoint.metadata, Some(EndpointMetadata::Machine(_))) { - return Ok(Arc::clone(endpoint)); - } +pub(super) async fn ensure_primary_system_uuid(endpoint: &BmcEndpoint) -> Result<(), HealthError> { + let Some(EndpointMetadata::Machine(machine)) = endpoint.metadata.as_ref() else { + return Ok(()); + }; - let root = ServiceRoot::new(Arc::clone(endpoint.bmc())).await?; - let systems = root.systems().await?.ok_or_else(|| { - HealthError::GenericError(format!( - "BMC {:?} does not expose a ComputerSystem collection", - endpoint.addr - )) - })?; - let systems = systems.members().await?; - let identities: Vec = systems - .iter() - .map(|system| { - let raw = system.raw(); - SystemIdentity { - id: raw.base.id.clone(), - uuid: raw.uuid.flatten(), - bios_version: raw.bios_version.clone().flatten(), + machine + .system_uuid + .get_or_try_init(|| async { + let root = ServiceRoot::new(endpoint.bmc().clone()).await?; + let Some(systems) = root.systems().await? else { + tracing::warn!( + bmc_address = ?endpoint.addr, + "BMC does not expose a ComputerSystem collection" + ); + return Ok(None); + }; + let systems = systems.members().await?; + let identities: Vec = systems + .iter() + .map(|system| { + let raw = system.raw(); + SystemIdentity { + id: raw.base.id.clone(), + uuid: raw.uuid.flatten(), + bios_version: raw.bios_version.clone().flatten(), + } + }) + .collect(); + let Some(primary) = select_primary_system(&identities) else { + tracing::warn!( + bmc_address = ?endpoint.addr, + "BMC exposes an empty ComputerSystem collection" + ); + return Ok(None); + }; + if primary.uuid.is_none() { + tracing::warn!( + bmc_address = ?endpoint.addr, + system_id = %primary.id, + "Primary ComputerSystem does not expose a UUID" + ); } + Ok::, HealthError>(primary.uuid) }) - .collect(); - let primary = select_primary_system(&identities).ok_or_else(|| { - HealthError::GenericError(format!( - "BMC {:?} exposes an empty ComputerSystem collection", - endpoint.addr - )) - })?; - let system_uuid = primary.uuid.ok_or_else(|| { - HealthError::GenericError(format!( - "primary ComputerSystem {} on BMC {:?} does not expose a UUID", - primary.id, endpoint.addr - )) - })?; - - let mut enriched = endpoint.as_ref().clone(); - let Some(EndpointMetadata::Machine(machine)) = enriched.metadata.as_mut() else { - unreachable!("machine endpoint checked above") - }; - machine.system_uuid = Some(system_uuid); + .await?; - Ok(Arc::new(enriched)) + Ok(()) } #[cfg(test)] @@ -143,6 +141,27 @@ mod tests { assert_eq!(primary.uuid, Some(FIRST_UUID)); } + #[test] + fn primary_system_prefers_host_bios_over_auxiliary_uuid() { + let systems = [ + SystemIdentity { + id: "HGX_Baseboard".to_string(), + uuid: Some(FIRST_UUID), + bios_version: None, + }, + SystemIdentity { + id: "host".to_string(), + uuid: None, + bios_version: Some("1.2.3".to_string()), + }, + ]; + + let primary = select_primary_system(&systems).expect("primary system"); + + assert_eq!(primary.id, "host"); + assert_eq!(primary.uuid, None); + } + #[test] fn primary_system_is_absent_for_empty_collection() { assert!(select_primary_system(&[]).is_none()); diff --git a/crates/health/src/discovery/iteration.rs b/crates/health/src/discovery/iteration.rs index f86deea050..78b32b8b4a 100644 --- a/crates/health/src/discovery/iteration.rs +++ b/crates/health/src/discovery/iteration.rs @@ -25,7 +25,7 @@ use futures::{StreamExt, stream}; use super::DiscoveryIterationStats; use super::cleanup::{stop_ineligible_nmxc_collectors, stop_removed_bmc_collectors}; use super::context::{CollectorKind, DiscoveryLoopContext}; -use super::identity::with_primary_system_uuid; +use super::identity::ensure_primary_system_uuid; use super::spawn::{spawn_collectors_for_endpoint, switch_supports_nmxc_subscription}; use crate::HealthError; use crate::config::Configurable; @@ -77,27 +77,22 @@ pub async fn run_discovery_iteration( .cloned() .collect(); - // Resolve machine identity before collectors start when possible. UUID - // enrichment is best-effort: a BMC connection or data error must not - // suppress all telemetry for the endpoint. Resolution is attempted again - // on the next discovery iteration. + // Resolve machine identity before collectors start when possible. Shared + // write-once state propagates the result to running collectors and caches + // both present and absent UUIDs, preventing repeated successful BMC queries. let identity_concurrency = ctx.discovery_config.discovery_concurrency.max(1); - let sharded_endpoints: Vec> = stream::iter(sharded_endpoints) + stream::iter(sharded_endpoints.iter().cloned()) .map(|endpoint| async move { - match with_primary_system_uuid(&endpoint).await { - Ok(endpoint) => endpoint, - Err(error) => { - tracing::warn!( - ?error, - bmc_address = ?endpoint.addr, - "Could not resolve primary ComputerSystem UUID; continuing without it" - ); - endpoint - } + if let Err(error) = ensure_primary_system_uuid(&endpoint).await { + tracing::warn!( + ?error, + bmc_address = ?endpoint.addr, + "Could not resolve primary ComputerSystem UUID; continuing without it" + ); } }) .buffer_unordered(identity_concurrency) - .collect() + .collect::>() .await; if sharded_endpoints.is_empty() { diff --git a/crates/health/src/discovery/spawn.rs b/crates/health/src/discovery/spawn.rs index 2fcd1ac69d..13364ee027 100644 --- a/crates/health/src/discovery/spawn.rs +++ b/crates/health/src/discovery/spawn.rs @@ -696,7 +696,8 @@ mod tests { }; use crate::endpoint::test_support::endpoint_with_creds; use crate::endpoint::{ - BmcAddr, BmcCredentials, EndpointMetadata, MachineData, SwitchData, SwitchEndpointRole, + BmcAddr, BmcCredentials, EndpointMetadata, MachineData, SharedSystemUuid, SwitchData, + SwitchEndpointRole, }; use crate::limiter::{NoopLimiter, RateLimiter}; use crate::metrics::MetricsManager; @@ -788,7 +789,7 @@ mod tests { .expect("valid machine id"), ), machine_serial: None, - system_uuid: None, + system_uuid: SharedSystemUuid::default(), slot_number: None, tray_index: None, nvlink_domain_uuid: None, diff --git a/crates/health/src/endpoint/mod.rs b/crates/health/src/endpoint/mod.rs index 8ae1f1cd57..d6564c8910 100644 --- a/crates/health/src/endpoint/mod.rs +++ b/crates/health/src/endpoint/mod.rs @@ -22,7 +22,7 @@ mod sources; pub use cluster::ClusterEndpointSource; pub use model::{ BmcAddr, BmcCredentials, BmcEndpoint, EndpointMetadata, EndpointSource, MachineData, - PowerShelfData, SwitchData, SwitchEndpointRole, + PowerShelfData, SharedSystemUuid, SwitchData, SwitchEndpointRole, }; pub use sources::{CompositeEndpointSource, StaticEndpointSource}; diff --git a/crates/health/src/endpoint/model.rs b/crates/health/src/endpoint/model.rs index 65dbe85083..f6eb67e9a6 100644 --- a/crates/health/src/endpoint/model.rs +++ b/crates/health/src/endpoint/model.rs @@ -17,6 +17,7 @@ use std::borrow::Cow; use std::collections::BTreeMap; +use std::future::Future; use std::net::IpAddr; use std::sync::Arc; @@ -26,11 +27,42 @@ use carbide_uuid::power_shelf::PowerShelfId; use carbide_uuid::rack::RackId; use carbide_uuid::switch::SwitchId; use mac_address::MacAddress; +use tokio::sync::OnceCell; use url::Url; use crate::HealthError; use crate::bmc::{BmcClient, BoxFuture}; +/// Shared, write-once UUID reported by a machine's primary ComputerSystem. +/// +/// Collectors clone endpoint metadata when they start, so this state must remain +/// shared for a UUID resolved after collector startup to reach emitted events. +#[derive(Clone, Debug, Default)] +pub struct SharedSystemUuid(Arc>>); + +impl SharedSystemUuid { + pub fn get(&self) -> Option { + self.0.get().copied().flatten() + } + + pub(crate) async fn get_or_try_init(&self, f: F) -> Result, E> + where + F: FnOnce() -> Fut, + Fut: Future, E>>, + { + self.0.get_or_try_init(f).await.copied() + } +} + +impl From> for SharedSystemUuid { + fn from(system_uuid: Option) -> Self { + match system_uuid { + Some(system_uuid) => Self(Arc::new(OnceCell::new_with(Some(Some(system_uuid))))), + None => Self::default(), + } + } +} + #[derive(Clone)] pub struct BmcEndpoint { pub addr: BmcAddr, @@ -130,10 +162,11 @@ pub struct MachineData { /// UUID reported by the primary Redfish ComputerSystem resource. /// - /// Endpoint discovery attempts to resolve this before machine collectors - /// start. It remains absent when the BMC is unavailable or does not expose - /// a usable UUID, allowing telemetry collection to continue. - pub system_uuid: Option, + /// Endpoint discovery resolves this on demand. The write-once shared state + /// propagates enrichment to collectors that already started and records + /// both present and absent UUID results so successful BMC queries happen + /// only once. + pub system_uuid: SharedSystemUuid, /// Physical rack slot where the machine is installed, when known. pub slot_number: Option, @@ -234,12 +267,14 @@ pub trait EndpointSource: Send + Sync { #[cfg(test)] mod tests { + use std::convert::Infallible; use std::net::IpAddr; use std::str::FromStr; + use std::sync::atomic::{AtomicUsize, Ordering}; use mac_address::MacAddress; - use super::{BmcAddr, BmcCredentials}; + use super::{BmcAddr, BmcCredentials, SharedSystemUuid}; use crate::endpoint::test_support::endpoint_with_creds; fn addr(ip: &str, port: Option) -> BmcAddr { @@ -288,4 +323,25 @@ mod tests { assert_eq!(endpoint.switch_connect_host_for_uri(), "[2001:db8::1]"); } + + #[tokio::test] + async fn shared_system_uuid_caches_absent_result_across_clones() { + let state = SharedSystemUuid::default(); + let clone = state.clone(); + let query_count = AtomicUsize::new(0); + + for state in [&state, &clone] { + state + .get_or_try_init(|| async { + query_count.fetch_add(1, Ordering::SeqCst); + Ok::<_, Infallible>(None) + }) + .await + .expect("infallible UUID initialization"); + } + + assert_eq!(query_count.load(Ordering::SeqCst), 1); + assert_eq!(state.get(), None); + assert_eq!(clone.get(), None); + } } diff --git a/crates/health/src/endpoint/sources.rs b/crates/health/src/endpoint/sources.rs index d3adf0f7fa..85d1a1388c 100644 --- a/crates/health/src/endpoint/sources.rs +++ b/crates/health/src/endpoint/sources.rs @@ -30,7 +30,7 @@ use crate::bmc::{BmcClient, FixedCredentialProvider}; use crate::config::{StaticBmcEndpoint, StaticSwitchEndpointRole}; use crate::endpoint::{ BmcAddr, BmcCredentials, BmcEndpoint, BoxFuture, EndpointMetadata, EndpointSource, MachineData, - PowerShelfData, SwitchData, SwitchEndpointRole, + PowerShelfData, SharedSystemUuid, SwitchData, SwitchEndpointRole, }; pub struct StaticEndpointSource { @@ -152,7 +152,7 @@ impl StaticEndpointSource { Some(EndpointMetadata::Machine(MachineData { machine_id, machine_serial: machine.serial.clone(), - system_uuid: None, + system_uuid: SharedSystemUuid::default(), slot_number: machine.slot_number, tray_index: machine.tray_index, nvlink_domain_uuid, diff --git a/crates/health/src/otlp/convert.rs b/crates/health/src/otlp/convert.rs index d264e54e93..0b18e3c6c2 100644 --- a/crates/health/src/otlp/convert.rs +++ b/crates/health/src/otlp/convert.rs @@ -337,7 +337,8 @@ mod tests { use super::*; use crate::endpoint::{ - BmcAddr, EndpointMetadata, MachineData, PowerShelfData, SwitchData, SwitchEndpointRole, + BmcAddr, EndpointMetadata, MachineData, PowerShelfData, SharedSystemUuid, SwitchData, + SwitchEndpointRole, }; use crate::sink::{ Classification, HealthReport, HealthReportAlert, LogRecord, Probe, ReportSource, @@ -420,7 +421,7 @@ mod tests { .expect("valid machine id"), ), machine_serial: Some("MN-001".to_string()), - system_uuid: Some(uuid::uuid!("4c4c4544-0044-4710-8052-cac04f4b4632")), + system_uuid: Some(uuid::uuid!("4c4c4544-0044-4710-8052-cac04f4b4632")).into(), slot_number: Some(15), tray_index: Some(5), nvlink_domain_uuid: Some(domain_uuid), @@ -463,7 +464,7 @@ mod tests { metadata: Some(EndpointMetadata::Machine(MachineData { machine_id: None, machine_serial: None, - system_uuid: None, + system_uuid: SharedSystemUuid::default(), slot_number: None, tray_index: None, nvlink_domain_uuid: None, diff --git a/crates/health/src/processor/health_report.rs b/crates/health/src/processor/health_report.rs index ce813db6db..046eeb6ee8 100644 --- a/crates/health/src/processor/health_report.rs +++ b/crates/health/src/processor/health_report.rs @@ -268,7 +268,7 @@ mod tests { use nv_redfish::resource::Health as BmcHealth; use super::*; - use crate::endpoint::{BmcAddr, EndpointMetadata, MachineData}; + use crate::endpoint::{BmcAddr, EndpointMetadata, MachineData, SharedSystemUuid}; use crate::sink::HealthReportTarget; fn test_context() -> EventContext { @@ -288,7 +288,7 @@ mod tests { .expect("valid machine id"), ), machine_serial: None, - system_uuid: None, + system_uuid: SharedSystemUuid::default(), slot_number: None, tray_index: None, nvlink_domain_uuid: None, diff --git a/crates/health/src/sink/events.rs b/crates/health/src/sink/events.rs index 07e491750f..2abd643a8d 100644 --- a/crates/health/src/sink/events.rs +++ b/crates/health/src/sink/events.rs @@ -96,7 +96,7 @@ impl EventContext { /// Returns the UUID reported by the primary Redfish ComputerSystem. pub fn system_uuid(&self) -> Option { self.machine_metadata() - .and_then(|machine| machine.system_uuid) + .and_then(|machine| machine.system_uuid.get()) } /// Returns the uniform GPU driver version when it is known for the machine. @@ -546,7 +546,7 @@ mod tests { use mac_address::MacAddress; use super::*; - use crate::endpoint::{MachineData, PowerShelfData, SwitchData}; + use crate::endpoint::{MachineData, PowerShelfData, SharedSystemUuid, SwitchData}; #[derive(Clone, Copy)] enum ContextKind { @@ -656,7 +656,7 @@ mod tests { ContextKind::Machine => Some(EndpointMetadata::Machine(MachineData { machine_id: Some(machine_id()), machine_serial: Some("MN-001".to_string()), - system_uuid: None, + system_uuid: SharedSystemUuid::default(), slot_number: Some(7), tray_index: Some(3), nvlink_domain_uuid: Some(nvlink_domain_id()), @@ -688,6 +688,25 @@ mod tests { } } + #[tokio::test] + async fn cloned_event_context_observes_later_system_uuid_resolution() { + let system_uuid = SharedSystemUuid::default(); + let mut context = context(ContextKind::Machine); + let Some(EndpointMetadata::Machine(machine)) = context.metadata.as_mut() else { + panic!("machine context"); + }; + machine.system_uuid = system_uuid.clone(); + let collector_context = context.clone(); + let expected = uuid::uuid!("4c4c4544-0044-4710-8052-cac04f4b4632"); + + system_uuid + .get_or_try_init(|| async { Ok::<_, std::convert::Infallible>(Some(expected)) }) + .await + .expect("infallible UUID initialization"); + + assert_eq!(collector_context.system_uuid(), Some(expected)); + } + fn summarize_context(context: EventContext) -> ContextSummary { ContextSummary { endpoint_key: context.endpoint_key().to_string(), diff --git a/crates/health/src/sink/health_report.rs b/crates/health/src/sink/health_report.rs index 8cfbdf7903..99c63458dd 100644 --- a/crates/health/src/sink/health_report.rs +++ b/crates/health/src/sink/health_report.rs @@ -250,7 +250,7 @@ mod tests { use mac_address::MacAddress; use super::*; - use crate::endpoint::{BmcAddr, EndpointMetadata, MachineData}; + use crate::endpoint::{BmcAddr, EndpointMetadata, MachineData, SharedSystemUuid}; use crate::sink::events::{Classification, HealthReportAlert, HealthReportSuccess, Probe}; fn machine_id(value: &str) -> MachineId { @@ -284,7 +284,7 @@ mod tests { metadata: Some(EndpointMetadata::Machine(MachineData { machine_id: Some(id), machine_serial: None, - system_uuid: None, + system_uuid: SharedSystemUuid::default(), slot_number: None, tray_index: None, nvlink_domain_uuid: None, diff --git a/crates/health/src/sink/log_file.rs b/crates/health/src/sink/log_file.rs index 9fba62c623..8e40620b47 100644 --- a/crates/health/src/sink/log_file.rs +++ b/crates/health/src/sink/log_file.rs @@ -314,7 +314,7 @@ mod tests { .expect("valid machine id"), ), machine_serial: Some("MN-001".to_string()), - system_uuid: Some(uuid::uuid!("4c4c4544-0044-4710-8052-cac04f4b4632")), + system_uuid: Some(uuid::uuid!("4c4c4544-0044-4710-8052-cac04f4b4632")).into(), slot_number: None, tray_index: None, nvlink_domain_uuid: Some(NvLinkDomainId::nil()), diff --git a/crates/health/src/sink/mod.rs b/crates/health/src/sink/mod.rs index 9dfbe54496..c6f390e394 100644 --- a/crates/health/src/sink/mod.rs +++ b/crates/health/src/sink/mod.rs @@ -132,7 +132,7 @@ mod tests { CollectorEvent, CompositeDataSink, DataSink, DiagnosticLogRecord, EventContext, LogRecord, MetricSample, PrometheusSink, }; - use crate::endpoint::{BmcAddr, EndpointMetadata, MachineData}; + use crate::endpoint::{BmcAddr, EndpointMetadata, MachineData, SharedSystemUuid}; use crate::metrics::MetricsManager; struct CountingSink { @@ -375,7 +375,7 @@ mod tests { .expect("valid machine id"), ), machine_serial: None, - system_uuid: None, + system_uuid: SharedSystemUuid::default(), slot_number: None, tray_index: None, nvlink_domain_uuid: None, @@ -453,7 +453,7 @@ mod tests { .expect("valid machine id"), ), machine_serial: None, - system_uuid: None, + system_uuid: SharedSystemUuid::default(), slot_number: None, tray_index: None, nvlink_domain_uuid: None, @@ -513,7 +513,7 @@ mod tests { .expect("valid machine id"), ), machine_serial: None, - system_uuid: None, + system_uuid: SharedSystemUuid::default(), slot_number: None, tray_index: None, nvlink_domain_uuid: None, diff --git a/crates/health/src/sink/prometheus.rs b/crates/health/src/sink/prometheus.rs index e99580f78c..a3cfeba8eb 100644 --- a/crates/health/src/sink/prometheus.rs +++ b/crates/health/src/sink/prometheus.rs @@ -291,7 +291,7 @@ mod tests { .expect("valid machine id"), ), machine_serial: Some("MN-001".to_string()), - system_uuid: Some(uuid::uuid!("4c4c4544-0044-4710-8052-cac04f4b4632")), + system_uuid: Some(uuid::uuid!("4c4c4544-0044-4710-8052-cac04f4b4632")).into(), slot_number: Some(15), tray_index: Some(5), nvlink_domain_uuid: Some(NvLinkDomainId::nil()), From 92457b9d03220c4c225f0cc0b5d31a1dd39afdc2 Mon Sep 17 00:00:00 2001 From: Jingxiang Zhang <72907213+jingxiang-z@users.noreply.github.com> Date: Fri, 24 Jul 2026 12:33:42 -0700 Subject: [PATCH 6/6] docs(health): fix docs for clarity Co-authored-by: Alex Ball Signed-off-by: Jingxiang Zhang <72907213+jingxiang-z@users.noreply.github.com> --- docs/architecture/health_aggregation.md | 18 +++++++++++++++--- docs/operations/monitoring-health.md | 4 ++-- 2 files changed, 17 insertions(+), 5 deletions(-) diff --git a/docs/architecture/health_aggregation.md b/docs/architecture/health_aggregation.md index 32de21b43c..5a7ed54e26 100644 --- a/docs/architecture/health_aggregation.md +++ b/docs/architecture/health_aggregation.md @@ -265,11 +265,23 @@ Finally, `nico-hw-health` also emits a health-rollup in `HealthReport` format to This assessed health status is built by comparing the metrics that are emitted from BMCs against well-defined ranges or by interpreting the `health_ok` values provided by BMCs. -For production deployments, `nico-hw-health` discovers machine, switch, and power-shelf BMC endpoints from NICo API via `[endpoint_sources.nico_api]`. Machine endpoints carry the inventory metadata needed to interpret hardware health in fleet context, including machine ID, primary Redfish system UUID, serial number, rack ID, rack placement, and NVLink domain UUID when present. Switch endpoints carry switch ID, serial number, and rack placement when present. Local and test deployments can instead configure explicit machine, switch, or power-shelf identity with `[[endpoint_sources.static_bmc_endpoints]]`; static machine endpoints can include the same serial number, rack placement, and NVLink domain UUID metadata, static switch endpoints can include serial number and rack placement metadata, and all static endpoints can provide `rack_id` and validated custom telemetry `labels`. The primary Redfish system UUID remains BMC-derived and cannot be overridden by a custom label. +**For production deployments**, `nico-hw-health` discovers machine, switch, and power-shelf BMC endpoints from NICo API via `[endpoint_sources.nico_api]`. + +Machine endpoints carry the inventory metadata needed to interpret hardware health in fleet context. This includes machine ID, primary Redfish system UUID, serial number, rack ID, rack placement, and NVLink domain UUID when present. + +Switch endpoints carry switch ID, serial number, and rack placement when present. + +**For local and test deployments**, you can configure explicit machine, switch, or power-shelf identity with `[[endpoint_sources.static_bmc_endpoints]]`. Note the following: + +- Static machine endpoints can include the same serial number, rack placement, and NVLink domain UUID metadata +- Static switch endpoints can include serial number and rack placement metadata +- All static endpoints can provide `rack_id` and validated custom telemetry `labels` +- The primary Redfish system UUID remains BMC-derived and cannot be overridden by a custom label The publishing sinks expose that inventory context using the conventions of the target backend: -- `[sinks.prometheus]` adds machine metadata as metric labels named `machine_id`, `system_uuid`, `serial_number`, `rack_id`, `machine_slot_number`, `machine_tray_index`, and `nvlink_domain_uuid`; switch metadata uses `switch_id`, `serial_number`, `rack_id`, `switch_slot_number`, and `switch_tray_index`. Static endpoint custom labels keep their configured names. -- `[sinks.otlp]` adds machine metadata as OTLP resource attributes named `machine.id`, `system.uuid`, `rack.id`, integer `machine.slot_number`, integer `machine.tray_index`, and `nvlink.domain.uuid`; switch metadata uses `switch.id`, `rack.id`, integer `switch.slot_number`, and integer `switch.tray_index`. Static endpoint custom labels keep their configured names. + +- `[sinks.prometheus]` adds _machine_ metadata as metric labels named `machine_id`, `system_uuid`, `serial_number`, `rack_id`, `machine_slot_number`, `machine_tray_index`, and `nvlink_domain_uuid` _Switch_ metadata labels are `switch_id`, `serial_number`, `rack_id`, `switch_slot_number`, and `switch_tray_index`. Static endpoint custom labels keep their configured names. +- `[sinks.otlp]` adds _machine_ metadata as OTLP resource attributes named `machine.id`, `system.uuid`, `rack.id`, integer `machine.slot_number`, integer `machine.tray_index`, and `nvlink.domain.uuid`. _Switch_ metadata labels are `switch.id`, `rack.id`, integer `switch.slot_number`, and integer `switch.tray_index`. Static endpoint custom labels keep their configured names. - `[sinks.health_report]`, `[sinks.rack_health_report]`, `[sinks.switch_health_report]`, and `[sinks.power_shelf_health_report]` use the same event context when submitting assessed health reports back to NICo API. The persisted `HealthReport` and `HealthProbeAlert` schemas remain the probe success/alert model described above. ### BMC inventory monitoring diff --git a/docs/operations/monitoring-health.md b/docs/operations/monitoring-health.md index b67c261767..658583d531 100644 --- a/docs/operations/monitoring-health.md +++ b/docs/operations/monitoring-health.md @@ -114,8 +114,8 @@ password = "secret" labels = { site = "rno-dev7", cluster = "cluster-01", environment = "development" } ``` -Static endpoints may define up to 32 custom telemetry labels. Label names must -match `[a-zA-Z_][a-zA-Z0-9_]*`; names owned by the health service, such as +Static endpoints can define up to 32 custom telemetry labels. Label names must +match `[a-zA-Z_][a-zA-Z0-9_]*`. Names owned by the health service, such as `machine_id`, `system_uuid`, `endpoint_ip`, and `collector_type`, are reserved. Custom labels are exported as Prometheus labels and OTLP resource attributes, as a `labels` object in JSONL logs, and as a structured map in tracing events.