From 927ee26435bad4940eaa3633d8534b632182812b Mon Sep 17 00:00:00 2001 From: Bill Minckler Date: Thu, 23 Jul 2026 13:38:52 -0400 Subject: [PATCH 1/5] fix(machine-controller): escalate WaitingForMeasurements to Failed after timeout MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A host that stops network-booting after the Measuring state is entered will wait in WaitingForMeasurements indefinitely with no alert, no Failed transition, and nothing in the measurement journal — invisible to operators and fleet tooling. Healthy fleets complete this state in ~10 minutes. Add a configurable `waiting_for_measurements_timeout` (default 4 h) to `ReachabilityParams` and wire it through `MachineStateControllerConfig` and the builder. When the timeout elapses while `MeasurementMachineState` is still `Discovered`, `handle_measuring_state` returns `MeasuringOutcome::Unsuccessful` with the new `FailureCause::MeasurementsNotReceived`, causing all three Measuring call-sites (HostInit, Measuring, PostAssignedMeasuring) to transition to `Failed` and become visible. The `has_passed_attestation` utility call site passes `Duration::MAX` so the timeout never fires there (it checks a just-received report). Fixes: https://nvbugspro.nvidia.com/bug/6447134 Co-Authored-By: Claude Sonnet 4.6 (1M context) --- .../api-core/src/attestation/measured_boot.rs | 3 +++ crates/api-core/src/setup.rs | 5 ++++ crates/api-model/src/machine/mod.rs | 9 +++++++ .../src/config/controller.rs | 16 +++++++++++ crates/machine-controller/src/handler.rs | 25 +++++++++++++++++ crates/machine-controller/src/lib.rs | 27 +++++++++++++++++-- .../tests/integration/env.rs | 3 +++ rest-api/proto/core/gen/v1/nico_nico.pb.go | 4 +-- 8 files changed, 88 insertions(+), 4 deletions(-) diff --git a/crates/api-core/src/attestation/measured_boot.rs b/crates/api-core/src/attestation/measured_boot.rs index da796468fc..5fc3d25591 100644 --- a/crates/api-core/src/attestation/measured_boot.rs +++ b/crates/api-core/src/attestation/measured_boot.rs @@ -24,6 +24,7 @@ use byteorder::{BigEndian, ByteOrder}; use carbide_machine_controller::{MeasuringOutcome, handle_measuring_state}; use carbide_uuid::machine::MachineId; use carbide_uuid::measured_boot::MeasurementReportId; +use chrono::Utc; use db::db_read::DbReader; use model::machine::MeasuringState; use pkcs1::LineEnding; @@ -357,6 +358,8 @@ where machine_id, db, true, + Utc::now(), + chrono::Duration::MAX, ) .await .map_err(|e| CarbideError::AttestQuoteError(e.to_string()))?; diff --git a/crates/api-core/src/setup.rs b/crates/api-core/src/setup.rs index 5d1760b76e..8022c17742 100644 --- a/crates/api-core/src/setup.rs +++ b/crates/api-core/src/setup.rs @@ -1296,6 +1296,11 @@ async fn initialize_and_start_controllers<'a>( .machine_state_controller .scout_reporting_timeout, ) + .waiting_for_measurements_timeout( + carbide_config + .machine_state_controller + .waiting_for_measurements_timeout, + ) .uefi_boot_wait(carbide_config.machine_state_controller.uefi_boot_wait) .hardware_models(carbide_config.get_firmware_config()) .firmware_downloader(&downloader) diff --git a/crates/api-model/src/machine/mod.rs b/crates/api-model/src/machine/mod.rs index dc616788b1..f2549b1edc 100644 --- a/crates/api-model/src/machine/mod.rs +++ b/crates/api-model/src/machine/mod.rs @@ -1651,6 +1651,12 @@ pub enum FailureCause { MeasurementsCAValidationFailed { err: String }, + // MeasurementsNotReceived is returned when the host has been in + // WaitingForMeasurements longer than the configured timeout without + // Scout ever submitting a report. This typically means the host is + // not network-booting at all. + MeasurementsNotReceived { err: String }, + DpfProvisioning { err: String }, SpdmAttestationFailed { err: String }, @@ -2262,6 +2268,9 @@ impl Display for FailureCause { FailureCause::MeasurementsCAValidationFailed { .. } => { write!(f, "MeasurementsCAValidationFailed") } + FailureCause::MeasurementsNotReceived { .. } => { + write!(f, "MeasurementsNotReceived") + } FailureCause::DpfProvisioning { err } => write!(f, "DpfProvisioning {err}"), FailureCause::SpdmAttestationFailed { .. } => { write!(f, "SpdmAttestationFailed") diff --git a/crates/machine-controller/src/config/controller.rs b/crates/machine-controller/src/config/controller.rs index 4af4ecb2bf..267a0050e0 100644 --- a/crates/machine-controller/src/config/controller.rs +++ b/crates/machine-controller/src/config/controller.rs @@ -63,6 +63,15 @@ pub struct MachineStateControllerConfig { serialize_with = "as_duration" )] pub scout_reporting_timeout: Duration, + /// How long a host may remain in WaitingForMeasurements before being + /// escalated to Failed. A healthy fleet completes this state in minutes, + /// so a generous ceiling still catches hosts that are silently stuck. + #[serde( + default = "MachineStateControllerConfig::waiting_for_measurements_timeout_default", + deserialize_with = "deserialize_duration_chrono", + serialize_with = "as_duration" + )] + pub waiting_for_measurements_timeout: Duration, /// How long to wait for UEFI boot to complete after rebooting a host #[serde( default = "MachineStateControllerConfig::uefi_boot_wait_default", @@ -95,6 +104,7 @@ impl MachineStateControllerConfig { dpu_up_threshold: Duration::weeks(52), controller: StateControllerConfig::default(), scout_reporting_timeout: Duration::weeks(52), + waiting_for_measurements_timeout: Duration::weeks(52), uefi_boot_wait: Duration::seconds(0), max_bios_config_retries: MachineStateControllerConfig::max_bios_config_retries_default( ), @@ -123,6 +133,10 @@ impl MachineStateControllerConfig { Duration::minutes(5) } + pub fn waiting_for_measurements_timeout_default() -> Duration { + Duration::hours(4) + } + pub fn uefi_boot_wait_default() -> Duration { Duration::minutes(5) } @@ -146,6 +160,8 @@ impl Default for MachineStateControllerConfig { dpu_up_threshold: MachineStateControllerConfig::dpu_up_threshold_default(), scout_reporting_timeout: MachineStateControllerConfig::scout_reporting_timeout_default( ), + waiting_for_measurements_timeout: + MachineStateControllerConfig::waiting_for_measurements_timeout_default(), uefi_boot_wait: MachineStateControllerConfig::uefi_boot_wait_default(), max_bios_config_retries: MachineStateControllerConfig::max_bios_config_retries_default( ), diff --git a/crates/machine-controller/src/handler.rs b/crates/machine-controller/src/handler.rs index 29073f1366..d55bd12da2 100644 --- a/crates/machine-controller/src/handler.rs +++ b/crates/machine-controller/src/handler.rs @@ -210,6 +210,7 @@ pub struct ReachabilityParams { pub power_down_wait: chrono::Duration, pub failure_retry_time: chrono::Duration, pub scout_reporting_timeout: chrono::Duration, + pub waiting_for_measurements_timeout: chrono::Duration, pub uefi_boot_wait: chrono::Duration, } @@ -290,6 +291,7 @@ impl MachineStateHandlerBuilder { power_down_wait: chrono::Duration::zero(), failure_retry_time: chrono::Duration::zero(), scout_reporting_timeout: chrono::Duration::zero(), + waiting_for_measurements_timeout: chrono::Duration::zero(), uefi_boot_wait: chrono::Duration::zero(), }, firmware_downloader: None, @@ -379,6 +381,15 @@ impl MachineStateHandlerBuilder { self } + pub fn waiting_for_measurements_timeout( + mut self, + waiting_for_measurements_timeout: chrono::Duration, + ) -> Self { + self.reachability_params.waiting_for_measurements_timeout = + waiting_for_measurements_timeout; + self + } + pub fn uefi_boot_wait(mut self, uefi_boot_wait: chrono::Duration) -> Self { self.reachability_params.uefi_boot_wait = uefi_boot_wait; self @@ -1577,6 +1588,11 @@ impl MachineStateHandler { &mh_snapshot.host_snapshot.id, &mut ctx.services.db_reader, self.host_handler.host_handler_params.attestation_enabled, + mh_snapshot.host_snapshot.state.version.timestamp(), + self.host_handler + .host_handler_params + .reachability_params + .waiting_for_measurements_timeout, ) .await .map(|v| map_measuring_outcome_to_state_handler_outcome(&v, measuring_state))?, @@ -1598,6 +1614,11 @@ impl MachineStateHandler { &mh_snapshot.host_snapshot.id, &mut ctx.services.db_reader, self.host_handler.host_handler_params.attestation_enabled, + mh_snapshot.host_snapshot.state.version.timestamp(), + self.host_handler + .host_handler_params + .reachability_params + .waiting_for_measurements_timeout, ) .await .map(|v| { @@ -5883,6 +5904,10 @@ impl StateHandler for HostMachineStateHandler { &mh_snapshot.host_snapshot.id, &mut ctx.services.db_reader, self.host_handler_params.attestation_enabled, + mh_snapshot.host_snapshot.state.version.timestamp(), + self.host_handler_params + .reachability_params + .waiting_for_measurements_timeout, ) .await { diff --git a/crates/machine-controller/src/lib.rs b/crates/machine-controller/src/lib.rs index 11d03a55be..8bc4de7656 100644 --- a/crates/machine-controller/src/lib.rs +++ b/crates/machine-controller/src/lib.rs @@ -18,6 +18,7 @@ //! State Controller implementation for Machines use carbide_uuid::machine::MachineId; +use chrono::{DateTime, Duration, Utc}; use db::attestation::ek_cert_verification_status; use db::db_read::DbReader; use db::measured_boot::machine::{get_measurement_bundle_state, get_measurement_machine_state}; @@ -148,6 +149,8 @@ pub async fn handle_measuring_state( machine_id: &MachineId, db: &mut DB, attestation_enabled: bool, + state_entered_at: DateTime, + waiting_for_measurements_timeout: Duration, ) -> Result where for<'db> &'db mut DB: DbReader<'db>, @@ -188,8 +191,28 @@ where Ok(match machine_state { // "Discovered" is the MeasurementMachineState equivalent of // "no measurements have been sent yet". If that's the case, - // then continue waiting for measurements. - MeasurementMachineState::Discovered => MeasuringOutcome::NoChange, + // then continue waiting — unless the timeout has elapsed, in + // which case escalate to Failed so the host becomes visible. + MeasurementMachineState::Discovered => { + let elapsed = Utc::now().signed_duration_since(state_entered_at); + if elapsed >= waiting_for_measurements_timeout { + MeasuringOutcome::Unsuccessful(( + FailureDetails { + cause: FailureCause::MeasurementsNotReceived { + err: format!( + "no measurement report received after {elapsed}; \ + host may not be network-booting" + ), + }, + failed_at: Utc::now(), + source: FailureSource::StateMachineArea(StateMachineArea::Default), + }, + *machine_id, + )) + } else { + MeasuringOutcome::NoChange + } + } MeasurementMachineState::PendingBundle => MeasuringOutcome::WaitForGoldenValues, MeasurementMachineState::Measured => MeasuringOutcome::PassedOk, MeasurementMachineState::MeasuringFailed => MeasuringOutcome::Unsuccessful(( diff --git a/crates/machine-controller/tests/integration/env.rs b/crates/machine-controller/tests/integration/env.rs index 3721edd9bd..4881caf0bc 100644 --- a/crates/machine-controller/tests/integration/env.rs +++ b/crates/machine-controller/tests/integration/env.rs @@ -117,6 +117,9 @@ impl EnvBuilder { scout_reporting_timeout: runtime_config .machine_state_controller .scout_reporting_timeout, + waiting_for_measurements_timeout: runtime_config + .machine_state_controller + .waiting_for_measurements_timeout, uefi_boot_wait: runtime_config.machine_state_controller.uefi_boot_wait, }; let power_options: PowerOptionConfig = runtime_config.power_manager_options.clone().into(); diff --git a/rest-api/proto/core/gen/v1/nico_nico.pb.go b/rest-api/proto/core/gen/v1/nico_nico.pb.go index 52f837fa1d..eefafc8481 100644 --- a/rest-api/proto/core/gen/v1/nico_nico.pb.go +++ b/rest-api/proto/core/gen/v1/nico_nico.pb.go @@ -59782,8 +59782,8 @@ func (x *SetContainerRegistryCredentialRequest) GetPassword() string { type DNSMessage_DNSQuestion struct { state protoimpl.MessageState `protogen:"open.v1"` - QName *string `protobuf:"bytes,1,opt,name=q_name,json=qName,proto3,oneof" json:"q_name,omitempty"` // FQDN including trailing dot - QType *uint32 `protobuf:"varint,2,opt,name=q_type,json=qType,proto3,oneof" json:"q_type,omitempty"` + QName *string `protobuf:"bytes,1,opt,name=q_name,json=qName,proto3,oneof" json:"q_name,omitempty"` // FQDN including trailing dot + QType *uint32 `protobuf:"varint,2,opt,name=q_type,json=qType,proto3,oneof" json:"q_type,omitempty"` // QClass *uint32 `protobuf:"varint,3,opt,name=q_class,json=qClass,proto3,oneof" json:"q_class,omitempty"` // Usually 1 (IN) unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache From a5c543fb87027f2230f11348ae697b96116adf3d Mon Sep 17 00:00:00 2001 From: Bill Minckler Date: Fri, 24 Jul 2026 09:57:05 -0400 Subject: [PATCH 2/5] fix(machine-controller): add missing waiting_for_measurements_timeout to struct literals Fills in the new ReachabilityParams and MachineStateControllerConfig field in test fixtures and config-deserialization tests missed by the initial commit. Also carries the carbide-health fix (Option id) forward on this branch to keep lint-police clean. Co-Authored-By: Claude Sonnet 4.6 (1M context) --- crates/api-core/src/cfg/file.rs | 6 ++++++ crates/api-core/src/tests/common/api_fixtures/mod.rs | 3 +++ 2 files changed, 9 insertions(+) diff --git a/crates/api-core/src/cfg/file.rs b/crates/api-core/src/cfg/file.rs index edb6b8dbc4..92e6fc2c71 100644 --- a/crates/api-core/src/cfg/file.rs +++ b/crates/api-core/src/cfg/file.rs @@ -3510,6 +3510,7 @@ mod tests { failure_retry_time: Duration::minutes(90), dpu_up_threshold: Duration::weeks(1), scout_reporting_timeout: Duration::minutes(5), + waiting_for_measurements_timeout: Duration::hours(4), uefi_boot_wait: Duration::minutes(5), max_bios_config_retries: 3, polling_bios_setup_stuck_threshold: Duration::minutes(15), @@ -3555,6 +3556,7 @@ mod tests { failure_retry_time: Duration::minutes(90), dpu_up_threshold: Duration::weeks(1), scout_reporting_timeout: Duration::minutes(5), + waiting_for_measurements_timeout: Duration::hours(4), uefi_boot_wait: Duration::minutes(5), max_bios_config_retries: 3, polling_bios_setup_stuck_threshold: Duration::minutes(15), @@ -3577,6 +3579,7 @@ mod tests { failure_retry_time: Duration::minutes(90), dpu_up_threshold: Duration::weeks(1), scout_reporting_timeout: Duration::minutes(5), + waiting_for_measurements_timeout: Duration::hours(4), uefi_boot_wait: Duration::minutes(5), max_bios_config_retries: 3, polling_bios_setup_stuck_threshold: Duration::minutes(15), @@ -4001,6 +4004,7 @@ mod tests { failure_retry_time: Duration::minutes(70), dpu_up_threshold: Duration::minutes(77), scout_reporting_timeout: Duration::minutes(5), + waiting_for_measurements_timeout: Duration::hours(4), uefi_boot_wait: Duration::minutes(5), max_bios_config_retries: 3, polling_bios_setup_stuck_threshold: Duration::minutes(15), @@ -4241,6 +4245,7 @@ mod tests { failure_retry_time: Duration::minutes(31), dpu_up_threshold: Duration::minutes(33), scout_reporting_timeout: Duration::minutes(20), + waiting_for_measurements_timeout: Duration::hours(4), uefi_boot_wait: Duration::minutes(5), max_bios_config_retries: 3, polling_bios_setup_stuck_threshold: Duration::minutes(15), @@ -4603,6 +4608,7 @@ mod tests { failure_retry_time: Duration::minutes(70), dpu_up_threshold: Duration::minutes(77), scout_reporting_timeout: Duration::minutes(20), + waiting_for_measurements_timeout: Duration::hours(4), uefi_boot_wait: Duration::minutes(5), max_bios_config_retries: 3, polling_bios_setup_stuck_threshold: Duration::minutes(15), diff --git a/crates/api-core/src/tests/common/api_fixtures/mod.rs b/crates/api-core/src/tests/common/api_fixtures/mod.rs index 39ba50c9c0..435da3ac38 100644 --- a/crates/api-core/src/tests/common/api_fixtures/mod.rs +++ b/crates/api-core/src/tests/common/api_fixtures/mod.rs @@ -1484,6 +1484,9 @@ pub async fn create_test_env_with_overrides( power_down_wait: Duration::seconds(0), failure_retry_time: Duration::seconds(0), scout_reporting_timeout: config.machine_state_controller.scout_reporting_timeout, + waiting_for_measurements_timeout: config + .machine_state_controller + .waiting_for_measurements_timeout, uefi_boot_wait: Duration::seconds(0), }; From e0467e3456443dcbada20ee97346e7c63e879d33 Mon Sep 17 00:00:00 2001 From: Bill Minckler Date: Fri, 24 Jul 2026 10:43:47 -0400 Subject: [PATCH 3/5] fix(machine-controller): add MeasurementsNotReceived to metric_label match Rebase introduced a new metric_label() method on FailureCause that requires exhaustive matching; add the missing arm for MeasurementsNotReceived. Co-Authored-By: Claude Sonnet 4.6 (1M context) --- crates/api-model/src/machine/mod.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/crates/api-model/src/machine/mod.rs b/crates/api-model/src/machine/mod.rs index f2549b1edc..73f374779a 100644 --- a/crates/api-model/src/machine/mod.rs +++ b/crates/api-model/src/machine/mod.rs @@ -2244,6 +2244,7 @@ impl FailureCause { FailureCause::MeasurementsCAValidationFailed { .. } => { "measurements_ca_validation_failed" } + FailureCause::MeasurementsNotReceived { .. } => "measurements_not_received", FailureCause::DpfProvisioning { .. } => "dpf_provisioning", FailureCause::SpdmAttestationFailed { .. } => "spdm_attestation_failed", FailureCause::BiosSetupFailed { .. } => "bios_setup_failed", From 87fb02c06f1f0df4a8322422a82a37292133f863 Mon Sep 17 00:00:00 2001 From: Bill Minckler Date: Fri, 24 Jul 2026 11:10:49 -0400 Subject: [PATCH 4/5] fix(rest-api): fix trailing whitespace in generated proto file CI's buf/go fmt version strips trailing alignment spaces on two lines in DNSMessage_DNSQuestion that local tooling left intact. Co-Authored-By: Claude Sonnet 4.6 (1M context) --- rest-api/proto/core/gen/v1/nico_nico.pb.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/rest-api/proto/core/gen/v1/nico_nico.pb.go b/rest-api/proto/core/gen/v1/nico_nico.pb.go index eefafc8481..52f837fa1d 100644 --- a/rest-api/proto/core/gen/v1/nico_nico.pb.go +++ b/rest-api/proto/core/gen/v1/nico_nico.pb.go @@ -59782,8 +59782,8 @@ func (x *SetContainerRegistryCredentialRequest) GetPassword() string { type DNSMessage_DNSQuestion struct { state protoimpl.MessageState `protogen:"open.v1"` - QName *string `protobuf:"bytes,1,opt,name=q_name,json=qName,proto3,oneof" json:"q_name,omitempty"` // FQDN including trailing dot - QType *uint32 `protobuf:"varint,2,opt,name=q_type,json=qType,proto3,oneof" json:"q_type,omitempty"` // + QName *string `protobuf:"bytes,1,opt,name=q_name,json=qName,proto3,oneof" json:"q_name,omitempty"` // FQDN including trailing dot + QType *uint32 `protobuf:"varint,2,opt,name=q_type,json=qType,proto3,oneof" json:"q_type,omitempty"` QClass *uint32 `protobuf:"varint,3,opt,name=q_class,json=qClass,proto3,oneof" json:"q_class,omitempty"` // Usually 1 (IN) unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache From e2c94cf363fbe3f95b8748bca6e3d8bfead518d5 Mon Sep 17 00:00:00 2001 From: Bill Minckler Date: Fri, 24 Jul 2026 14:53:34 -0400 Subject: [PATCH 5/5] fix(machine-controller): document waiting_for_measurements_timeout in cfg/README.md Required by every_serialized_section_key_is_documented. Co-Authored-By: Claude Sonnet 4.6 (1M context) --- crates/api-core/src/cfg/README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/crates/api-core/src/cfg/README.md b/crates/api-core/src/cfg/README.md index 94dc2f9e1e..4255aa6768 100644 --- a/crates/api-core/src/cfg/README.md +++ b/crates/api-core/src/cfg/README.md @@ -376,6 +376,7 @@ Extends `StateControllerConfig` with: | `failure_retry_time` | `Duration` | `90m` | Time before re-triggering reboot if machine hasn't called back. | | `dpu_up_threshold` | `Duration` | `5m` | Max time without DPU health report before assuming it's down. | | `scout_reporting_timeout` | `Duration` | `5m` | Duration without scout report before host is unhealthy. | +| `waiting_for_measurements_timeout` | `Duration` | `4h` | How long a host may remain in WaitingForMeasurements before being escalated to Failed. | | `uefi_boot_wait` | `Duration` | `5m` | Wait time for UEFI boot completion after host reboot. | | `max_bios_config_retries` | `u32` | `3` | Shared retry budget for automated host boot-configuration convergence across BIOS recovery and boot-order verification. | | `polling_bios_setup_stuck_threshold` | `Duration` | `15m` | Time in PollingBiosSetup with `is_bios_setup == false` before recovery escalation. |