diff --git a/crates/api-core/src/attestation/measured_boot.rs b/crates/api-core/src/attestation/measured_boot.rs index da796468fc..5fc3d25591 100644 --- a/crates/api-core/src/attestation/measured_boot.rs +++ b/crates/api-core/src/attestation/measured_boot.rs @@ -24,6 +24,7 @@ use byteorder::{BigEndian, ByteOrder}; use carbide_machine_controller::{MeasuringOutcome, handle_measuring_state}; use carbide_uuid::machine::MachineId; use carbide_uuid::measured_boot::MeasurementReportId; +use chrono::Utc; use db::db_read::DbReader; use model::machine::MeasuringState; use pkcs1::LineEnding; @@ -357,6 +358,8 @@ where machine_id, db, true, + Utc::now(), + chrono::Duration::MAX, ) .await .map_err(|e| CarbideError::AttestQuoteError(e.to_string()))?; diff --git a/crates/api-core/src/cfg/README.md b/crates/api-core/src/cfg/README.md index 94dc2f9e1e..4255aa6768 100644 --- a/crates/api-core/src/cfg/README.md +++ b/crates/api-core/src/cfg/README.md @@ -376,6 +376,7 @@ Extends `StateControllerConfig` with: | `failure_retry_time` | `Duration` | `90m` | Time before re-triggering reboot if machine hasn't called back. | | `dpu_up_threshold` | `Duration` | `5m` | Max time without DPU health report before assuming it's down. | | `scout_reporting_timeout` | `Duration` | `5m` | Duration without scout report before host is unhealthy. | +| `waiting_for_measurements_timeout` | `Duration` | `4h` | How long a host may remain in WaitingForMeasurements before being escalated to Failed. | | `uefi_boot_wait` | `Duration` | `5m` | Wait time for UEFI boot completion after host reboot. | | `max_bios_config_retries` | `u32` | `3` | Shared retry budget for automated host boot-configuration convergence across BIOS recovery and boot-order verification. | | `polling_bios_setup_stuck_threshold` | `Duration` | `15m` | Time in PollingBiosSetup with `is_bios_setup == false` before recovery escalation. | diff --git a/crates/api-core/src/cfg/file.rs b/crates/api-core/src/cfg/file.rs index edb6b8dbc4..92e6fc2c71 100644 --- a/crates/api-core/src/cfg/file.rs +++ b/crates/api-core/src/cfg/file.rs @@ -3510,6 +3510,7 @@ mod tests { failure_retry_time: Duration::minutes(90), dpu_up_threshold: Duration::weeks(1), scout_reporting_timeout: Duration::minutes(5), + waiting_for_measurements_timeout: Duration::hours(4), uefi_boot_wait: Duration::minutes(5), max_bios_config_retries: 3, polling_bios_setup_stuck_threshold: Duration::minutes(15), @@ -3555,6 +3556,7 @@ mod tests { failure_retry_time: Duration::minutes(90), dpu_up_threshold: Duration::weeks(1), scout_reporting_timeout: Duration::minutes(5), + waiting_for_measurements_timeout: Duration::hours(4), uefi_boot_wait: Duration::minutes(5), max_bios_config_retries: 3, polling_bios_setup_stuck_threshold: Duration::minutes(15), @@ -3577,6 +3579,7 @@ mod tests { failure_retry_time: Duration::minutes(90), dpu_up_threshold: Duration::weeks(1), scout_reporting_timeout: Duration::minutes(5), + waiting_for_measurements_timeout: Duration::hours(4), uefi_boot_wait: Duration::minutes(5), max_bios_config_retries: 3, polling_bios_setup_stuck_threshold: Duration::minutes(15), @@ -4001,6 +4004,7 @@ mod tests { failure_retry_time: Duration::minutes(70), dpu_up_threshold: Duration::minutes(77), scout_reporting_timeout: Duration::minutes(5), + waiting_for_measurements_timeout: Duration::hours(4), uefi_boot_wait: Duration::minutes(5), max_bios_config_retries: 3, polling_bios_setup_stuck_threshold: Duration::minutes(15), @@ -4241,6 +4245,7 @@ mod tests { failure_retry_time: Duration::minutes(31), dpu_up_threshold: Duration::minutes(33), scout_reporting_timeout: Duration::minutes(20), + waiting_for_measurements_timeout: Duration::hours(4), uefi_boot_wait: Duration::minutes(5), max_bios_config_retries: 3, polling_bios_setup_stuck_threshold: Duration::minutes(15), @@ -4603,6 +4608,7 @@ mod tests { failure_retry_time: Duration::minutes(70), dpu_up_threshold: Duration::minutes(77), scout_reporting_timeout: Duration::minutes(20), + waiting_for_measurements_timeout: Duration::hours(4), uefi_boot_wait: Duration::minutes(5), max_bios_config_retries: 3, polling_bios_setup_stuck_threshold: Duration::minutes(15), diff --git a/crates/api-core/src/setup.rs b/crates/api-core/src/setup.rs index 5d1760b76e..8022c17742 100644 --- a/crates/api-core/src/setup.rs +++ b/crates/api-core/src/setup.rs @@ -1296,6 +1296,11 @@ async fn initialize_and_start_controllers<'a>( .machine_state_controller .scout_reporting_timeout, ) + .waiting_for_measurements_timeout( + carbide_config + .machine_state_controller + .waiting_for_measurements_timeout, + ) .uefi_boot_wait(carbide_config.machine_state_controller.uefi_boot_wait) .hardware_models(carbide_config.get_firmware_config()) .firmware_downloader(&downloader) diff --git a/crates/api-core/src/tests/common/api_fixtures/mod.rs b/crates/api-core/src/tests/common/api_fixtures/mod.rs index 39ba50c9c0..435da3ac38 100644 --- a/crates/api-core/src/tests/common/api_fixtures/mod.rs +++ b/crates/api-core/src/tests/common/api_fixtures/mod.rs @@ -1484,6 +1484,9 @@ pub async fn create_test_env_with_overrides( power_down_wait: Duration::seconds(0), failure_retry_time: Duration::seconds(0), scout_reporting_timeout: config.machine_state_controller.scout_reporting_timeout, + waiting_for_measurements_timeout: config + .machine_state_controller + .waiting_for_measurements_timeout, uefi_boot_wait: Duration::seconds(0), }; diff --git a/crates/api-model/src/machine/mod.rs b/crates/api-model/src/machine/mod.rs index dc616788b1..73f374779a 100644 --- a/crates/api-model/src/machine/mod.rs +++ b/crates/api-model/src/machine/mod.rs @@ -1651,6 +1651,12 @@ pub enum FailureCause { MeasurementsCAValidationFailed { err: String }, + // MeasurementsNotReceived is returned when the host has been in + // WaitingForMeasurements longer than the configured timeout without + // Scout ever submitting a report. This typically means the host is + // not network-booting at all. + MeasurementsNotReceived { err: String }, + DpfProvisioning { err: String }, SpdmAttestationFailed { err: String }, @@ -2238,6 +2244,7 @@ impl FailureCause { FailureCause::MeasurementsCAValidationFailed { .. } => { "measurements_ca_validation_failed" } + FailureCause::MeasurementsNotReceived { .. } => "measurements_not_received", FailureCause::DpfProvisioning { .. } => "dpf_provisioning", FailureCause::SpdmAttestationFailed { .. } => "spdm_attestation_failed", FailureCause::BiosSetupFailed { .. } => "bios_setup_failed", @@ -2262,6 +2269,9 @@ impl Display for FailureCause { FailureCause::MeasurementsCAValidationFailed { .. } => { write!(f, "MeasurementsCAValidationFailed") } + FailureCause::MeasurementsNotReceived { .. } => { + write!(f, "MeasurementsNotReceived") + } FailureCause::DpfProvisioning { err } => write!(f, "DpfProvisioning {err}"), FailureCause::SpdmAttestationFailed { .. } => { write!(f, "SpdmAttestationFailed") diff --git a/crates/machine-controller/src/config/controller.rs b/crates/machine-controller/src/config/controller.rs index 4af4ecb2bf..267a0050e0 100644 --- a/crates/machine-controller/src/config/controller.rs +++ b/crates/machine-controller/src/config/controller.rs @@ -63,6 +63,15 @@ pub struct MachineStateControllerConfig { serialize_with = "as_duration" )] pub scout_reporting_timeout: Duration, + /// How long a host may remain in WaitingForMeasurements before being + /// escalated to Failed. A healthy fleet completes this state in minutes, + /// so a generous ceiling still catches hosts that are silently stuck. + #[serde( + default = "MachineStateControllerConfig::waiting_for_measurements_timeout_default", + deserialize_with = "deserialize_duration_chrono", + serialize_with = "as_duration" + )] + pub waiting_for_measurements_timeout: Duration, /// How long to wait for UEFI boot to complete after rebooting a host #[serde( default = "MachineStateControllerConfig::uefi_boot_wait_default", @@ -95,6 +104,7 @@ impl MachineStateControllerConfig { dpu_up_threshold: Duration::weeks(52), controller: StateControllerConfig::default(), scout_reporting_timeout: Duration::weeks(52), + waiting_for_measurements_timeout: Duration::weeks(52), uefi_boot_wait: Duration::seconds(0), max_bios_config_retries: MachineStateControllerConfig::max_bios_config_retries_default( ), @@ -123,6 +133,10 @@ impl MachineStateControllerConfig { Duration::minutes(5) } + pub fn waiting_for_measurements_timeout_default() -> Duration { + Duration::hours(4) + } + pub fn uefi_boot_wait_default() -> Duration { Duration::minutes(5) } @@ -146,6 +160,8 @@ impl Default for MachineStateControllerConfig { dpu_up_threshold: MachineStateControllerConfig::dpu_up_threshold_default(), scout_reporting_timeout: MachineStateControllerConfig::scout_reporting_timeout_default( ), + waiting_for_measurements_timeout: + MachineStateControllerConfig::waiting_for_measurements_timeout_default(), uefi_boot_wait: MachineStateControllerConfig::uefi_boot_wait_default(), max_bios_config_retries: MachineStateControllerConfig::max_bios_config_retries_default( ), diff --git a/crates/machine-controller/src/handler.rs b/crates/machine-controller/src/handler.rs index 29073f1366..d55bd12da2 100644 --- a/crates/machine-controller/src/handler.rs +++ b/crates/machine-controller/src/handler.rs @@ -210,6 +210,7 @@ pub struct ReachabilityParams { pub power_down_wait: chrono::Duration, pub failure_retry_time: chrono::Duration, pub scout_reporting_timeout: chrono::Duration, + pub waiting_for_measurements_timeout: chrono::Duration, pub uefi_boot_wait: chrono::Duration, } @@ -290,6 +291,7 @@ impl MachineStateHandlerBuilder { power_down_wait: chrono::Duration::zero(), failure_retry_time: chrono::Duration::zero(), scout_reporting_timeout: chrono::Duration::zero(), + waiting_for_measurements_timeout: chrono::Duration::zero(), uefi_boot_wait: chrono::Duration::zero(), }, firmware_downloader: None, @@ -379,6 +381,15 @@ impl MachineStateHandlerBuilder { self } + pub fn waiting_for_measurements_timeout( + mut self, + waiting_for_measurements_timeout: chrono::Duration, + ) -> Self { + self.reachability_params.waiting_for_measurements_timeout = + waiting_for_measurements_timeout; + self + } + pub fn uefi_boot_wait(mut self, uefi_boot_wait: chrono::Duration) -> Self { self.reachability_params.uefi_boot_wait = uefi_boot_wait; self @@ -1577,6 +1588,11 @@ impl MachineStateHandler { &mh_snapshot.host_snapshot.id, &mut ctx.services.db_reader, self.host_handler.host_handler_params.attestation_enabled, + mh_snapshot.host_snapshot.state.version.timestamp(), + self.host_handler + .host_handler_params + .reachability_params + .waiting_for_measurements_timeout, ) .await .map(|v| map_measuring_outcome_to_state_handler_outcome(&v, measuring_state))?, @@ -1598,6 +1614,11 @@ impl MachineStateHandler { &mh_snapshot.host_snapshot.id, &mut ctx.services.db_reader, self.host_handler.host_handler_params.attestation_enabled, + mh_snapshot.host_snapshot.state.version.timestamp(), + self.host_handler + .host_handler_params + .reachability_params + .waiting_for_measurements_timeout, ) .await .map(|v| { @@ -5883,6 +5904,10 @@ impl StateHandler for HostMachineStateHandler { &mh_snapshot.host_snapshot.id, &mut ctx.services.db_reader, self.host_handler_params.attestation_enabled, + mh_snapshot.host_snapshot.state.version.timestamp(), + self.host_handler_params + .reachability_params + .waiting_for_measurements_timeout, ) .await { diff --git a/crates/machine-controller/src/lib.rs b/crates/machine-controller/src/lib.rs index 11d03a55be..8bc4de7656 100644 --- a/crates/machine-controller/src/lib.rs +++ b/crates/machine-controller/src/lib.rs @@ -18,6 +18,7 @@ //! State Controller implementation for Machines use carbide_uuid::machine::MachineId; +use chrono::{DateTime, Duration, Utc}; use db::attestation::ek_cert_verification_status; use db::db_read::DbReader; use db::measured_boot::machine::{get_measurement_bundle_state, get_measurement_machine_state}; @@ -148,6 +149,8 @@ pub async fn handle_measuring_state( machine_id: &MachineId, db: &mut DB, attestation_enabled: bool, + state_entered_at: DateTime, + waiting_for_measurements_timeout: Duration, ) -> Result where for<'db> &'db mut DB: DbReader<'db>, @@ -188,8 +191,28 @@ where Ok(match machine_state { // "Discovered" is the MeasurementMachineState equivalent of // "no measurements have been sent yet". If that's the case, - // then continue waiting for measurements. - MeasurementMachineState::Discovered => MeasuringOutcome::NoChange, + // then continue waiting — unless the timeout has elapsed, in + // which case escalate to Failed so the host becomes visible. + MeasurementMachineState::Discovered => { + let elapsed = Utc::now().signed_duration_since(state_entered_at); + if elapsed >= waiting_for_measurements_timeout { + MeasuringOutcome::Unsuccessful(( + FailureDetails { + cause: FailureCause::MeasurementsNotReceived { + err: format!( + "no measurement report received after {elapsed}; \ + host may not be network-booting" + ), + }, + failed_at: Utc::now(), + source: FailureSource::StateMachineArea(StateMachineArea::Default), + }, + *machine_id, + )) + } else { + MeasuringOutcome::NoChange + } + } MeasurementMachineState::PendingBundle => MeasuringOutcome::WaitForGoldenValues, MeasurementMachineState::Measured => MeasuringOutcome::PassedOk, MeasurementMachineState::MeasuringFailed => MeasuringOutcome::Unsuccessful(( diff --git a/crates/machine-controller/tests/integration/env.rs b/crates/machine-controller/tests/integration/env.rs index 3721edd9bd..4881caf0bc 100644 --- a/crates/machine-controller/tests/integration/env.rs +++ b/crates/machine-controller/tests/integration/env.rs @@ -117,6 +117,9 @@ impl EnvBuilder { scout_reporting_timeout: runtime_config .machine_state_controller .scout_reporting_timeout, + waiting_for_measurements_timeout: runtime_config + .machine_state_controller + .waiting_for_measurements_timeout, uefi_boot_wait: runtime_config.machine_state_controller.uefi_boot_wait, }; let power_options: PowerOptionConfig = runtime_config.power_manager_options.clone().into();