fix(machine-controller): escalate WaitingForMeasurements to Failed after timeout

crates/api-core/src/attestation/measured_boot.rs

@@ -24,6 +24,7 @@ use byteorder::{BigEndian, ByteOrder};
 use carbide_machine_controller::{MeasuringOutcome, handle_measuring_state};
 use carbide_uuid::machine::MachineId;
 use carbide_uuid::measured_boot::MeasurementReportId;
+use chrono::Utc;
 use db::db_read::DbReader;
 use model::machine::MeasuringState;
 use pkcs1::LineEnding;
@@ -357,6 +358,8 @@ where
         machine_id,
         db,
         true,
+        Utc::now(),
+        chrono::Duration::MAX,
     )
     .await
     .map_err(|e| CarbideError::AttestQuoteError(e.to_string()))?;

crates/api-core/src/cfg/README.md

@@ -376,6 +376,7 @@ Extends `StateControllerConfig` with:
 | `failure_retry_time` | `Duration` | `90m`   | Time before re-triggering reboot if machine hasn't called back. |
 | `dpu_up_threshold` | `Duration` | `5m`    | Max time without DPU health report before assuming it's down. |
 | `scout_reporting_timeout` | `Duration` | `5m`    | Duration without scout report before host is unhealthy. |
+| `waiting_for_measurements_timeout` | `Duration` | `4h`    | How long a host may remain in WaitingForMeasurements before being escalated to Failed. |
 | `uefi_boot_wait` | `Duration` | `5m`    | Wait time for UEFI boot completion after host reboot. |
 | `max_bios_config_retries` | `u32` | `3` | Shared retry budget for automated host boot-configuration convergence across BIOS recovery and boot-order verification. |
 | `polling_bios_setup_stuck_threshold` | `Duration` | `15m` | Time in PollingBiosSetup with `is_bios_setup == false` before recovery escalation. |

crates/api-core/src/cfg/file.rs

@@ -3510,6 +3510,7 @@ mod tests {
             failure_retry_time: Duration::minutes(90),
             dpu_up_threshold: Duration::weeks(1),
             scout_reporting_timeout: Duration::minutes(5),
+            waiting_for_measurements_timeout: Duration::hours(4),
             uefi_boot_wait: Duration::minutes(5),
             max_bios_config_retries: 3,
             polling_bios_setup_stuck_threshold: Duration::minutes(15),
@@ -3555,6 +3556,7 @@ mod tests {
                 failure_retry_time: Duration::minutes(90),
                 dpu_up_threshold: Duration::weeks(1),
                 scout_reporting_timeout: Duration::minutes(5),
+                waiting_for_measurements_timeout: Duration::hours(4),
                 uefi_boot_wait: Duration::minutes(5),
                 max_bios_config_retries: 3,
                 polling_bios_setup_stuck_threshold: Duration::minutes(15),
@@ -3577,6 +3579,7 @@ mod tests {
                 failure_retry_time: Duration::minutes(90),
                 dpu_up_threshold: Duration::weeks(1),
                 scout_reporting_timeout: Duration::minutes(5),
+                waiting_for_measurements_timeout: Duration::hours(4),
                 uefi_boot_wait: Duration::minutes(5),
                 max_bios_config_retries: 3,
                 polling_bios_setup_stuck_threshold: Duration::minutes(15),
@@ -4001,6 +4004,7 @@ mod tests {
                 failure_retry_time: Duration::minutes(70),
                 dpu_up_threshold: Duration::minutes(77),
                 scout_reporting_timeout: Duration::minutes(5),
+                waiting_for_measurements_timeout: Duration::hours(4),
                 uefi_boot_wait: Duration::minutes(5),
                 max_bios_config_retries: 3,
                 polling_bios_setup_stuck_threshold: Duration::minutes(15),
@@ -4241,6 +4245,7 @@ mod tests {
                 failure_retry_time: Duration::minutes(31),
                 dpu_up_threshold: Duration::minutes(33),
                 scout_reporting_timeout: Duration::minutes(20),
+                waiting_for_measurements_timeout: Duration::hours(4),
                 uefi_boot_wait: Duration::minutes(5),
                 max_bios_config_retries: 3,
                 polling_bios_setup_stuck_threshold: Duration::minutes(15),
@@ -4603,6 +4608,7 @@ mod tests {
                 failure_retry_time: Duration::minutes(70),
                 dpu_up_threshold: Duration::minutes(77),
                 scout_reporting_timeout: Duration::minutes(20),
+                waiting_for_measurements_timeout: Duration::hours(4),
                 uefi_boot_wait: Duration::minutes(5),
                 max_bios_config_retries: 3,
                 polling_bios_setup_stuck_threshold: Duration::minutes(15),

crates/api-core/src/setup.rs

@@ -1296,6 +1296,11 @@ async fn initialize_and_start_controllers<'a>(
                         .machine_state_controller
                         .scout_reporting_timeout,
                 )
+                .waiting_for_measurements_timeout(
+                    carbide_config
+                        .machine_state_controller
+                        .waiting_for_measurements_timeout,
+                )
                 .uefi_boot_wait(carbide_config.machine_state_controller.uefi_boot_wait)
                 .hardware_models(carbide_config.get_firmware_config())
                 .firmware_downloader(&downloader)

crates/api-core/src/tests/common/api_fixtures/mod.rs

@@ -1484,6 +1484,9 @@ pub async fn create_test_env_with_overrides(
         power_down_wait: Duration::seconds(0),
         failure_retry_time: Duration::seconds(0),
         scout_reporting_timeout: config.machine_state_controller.scout_reporting_timeout,
+        waiting_for_measurements_timeout: config
+            .machine_state_controller
+            .waiting_for_measurements_timeout,
         uefi_boot_wait: Duration::seconds(0),
     };
 

crates/api-model/src/machine/mod.rs

@@ -1651,6 +1651,12 @@ pub enum FailureCause {
 
     MeasurementsCAValidationFailed { err: String },
 
+    // MeasurementsNotReceived is returned when the host has been in
+    // WaitingForMeasurements longer than the configured timeout without
+    // Scout ever submitting a report. This typically means the host is
+    // not network-booting at all.
+    MeasurementsNotReceived { err: String },
+
     DpfProvisioning { err: String },
 
     SpdmAttestationFailed { err: String },
@@ -2238,6 +2244,7 @@ impl FailureCause {
             FailureCause::MeasurementsCAValidationFailed { .. } => {
                 "measurements_ca_validation_failed"
             }
+            FailureCause::MeasurementsNotReceived { .. } => "measurements_not_received",
             FailureCause::DpfProvisioning { .. } => "dpf_provisioning",
             FailureCause::SpdmAttestationFailed { .. } => "spdm_attestation_failed",
             FailureCause::BiosSetupFailed { .. } => "bios_setup_failed",
@@ -2262,6 +2269,9 @@ impl Display for FailureCause {
             FailureCause::MeasurementsCAValidationFailed { .. } => {
                 write!(f, "MeasurementsCAValidationFailed")
             }
+            FailureCause::MeasurementsNotReceived { .. } => {
+                write!(f, "MeasurementsNotReceived")
+            }
             FailureCause::DpfProvisioning { err } => write!(f, "DpfProvisioning {err}"),
             FailureCause::SpdmAttestationFailed { .. } => {
                 write!(f, "SpdmAttestationFailed")

crates/machine-controller/src/config/controller.rs

@@ -63,6 +63,15 @@ pub struct MachineStateControllerConfig {
         serialize_with = "as_duration"
     )]
     pub scout_reporting_timeout: Duration,
+    /// How long a host may remain in WaitingForMeasurements before being
+    /// escalated to Failed. A healthy fleet completes this state in minutes,
+    /// so a generous ceiling still catches hosts that are silently stuck.
+    #[serde(
+        default = "MachineStateControllerConfig::waiting_for_measurements_timeout_default",
+        deserialize_with = "deserialize_duration_chrono",
+        serialize_with = "as_duration"
+    )]
+    pub waiting_for_measurements_timeout: Duration,
     /// How long to wait for UEFI boot to complete after rebooting a host
     #[serde(
         default = "MachineStateControllerConfig::uefi_boot_wait_default",
@@ -95,6 +104,7 @@ impl MachineStateControllerConfig {
             dpu_up_threshold: Duration::weeks(52),
             controller: StateControllerConfig::default(),
             scout_reporting_timeout: Duration::weeks(52),
+            waiting_for_measurements_timeout: Duration::weeks(52),
             uefi_boot_wait: Duration::seconds(0),
             max_bios_config_retries: MachineStateControllerConfig::max_bios_config_retries_default(
             ),
@@ -123,6 +133,10 @@ impl MachineStateControllerConfig {
         Duration::minutes(5)
     }
 
+    pub fn waiting_for_measurements_timeout_default() -> Duration {
+        Duration::hours(4)
+    }
+
     pub fn uefi_boot_wait_default() -> Duration {
         Duration::minutes(5)
     }
@@ -146,6 +160,8 @@ impl Default for MachineStateControllerConfig {
             dpu_up_threshold: MachineStateControllerConfig::dpu_up_threshold_default(),
             scout_reporting_timeout: MachineStateControllerConfig::scout_reporting_timeout_default(
             ),
+            waiting_for_measurements_timeout:
+                MachineStateControllerConfig::waiting_for_measurements_timeout_default(),
             uefi_boot_wait: MachineStateControllerConfig::uefi_boot_wait_default(),
             max_bios_config_retries: MachineStateControllerConfig::max_bios_config_retries_default(
             ),

crates/machine-controller/src/handler.rs

@@ -210,6 +210,7 @@ pub struct ReachabilityParams {
     pub power_down_wait: chrono::Duration,
     pub failure_retry_time: chrono::Duration,
     pub scout_reporting_timeout: chrono::Duration,
+    pub waiting_for_measurements_timeout: chrono::Duration,
     pub uefi_boot_wait: chrono::Duration,
 }
 
@@ -290,6 +291,7 @@ impl MachineStateHandlerBuilder {
                 power_down_wait: chrono::Duration::zero(),
                 failure_retry_time: chrono::Duration::zero(),
                 scout_reporting_timeout: chrono::Duration::zero(),
+                waiting_for_measurements_timeout: chrono::Duration::zero(),
                 uefi_boot_wait: chrono::Duration::zero(),
             },
             firmware_downloader: None,
@@ -379,6 +381,15 @@ impl MachineStateHandlerBuilder {
         self
     }
 
+    pub fn waiting_for_measurements_timeout(
+        mut self,
+        waiting_for_measurements_timeout: chrono::Duration,
+    ) -> Self {
+        self.reachability_params.waiting_for_measurements_timeout =
+            waiting_for_measurements_timeout;
+        self
+    }
+
     pub fn uefi_boot_wait(mut self, uefi_boot_wait: chrono::Duration) -> Self {
         self.reachability_params.uefi_boot_wait = uefi_boot_wait;
         self
@@ -1577,6 +1588,11 @@ impl MachineStateHandler {
                 &mh_snapshot.host_snapshot.id,
                 &mut ctx.services.db_reader,
                 self.host_handler.host_handler_params.attestation_enabled,
+                mh_snapshot.host_snapshot.state.version.timestamp(),
+                self.host_handler
+                    .host_handler_params
+                    .reachability_params
+                    .waiting_for_measurements_timeout,
             )
             .await
             .map(|v| map_measuring_outcome_to_state_handler_outcome(&v, measuring_state))?,
@@ -1598,6 +1614,11 @@ impl MachineStateHandler {
                             &mh_snapshot.host_snapshot.id,
                             &mut ctx.services.db_reader,
                             self.host_handler.host_handler_params.attestation_enabled,
+                            mh_snapshot.host_snapshot.state.version.timestamp(),
+                            self.host_handler
+                                .host_handler_params
+                                .reachability_params
+                                .waiting_for_measurements_timeout,
                         )
                         .await
                         .map(|v| {
@@ -5883,6 +5904,10 @@ impl StateHandler for HostMachineStateHandler {
                         &mh_snapshot.host_snapshot.id,
                         &mut ctx.services.db_reader,
                         self.host_handler_params.attestation_enabled,
+                        mh_snapshot.host_snapshot.state.version.timestamp(),
+                        self.host_handler_params
+                            .reachability_params
+                            .waiting_for_measurements_timeout,
                     )
                     .await
                     {

crates/machine-controller/src/lib.rs

@@ -18,6 +18,7 @@
 //! State Controller implementation for Machines
 
 use carbide_uuid::machine::MachineId;
+use chrono::{DateTime, Duration, Utc};
 use db::attestation::ek_cert_verification_status;
 use db::db_read::DbReader;
 use db::measured_boot::machine::{get_measurement_bundle_state, get_measurement_machine_state};
@@ -148,6 +149,8 @@ pub async fn handle_measuring_state<DB>(
     machine_id: &MachineId,
     db: &mut DB,
     attestation_enabled: bool,
+    state_entered_at: DateTime<Utc>,
+    waiting_for_measurements_timeout: Duration,
 ) -> Result<MeasuringOutcome, StateHandlerError>
 where
     for<'db> &'db mut DB: DbReader<'db>,
@@ -188,8 +191,28 @@ where
             Ok(match machine_state {
                 // "Discovered" is the MeasurementMachineState equivalent of
                 // "no measurements have been sent yet". If that's the case,
-                // then continue waiting for measurements.
-                MeasurementMachineState::Discovered => MeasuringOutcome::NoChange,
+                // then continue waiting — unless the timeout has elapsed, in
+                // which case escalate to Failed so the host becomes visible.
+                MeasurementMachineState::Discovered => {
+                    let elapsed = Utc::now().signed_duration_since(state_entered_at);
+                    if elapsed >= waiting_for_measurements_timeout {
+                        MeasuringOutcome::Unsuccessful((
+                            FailureDetails {
+                                cause: FailureCause::MeasurementsNotReceived {
+                                    err: format!(
+                                        "no measurement report received after {elapsed}; \
+                                         host may not be network-booting"
+                                    ),
+                                },
+                                failed_at: Utc::now(),
+                                source: FailureSource::StateMachineArea(StateMachineArea::Default),
+                            },
+                            *machine_id,
+                        ))
+                    } else {
+                        MeasuringOutcome::NoChange
+                    }
+                }
                 MeasurementMachineState::PendingBundle => MeasuringOutcome::WaitForGoldenValues,
                 MeasurementMachineState::Measured => MeasuringOutcome::PassedOk,
                 MeasurementMachineState::MeasuringFailed => MeasuringOutcome::Unsuccessful((

crates/machine-controller/tests/integration/env.rs

@@ -117,6 +117,9 @@ impl EnvBuilder {
             scout_reporting_timeout: runtime_config
                 .machine_state_controller
                 .scout_reporting_timeout,
+            waiting_for_measurements_timeout: runtime_config
+                .machine_state_controller
+                .waiting_for_measurements_timeout,
             uefi_boot_wait: runtime_config.machine_state_controller.uefi_boot_wait,
         };
         let power_options: PowerOptionConfig = runtime_config.power_manager_options.clone().into();