From 07e44eb93bc195c2fd2ecb0afb2d9300bedc4e0d Mon Sep 17 00:00:00 2001 From: Chet Nichols III Date: Mon, 6 Jul 2026 15:37:50 -0700 Subject: [PATCH 1/5] feat: skip the SetBootOrder re-apply when the boot config is already set SetBootOrder re-applied a host's boot config on every pass -- re-asserting the HTTP-boot device (`machine_setup`) and setting the boot order -- even when it was already in place. It now checks first, and when the config is already set it goes straight to verification instead of re-writing it: an already-configured host skips a needless BIOS write and the reboot that applies it. That re-apply also caused a stall. On Dell, two BIOS config writes can't share one pending job, so re-asserting `machine_setup` (which commits a job) and then setting the boot order in the same pass collided -- iDRAC rejects the second with `SYS011` ("a config job is already committed") -- and the flow looped there. Now the re-assert runs only when the HTTP-boot device is actually reverted (the boot NIC dropped off the BMC's Redfish inventory on a reboot), and reboots to apply it before the boot-order set, so the two writes never share a pass. - Already configured -- skip the re-assert and the boot-order set; verify and move on. - HTTP-boot device reverted -- re-assert `machine_setup`, reboot to apply, then set the order. - Boot order not yet set (device fine) -- set the order without re-asserting; no colliding job. Tests updated! This supports https://github.com/NVIDIA/infra-controller/issues/3137 Signed-off-by: Chet Nichols III (cherry picked from commit 78aa3766a41029917daa4dec14dd22210775fa1e) --- crates/api-core/src/tests/machine_history.rs | 3 - crates/api-core/src/tests/machine_states.rs | 159 +++++++++++++------ crates/machine-controller/src/handler.rs | 96 ++++++++--- 3 files changed, 191 insertions(+), 67 deletions(-) diff --git a/crates/api-core/src/tests/machine_history.rs b/crates/api-core/src/tests/machine_history.rs index 28f57bb283..16660dfaff 100644 --- a/crates/api-core/src/tests/machine_history.rs +++ b/crates/api-core/src/tests/machine_history.rs @@ -52,9 +52,6 @@ async fn test_machine_state_history(pool: sqlx::PgPool) -> Result<(), Box Date: Mon, 6 Jul 2026 21:06:59 -0700 Subject: [PATCH 2/5] feat: apply SetBootOrder boot config once per reboot When SetBootOrder finds the HTTP-boot device reverted (the boot NIC dropped off the BMC's Redfish inventory on a reboot), it restores the device with `machine_setup` and restarts the host to apply it. That restore now happens once per revert: a new `WaitForHttpBootDeviceApplied` substate polls the device across the reboot -- the same wait choreography the boot-order job already has -- instead of the SetBootOrder pass re-writing the staged BIOS settings and re-creating the config job on every pass until the reboot lands. - While the reboot is in flight, the host waits in `WaitForHttpBootDeviceApplied`; once the device verifies, the flow returns to `SetBootOrder` to set the boot order. - A device still reverted after the wait window (it can drop off again on the apply reboot itself) returns to `SetBootOrder` for a fresh restore, so re-applies stay bounded to one per window rather than one per pass. - The restore path drops its two-way reboot branch for the unconditional restart the `RebootHost` substate already uses. Tests updated! This supports https://github.com/NVIDIA/infra-controller/issues/3166 Signed-off-by: Chet Nichols III (cherry picked from commit 064ce21bc2f7a59b7802df5b50b1920886a39df9) --- crates/api-core/src/tests/machine_states.rs | 152 ++++++++++++++++++++ crates/api-model/src/machine/mod.rs | 4 + crates/machine-controller/src/handler.rs | 124 ++++++++++++---- 3 files changed, 253 insertions(+), 27 deletions(-) diff --git a/crates/api-core/src/tests/machine_states.rs b/crates/api-core/src/tests/machine_states.rs index d17844f5a4..3f291876b1 100644 --- a/crates/api-core/src/tests/machine_states.rs +++ b/crates/api-core/src/tests/machine_states.rs @@ -3078,6 +3078,61 @@ async fn test_set_boot_order_reasserts_http_boot_device_when_reverted(pool: sqlx "the boot order must not be set in the same pass as the re-assert (shared BIOS job), got: {first_pass:?}" ); + // The re-assert is committed once. While the device is still reverted (the + // apply reboot hasn't landed yet), the host polls in + // WaitForHttpBootDeviceApplied without re-running machine_setup. + { + let mut txn = env.db_txn().await; + let host = mh.host().db_machine(&mut txn).await; + assert!( + matches!( + host.current_state(), + ManagedHostState::HostInit { + machine_state: MachineState::SetBootOrder { + set_boot_order_info: Some(SetBootOrderInfo { + set_boot_order_state: SetBootOrderState::WaitForHttpBootDeviceApplied, + .. + }), + }, + } + ), + "after the re-assert the host should wait for the device to apply, got: {:?}", + host.current_state() + ); + } + let after_reassert = env.redfish_sim.timepoint(); + env.run_machine_state_controller_iteration().await; + let second_pass = env.redfish_sim.actions_since(&after_reassert).all_hosts(); + assert!( + !second_pass + .iter() + .any(|a| matches!(a, RedfishSimAction::MachineSetup { .. })), + "machine_setup must not re-run while waiting for the re-asserted device to apply, got: {second_pass:?}" + ); + + // Within the wait window the host stays parked in + // WaitForHttpBootDeviceApplied -- it neither advances nor bounces back to + // SetBootOrder for another re-assert. + { + let mut txn = env.db_txn().await; + let host = mh.host().db_machine(&mut txn).await; + assert!( + matches!( + host.current_state(), + ManagedHostState::HostInit { + machine_state: MachineState::SetBootOrder { + set_boot_order_info: Some(SetBootOrderInfo { + set_boot_order_state: SetBootOrderState::WaitForHttpBootDeviceApplied, + .. + }), + }, + } + ), + "the host should keep waiting for the device to apply within the wait window, got: {:?}", + host.current_state() + ); + } + // Model the reboot applying the re-asserted config: the device now verifies. env.redfish_sim.set_is_bios_setup(true); @@ -3096,6 +3151,103 @@ async fn test_set_boot_order_reasserts_http_boot_device_when_reverted(pool: sqlx assert_machine_setup_precedes_reorder_for(&actions, boot_nic_mac); } +/// When the re-asserted HTTP-boot device does not verify within the wait +/// window (the boot NIC can drop off the BMC's Redfish inventory again on the +/// apply reboot itself), the host returns to `SetBootOrder` for a fresh +/// re-assert -- one `machine_setup` per window, not per pass. Once the state +/// machine's retry budget is exhausted it stops re-asserting and surfaces the +/// host for manual intervention. +#[crate::sqlx_test] +async fn test_set_boot_order_reassert_window_expiry_bounds_reapplies(pool: sqlx::PgPool) { + let env = create_zero_dpu_test_env(pool).await; + + let mh = create_managed_host_with_config(&env, ManagedHostConfig::zero_dpu()).await; + let host_id = mh.host().id; + + // The device stays reverted throughout: every verify reads false. + env.redfish_sim.set_is_bios_setup(false); + env.redfish_sim.set_is_boot_order_setup(false); + + let stuck_waiting = |retry_count: u32| ManagedHostState::HostInit { + machine_state: MachineState::SetBootOrder { + set_boot_order_info: Some(SetBootOrderInfo { + set_boot_order_jid: None, + set_boot_order_state: SetBootOrderState::WaitForHttpBootDeviceApplied, + retry_count, + }), + }, + }; + + // Plant the host mid-wait with the window already expired (backdated past + // the 10-minute apply window). Pass 1 returns it to SetBootOrder; pass 2 + // re-asserts once and parks it back in the wait substate. + set_host_controller_state_stuck_in(&env, host_id, &stuck_waiting(0), 11).await; + let checkpoint = env.redfish_sim.timepoint(); + env.run_machine_state_controller_iteration().await; + env.run_machine_state_controller_iteration().await; + let actions = env.redfish_sim.actions_since(&checkpoint).all_hosts(); + assert_eq!( + actions + .iter() + .filter(|a| matches!(a, RedfishSimAction::MachineSetup { .. })) + .count(), + 1, + "an expired wait window should produce exactly one fresh re-assert, got: {actions:?}" + ); + { + let mut txn = env.db_txn().await; + let host = mh.host().db_machine(&mut txn).await; + assert!( + matches!( + host.current_state(), + ManagedHostState::HostInit { + machine_state: MachineState::SetBootOrder { + set_boot_order_info: Some(SetBootOrderInfo { + set_boot_order_state: SetBootOrderState::WaitForHttpBootDeviceApplied, + retry_count: 1, + .. + }), + }, + } + ), + "the fresh re-assert should park the host back in the wait substate with the retry spent, got: {:?}", + host.current_state() + ); + } + + // Budget exhausted: the same expired window with the retry budget spent + // must not re-assert again -- the host stays parked for an operator. + set_host_controller_state_stuck_in(&env, host_id, &stuck_waiting(3), 11).await; + let checkpoint = env.redfish_sim.timepoint(); + env.run_machine_state_controller_iteration().await; + let actions = env.redfish_sim.actions_since(&checkpoint).all_hosts(); + assert!( + !actions + .iter() + .any(|a| matches!(a, RedfishSimAction::MachineSetup { .. })), + "an exhausted retry budget must not re-assert the device again, got: {actions:?}" + ); + { + let mut txn = env.db_txn().await; + let host = mh.host().db_machine(&mut txn).await; + assert!( + matches!( + host.current_state(), + ManagedHostState::HostInit { + machine_state: MachineState::SetBootOrder { + set_boot_order_info: Some(SetBootOrderInfo { + set_boot_order_state: SetBootOrderState::WaitForHttpBootDeviceApplied, + .. + }), + }, + } + ), + "the capped host should stay parked awaiting manual intervention, got: {:?}", + host.current_state() + ); + } +} + /// When HostInit/PollingBiosSetup retry budget is exhausted, enter Failed and recover via is_bios_setup. #[crate::sqlx_test] async fn test_polling_bios_setup_exhausted_enters_failed_and_recovers_when_bios_setup_true( diff --git a/crates/api-model/src/machine/mod.rs b/crates/api-model/src/machine/mod.rs index 02d01f9056..5e923fca26 100644 --- a/crates/api-model/src/machine/mod.rs +++ b/crates/api-model/src/machine/mod.rs @@ -1904,6 +1904,10 @@ pub struct SetBootOrderInfo { #[serde(tag = "state", rename_all = "lowercase")] pub enum SetBootOrderState { SetBootOrder, + /// A reverted HTTP-boot device was re-asserted (`machine_setup`) and the + /// host restarted to apply it; polls the device across that reboot, then + /// returns to `SetBootOrder` to set the boot order. + WaitForHttpBootDeviceApplied, WaitForSetBootOrderJobScheduled, RebootHost, WaitForSetBootOrderJobCompletion, diff --git a/crates/machine-controller/src/handler.rs b/crates/machine-controller/src/handler.rs index 9e2ecfe923..22120fd0ee 100644 --- a/crates/machine-controller/src/handler.rs +++ b/crates/machine-controller/src/handler.rs @@ -11263,10 +11263,13 @@ async fn set_host_boot_order( if !is_bios_setup { // The HTTP-boot device was reverted (the boot NIC dropped off the // BMC's Redfish inventory on a reboot), so re-assert it with - // `machine_setup` and reboot to apply it *before* setting the boot - // order -- the two BIOS writes never share one pass (they can't - // share one Dell config job). The next pass reads the device - // applied (`is_bios_setup` true) and continues to the order. + // `machine_setup` and restart the host to apply it *before* + // setting the boot order -- the two BIOS writes never share one + // pass (they can't share one Dell config job). The re-assert + // commits once: `WaitForHttpBootDeviceApplied` polls the device + // across the reboot instead of this arm re-writing the staged + // settings and re-creating the config job every pass while the + // reboot lands. call_machine_setup_and_handle_no_dpu_error( redfish_client, Some(&boot_interface), @@ -11276,29 +11279,15 @@ async fn set_host_boot_order( .await .map_err(|e| redfish_error("machine_setup", e))?; - let reboot_status = if mh_snapshot.host_snapshot.last_reboot_requested.is_none() { - handler_host_power_control(mh_snapshot, ctx, SystemPowerControl::ForceRestart) - .await?; - RebootStatus { - increase_retry_count: true, - status: "Restarted host to apply re-asserted HTTP boot device".to_string(), - } - } else { - trigger_reboot_if_needed( - &mh_snapshot.host_snapshot, - mh_snapshot, - None, - reachability_params, - ctx, - ) - .await? - }; - if reboot_status.increase_retry_count { - log_host_config(redfish_client, mh_snapshot).await; - } - return Ok(SetBootOrderOutcome::WaitingForReboot(format!( - "HTTP boot device was reverted; re-asserted it and rebooting to apply before setting the boot order: {reboot_status:#?}" - ))); + handler_host_power_control(mh_snapshot, ctx, SystemPowerControl::ForceRestart) + .await?; + log_host_config(redfish_client, mh_snapshot).await; + + return Ok(SetBootOrderOutcome::Continue(SetBootOrderInfo { + set_boot_order_jid: None, + set_boot_order_state: SetBootOrderState::WaitForHttpBootDeviceApplied, + retry_count: set_boot_order_info.retry_count, + })); } // HTTP-boot device is already set, so `machine_setup` was not re-run @@ -11388,6 +11377,87 @@ async fn set_host_boot_order( retry_count: set_boot_order_info.retry_count, })) } + SetBootOrderState::WaitForHttpBootDeviceApplied => { + // The SetBootOrder substate re-asserted the reverted HTTP-boot device + // and restarted the host; the staged BIOS settings apply across that + // reboot. Poll until the device verifies, then return to SetBootOrder + // to set the boot order. If it still hasn't applied once the reboot + // has had time to land (the boot NIC can drop off the BMC's Redfish + // inventory again on the apply reboot itself), return to SetBootOrder + // for a fresh re-assert -- re-commits stay bounded to one per wait + // window instead of one per pass. + const HTTP_BOOT_DEVICE_APPLY_WAIT_MINUTES: i64 = 10; + const MAX_HTTP_BOOT_DEVICE_APPLY_RETRIES: u32 = 3; + + let boot_interface = match resolve_boot_interface(mh_snapshot, &predictions) { + BootInterfaceResolution::Ready(target) => target, + BootInterfaceResolution::AwaitingNic => { + return Ok(SetBootOrderOutcome::Wait(format!( + "Waiting for zero-DPU host {} to discover its boot NIC before verifying the re-asserted HTTP boot device.", + mh_snapshot.host_snapshot.id + ))); + } + BootInterfaceResolution::Missing => { + return Err(StateHandlerError::GenericError(eyre::eyre!( + "Missing boot interface for host: {}", + mh_snapshot.host_snapshot.id + ))); + } + }; + + let is_bios_setup = boot_interface + .run(|bi| redfish_client.is_bios_setup(Some(bi))) + .await + .map_err(|e| redfish_error("is_bios_setup", e))?; + + if is_bios_setup { + tracing::info!( + machine_id = %mh_snapshot.host_snapshot.id, + "Re-asserted HTTP boot device applied; returning to SetBootOrder to set the boot order", + ); + return Ok(SetBootOrderOutcome::Continue(SetBootOrderInfo { + set_boot_order_jid: None, + set_boot_order_state: SetBootOrderState::SetBootOrder, + retry_count: set_boot_order_info.retry_count, + })); + } + + let minutes_waiting = mh_snapshot + .host_snapshot + .state + .version + .since_state_change() + .num_minutes(); + + if minutes_waiting >= HTTP_BOOT_DEVICE_APPLY_WAIT_MINUTES { + // Each expired window spends one retry from the SetBootOrder + // state machine's shared budget. Once it's exhausted the device + // is not coming back on its own -- stop re-asserting and surface + // the host for an operator, the same terminal escalation the + // reboot pacer applies to a host that never comes up. + if set_boot_order_info.retry_count >= MAX_HTTP_BOOT_DEVICE_APPLY_RETRIES { + return Err(StateHandlerError::ManualInterventionRequired(format!( + "HTTP boot device on host {} still not applied after {} re-asserts; manual intervention required", + mh_snapshot.host_snapshot.id, MAX_HTTP_BOOT_DEVICE_APPLY_RETRIES + ))); + } + tracing::warn!( + machine_id = %mh_snapshot.host_snapshot.id, + minutes_waiting, + "Re-asserted HTTP boot device still not applied after the wait window; returning to SetBootOrder to re-assert", + ); + return Ok(SetBootOrderOutcome::Continue(SetBootOrderInfo { + set_boot_order_jid: None, + set_boot_order_state: SetBootOrderState::SetBootOrder, + retry_count: set_boot_order_info.retry_count + 1, + })); + } + + Ok(SetBootOrderOutcome::WaitingForReboot(format!( + "Waiting for the re-asserted HTTP boot device to apply on host {} ({minutes_waiting}m of {HTTP_BOOT_DEVICE_APPLY_WAIT_MINUTES}m)", + mh_snapshot.host_snapshot.id + ))) + } SetBootOrderState::WaitForSetBootOrderJobScheduled => { if let Some(job_id) = &set_boot_order_info.set_boot_order_jid { let job_state = redfish_client From bed758ff6eadffb25450761cad8d15363914c7bc Mon Sep 17 00:00:00 2001 From: Chet Nichols III Date: Tue, 7 Jul 2026 06:41:09 -0700 Subject: [PATCH 3/5] refactor: resolve the boot NIC through one shared step Four Redfish boot steps -- the three boot-order substates (SetBootOrder, WaitForHttpBootDeviceApplied, CheckBootOrder) and host boot repair -- each opened with the same fifteen-line block resolving the host's boot NIC, with only the wait message differing. The resolution step now lives in one helper, `require_boot_interface`, and each caller states just the activity it's blocked on: a zero-DPU host still discovering its boot NIC maps to the caller's wait outcome, and a host with no resolvable interface stays a hard error. - One source of truth for the resolve-or-wait-or-fail choreography, so the boot steps can't drift apart as the state machine grows. - Behavior-preserving: every caller keeps its exact wait message (the helper templates "before "), and wait-vs-error semantics are unchanged. - Call sites shrink from fifteen lines to a handful each. This supports https://github.com/NVIDIA/infra-controller/issues/3193 Signed-off-by: Chet Nichols III (cherry picked from commit 428135d4fe7b67a466fc96805f1b82ed21cded19) --- crates/machine-controller/src/handler.rs | 199 ++++++++++++++++------- 1 file changed, 143 insertions(+), 56 deletions(-) diff --git a/crates/machine-controller/src/handler.rs b/crates/machine-controller/src/handler.rs index 22120fd0ee..42e621fd0f 100644 --- a/crates/machine-controller/src/handler.rs +++ b/crates/machine-controller/src/handler.rs @@ -3573,20 +3573,14 @@ async fn check_host_boot_config( // falls back to its predicted boot NIC, and only waits when even that is // unavailable. let predictions = load_boot_predictions(ctx, &mh_snapshot.host_snapshot.id).await?; - let boot_interface = match resolve_boot_interface(mh_snapshot, &predictions) { - BootInterfaceResolution::Ready(target) => target, - BootInterfaceResolution::AwaitingNic => { - return Ok(HostBootConfigDecision::Wait(format!( - "Waiting for zero-DPU host {} to discover its boot NIC before configuring boot.", - mh_snapshot.host_snapshot.id - ))); - } - BootInterfaceResolution::Missing => { - return Err(StateHandlerError::GenericError(eyre::eyre!( - "Missing boot interface for host: {}", - mh_snapshot.host_snapshot.id - ))); - } + let boot_interface = match require_boot_interface( + mh_snapshot, + &predictions, + "configuring boot", + HostBootConfigDecision::Wait, + )? { + RequiredBootInterface::Ready(target) => target, + RequiredBootInterface::Wait(decision) => return Ok(decision), }; let vendor = mh_snapshot.host_snapshot.bmc_vendor(); @@ -4956,6 +4950,117 @@ enum HostBootConfigDpuFreshness { SinceLastHostRebootRequest, } +/// Outcome of [`require_boot_interface`]: the resolved boot NIC, or the +/// caller's wait outcome for a zero-DPU host still discovering its boot NIC. +#[derive(Debug)] +enum RequiredBootInterface { + Ready(BootInterfaceTarget), + Wait(W), +} + +/// Resolve the boot NIC for a Redfish boot step, folding the not-ready cases +/// every caller handles the same way: a zero-DPU host that has not discovered +/// its boot NIC yet maps to the caller's wait outcome (`wait` wraps the shared +/// message; `activity` names the blocked step), and a host with no resolvable +/// interface is a hard error. Keeps the boot-order substates and host boot +/// repair resolving the boot NIC identically. +fn require_boot_interface( + mh_snapshot: &ManagedHostStateSnapshot, + predictions: &[PredictedMachineInterface], + activity: &str, + wait: impl FnOnce(String) -> W, +) -> Result, StateHandlerError> { + map_boot_interface_resolution( + resolve_boot_interface(mh_snapshot, predictions), + &mh_snapshot.host_snapshot.id, + activity, + wait, + ) +} + +/// The mapping behind [`require_boot_interface`], split out from the snapshot +/// lookup so it can be unit-tested directly. +fn map_boot_interface_resolution( + resolution: BootInterfaceResolution, + host_id: &MachineId, + activity: &str, + wait: impl FnOnce(String) -> W, +) -> Result, StateHandlerError> { + match resolution { + BootInterfaceResolution::Ready(target) => Ok(RequiredBootInterface::Ready(target)), + BootInterfaceResolution::AwaitingNic => Ok(RequiredBootInterface::Wait(wait(format!( + "Waiting for zero-DPU host {host_id} to discover its boot NIC before {activity}." + )))), + BootInterfaceResolution::Missing => Err(StateHandlerError::GenericError(eyre::eyre!( + "Missing boot interface for host: {host_id}" + ))), + } +} + +#[cfg(test)] +mod require_boot_interface_tests { + use super::*; + + fn host_id() -> MachineId { + "fm100ds7blqjsadm2uuh3qqbf1h7k8pmf47um6v9uckrg7l03po8mhqgvng" + .parse() + .unwrap() + } + + // Ready passes the resolved target through untouched. + #[test] + fn ready_passes_the_target_through() { + let target = BootInterfaceTarget::MacOnly("20:00:00:00:00:01".parse().unwrap()); + let resolved = map_boot_interface_resolution::( + BootInterfaceResolution::Ready(target.clone()), + &host_id(), + "setting boot order", + |msg| msg, + ) + .unwrap(); + let RequiredBootInterface::Ready(out) = resolved else { + panic!("expected Ready"); + }; + assert_eq!(out, target); + } + + // AwaitingNic becomes the caller's wait outcome, built from the shared + // message with the caller's activity spliced in. + #[test] + fn awaiting_nic_maps_to_the_callers_wait_outcome() { + let resolved = map_boot_interface_resolution::( + BootInterfaceResolution::AwaitingNic, + &host_id(), + "setting boot order", + |msg| msg, + ) + .unwrap(); + let RequiredBootInterface::Wait(msg) = resolved else { + panic!("expected Wait"); + }; + assert_eq!( + msg, + format!( + "Waiting for zero-DPU host {} to discover its boot NIC before setting boot order.", + host_id() + ) + ); + } + + // Missing is a hard error, not a wait. + #[test] + fn missing_is_a_hard_error() { + let err = map_boot_interface_resolution::( + BootInterfaceResolution::Missing, + &host_id(), + "setting boot order", + |msg| msg, + ) + .unwrap_err(); + assert!(matches!(err, StateHandlerError::GenericError(_))); + } +} + /// In case machine does not come up until a specified duration, this function tries to reboot /// it again. The reboot continues till 6 hours only. After that this function gives up. /// WARNING: @@ -11227,20 +11332,14 @@ async fn set_host_boot_order( // HostInband lease creates a real row, a zero-DPU/NIC-mode host // resolves via its predictions; it waits only when neither a real row // nor a usable prediction exists. - let boot_interface = match resolve_boot_interface(mh_snapshot, &predictions) { - BootInterfaceResolution::Ready(target) => target, - BootInterfaceResolution::AwaitingNic => { - return Ok(SetBootOrderOutcome::Wait(format!( - "Waiting for zero-DPU host {} to discover its boot NIC before setting boot order.", - mh_snapshot.host_snapshot.id - ))); - } - BootInterfaceResolution::Missing => { - return Err(StateHandlerError::GenericError(eyre::eyre!( - "Missing boot interface for host: {}", - mh_snapshot.host_snapshot.id - ))); - } + let boot_interface = match require_boot_interface( + mh_snapshot, + &predictions, + "setting boot order", + SetBootOrderOutcome::Wait, + )? { + RequiredBootInterface::Ready(target) => target, + RequiredBootInterface::Wait(outcome) => return Ok(outcome), }; // Don't re-apply a boot config that's already in place. `SetBootOrder` @@ -11389,20 +11488,14 @@ async fn set_host_boot_order( const HTTP_BOOT_DEVICE_APPLY_WAIT_MINUTES: i64 = 10; const MAX_HTTP_BOOT_DEVICE_APPLY_RETRIES: u32 = 3; - let boot_interface = match resolve_boot_interface(mh_snapshot, &predictions) { - BootInterfaceResolution::Ready(target) => target, - BootInterfaceResolution::AwaitingNic => { - return Ok(SetBootOrderOutcome::Wait(format!( - "Waiting for zero-DPU host {} to discover its boot NIC before verifying the re-asserted HTTP boot device.", - mh_snapshot.host_snapshot.id - ))); - } - BootInterfaceResolution::Missing => { - return Err(StateHandlerError::GenericError(eyre::eyre!( - "Missing boot interface for host: {}", - mh_snapshot.host_snapshot.id - ))); - } + let boot_interface = match require_boot_interface( + mh_snapshot, + &predictions, + "verifying the re-asserted HTTP boot device", + SetBootOrderOutcome::Wait, + )? { + RequiredBootInterface::Ready(target) => target, + RequiredBootInterface::Wait(outcome) => return Ok(outcome), }; let is_bios_setup = boot_interface @@ -11669,20 +11762,14 @@ async fn set_host_boot_order( let retry_count = set_boot_order_info.retry_count; - let boot_interface = match resolve_boot_interface(mh_snapshot, &predictions) { - BootInterfaceResolution::Ready(target) => target, - BootInterfaceResolution::AwaitingNic => { - return Ok(SetBootOrderOutcome::Wait(format!( - "Waiting for zero-DPU host {} to discover its boot NIC before verifying boot order.", - mh_snapshot.host_snapshot.id - ))); - } - BootInterfaceResolution::Missing => { - return Err(StateHandlerError::GenericError(eyre::eyre!( - "Missing boot interface for host: {}", - mh_snapshot.host_snapshot.id - ))); - } + let boot_interface = match require_boot_interface( + mh_snapshot, + &predictions, + "verifying boot order", + SetBootOrderOutcome::Wait, + )? { + RequiredBootInterface::Ready(target) => target, + RequiredBootInterface::Wait(outcome) => return Ok(outcome), }; let boot_order_configured = boot_interface From fd22f0a0299df3fc797e0fe7842188e4403de11c Mon Sep 17 00:00:00 2001 From: Chet Nichols III Date: Fri, 10 Jul 2026 13:07:05 -0700 Subject: [PATCH 4/5] feat: ensure a correct boot config during machine validation (#3369) Machine validation now ensures the host's boot device config is what it should be, and corrects it when it isn't. While waiting for the host to boot into the validation environment, validation verifies the boot config; if it reads reverted -- however it drifted, whether someone changed it externally, a BIOS quirk, or the boot NIC dropping off the BMC's inventory during the reboot's POST -- validation unlocks the BMC, re-drives the boot-order flow (re-assert the HTTP-boot device, reboot to apply, set and verify the order), re-locks, and resumes from its reboot step. Without this, a drifted boot config was unrecoverable at this stage: the host can't boot into scout, more reboots can't restore a setting that is gone, and the BMC lockdown enabled earlier in HostInit blocks any fix -- so validation paced reboots indefinitely. - Verification lives in the wait-for-reboot path: an intact config keeps the normal reboot pacing; only a reverted one enters correction. - The correction reuses the boot-order flow and the host-boot-repair unlock/re-lock choreography, honoring the expected-machine lockdown policy. - New validation substates (PrepareBootRepair, UnlockForBootRepair, RepairBootConfig, LockAfterBootRepair) surface the correction in state history and metrics. Tests updated! This supports https://github.com/NVIDIA/infra-controller/issues/3365 Signed-off-by: Chet Nichols III Signed-off-by: Chet Nichols III (cherry picked from commit a2ab63f2ca123b3c9335e99bd12f1717c0184ad7) --- .../src/tests/common/api_fixtures/mod.rs | 4 + crates/api-core/src/tests/machine_states.rs | 111 ++++++++ crates/api-model/src/machine/mod.rs | 25 ++ .../src/handler/machine_validation.rs | 269 +++++++++++++++++- crates/machine-controller/src/io.rs | 4 + 5 files changed, 410 insertions(+), 3 deletions(-) diff --git a/crates/api-core/src/tests/common/api_fixtures/mod.rs b/crates/api-core/src/tests/common/api_fixtures/mod.rs index 0c1e5251f0..f1a7770545 100644 --- a/crates/api-core/src/tests/common/api_fixtures/mod.rs +++ b/crates/api-core/src/tests/common/api_fixtures/mod.rs @@ -469,6 +469,10 @@ impl TestEnv { } } MachineValidatingState::RebootHost { .. } => state.clone(), + MachineValidatingState::PrepareBootRepair { .. } + | MachineValidatingState::UnlockForBootRepair { .. } + | MachineValidatingState::RepairBootConfig { .. } + | MachineValidatingState::LockAfterBootRepair { .. } => state.clone(), } } }, diff --git a/crates/api-core/src/tests/machine_states.rs b/crates/api-core/src/tests/machine_states.rs index 3f291876b1..088950fda8 100644 --- a/crates/api-core/src/tests/machine_states.rs +++ b/crates/api-core/src/tests/machine_states.rs @@ -3248,6 +3248,117 @@ async fn test_set_boot_order_reassert_window_expiry_bounds_reapplies(pool: sqlx: } } +/// A host that cannot return from a validation reboot because its boot config +/// reverted (the boot NIC dropped off the BMC's inventory during POST, so the +/// BIOS reset the HTTP-boot device) gets repaired in place: validation unlocks +/// the BMC, re-drives the boot-order flow (re-assert, reboot to apply, +/// reorder, verify), re-locks, and resumes validation from its reboot step -- +/// instead of pacing reboots that can never succeed. +#[crate::sqlx_test] +async fn test_machine_validation_repairs_reverted_boot_config(pool: sqlx::PgPool) { + let env = create_zero_dpu_test_env(pool).await; + + let mh = create_managed_host_with_config(&env, ManagedHostConfig::zero_dpu()).await; + let host_id = mh.host().id; + let boot_nic_mac = host_inband_nic_mac(&env, host_id).await; + + // The validation reboot's POST reverted the boot config, and HostInit + // enabled lockdown before validation reached this point: BIOS writes + // bounce off the BMC until it is unlocked. + env.redfish_sim.set_is_bios_setup(false); + env.redfish_sim.set_is_boot_order_setup(false); + env.redfish_sim + .set_lockdown(libredfish::EnabledDisabled::Enabled); + + // Park the host mid-validation, waiting on a reboot that cannot land + // (the state is newer than the host's last reported boot). + set_host_controller_state_stuck_in( + &env, + host_id, + &ManagedHostState::Validation { + validation_state: ValidationState::MachineValidation { + machine_validation: MachineValidatingState::MachineValidating { + context: "Discovery".to_string(), + id: MachineValidationId::new(), + completed: 1, + total: 1, + is_enabled: true, + }, + }, + }, + 0, + ) + .await; + + let checkpoint = env.redfish_sim.timepoint(); + + // Detection through re-assert: the flow notices the reverted config, + // unlocks the BMC, and re-asserts the HTTP-boot device. + for _ in 0..6 { + env.run_machine_state_controller_iteration().await; + if env + .redfish_sim + .actions_since(&checkpoint) + .all_hosts() + .iter() + .any(|a| matches!(a, RedfishSimAction::MachineSetup { .. })) + { + break; + } + } + let actions = env.redfish_sim.actions_since(&checkpoint).all_hosts(); + assert!( + actions + .iter() + .any(|a| matches!(a, RedfishSimAction::MachineSetup { .. })), + "validation boot repair should re-assert the reverted HTTP boot device, got: {actions:?}" + ); + assert!( + env.redfish_sim + .lockdown_states() + .contains(&libredfish::EnabledDisabled::Disabled), + "boot repair should unlock the BMC before writing BIOS settings" + ); + + // The repair reboot applies the re-asserted device. + env.redfish_sim.set_is_bios_setup(true); + + // The repair completes (reorder + verify), re-locks the BMC, and resumes + // validation from its reboot step. + let mut resumed = false; + for _ in 0..20 { + env.run_machine_state_controller_iteration().await; + let mut txn = env.db_txn().await; + let host = mh.host().db_machine(&mut txn).await; + if matches!( + host.current_state(), + ManagedHostState::Validation { + validation_state: ValidationState::MachineValidation { + machine_validation: MachineValidatingState::RebootHost { .. }, + }, + } + ) { + resumed = true; + break; + } + } + assert!( + resumed, + "the repaired host should resume validation from its reboot step" + ); + assert!( + env.redfish_sim + .lockdown_states() + .iter() + .all(|l| *l == libredfish::EnabledDisabled::Enabled), + "boot repair should re-lock the BMC once the boot config is repaired" + ); + + // Across the repair, the re-assert targeting the boot NIC preceded the reorder. + let actions = env.redfish_sim.actions_since(&checkpoint).all_hosts(); + assert_machine_setup_precedes_reorder_for(&actions, boot_nic_mac); +} + /// When HostInit/PollingBiosSetup retry budget is exhausted, enter Failed and recover via is_bios_setup. #[crate::sqlx_test] async fn test_polling_bios_setup_exhausted_enters_failed_and_recovers_when_bios_setup_true( diff --git a/crates/api-model/src/machine/mod.rs b/crates/api-model/src/machine/mod.rs index 5e923fca26..5f35ff8263 100644 --- a/crates/api-model/src/machine/mod.rs +++ b/crates/api-model/src/machine/mod.rs @@ -1284,6 +1284,25 @@ pub enum MachineValidatingState { #[serde(default = "default_true")] is_enabled: bool, }, + /// Machine validation ensures the host's boot device config is in place. + /// When it reads reverted -- however it drifted (changed externally, a + /// BIOS quirk, or the boot NIC dropping off the BMC's inventory during a + /// reboot's POST) -- these states correct it, mirroring host boot repair: + /// unlock the BMC, drive the boot-order flow, re-lock, resume validation. + PrepareBootRepair { + validation_id: MachineValidationId, + }, + UnlockForBootRepair { + validation_id: MachineValidationId, + unlock_host_state: UnlockHostState, + }, + RepairBootConfig { + validation_id: MachineValidationId, + set_boot_order_info: SetBootOrderInfo, + }, + LockAfterBootRepair { + validation_id: MachineValidationId, + }, } #[derive(Debug, Clone, Serialize, Deserialize, Eq, PartialEq)] #[serde(tag = "validation_type", rename_all = "lowercase")] @@ -2654,6 +2673,12 @@ pub fn state_sla( MachineValidatingState::RebootHost { .. } => { StateSla::with_sla(slas::VALIDATION, time_in_state) } + MachineValidatingState::PrepareBootRepair { .. } + | MachineValidatingState::UnlockForBootRepair { .. } + | MachineValidatingState::RepairBootConfig { .. } + | MachineValidatingState::LockAfterBootRepair { .. } => { + StateSla::with_sla(slas::VALIDATION, time_in_state) + } }, }, } diff --git a/crates/machine-controller/src/handler/machine_validation.rs b/crates/machine-controller/src/handler/machine_validation.rs index 79f975d0cd..7950f844cd 100644 --- a/crates/machine-controller/src/handler/machine_validation.rs +++ b/crates/machine-controller/src/handler/machine_validation.rs @@ -15,10 +15,10 @@ * limitations under the License. */ use carbide_uuid::machine_validation::MachineValidationId; -use libredfish::SystemPowerControl; +use libredfish::{EnabledDisabled, RedfishError, SystemPowerControl}; use model::machine::{ FailureCause, MachineState, MachineValidatingState, ManagedHostState, ManagedHostStateSnapshot, - ValidationState, + SetBootOrderInfo, SetBootOrderState, UnlockHostState, ValidationState, }; use model::machine_validation::{MachineValidationState, MachineValidationStatus}; use state_controller::state_handler::{ @@ -27,7 +27,30 @@ use state_controller::state_handler::{ use super::{HostHandlerParams, is_machine_validation_requested, machine_validation_completed}; use crate::context::{MachineStateHandlerContextObjects, MachineStateHandlerServices}; -use crate::handler::{handler_host_power_control, rebooted, trigger_reboot_if_needed}; +use crate::handler::{ + RequiredBootInterface, SetBootOrderOutcome, handler_host_power_control, host_power_control, + load_boot_predictions, rebooted, redfish_error, require_boot_interface, set_host_boot_order, + trigger_reboot_if_needed, wait, +}; + +/// The validation flavor of the managed-host state, so the repair arms can +/// transition between validation substates without repeating the wrapper. +fn validating(machine_validation: MachineValidatingState) -> ManagedHostState { + ManagedHostState::Validation { + validation_state: ValidationState::MachineValidation { machine_validation }, + } +} + +/// The starting point for the boot-order flow when validation boot repair +/// re-drives it: the flow itself checks what is already in place and only +/// re-applies what the reboot reverted. +fn boot_repair_start() -> SetBootOrderInfo { + SetBootOrderInfo { + set_boot_order_jid: None, + set_boot_order_state: SetBootOrderState::SetBootOrder, + retry_count: 0, + } +} async fn skip_machine_validation( ctx: &mut StateHandlerContext<'_, MachineStateHandlerContextObjects>, @@ -122,6 +145,46 @@ pub(crate) async fn handle_machine_validation_state( is_enabled, ); if !rebooted(&mh_snapshot.host_snapshot) { + // Ensure the boot config is still what it should be while + // waiting. If it reads reverted -- changed externally, a BIOS + // quirk, or the boot NIC dropping off the BMC's inventory + // during the reboot's POST -- the host can't boot into the + // validation environment, and further reboots can't restore a + // setting that is gone. Correct it instead of pacing forever. + let predictions = load_boot_predictions(ctx, &mh_snapshot.host_snapshot.id).await?; + if let RequiredBootInterface::Ready(boot_interface) = require_boot_interface( + mh_snapshot, + &predictions, + "verifying the boot config during validation", + |message| message, + )? { + let redfish_client = ctx + .services + .create_redfish_client_from_machine(&mh_snapshot.host_snapshot) + .await?; + match boot_interface + .run(|bi| redfish_client.is_bios_setup(Some(bi))) + .await + { + Ok(false) => { + tracing::warn!( + machine_id = %mh_snapshot.host_snapshot.id, + "Boot config reads reverted while waiting for the validation reboot; correcting it via boot repair", + ); + return Ok(StateHandlerOutcome::transition(validating( + MachineValidatingState::PrepareBootRepair { validation_id: *id }, + ))); + } + Ok(true) => {} + Err(e) => { + tracing::warn!( + machine_id = %mh_snapshot.host_snapshot.id, + error = %e, + "Could not verify the boot config while waiting for the validation reboot; will retry", + ); + } + } + } let status = trigger_reboot_if_needed( &mh_snapshot.host_snapshot, mh_snapshot, @@ -174,6 +237,206 @@ pub(crate) async fn handle_machine_validation_state( } Ok(StateHandlerOutcome::do_nothing()) } + MachineValidatingState::PrepareBootRepair { validation_id } => { + // Boot repair writes BIOS settings, and lockdown was enabled + // earlier in HostInit -- writes bounce off a locked BMC. Check and + // unlock first, mirroring host boot repair. + let redfish_client = ctx + .services + .create_redfish_client_from_machine(&mh_snapshot.host_snapshot) + .await?; + + let next = match redfish_client.lockdown_status().await { + Err(RedfishError::NotSupported(_)) => { + tracing::info!( + machine_id = %mh_snapshot.host_snapshot.id, + "BMC vendor does not support checking lockdown status during validation boot repair", + ); + MachineValidatingState::RepairBootConfig { + validation_id: *validation_id, + set_boot_order_info: boot_repair_start(), + } + } + Err(e) => { + tracing::warn!( + machine_id = %mh_snapshot.host_snapshot.id, + error = %e, + "Failed to fetch lockdown status during validation boot repair", + ); + return Ok(StateHandlerOutcome::wait(format!( + "Failed to fetch lockdown status: {e}" + ))); + } + Ok(lockdown_status) if !lockdown_status.is_fully_disabled() => { + tracing::info!( + machine_id = %mh_snapshot.host_snapshot.id, + "Lockdown is enabled during validation boot repair; disabling before boot config writes", + ); + MachineValidatingState::UnlockForBootRepair { + validation_id: *validation_id, + unlock_host_state: UnlockHostState::DisableLockdown, + } + } + Ok(_) => MachineValidatingState::RepairBootConfig { + validation_id: *validation_id, + set_boot_order_info: boot_repair_start(), + }, + }; + + Ok(StateHandlerOutcome::transition(validating(next))) + } + MachineValidatingState::UnlockForBootRepair { + validation_id, + unlock_host_state, + } => { + // Mirror host boot repair's unlock choreography. + let redfish_client = ctx + .services + .create_redfish_client_from_machine(&mh_snapshot.host_snapshot) + .await?; + + let next = match unlock_host_state { + UnlockHostState::DisableLockdown => { + // Tolerate a vendor that reports lockdown status but does + // not support setting it, symmetric with the re-lock step. + match redfish_client.lockdown_bmc(EnabledDisabled::Disabled).await { + Ok(()) => {} + Err(RedfishError::NotSupported(_)) => { + tracing::info!( + machine_id = %mh_snapshot.host_snapshot.id, + "BMC vendor does not support disabling lockdown during validation boot repair", + ); + } + Err(e) => return Err(redfish_error("lockdown_bmc", e)), + } + + let vendor = mh_snapshot.host_snapshot.bmc_vendor(); + if vendor.is_supermicro() { + tracing::info!( + machine_id = %mh_snapshot.host_snapshot.id, + %vendor, + "BMC lockdown disabled; rebooting host so Redfish reflects actual boot state", + ); + MachineValidatingState::UnlockForBootRepair { + validation_id: *validation_id, + unlock_host_state: UnlockHostState::RebootHost, + } + } else { + MachineValidatingState::RepairBootConfig { + validation_id: *validation_id, + set_boot_order_info: boot_repair_start(), + } + } + } + UnlockHostState::RebootHost => { + host_power_control( + redfish_client.as_ref(), + &mh_snapshot.host_snapshot, + SystemPowerControl::ForceRestart, + ctx, + ) + .await + .map_err(|e| { + StateHandlerError::GenericError(eyre::eyre!( + "failed to ForceRestart host after disabling BMC lockdown: {}", + e + )) + })?; + + MachineValidatingState::UnlockForBootRepair { + validation_id: *validation_id, + unlock_host_state: UnlockHostState::WaitForUefiBoot, + } + } + UnlockHostState::WaitForUefiBoot => { + let entered_at = mh_snapshot.host_snapshot.state.version.timestamp(); + if wait( + &entered_at, + host_handler_params.reachability_params.uefi_boot_wait, + ) { + return Ok(StateHandlerOutcome::wait(format!( + "Waiting for UEFI boot to complete on {} after post-unlock reboot", + mh_snapshot.host_snapshot.id + ))); + } + MachineValidatingState::RepairBootConfig { + validation_id: *validation_id, + set_boot_order_info: boot_repair_start(), + } + } + }; + + Ok(StateHandlerOutcome::transition(validating(next))) + } + MachineValidatingState::RepairBootConfig { + validation_id, + set_boot_order_info, + } => { + // Re-drive the boot-order flow: it checks what is already in + // place, re-asserts the reverted HTTP-boot device (rebooting to + // apply it), sets the boot order, and verifies -- the same engine + // the boot-configuration stages use. + let redfish_client = ctx + .services + .create_redfish_client_from_machine(&mh_snapshot.host_snapshot) + .await?; + + match set_host_boot_order( + ctx, + &host_handler_params.reachability_params, + redfish_client.as_ref(), + mh_snapshot, + set_boot_order_info.clone(), + ) + .await? + { + SetBootOrderOutcome::Continue(info) => Ok(StateHandlerOutcome::transition( + validating(MachineValidatingState::RepairBootConfig { + validation_id: *validation_id, + set_boot_order_info: info, + }), + )), + SetBootOrderOutcome::Done => Ok(StateHandlerOutcome::transition(validating( + MachineValidatingState::LockAfterBootRepair { + validation_id: *validation_id, + }, + ))), + SetBootOrderOutcome::WaitingForReboot(reason) + | SetBootOrderOutcome::Wait(reason) => Ok(StateHandlerOutcome::wait(reason)), + } + } + MachineValidatingState::LockAfterBootRepair { validation_id } => { + // Restore the lockdown that boot repair temporarily opened, then + // resume validation from its reboot step. + let redfish_client = ctx + .services + .create_redfish_client_from_machine(&mh_snapshot.host_snapshot) + .await?; + + if mh_snapshot.host_snapshot.host_profile.disable_lockdown { + tracing::info!( + machine_id = %mh_snapshot.host_snapshot.id, + "Skipping lockdown re-enable after validation boot repair per expected-machine config", + ); + } else { + match redfish_client.lockdown_bmc(EnabledDisabled::Enabled).await { + Ok(()) => {} + Err(RedfishError::NotSupported(_)) => { + tracing::info!( + machine_id = %mh_snapshot.host_snapshot.id, + "BMC vendor does not support re-enabling lockdown after validation boot repair", + ); + } + Err(e) => return Err(redfish_error("lockdown_bmc", e)), + } + } + + Ok(StateHandlerOutcome::transition(validating( + MachineValidatingState::RebootHost { + validation_id: *validation_id, + }, + ))) + } } } diff --git a/crates/machine-controller/src/io.rs b/crates/machine-controller/src/io.rs index d514116ff1..e831a4d975 100644 --- a/crates/machine-controller/src/io.rs +++ b/crates/machine-controller/src/io.rs @@ -264,6 +264,10 @@ impl StateControllerIO for MachineStateControllerIO { match validation_state { MachineValidatingState::MachineValidating { .. } => "machinevalidating", MachineValidatingState::RebootHost { .. } => "reboothost", + MachineValidatingState::PrepareBootRepair { .. } => "preparebootrepair", + MachineValidatingState::UnlockForBootRepair { .. } => "unlockforbootrepair", + MachineValidatingState::RepairBootConfig { .. } => "repairbootconfig", + MachineValidatingState::LockAfterBootRepair { .. } => "lockafterbootrepair", } } match state { From 6ad9361aade053db45775a620f9b2f8bac52d89c Mon Sep 17 00:00:00 2001 From: Chet Nichols III Date: Tue, 14 Jul 2026 11:43:35 -0700 Subject: [PATCH 5/5] refactor: generalized convergence for host boot interface configuration (#3454) Host boot configuration now runs through one controller path across host initialization, assigned platform setup, DPU reprovision, and validation. Each lifecycle keeps its own persisted state while sharing BIOS setup, vendor jobs, polling, boot order, and final verification. Primary callouts are: - `host_boot_config` selects the smallest repair and advances the shared flow without introducing a new serialized wrapper. - Order-only drift skips `machine_setup`, while late BIOS drift returns to the job-aware path and carries a bounded convergence budget. - Dell job IDs stay persisted through reboot and recovery; validation keeps its run identity and closes the run on terminal repair failure. - Viking and legacy persisted states keep their safe recovery behavior. Tests updated! This supports https://github.com/NVIDIA/infra-controller/issues/3391 Signed-off-by: Chet Nichols III (cherry picked from commit 9d80c7fcda5866340abdceb857c3e13ef49a4c48) --- Cargo.lock | 2 + crates/api-core/src/cfg/README.md | 2 +- .../src/tests/common/api_fixtures/mod.rs | 4 + .../api-core/src/tests/dpu_reprovisioning.rs | 190 ++- crates/api-core/src/tests/machine_states.rs | 680 +++++++-- crates/api-model/src/machine/mod.rs | 33 +- crates/machine-controller/Cargo.toml | 2 + .../src/config/controller.rs | 4 +- crates/machine-controller/src/handler.rs | 1233 ++++++----------- .../src/handler/bios_config.rs | 2 +- .../src/handler/host_boot_config.rs | 542 ++++++++ .../src/handler/machine_validation.rs | 259 +++- crates/machine-controller/src/io.rs | 6 + 13 files changed, 2029 insertions(+), 930 deletions(-) create mode 100644 crates/machine-controller/src/handler/host_boot_config.rs diff --git a/Cargo.lock b/Cargo.lock index 9761f8d4fc..c920eaab58 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2310,6 +2310,8 @@ dependencies = [ "carbide-redfish", "carbide-secrets", "carbide-state-controller-common", + "carbide-test-harness", + "carbide-test-support", "carbide-utils", "carbide-uuid", "chrono", diff --git a/crates/api-core/src/cfg/README.md b/crates/api-core/src/cfg/README.md index 7cf3b41736..06bec6264d 100644 --- a/crates/api-core/src/cfg/README.md +++ b/crates/api-core/src/cfg/README.md @@ -329,7 +329,7 @@ Extends `StateControllerConfig` with: | `dpu_up_threshold` | `Duration` | `5m` | Max time without DPU health report before assuming it's down. | | `scout_reporting_timeout` | `Duration` | `5m` | Duration without scout report before host is unhealthy. | | `uefi_boot_wait` | `Duration` | `5m` | Wait time for UEFI boot completion after host reboot. | -| `max_bios_config_retries` | `u32` | `3` | Max HandleBiosJobFailure recovery cycles during BIOS configuration. | +| `max_bios_config_retries` | `u32` | `3` | Shared retry budget for automated host boot-configuration convergence across BIOS recovery and boot-order verification. | | `polling_bios_setup_stuck_threshold` | `Duration` | `15m` | Time in PollingBiosSetup with `is_bios_setup == false` before recovery escalation. | ### `NetworkSegmentStateControllerConfig` diff --git a/crates/api-core/src/tests/common/api_fixtures/mod.rs b/crates/api-core/src/tests/common/api_fixtures/mod.rs index f1a7770545..a034f2a441 100644 --- a/crates/api-core/src/tests/common/api_fixtures/mod.rs +++ b/crates/api-core/src/tests/common/api_fixtures/mod.rs @@ -471,6 +471,10 @@ impl TestEnv { MachineValidatingState::RebootHost { .. } => state.clone(), MachineValidatingState::PrepareBootRepair { .. } | MachineValidatingState::UnlockForBootRepair { .. } + | MachineValidatingState::CheckBootConfigForRepair { .. } + | MachineValidatingState::ConfigureBootBios { .. } + | MachineValidatingState::WaitingForBootBiosJob { .. } + | MachineValidatingState::PollingBootBiosSetup { .. } | MachineValidatingState::RepairBootConfig { .. } | MachineValidatingState::LockAfterBootRepair { .. } => state.clone(), } diff --git a/crates/api-core/src/tests/dpu_reprovisioning.rs b/crates/api-core/src/tests/dpu_reprovisioning.rs index 20c6a1f82a..6c3422ff50 100644 --- a/crates/api-core/src/tests/dpu_reprovisioning.rs +++ b/crates/api-core/src/tests/dpu_reprovisioning.rs @@ -28,8 +28,8 @@ use libredfish::{EnabledDisabled, SystemPowerControl}; use model::instance::status::tenant::TenantState; use model::machine::{ DpuInitState, FailureCause, FailureDetails, FailureSource, InstallDpuOsState, InstanceState, - Machine, MachineLastRebootRequestedMode, MachineState, ManagedHostState, ReprovisionState, - SetBootOrderInfo, SetBootOrderState, StateMachineArea, UnlockHostState, + Machine, MachineLastRebootRequestedMode, MachineState, ManagedHostState, PowerState, + ReprovisionState, SetBootOrderInfo, SetBootOrderState, StateMachineArea, UnlockHostState, }; use model::test_support::HardwareInfoTemplate; use rpc::forge::MachineArchitecture; @@ -86,8 +86,6 @@ fn reprovision_host_boot_repair_states( unlock_host_state: UnlockHostState::DisableLockdown, }, ReprovisionState::CheckHostBootConfig, - ReprovisionState::ConfigureHostBoot { retry_count: 0 }, - ReprovisionState::PollingHostBiosSetup { retry_count: 0 }, reprovision_set_host_boot_order_state(SetBootOrderState::SetBootOrder), reprovision_set_host_boot_order_state(SetBootOrderState::WaitForSetBootOrderJobScheduled), reprovision_set_host_boot_order_state(SetBootOrderState::RebootHost), @@ -137,6 +135,7 @@ async fn assert_dpu_reprovision_host_boot_repair( expected_states: Vec, ) -> Machine { env.redfish_sim.set_lockdown(EnabledDisabled::Enabled); + env.redfish_sim.set_is_bios_setup(true); env.redfish_sim.set_is_boot_order_setup(false); let redfish_timepoint = env.redfish_sim.timepoint(); @@ -179,27 +178,56 @@ async fn assert_dpu_reprovision_host_boot_repair( } } - // machine_setup enables the bootable DPU interface before boot-order promotion. let actions = env .redfish_sim .actions_since(&redfish_timepoint) .all_hosts(); - let machine_setup_pos = actions - .iter() - .position(|action| matches!(action, RedfishSimAction::MachineSetup { .. })) - .expect("expected DPU reprovision boot repair to call machine_setup"); - let set_boot_order_pos = actions + let (set_boot_order_pos, set_boot_order_mac) = actions .iter() - .position(|action| matches!(action, RedfishSimAction::SetBootOrderDpuFirst { .. })) + .enumerate() + .find_map(|(position, action)| match action { + RedfishSimAction::SetBootOrderDpuFirst { boot_interface_mac } => { + Some((position, boot_interface_mac)) + } + _ => None, + }) .expect("expected DPU reprovision boot repair to set DPU-first boot order"); - let check_boot_order_pos = actions + let boot_order_checks = actions .iter() - .rposition(|action| matches!(action, RedfishSimAction::IsBootOrderSetup { .. })) - .expect("expected DPU reprovision boot repair to verify boot order"); + .enumerate() + .filter_map(|(position, action)| match action { + RedfishSimAction::IsBootOrderSetup { boot_interface_mac } => { + Some((position, boot_interface_mac)) + } + _ => None, + }) + .collect::>(); assert!( - machine_setup_pos < set_boot_order_pos && set_boot_order_pos < check_boot_order_pos, - "expected machine_setup before set_boot_order_dpu_first before is_boot_order_setup; got: {actions:?}" + actions + .iter() + .all(|action| !matches!(action, RedfishSimAction::MachineSetup { .. })), + "expected order-only DPU reprovision boot repair to preserve the configured HTTP boot device; got: {actions:?}" + ); + assert!( + boot_order_checks + .iter() + .filter(|(position, _)| *position < set_boot_order_pos) + .count() + >= 2, + "expected both CheckHostBootConfig and SetBootOrder to inspect the order before writing it; got: {actions:?}" + ); + assert!( + boot_order_checks + .iter() + .any(|(position, _)| *position > set_boot_order_pos), + "expected boot-order verification after the write; got: {actions:?}" + ); + assert!( + boot_order_checks + .iter() + .all(|(_, boot_interface_mac)| *boot_interface_mac == set_boot_order_mac), + "expected every order check and write to target the same interface; got: {actions:?}" ); let rebooting_machine = machine.next_iteration_machine(env).await; @@ -564,6 +592,136 @@ async fn test_dpu_reprovision_viking_skips_boot_order_when_bios_setup(pool: sqlx ); } +#[crate::sqlx_test] +async fn test_dpu_reprovision_viking_finishes_parked_boot_order_recovery(pool: sqlx::PgPool) { + let env = create_test_env(pool).await; + let mh = create_managed_host_with_hardware_info_template( + &env, + HardwareInfoTemplate::Custom(DGX_H100_INFO_JSON), + ) + .await; + let dpu_machine = prepare_dpu_reprovision_host_boot_check(&env, &mh).await; + + // A controller upgrade can find a Viking in a persisted recovery substate + // created before boot-order remediation was disabled for this platform. + // Finish restoring host power before taking the safe terminal shortcut. + let parked_recovery = mh.new_dpu_reprovision_state(reprovision_set_host_boot_order_state( + SetBootOrderState::HandleJobFailure { + failure: "persisted failed boot-order job".to_string(), + power_state: PowerState::Off, + }, + )); + let mut txn = env.pool.begin().await.unwrap(); + db::machine::update_state(&mut txn, &mh.id, &parked_recovery) + .await + .unwrap(); + txn.commit().await.unwrap(); + + let redfish_timepoint = env.redfish_sim.timepoint(); + let dpu = dpu_machine.next_iteration_machine(&env).await; + assert_eq!(dpu.current_state(), &parked_recovery); + assert_eq!( + env.redfish_sim + .actions_since(&redfish_timepoint) + .all_hosts(), + vec![RedfishSimAction::Power(SystemPowerControl::ForceOff)], + "Viking policy must not abandon an in-flight power recovery" + ); + + let recovering_power_on = mh.new_dpu_reprovision_state(reprovision_set_host_boot_order_state( + SetBootOrderState::HandleJobFailure { + failure: "persisted failed boot-order job".to_string(), + power_state: PowerState::On, + }, + )); + let redfish_timepoint = env.redfish_sim.timepoint(); + let dpu = dpu_machine.next_iteration_machine(&env).await; + assert_eq!(dpu.current_state(), &recovering_power_on); + assert_eq!( + env.redfish_sim + .actions_since(&redfish_timepoint) + .all_hosts(), + vec![RedfishSimAction::BmcReset], + "parked recovery should reset the BMC after the host powers off" + ); + + // ForceOff records last_reboot_requested; backdate it so power_down_wait + // elapses in-process before the controller restores host power. + { + let mut txn = env.db_txn().await; + let host = mh.host().db_machine(&mut txn).await; + update_time_params(&env.pool, &host, 1, None).await; + } + + let redfish_timepoint = env.redfish_sim.timepoint(); + let dpu = dpu_machine.next_iteration_machine(&env).await; + assert_eq!(dpu.current_state(), &recovering_power_on); + assert_eq!( + env.redfish_sim + .actions_since(&redfish_timepoint) + .all_hosts(), + vec![RedfishSimAction::Power(SystemPowerControl::On)], + "parked recovery should restore host power after the BMC reset" + ); + + let safe_terminal = mh.new_dpu_reprovision_state(reprovision_set_host_boot_order_state( + SetBootOrderState::CheckBootOrder, + )); + let redfish_timepoint = env.redfish_sim.timepoint(); + let dpu = dpu_machine.next_iteration_machine(&env).await; + assert_eq!(dpu.current_state(), &safe_terminal); + assert!( + env.redfish_sim + .actions_since(&redfish_timepoint) + .all_hosts() + .is_empty(), + "completed recovery should naturally reach CheckBootOrder" + ); + + // CheckBootOrder has no unfinished operation behind it, but its preceding + // reboot may have reverted the HTTP boot device. Repair that BIOS drift + // without touching the unsupported Viking boot-order APIs, and spend one + // attempt from the shared convergence budget. + env.redfish_sim.set_is_bios_setup(false); + + let redfish_timepoint = env.redfish_sim.timepoint(); + let dpu = dpu_machine.next_iteration_machine(&env).await; + assert_eq!( + dpu.current_state(), + &mh.new_dpu_reprovision_state(ReprovisionState::ConfigureHostBoot { retry_count: 1 }) + ); + assert!( + env.redfish_sim + .actions_since(&redfish_timepoint) + .all_hosts() + .is_empty(), + "Viking BIOS recovery must not read or write boot order" + ); + + // Once BIOS is intact, the same safe terminal skips boot-order + // verification and completes the repair. + env.redfish_sim.set_is_bios_setup(true); + let mut txn = env.pool.begin().await.unwrap(); + db::machine::update_state(&mut txn, &mh.id, &safe_terminal) + .await + .unwrap(); + txn.commit().await.unwrap(); + + let redfish_timepoint = env.redfish_sim.timepoint(); + let dpu = dpu_machine.next_iteration_machine(&env).await; + assert_eq!( + dpu.current_state(), + &mh.new_dpu_reprovision_state(ReprovisionState::LockHostAfterBootRepair) + ); + assert!( + env.redfish_sim + .actions_since(&redfish_timepoint) + .all_hosts() + .is_empty(), + "safe Viking completion should not touch Redfish boot order" + ); +} + #[crate::sqlx_test] async fn test_dpu_for_reprovisioning_fail_if_maintenance_not_set(pool: sqlx::PgPool) { let env = create_test_env(pool).await; diff --git a/crates/api-core/src/tests/machine_states.rs b/crates/api-core/src/tests/machine_states.rs index 088950fda8..8fb6af4963 100644 --- a/crates/api-core/src/tests/machine_states.rs +++ b/crates/api-core/src/tests/machine_states.rs @@ -63,6 +63,7 @@ use model::machine::{ MachineValidatingState, ManagedHostState, MeasuringState, PowerState, SetBootOrderInfo, SetBootOrderState, SetSecureBootState, SpdmMeasuringState, StateMachineArea, ValidationState, }; +use model::machine_validation::MachineValidationState; use model::network_segment::NetworkSegmentType; use model::site_explorer::{EndpointExplorationReport, ExploredDpu, ExploredManagedHost}; use model::test_support::{DpuConfig, ManagedHostConfig}; @@ -2703,6 +2704,77 @@ async fn test_hpc_polling_bios_setup_stuck_enters_handle_bios_job_failure(pool: ); } +/// Assigned platform configuration should repair boot-order-only drift without +/// rerunning broad BIOS setup. +#[crate::sqlx_test] +async fn test_hpc_order_only_drift_skips_bios_configuration(pool: sqlx::PgPool) { + let env = create_test_env(pool).await; + + let mh = common::api_fixtures::create_managed_host(&env).await; + let segment_id = env.create_vpc_and_tenant_segment().await; + create_instance(&env, &mh, false, segment_id).await; + let host_id = mh.host().id; + + set_host_controller_state_stuck_in( + &env, + host_id, + &ManagedHostState::Assigned { + instance_state: InstanceState::HostPlatformConfiguration { + platform_config_state: HostPlatformConfigurationState::CheckHostConfig, + }, + }, + 0, + ) + .await; + // CheckHostConfig requires a DPU observation associated with this host + // state before Redfish results are trusted. + mh.network_configured(&env).await; + + env.redfish_sim.set_is_bios_setup(true); + env.redfish_sim.set_is_boot_order_setup(false); + env.redfish_sim + .set_lockdown(libredfish::EnabledDisabled::Disabled); + let checkpoint = env.redfish_sim.timepoint(); + + env.run_machine_state_controller_iteration().await; + { + let mut txn = env.db_txn().await; + let host = mh.host().db_machine(&mut txn).await; + assert!( + matches!( + host.current_state(), + ManagedHostState::Assigned { + instance_state: InstanceState::HostPlatformConfiguration { + platform_config_state: HostPlatformConfigurationState::SetBootOrder { + set_boot_order_info: SetBootOrderInfo { + set_boot_order_state: SetBootOrderState::SetBootOrder, + .. + }, + }, + }, + } + ), + "order-only drift should route directly to SetBootOrder, got: {:?}", + host.current_state() + ); + } + + env.run_machine_state_controller_iteration().await; + let actions = env.redfish_sim.actions_since(&checkpoint).all_hosts(); + assert!( + actions + .iter() + .any(|action| matches!(action, RedfishSimAction::SetBootOrderDpuFirst { .. })), + "assigned platform configuration should restore the drifted order, got: {actions:?}" + ); + assert!( + actions + .iter() + .all(|action| !matches!(action, RedfishSimAction::MachineSetup { .. })), + "order-only drift should not rerun machine_setup, got: {actions:?}" + ); +} + /// Stuck HostInit/PollingBiosSetup recovery re-runs machine_setup and reaches HostInit/SetBootOrder. #[crate::sqlx_test] async fn test_polling_bios_setup_full_recovery_reruns_machine_setup_and_succeeds( @@ -2805,25 +2877,27 @@ async fn test_polling_bios_setup_full_recovery_reruns_machine_setup_and_succeeds /// so a zero-DPU host's in-band NIC takes a real `machine_interfaces` row that /// the boot-order phase can resolve. Mirrors `test_dhcp_allows_zero_dpu_host`. async fn create_zero_dpu_test_env(pool: sqlx::PgPool) -> TestEnv { - let env = create_test_env_with_overrides( - pool, - TestEnvOverrides { - site_prefixes: Some(vec![ - IpNetwork::new( - FIXTURE_ADMIN_NETWORK_SEGMENT_GATEWAY.network(), - FIXTURE_ADMIN_NETWORK_SEGMENT_GATEWAY.prefix(), - ) - .unwrap(), - IpNetwork::new( - FIXTURE_HOST_INBAND_NETWORK_SEGMENT_GATEWAY.network(), - FIXTURE_HOST_INBAND_NETWORK_SEGMENT_GATEWAY.prefix(), - ) - .unwrap(), - ]), - ..Default::default() - }, - ) - .await; + create_zero_dpu_test_env_with_overrides(pool, TestEnvOverrides::default()).await +} + +async fn create_zero_dpu_test_env_with_overrides( + pool: sqlx::PgPool, + mut overrides: TestEnvOverrides, +) -> TestEnv { + overrides.site_prefixes = Some(vec![ + IpNetwork::new( + FIXTURE_ADMIN_NETWORK_SEGMENT_GATEWAY.network(), + FIXTURE_ADMIN_NETWORK_SEGMENT_GATEWAY.prefix(), + ) + .unwrap(), + IpNetwork::new( + FIXTURE_HOST_INBAND_NETWORK_SEGMENT_GATEWAY.network(), + FIXTURE_HOST_INBAND_NETWORK_SEGMENT_GATEWAY.prefix(), + ) + .unwrap(), + ]); + + let env = create_test_env_with_overrides(pool, overrides).await; create_host_inband_network_segment(&env.api, None).await; env } @@ -3035,14 +3109,265 @@ async fn test_set_boot_order_sets_order_without_reasserting_when_device_configur ); } -/// Recovery path: when the HTTP-boot device is reverted (the boot NIC dropped off -/// the BMC's Redfish inventory on a reboot, so `is_bios_setup` reads false), -/// SetBootOrder re-asserts `machine_setup` and reboots to apply it *before* the -/// boot-order set -- the two BIOS writes never share one pass (they can't share -/// one Dell config job). Once the device verifies again, the order is set and the -/// host advances. +/// The reboot that applies a boot-order job can independently revert a managed +/// BIOS setting. Final verification checks BIOS before order, routes back to +/// the shared job-aware stages, and carries the existing convergence budget. +#[crate::sqlx_test] +async fn test_check_boot_order_repairs_bios_drift_after_order_reboot(pool: sqlx::PgPool) { + let env = create_zero_dpu_test_env(pool).await; + + let mh = create_managed_host_with_config(&env, ManagedHostConfig::zero_dpu()).await; + let host_id = mh.host().id; + + env.redfish_sim.set_is_bios_setup(false); + env.redfish_sim.set_is_boot_order_setup(true); + env.redfish_sim + .set_lockdown(libredfish::EnabledDisabled::Disabled); + + set_host_controller_state_stuck_in( + &env, + host_id, + &ManagedHostState::HostInit { + machine_state: MachineState::SetBootOrder { + set_boot_order_info: Some(SetBootOrderInfo { + set_boot_order_jid: None, + set_boot_order_state: SetBootOrderState::CheckBootOrder, + retry_count: 2, + }), + }, + }, + 1, + ) + .await; + + let checkpoint = env.redfish_sim.timepoint(); + env.run_machine_state_controller_iteration().await; + + let mut txn = env.db_txn().await; + let host = mh.host().db_machine(&mut txn).await; + assert!( + matches!( + host.current_state(), + ManagedHostState::HostInit { + machine_state: MachineState::WaitingForPlatformConfiguration { retry_count: 3 }, + } + ), + "post-order BIOS drift should carry the convergence budget into shared BIOS repair, got: {:?}", + host.current_state() + ); + + let actions = env.redfish_sim.actions_since(&checkpoint).all_hosts(); + assert!( + actions + .iter() + .all(|action| !matches!(action, RedfishSimAction::IsBootOrderSetup { .. })), + "BIOS drift should short-circuit final boot-order verification, got: {actions:?}" + ); +} + +/// Repeated BIOS drift across boot-order reboots consumes the carried budget +/// instead of cycling indefinitely through machine_setup and another reboot. +#[crate::sqlx_test] +async fn test_check_boot_order_bios_drift_exhausts_convergence_budget(pool: sqlx::PgPool) { + let env = create_zero_dpu_test_env(pool).await; + + let mh = create_managed_host_with_config(&env, ManagedHostConfig::zero_dpu()).await; + let host_id = mh.host().id; + + env.redfish_sim.set_is_bios_setup(false); + env.redfish_sim + .set_lockdown(libredfish::EnabledDisabled::Disabled); + set_host_controller_state_stuck_in( + &env, + host_id, + &ManagedHostState::HostInit { + machine_state: MachineState::SetBootOrder { + set_boot_order_info: Some(SetBootOrderInfo { + set_boot_order_jid: None, + set_boot_order_state: SetBootOrderState::CheckBootOrder, + retry_count: 3, + }), + }, + }, + 1, + ) + .await; + + let checkpoint = env.redfish_sim.timepoint(); + env.run_machine_state_controller_iteration().await; + + let mut txn = env.db_txn().await; + let host = mh.host().db_machine(&mut txn).await; + assert!( + matches!( + host.current_state(), + ManagedHostState::Failed { + details: FailureDetails { + cause: FailureCause::BiosSetupFailed { .. }, + source: FailureSource::StateMachineArea(StateMachineArea::HostInit), + .. + }, + .. + } + ), + "expected repeated cross-phase drift to exhaust into Failed, got: {:?}", + host.current_state() + ); + + let actions = env.redfish_sim.actions_since(&checkpoint).all_hosts(); + assert!( + actions + .iter() + .all(|action| !matches!(action, RedfishSimAction::MachineSetup { .. })), + "an exhausted convergence budget must not issue another BIOS write, got: {actions:?}" + ); +} + +/// Boot-order verification uses the same configured convergence budget as +/// BIOS recovery, then remains able to observe an operator's repair while +/// parked at that limit. +#[crate::sqlx_test] +async fn test_check_boot_order_respects_configured_convergence_budget(pool: sqlx::PgPool) { + let mut config = get_config(); + config.machine_state_controller.max_bios_config_retries = 1; + let env = + create_zero_dpu_test_env_with_overrides(pool, TestEnvOverrides::with_config(config)).await; + + let mh = create_managed_host_with_config(&env, ManagedHostConfig::zero_dpu()).await; + let host_id = mh.host().id; + + env.redfish_sim.set_is_bios_setup(true); + env.redfish_sim.set_is_boot_order_setup(false); + env.redfish_sim + .set_lockdown(libredfish::EnabledDisabled::Disabled); + set_host_controller_state_stuck_in( + &env, + host_id, + &ManagedHostState::HostInit { + machine_state: MachineState::SetBootOrder { + set_boot_order_info: Some(SetBootOrderInfo { + set_boot_order_jid: None, + set_boot_order_state: SetBootOrderState::CheckBootOrder, + retry_count: 1, + }), + }, + }, + 31, + ) + .await; + + env.run_machine_state_controller_iteration().await; + + { + let mut txn = env.db_txn().await; + let host = mh.host().db_machine(&mut txn).await; + assert!( + matches!( + host.current_state(), + ManagedHostState::HostInit { + machine_state: MachineState::SetBootOrder { + set_boot_order_info: Some(SetBootOrderInfo { + set_boot_order_state: SetBootOrderState::CheckBootOrder, + retry_count: 1, + .. + }), + }, + } + ), + "an exhausted configured budget must not advance to another boot-order attempt, got: {:?}", + host.current_state() + ); + assert!( + matches!( + host.controller_state_outcome.as_ref(), + Some(PersistentStateHandlerOutcome::Error { err, .. }) + if err.contains("Manual intervention required") + && err.contains("max_retries: 1") + ), + "expected boot-order exhaustion to report the configured limit, got: {:?}", + host.controller_state_outcome + ); + } + + env.redfish_sim.set_is_boot_order_setup(true); + env.run_machine_state_controller_iteration().await; + + let mut txn = env.db_txn().await; + let host = mh.host().db_machine(&mut txn).await; + assert!( + matches!( + host.current_state(), + ManagedHostState::HostInit { + machine_state: MachineState::Measuring { + measuring_state: MeasuringState::WaitingForMeasurements, + }, + } + ), + "a manually repaired boot order should complete from the parked verification state, got: {:?}", + host.current_state() + ); +} + +/// A configured budget above the historical fixed ceiling permits the shared +/// boot-config flow to spend the additional retries. +#[crate::sqlx_test] +async fn test_check_boot_order_allows_configured_budget_above_legacy_limit(pool: sqlx::PgPool) { + let mut config = get_config(); + config.machine_state_controller.max_bios_config_retries = 5; + let env = + create_zero_dpu_test_env_with_overrides(pool, TestEnvOverrides::with_config(config)).await; + + let mh = create_managed_host_with_config(&env, ManagedHostConfig::zero_dpu()).await; + let host_id = mh.host().id; + + env.redfish_sim.set_is_bios_setup(true); + env.redfish_sim.set_is_boot_order_setup(false); + env.redfish_sim + .set_lockdown(libredfish::EnabledDisabled::Disabled); + set_host_controller_state_stuck_in( + &env, + host_id, + &ManagedHostState::HostInit { + machine_state: MachineState::SetBootOrder { + set_boot_order_info: Some(SetBootOrderInfo { + set_boot_order_jid: None, + set_boot_order_state: SetBootOrderState::CheckBootOrder, + retry_count: 3, + }), + }, + }, + 31, + ) + .await; + + env.run_machine_state_controller_iteration().await; + + let mut txn = env.db_txn().await; + let host = mh.host().db_machine(&mut txn).await; + assert!( + matches!( + host.current_state(), + ManagedHostState::HostInit { + machine_state: MachineState::SetBootOrder { + set_boot_order_info: Some(SetBootOrderInfo { + set_boot_order_state: SetBootOrderState::SetBootOrder, + retry_count: 4, + .. + }), + }, + } + ), + "a configured budget above three should permit another boot-order attempt, got: {:?}", + host.current_state() + ); +} + +/// If the HTTP-boot device reverts after inspection but before SetBootOrder, +/// the order actuator routes back through the shared BIOS driver. In +/// particular, a Dell-style job returned by machine_setup is persisted and +/// completed before the order write. #[crate::sqlx_test] -async fn test_set_boot_order_reasserts_http_boot_device_when_reverted(pool: sqlx::PgPool) { +async fn test_set_boot_order_routes_reverted_bios_through_shared_job_flow(pool: sqlx::PgPool) { let env = create_zero_dpu_test_env(pool).await; let mh = create_managed_host_with_config(&env, ManagedHostConfig::zero_dpu()).await; @@ -3054,12 +3379,22 @@ async fn test_set_boot_order_reasserts_http_boot_device_when_reverted(pool: sqlx // while the device is gone. env.redfish_sim.set_is_bios_setup(false); env.redfish_sim.set_is_boot_order_setup(false); + // SetBootOrder is reached only after HostInit has opened lockdown for BIOS + // writes; model that precondition when planting this synthetic state. + env.redfish_sim + .set_lockdown(libredfish::EnabledDisabled::Disabled); + env.redfish_sim + .set_machine_setup_bios_job_id(Some("JID_LATE_BOOT_DRIFT".to_string())); + env.redfish_sim.set_job_state_sequence(vec![ + libredfish::JobState::Scheduled, + libredfish::JobState::Completed, + ]); set_host_stuck_in_set_boot_order(&env, host_id).await; let redfish_timepoint = env.redfish_sim.timepoint(); - // First pass: re-assert the device and reboot to apply it. It must NOT set the - // boot order in the same pass -- two BIOS writes can't share one config job. + // First pass only detects the late drift and moves back to the shared BIOS + // stage. It must not issue either BIOS write from the order actuator. env.run_machine_state_controller_iteration().await; let first_pass = env .redfish_sim @@ -3068,8 +3403,8 @@ async fn test_set_boot_order_reasserts_http_boot_device_when_reverted(pool: sqlx assert!( first_pass .iter() - .any(|a| matches!(a, RedfishSimAction::MachineSetup { .. })), - "a reverted HTTP-boot device should be re-asserted at SetBootOrder, got: {first_pass:?}" + .all(|a| !matches!(a, RedfishSimAction::MachineSetup { .. })), + "SetBootOrder should delegate late BIOS drift to the shared driver, got: {first_pass:?}" ); assert!( !first_pass @@ -3078,9 +3413,6 @@ async fn test_set_boot_order_reasserts_http_boot_device_when_reverted(pool: sqlx "the boot order must not be set in the same pass as the re-assert (shared BIOS job), got: {first_pass:?}" ); - // The re-assert is committed once. While the device is still reverted (the - // apply reboot hasn't landed yet), the host polls in - // WaitForHttpBootDeviceApplied without re-running machine_setup. { let mut txn = env.db_txn().await; let host = mh.host().db_machine(&mut txn).await; @@ -3088,31 +3420,17 @@ async fn test_set_boot_order_reasserts_http_boot_device_when_reverted(pool: sqlx matches!( host.current_state(), ManagedHostState::HostInit { - machine_state: MachineState::SetBootOrder { - set_boot_order_info: Some(SetBootOrderInfo { - set_boot_order_state: SetBootOrderState::WaitForHttpBootDeviceApplied, - .. - }), - }, + machine_state: MachineState::WaitingForPlatformConfiguration { retry_count: 1 }, } ), - "after the re-assert the host should wait for the device to apply, got: {:?}", + "late BIOS drift should return to shared configuration, got: {:?}", host.current_state() ); } - let after_reassert = env.redfish_sim.timepoint(); - env.run_machine_state_controller_iteration().await; - let second_pass = env.redfish_sim.actions_since(&after_reassert).all_hosts(); - assert!( - !second_pass - .iter() - .any(|a| matches!(a, RedfishSimAction::MachineSetup { .. })), - "machine_setup must not re-run while waiting for the re-asserted device to apply, got: {second_pass:?}" - ); - // Within the wait window the host stays parked in - // WaitForHttpBootDeviceApplied -- it neither advances nor bounces back to - // SetBootOrder for another re-assert. + // The shared driver persists the returned job instead of discarding it and + // rebooting immediately. + env.run_machine_state_controller_iteration().await; { let mut txn = env.db_txn().await; let host = mh.host().db_machine(&mut txn).await; @@ -3120,20 +3438,21 @@ async fn test_set_boot_order_reasserts_http_boot_device_when_reverted(pool: sqlx matches!( host.current_state(), ManagedHostState::HostInit { - machine_state: MachineState::SetBootOrder { - set_boot_order_info: Some(SetBootOrderInfo { - set_boot_order_state: SetBootOrderState::WaitForHttpBootDeviceApplied, + machine_state: MachineState::WaitingForBiosJob { + bios_config_info: BiosConfigInfo { + bios_job_id: Some(job_id), + retry_count: 1, .. - }), + }, }, - } + } if job_id == "JID_LATE_BOOT_DRIFT" ), - "the host should keep waiting for the device to apply within the wait window, got: {:?}", + "the late-drift BIOS job should be persisted, got: {:?}", host.current_state() ); } - // Model the reboot applying the re-asserted config: the device now verifies. + // Model the BIOS job applying the restored device. env.redfish_sim.set_is_bios_setup(true); // With the device restored, the host sets the boot order and advances. @@ -3151,14 +3470,12 @@ async fn test_set_boot_order_reasserts_http_boot_device_when_reverted(pool: sqlx assert_machine_setup_precedes_reorder_for(&actions, boot_nic_mac); } -/// When the re-asserted HTTP-boot device does not verify within the wait -/// window (the boot NIC can drop off the BMC's Redfish inventory again on the -/// apply reboot itself), the host returns to `SetBootOrder` for a fresh -/// re-assert -- one `machine_setup` per window, not per pass. Once the state -/// machine's retry budget is exhausted it stops re-asserting and surfaces the -/// host for manual intervention. +/// Hosts persisted in the legacy HTTP-device wait state migrate to the shared +/// BIOS driver when their wait expires. The existing retry count becomes the +/// shared convergence budget, and an exhausted budget still stops for manual +/// intervention. #[crate::sqlx_test] -async fn test_set_boot_order_reassert_window_expiry_bounds_reapplies(pool: sqlx::PgPool) { +async fn test_legacy_http_boot_wait_migrates_to_shared_bios_repair(pool: sqlx::PgPool) { let env = create_zero_dpu_test_env(pool).await; let mh = create_managed_host_with_config(&env, ManagedHostConfig::zero_dpu()).await; @@ -3167,6 +3484,8 @@ async fn test_set_boot_order_reassert_window_expiry_bounds_reapplies(pool: sqlx: // The device stays reverted throughout: every verify reads false. env.redfish_sim.set_is_bios_setup(false); env.redfish_sim.set_is_boot_order_setup(false); + env.redfish_sim + .set_lockdown(libredfish::EnabledDisabled::Disabled); let stuck_waiting = |retry_count: u32| ManagedHostState::HostInit { machine_state: MachineState::SetBootOrder { @@ -3179,8 +3498,8 @@ async fn test_set_boot_order_reassert_window_expiry_bounds_reapplies(pool: sqlx: }; // Plant the host mid-wait with the window already expired (backdated past - // the 10-minute apply window). Pass 1 returns it to SetBootOrder; pass 2 - // re-asserts once and parks it back in the wait substate. + // the 10-minute apply window). Pass 1 migrates its persisted state; pass 2 + // runs machine_setup through the common BIOS driver. set_host_controller_state_stuck_in(&env, host_id, &stuck_waiting(0), 11).await; let checkpoint = env.redfish_sim.timepoint(); env.run_machine_state_controller_iteration().await; @@ -3201,16 +3520,10 @@ async fn test_set_boot_order_reassert_window_expiry_bounds_reapplies(pool: sqlx: matches!( host.current_state(), ManagedHostState::HostInit { - machine_state: MachineState::SetBootOrder { - set_boot_order_info: Some(SetBootOrderInfo { - set_boot_order_state: SetBootOrderState::WaitForHttpBootDeviceApplied, - retry_count: 1, - .. - }), - }, + machine_state: MachineState::PollingBiosSetup { retry_count: 1 }, } ), - "the fresh re-assert should park the host back in the wait substate with the retry spent, got: {:?}", + "the legacy wait should carry its retry count into shared BIOS polling, got: {:?}", host.current_state() ); } @@ -3269,9 +3582,16 @@ async fn test_machine_validation_repairs_reverted_boot_config(pool: sqlx::PgPool env.redfish_sim.set_is_boot_order_setup(false); env.redfish_sim .set_lockdown(libredfish::EnabledDisabled::Enabled); + env.redfish_sim + .set_machine_setup_bios_job_id(Some("JID_VALIDATION_BOOT_REPAIR".to_string())); + env.redfish_sim.set_job_state_sequence(vec![ + libredfish::JobState::Scheduled, + libredfish::JobState::Completed, + ]); // Park the host mid-validation, waiting on a reboot that cannot land // (the state is newer than the host's last reported boot). + let validation_id = MachineValidationId::new(); set_host_controller_state_stuck_in( &env, host_id, @@ -3279,7 +3599,7 @@ async fn test_machine_validation_repairs_reverted_boot_config(pool: sqlx::PgPool validation_state: ValidationState::MachineValidation { machine_validation: MachineValidatingState::MachineValidating { context: "Discovery".to_string(), - id: MachineValidationId::new(), + id: validation_id, completed: 1, total: 1, is_enabled: true, @@ -3319,6 +3639,29 @@ async fn test_machine_validation_repairs_reverted_boot_config(pool: sqlx::PgPool .contains(&libredfish::EnabledDisabled::Disabled), "boot repair should unlock the BMC before writing BIOS settings" ); + { + let mut txn = env.db_txn().await; + let host = mh.host().db_machine(&mut txn).await; + assert!( + matches!( + host.current_state(), + ManagedHostState::Validation { + validation_state: ValidationState::MachineValidation { + machine_validation: MachineValidatingState::WaitingForBootBiosJob { + validation_id: current_validation_id, + bios_config_info: BiosConfigInfo { + bios_job_id: Some(job_id), + .. + }, + }, + }, + } if *current_validation_id == validation_id + && job_id == "JID_VALIDATION_BOOT_REPAIR" + ), + "validation should preserve the BIOS job before rebooting, got: {:?}", + host.current_state() + ); + } // The repair reboot applies the re-asserted device. env.redfish_sim.set_is_bios_setup(true); @@ -3334,9 +3677,11 @@ async fn test_machine_validation_repairs_reverted_boot_config(pool: sqlx::PgPool host.current_state(), ManagedHostState::Validation { validation_state: ValidationState::MachineValidation { - machine_validation: MachineValidatingState::RebootHost { .. }, + machine_validation: MachineValidatingState::RebootHost { + validation_id: current_validation_id, + }, }, - } + } if *current_validation_id == validation_id ) { resumed = true; break; @@ -3359,6 +3704,183 @@ async fn test_machine_validation_repairs_reverted_boot_config(pool: sqlx::PgPool assert_machine_setup_precedes_reorder_for(&actions, boot_nic_mac); } +/// Boot-order-only drift (the HTTP boot device is still configured, but its +/// priority changed) is detected while validation waits for a reboot. The +/// repair restores only the order, re-locks the BMC, and resumes validation. +#[crate::sqlx_test] +async fn test_machine_validation_repairs_drifted_boot_order(pool: sqlx::PgPool) { + let env = create_zero_dpu_test_env(pool).await; + + let mh = create_managed_host_with_config(&env, ManagedHostConfig::zero_dpu()).await; + let host_id = mh.host().id; + let boot_nic_mac = host_inband_nic_mac(&env, host_id).await; + + // The HTTP boot device is intact; only the order drifted. Set the BIOS + // status explicitly so this regression does not rely on simulator defaults. + env.redfish_sim.set_is_bios_setup(true); + env.redfish_sim.set_is_boot_order_setup(false); + env.redfish_sim + .set_lockdown(libredfish::EnabledDisabled::Enabled); + + let validation_id = MachineValidationId::new(); + set_host_controller_state_stuck_in( + &env, + host_id, + &ManagedHostState::Validation { + validation_state: ValidationState::MachineValidation { + machine_validation: MachineValidatingState::MachineValidating { + context: "Discovery".to_string(), + id: validation_id, + completed: 1, + total: 1, + is_enabled: true, + }, + }, + }, + 0, + ) + .await; + + let checkpoint = env.redfish_sim.timepoint(); + + let mut unlocked = false; + let mut resumed = false; + for _ in 0..20 { + env.run_machine_state_controller_iteration().await; + unlocked |= env + .redfish_sim + .lockdown_states() + .contains(&libredfish::EnabledDisabled::Disabled); + let mut txn = env.db_txn().await; + let host = mh.host().db_machine(&mut txn).await; + if matches!( + host.current_state(), + ManagedHostState::Validation { + validation_state: ValidationState::MachineValidation { + machine_validation: MachineValidatingState::RebootHost { + validation_id: current_validation_id, + }, + }, + } if *current_validation_id == validation_id + ) { + resumed = true; + break; + } + } + + assert!( + resumed, + "order-only drift should be repaired before validation resumes" + ); + assert!( + unlocked, + "validation should unlock the BMC before repairing boot-order drift" + ); + + let actions = env.redfish_sim.actions_since(&checkpoint).all_hosts(); + assert!( + actions.iter().any(|a| matches!( + a, + RedfishSimAction::IsBootOrderSetup { boot_interface_mac } + if *boot_interface_mac == boot_nic_mac.to_string() + )), + "validation should check the boot order for the boot NIC, got: {actions:?}" + ); + assert!( + actions.iter().any(|a| matches!( + a, + RedfishSimAction::SetBootOrderDpuFirst { boot_interface_mac } + if *boot_interface_mac == boot_nic_mac.to_string() + )), + "the drifted boot order should be restored for the boot NIC, got: {actions:?}" + ); + assert!( + !actions + .iter() + .any(|a| matches!(a, RedfishSimAction::MachineSetup { .. })), + "order-only repair should preserve the configured HTTP boot device, got: {actions:?}" + ); + assert!( + env.redfish_sim + .lockdown_states() + .iter() + .all(|l| *l == libredfish::EnabledDisabled::Enabled), + "the BMC should be re-locked once the boot order is repaired" + ); +} + +/// Exhausting the shared BIOS recovery budget during validation is terminal +/// for both the host repair and the active validation run. +#[crate::sqlx_test] +async fn test_validation_bios_setup_exhaustion_completes_active_validation(pool: sqlx::PgPool) { + let env = create_test_env(pool).await; + + let mh = create_managed_host(&env).await; + let host_id = mh.host().id; + let validation_id = + on_demand_machine_validation(&env, host_id, Vec::new(), Vec::new(), false, Vec::new()) + .await + .validation_id + .expect("on-demand validation should return an id"); + + env.redfish_sim.set_is_bios_setup(false); + set_host_controller_state_stuck_in( + &env, + host_id, + &ManagedHostState::Validation { + validation_state: ValidationState::MachineValidation { + machine_validation: MachineValidatingState::PollingBootBiosSetup { + validation_id, + retry_count: 3, + }, + }, + }, + 16, + ) + .await; + + env.run_machine_state_controller_iteration().await; + + let mut txn = env.db_txn().await; + let host = mh.host().db_machine(&mut txn).await; + assert!( + matches!( + host.current_state(), + ManagedHostState::Failed { + details: FailureDetails { + cause: FailureCause::BiosSetupFailed { .. }, + source: FailureSource::StateMachineArea(StateMachineArea::MainFlow), + .. + }, + .. + } + ), + "expected validation boot repair exhaustion to fail the host, got: {:?}", + host.current_state() + ); + assert_eq!( + host.on_demand_machine_validation_request, + Some(false), + "terminal boot repair should clear the validation request" + ); + + let validation = db::machine_validation::find_by_id(&env.pool, &validation_id) + .await + .expect("validation run should still be queryable"); + assert!( + matches!( + validation.status.as_ref().map(|status| &status.state), + Some(&MachineValidationState::Failed) + ), + "validation run should be marked failed, got: {:?}", + validation.status + ); + assert!( + validation.end_time.is_some(), + "terminal validation run should record an end time" + ); +} + /// When HostInit/PollingBiosSetup retry budget is exhausted, enter Failed and recover via is_bios_setup. #[crate::sqlx_test] async fn test_polling_bios_setup_exhausted_enters_failed_and_recovers_when_bios_setup_true( diff --git a/crates/api-model/src/machine/mod.rs b/crates/api-model/src/machine/mod.rs index 5f35ff8263..1d360b9b34 100644 --- a/crates/api-model/src/machine/mod.rs +++ b/crates/api-model/src/machine/mod.rs @@ -1296,6 +1296,23 @@ pub enum MachineValidatingState { validation_id: MachineValidationId, unlock_host_state: UnlockHostState, }, + CheckBootConfigForRepair { + validation_id: MachineValidationId, + }, + ConfigureBootBios { + validation_id: MachineValidationId, + #[serde(default)] + retry_count: u32, + }, + WaitingForBootBiosJob { + validation_id: MachineValidationId, + bios_config_info: BiosConfigInfo, + }, + PollingBootBiosSetup { + validation_id: MachineValidationId, + #[serde(default)] + retry_count: u32, + }, RepairBootConfig { validation_id: MachineValidationId, set_boot_order_info: SetBootOrderInfo, @@ -1887,7 +1904,8 @@ pub struct BiosConfigInfo { #[serde(default, skip_serializing_if = "Option::is_none")] pub bios_job_id: Option, pub bios_config_state: BiosConfigState, - /// Full configure_host_bios retry count across HandleBiosJobFailure recovery cycles. + /// Shared host boot-configuration convergence retry count, including BIOS + /// job-failure recovery cycles. #[serde(default)] pub retry_count: u32, } @@ -1913,7 +1931,8 @@ pub struct SetBootOrderInfo { #[serde(default, skip_serializing_if = "Option::is_none")] pub set_boot_order_jid: Option, pub set_boot_order_state: SetBootOrderState, - /// Retry counter for SetBootOrder state machine. Defaults to 0 for backwards compatibility. + /// Shared host boot-configuration convergence retry count. Defaults to 0 + /// for backwards compatibility. #[serde(default)] pub retry_count: u32, } @@ -1923,9 +1942,9 @@ pub struct SetBootOrderInfo { #[serde(tag = "state", rename_all = "lowercase")] pub enum SetBootOrderState { SetBootOrder, - /// A reverted HTTP-boot device was re-asserted (`machine_setup`) and the - /// host restarted to apply it; polls the device across that reboot, then - /// returns to `SetBootOrder` to set the boot order. + /// Legacy persisted state from the former inline HTTP-device repair flow. + /// It polls the device across the already-started reboot, then either + /// resumes `SetBootOrder` or migrates to the shared BIOS repair stages. WaitForHttpBootDeviceApplied, WaitForSetBootOrderJobScheduled, RebootHost, @@ -2675,6 +2694,10 @@ pub fn state_sla( } MachineValidatingState::PrepareBootRepair { .. } | MachineValidatingState::UnlockForBootRepair { .. } + | MachineValidatingState::CheckBootConfigForRepair { .. } + | MachineValidatingState::ConfigureBootBios { .. } + | MachineValidatingState::WaitingForBootBiosJob { .. } + | MachineValidatingState::PollingBootBiosSetup { .. } | MachineValidatingState::RepairBootConfig { .. } | MachineValidatingState::LockAfterBootRepair { .. } => { StateSla::with_sla(slas::VALIDATION, time_in_state) diff --git a/crates/machine-controller/Cargo.toml b/crates/machine-controller/Cargo.toml index 9cd99b9978..74c865bd3e 100644 --- a/crates/machine-controller/Cargo.toml +++ b/crates/machine-controller/Cargo.toml @@ -70,6 +70,8 @@ version-compare = { workspace = true } [dev-dependencies] carbide-redfish = { path = "../redfish", features = ["test-support"] } +carbide-test-harness = { path = "../test-harness" } +carbide-test-support = { path = "../test-support" } figment = { workspace = true, features = ["env", "test", "toml"] } regex = { workspace = true } diff --git a/crates/machine-controller/src/config/controller.rs b/crates/machine-controller/src/config/controller.rs index 76e964eaaf..4af4ecb2bf 100644 --- a/crates/machine-controller/src/config/controller.rs +++ b/crates/machine-controller/src/config/controller.rs @@ -70,7 +70,9 @@ pub struct MachineStateControllerConfig { serialize_with = "as_duration" )] pub uefi_boot_wait: Duration, - /// Max configure_host_bios retry cycles through HandleBiosJobFailure recovery. + /// Retry budget for automated host boot-configuration convergence, shared + /// across BIOS recovery and boot-order verification. The field name is + /// retained for configuration compatibility. #[serde(default = "MachineStateControllerConfig::max_bios_config_retries_default")] pub max_bios_config_retries: u32, /// How long PollingBiosSetup may sit on Ok(false) before escalating into diff --git a/crates/machine-controller/src/handler.rs b/crates/machine-controller/src/handler.rs index 42e621fd0f..9489cf9774 100644 --- a/crates/machine-controller/src/handler.rs +++ b/crates/machine-controller/src/handler.rs @@ -119,21 +119,23 @@ mod bios_config; mod dpf; mod firmware_artifact; mod helpers; +mod host_boot_config; mod machine_validation; mod power; mod sku; #[cfg(test)] mod test_machine_setup; -use bios_config::{ - BiosConfigJobAdvanceOutcome, BiosConfigOutcome, PollingBiosSetupOutcome, - advance_bios_config_job, advance_polling_bios_setup, configure_host_bios, - handle_bios_setup_failed_recovery, -}; +use bios_config::handle_bios_setup_failed_recovery; use helpers::{ DpuDiscoveringStateHelper, DpuInitStateHelper, ManagedHostStateHelper, NextState, ReprovisionStateHelper, all_equal, }; +use host_boot_config::{ + HostBootConfigCheckOutcome, HostBootConfigDecision, HostBootConfigDpuFreshness, + HostBootConfigOutcome, HostBootConfigStage, check_host_boot_config, + initial_set_boot_order_info, run_host_boot_config_stage, should_skip_boot_order_remediation, +}; use state_controller::db_write_batch::DbWriteBatch; use crate::config::{BomValidationConfig, PowerManagerOptions}; @@ -1447,11 +1449,7 @@ impl MachineStateHandler { FailureCause::BiosSetupFailed { .. } if machine_id.machine_type().is_host() => { let recovered = ManagedHostState::HostInit { machine_state: MachineState::SetBootOrder { - set_boot_order_info: Some(SetBootOrderInfo { - set_boot_order_jid: None, - set_boot_order_state: SetBootOrderState::SetBootOrder, - retry_count: 0, - }), + set_boot_order_info: Some(initial_set_boot_order_info()), }, }; handle_bios_setup_failed_recovery(ctx, mh_snapshot, recovered).await @@ -3127,243 +3125,70 @@ async fn handle_dpu_reprovision( // WaitingForNetworkConfig already accepted the DPU observation. Do // not require a newer observation just because the host state // version advanced while entering host boot repair. - let redfish_client = ctx - .services - .create_redfish_client_from_machine(&state.host_snapshot) - .await?; - - let next_state = match check_host_boot_config( - redfish_client.as_ref(), + handle_dpu_reprovision_host_boot_config_check( + ctx, state, reachability_params, HostBootConfigDpuFreshness::AlreadyValidated, - ctx, ) - .await? - { - HostBootConfigDecision::Wait(reason) => { - return Ok(StateHandlerOutcome::wait(reason)); - } - HostBootConfigDecision::ConfigureBoot => { - ReprovisionState::ConfigureHostBoot { retry_count: 0 } - } - HostBootConfigDecision::LockHost => ReprovisionState::LockHostAfterBootRepair, - }; - - Ok(StateHandlerOutcome::transition( - update_reprovision_targets_to_reprovision_state(state, next_state)?, - )) + .await } ReprovisionState::CheckHostBootConfigAfterHostReboot => { // This path rebooted the host after unlocking, so require a DPU // observation newer than that reboot before trusting boot checks. - let redfish_client = ctx - .services - .create_redfish_client_from_machine(&state.host_snapshot) - .await?; - - let next_state = match check_host_boot_config( - redfish_client.as_ref(), + handle_dpu_reprovision_host_boot_config_check( + ctx, state, reachability_params, HostBootConfigDpuFreshness::SinceLastHostRebootRequest, - ctx, ) - .await? - { - HostBootConfigDecision::Wait(reason) => { - return Ok(StateHandlerOutcome::wait(reason)); - } - HostBootConfigDecision::ConfigureBoot => { - ReprovisionState::ConfigureHostBoot { retry_count: 0 } - } - HostBootConfigDecision::LockHost => ReprovisionState::LockHostAfterBootRepair, - }; - - Ok(StateHandlerOutcome::transition( - update_reprovision_targets_to_reprovision_state(state, next_state)?, - )) + .await } ReprovisionState::ConfigureHostBoot { retry_count } => { - // Run machine_setup only after the reprovisioned DPU is healthy; it - // may patch BIOS settings and trigger host-impacting recovery. - let redfish_client = ctx - .services - .create_redfish_client_from_machine(&state.host_snapshot) - .await?; - - match configure_host_bios( + handle_dpu_reprovision_host_boot_config_stage( ctx, - reachability_params, - redfish_client.as_ref(), state, - *retry_count, + reachability_params, + HostBootConfigStage::ConfigureBios { + retry_count: *retry_count, + }, ) - .await? - { - BiosConfigOutcome::Done => Ok(StateHandlerOutcome::transition( - update_reprovision_targets_to_reprovision_state( - state, - ReprovisionState::PollingHostBiosSetup { - retry_count: *retry_count, - }, - )?, - )), - BiosConfigOutcome::WaitingForBiosJob(bios_config_info) => { - Ok(StateHandlerOutcome::transition( - update_reprovision_targets_to_reprovision_state( - state, - ReprovisionState::WaitingForHostBiosJob { bios_config_info }, - )?, - )) - } - BiosConfigOutcome::WaitingForReboot(reason) => { - Ok(StateHandlerOutcome::wait(reason)) - } - } + .await } ReprovisionState::WaitingForHostBiosJob { bios_config_info } => { - // Poll vendor BIOS jobs before verifying the setup and boot order. - let redfish_client = ctx - .services - .create_redfish_client_from_machine(&state.host_snapshot) - .await?; - - match advance_bios_config_job( + handle_dpu_reprovision_host_boot_config_stage( ctx, - redfish_client.as_ref(), state, - bios_config_info.clone(), + reachability_params, + HostBootConfigStage::WaitingForBiosJob { + bios_config_info: bios_config_info.clone(), + }, ) - .await? - { - BiosConfigJobAdvanceOutcome::Continue(updated) => { - Ok(StateHandlerOutcome::transition( - update_reprovision_targets_to_reprovision_state( - state, - ReprovisionState::WaitingForHostBiosJob { - bios_config_info: updated, - }, - )?, - )) - } - BiosConfigJobAdvanceOutcome::Done => Ok(StateHandlerOutcome::transition( - update_reprovision_targets_to_reprovision_state( - state, - ReprovisionState::PollingHostBiosSetup { - retry_count: bios_config_info.retry_count, - }, - )?, - )), - BiosConfigJobAdvanceOutcome::Failed { failure } => Ok( - StateHandlerOutcome::transition(dpu_reprovision_host_boot_failed_state( - &state.managed_state, - state.host_snapshot.id, - failure, - )), - ), - BiosConfigJobAdvanceOutcome::RetryPlatformConfiguration { retry_count } => { - Ok(StateHandlerOutcome::transition( - update_reprovision_targets_to_reprovision_state( - state, - ReprovisionState::ConfigureHostBoot { retry_count }, - )?, - )) - } - BiosConfigJobAdvanceOutcome::Wait(reason) => Ok(StateHandlerOutcome::wait(reason)), - } + .await } ReprovisionState::PollingHostBiosSetup { retry_count } => { - // Verify machine_setup effects before promoting the DPU boot option. - let redfish_client = ctx - .services - .create_redfish_client_from_machine(&state.host_snapshot) - .await?; - - let predictions = load_boot_predictions(ctx, &state.host_snapshot.id).await?; - match advance_polling_bios_setup( - redfish_client.as_ref(), + handle_dpu_reprovision_host_boot_config_stage( + ctx, state, - *retry_count, - &ctx.services.site_config.machine_state_controller, - &predictions, + reachability_params, + HostBootConfigStage::PollingBiosSetup { + retry_count: *retry_count, + }, ) - .await? - { - PollingBiosSetupOutcome::Verified => { - let next_state = if should_skip_boot_order_remediation(state) { - ReprovisionState::LockHostAfterBootRepair - } else { - ReprovisionState::SetHostBootOrder { - set_boot_order_info: SetBootOrderInfo { - set_boot_order_jid: None, - set_boot_order_state: SetBootOrderState::SetBootOrder, - retry_count: 0, - }, - } - }; - - Ok(StateHandlerOutcome::transition( - update_reprovision_targets_to_reprovision_state(state, next_state)?, - )) - } - PollingBiosSetupOutcome::EnterRecovery(bios_config_info) => { - Ok(StateHandlerOutcome::transition( - update_reprovision_targets_to_reprovision_state( - state, - ReprovisionState::WaitingForHostBiosJob { bios_config_info }, - )?, - )) - } - PollingBiosSetupOutcome::Failed { failure } => Ok(StateHandlerOutcome::transition( - dpu_reprovision_host_boot_failed_state( - &state.managed_state, - state.host_snapshot.id, - failure, - ), - )), - PollingBiosSetupOutcome::Wait(reason) => Ok(StateHandlerOutcome::wait(reason)), - } + .await } ReprovisionState::SetHostBootOrder { set_boot_order_info, } => { - // Promote the selected DPU boot option after machine_setup has enabled it. - let redfish_client = ctx - .services - .create_redfish_client_from_machine(&state.host_snapshot) - .await?; - - match set_host_boot_order( + handle_dpu_reprovision_host_boot_config_stage( ctx, - reachability_params, - redfish_client.as_ref(), state, - set_boot_order_info.clone(), + reachability_params, + HostBootConfigStage::SetBootOrder { + set_boot_order_info: set_boot_order_info.clone(), + }, ) - .await? - { - SetBootOrderOutcome::Continue(boot_order_info) => { - Ok(StateHandlerOutcome::transition( - update_reprovision_targets_to_reprovision_state( - state, - ReprovisionState::SetHostBootOrder { - set_boot_order_info: boot_order_info, - }, - )?, - )) - } - SetBootOrderOutcome::Done => Ok(StateHandlerOutcome::transition( - update_reprovision_targets_to_reprovision_state( - state, - ReprovisionState::LockHostAfterBootRepair, - )?, - )), - SetBootOrderOutcome::WaitingForReboot(reason) => { - Ok(StateHandlerOutcome::wait(reason)) - } - SetBootOrderOutcome::Wait(reason) => Ok(StateHandlerOutcome::wait(reason)), - } + .await } ReprovisionState::LockHostAfterBootRepair => { // Preserve expected-machine lockdown policy after temporarily @@ -3489,6 +3314,117 @@ async fn handle_dpu_reprovision( } } +/// Checks host boot configuration for DPU reprovisioning and maps the shared +/// decision back into the reprovision state while preserving all active DPU +/// targets. The caller selects the observation-freshness requirement because +/// only the post-unlock path has rebooted the host. +async fn handle_dpu_reprovision_host_boot_config_check( + ctx: &mut StateHandlerContext<'_, MachineStateHandlerContextObjects>, + state: &ManagedHostStateSnapshot, + reachability_params: &ReachabilityParams, + dpu_freshness: HostBootConfigDpuFreshness, +) -> Result, StateHandlerError> { + let redfish_client = ctx + .services + .create_redfish_client_from_machine(&state.host_snapshot) + .await?; + + let next_state = match check_host_boot_config( + redfish_client.as_ref(), + state, + reachability_params, + dpu_freshness, + ctx, + ) + .await? + { + HostBootConfigCheckOutcome::Wait(reason) => { + return Ok(StateHandlerOutcome::wait(reason)); + } + HostBootConfigCheckOutcome::Ready(HostBootConfigDecision::ConfigureBios) => { + ReprovisionState::ConfigureHostBoot { retry_count: 0 } + } + HostBootConfigCheckOutcome::Ready(HostBootConfigDecision::SetBootOrder) => { + ReprovisionState::SetHostBootOrder { + set_boot_order_info: initial_set_boot_order_info(), + } + } + HostBootConfigCheckOutcome::Ready(HostBootConfigDecision::Complete) => { + ReprovisionState::LockHostAfterBootRepair + } + }; + + Ok(StateHandlerOutcome::transition( + update_reprovision_targets_to_reprovision_state(state, next_state)?, + )) +} + +/// Handles a host boot-configuration stage owned by DPU reprovisioning. +/// +/// Called by `handle_dpu_reprovision` for its host BIOS configuration, +/// vendor-job, polling, and boot-order substates. This adapter preserves the +/// active DPU reprovision targets, continues to `LockHostAfterBootRepair` on +/// completion, and attributes terminal failure to the lifecycle that owns the +/// reprovision. +async fn handle_dpu_reprovision_host_boot_config_stage( + ctx: &mut StateHandlerContext<'_, MachineStateHandlerContextObjects>, + state: &ManagedHostStateSnapshot, + reachability_params: &ReachabilityParams, + stage: HostBootConfigStage, +) -> Result, StateHandlerError> { + let redfish_client = ctx + .services + .create_redfish_client_from_machine(&state.host_snapshot) + .await?; + + match run_host_boot_config_stage( + ctx, + reachability_params, + redfish_client.as_ref(), + state, + stage, + ) + .await? + { + HostBootConfigOutcome::Continue(stage) => { + let reprovision_state = match stage { + HostBootConfigStage::ConfigureBios { retry_count } => { + ReprovisionState::ConfigureHostBoot { retry_count } + } + HostBootConfigStage::WaitingForBiosJob { bios_config_info } => { + ReprovisionState::WaitingForHostBiosJob { bios_config_info } + } + HostBootConfigStage::PollingBiosSetup { retry_count } => { + ReprovisionState::PollingHostBiosSetup { retry_count } + } + HostBootConfigStage::SetBootOrder { + set_boot_order_info, + } => ReprovisionState::SetHostBootOrder { + set_boot_order_info, + }, + }; + + Ok(StateHandlerOutcome::transition( + update_reprovision_targets_to_reprovision_state(state, reprovision_state)?, + )) + } + HostBootConfigOutcome::Complete => Ok(StateHandlerOutcome::transition( + update_reprovision_targets_to_reprovision_state( + state, + ReprovisionState::LockHostAfterBootRepair, + )?, + )), + HostBootConfigOutcome::Wait(reason) => Ok(StateHandlerOutcome::wait(reason)), + HostBootConfigOutcome::Failed { failure } => Ok(StateHandlerOutcome::transition( + dpu_reprovision_host_boot_failed_state( + &state.managed_state, + state.host_snapshot.id, + failure, + ), + )), + } +} + /// Build the correct failed state for host boot repair during DPU reprovision. fn dpu_reprovision_host_boot_failed_state( current_state: &ManagedHostState, @@ -3545,152 +3481,6 @@ async fn load_boot_predictions( Ok(predictions) } -/// Check whether host BIOS and DPU-first boot order remediation is required. -async fn check_host_boot_config( - redfish_client: &dyn Redfish, - mh_snapshot: &ManagedHostStateSnapshot, - reachability_params: &ReachabilityParams, - dpu_freshness: HostBootConfigDpuFreshness, - ctx: &mut StateHandlerContext<'_, MachineStateHandlerContextObjects>, -) -> Result { - // Wait for DPUs only when this caller needs a fresh observation. DPU - // reprovision already validated DPU health before entering host boot repair. - if should_wait_for_dpus_before_host_boot_config( - mh_snapshot, - reachability_params, - dpu_freshness, - ctx, - ) - .await - { - return Ok(HostBootConfigDecision::Wait( - "Waiting for DPUs to come up.".to_string(), - )); - } - - // Resolve the interface whose boot option should be first in host UEFI. A - // zero-DPU host whose boot NIC has not taken its first HostInband lease yet - // falls back to its predicted boot NIC, and only waits when even that is - // unavailable. - let predictions = load_boot_predictions(ctx, &mh_snapshot.host_snapshot.id).await?; - let boot_interface = match require_boot_interface( - mh_snapshot, - &predictions, - "configuring boot", - HostBootConfigDecision::Wait, - )? { - RequiredBootInterface::Ready(target) => target, - RequiredBootInterface::Wait(decision) => return Ok(decision), - }; - - let vendor = mh_snapshot.host_snapshot.bmc_vendor(); - - log_host_config(redfish_client, mh_snapshot).await; - - let is_bios_setup = boot_interface - .run(|bi| redfish_client.is_bios_setup(Some(bi))) - .await - .map_err(|e| redfish_error("is_bios_setup", e))?; - - if should_skip_boot_order_remediation(mh_snapshot) { - if is_bios_setup { - tracing::info!( - machine_id = %mh_snapshot.host_snapshot.id, - bmc_vendor = %vendor, - "Skipping boot order remediation on Viking (known FW/BMC issue)" - ); - return Ok(HostBootConfigDecision::LockHost); - } - - tracing::warn!( - machine_id = %mh_snapshot.host_snapshot.id, - bmc_vendor = %vendor, - "Host BIOS setup is not configured properly on Viking; running BIOS repair before skipping boot order remediation" - ); - return Ok(HostBootConfigDecision::ConfigureBoot); - } - - let is_boot_order_setup = boot_interface - .run(|bi| redfish_client.is_boot_order_setup(bi)) - .await - .map_err(|e| redfish_error("is_boot_order_setup", e))?; - - if is_bios_setup && is_boot_order_setup { - tracing::info!( - machine_id = %mh_snapshot.host_snapshot.id, - bmc_vendor = %vendor, - "Host BIOS setup and boot order are configured properly" - ); - Ok(HostBootConfigDecision::LockHost) - } else { - tracing::warn!( - machine_id = %mh_snapshot.host_snapshot.id, - bmc_vendor = %vendor, - is_bios_setup, - is_boot_order_setup, - "Host BIOS setup or boot order is not configured properly" - ); - Ok(HostBootConfigDecision::ConfigureBoot) - } -} - -/// Viking BMC firmware cannot safely run boot-order remediation; BIOS repair still applies. -fn should_skip_boot_order_remediation(mh_snapshot: &ManagedHostStateSnapshot) -> bool { - mh_snapshot - .host_snapshot - .hardware_info - .as_ref() - .is_some_and(|hw| hw.is_dgx_h100()) -} - -async fn should_wait_for_dpus_before_host_boot_config( - mh_snapshot: &ManagedHostStateSnapshot, - reachability_params: &ReachabilityParams, - dpu_freshness: HostBootConfigDpuFreshness, - ctx: &mut StateHandlerContext<'_, MachineStateHandlerContextObjects>, -) -> bool { - if !mh_snapshot.has_managed_dpus() { - return false; - } - - match dpu_freshness { - HostBootConfigDpuFreshness::AlreadyValidated => false, - HostBootConfigDpuFreshness::CurrentHostState => { - !are_dpus_up_trigger_reboot_if_needed(mh_snapshot, reachability_params, ctx).await - } - HostBootConfigDpuFreshness::SinceLastHostRebootRequest => { - let Some(last_reboot_requested) = mh_snapshot.host_snapshot.last_reboot_requested - else { - tracing::warn!( - machine_id = %mh_snapshot.host_snapshot.id, - "No host reboot request timestamp found before post-reboot host boot config check" - ); - return false; - }; - - for dpu_snapshot in &mh_snapshot.dpu_snapshots { - if !is_dpu_observed_since(dpu_snapshot, last_reboot_requested.time) { - match trigger_reboot_if_needed( - dpu_snapshot, - mh_snapshot, - None, - reachability_params, - ctx, - ) - .await - { - Ok(_) => {} - Err(e) => tracing::warn!("could not reboot dpu {}: {e}", dpu_snapshot.id), - } - return true; - } - } - - false - } - } -} - // Returns true if update_manager flagged this managed host as needing its firmware examined fn host_reprovisioning_requested(state: &ManagedHostStateSnapshot) -> bool { state.host_snapshot.host_reprovision_requested.is_some() @@ -4928,6 +4718,9 @@ pub struct RebootStatus { /// Outcome of set_host_boot_order function. enum SetBootOrderOutcome { Continue(SetBootOrderInfo), + /// BIOS drifted after the caller's inspection; return to the shared BIOS + /// configuration stages so any Redfish job ID is persisted and polled. + ConfigureBios, Done, WaitingForReboot(String), /// No boot interface to act on yet -- e.g. a zero-DPU host whose boot NIC @@ -4936,20 +4729,6 @@ enum SetBootOrderOutcome { Wait(String), } -/// Decision from checking whether host boot repair is still required. -enum HostBootConfigDecision { - ConfigureBoot, - LockHost, - Wait(String), -} - -/// DPU observation freshness required before checking host boot config. -enum HostBootConfigDpuFreshness { - AlreadyValidated, - CurrentHostState, - SinceLastHostRebootRequest, -} - /// Outcome of [`require_boot_interface`]: the resolved boot NIC, or the /// caller's wait outcome for a zero-DPU host still discovering its boot NIC. #[derive(Debug)] @@ -5407,63 +5186,63 @@ fn check_host_health_for_alerts(state: &ManagedHostStateSnapshot) -> Result<(), } } -async fn handle_host_boot_order_setup( +/// Handles a shared boot-configuration stage during host initialization. +/// +/// Called by `HostMachineStateHandler::handle_object_state` for its platform +/// configuration, vendor-job, polling, and boot-order states. This adapter maps +/// continued stages back into `MachineState`, proceeds to measurements on +/// completion, and reports terminal failure as a host-init BIOS setup failure. +async fn handle_host_init_boot_config_stage( ctx: &mut StateHandlerContext<'_, MachineStateHandlerContextObjects>, - host_handler_params: HostHandlerParams, - mh_snapshot: &mut ManagedHostStateSnapshot, - set_boot_order_info: Option, + mh_snapshot: &ManagedHostStateSnapshot, + reachability_params: &ReachabilityParams, + redfish_client: &dyn Redfish, + stage: HostBootConfigStage, ) -> Result, StateHandlerError> { - tracing::info!( - "Starting Boot Order Configuration for {}: {set_boot_order_info:#?}", - mh_snapshot.host_snapshot.id - ); - - let redfish_client = ctx - .services - .create_redfish_client_from_machine(&mh_snapshot.host_snapshot) - .await?; - - let next_state = match set_boot_order_info { - Some(info) => { - match set_host_boot_order( - ctx, - &host_handler_params.reachability_params, - redfish_client.as_ref(), - mh_snapshot, - info, - ) - .await? - { - SetBootOrderOutcome::Continue(boot_order_info) => ManagedHostState::HostInit { - machine_state: MachineState::SetBootOrder { - set_boot_order_info: Some(boot_order_info), - }, - }, - SetBootOrderOutcome::Done => ManagedHostState::HostInit { - machine_state: MachineState::Measuring { - measuring_state: MeasuringState::WaitingForMeasurements, - }, - }, - SetBootOrderOutcome::WaitingForReboot(reason) => { - return Ok(StateHandlerOutcome::wait(reason)); + match run_host_boot_config_stage(ctx, reachability_params, redfish_client, mh_snapshot, stage) + .await? + { + HostBootConfigOutcome::Continue(stage) => { + let machine_state = match stage { + HostBootConfigStage::ConfigureBios { retry_count } => { + MachineState::WaitingForPlatformConfiguration { retry_count } } - SetBootOrderOutcome::Wait(reason) => { - return Ok(StateHandlerOutcome::wait(reason)); + HostBootConfigStage::WaitingForBiosJob { bios_config_info } => { + MachineState::WaitingForBiosJob { bios_config_info } } - } + HostBootConfigStage::PollingBiosSetup { retry_count } => { + MachineState::PollingBiosSetup { retry_count } + } + HostBootConfigStage::SetBootOrder { + set_boot_order_info, + } => MachineState::SetBootOrder { + set_boot_order_info: Some(set_boot_order_info), + }, + }; + Ok(StateHandlerOutcome::transition( + ManagedHostState::HostInit { machine_state }, + )) } - None => ManagedHostState::HostInit { - machine_state: MachineState::SetBootOrder { - set_boot_order_info: Some(SetBootOrderInfo { - set_boot_order_jid: None, - set_boot_order_state: SetBootOrderState::SetBootOrder, - retry_count: 0, - }), + HostBootConfigOutcome::Complete => Ok(StateHandlerOutcome::transition( + ManagedHostState::HostInit { + machine_state: MachineState::Measuring { + measuring_state: MeasuringState::WaitingForMeasurements, + }, }, - }, - }; - - Ok(StateHandlerOutcome::transition(next_state)) + )), + HostBootConfigOutcome::Wait(reason) => Ok(StateHandlerOutcome::wait(reason)), + HostBootConfigOutcome::Failed { failure } => { + Ok(StateHandlerOutcome::transition(ManagedHostState::Failed { + details: FailureDetails { + cause: FailureCause::BiosSetupFailed { err: failure }, + failed_at: Utc::now(), + source: FailureSource::StateMachineArea(StateMachineArea::HostInit), + }, + machine_id: mh_snapshot.host_snapshot.id, + retry_count: 0, + })) + } + } } /// TODO: we need to handle the case where the job is deleted for some reason @@ -5847,147 +5626,76 @@ impl StateHandler for HostMachineStateHandler { } } - match configure_host_bios( + handle_host_init_boot_config_stage( ctx, + mh_snapshot, &self.host_handler_params.reachability_params, redfish_client.as_ref(), - mh_snapshot, - *retry_count, + HostBootConfigStage::ConfigureBios { + retry_count: *retry_count, + }, ) - .await? - { - BiosConfigOutcome::Done => Ok(StateHandlerOutcome::transition( - ManagedHostState::HostInit { - machine_state: MachineState::PollingBiosSetup { - retry_count: *retry_count, - }, - }, - )), - BiosConfigOutcome::WaitingForBiosJob(bios_config_info) => Ok( - StateHandlerOutcome::transition(ManagedHostState::HostInit { - machine_state: MachineState::WaitingForBiosJob { bios_config_info }, - }), - ), - BiosConfigOutcome::WaitingForReboot(reason) => { - Ok(StateHandlerOutcome::wait(reason)) - } - } + .await } MachineState::WaitingForBiosJob { bios_config_info } => { let redfish_client = ctx .services .create_redfish_client_from_machine(&mh_snapshot.host_snapshot) .await?; - match advance_bios_config_job( + handle_host_init_boot_config_stage( ctx, - redfish_client.as_ref(), mh_snapshot, - bios_config_info.clone(), - ) - .await? - { - BiosConfigJobAdvanceOutcome::Continue(updated) => Ok( - StateHandlerOutcome::transition(ManagedHostState::HostInit { - machine_state: MachineState::WaitingForBiosJob { - bios_config_info: updated, - }, - }), - ), - BiosConfigJobAdvanceOutcome::Done => Ok(StateHandlerOutcome::transition( - ManagedHostState::HostInit { - machine_state: MachineState::PollingBiosSetup { - retry_count: bios_config_info.retry_count, - }, - }, - )), - BiosConfigJobAdvanceOutcome::Failed { failure } => { - Ok(StateHandlerOutcome::transition(ManagedHostState::Failed { - details: FailureDetails { - cause: FailureCause::BiosSetupFailed { err: failure }, - failed_at: Utc::now(), - source: FailureSource::StateMachineArea( - StateMachineArea::HostInit, - ), - }, - machine_id: mh_snapshot.host_snapshot.id, - retry_count: 0, - })) - } - BiosConfigJobAdvanceOutcome::RetryPlatformConfiguration { retry_count } => { - Ok(StateHandlerOutcome::transition( - ManagedHostState::HostInit { - machine_state: MachineState::WaitingForPlatformConfiguration { - retry_count, - }, - }, - )) - } - BiosConfigJobAdvanceOutcome::Wait(reason) => { - Ok(StateHandlerOutcome::wait(reason)) - } - } - } - MachineState::PollingBiosSetup { retry_count } => { - let next_state = ManagedHostState::HostInit { - machine_state: MachineState::SetBootOrder { - set_boot_order_info: Some(SetBootOrderInfo { - set_boot_order_jid: None, - set_boot_order_state: SetBootOrderState::SetBootOrder, - retry_count: 0, - }), + &self.host_handler_params.reachability_params, + redfish_client.as_ref(), + HostBootConfigStage::WaitingForBiosJob { + bios_config_info: bios_config_info.clone(), }, - }; - + ) + .await + } + MachineState::PollingBiosSetup { retry_count } => { let redfish_client = ctx .services .create_redfish_client_from_machine(&mh_snapshot.host_snapshot) .await?; - let predictions = - load_boot_predictions(ctx, &mh_snapshot.host_snapshot.id).await?; - match advance_polling_bios_setup( - redfish_client.as_ref(), + handle_host_init_boot_config_stage( + ctx, mh_snapshot, - *retry_count, - &ctx.services.site_config.machine_state_controller, - &predictions, + &self.host_handler_params.reachability_params, + redfish_client.as_ref(), + HostBootConfigStage::PollingBiosSetup { + retry_count: *retry_count, + }, ) - .await? - { - PollingBiosSetupOutcome::Verified => { - Ok(StateHandlerOutcome::transition(next_state)) - } - PollingBiosSetupOutcome::EnterRecovery(bios_config_info) => Ok( - StateHandlerOutcome::transition(ManagedHostState::HostInit { - machine_state: MachineState::WaitingForBiosJob { bios_config_info }, - }), - ), - PollingBiosSetupOutcome::Failed { failure } => { - Ok(StateHandlerOutcome::transition(ManagedHostState::Failed { - details: FailureDetails { - cause: FailureCause::BiosSetupFailed { err: failure }, - failed_at: Utc::now(), - source: FailureSource::StateMachineArea( - StateMachineArea::HostInit, - ), - }, - machine_id: mh_snapshot.host_snapshot.id, - retry_count: 0, - })) - } - PollingBiosSetupOutcome::Wait(reason) => { - Ok(StateHandlerOutcome::wait(reason)) - } - } + .await } MachineState::SetBootOrder { set_boot_order_info, - } => Ok(handle_host_boot_order_setup( - ctx, - self.host_handler_params.clone(), - mh_snapshot, - set_boot_order_info.clone(), - ) - .await?), + } => { + let Some(set_boot_order_info) = set_boot_order_info.clone() else { + return Ok(StateHandlerOutcome::transition( + ManagedHostState::HostInit { + machine_state: MachineState::SetBootOrder { + set_boot_order_info: Some(initial_set_boot_order_info()), + }, + }, + )); + }; + let redfish_client = ctx + .services + .create_redfish_client_from_machine(&mh_snapshot.host_snapshot) + .await?; + handle_host_init_boot_config_stage( + ctx, + mh_snapshot, + &self.host_handler_params.reachability_params, + redfish_client.as_ref(), + HostBootConfigStage::SetBootOrder { + set_boot_order_info, + }, + ) + .await + } MachineState::Measuring { measuring_state } => { if !self.host_handler_params.attestation_enabled { return Ok(StateHandlerOutcome::transition( @@ -7259,11 +6967,7 @@ impl StateHandler for InstanceStateHandler { instance_state: InstanceState::HostPlatformConfiguration { platform_config_state: HostPlatformConfigurationState::SetBootOrder { - set_boot_order_info: SetBootOrderInfo { - set_boot_order_jid: None, - set_boot_order_state: SetBootOrderState::SetBootOrder, - retry_count: 0, - }, + set_boot_order_info: initial_set_boot_order_info(), }, }, }; @@ -10854,6 +10558,75 @@ async fn log_host_config(redfish_client: &dyn Redfish, mh_snapshot: &ManagedHost ); } +/// Handles a shared boot-configuration stage during assigned-host platform +/// configuration. +/// +/// Called by `handle_instance_host_platform_config` for its BIOS configuration, +/// vendor-job, polling, and boot-order states. This adapter maps continued +/// stages back into `HostPlatformConfigurationState`, proceeds to `LockHost` on +/// completion, and scopes terminal failure to the assigned instance. +async fn handle_instance_host_boot_config_stage( + ctx: &mut StateHandlerContext<'_, MachineStateHandlerContextObjects>, + mh_snapshot: &ManagedHostStateSnapshot, + reachability_params: &ReachabilityParams, + redfish_client: &dyn Redfish, + stage: HostBootConfigStage, +) -> Result, StateHandlerError> { + match run_host_boot_config_stage(ctx, reachability_params, redfish_client, mh_snapshot, stage) + .await? + { + HostBootConfigOutcome::Continue(stage) => { + let platform_config_state = match stage { + HostBootConfigStage::ConfigureBios { retry_count } => { + HostPlatformConfigurationState::ConfigureBios { + bios_config_info: None, + retry_count, + } + } + HostBootConfigStage::WaitingForBiosJob { bios_config_info } => { + HostPlatformConfigurationState::WaitingForBiosJob { bios_config_info } + } + HostBootConfigStage::PollingBiosSetup { retry_count } => { + HostPlatformConfigurationState::PollingBiosSetup { retry_count } + } + HostBootConfigStage::SetBootOrder { + set_boot_order_info, + } => HostPlatformConfigurationState::SetBootOrder { + set_boot_order_info, + }, + }; + + Ok(StateHandlerOutcome::transition( + ManagedHostState::Assigned { + instance_state: InstanceState::HostPlatformConfiguration { + platform_config_state, + }, + }, + )) + } + HostBootConfigOutcome::Complete => Ok(StateHandlerOutcome::transition( + ManagedHostState::Assigned { + instance_state: InstanceState::HostPlatformConfiguration { + platform_config_state: HostPlatformConfigurationState::LockHost, + }, + }, + )), + HostBootConfigOutcome::Wait(reason) => Ok(StateHandlerOutcome::wait(reason)), + HostBootConfigOutcome::Failed { failure } => Ok(StateHandlerOutcome::transition( + ManagedHostState::Assigned { + instance_state: InstanceState::Failed { + details: FailureDetails { + cause: FailureCause::BiosSetupFailed { err: failure }, + failed_at: Utc::now(), + source: FailureSource::StateMachineArea(StateMachineArea::AssignedInstance), + }, + machine_id: mh_snapshot.host_snapshot.id, + }, + }, + )), + } +} + async fn handle_instance_host_platform_config( ctx: &mut StateHandlerContext<'_, MachineStateHandlerContextObjects>, mh_snapshot: &mut ManagedHostStateSnapshot, @@ -11082,18 +10855,29 @@ async fn handle_instance_host_platform_config( ) .await? { - HostBootConfigDecision::Wait(reason) => { + HostBootConfigCheckOutcome::Wait(reason) => { return Ok(StateHandlerOutcome::wait(reason)); } - HostBootConfigDecision::ConfigureBoot => InstanceState::HostPlatformConfiguration { - platform_config_state: HostPlatformConfigurationState::ConfigureBios { - bios_config_info: None, - retry_count: 0, - }, - }, - HostBootConfigDecision::LockHost => InstanceState::HostPlatformConfiguration { - platform_config_state: HostPlatformConfigurationState::LockHost, - }, + HostBootConfigCheckOutcome::Ready(HostBootConfigDecision::ConfigureBios) => { + InstanceState::HostPlatformConfiguration { + platform_config_state: HostPlatformConfigurationState::ConfigureBios { + bios_config_info: None, + retry_count: 0, + }, + } + } + HostBootConfigCheckOutcome::Ready(HostBootConfigDecision::SetBootOrder) => { + InstanceState::HostPlatformConfiguration { + platform_config_state: HostPlatformConfigurationState::SetBootOrder { + set_boot_order_info: initial_set_boot_order_info(), + }, + } + } + HostBootConfigCheckOutcome::Ready(HostBootConfigDecision::Complete) => { + InstanceState::HostPlatformConfiguration { + platform_config_state: HostPlatformConfigurationState::LockHost, + } + } } } HostPlatformConfigurationState::ConfigureBios { @@ -11114,174 +10898,48 @@ async fn handle_instance_host_platform_config( )); } - let next_platform = match configure_host_bios( + return handle_instance_host_boot_config_stage( ctx, + mh_snapshot, reachability_params, redfish_client.as_ref(), - mh_snapshot, - retry_count, + HostBootConfigStage::ConfigureBios { retry_count }, ) - .await? - { - BiosConfigOutcome::Done => { - HostPlatformConfigurationState::PollingBiosSetup { retry_count } - } - BiosConfigOutcome::WaitingForBiosJob(bios_config_info) => { - HostPlatformConfigurationState::WaitingForBiosJob { bios_config_info } - } - BiosConfigOutcome::WaitingForReboot(reason) => { - return Ok(StateHandlerOutcome::wait(reason)); - } - }; - return Ok(StateHandlerOutcome::transition( - ManagedHostState::Assigned { - instance_state: InstanceState::HostPlatformConfiguration { - platform_config_state: next_platform, - }, - }, - )); + .await; } HostPlatformConfigurationState::WaitingForBiosJob { bios_config_info } => { - let next_platform = match advance_bios_config_job( + return handle_instance_host_boot_config_stage( ctx, - redfish_client.as_ref(), mh_snapshot, - bios_config_info.clone(), + reachability_params, + redfish_client.as_ref(), + HostBootConfigStage::WaitingForBiosJob { bios_config_info }, ) - .await? - { - BiosConfigJobAdvanceOutcome::Continue(updated) => { - HostPlatformConfigurationState::WaitingForBiosJob { - bios_config_info: updated, - } - } - BiosConfigJobAdvanceOutcome::Done => { - HostPlatformConfigurationState::PollingBiosSetup { - retry_count: bios_config_info.retry_count, - } - } - BiosConfigJobAdvanceOutcome::Failed { failure } => { - return Ok(StateHandlerOutcome::transition( - ManagedHostState::Assigned { - instance_state: InstanceState::Failed { - details: FailureDetails { - cause: FailureCause::BiosSetupFailed { err: failure }, - failed_at: Utc::now(), - source: FailureSource::StateMachineArea( - StateMachineArea::AssignedInstance, - ), - }, - machine_id: mh_snapshot.host_snapshot.id, - }, - }, - )); - } - BiosConfigJobAdvanceOutcome::RetryPlatformConfiguration { - retry_count: next_count, - } => HostPlatformConfigurationState::ConfigureBios { - bios_config_info: None, - retry_count: next_count, - }, - BiosConfigJobAdvanceOutcome::Wait(reason) => { - return Ok(StateHandlerOutcome::wait(reason)); - } - }; - return Ok(StateHandlerOutcome::transition( - ManagedHostState::Assigned { - instance_state: InstanceState::HostPlatformConfiguration { - platform_config_state: next_platform, - }, - }, - )); + .await; } HostPlatformConfigurationState::PollingBiosSetup { retry_count } => { - let next_instance_state = InstanceState::HostPlatformConfiguration { - platform_config_state: if should_skip_boot_order_remediation(mh_snapshot) { - HostPlatformConfigurationState::LockHost - } else { - HostPlatformConfigurationState::SetBootOrder { - set_boot_order_info: SetBootOrderInfo { - set_boot_order_jid: None, - set_boot_order_state: SetBootOrderState::SetBootOrder, - retry_count: 0, - }, - } - }, - }; - - let predictions = load_boot_predictions(ctx, &mh_snapshot.host_snapshot.id).await?; - match advance_polling_bios_setup( - redfish_client.as_ref(), + return handle_instance_host_boot_config_stage( + ctx, mh_snapshot, - retry_count, - &ctx.services.site_config.machine_state_controller, - &predictions, + reachability_params, + redfish_client.as_ref(), + HostBootConfigStage::PollingBiosSetup { retry_count }, ) - .await? - { - PollingBiosSetupOutcome::Verified => next_instance_state, - PollingBiosSetupOutcome::EnterRecovery(bios_config_info) => { - return Ok(StateHandlerOutcome::transition( - ManagedHostState::Assigned { - instance_state: InstanceState::HostPlatformConfiguration { - platform_config_state: - HostPlatformConfigurationState::WaitingForBiosJob { - bios_config_info, - }, - }, - }, - )); - } - PollingBiosSetupOutcome::Failed { failure } => { - return Ok(StateHandlerOutcome::transition( - ManagedHostState::Assigned { - instance_state: InstanceState::Failed { - details: FailureDetails { - cause: FailureCause::BiosSetupFailed { err: failure }, - failed_at: Utc::now(), - source: FailureSource::StateMachineArea( - StateMachineArea::AssignedInstance, - ), - }, - machine_id: mh_snapshot.host_snapshot.id, - }, - }, - )); - } - PollingBiosSetupOutcome::Wait(reason) => { - return Ok(StateHandlerOutcome::wait(reason)); - } - } + .await; } HostPlatformConfigurationState::SetBootOrder { set_boot_order_info, } => { - match set_host_boot_order( + return handle_instance_host_boot_config_stage( ctx, + mh_snapshot, reachability_params, redfish_client.as_ref(), - mh_snapshot, - set_boot_order_info, - ) - .await? - { - SetBootOrderOutcome::Continue(boot_order_info) => { - InstanceState::HostPlatformConfiguration { - platform_config_state: HostPlatformConfigurationState::SetBootOrder { - set_boot_order_info: boot_order_info, - }, - } - } - SetBootOrderOutcome::Done => InstanceState::HostPlatformConfiguration { - platform_config_state: HostPlatformConfigurationState::LockHost, + HostBootConfigStage::SetBootOrder { + set_boot_order_info, }, - SetBootOrderOutcome::WaitingForReboot(reason) => { - return Ok(StateHandlerOutcome::wait(reason)); - } - SetBootOrderOutcome::Wait(reason) => { - return Ok(StateHandlerOutcome::wait(reason)); - } - } + ) + .await; } HostPlatformConfigurationState::LockHost => { if mh_snapshot.host_snapshot.host_profile.disable_lockdown { @@ -11342,51 +11000,36 @@ async fn set_host_boot_order( RequiredBootInterface::Wait(outcome) => return Ok(outcome), }; - // Don't re-apply a boot config that's already in place. `SetBootOrder` - // re-asserts the HTTP-boot device (`machine_setup`) and then sets the - // boot order, but on Dell two BIOS config writes can't share one - // pending job: re-asserting commits a config job, and - // `set_boot_order_dpu_first` then can't set the order (iDRAC `SYS011`, - // "a config job is already committed"), so re-applying an - // already-correct config collides and loops. Check first -- if the - // config is already set, skip straight to verification. Only when the - // HTTP-boot device was actually reverted (the boot NIC dropped off the - // BMC's Redfish inventory on a reboot) do we re-assert it, and then - // reboot to apply it before the boot-order set, so the two writes - // never land in one pass. + // Don't re-apply a boot config that's already in place. On Dell two + // BIOS config writes cannot share one pending job: machine_setup can + // commit a job that collides with set_boot_order_dpu_first (iDRAC + // `SYS011`). If the HTTP device drifted since the owning lifecycle's + // inspection, return to the shared BIOS/job stages before attempting + // the order write. let is_bios_setup = boot_interface .run(|bi| redfish_client.is_bios_setup(Some(bi))) .await .map_err(|e| redfish_error("is_bios_setup", e))?; if !is_bios_setup { - // The HTTP-boot device was reverted (the boot NIC dropped off the - // BMC's Redfish inventory on a reboot), so re-assert it with - // `machine_setup` and restart the host to apply it *before* - // setting the boot order -- the two BIOS writes never share one - // pass (they can't share one Dell config job). The re-assert - // commits once: `WaitForHttpBootDeviceApplied` polls the device - // across the reboot instead of this arm re-writing the staged - // settings and re-creating the config job every pass while the - // reboot lands. - call_machine_setup_and_handle_no_dpu_error( - redfish_client, - Some(&boot_interface), - mh_snapshot.host_snapshot.associated_dpu_machine_ids().len(), - &ctx.services.site_config, - ) - .await - .map_err(|e| redfish_error("machine_setup", e))?; - - handler_host_power_control(mh_snapshot, ctx, SystemPowerControl::ForceRestart) - .await?; - log_host_config(redfish_client, mh_snapshot).await; - - return Ok(SetBootOrderOutcome::Continue(SetBootOrderInfo { - set_boot_order_jid: None, - set_boot_order_state: SetBootOrderState::WaitForHttpBootDeviceApplied, - retry_count: set_boot_order_info.retry_count, - })); + // The HTTP-boot device drifted after the owning lifecycle's + // inspection. Route back through the shared BIOS driver rather + // than issuing machine_setup here: vendor jobs must be persisted + // and polled before boot-order work can safely continue. + return Ok(SetBootOrderOutcome::ConfigureBios); + } + + // Keep the vendor safety policy at the actuator boundary. This is + // intentionally after the BIOS check: validation reuses this state + // to restore a reverted HTTP boot device on Viking, where BIOS + // repair is supported but boot-order remediation is not. + if should_skip_boot_order_remediation(mh_snapshot) { + tracing::info!( + machine_id = %mh_snapshot.host_snapshot.id, + bmc_vendor = %mh_snapshot.host_snapshot.bmc_vendor(), + "Skipping boot order remediation on Viking (known FW/BMC issue)" + ); + return Ok(SetBootOrderOutcome::Done); } // HTTP-boot device is already set, so `machine_setup` was not re-run @@ -11477,14 +11120,9 @@ async fn set_host_boot_order( })) } SetBootOrderState::WaitForHttpBootDeviceApplied => { - // The SetBootOrder substate re-asserted the reverted HTTP-boot device - // and restarted the host; the staged BIOS settings apply across that - // reboot. Poll until the device verifies, then return to SetBootOrder - // to set the boot order. If it still hasn't applied once the reboot - // has had time to land (the boot NIC can drop off the BMC's Redfish - // inventory again on the apply reboot itself), return to SetBootOrder - // for a fresh re-assert -- re-commits stay bounded to one per wait - // window instead of one per pass. + // Backward compatibility for hosts persisted in the former inline + // machine_setup flow. Once its wait window expires, migrate into the + // shared BIOS/job stages so any new vendor job is retained. const HTTP_BOOT_DEVICE_APPLY_WAIT_MINUTES: i64 = 10; const MAX_HTTP_BOOT_DEVICE_APPLY_RETRIES: u32 = 3; @@ -11523,11 +11161,10 @@ async fn set_host_boot_order( .num_minutes(); if minutes_waiting >= HTTP_BOOT_DEVICE_APPLY_WAIT_MINUTES { - // Each expired window spends one retry from the SetBootOrder - // state machine's shared budget. Once it's exhausted the device - // is not coming back on its own -- stop re-asserting and surface - // the host for an operator, the same terminal escalation the - // reboot pacer applies to a host that never comes up. + // Preserve the terminal behavior of a legacy state whose old + // retry budget was already exhausted. Any nonterminal state + // carries its existing count into the shared convergence + // budget. if set_boot_order_info.retry_count >= MAX_HTTP_BOOT_DEVICE_APPLY_RETRIES { return Err(StateHandlerError::ManualInterventionRequired(format!( "HTTP boot device on host {} still not applied after {} re-asserts; manual intervention required", @@ -11537,13 +11174,9 @@ async fn set_host_boot_order( tracing::warn!( machine_id = %mh_snapshot.host_snapshot.id, minutes_waiting, - "Re-asserted HTTP boot device still not applied after the wait window; returning to SetBootOrder to re-assert", + "Re-asserted HTTP boot device still not applied after the wait window; migrating to shared BIOS repair", ); - return Ok(SetBootOrderOutcome::Continue(SetBootOrderInfo { - set_boot_order_jid: None, - set_boot_order_state: SetBootOrderState::SetBootOrder, - retry_count: set_boot_order_info.retry_count + 1, - })); + return Ok(SetBootOrderOutcome::ConfigureBios); } Ok(SetBootOrderOutcome::WaitingForReboot(format!( @@ -11573,7 +11206,8 @@ async fn set_host_boot_order( })) } SetBootOrderState::RebootHost => { - // Host needs to be rebooted to pick up the changes after calling machine_setup + // Reboot before final verification so any accepted boot-order + // change or vendor job can apply. handler_host_power_control(mh_snapshot, ctx, SystemPowerControl::ForceRestart).await?; Ok(SetBootOrderOutcome::Continue(SetBootOrderInfo { @@ -11757,10 +11391,14 @@ async fn set_host_boot_order( } } SetBootOrderState::CheckBootOrder => { - const MAX_BOOT_ORDER_RETRIES: u32 = 3; const CHECK_BOOT_ORDER_TIMEOUT_MINUTES: i64 = 30; let retry_count = set_boot_order_info.retry_count; + let max_retries = ctx + .services + .site_config + .machine_state_controller + .max_bios_config_retries; let boot_interface = match require_boot_interface( mh_snapshot, @@ -11772,6 +11410,36 @@ async fn set_host_boot_order( RequiredBootInterface::Wait(outcome) => return Ok(outcome), }; + // The boot-order job/recovery reboot can itself revert the HTTP + // device or another managed BIOS attribute. Verify BIOS first for + // every platform and return to the shared job-aware repair before + // declaring the overall boot configuration complete. + let is_bios_setup = boot_interface + .run(|bi| redfish_client.is_bios_setup(Some(bi))) + .await + .map_err(|e| redfish_error("is_bios_setup", e))?; + + if !is_bios_setup { + tracing::warn!( + machine_id = %mh_snapshot.host_snapshot.id, + bmc_vendor = %mh_snapshot.host_snapshot.bmc_vendor(), + "Host BIOS drifted during boot-order recovery; returning to shared BIOS repair" + ); + return Ok(SetBootOrderOutcome::ConfigureBios); + } + + // A controller upgrade can resume a Viking after an old boot-order + // job or power-recovery substate. Once the operation and BIOS + // verification are complete, skip the unsupported order read. + if should_skip_boot_order_remediation(mh_snapshot) { + tracing::info!( + machine_id = %mh_snapshot.host_snapshot.id, + bmc_vendor = %mh_snapshot.host_snapshot.bmc_vendor(), + "Skipping boot order verification on Viking (known FW/BMC issue)" + ); + return Ok(SetBootOrderOutcome::Done); + } + let boot_order_configured = boot_interface .run(|bi| redfish_client.is_boot_order_setup(bi)) .await @@ -11796,16 +11464,23 @@ async fn set_host_boot_order( time_since_state_change.num_minutes() ); - // If we've been stuck for 30+ minutes and haven't exhausted retries, retry SetBootOrder - if time_since_state_change.num_minutes() >= CHECK_BOOT_ORDER_TIMEOUT_MINUTES - && retry_count < MAX_BOOT_ORDER_RETRIES - { - tracing::info!( - "Boot order check timed out for {} after {} minutes, retrying SetBootOrder (retry {} of {})", + if time_since_state_change.num_minutes() < CHECK_BOOT_ORDER_TIMEOUT_MINUTES { + return Err(StateHandlerError::GenericError(eyre::eyre!( + "Boot order is not configured properly for host {} after SetBootOrder completed (retry_count: {}, max_retries: {}, time_in_state: {} minutes)", mh_snapshot.host_snapshot.id, - time_since_state_change.num_minutes(), - retry_count + 1, - MAX_BOOT_ORDER_RETRIES + retry_count, + max_retries, + time_since_state_change.num_minutes() + ))); + } + + if retry_count < max_retries { + tracing::info!( + machine_id = %mh_snapshot.host_snapshot.id, + time_in_state_minutes = time_since_state_change.num_minutes(), + retry_count = retry_count + 1, + max_retries, + "Boot order check timed out; retrying SetBootOrder" ); return Ok(SetBootOrderOutcome::Continue(SetBootOrderInfo { @@ -11815,11 +11490,13 @@ async fn set_host_boot_order( })); } - // Either still within timeout window or exhausted retries - return error - Err(StateHandlerError::GenericError(eyre::eyre!( - "Boot order is not configured properly for host {} after SetBootOrder completed (retry_count: {}, time_in_state: {} minutes)", + // Keep the state parked so an operator can repair the BMC and the + // next verification can complete without resetting the lifecycle. + Err(StateHandlerError::ManualInterventionRequired(format!( + "Boot order is not configured properly for host {} after SetBootOrder completed; automated boot-config convergence exhausted (retry_count: {}, max_retries: {}, time_in_state: {} minutes)", mh_snapshot.host_snapshot.id, retry_count, + max_retries, time_since_state_change.num_minutes() ))) } diff --git a/crates/machine-controller/src/handler/bios_config.rs b/crates/machine-controller/src/handler/bios_config.rs index b94aaa5992..b8644f809e 100644 --- a/crates/machine-controller/src/handler/bios_config.rs +++ b/crates/machine-controller/src/handler/bios_config.rs @@ -363,7 +363,7 @@ fn try_bios_recovery_attempt( ); return Ok(BiosRecoveryAttemptOutcome::Failed { failure: format!( - "{failure} (automated BIOS recovery exhausted after {} attempts)", + "{failure} (automated BIOS recovery exhausted after {} retries)", machine_controller_config.max_bios_config_retries ), }); diff --git a/crates/machine-controller/src/handler/host_boot_config.rs b/crates/machine-controller/src/handler/host_boot_config.rs new file mode 100644 index 0000000000..076f6a1527 --- /dev/null +++ b/crates/machine-controller/src/handler/host_boot_config.rs @@ -0,0 +1,542 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +//! Shared host boot-configuration convergence. +//! +//! The lifecycle state machines retain ownership of their persisted state +//! shapes. [`HostBootConfigStage`] is only a runtime projection of those +//! states, so this module can drive the common BIOS/job/poll/boot-order +//! sequence without introducing another serialized controller wrapper. +//! +//! [`run_host_boot_config_stage`] owns the lifecycle-neutral Redfish work. Its +//! four lifecycle adapters stay beside the state machines that own their +//! persisted state and terminal behavior: +//! - `handle_host_init_boot_config_stage` in `handler.rs`; +//! - `handle_instance_host_boot_config_stage` in `handler.rs`; +//! - `handle_dpu_reprovision_host_boot_config_stage` in `handler.rs`; +//! - `handle_validation_boot_config_stage` in `machine_validation.rs`. + +use carbide_redfish::boot_interface::BootInterfaceTarget; +use carbide_redfish::libredfish::error::state_handler_redfish_error as redfish_error; +use libredfish::Redfish; +use model::machine::{ + BiosConfigInfo, ManagedHostStateSnapshot, SetBootOrderInfo, SetBootOrderState, +}; +use state_controller::state_handler::{StateHandlerContext, StateHandlerError}; + +use super::bios_config::{ + BiosConfigJobAdvanceOutcome, BiosConfigOutcome, PollingBiosSetupOutcome, + advance_bios_config_job, advance_polling_bios_setup, configure_host_bios, +}; +use super::{ + ReachabilityParams, RequiredBootInterface, SetBootOrderOutcome, + are_dpus_up_trigger_reboot_if_needed, is_dpu_observed_since, load_boot_predictions, + log_host_config, require_boot_interface, set_host_boot_order, trigger_reboot_if_needed, +}; +use crate::context::MachineStateHandlerContextObjects; + +/// Runtime projection of the persisted BIOS and boot-order states shared by +/// HostInit, assigned platform configuration, DPU reprovision, and validation. +/// +/// This type must remain controller-internal: the owning lifecycle maps each +/// `Continue` outcome back to its existing serialized state variant. +#[derive(Debug, Clone, Eq, PartialEq)] +pub(super) enum HostBootConfigStage { + ConfigureBios { + retry_count: u32, + }, + WaitingForBiosJob { + bios_config_info: BiosConfigInfo, + }, + PollingBiosSetup { + retry_count: u32, + }, + SetBootOrder { + set_boot_order_info: SetBootOrderInfo, + }, +} + +/// Result of running one common boot-configuration stage. +#[derive(Debug, Clone, Eq, PartialEq)] +pub(super) enum HostBootConfigOutcome { + Continue(HostBootConfigStage), + Complete, + Wait(String), + Failed { failure: String }, +} + +/// The next remediation required after inspecting the desired and actual host +/// boot configuration. +#[derive(Debug, Clone, Eq, PartialEq)] +pub(super) enum HostBootConfigDecision { + /// The platform BIOS settings (including the HTTP boot device) drifted. + ConfigureBios, + /// BIOS is already correct; only boot order drifted. + SetBootOrder, + /// BIOS and every NICo-managed boot-order setting are already correct. + Complete, +} + +/// Result of pacing prerequisites and inspecting the host boot config. +#[derive(Debug, Clone, Eq, PartialEq)] +pub(super) enum HostBootConfigCheckOutcome { + Ready(HostBootConfigDecision), + /// The host cannot be inspected yet, for example while discovering a + /// zero-DPU host's boot NIC or waiting for a fresh DPU observation. + Wait(String), +} + +/// DPU observation freshness required before trusting Redfish boot state. +#[derive(Debug, Clone, Copy, Eq, PartialEq)] +pub(super) enum HostBootConfigDpuFreshness { + /// The caller already gated on healthy, synchronized DPUs. + AlreadyValidated, + /// Require observations associated with the current host lifecycle state. + CurrentHostState, + /// Require observations newer than the most recent host reboot request. + SinceLastHostRebootRequest, +} + +/// Actual host boot settings read from Redfish. +/// +/// `boot_order_setup` is `None` when BIOS drift already determines the next +/// action, or when boot order is intentionally not managed on this hardware. +#[derive(Debug, Clone, Copy, Eq, PartialEq)] +pub(super) struct HostBootConfigInspection { + pub is_bios_setup: bool, + pub is_boot_order_setup: Option, +} + +/// Convert an inspection into the smallest remediation that can converge it. +/// BIOS repair takes precedence because it may recreate the boot option whose +/// order would otherwise be inspected or changed. +pub(super) fn decide_host_boot_config( + inspection: HostBootConfigInspection, +) -> HostBootConfigDecision { + if !inspection.is_bios_setup { + HostBootConfigDecision::ConfigureBios + } else if inspection.is_boot_order_setup == Some(false) { + HostBootConfigDecision::SetBootOrder + } else { + HostBootConfigDecision::Complete + } +} + +/// Inspect a host whose boot interface has already been resolved. +/// +/// Keeping interface resolution outside this primitive lets validation retain +/// its existing distinction between a missing interface (hard error), an +/// undiscovered zero-DPU NIC (pace the reboot), and a Redfish read error (log +/// and retry). +pub(super) async fn inspect_host_boot_config( + redfish_client: &dyn Redfish, + mh_snapshot: &ManagedHostStateSnapshot, + boot_interface: &BootInterfaceTarget, +) -> Result { + let is_bios_setup = boot_interface + .run(|bi| redfish_client.is_bios_setup(Some(bi))) + .await + .map_err(|e| redfish_error("is_bios_setup", e))?; + + if !is_bios_setup || should_skip_boot_order_remediation(mh_snapshot) { + return Ok(HostBootConfigInspection { + is_bios_setup, + is_boot_order_setup: None, + }); + } + + let is_boot_order_setup = boot_interface + .run(|bi| redfish_client.is_boot_order_setup(bi)) + .await + .map_err(|e| redfish_error("is_boot_order_setup", e))?; + + Ok(HostBootConfigInspection { + is_bios_setup, + is_boot_order_setup: Some(is_boot_order_setup), + }) +} + +/// Check whether host BIOS or boot order needs remediation. +pub(super) async fn check_host_boot_config( + redfish_client: &dyn Redfish, + mh_snapshot: &ManagedHostStateSnapshot, + reachability_params: &ReachabilityParams, + dpu_freshness: HostBootConfigDpuFreshness, + ctx: &mut StateHandlerContext<'_, MachineStateHandlerContextObjects>, +) -> Result { + if should_wait_for_dpus_before_host_boot_config( + mh_snapshot, + reachability_params, + dpu_freshness, + ctx, + ) + .await + { + return Ok(HostBootConfigCheckOutcome::Wait( + "Waiting for DPUs to come up.".to_string(), + )); + } + + let predictions = load_boot_predictions(ctx, &mh_snapshot.host_snapshot.id).await?; + let boot_interface = match require_boot_interface( + mh_snapshot, + &predictions, + "configuring boot", + HostBootConfigCheckOutcome::Wait, + )? { + RequiredBootInterface::Ready(target) => target, + RequiredBootInterface::Wait(outcome) => return Ok(outcome), + }; + + log_host_config(redfish_client, mh_snapshot).await; + + let inspection = inspect_host_boot_config(redfish_client, mh_snapshot, &boot_interface).await?; + let decision = decide_host_boot_config(inspection); + let vendor = mh_snapshot.host_snapshot.bmc_vendor(); + + match &decision { + HostBootConfigDecision::ConfigureBios => tracing::warn!( + machine_id = %mh_snapshot.host_snapshot.id, + bmc_vendor = %vendor, + "Host BIOS setup is not configured properly" + ), + HostBootConfigDecision::SetBootOrder => tracing::warn!( + machine_id = %mh_snapshot.host_snapshot.id, + bmc_vendor = %vendor, + "Host BIOS setup is correct but boot order is not configured properly" + ), + HostBootConfigDecision::Complete if should_skip_boot_order_remediation(mh_snapshot) => { + tracing::info!( + machine_id = %mh_snapshot.host_snapshot.id, + bmc_vendor = %vendor, + "Host BIOS setup is configured; skipping boot order remediation on Viking" + ); + } + HostBootConfigDecision::Complete => tracing::info!( + machine_id = %mh_snapshot.host_snapshot.id, + bmc_vendor = %vendor, + "Host BIOS setup and boot order are configured properly" + ), + } + + Ok(HostBootConfigCheckOutcome::Ready(decision)) +} + +/// Runs one lifecycle-neutral host boot-configuration stage. +/// +/// Called by the host-init, assigned-instance, DPU-reprovision, and validation +/// adapters. This function performs the Redfish BIOS, vendor-job, polling, or +/// boot-order work for one [`HostBootConfigStage`] and returns a +/// [`HostBootConfigOutcome`]. The caller owns the persisted lifecycle state and +/// maps the outcome back into that state. +pub(super) async fn run_host_boot_config_stage( + ctx: &mut StateHandlerContext<'_, MachineStateHandlerContextObjects>, + reachability_params: &ReachabilityParams, + redfish_client: &dyn Redfish, + mh_snapshot: &ManagedHostStateSnapshot, + stage: HostBootConfigStage, +) -> Result { + match stage { + HostBootConfigStage::ConfigureBios { retry_count } => { + match configure_host_bios( + ctx, + reachability_params, + redfish_client, + mh_snapshot, + retry_count, + ) + .await? + { + BiosConfigOutcome::Done => Ok(HostBootConfigOutcome::Continue( + HostBootConfigStage::PollingBiosSetup { retry_count }, + )), + BiosConfigOutcome::WaitingForBiosJob(bios_config_info) => { + Ok(HostBootConfigOutcome::Continue( + HostBootConfigStage::WaitingForBiosJob { bios_config_info }, + )) + } + BiosConfigOutcome::WaitingForReboot(reason) => { + Ok(HostBootConfigOutcome::Wait(reason)) + } + } + } + HostBootConfigStage::WaitingForBiosJob { bios_config_info } => { + let retry_count = bios_config_info.retry_count; + match advance_bios_config_job(ctx, redfish_client, mh_snapshot, bios_config_info) + .await? + { + BiosConfigJobAdvanceOutcome::Continue(bios_config_info) => { + Ok(HostBootConfigOutcome::Continue( + HostBootConfigStage::WaitingForBiosJob { bios_config_info }, + )) + } + BiosConfigJobAdvanceOutcome::Done => Ok(HostBootConfigOutcome::Continue( + HostBootConfigStage::PollingBiosSetup { retry_count }, + )), + BiosConfigJobAdvanceOutcome::Failed { failure } => { + Ok(HostBootConfigOutcome::Failed { failure }) + } + BiosConfigJobAdvanceOutcome::Wait(reason) => { + Ok(HostBootConfigOutcome::Wait(reason)) + } + BiosConfigJobAdvanceOutcome::RetryPlatformConfiguration { retry_count } => { + Ok(HostBootConfigOutcome::Continue( + HostBootConfigStage::ConfigureBios { retry_count }, + )) + } + } + } + HostBootConfigStage::PollingBiosSetup { retry_count } => { + let predictions = load_boot_predictions(ctx, &mh_snapshot.host_snapshot.id).await?; + match advance_polling_bios_setup( + redfish_client, + mh_snapshot, + retry_count, + &ctx.services.site_config.machine_state_controller, + &predictions, + ) + .await? + { + PollingBiosSetupOutcome::Verified => { + if should_skip_boot_order_remediation(mh_snapshot) { + Ok(HostBootConfigOutcome::Complete) + } else { + Ok(HostBootConfigOutcome::Continue( + HostBootConfigStage::SetBootOrder { + set_boot_order_info: set_boot_order_info(retry_count), + }, + )) + } + } + PollingBiosSetupOutcome::Wait(reason) => Ok(HostBootConfigOutcome::Wait(reason)), + PollingBiosSetupOutcome::EnterRecovery(bios_config_info) => { + Ok(HostBootConfigOutcome::Continue( + HostBootConfigStage::WaitingForBiosJob { bios_config_info }, + )) + } + PollingBiosSetupOutcome::Failed { failure } => { + Ok(HostBootConfigOutcome::Failed { failure }) + } + } + } + HostBootConfigStage::SetBootOrder { + set_boot_order_info, + } => { + let retry_count = set_boot_order_info.retry_count; + match set_host_boot_order( + ctx, + reachability_params, + redfish_client, + mh_snapshot, + set_boot_order_info, + ) + .await? + { + SetBootOrderOutcome::Continue(set_boot_order_info) => Ok( + HostBootConfigOutcome::Continue(HostBootConfigStage::SetBootOrder { + set_boot_order_info, + }), + ), + SetBootOrderOutcome::ConfigureBios => { + let max_retries = ctx + .services + .site_config + .machine_state_controller + .max_bios_config_retries; + let Some(retry_count) = next_boot_config_retry_count(retry_count, max_retries) + else { + return Ok(HostBootConfigOutcome::Failed { + failure: format!( + "BIOS settings repeatedly drifted during boot-order repair; automated boot-config convergence exhausted after {max_retries} retries" + ), + }); + }; + + Ok(HostBootConfigOutcome::Continue( + HostBootConfigStage::ConfigureBios { retry_count }, + )) + } + SetBootOrderOutcome::Done => Ok(HostBootConfigOutcome::Complete), + SetBootOrderOutcome::WaitingForReboot(reason) + | SetBootOrderOutcome::Wait(reason) => Ok(HostBootConfigOutcome::Wait(reason)), + } + } + } +} + +pub(super) fn initial_set_boot_order_info() -> SetBootOrderInfo { + set_boot_order_info(0) +} + +fn set_boot_order_info(retry_count: u32) -> SetBootOrderInfo { + SetBootOrderInfo { + set_boot_order_jid: None, + set_boot_order_state: SetBootOrderState::SetBootOrder, + retry_count, + } +} + +/// Carry one bounded convergence budget when boot-order work discovers that +/// BIOS must be repaired again. Existing owner states already persist this +/// counter through both phases, so no serialized wrapper is required. +fn next_boot_config_retry_count(retry_count: u32, max_retries: u32) -> Option { + (retry_count < max_retries).then(|| retry_count + 1) +} + +/// Viking BMC firmware cannot safely run boot-order remediation; BIOS repair +/// still applies. +pub(super) fn should_skip_boot_order_remediation(mh_snapshot: &ManagedHostStateSnapshot) -> bool { + mh_snapshot + .host_snapshot + .hardware_info + .as_ref() + .is_some_and(|hw| hw.is_dgx_h100()) +} + +async fn should_wait_for_dpus_before_host_boot_config( + mh_snapshot: &ManagedHostStateSnapshot, + reachability_params: &ReachabilityParams, + dpu_freshness: HostBootConfigDpuFreshness, + ctx: &mut StateHandlerContext<'_, MachineStateHandlerContextObjects>, +) -> bool { + if !mh_snapshot.has_managed_dpus() { + return false; + } + + match dpu_freshness { + HostBootConfigDpuFreshness::AlreadyValidated => false, + HostBootConfigDpuFreshness::CurrentHostState => { + !are_dpus_up_trigger_reboot_if_needed(mh_snapshot, reachability_params, ctx).await + } + HostBootConfigDpuFreshness::SinceLastHostRebootRequest => { + let Some(last_reboot_requested) = mh_snapshot.host_snapshot.last_reboot_requested + else { + tracing::warn!( + machine_id = %mh_snapshot.host_snapshot.id, + "No host reboot request timestamp found before post-reboot host boot config check" + ); + return false; + }; + + for dpu_snapshot in &mh_snapshot.dpu_snapshots { + if !is_dpu_observed_since(dpu_snapshot, last_reboot_requested.time) { + match trigger_reboot_if_needed( + dpu_snapshot, + mh_snapshot, + None, + reachability_params, + ctx, + ) + .await + { + Ok(_) => {} + Err(e) => tracing::warn!( + dpu_id = %dpu_snapshot.id, + error = %e, + "Could not reboot DPU while waiting to check host boot config" + ), + } + return true; + } + } + + false + } + } +} + +#[cfg(test)] +mod tests { + use carbide_test_support::value_scenarios; + + use super::*; + + #[test] + fn boot_config_decision_chooses_the_smallest_remediation() { + value_scenarios!(decide_host_boot_config: + "BIOS drift takes precedence" { + HostBootConfigInspection { + is_bios_setup: false, + is_boot_order_setup: None, + } => HostBootConfigDecision::ConfigureBios, + HostBootConfigInspection { + is_bios_setup: false, + is_boot_order_setup: Some(false), + } => HostBootConfigDecision::ConfigureBios, + HostBootConfigInspection { + is_bios_setup: false, + is_boot_order_setup: Some(true), + } => HostBootConfigDecision::ConfigureBios, + } + + "configured BIOS chooses the narrowest remaining action" { + HostBootConfigInspection { + is_bios_setup: true, + is_boot_order_setup: Some(false), + } => HostBootConfigDecision::SetBootOrder, + HostBootConfigInspection { + is_bios_setup: true, + is_boot_order_setup: Some(true), + } => HostBootConfigDecision::Complete, + HostBootConfigInspection { + is_bios_setup: true, + is_boot_order_setup: None, + } => HostBootConfigDecision::Complete, + } + ); + } + + #[test] + fn boot_config_retry_count_is_bounded_across_bios_and_order() { + struct RetryCount { + retry_count: u32, + max_retries: u32, + } + + value_scenarios!( + run = |RetryCount { + retry_count, + max_retries, + }: RetryCount| next_boot_config_retry_count(retry_count, max_retries); + "retry budget remains" { + RetryCount { + retry_count: 0, + max_retries: 1, + } => Some(1), + RetryCount { + retry_count: 4, + max_retries: 5, + } => Some(5), + } + + "retry budget is unavailable" { + RetryCount { + retry_count: 1, + max_retries: 1, + } => None, + RetryCount { + retry_count: 0, + max_retries: 0, + } => None, + RetryCount { + retry_count: 4, + max_retries: 1, + } => None, + } + ); + } +} diff --git a/crates/machine-controller/src/handler/machine_validation.rs b/crates/machine-controller/src/handler/machine_validation.rs index 7950f844cd..d29cfab079 100644 --- a/crates/machine-controller/src/handler/machine_validation.rs +++ b/crates/machine-controller/src/handler/machine_validation.rs @@ -15,10 +15,11 @@ * limitations under the License. */ use carbide_uuid::machine_validation::MachineValidationId; +use chrono::Utc; use libredfish::{EnabledDisabled, RedfishError, SystemPowerControl}; use model::machine::{ - FailureCause, MachineState, MachineValidatingState, ManagedHostState, ManagedHostStateSnapshot, - SetBootOrderInfo, SetBootOrderState, UnlockHostState, ValidationState, + FailureCause, FailureDetails, FailureSource, MachineState, MachineValidatingState, + ManagedHostState, ManagedHostStateSnapshot, StateMachineArea, UnlockHostState, ValidationState, }; use model::machine_validation::{MachineValidationState, MachineValidationStatus}; use state_controller::state_handler::{ @@ -27,10 +28,14 @@ use state_controller::state_handler::{ use super::{HostHandlerParams, is_machine_validation_requested, machine_validation_completed}; use crate::context::{MachineStateHandlerContextObjects, MachineStateHandlerServices}; +use crate::handler::host_boot_config::{ + HostBootConfigCheckOutcome, HostBootConfigDecision, HostBootConfigDpuFreshness, + HostBootConfigOutcome, HostBootConfigStage, check_host_boot_config, decide_host_boot_config, + initial_set_boot_order_info, inspect_host_boot_config, run_host_boot_config_stage, +}; use crate::handler::{ - RequiredBootInterface, SetBootOrderOutcome, handler_host_power_control, host_power_control, - load_boot_predictions, rebooted, redfish_error, require_boot_interface, set_host_boot_order, - trigger_reboot_if_needed, wait, + RequiredBootInterface, handler_host_power_control, host_power_control, load_boot_predictions, + rebooted, redfish_error, require_boot_interface, trigger_reboot_if_needed, wait, }; /// The validation flavor of the managed-host state, so the repair arms can @@ -41,14 +46,103 @@ fn validating(machine_validation: MachineValidatingState) -> ManagedHostState { } } -/// The starting point for the boot-order flow when validation boot repair -/// re-drives it: the flow itself checks what is already in place and only -/// re-applies what the reboot reverted. -fn boot_repair_start() -> SetBootOrderInfo { - SetBootOrderInfo { - set_boot_order_jid: None, - set_boot_order_state: SetBootOrderState::SetBootOrder, - retry_count: 0, +/// Handles a shared boot-configuration stage for machine validation. +/// +/// Called by `handle_machine_validation_state` for its BIOS configuration, +/// vendor-job, polling, and boot-order repair substates. This adapter preserves +/// the validation ID, maps continued stages back into `MachineValidatingState`, +/// continues to `LockAfterBootRepair` on completion, and closes the validation +/// run before reporting a terminal boot-configuration failure. +async fn handle_validation_boot_config_stage( + ctx: &mut StateHandlerContext<'_, MachineStateHandlerContextObjects>, + host_handler_params: &HostHandlerParams, + mh_snapshot: &ManagedHostStateSnapshot, + validation_id: MachineValidationId, + stage: HostBootConfigStage, +) -> Result, StateHandlerError> { + let redfish_client = ctx + .services + .create_redfish_client_from_machine(&mh_snapshot.host_snapshot) + .await?; + + match run_host_boot_config_stage( + ctx, + &host_handler_params.reachability_params, + redfish_client.as_ref(), + mh_snapshot, + stage, + ) + .await? + { + HostBootConfigOutcome::Continue(stage) => { + let machine_validation = match stage { + HostBootConfigStage::ConfigureBios { retry_count } => { + MachineValidatingState::ConfigureBootBios { + validation_id, + retry_count, + } + } + HostBootConfigStage::WaitingForBiosJob { bios_config_info } => { + MachineValidatingState::WaitingForBootBiosJob { + validation_id, + bios_config_info, + } + } + HostBootConfigStage::PollingBiosSetup { retry_count } => { + MachineValidatingState::PollingBootBiosSetup { + validation_id, + retry_count, + } + } + HostBootConfigStage::SetBootOrder { + set_boot_order_info, + } => MachineValidatingState::RepairBootConfig { + validation_id, + set_boot_order_info, + }, + }; + + Ok(StateHandlerOutcome::transition(validating( + machine_validation, + ))) + } + HostBootConfigOutcome::Complete => Ok(StateHandlerOutcome::transition(validating( + MachineValidatingState::LockAfterBootRepair { validation_id }, + ))), + HostBootConfigOutcome::Wait(reason) => Ok(StateHandlerOutcome::wait(reason)), + HostBootConfigOutcome::Failed { failure } => { + let machine_id = mh_snapshot.host_snapshot.id; + let mut txn = ctx.services.db_pool.begin().await?; + let completed = db::machine_validation::mark_machine_validation_complete( + txn.as_mut(), + &machine_id, + &validation_id, + MachineValidationStatus { + state: MachineValidationState::Failed, + ..MachineValidationStatus::default() + }, + ) + .await?; + + if !completed { + tracing::info!( + %machine_id, + %validation_id, + "Machine validation boot-config failure observed after the run was already terminal" + ); + } + + Ok(StateHandlerOutcome::transition(ManagedHostState::Failed { + details: FailureDetails { + cause: FailureCause::BiosSetupFailed { err: failure }, + failed_at: Utc::now(), + source: FailureSource::StateMachineArea(StateMachineArea::MainFlow), + }, + machine_id, + retry_count: 0, + }) + .with_txn(txn)) + } } } @@ -162,11 +256,20 @@ pub(crate) async fn handle_machine_validation_state( .services .create_redfish_client_from_machine(&mh_snapshot.host_snapshot) .await?; - match boot_interface - .run(|bi| redfish_client.is_bios_setup(Some(bi))) - .await + match inspect_host_boot_config( + redfish_client.as_ref(), + mh_snapshot, + &boot_interface, + ) + .await { - Ok(false) => { + Ok(inspection) + if matches!( + decide_host_boot_config(inspection), + HostBootConfigDecision::ConfigureBios + | HostBootConfigDecision::SetBootOrder + ) => + { tracing::warn!( machine_id = %mh_snapshot.host_snapshot.id, "Boot config reads reverted while waiting for the validation reboot; correcting it via boot repair", @@ -175,7 +278,7 @@ pub(crate) async fn handle_machine_validation_state( MachineValidatingState::PrepareBootRepair { validation_id: *id }, ))); } - Ok(true) => {} + Ok(_) => {} Err(e) => { tracing::warn!( machine_id = %mh_snapshot.host_snapshot.id, @@ -252,9 +355,8 @@ pub(crate) async fn handle_machine_validation_state( machine_id = %mh_snapshot.host_snapshot.id, "BMC vendor does not support checking lockdown status during validation boot repair", ); - MachineValidatingState::RepairBootConfig { + MachineValidatingState::CheckBootConfigForRepair { validation_id: *validation_id, - set_boot_order_info: boot_repair_start(), } } Err(e) => { @@ -277,9 +379,8 @@ pub(crate) async fn handle_machine_validation_state( unlock_host_state: UnlockHostState::DisableLockdown, } } - Ok(_) => MachineValidatingState::RepairBootConfig { + Ok(_) => MachineValidatingState::CheckBootConfigForRepair { validation_id: *validation_id, - set_boot_order_info: boot_repair_start(), }, }; @@ -322,9 +423,8 @@ pub(crate) async fn handle_machine_validation_state( unlock_host_state: UnlockHostState::RebootHost, } } else { - MachineValidatingState::RepairBootConfig { + MachineValidatingState::CheckBootConfigForRepair { validation_id: *validation_id, - set_boot_order_info: boot_repair_start(), } } } @@ -359,51 +459,112 @@ pub(crate) async fn handle_machine_validation_state( mh_snapshot.host_snapshot.id ))); } - MachineValidatingState::RepairBootConfig { + MachineValidatingState::CheckBootConfigForRepair { validation_id: *validation_id, - set_boot_order_info: boot_repair_start(), } } }; Ok(StateHandlerOutcome::transition(validating(next))) } - MachineValidatingState::RepairBootConfig { - validation_id, - set_boot_order_info, - } => { - // Re-drive the boot-order flow: it checks what is already in - // place, re-asserts the reverted HTTP-boot device (rebooting to - // apply it), sets the boot order, and verifies -- the same engine - // the boot-configuration stages use. + MachineValidatingState::CheckBootConfigForRepair { validation_id } => { let redfish_client = ctx .services .create_redfish_client_from_machine(&mh_snapshot.host_snapshot) .await?; - match set_host_boot_order( - ctx, - &host_handler_params.reachability_params, + let next = match check_host_boot_config( redfish_client.as_ref(), mh_snapshot, - set_boot_order_info.clone(), + &host_handler_params.reachability_params, + HostBootConfigDpuFreshness::CurrentHostState, + ctx, ) .await? { - SetBootOrderOutcome::Continue(info) => Ok(StateHandlerOutcome::transition( - validating(MachineValidatingState::RepairBootConfig { + HostBootConfigCheckOutcome::Wait(reason) => { + return Ok(StateHandlerOutcome::wait(reason)); + } + HostBootConfigCheckOutcome::Ready(HostBootConfigDecision::ConfigureBios) => { + MachineValidatingState::ConfigureBootBios { validation_id: *validation_id, - set_boot_order_info: info, - }), - )), - SetBootOrderOutcome::Done => Ok(StateHandlerOutcome::transition(validating( + retry_count: 0, + } + } + HostBootConfigCheckOutcome::Ready(HostBootConfigDecision::SetBootOrder) => { + MachineValidatingState::RepairBootConfig { + validation_id: *validation_id, + set_boot_order_info: initial_set_boot_order_info(), + } + } + HostBootConfigCheckOutcome::Ready(HostBootConfigDecision::Complete) => { MachineValidatingState::LockAfterBootRepair { validation_id: *validation_id, - }, - ))), - SetBootOrderOutcome::WaitingForReboot(reason) - | SetBootOrderOutcome::Wait(reason) => Ok(StateHandlerOutcome::wait(reason)), - } + } + } + }; + + Ok(StateHandlerOutcome::transition(validating(next))) + } + MachineValidatingState::ConfigureBootBios { + validation_id, + retry_count, + } => { + handle_validation_boot_config_stage( + ctx, + host_handler_params, + mh_snapshot, + *validation_id, + HostBootConfigStage::ConfigureBios { + retry_count: *retry_count, + }, + ) + .await + } + MachineValidatingState::WaitingForBootBiosJob { + validation_id, + bios_config_info, + } => { + handle_validation_boot_config_stage( + ctx, + host_handler_params, + mh_snapshot, + *validation_id, + HostBootConfigStage::WaitingForBiosJob { + bios_config_info: bios_config_info.clone(), + }, + ) + .await + } + MachineValidatingState::PollingBootBiosSetup { + validation_id, + retry_count, + } => { + handle_validation_boot_config_stage( + ctx, + host_handler_params, + mh_snapshot, + *validation_id, + HostBootConfigStage::PollingBiosSetup { + retry_count: *retry_count, + }, + ) + .await + } + MachineValidatingState::RepairBootConfig { + validation_id, + set_boot_order_info, + } => { + handle_validation_boot_config_stage( + ctx, + host_handler_params, + mh_snapshot, + *validation_id, + HostBootConfigStage::SetBootOrder { + set_boot_order_info: set_boot_order_info.clone(), + }, + ) + .await } MachineValidatingState::LockAfterBootRepair { validation_id } => { // Restore the lockdown that boot repair temporarily opened, then diff --git a/crates/machine-controller/src/io.rs b/crates/machine-controller/src/io.rs index e831a4d975..a80411c47b 100644 --- a/crates/machine-controller/src/io.rs +++ b/crates/machine-controller/src/io.rs @@ -266,6 +266,12 @@ impl StateControllerIO for MachineStateControllerIO { MachineValidatingState::RebootHost { .. } => "reboothost", MachineValidatingState::PrepareBootRepair { .. } => "preparebootrepair", MachineValidatingState::UnlockForBootRepair { .. } => "unlockforbootrepair", + MachineValidatingState::CheckBootConfigForRepair { .. } => { + "checkbootconfigforrepair" + } + MachineValidatingState::ConfigureBootBios { .. } => "configurebootbios", + MachineValidatingState::WaitingForBootBiosJob { .. } => "waitingforbootbiosjob", + MachineValidatingState::PollingBootBiosSetup { .. } => "pollingbootbiossetup", MachineValidatingState::RepairBootConfig { .. } => "repairbootconfig", MachineValidatingState::LockAfterBootRepair { .. } => "lockafterbootrepair", }