diff --git a/Cargo.lock b/Cargo.lock index 9761f8d4fc..c920eaab58 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2310,6 +2310,8 @@ dependencies = [ "carbide-redfish", "carbide-secrets", "carbide-state-controller-common", + "carbide-test-harness", + "carbide-test-support", "carbide-utils", "carbide-uuid", "chrono", diff --git a/crates/api-core/src/cfg/README.md b/crates/api-core/src/cfg/README.md index 7cf3b41736..06bec6264d 100644 --- a/crates/api-core/src/cfg/README.md +++ b/crates/api-core/src/cfg/README.md @@ -329,7 +329,7 @@ Extends `StateControllerConfig` with: | `dpu_up_threshold` | `Duration` | `5m` | Max time without DPU health report before assuming it's down. | | `scout_reporting_timeout` | `Duration` | `5m` | Duration without scout report before host is unhealthy. | | `uefi_boot_wait` | `Duration` | `5m` | Wait time for UEFI boot completion after host reboot. | -| `max_bios_config_retries` | `u32` | `3` | Max HandleBiosJobFailure recovery cycles during BIOS configuration. | +| `max_bios_config_retries` | `u32` | `3` | Shared retry budget for automated host boot-configuration convergence across BIOS recovery and boot-order verification. | | `polling_bios_setup_stuck_threshold` | `Duration` | `15m` | Time in PollingBiosSetup with `is_bios_setup == false` before recovery escalation. | ### `NetworkSegmentStateControllerConfig` diff --git a/crates/api-core/src/tests/common/api_fixtures/mod.rs b/crates/api-core/src/tests/common/api_fixtures/mod.rs index 0c1e5251f0..a034f2a441 100644 --- a/crates/api-core/src/tests/common/api_fixtures/mod.rs +++ b/crates/api-core/src/tests/common/api_fixtures/mod.rs @@ -469,6 +469,14 @@ impl TestEnv { } } MachineValidatingState::RebootHost { .. } => state.clone(), + MachineValidatingState::PrepareBootRepair { .. } + | MachineValidatingState::UnlockForBootRepair { .. } + | MachineValidatingState::CheckBootConfigForRepair { .. } + | MachineValidatingState::ConfigureBootBios { .. } + | MachineValidatingState::WaitingForBootBiosJob { .. } + | MachineValidatingState::PollingBootBiosSetup { .. } + | MachineValidatingState::RepairBootConfig { .. } + | MachineValidatingState::LockAfterBootRepair { .. } => state.clone(), } } }, diff --git a/crates/api-core/src/tests/dpu_reprovisioning.rs b/crates/api-core/src/tests/dpu_reprovisioning.rs index 20c6a1f82a..6c3422ff50 100644 --- a/crates/api-core/src/tests/dpu_reprovisioning.rs +++ b/crates/api-core/src/tests/dpu_reprovisioning.rs @@ -28,8 +28,8 @@ use libredfish::{EnabledDisabled, SystemPowerControl}; use model::instance::status::tenant::TenantState; use model::machine::{ DpuInitState, FailureCause, FailureDetails, FailureSource, InstallDpuOsState, InstanceState, - Machine, MachineLastRebootRequestedMode, MachineState, ManagedHostState, ReprovisionState, - SetBootOrderInfo, SetBootOrderState, StateMachineArea, UnlockHostState, + Machine, MachineLastRebootRequestedMode, MachineState, ManagedHostState, PowerState, + ReprovisionState, SetBootOrderInfo, SetBootOrderState, StateMachineArea, UnlockHostState, }; use model::test_support::HardwareInfoTemplate; use rpc::forge::MachineArchitecture; @@ -86,8 +86,6 @@ fn reprovision_host_boot_repair_states( unlock_host_state: UnlockHostState::DisableLockdown, }, ReprovisionState::CheckHostBootConfig, - ReprovisionState::ConfigureHostBoot { retry_count: 0 }, - ReprovisionState::PollingHostBiosSetup { retry_count: 0 }, reprovision_set_host_boot_order_state(SetBootOrderState::SetBootOrder), reprovision_set_host_boot_order_state(SetBootOrderState::WaitForSetBootOrderJobScheduled), reprovision_set_host_boot_order_state(SetBootOrderState::RebootHost), @@ -137,6 +135,7 @@ async fn assert_dpu_reprovision_host_boot_repair( expected_states: Vec, ) -> Machine { env.redfish_sim.set_lockdown(EnabledDisabled::Enabled); + env.redfish_sim.set_is_bios_setup(true); env.redfish_sim.set_is_boot_order_setup(false); let redfish_timepoint = env.redfish_sim.timepoint(); @@ -179,27 +178,56 @@ async fn assert_dpu_reprovision_host_boot_repair( } } - // machine_setup enables the bootable DPU interface before boot-order promotion. let actions = env .redfish_sim .actions_since(&redfish_timepoint) .all_hosts(); - let machine_setup_pos = actions - .iter() - .position(|action| matches!(action, RedfishSimAction::MachineSetup { .. })) - .expect("expected DPU reprovision boot repair to call machine_setup"); - let set_boot_order_pos = actions + let (set_boot_order_pos, set_boot_order_mac) = actions .iter() - .position(|action| matches!(action, RedfishSimAction::SetBootOrderDpuFirst { .. })) + .enumerate() + .find_map(|(position, action)| match action { + RedfishSimAction::SetBootOrderDpuFirst { boot_interface_mac } => { + Some((position, boot_interface_mac)) + } + _ => None, + }) .expect("expected DPU reprovision boot repair to set DPU-first boot order"); - let check_boot_order_pos = actions + let boot_order_checks = actions .iter() - .rposition(|action| matches!(action, RedfishSimAction::IsBootOrderSetup { .. })) - .expect("expected DPU reprovision boot repair to verify boot order"); + .enumerate() + .filter_map(|(position, action)| match action { + RedfishSimAction::IsBootOrderSetup { boot_interface_mac } => { + Some((position, boot_interface_mac)) + } + _ => None, + }) + .collect::>(); assert!( - machine_setup_pos < set_boot_order_pos && set_boot_order_pos < check_boot_order_pos, - "expected machine_setup before set_boot_order_dpu_first before is_boot_order_setup; got: {actions:?}" + actions + .iter() + .all(|action| !matches!(action, RedfishSimAction::MachineSetup { .. })), + "expected order-only DPU reprovision boot repair to preserve the configured HTTP boot device; got: {actions:?}" + ); + assert!( + boot_order_checks + .iter() + .filter(|(position, _)| *position < set_boot_order_pos) + .count() + >= 2, + "expected both CheckHostBootConfig and SetBootOrder to inspect the order before writing it; got: {actions:?}" + ); + assert!( + boot_order_checks + .iter() + .any(|(position, _)| *position > set_boot_order_pos), + "expected boot-order verification after the write; got: {actions:?}" + ); + assert!( + boot_order_checks + .iter() + .all(|(_, boot_interface_mac)| *boot_interface_mac == set_boot_order_mac), + "expected every order check and write to target the same interface; got: {actions:?}" ); let rebooting_machine = machine.next_iteration_machine(env).await; @@ -564,6 +592,136 @@ async fn test_dpu_reprovision_viking_skips_boot_order_when_bios_setup(pool: sqlx ); } +#[crate::sqlx_test] +async fn test_dpu_reprovision_viking_finishes_parked_boot_order_recovery(pool: sqlx::PgPool) { + let env = create_test_env(pool).await; + let mh = create_managed_host_with_hardware_info_template( + &env, + HardwareInfoTemplate::Custom(DGX_H100_INFO_JSON), + ) + .await; + let dpu_machine = prepare_dpu_reprovision_host_boot_check(&env, &mh).await; + + // A controller upgrade can find a Viking in a persisted recovery substate + // created before boot-order remediation was disabled for this platform. + // Finish restoring host power before taking the safe terminal shortcut. + let parked_recovery = mh.new_dpu_reprovision_state(reprovision_set_host_boot_order_state( + SetBootOrderState::HandleJobFailure { + failure: "persisted failed boot-order job".to_string(), + power_state: PowerState::Off, + }, + )); + let mut txn = env.pool.begin().await.unwrap(); + db::machine::update_state(&mut txn, &mh.id, &parked_recovery) + .await + .unwrap(); + txn.commit().await.unwrap(); + + let redfish_timepoint = env.redfish_sim.timepoint(); + let dpu = dpu_machine.next_iteration_machine(&env).await; + assert_eq!(dpu.current_state(), &parked_recovery); + assert_eq!( + env.redfish_sim + .actions_since(&redfish_timepoint) + .all_hosts(), + vec![RedfishSimAction::Power(SystemPowerControl::ForceOff)], + "Viking policy must not abandon an in-flight power recovery" + ); + + let recovering_power_on = mh.new_dpu_reprovision_state(reprovision_set_host_boot_order_state( + SetBootOrderState::HandleJobFailure { + failure: "persisted failed boot-order job".to_string(), + power_state: PowerState::On, + }, + )); + let redfish_timepoint = env.redfish_sim.timepoint(); + let dpu = dpu_machine.next_iteration_machine(&env).await; + assert_eq!(dpu.current_state(), &recovering_power_on); + assert_eq!( + env.redfish_sim + .actions_since(&redfish_timepoint) + .all_hosts(), + vec![RedfishSimAction::BmcReset], + "parked recovery should reset the BMC after the host powers off" + ); + + // ForceOff records last_reboot_requested; backdate it so power_down_wait + // elapses in-process before the controller restores host power. + { + let mut txn = env.db_txn().await; + let host = mh.host().db_machine(&mut txn).await; + update_time_params(&env.pool, &host, 1, None).await; + } + + let redfish_timepoint = env.redfish_sim.timepoint(); + let dpu = dpu_machine.next_iteration_machine(&env).await; + assert_eq!(dpu.current_state(), &recovering_power_on); + assert_eq!( + env.redfish_sim + .actions_since(&redfish_timepoint) + .all_hosts(), + vec![RedfishSimAction::Power(SystemPowerControl::On)], + "parked recovery should restore host power after the BMC reset" + ); + + let safe_terminal = mh.new_dpu_reprovision_state(reprovision_set_host_boot_order_state( + SetBootOrderState::CheckBootOrder, + )); + let redfish_timepoint = env.redfish_sim.timepoint(); + let dpu = dpu_machine.next_iteration_machine(&env).await; + assert_eq!(dpu.current_state(), &safe_terminal); + assert!( + env.redfish_sim + .actions_since(&redfish_timepoint) + .all_hosts() + .is_empty(), + "completed recovery should naturally reach CheckBootOrder" + ); + + // CheckBootOrder has no unfinished operation behind it, but its preceding + // reboot may have reverted the HTTP boot device. Repair that BIOS drift + // without touching the unsupported Viking boot-order APIs, and spend one + // attempt from the shared convergence budget. + env.redfish_sim.set_is_bios_setup(false); + + let redfish_timepoint = env.redfish_sim.timepoint(); + let dpu = dpu_machine.next_iteration_machine(&env).await; + assert_eq!( + dpu.current_state(), + &mh.new_dpu_reprovision_state(ReprovisionState::ConfigureHostBoot { retry_count: 1 }) + ); + assert!( + env.redfish_sim + .actions_since(&redfish_timepoint) + .all_hosts() + .is_empty(), + "Viking BIOS recovery must not read or write boot order" + ); + + // Once BIOS is intact, the same safe terminal skips boot-order + // verification and completes the repair. + env.redfish_sim.set_is_bios_setup(true); + let mut txn = env.pool.begin().await.unwrap(); + db::machine::update_state(&mut txn, &mh.id, &safe_terminal) + .await + .unwrap(); + txn.commit().await.unwrap(); + + let redfish_timepoint = env.redfish_sim.timepoint(); + let dpu = dpu_machine.next_iteration_machine(&env).await; + assert_eq!( + dpu.current_state(), + &mh.new_dpu_reprovision_state(ReprovisionState::LockHostAfterBootRepair) + ); + assert!( + env.redfish_sim + .actions_since(&redfish_timepoint) + .all_hosts() + .is_empty(), + "safe Viking completion should not touch Redfish boot order" + ); +} + #[crate::sqlx_test] async fn test_dpu_for_reprovisioning_fail_if_maintenance_not_set(pool: sqlx::PgPool) { let env = create_test_env(pool).await; diff --git a/crates/api-core/src/tests/machine_history.rs b/crates/api-core/src/tests/machine_history.rs index 28f57bb283..16660dfaff 100644 --- a/crates/api-core/src/tests/machine_history.rs +++ b/crates/api-core/src/tests/machine_history.rs @@ -52,9 +52,6 @@ async fn test_machine_state_history(pool: sqlx::PgPool) -> Result<(), Box TestEnv { - let env = create_test_env_with_overrides( - pool, - TestEnvOverrides { - site_prefixes: Some(vec![ - IpNetwork::new( - FIXTURE_ADMIN_NETWORK_SEGMENT_GATEWAY.network(), - FIXTURE_ADMIN_NETWORK_SEGMENT_GATEWAY.prefix(), - ) - .unwrap(), - IpNetwork::new( - FIXTURE_HOST_INBAND_NETWORK_SEGMENT_GATEWAY.network(), - FIXTURE_HOST_INBAND_NETWORK_SEGMENT_GATEWAY.prefix(), - ) - .unwrap(), - ]), - ..Default::default() - }, - ) - .await; + create_zero_dpu_test_env_with_overrides(pool, TestEnvOverrides::default()).await +} + +async fn create_zero_dpu_test_env_with_overrides( + pool: sqlx::PgPool, + mut overrides: TestEnvOverrides, +) -> TestEnv { + overrides.site_prefixes = Some(vec![ + IpNetwork::new( + FIXTURE_ADMIN_NETWORK_SEGMENT_GATEWAY.network(), + FIXTURE_ADMIN_NETWORK_SEGMENT_GATEWAY.prefix(), + ) + .unwrap(), + IpNetwork::new( + FIXTURE_HOST_INBAND_NETWORK_SEGMENT_GATEWAY.network(), + FIXTURE_HOST_INBAND_NETWORK_SEGMENT_GATEWAY.prefix(), + ) + .unwrap(), + ]); + + let env = create_test_env_with_overrides(pool, overrides).await; create_host_inband_network_segment(&env.api, None).await; env } @@ -2850,9 +2924,9 @@ async fn set_host_stuck_in_set_boot_order(env: &TestEnv, host_id: MachineId) { /// Drives the machine state controller until the host leaves HostInit/SetBootOrder /// for the next phase (HostInit/Measuring), returning whether it got there. A -/// zero-DPU host whose `HttpDev1` device has de-enumerated never reaches it -/// unless SetBootOrder re-asserts `machine_setup`, so callers assert on the -/// result to distinguish recovery from a host wedged at CheckBootOrder. +/// zero-DPU host whose `HttpDev1` device dropped off the BMC's Redfish inventory +/// never reaches it unless SetBootOrder re-asserts `machine_setup`, so callers +/// assert on the result to distinguish recovery from a host wedged at CheckBootOrder. async fn drive_until_past_set_boot_order( env: &TestEnv, mh: &TestManagedHost, @@ -2941,23 +3015,15 @@ fn assert_machine_setup_precedes_reorder_for( ); } -/// Regression test for the NIC-mode `HttpDev1` de-enumeration hang. -/// -/// On a zero-DPU host, a reboot during the boot-order phase can de-enumerate the -/// BlueField from Redfish, reverting the `HttpDev1` UEFI HTTP-boot device to the -/// onboard default. `set_boot_order_dpu_first` only reorders the boot options it -/// finds, so it cannot bring `HttpDev1` back -- the boot order never verifies and -/// the host wedges in SetBootOrder/CheckBootOrder. SetBootOrder now re-asserts -/// `machine_setup` on each attempt, re-enabling the device so the reorder sticks. -/// -/// The sim models this: `set_http_dev1_reverted` disables the device, so the -/// first `set_boot_order_dpu_first` records the boot order as *not* configured; -/// only the re-assert's `machine_setup` re-enables it. Without the re-assert the -/// host never leaves CheckBootOrder; with it, the host recovers to Measuring. +/// The core of this enhancement: when the boot config is already in place, +/// SetBootOrder does not re-apply it. `is_bios_setup` and `is_boot_order_setup` +/// both pass, so it skips the `machine_setup` re-assert AND the boot-order set -- +/// no redundant BIOS write, no reboot to apply one -- and goes straight to +/// verification. This is the case that wedged a real host: re-applying an +/// already-correct config committed a BIOS job that then collided with the +/// boot-order set on Dell (`SYS011`), and the flow looped there. #[crate::sqlx_test] -async fn test_set_boot_order_reasserts_machine_setup_when_http_dev1_de_enumerates( - pool: sqlx::PgPool, -) { +async fn test_set_boot_order_skips_reapply_when_config_already_in_place(pool: sqlx::PgPool) { let env = create_zero_dpu_test_env(pool).await; let mh = create_managed_host_with_config(&env, ManagedHostConfig::zero_dpu()).await; @@ -2966,67 +3032,853 @@ async fn test_set_boot_order_reasserts_machine_setup_when_http_dev1_de_enumerate "zero-DPU fixture should produce no DPU machines" ); let host_id = mh.host().id; - let boot_nic_mac = host_inband_nic_mac(&env, host_id).await; - - // Model the BlueField de-enumerating: HttpDev1 reverts to the onboard - // default, so the first reorder won't make the boot order verify. - env.redfish_sim.set_http_dev1_reverted(); + // Default sim: the HTTP-boot device and the boot order both read configured. set_host_stuck_in_set_boot_order(&env, host_id).await; - let redfish_timepoint = env.redfish_sim.timepoint(); let recovered = drive_until_past_set_boot_order(&env, &mh, 15).await; - if !recovered { - let mut txn = env.db_txn().await; - let host = mh.host().db_machine(&mut txn).await; - panic!( - "expected SetBootOrder to re-assert machine_setup and recover a de-enumerated \ - HttpDev1 (reaching HostInit/Measuring), but the host wedged at: {:?}", - host.current_state() - ); - } + assert!( + recovered, + "an already-configured host should advance past SetBootOrder" + ); - // The recovery hinges on machine_setup running -- targeting the boot NIC -- - // before the reorder during the boot-order phase. + // Nothing was re-applied: no re-assert, no reorder. let actions = env .redfish_sim .actions_since(&redfish_timepoint) .all_hosts(); - assert_machine_setup_precedes_reorder_for(&actions, boot_nic_mac); + assert!( + !actions + .iter() + .any(|a| matches!(a, RedfishSimAction::MachineSetup { .. })), + "an already-configured host should not re-assert machine_setup at SetBootOrder, got: {actions:?}" + ); + assert!( + !actions + .iter() + .any(|a| matches!(a, RedfishSimAction::SetBootOrderDpuFirst { .. })), + "an already-configured host should not re-set the boot order at SetBootOrder, got: {actions:?}" + ); } -/// Healthy-path counterpart: when `HttpDev1` was never de-enumerated, the -/// SetBootOrder `machine_setup` re-assert is idempotent -- the host still -/// advances out of SetBootOrder exactly as before, and machine_setup is invoked -/// without changing the outcome. Guards against the re-assert regressing the -/// normal zero-DPU boot-order path. +/// When the HTTP-boot device is already configured but the boot order is not yet +/// set -- the first-time SetBootOrder case -- SetBootOrder sets the order without +/// re-asserting `machine_setup`, so there is no pending BIOS job for the +/// boot-order set to collide with. #[crate::sqlx_test] -async fn test_set_boot_order_reassert_is_idempotent_on_healthy_zero_dpu_host(pool: sqlx::PgPool) { +async fn test_set_boot_order_sets_order_without_reasserting_when_device_configured( + pool: sqlx::PgPool, +) { let env = create_zero_dpu_test_env(pool).await; let mh = create_managed_host_with_config(&env, ManagedHostConfig::zero_dpu()).await; let host_id = mh.host().id; let boot_nic_mac = host_inband_nic_mac(&env, host_id).await; - // No de-enumeration this time: HttpDev1 stays enabled throughout. - set_host_stuck_in_set_boot_order(&env, host_id).await; + // Device configured (is_bios_setup true, the sim default), but the boot order + // not yet set. + env.redfish_sim.set_is_boot_order_setup(false); + set_host_stuck_in_set_boot_order(&env, host_id).await; let redfish_timepoint = env.redfish_sim.timepoint(); let recovered = drive_until_past_set_boot_order(&env, &mh, 15).await; assert!( recovered, - "a healthy zero-DPU host should still advance past SetBootOrder with the re-assert in place" + "host should set the boot order and advance past SetBootOrder" ); - // The re-assert still runs (targeting the boot NIC) before the reorder; on - // the healthy path it just doesn't change the outcome. let actions = env .redfish_sim .actions_since(&redfish_timepoint) .all_hosts(); - assert_machine_setup_precedes_reorder_for(&actions, boot_nic_mac); + assert!( + actions.iter().any(|a| matches!( + a, + RedfishSimAction::SetBootOrderDpuFirst { boot_interface_mac } + if *boot_interface_mac == boot_nic_mac.to_string() + )), + "the boot order should be set for the boot NIC, got: {actions:?}" + ); + assert!( + !actions + .iter() + .any(|a| matches!(a, RedfishSimAction::MachineSetup { .. })), + "the device is already configured, so machine_setup should not be re-asserted, got: {actions:?}" + ); +} + +/// The reboot that applies a boot-order job can independently revert a managed +/// BIOS setting. Final verification checks BIOS before order, routes back to +/// the shared job-aware stages, and carries the existing convergence budget. +#[crate::sqlx_test] +async fn test_check_boot_order_repairs_bios_drift_after_order_reboot(pool: sqlx::PgPool) { + let env = create_zero_dpu_test_env(pool).await; + + let mh = create_managed_host_with_config(&env, ManagedHostConfig::zero_dpu()).await; + let host_id = mh.host().id; + + env.redfish_sim.set_is_bios_setup(false); + env.redfish_sim.set_is_boot_order_setup(true); + env.redfish_sim + .set_lockdown(libredfish::EnabledDisabled::Disabled); + + set_host_controller_state_stuck_in( + &env, + host_id, + &ManagedHostState::HostInit { + machine_state: MachineState::SetBootOrder { + set_boot_order_info: Some(SetBootOrderInfo { + set_boot_order_jid: None, + set_boot_order_state: SetBootOrderState::CheckBootOrder, + retry_count: 2, + }), + }, + }, + 1, + ) + .await; + + let checkpoint = env.redfish_sim.timepoint(); + env.run_machine_state_controller_iteration().await; + + let mut txn = env.db_txn().await; + let host = mh.host().db_machine(&mut txn).await; + assert!( + matches!( + host.current_state(), + ManagedHostState::HostInit { + machine_state: MachineState::WaitingForPlatformConfiguration { retry_count: 3 }, + } + ), + "post-order BIOS drift should carry the convergence budget into shared BIOS repair, got: {:?}", + host.current_state() + ); + + let actions = env.redfish_sim.actions_since(&checkpoint).all_hosts(); + assert!( + actions + .iter() + .all(|action| !matches!(action, RedfishSimAction::IsBootOrderSetup { .. })), + "BIOS drift should short-circuit final boot-order verification, got: {actions:?}" + ); +} + +/// Repeated BIOS drift across boot-order reboots consumes the carried budget +/// instead of cycling indefinitely through machine_setup and another reboot. +#[crate::sqlx_test] +async fn test_check_boot_order_bios_drift_exhausts_convergence_budget(pool: sqlx::PgPool) { + let env = create_zero_dpu_test_env(pool).await; + + let mh = create_managed_host_with_config(&env, ManagedHostConfig::zero_dpu()).await; + let host_id = mh.host().id; + + env.redfish_sim.set_is_bios_setup(false); + env.redfish_sim + .set_lockdown(libredfish::EnabledDisabled::Disabled); + set_host_controller_state_stuck_in( + &env, + host_id, + &ManagedHostState::HostInit { + machine_state: MachineState::SetBootOrder { + set_boot_order_info: Some(SetBootOrderInfo { + set_boot_order_jid: None, + set_boot_order_state: SetBootOrderState::CheckBootOrder, + retry_count: 3, + }), + }, + }, + 1, + ) + .await; + + let checkpoint = env.redfish_sim.timepoint(); + env.run_machine_state_controller_iteration().await; + + let mut txn = env.db_txn().await; + let host = mh.host().db_machine(&mut txn).await; + assert!( + matches!( + host.current_state(), + ManagedHostState::Failed { + details: FailureDetails { + cause: FailureCause::BiosSetupFailed { .. }, + source: FailureSource::StateMachineArea(StateMachineArea::HostInit), + .. + }, + .. + } + ), + "expected repeated cross-phase drift to exhaust into Failed, got: {:?}", + host.current_state() + ); + + let actions = env.redfish_sim.actions_since(&checkpoint).all_hosts(); + assert!( + actions + .iter() + .all(|action| !matches!(action, RedfishSimAction::MachineSetup { .. })), + "an exhausted convergence budget must not issue another BIOS write, got: {actions:?}" + ); +} + +/// Boot-order verification uses the same configured convergence budget as +/// BIOS recovery, then remains able to observe an operator's repair while +/// parked at that limit. +#[crate::sqlx_test] +async fn test_check_boot_order_respects_configured_convergence_budget(pool: sqlx::PgPool) { + let mut config = get_config(); + config.machine_state_controller.max_bios_config_retries = 1; + let env = + create_zero_dpu_test_env_with_overrides(pool, TestEnvOverrides::with_config(config)).await; + + let mh = create_managed_host_with_config(&env, ManagedHostConfig::zero_dpu()).await; + let host_id = mh.host().id; + + env.redfish_sim.set_is_bios_setup(true); + env.redfish_sim.set_is_boot_order_setup(false); + env.redfish_sim + .set_lockdown(libredfish::EnabledDisabled::Disabled); + set_host_controller_state_stuck_in( + &env, + host_id, + &ManagedHostState::HostInit { + machine_state: MachineState::SetBootOrder { + set_boot_order_info: Some(SetBootOrderInfo { + set_boot_order_jid: None, + set_boot_order_state: SetBootOrderState::CheckBootOrder, + retry_count: 1, + }), + }, + }, + 31, + ) + .await; + + env.run_machine_state_controller_iteration().await; + + { + let mut txn = env.db_txn().await; + let host = mh.host().db_machine(&mut txn).await; + assert!( + matches!( + host.current_state(), + ManagedHostState::HostInit { + machine_state: MachineState::SetBootOrder { + set_boot_order_info: Some(SetBootOrderInfo { + set_boot_order_state: SetBootOrderState::CheckBootOrder, + retry_count: 1, + .. + }), + }, + } + ), + "an exhausted configured budget must not advance to another boot-order attempt, got: {:?}", + host.current_state() + ); + assert!( + matches!( + host.controller_state_outcome.as_ref(), + Some(PersistentStateHandlerOutcome::Error { err, .. }) + if err.contains("Manual intervention required") + && err.contains("max_retries: 1") + ), + "expected boot-order exhaustion to report the configured limit, got: {:?}", + host.controller_state_outcome + ); + } + + env.redfish_sim.set_is_boot_order_setup(true); + env.run_machine_state_controller_iteration().await; + + let mut txn = env.db_txn().await; + let host = mh.host().db_machine(&mut txn).await; + assert!( + matches!( + host.current_state(), + ManagedHostState::HostInit { + machine_state: MachineState::Measuring { + measuring_state: MeasuringState::WaitingForMeasurements, + }, + } + ), + "a manually repaired boot order should complete from the parked verification state, got: {:?}", + host.current_state() + ); +} + +/// A configured budget above the historical fixed ceiling permits the shared +/// boot-config flow to spend the additional retries. +#[crate::sqlx_test] +async fn test_check_boot_order_allows_configured_budget_above_legacy_limit(pool: sqlx::PgPool) { + let mut config = get_config(); + config.machine_state_controller.max_bios_config_retries = 5; + let env = + create_zero_dpu_test_env_with_overrides(pool, TestEnvOverrides::with_config(config)).await; + + let mh = create_managed_host_with_config(&env, ManagedHostConfig::zero_dpu()).await; + let host_id = mh.host().id; + + env.redfish_sim.set_is_bios_setup(true); + env.redfish_sim.set_is_boot_order_setup(false); + env.redfish_sim + .set_lockdown(libredfish::EnabledDisabled::Disabled); + set_host_controller_state_stuck_in( + &env, + host_id, + &ManagedHostState::HostInit { + machine_state: MachineState::SetBootOrder { + set_boot_order_info: Some(SetBootOrderInfo { + set_boot_order_jid: None, + set_boot_order_state: SetBootOrderState::CheckBootOrder, + retry_count: 3, + }), + }, + }, + 31, + ) + .await; + + env.run_machine_state_controller_iteration().await; + + let mut txn = env.db_txn().await; + let host = mh.host().db_machine(&mut txn).await; + assert!( + matches!( + host.current_state(), + ManagedHostState::HostInit { + machine_state: MachineState::SetBootOrder { + set_boot_order_info: Some(SetBootOrderInfo { + set_boot_order_state: SetBootOrderState::SetBootOrder, + retry_count: 4, + .. + }), + }, + } + ), + "a configured budget above three should permit another boot-order attempt, got: {:?}", + host.current_state() + ); +} + +/// If the HTTP-boot device reverts after inspection but before SetBootOrder, +/// the order actuator routes back through the shared BIOS driver. In +/// particular, a Dell-style job returned by machine_setup is persisted and +/// completed before the order write. +#[crate::sqlx_test] +async fn test_set_boot_order_routes_reverted_bios_through_shared_job_flow(pool: sqlx::PgPool) { + let env = create_zero_dpu_test_env(pool).await; + + let mh = create_managed_host_with_config(&env, ManagedHostConfig::zero_dpu()).await; + let host_id = mh.host().id; + let boot_nic_mac = host_inband_nic_mac(&env, host_id).await; + + // Model the HTTP-boot device reverted: the config verify fails until a + // machine_setup re-assert restores it, and the boot order reads unconfigured + // while the device is gone. + env.redfish_sim.set_is_bios_setup(false); + env.redfish_sim.set_is_boot_order_setup(false); + // SetBootOrder is reached only after HostInit has opened lockdown for BIOS + // writes; model that precondition when planting this synthetic state. + env.redfish_sim + .set_lockdown(libredfish::EnabledDisabled::Disabled); + env.redfish_sim + .set_machine_setup_bios_job_id(Some("JID_LATE_BOOT_DRIFT".to_string())); + env.redfish_sim.set_job_state_sequence(vec![ + libredfish::JobState::Scheduled, + libredfish::JobState::Completed, + ]); + + set_host_stuck_in_set_boot_order(&env, host_id).await; + let redfish_timepoint = env.redfish_sim.timepoint(); + + // First pass only detects the late drift and moves back to the shared BIOS + // stage. It must not issue either BIOS write from the order actuator. + env.run_machine_state_controller_iteration().await; + let first_pass = env + .redfish_sim + .actions_since(&redfish_timepoint) + .all_hosts(); + assert!( + first_pass + .iter() + .all(|a| !matches!(a, RedfishSimAction::MachineSetup { .. })), + "SetBootOrder should delegate late BIOS drift to the shared driver, got: {first_pass:?}" + ); + assert!( + !first_pass + .iter() + .any(|a| matches!(a, RedfishSimAction::SetBootOrderDpuFirst { .. })), + "the boot order must not be set in the same pass as the re-assert (shared BIOS job), got: {first_pass:?}" + ); + + { + let mut txn = env.db_txn().await; + let host = mh.host().db_machine(&mut txn).await; + assert!( + matches!( + host.current_state(), + ManagedHostState::HostInit { + machine_state: MachineState::WaitingForPlatformConfiguration { retry_count: 1 }, + } + ), + "late BIOS drift should return to shared configuration, got: {:?}", + host.current_state() + ); + } + + // The shared driver persists the returned job instead of discarding it and + // rebooting immediately. + env.run_machine_state_controller_iteration().await; + { + let mut txn = env.db_txn().await; + let host = mh.host().db_machine(&mut txn).await; + assert!( + matches!( + host.current_state(), + ManagedHostState::HostInit { + machine_state: MachineState::WaitingForBiosJob { + bios_config_info: BiosConfigInfo { + bios_job_id: Some(job_id), + retry_count: 1, + .. + }, + }, + } if job_id == "JID_LATE_BOOT_DRIFT" + ), + "the late-drift BIOS job should be persisted, got: {:?}", + host.current_state() + ); + } + + // Model the BIOS job applying the restored device. + env.redfish_sim.set_is_bios_setup(true); + + // With the device restored, the host sets the boot order and advances. + let recovered = drive_until_past_set_boot_order(&env, &mh, 15).await; + assert!( + recovered, + "host should recover past SetBootOrder once the HTTP-boot device is restored" + ); + + // Across the recovery, the re-assert targeting the boot NIC preceded the reorder. + let actions = env + .redfish_sim + .actions_since(&redfish_timepoint) + .all_hosts(); + assert_machine_setup_precedes_reorder_for(&actions, boot_nic_mac); +} + +/// Hosts persisted in the legacy HTTP-device wait state migrate to the shared +/// BIOS driver when their wait expires. The existing retry count becomes the +/// shared convergence budget, and an exhausted budget still stops for manual +/// intervention. +#[crate::sqlx_test] +async fn test_legacy_http_boot_wait_migrates_to_shared_bios_repair(pool: sqlx::PgPool) { + let env = create_zero_dpu_test_env(pool).await; + + let mh = create_managed_host_with_config(&env, ManagedHostConfig::zero_dpu()).await; + let host_id = mh.host().id; + + // The device stays reverted throughout: every verify reads false. + env.redfish_sim.set_is_bios_setup(false); + env.redfish_sim.set_is_boot_order_setup(false); + env.redfish_sim + .set_lockdown(libredfish::EnabledDisabled::Disabled); + + let stuck_waiting = |retry_count: u32| ManagedHostState::HostInit { + machine_state: MachineState::SetBootOrder { + set_boot_order_info: Some(SetBootOrderInfo { + set_boot_order_jid: None, + set_boot_order_state: SetBootOrderState::WaitForHttpBootDeviceApplied, + retry_count, + }), + }, + }; + + // Plant the host mid-wait with the window already expired (backdated past + // the 10-minute apply window). Pass 1 migrates its persisted state; pass 2 + // runs machine_setup through the common BIOS driver. + set_host_controller_state_stuck_in(&env, host_id, &stuck_waiting(0), 11).await; + let checkpoint = env.redfish_sim.timepoint(); + env.run_machine_state_controller_iteration().await; + env.run_machine_state_controller_iteration().await; + let actions = env.redfish_sim.actions_since(&checkpoint).all_hosts(); + assert_eq!( + actions + .iter() + .filter(|a| matches!(a, RedfishSimAction::MachineSetup { .. })) + .count(), + 1, + "an expired wait window should produce exactly one fresh re-assert, got: {actions:?}" + ); + { + let mut txn = env.db_txn().await; + let host = mh.host().db_machine(&mut txn).await; + assert!( + matches!( + host.current_state(), + ManagedHostState::HostInit { + machine_state: MachineState::PollingBiosSetup { retry_count: 1 }, + } + ), + "the legacy wait should carry its retry count into shared BIOS polling, got: {:?}", + host.current_state() + ); + } + + // Budget exhausted: the same expired window with the retry budget spent + // must not re-assert again -- the host stays parked for an operator. + set_host_controller_state_stuck_in(&env, host_id, &stuck_waiting(3), 11).await; + let checkpoint = env.redfish_sim.timepoint(); + env.run_machine_state_controller_iteration().await; + let actions = env.redfish_sim.actions_since(&checkpoint).all_hosts(); + assert!( + !actions + .iter() + .any(|a| matches!(a, RedfishSimAction::MachineSetup { .. })), + "an exhausted retry budget must not re-assert the device again, got: {actions:?}" + ); + { + let mut txn = env.db_txn().await; + let host = mh.host().db_machine(&mut txn).await; + assert!( + matches!( + host.current_state(), + ManagedHostState::HostInit { + machine_state: MachineState::SetBootOrder { + set_boot_order_info: Some(SetBootOrderInfo { + set_boot_order_state: SetBootOrderState::WaitForHttpBootDeviceApplied, + .. + }), + }, + } + ), + "the capped host should stay parked awaiting manual intervention, got: {:?}", + host.current_state() + ); + } +} + +/// A host that cannot return from a validation reboot because its boot config +/// reverted (the boot NIC dropped off the BMC's inventory during POST, so the +/// BIOS reset the HTTP-boot device) gets repaired in place: validation unlocks +/// the BMC, re-drives the boot-order flow (re-assert, reboot to apply, +/// reorder, verify), re-locks, and resumes validation from its reboot step -- +/// instead of pacing reboots that can never succeed. +#[crate::sqlx_test] +async fn test_machine_validation_repairs_reverted_boot_config(pool: sqlx::PgPool) { + let env = create_zero_dpu_test_env(pool).await; + + let mh = create_managed_host_with_config(&env, ManagedHostConfig::zero_dpu()).await; + let host_id = mh.host().id; + let boot_nic_mac = host_inband_nic_mac(&env, host_id).await; + + // The validation reboot's POST reverted the boot config, and HostInit + // enabled lockdown before validation reached this point: BIOS writes + // bounce off the BMC until it is unlocked. + env.redfish_sim.set_is_bios_setup(false); + env.redfish_sim.set_is_boot_order_setup(false); + env.redfish_sim + .set_lockdown(libredfish::EnabledDisabled::Enabled); + env.redfish_sim + .set_machine_setup_bios_job_id(Some("JID_VALIDATION_BOOT_REPAIR".to_string())); + env.redfish_sim.set_job_state_sequence(vec![ + libredfish::JobState::Scheduled, + libredfish::JobState::Completed, + ]); + + // Park the host mid-validation, waiting on a reboot that cannot land + // (the state is newer than the host's last reported boot). + let validation_id = MachineValidationId::new(); + set_host_controller_state_stuck_in( + &env, + host_id, + &ManagedHostState::Validation { + validation_state: ValidationState::MachineValidation { + machine_validation: MachineValidatingState::MachineValidating { + context: "Discovery".to_string(), + id: validation_id, + completed: 1, + total: 1, + is_enabled: true, + }, + }, + }, + 0, + ) + .await; + + let checkpoint = env.redfish_sim.timepoint(); + + // Detection through re-assert: the flow notices the reverted config, + // unlocks the BMC, and re-asserts the HTTP-boot device. + for _ in 0..6 { + env.run_machine_state_controller_iteration().await; + if env + .redfish_sim + .actions_since(&checkpoint) + .all_hosts() + .iter() + .any(|a| matches!(a, RedfishSimAction::MachineSetup { .. })) + { + break; + } + } + let actions = env.redfish_sim.actions_since(&checkpoint).all_hosts(); + assert!( + actions + .iter() + .any(|a| matches!(a, RedfishSimAction::MachineSetup { .. })), + "validation boot repair should re-assert the reverted HTTP boot device, got: {actions:?}" + ); + assert!( + env.redfish_sim + .lockdown_states() + .contains(&libredfish::EnabledDisabled::Disabled), + "boot repair should unlock the BMC before writing BIOS settings" + ); + { + let mut txn = env.db_txn().await; + let host = mh.host().db_machine(&mut txn).await; + assert!( + matches!( + host.current_state(), + ManagedHostState::Validation { + validation_state: ValidationState::MachineValidation { + machine_validation: MachineValidatingState::WaitingForBootBiosJob { + validation_id: current_validation_id, + bios_config_info: BiosConfigInfo { + bios_job_id: Some(job_id), + .. + }, + }, + }, + } if *current_validation_id == validation_id + && job_id == "JID_VALIDATION_BOOT_REPAIR" + ), + "validation should preserve the BIOS job before rebooting, got: {:?}", + host.current_state() + ); + } + + // The repair reboot applies the re-asserted device. + env.redfish_sim.set_is_bios_setup(true); + + // The repair completes (reorder + verify), re-locks the BMC, and resumes + // validation from its reboot step. + let mut resumed = false; + for _ in 0..20 { + env.run_machine_state_controller_iteration().await; + let mut txn = env.db_txn().await; + let host = mh.host().db_machine(&mut txn).await; + if matches!( + host.current_state(), + ManagedHostState::Validation { + validation_state: ValidationState::MachineValidation { + machine_validation: MachineValidatingState::RebootHost { + validation_id: current_validation_id, + }, + }, + } if *current_validation_id == validation_id + ) { + resumed = true; + break; + } + } + assert!( + resumed, + "the repaired host should resume validation from its reboot step" + ); + assert!( + env.redfish_sim + .lockdown_states() + .iter() + .all(|l| *l == libredfish::EnabledDisabled::Enabled), + "boot repair should re-lock the BMC once the boot config is repaired" + ); + + // Across the repair, the re-assert targeting the boot NIC preceded the reorder. + let actions = env.redfish_sim.actions_since(&checkpoint).all_hosts(); + assert_machine_setup_precedes_reorder_for(&actions, boot_nic_mac); +} + +/// Boot-order-only drift (the HTTP boot device is still configured, but its +/// priority changed) is detected while validation waits for a reboot. The +/// repair restores only the order, re-locks the BMC, and resumes validation. +#[crate::sqlx_test] +async fn test_machine_validation_repairs_drifted_boot_order(pool: sqlx::PgPool) { + let env = create_zero_dpu_test_env(pool).await; + + let mh = create_managed_host_with_config(&env, ManagedHostConfig::zero_dpu()).await; + let host_id = mh.host().id; + let boot_nic_mac = host_inband_nic_mac(&env, host_id).await; + + // The HTTP boot device is intact; only the order drifted. Set the BIOS + // status explicitly so this regression does not rely on simulator defaults. + env.redfish_sim.set_is_bios_setup(true); + env.redfish_sim.set_is_boot_order_setup(false); + env.redfish_sim + .set_lockdown(libredfish::EnabledDisabled::Enabled); + + let validation_id = MachineValidationId::new(); + set_host_controller_state_stuck_in( + &env, + host_id, + &ManagedHostState::Validation { + validation_state: ValidationState::MachineValidation { + machine_validation: MachineValidatingState::MachineValidating { + context: "Discovery".to_string(), + id: validation_id, + completed: 1, + total: 1, + is_enabled: true, + }, + }, + }, + 0, + ) + .await; + + let checkpoint = env.redfish_sim.timepoint(); + + let mut unlocked = false; + let mut resumed = false; + for _ in 0..20 { + env.run_machine_state_controller_iteration().await; + unlocked |= env + .redfish_sim + .lockdown_states() + .contains(&libredfish::EnabledDisabled::Disabled); + let mut txn = env.db_txn().await; + let host = mh.host().db_machine(&mut txn).await; + if matches!( + host.current_state(), + ManagedHostState::Validation { + validation_state: ValidationState::MachineValidation { + machine_validation: MachineValidatingState::RebootHost { + validation_id: current_validation_id, + }, + }, + } if *current_validation_id == validation_id + ) { + resumed = true; + break; + } + } + + assert!( + resumed, + "order-only drift should be repaired before validation resumes" + ); + assert!( + unlocked, + "validation should unlock the BMC before repairing boot-order drift" + ); + + let actions = env.redfish_sim.actions_since(&checkpoint).all_hosts(); + assert!( + actions.iter().any(|a| matches!( + a, + RedfishSimAction::IsBootOrderSetup { boot_interface_mac } + if *boot_interface_mac == boot_nic_mac.to_string() + )), + "validation should check the boot order for the boot NIC, got: {actions:?}" + ); + assert!( + actions.iter().any(|a| matches!( + a, + RedfishSimAction::SetBootOrderDpuFirst { boot_interface_mac } + if *boot_interface_mac == boot_nic_mac.to_string() + )), + "the drifted boot order should be restored for the boot NIC, got: {actions:?}" + ); + assert!( + !actions + .iter() + .any(|a| matches!(a, RedfishSimAction::MachineSetup { .. })), + "order-only repair should preserve the configured HTTP boot device, got: {actions:?}" + ); + assert!( + env.redfish_sim + .lockdown_states() + .iter() + .all(|l| *l == libredfish::EnabledDisabled::Enabled), + "the BMC should be re-locked once the boot order is repaired" + ); +} + +/// Exhausting the shared BIOS recovery budget during validation is terminal +/// for both the host repair and the active validation run. +#[crate::sqlx_test] +async fn test_validation_bios_setup_exhaustion_completes_active_validation(pool: sqlx::PgPool) { + let env = create_test_env(pool).await; + + let mh = create_managed_host(&env).await; + let host_id = mh.host().id; + let validation_id = + on_demand_machine_validation(&env, host_id, Vec::new(), Vec::new(), false, Vec::new()) + .await + .validation_id + .expect("on-demand validation should return an id"); + + env.redfish_sim.set_is_bios_setup(false); + set_host_controller_state_stuck_in( + &env, + host_id, + &ManagedHostState::Validation { + validation_state: ValidationState::MachineValidation { + machine_validation: MachineValidatingState::PollingBootBiosSetup { + validation_id, + retry_count: 3, + }, + }, + }, + 16, + ) + .await; + + env.run_machine_state_controller_iteration().await; + + let mut txn = env.db_txn().await; + let host = mh.host().db_machine(&mut txn).await; + assert!( + matches!( + host.current_state(), + ManagedHostState::Failed { + details: FailureDetails { + cause: FailureCause::BiosSetupFailed { .. }, + source: FailureSource::StateMachineArea(StateMachineArea::MainFlow), + .. + }, + .. + } + ), + "expected validation boot repair exhaustion to fail the host, got: {:?}", + host.current_state() + ); + assert_eq!( + host.on_demand_machine_validation_request, + Some(false), + "terminal boot repair should clear the validation request" + ); + + let validation = db::machine_validation::find_by_id(&env.pool, &validation_id) + .await + .expect("validation run should still be queryable"); + assert!( + matches!( + validation.status.as_ref().map(|status| &status.state), + Some(&MachineValidationState::Failed) + ), + "validation run should be marked failed, got: {:?}", + validation.status + ); + assert!( + validation.end_time.is_some(), + "terminal validation run should record an end time" + ); } /// When HostInit/PollingBiosSetup retry budget is exhausted, enter Failed and recover via is_bios_setup. diff --git a/crates/api-model/src/machine/mod.rs b/crates/api-model/src/machine/mod.rs index 02d01f9056..1d360b9b34 100644 --- a/crates/api-model/src/machine/mod.rs +++ b/crates/api-model/src/machine/mod.rs @@ -1284,6 +1284,42 @@ pub enum MachineValidatingState { #[serde(default = "default_true")] is_enabled: bool, }, + /// Machine validation ensures the host's boot device config is in place. + /// When it reads reverted -- however it drifted (changed externally, a + /// BIOS quirk, or the boot NIC dropping off the BMC's inventory during a + /// reboot's POST) -- these states correct it, mirroring host boot repair: + /// unlock the BMC, drive the boot-order flow, re-lock, resume validation. + PrepareBootRepair { + validation_id: MachineValidationId, + }, + UnlockForBootRepair { + validation_id: MachineValidationId, + unlock_host_state: UnlockHostState, + }, + CheckBootConfigForRepair { + validation_id: MachineValidationId, + }, + ConfigureBootBios { + validation_id: MachineValidationId, + #[serde(default)] + retry_count: u32, + }, + WaitingForBootBiosJob { + validation_id: MachineValidationId, + bios_config_info: BiosConfigInfo, + }, + PollingBootBiosSetup { + validation_id: MachineValidationId, + #[serde(default)] + retry_count: u32, + }, + RepairBootConfig { + validation_id: MachineValidationId, + set_boot_order_info: SetBootOrderInfo, + }, + LockAfterBootRepair { + validation_id: MachineValidationId, + }, } #[derive(Debug, Clone, Serialize, Deserialize, Eq, PartialEq)] #[serde(tag = "validation_type", rename_all = "lowercase")] @@ -1868,7 +1904,8 @@ pub struct BiosConfigInfo { #[serde(default, skip_serializing_if = "Option::is_none")] pub bios_job_id: Option, pub bios_config_state: BiosConfigState, - /// Full configure_host_bios retry count across HandleBiosJobFailure recovery cycles. + /// Shared host boot-configuration convergence retry count, including BIOS + /// job-failure recovery cycles. #[serde(default)] pub retry_count: u32, } @@ -1894,7 +1931,8 @@ pub struct SetBootOrderInfo { #[serde(default, skip_serializing_if = "Option::is_none")] pub set_boot_order_jid: Option, pub set_boot_order_state: SetBootOrderState, - /// Retry counter for SetBootOrder state machine. Defaults to 0 for backwards compatibility. + /// Shared host boot-configuration convergence retry count. Defaults to 0 + /// for backwards compatibility. #[serde(default)] pub retry_count: u32, } @@ -1904,6 +1942,10 @@ pub struct SetBootOrderInfo { #[serde(tag = "state", rename_all = "lowercase")] pub enum SetBootOrderState { SetBootOrder, + /// Legacy persisted state from the former inline HTTP-device repair flow. + /// It polls the device across the already-started reboot, then either + /// resumes `SetBootOrder` or migrates to the shared BIOS repair stages. + WaitForHttpBootDeviceApplied, WaitForSetBootOrderJobScheduled, RebootHost, WaitForSetBootOrderJobCompletion, @@ -2650,6 +2692,16 @@ pub fn state_sla( MachineValidatingState::RebootHost { .. } => { StateSla::with_sla(slas::VALIDATION, time_in_state) } + MachineValidatingState::PrepareBootRepair { .. } + | MachineValidatingState::UnlockForBootRepair { .. } + | MachineValidatingState::CheckBootConfigForRepair { .. } + | MachineValidatingState::ConfigureBootBios { .. } + | MachineValidatingState::WaitingForBootBiosJob { .. } + | MachineValidatingState::PollingBootBiosSetup { .. } + | MachineValidatingState::RepairBootConfig { .. } + | MachineValidatingState::LockAfterBootRepair { .. } => { + StateSla::with_sla(slas::VALIDATION, time_in_state) + } }, }, } diff --git a/crates/machine-controller/Cargo.toml b/crates/machine-controller/Cargo.toml index 9cd99b9978..74c865bd3e 100644 --- a/crates/machine-controller/Cargo.toml +++ b/crates/machine-controller/Cargo.toml @@ -70,6 +70,8 @@ version-compare = { workspace = true } [dev-dependencies] carbide-redfish = { path = "../redfish", features = ["test-support"] } +carbide-test-harness = { path = "../test-harness" } +carbide-test-support = { path = "../test-support" } figment = { workspace = true, features = ["env", "test", "toml"] } regex = { workspace = true } diff --git a/crates/machine-controller/src/config/controller.rs b/crates/machine-controller/src/config/controller.rs index 76e964eaaf..4af4ecb2bf 100644 --- a/crates/machine-controller/src/config/controller.rs +++ b/crates/machine-controller/src/config/controller.rs @@ -70,7 +70,9 @@ pub struct MachineStateControllerConfig { serialize_with = "as_duration" )] pub uefi_boot_wait: Duration, - /// Max configure_host_bios retry cycles through HandleBiosJobFailure recovery. + /// Retry budget for automated host boot-configuration convergence, shared + /// across BIOS recovery and boot-order verification. The field name is + /// retained for configuration compatibility. #[serde(default = "MachineStateControllerConfig::max_bios_config_retries_default")] pub max_bios_config_retries: u32, /// How long PollingBiosSetup may sit on Ok(false) before escalating into diff --git a/crates/machine-controller/src/handler.rs b/crates/machine-controller/src/handler.rs index daf3433c45..9489cf9774 100644 --- a/crates/machine-controller/src/handler.rs +++ b/crates/machine-controller/src/handler.rs @@ -119,21 +119,23 @@ mod bios_config; mod dpf; mod firmware_artifact; mod helpers; +mod host_boot_config; mod machine_validation; mod power; mod sku; #[cfg(test)] mod test_machine_setup; -use bios_config::{ - BiosConfigJobAdvanceOutcome, BiosConfigOutcome, PollingBiosSetupOutcome, - advance_bios_config_job, advance_polling_bios_setup, configure_host_bios, - handle_bios_setup_failed_recovery, -}; +use bios_config::handle_bios_setup_failed_recovery; use helpers::{ DpuDiscoveringStateHelper, DpuInitStateHelper, ManagedHostStateHelper, NextState, ReprovisionStateHelper, all_equal, }; +use host_boot_config::{ + HostBootConfigCheckOutcome, HostBootConfigDecision, HostBootConfigDpuFreshness, + HostBootConfigOutcome, HostBootConfigStage, check_host_boot_config, + initial_set_boot_order_info, run_host_boot_config_stage, should_skip_boot_order_remediation, +}; use state_controller::db_write_batch::DbWriteBatch; use crate::config::{BomValidationConfig, PowerManagerOptions}; @@ -1447,11 +1449,7 @@ impl MachineStateHandler { FailureCause::BiosSetupFailed { .. } if machine_id.machine_type().is_host() => { let recovered = ManagedHostState::HostInit { machine_state: MachineState::SetBootOrder { - set_boot_order_info: Some(SetBootOrderInfo { - set_boot_order_jid: None, - set_boot_order_state: SetBootOrderState::SetBootOrder, - retry_count: 0, - }), + set_boot_order_info: Some(initial_set_boot_order_info()), }, }; handle_bios_setup_failed_recovery(ctx, mh_snapshot, recovered).await @@ -3127,243 +3125,70 @@ async fn handle_dpu_reprovision( // WaitingForNetworkConfig already accepted the DPU observation. Do // not require a newer observation just because the host state // version advanced while entering host boot repair. - let redfish_client = ctx - .services - .create_redfish_client_from_machine(&state.host_snapshot) - .await?; - - let next_state = match check_host_boot_config( - redfish_client.as_ref(), + handle_dpu_reprovision_host_boot_config_check( + ctx, state, reachability_params, HostBootConfigDpuFreshness::AlreadyValidated, - ctx, ) - .await? - { - HostBootConfigDecision::Wait(reason) => { - return Ok(StateHandlerOutcome::wait(reason)); - } - HostBootConfigDecision::ConfigureBoot => { - ReprovisionState::ConfigureHostBoot { retry_count: 0 } - } - HostBootConfigDecision::LockHost => ReprovisionState::LockHostAfterBootRepair, - }; - - Ok(StateHandlerOutcome::transition( - update_reprovision_targets_to_reprovision_state(state, next_state)?, - )) + .await } ReprovisionState::CheckHostBootConfigAfterHostReboot => { // This path rebooted the host after unlocking, so require a DPU // observation newer than that reboot before trusting boot checks. - let redfish_client = ctx - .services - .create_redfish_client_from_machine(&state.host_snapshot) - .await?; - - let next_state = match check_host_boot_config( - redfish_client.as_ref(), + handle_dpu_reprovision_host_boot_config_check( + ctx, state, reachability_params, HostBootConfigDpuFreshness::SinceLastHostRebootRequest, - ctx, ) - .await? - { - HostBootConfigDecision::Wait(reason) => { - return Ok(StateHandlerOutcome::wait(reason)); - } - HostBootConfigDecision::ConfigureBoot => { - ReprovisionState::ConfigureHostBoot { retry_count: 0 } - } - HostBootConfigDecision::LockHost => ReprovisionState::LockHostAfterBootRepair, - }; - - Ok(StateHandlerOutcome::transition( - update_reprovision_targets_to_reprovision_state(state, next_state)?, - )) + .await } ReprovisionState::ConfigureHostBoot { retry_count } => { - // Run machine_setup only after the reprovisioned DPU is healthy; it - // may patch BIOS settings and trigger host-impacting recovery. - let redfish_client = ctx - .services - .create_redfish_client_from_machine(&state.host_snapshot) - .await?; - - match configure_host_bios( + handle_dpu_reprovision_host_boot_config_stage( ctx, - reachability_params, - redfish_client.as_ref(), state, - *retry_count, + reachability_params, + HostBootConfigStage::ConfigureBios { + retry_count: *retry_count, + }, ) - .await? - { - BiosConfigOutcome::Done => Ok(StateHandlerOutcome::transition( - update_reprovision_targets_to_reprovision_state( - state, - ReprovisionState::PollingHostBiosSetup { - retry_count: *retry_count, - }, - )?, - )), - BiosConfigOutcome::WaitingForBiosJob(bios_config_info) => { - Ok(StateHandlerOutcome::transition( - update_reprovision_targets_to_reprovision_state( - state, - ReprovisionState::WaitingForHostBiosJob { bios_config_info }, - )?, - )) - } - BiosConfigOutcome::WaitingForReboot(reason) => { - Ok(StateHandlerOutcome::wait(reason)) - } - } + .await } ReprovisionState::WaitingForHostBiosJob { bios_config_info } => { - // Poll vendor BIOS jobs before verifying the setup and boot order. - let redfish_client = ctx - .services - .create_redfish_client_from_machine(&state.host_snapshot) - .await?; - - match advance_bios_config_job( + handle_dpu_reprovision_host_boot_config_stage( ctx, - redfish_client.as_ref(), state, - bios_config_info.clone(), + reachability_params, + HostBootConfigStage::WaitingForBiosJob { + bios_config_info: bios_config_info.clone(), + }, ) - .await? - { - BiosConfigJobAdvanceOutcome::Continue(updated) => { - Ok(StateHandlerOutcome::transition( - update_reprovision_targets_to_reprovision_state( - state, - ReprovisionState::WaitingForHostBiosJob { - bios_config_info: updated, - }, - )?, - )) - } - BiosConfigJobAdvanceOutcome::Done => Ok(StateHandlerOutcome::transition( - update_reprovision_targets_to_reprovision_state( - state, - ReprovisionState::PollingHostBiosSetup { - retry_count: bios_config_info.retry_count, - }, - )?, - )), - BiosConfigJobAdvanceOutcome::Failed { failure } => Ok( - StateHandlerOutcome::transition(dpu_reprovision_host_boot_failed_state( - &state.managed_state, - state.host_snapshot.id, - failure, - )), - ), - BiosConfigJobAdvanceOutcome::RetryPlatformConfiguration { retry_count } => { - Ok(StateHandlerOutcome::transition( - update_reprovision_targets_to_reprovision_state( - state, - ReprovisionState::ConfigureHostBoot { retry_count }, - )?, - )) - } - BiosConfigJobAdvanceOutcome::Wait(reason) => Ok(StateHandlerOutcome::wait(reason)), - } + .await } ReprovisionState::PollingHostBiosSetup { retry_count } => { - // Verify machine_setup effects before promoting the DPU boot option. - let redfish_client = ctx - .services - .create_redfish_client_from_machine(&state.host_snapshot) - .await?; - - let predictions = load_boot_predictions(ctx, &state.host_snapshot.id).await?; - match advance_polling_bios_setup( - redfish_client.as_ref(), + handle_dpu_reprovision_host_boot_config_stage( + ctx, state, - *retry_count, - &ctx.services.site_config.machine_state_controller, - &predictions, + reachability_params, + HostBootConfigStage::PollingBiosSetup { + retry_count: *retry_count, + }, ) - .await? - { - PollingBiosSetupOutcome::Verified => { - let next_state = if should_skip_boot_order_remediation(state) { - ReprovisionState::LockHostAfterBootRepair - } else { - ReprovisionState::SetHostBootOrder { - set_boot_order_info: SetBootOrderInfo { - set_boot_order_jid: None, - set_boot_order_state: SetBootOrderState::SetBootOrder, - retry_count: 0, - }, - } - }; - - Ok(StateHandlerOutcome::transition( - update_reprovision_targets_to_reprovision_state(state, next_state)?, - )) - } - PollingBiosSetupOutcome::EnterRecovery(bios_config_info) => { - Ok(StateHandlerOutcome::transition( - update_reprovision_targets_to_reprovision_state( - state, - ReprovisionState::WaitingForHostBiosJob { bios_config_info }, - )?, - )) - } - PollingBiosSetupOutcome::Failed { failure } => Ok(StateHandlerOutcome::transition( - dpu_reprovision_host_boot_failed_state( - &state.managed_state, - state.host_snapshot.id, - failure, - ), - )), - PollingBiosSetupOutcome::Wait(reason) => Ok(StateHandlerOutcome::wait(reason)), - } + .await } ReprovisionState::SetHostBootOrder { set_boot_order_info, } => { - // Promote the selected DPU boot option after machine_setup has enabled it. - let redfish_client = ctx - .services - .create_redfish_client_from_machine(&state.host_snapshot) - .await?; - - match set_host_boot_order( + handle_dpu_reprovision_host_boot_config_stage( ctx, - reachability_params, - redfish_client.as_ref(), state, - set_boot_order_info.clone(), + reachability_params, + HostBootConfigStage::SetBootOrder { + set_boot_order_info: set_boot_order_info.clone(), + }, ) - .await? - { - SetBootOrderOutcome::Continue(boot_order_info) => { - Ok(StateHandlerOutcome::transition( - update_reprovision_targets_to_reprovision_state( - state, - ReprovisionState::SetHostBootOrder { - set_boot_order_info: boot_order_info, - }, - )?, - )) - } - SetBootOrderOutcome::Done => Ok(StateHandlerOutcome::transition( - update_reprovision_targets_to_reprovision_state( - state, - ReprovisionState::LockHostAfterBootRepair, - )?, - )), - SetBootOrderOutcome::WaitingForReboot(reason) => { - Ok(StateHandlerOutcome::wait(reason)) - } - SetBootOrderOutcome::Wait(reason) => Ok(StateHandlerOutcome::wait(reason)), - } + .await } ReprovisionState::LockHostAfterBootRepair => { // Preserve expected-machine lockdown policy after temporarily @@ -3489,6 +3314,117 @@ async fn handle_dpu_reprovision( } } +/// Checks host boot configuration for DPU reprovisioning and maps the shared +/// decision back into the reprovision state while preserving all active DPU +/// targets. The caller selects the observation-freshness requirement because +/// only the post-unlock path has rebooted the host. +async fn handle_dpu_reprovision_host_boot_config_check( + ctx: &mut StateHandlerContext<'_, MachineStateHandlerContextObjects>, + state: &ManagedHostStateSnapshot, + reachability_params: &ReachabilityParams, + dpu_freshness: HostBootConfigDpuFreshness, +) -> Result, StateHandlerError> { + let redfish_client = ctx + .services + .create_redfish_client_from_machine(&state.host_snapshot) + .await?; + + let next_state = match check_host_boot_config( + redfish_client.as_ref(), + state, + reachability_params, + dpu_freshness, + ctx, + ) + .await? + { + HostBootConfigCheckOutcome::Wait(reason) => { + return Ok(StateHandlerOutcome::wait(reason)); + } + HostBootConfigCheckOutcome::Ready(HostBootConfigDecision::ConfigureBios) => { + ReprovisionState::ConfigureHostBoot { retry_count: 0 } + } + HostBootConfigCheckOutcome::Ready(HostBootConfigDecision::SetBootOrder) => { + ReprovisionState::SetHostBootOrder { + set_boot_order_info: initial_set_boot_order_info(), + } + } + HostBootConfigCheckOutcome::Ready(HostBootConfigDecision::Complete) => { + ReprovisionState::LockHostAfterBootRepair + } + }; + + Ok(StateHandlerOutcome::transition( + update_reprovision_targets_to_reprovision_state(state, next_state)?, + )) +} + +/// Handles a host boot-configuration stage owned by DPU reprovisioning. +/// +/// Called by `handle_dpu_reprovision` for its host BIOS configuration, +/// vendor-job, polling, and boot-order substates. This adapter preserves the +/// active DPU reprovision targets, continues to `LockHostAfterBootRepair` on +/// completion, and attributes terminal failure to the lifecycle that owns the +/// reprovision. +async fn handle_dpu_reprovision_host_boot_config_stage( + ctx: &mut StateHandlerContext<'_, MachineStateHandlerContextObjects>, + state: &ManagedHostStateSnapshot, + reachability_params: &ReachabilityParams, + stage: HostBootConfigStage, +) -> Result, StateHandlerError> { + let redfish_client = ctx + .services + .create_redfish_client_from_machine(&state.host_snapshot) + .await?; + + match run_host_boot_config_stage( + ctx, + reachability_params, + redfish_client.as_ref(), + state, + stage, + ) + .await? + { + HostBootConfigOutcome::Continue(stage) => { + let reprovision_state = match stage { + HostBootConfigStage::ConfigureBios { retry_count } => { + ReprovisionState::ConfigureHostBoot { retry_count } + } + HostBootConfigStage::WaitingForBiosJob { bios_config_info } => { + ReprovisionState::WaitingForHostBiosJob { bios_config_info } + } + HostBootConfigStage::PollingBiosSetup { retry_count } => { + ReprovisionState::PollingHostBiosSetup { retry_count } + } + HostBootConfigStage::SetBootOrder { + set_boot_order_info, + } => ReprovisionState::SetHostBootOrder { + set_boot_order_info, + }, + }; + + Ok(StateHandlerOutcome::transition( + update_reprovision_targets_to_reprovision_state(state, reprovision_state)?, + )) + } + HostBootConfigOutcome::Complete => Ok(StateHandlerOutcome::transition( + update_reprovision_targets_to_reprovision_state( + state, + ReprovisionState::LockHostAfterBootRepair, + )?, + )), + HostBootConfigOutcome::Wait(reason) => Ok(StateHandlerOutcome::wait(reason)), + HostBootConfigOutcome::Failed { failure } => Ok(StateHandlerOutcome::transition( + dpu_reprovision_host_boot_failed_state( + &state.managed_state, + state.host_snapshot.id, + failure, + ), + )), + } +} + /// Build the correct failed state for host boot repair during DPU reprovision. fn dpu_reprovision_host_boot_failed_state( current_state: &ManagedHostState, @@ -3545,188 +3481,36 @@ async fn load_boot_predictions( Ok(predictions) } -/// Check whether host BIOS and DPU-first boot order remediation is required. -async fn check_host_boot_config( - redfish_client: &dyn Redfish, - mh_snapshot: &ManagedHostStateSnapshot, +// Returns true if update_manager flagged this managed host as needing its firmware examined +fn host_reprovisioning_requested(state: &ManagedHostStateSnapshot) -> bool { + state.host_snapshot.host_reprovision_requested.is_some() +} + +/// Returns true if the host reprovisioning request was initiated by a rack-level service +/// (i.e. the rack firmware upgrade flow). +fn is_rack_level_reprovisioning(state: &ManagedHostStateSnapshot) -> bool { + state + .host_snapshot + .host_reprovision_requested + .as_ref() + .is_some_and(|req| req.initiator.starts_with("rack-")) +} + +/// This function waits for DPU to finish discovery and reboots it. +pub async fn try_wait_for_dpu_discovery( + state: &ManagedHostStateSnapshot, reachability_params: &ReachabilityParams, - dpu_freshness: HostBootConfigDpuFreshness, ctx: &mut StateHandlerContext<'_, MachineStateHandlerContextObjects>, -) -> Result { - // Wait for DPUs only when this caller needs a fresh observation. DPU - // reprovision already validated DPU health before entering host boot repair. - if should_wait_for_dpus_before_host_boot_config( - mh_snapshot, - reachability_params, - dpu_freshness, - ctx, - ) - .await - { - return Ok(HostBootConfigDecision::Wait( - "Waiting for DPUs to come up.".to_string(), - )); - } - - // Resolve the interface whose boot option should be first in host UEFI. A - // zero-DPU host whose boot NIC has not taken its first HostInband lease yet - // falls back to its predicted boot NIC, and only waits when even that is - // unavailable. - let predictions = load_boot_predictions(ctx, &mh_snapshot.host_snapshot.id).await?; - let boot_interface = match resolve_boot_interface(mh_snapshot, &predictions) { - BootInterfaceResolution::Ready(target) => target, - BootInterfaceResolution::AwaitingNic => { - return Ok(HostBootConfigDecision::Wait(format!( - "Waiting for zero-DPU host {} to discover its boot NIC before configuring boot.", - mh_snapshot.host_snapshot.id - ))); - } - BootInterfaceResolution::Missing => { - return Err(StateHandlerError::GenericError(eyre::eyre!( - "Missing boot interface for host: {}", - mh_snapshot.host_snapshot.id - ))); - } - }; - - let vendor = mh_snapshot.host_snapshot.bmc_vendor(); - - log_host_config(redfish_client, mh_snapshot).await; - - let is_bios_setup = boot_interface - .run(|bi| redfish_client.is_bios_setup(Some(bi))) - .await - .map_err(|e| redfish_error("is_bios_setup", e))?; - - if should_skip_boot_order_remediation(mh_snapshot) { - if is_bios_setup { - tracing::info!( - machine_id = %mh_snapshot.host_snapshot.id, - bmc_vendor = %vendor, - "Skipping boot order remediation on Viking (known FW/BMC issue)" - ); - return Ok(HostBootConfigDecision::LockHost); - } - - tracing::warn!( - machine_id = %mh_snapshot.host_snapshot.id, - bmc_vendor = %vendor, - "Host BIOS setup is not configured properly on Viking; running BIOS repair before skipping boot order remediation" - ); - return Ok(HostBootConfigDecision::ConfigureBoot); - } - - let is_boot_order_setup = boot_interface - .run(|bi| redfish_client.is_boot_order_setup(bi)) - .await - .map_err(|e| redfish_error("is_boot_order_setup", e))?; - - if is_bios_setup && is_boot_order_setup { - tracing::info!( - machine_id = %mh_snapshot.host_snapshot.id, - bmc_vendor = %vendor, - "Host BIOS setup and boot order are configured properly" - ); - Ok(HostBootConfigDecision::LockHost) - } else { - tracing::warn!( - machine_id = %mh_snapshot.host_snapshot.id, - bmc_vendor = %vendor, - is_bios_setup, - is_boot_order_setup, - "Host BIOS setup or boot order is not configured properly" - ); - Ok(HostBootConfigDecision::ConfigureBoot) - } -} - -/// Viking BMC firmware cannot safely run boot-order remediation; BIOS repair still applies. -fn should_skip_boot_order_remediation(mh_snapshot: &ManagedHostStateSnapshot) -> bool { - mh_snapshot - .host_snapshot - .hardware_info - .as_ref() - .is_some_and(|hw| hw.is_dgx_h100()) -} - -async fn should_wait_for_dpus_before_host_boot_config( - mh_snapshot: &ManagedHostStateSnapshot, - reachability_params: &ReachabilityParams, - dpu_freshness: HostBootConfigDpuFreshness, - ctx: &mut StateHandlerContext<'_, MachineStateHandlerContextObjects>, -) -> bool { - if !mh_snapshot.has_managed_dpus() { - return false; - } - - match dpu_freshness { - HostBootConfigDpuFreshness::AlreadyValidated => false, - HostBootConfigDpuFreshness::CurrentHostState => { - !are_dpus_up_trigger_reboot_if_needed(mh_snapshot, reachability_params, ctx).await - } - HostBootConfigDpuFreshness::SinceLastHostRebootRequest => { - let Some(last_reboot_requested) = mh_snapshot.host_snapshot.last_reboot_requested - else { - tracing::warn!( - machine_id = %mh_snapshot.host_snapshot.id, - "No host reboot request timestamp found before post-reboot host boot config check" - ); - return false; - }; - - for dpu_snapshot in &mh_snapshot.dpu_snapshots { - if !is_dpu_observed_since(dpu_snapshot, last_reboot_requested.time) { - match trigger_reboot_if_needed( - dpu_snapshot, - mh_snapshot, - None, - reachability_params, - ctx, - ) - .await - { - Ok(_) => {} - Err(e) => tracing::warn!("could not reboot dpu {}: {e}", dpu_snapshot.id), - } - return true; - } - } - - false - } - } -} - -// Returns true if update_manager flagged this managed host as needing its firmware examined -fn host_reprovisioning_requested(state: &ManagedHostStateSnapshot) -> bool { - state.host_snapshot.host_reprovision_requested.is_some() -} - -/// Returns true if the host reprovisioning request was initiated by a rack-level service -/// (i.e. the rack firmware upgrade flow). -fn is_rack_level_reprovisioning(state: &ManagedHostStateSnapshot) -> bool { - state - .host_snapshot - .host_reprovision_requested - .as_ref() - .is_some_and(|req| req.initiator.starts_with("rack-")) -} - -/// This function waits for DPU to finish discovery and reboots it. -pub async fn try_wait_for_dpu_discovery( - state: &ManagedHostStateSnapshot, - reachability_params: &ReachabilityParams, - ctx: &mut StateHandlerContext<'_, MachineStateHandlerContextObjects>, - is_reprovision_case: bool, - current_dpu_machine_id: &MachineId, -) -> Result, StateHandlerError> { - // We are waiting for the `DiscoveryCompleted` RPC call to update the - // `last_discovery_time` timestamp. - // This indicates that all forge-scout actions have succeeded. - for dpu_snapshot in &state.dpu_snapshots { - if is_reprovision_case && dpu_snapshot.reprovision_requested.is_none() { - // This is reprovision handling and this DPU is not under reprovisioning. - continue; + is_reprovision_case: bool, + current_dpu_machine_id: &MachineId, +) -> Result, StateHandlerError> { + // We are waiting for the `DiscoveryCompleted` RPC call to update the + // `last_discovery_time` timestamp. + // This indicates that all forge-scout actions have succeeded. + for dpu_snapshot in &state.dpu_snapshots { + if is_reprovision_case && dpu_snapshot.reprovision_requested.is_none() { + // This is reprovision handling and this DPU is not under reprovisioning. + continue; } if !discovered_after_state_transition( dpu_snapshot.state.version, @@ -4934,6 +4718,9 @@ pub struct RebootStatus { /// Outcome of set_host_boot_order function. enum SetBootOrderOutcome { Continue(SetBootOrderInfo), + /// BIOS drifted after the caller's inspection; return to the shared BIOS + /// configuration stages so any Redfish job ID is persisted and polled. + ConfigureBios, Done, WaitingForReboot(String), /// No boot interface to act on yet -- e.g. a zero-DPU host whose boot NIC @@ -4942,18 +4729,115 @@ enum SetBootOrderOutcome { Wait(String), } -/// Decision from checking whether host boot repair is still required. -enum HostBootConfigDecision { - ConfigureBoot, - LockHost, - Wait(String), +/// Outcome of [`require_boot_interface`]: the resolved boot NIC, or the +/// caller's wait outcome for a zero-DPU host still discovering its boot NIC. +#[derive(Debug)] +enum RequiredBootInterface { + Ready(BootInterfaceTarget), + Wait(W), +} + +/// Resolve the boot NIC for a Redfish boot step, folding the not-ready cases +/// every caller handles the same way: a zero-DPU host that has not discovered +/// its boot NIC yet maps to the caller's wait outcome (`wait` wraps the shared +/// message; `activity` names the blocked step), and a host with no resolvable +/// interface is a hard error. Keeps the boot-order substates and host boot +/// repair resolving the boot NIC identically. +fn require_boot_interface( + mh_snapshot: &ManagedHostStateSnapshot, + predictions: &[PredictedMachineInterface], + activity: &str, + wait: impl FnOnce(String) -> W, +) -> Result, StateHandlerError> { + map_boot_interface_resolution( + resolve_boot_interface(mh_snapshot, predictions), + &mh_snapshot.host_snapshot.id, + activity, + wait, + ) +} + +/// The mapping behind [`require_boot_interface`], split out from the snapshot +/// lookup so it can be unit-tested directly. +fn map_boot_interface_resolution( + resolution: BootInterfaceResolution, + host_id: &MachineId, + activity: &str, + wait: impl FnOnce(String) -> W, +) -> Result, StateHandlerError> { + match resolution { + BootInterfaceResolution::Ready(target) => Ok(RequiredBootInterface::Ready(target)), + BootInterfaceResolution::AwaitingNic => Ok(RequiredBootInterface::Wait(wait(format!( + "Waiting for zero-DPU host {host_id} to discover its boot NIC before {activity}." + )))), + BootInterfaceResolution::Missing => Err(StateHandlerError::GenericError(eyre::eyre!( + "Missing boot interface for host: {host_id}" + ))), + } } -/// DPU observation freshness required before checking host boot config. -enum HostBootConfigDpuFreshness { - AlreadyValidated, - CurrentHostState, - SinceLastHostRebootRequest, +#[cfg(test)] +mod require_boot_interface_tests { + use super::*; + + fn host_id() -> MachineId { + "fm100ds7blqjsadm2uuh3qqbf1h7k8pmf47um6v9uckrg7l03po8mhqgvng" + .parse() + .unwrap() + } + + // Ready passes the resolved target through untouched. + #[test] + fn ready_passes_the_target_through() { + let target = BootInterfaceTarget::MacOnly("20:00:00:00:00:01".parse().unwrap()); + let resolved = map_boot_interface_resolution::( + BootInterfaceResolution::Ready(target.clone()), + &host_id(), + "setting boot order", + |msg| msg, + ) + .unwrap(); + let RequiredBootInterface::Ready(out) = resolved else { + panic!("expected Ready"); + }; + assert_eq!(out, target); + } + + // AwaitingNic becomes the caller's wait outcome, built from the shared + // message with the caller's activity spliced in. + #[test] + fn awaiting_nic_maps_to_the_callers_wait_outcome() { + let resolved = map_boot_interface_resolution::( + BootInterfaceResolution::AwaitingNic, + &host_id(), + "setting boot order", + |msg| msg, + ) + .unwrap(); + let RequiredBootInterface::Wait(msg) = resolved else { + panic!("expected Wait"); + }; + assert_eq!( + msg, + format!( + "Waiting for zero-DPU host {} to discover its boot NIC before setting boot order.", + host_id() + ) + ); + } + + // Missing is a hard error, not a wait. + #[test] + fn missing_is_a_hard_error() { + let err = map_boot_interface_resolution::( + BootInterfaceResolution::Missing, + &host_id(), + "setting boot order", + |msg| msg, + ) + .unwrap_err(); + assert!(matches!(err, StateHandlerError::GenericError(_))); + } } /// In case machine does not come up until a specified duration, this function tries to reboot @@ -5302,63 +5186,63 @@ fn check_host_health_for_alerts(state: &ManagedHostStateSnapshot) -> Result<(), } } -async fn handle_host_boot_order_setup( +/// Handles a shared boot-configuration stage during host initialization. +/// +/// Called by `HostMachineStateHandler::handle_object_state` for its platform +/// configuration, vendor-job, polling, and boot-order states. This adapter maps +/// continued stages back into `MachineState`, proceeds to measurements on +/// completion, and reports terminal failure as a host-init BIOS setup failure. +async fn handle_host_init_boot_config_stage( ctx: &mut StateHandlerContext<'_, MachineStateHandlerContextObjects>, - host_handler_params: HostHandlerParams, - mh_snapshot: &mut ManagedHostStateSnapshot, - set_boot_order_info: Option, + mh_snapshot: &ManagedHostStateSnapshot, + reachability_params: &ReachabilityParams, + redfish_client: &dyn Redfish, + stage: HostBootConfigStage, ) -> Result, StateHandlerError> { - tracing::info!( - "Starting Boot Order Configuration for {}: {set_boot_order_info:#?}", - mh_snapshot.host_snapshot.id - ); - - let redfish_client = ctx - .services - .create_redfish_client_from_machine(&mh_snapshot.host_snapshot) - .await?; - - let next_state = match set_boot_order_info { - Some(info) => { - match set_host_boot_order( - ctx, - &host_handler_params.reachability_params, - redfish_client.as_ref(), - mh_snapshot, - info, - ) - .await? - { - SetBootOrderOutcome::Continue(boot_order_info) => ManagedHostState::HostInit { - machine_state: MachineState::SetBootOrder { - set_boot_order_info: Some(boot_order_info), - }, - }, - SetBootOrderOutcome::Done => ManagedHostState::HostInit { - machine_state: MachineState::Measuring { - measuring_state: MeasuringState::WaitingForMeasurements, - }, - }, - SetBootOrderOutcome::WaitingForReboot(reason) => { - return Ok(StateHandlerOutcome::wait(reason)); + match run_host_boot_config_stage(ctx, reachability_params, redfish_client, mh_snapshot, stage) + .await? + { + HostBootConfigOutcome::Continue(stage) => { + let machine_state = match stage { + HostBootConfigStage::ConfigureBios { retry_count } => { + MachineState::WaitingForPlatformConfiguration { retry_count } } - SetBootOrderOutcome::Wait(reason) => { - return Ok(StateHandlerOutcome::wait(reason)); + HostBootConfigStage::WaitingForBiosJob { bios_config_info } => { + MachineState::WaitingForBiosJob { bios_config_info } } - } + HostBootConfigStage::PollingBiosSetup { retry_count } => { + MachineState::PollingBiosSetup { retry_count } + } + HostBootConfigStage::SetBootOrder { + set_boot_order_info, + } => MachineState::SetBootOrder { + set_boot_order_info: Some(set_boot_order_info), + }, + }; + Ok(StateHandlerOutcome::transition( + ManagedHostState::HostInit { machine_state }, + )) } - None => ManagedHostState::HostInit { - machine_state: MachineState::SetBootOrder { - set_boot_order_info: Some(SetBootOrderInfo { - set_boot_order_jid: None, - set_boot_order_state: SetBootOrderState::SetBootOrder, - retry_count: 0, - }), + HostBootConfigOutcome::Complete => Ok(StateHandlerOutcome::transition( + ManagedHostState::HostInit { + machine_state: MachineState::Measuring { + measuring_state: MeasuringState::WaitingForMeasurements, + }, }, - }, - }; - - Ok(StateHandlerOutcome::transition(next_state)) + )), + HostBootConfigOutcome::Wait(reason) => Ok(StateHandlerOutcome::wait(reason)), + HostBootConfigOutcome::Failed { failure } => { + Ok(StateHandlerOutcome::transition(ManagedHostState::Failed { + details: FailureDetails { + cause: FailureCause::BiosSetupFailed { err: failure }, + failed_at: Utc::now(), + source: FailureSource::StateMachineArea(StateMachineArea::HostInit), + }, + machine_id: mh_snapshot.host_snapshot.id, + retry_count: 0, + })) + } + } } /// TODO: we need to handle the case where the job is deleted for some reason @@ -5742,163 +5626,92 @@ impl StateHandler for HostMachineStateHandler { } } - match configure_host_bios( + handle_host_init_boot_config_stage( ctx, + mh_snapshot, &self.host_handler_params.reachability_params, redfish_client.as_ref(), - mh_snapshot, - *retry_count, + HostBootConfigStage::ConfigureBios { + retry_count: *retry_count, + }, ) - .await? - { - BiosConfigOutcome::Done => Ok(StateHandlerOutcome::transition( - ManagedHostState::HostInit { - machine_state: MachineState::PollingBiosSetup { - retry_count: *retry_count, - }, - }, - )), - BiosConfigOutcome::WaitingForBiosJob(bios_config_info) => Ok( - StateHandlerOutcome::transition(ManagedHostState::HostInit { - machine_state: MachineState::WaitingForBiosJob { bios_config_info }, - }), - ), - BiosConfigOutcome::WaitingForReboot(reason) => { - Ok(StateHandlerOutcome::wait(reason)) - } - } + .await } MachineState::WaitingForBiosJob { bios_config_info } => { let redfish_client = ctx .services .create_redfish_client_from_machine(&mh_snapshot.host_snapshot) .await?; - match advance_bios_config_job( + handle_host_init_boot_config_stage( ctx, - redfish_client.as_ref(), mh_snapshot, - bios_config_info.clone(), + &self.host_handler_params.reachability_params, + redfish_client.as_ref(), + HostBootConfigStage::WaitingForBiosJob { + bios_config_info: bios_config_info.clone(), + }, ) - .await? - { - BiosConfigJobAdvanceOutcome::Continue(updated) => Ok( - StateHandlerOutcome::transition(ManagedHostState::HostInit { - machine_state: MachineState::WaitingForBiosJob { - bios_config_info: updated, - }, - }), - ), - BiosConfigJobAdvanceOutcome::Done => Ok(StateHandlerOutcome::transition( - ManagedHostState::HostInit { - machine_state: MachineState::PollingBiosSetup { - retry_count: bios_config_info.retry_count, - }, - }, - )), - BiosConfigJobAdvanceOutcome::Failed { failure } => { - Ok(StateHandlerOutcome::transition(ManagedHostState::Failed { - details: FailureDetails { - cause: FailureCause::BiosSetupFailed { err: failure }, - failed_at: Utc::now(), - source: FailureSource::StateMachineArea( - StateMachineArea::HostInit, - ), - }, - machine_id: mh_snapshot.host_snapshot.id, - retry_count: 0, - })) - } - BiosConfigJobAdvanceOutcome::RetryPlatformConfiguration { retry_count } => { - Ok(StateHandlerOutcome::transition( - ManagedHostState::HostInit { - machine_state: MachineState::WaitingForPlatformConfiguration { - retry_count, - }, - }, - )) - } - BiosConfigJobAdvanceOutcome::Wait(reason) => { - Ok(StateHandlerOutcome::wait(reason)) - } - } + .await } MachineState::PollingBiosSetup { retry_count } => { - let next_state = ManagedHostState::HostInit { - machine_state: MachineState::SetBootOrder { - set_boot_order_info: Some(SetBootOrderInfo { - set_boot_order_jid: None, - set_boot_order_state: SetBootOrderState::SetBootOrder, - retry_count: 0, - }), - }, - }; - let redfish_client = ctx .services .create_redfish_client_from_machine(&mh_snapshot.host_snapshot) .await?; - let predictions = - load_boot_predictions(ctx, &mh_snapshot.host_snapshot.id).await?; - match advance_polling_bios_setup( - redfish_client.as_ref(), + handle_host_init_boot_config_stage( + ctx, mh_snapshot, - *retry_count, - &ctx.services.site_config.machine_state_controller, - &predictions, + &self.host_handler_params.reachability_params, + redfish_client.as_ref(), + HostBootConfigStage::PollingBiosSetup { + retry_count: *retry_count, + }, ) - .await? - { - PollingBiosSetupOutcome::Verified => { - Ok(StateHandlerOutcome::transition(next_state)) - } - PollingBiosSetupOutcome::EnterRecovery(bios_config_info) => Ok( - StateHandlerOutcome::transition(ManagedHostState::HostInit { - machine_state: MachineState::WaitingForBiosJob { bios_config_info }, - }), - ), - PollingBiosSetupOutcome::Failed { failure } => { - Ok(StateHandlerOutcome::transition(ManagedHostState::Failed { - details: FailureDetails { - cause: FailureCause::BiosSetupFailed { err: failure }, - failed_at: Utc::now(), - source: FailureSource::StateMachineArea( - StateMachineArea::HostInit, - ), - }, - machine_id: mh_snapshot.host_snapshot.id, - retry_count: 0, - })) - } - PollingBiosSetupOutcome::Wait(reason) => { - Ok(StateHandlerOutcome::wait(reason)) - } - } + .await } MachineState::SetBootOrder { set_boot_order_info, - } => Ok(handle_host_boot_order_setup( - ctx, - self.host_handler_params.clone(), - mh_snapshot, - set_boot_order_info.clone(), - ) - .await?), - MachineState::Measuring { measuring_state } => { - if !self.host_handler_params.attestation_enabled { + } => { + let Some(set_boot_order_info) = set_boot_order_info.clone() else { return Ok(StateHandlerOutcome::transition( ManagedHostState::HostInit { - machine_state: MachineState::SpdmMeasuring { - spdm_measuring_state: SpdmMeasuringState::TriggerMeasurements, + machine_state: MachineState::SetBootOrder { + set_boot_order_info: Some(initial_set_boot_order_info()), }, }, )); - } - match handle_measuring_state( - measuring_state, - &mh_snapshot.host_snapshot.id, - &mut ctx.services.db_reader, - self.host_handler_params.attestation_enabled, - ) + }; + let redfish_client = ctx + .services + .create_redfish_client_from_machine(&mh_snapshot.host_snapshot) + .await?; + handle_host_init_boot_config_stage( + ctx, + mh_snapshot, + &self.host_handler_params.reachability_params, + redfish_client.as_ref(), + HostBootConfigStage::SetBootOrder { + set_boot_order_info, + }, + ) + .await + } + MachineState::Measuring { measuring_state } => { + if !self.host_handler_params.attestation_enabled { + return Ok(StateHandlerOutcome::transition( + ManagedHostState::HostInit { + machine_state: MachineState::SpdmMeasuring { + spdm_measuring_state: SpdmMeasuringState::TriggerMeasurements, + }, + }, + )); + } + match handle_measuring_state( + measuring_state, + &mh_snapshot.host_snapshot.id, + &mut ctx.services.db_reader, + self.host_handler_params.attestation_enabled, + ) .await { Ok(measuring_outcome) => { @@ -7154,11 +6967,7 @@ impl StateHandler for InstanceStateHandler { instance_state: InstanceState::HostPlatformConfiguration { platform_config_state: HostPlatformConfigurationState::SetBootOrder { - set_boot_order_info: SetBootOrderInfo { - set_boot_order_jid: None, - set_boot_order_state: SetBootOrderState::SetBootOrder, - retry_count: 0, - }, + set_boot_order_info: initial_set_boot_order_info(), }, }, }; @@ -10749,6 +10558,75 @@ async fn log_host_config(redfish_client: &dyn Redfish, mh_snapshot: &ManagedHost ); } +/// Handles a shared boot-configuration stage during assigned-host platform +/// configuration. +/// +/// Called by `handle_instance_host_platform_config` for its BIOS configuration, +/// vendor-job, polling, and boot-order states. This adapter maps continued +/// stages back into `HostPlatformConfigurationState`, proceeds to `LockHost` on +/// completion, and scopes terminal failure to the assigned instance. +async fn handle_instance_host_boot_config_stage( + ctx: &mut StateHandlerContext<'_, MachineStateHandlerContextObjects>, + mh_snapshot: &ManagedHostStateSnapshot, + reachability_params: &ReachabilityParams, + redfish_client: &dyn Redfish, + stage: HostBootConfigStage, +) -> Result, StateHandlerError> { + match run_host_boot_config_stage(ctx, reachability_params, redfish_client, mh_snapshot, stage) + .await? + { + HostBootConfigOutcome::Continue(stage) => { + let platform_config_state = match stage { + HostBootConfigStage::ConfigureBios { retry_count } => { + HostPlatformConfigurationState::ConfigureBios { + bios_config_info: None, + retry_count, + } + } + HostBootConfigStage::WaitingForBiosJob { bios_config_info } => { + HostPlatformConfigurationState::WaitingForBiosJob { bios_config_info } + } + HostBootConfigStage::PollingBiosSetup { retry_count } => { + HostPlatformConfigurationState::PollingBiosSetup { retry_count } + } + HostBootConfigStage::SetBootOrder { + set_boot_order_info, + } => HostPlatformConfigurationState::SetBootOrder { + set_boot_order_info, + }, + }; + + Ok(StateHandlerOutcome::transition( + ManagedHostState::Assigned { + instance_state: InstanceState::HostPlatformConfiguration { + platform_config_state, + }, + }, + )) + } + HostBootConfigOutcome::Complete => Ok(StateHandlerOutcome::transition( + ManagedHostState::Assigned { + instance_state: InstanceState::HostPlatformConfiguration { + platform_config_state: HostPlatformConfigurationState::LockHost, + }, + }, + )), + HostBootConfigOutcome::Wait(reason) => Ok(StateHandlerOutcome::wait(reason)), + HostBootConfigOutcome::Failed { failure } => Ok(StateHandlerOutcome::transition( + ManagedHostState::Assigned { + instance_state: InstanceState::Failed { + details: FailureDetails { + cause: FailureCause::BiosSetupFailed { err: failure }, + failed_at: Utc::now(), + source: FailureSource::StateMachineArea(StateMachineArea::AssignedInstance), + }, + machine_id: mh_snapshot.host_snapshot.id, + }, + }, + )), + } +} + async fn handle_instance_host_platform_config( ctx: &mut StateHandlerContext<'_, MachineStateHandlerContextObjects>, mh_snapshot: &mut ManagedHostStateSnapshot, @@ -10977,18 +10855,29 @@ async fn handle_instance_host_platform_config( ) .await? { - HostBootConfigDecision::Wait(reason) => { + HostBootConfigCheckOutcome::Wait(reason) => { return Ok(StateHandlerOutcome::wait(reason)); } - HostBootConfigDecision::ConfigureBoot => InstanceState::HostPlatformConfiguration { - platform_config_state: HostPlatformConfigurationState::ConfigureBios { - bios_config_info: None, - retry_count: 0, - }, - }, - HostBootConfigDecision::LockHost => InstanceState::HostPlatformConfiguration { - platform_config_state: HostPlatformConfigurationState::LockHost, - }, + HostBootConfigCheckOutcome::Ready(HostBootConfigDecision::ConfigureBios) => { + InstanceState::HostPlatformConfiguration { + platform_config_state: HostPlatformConfigurationState::ConfigureBios { + bios_config_info: None, + retry_count: 0, + }, + } + } + HostBootConfigCheckOutcome::Ready(HostBootConfigDecision::SetBootOrder) => { + InstanceState::HostPlatformConfiguration { + platform_config_state: HostPlatformConfigurationState::SetBootOrder { + set_boot_order_info: initial_set_boot_order_info(), + }, + } + } + HostBootConfigCheckOutcome::Ready(HostBootConfigDecision::Complete) => { + InstanceState::HostPlatformConfiguration { + platform_config_state: HostPlatformConfigurationState::LockHost, + } + } } } HostPlatformConfigurationState::ConfigureBios { @@ -11009,174 +10898,48 @@ async fn handle_instance_host_platform_config( )); } - let next_platform = match configure_host_bios( + return handle_instance_host_boot_config_stage( ctx, + mh_snapshot, reachability_params, redfish_client.as_ref(), - mh_snapshot, - retry_count, + HostBootConfigStage::ConfigureBios { retry_count }, ) - .await? - { - BiosConfigOutcome::Done => { - HostPlatformConfigurationState::PollingBiosSetup { retry_count } - } - BiosConfigOutcome::WaitingForBiosJob(bios_config_info) => { - HostPlatformConfigurationState::WaitingForBiosJob { bios_config_info } - } - BiosConfigOutcome::WaitingForReboot(reason) => { - return Ok(StateHandlerOutcome::wait(reason)); - } - }; - return Ok(StateHandlerOutcome::transition( - ManagedHostState::Assigned { - instance_state: InstanceState::HostPlatformConfiguration { - platform_config_state: next_platform, - }, - }, - )); + .await; } HostPlatformConfigurationState::WaitingForBiosJob { bios_config_info } => { - let next_platform = match advance_bios_config_job( + return handle_instance_host_boot_config_stage( ctx, - redfish_client.as_ref(), mh_snapshot, - bios_config_info.clone(), + reachability_params, + redfish_client.as_ref(), + HostBootConfigStage::WaitingForBiosJob { bios_config_info }, ) - .await? - { - BiosConfigJobAdvanceOutcome::Continue(updated) => { - HostPlatformConfigurationState::WaitingForBiosJob { - bios_config_info: updated, - } - } - BiosConfigJobAdvanceOutcome::Done => { - HostPlatformConfigurationState::PollingBiosSetup { - retry_count: bios_config_info.retry_count, - } - } - BiosConfigJobAdvanceOutcome::Failed { failure } => { - return Ok(StateHandlerOutcome::transition( - ManagedHostState::Assigned { - instance_state: InstanceState::Failed { - details: FailureDetails { - cause: FailureCause::BiosSetupFailed { err: failure }, - failed_at: Utc::now(), - source: FailureSource::StateMachineArea( - StateMachineArea::AssignedInstance, - ), - }, - machine_id: mh_snapshot.host_snapshot.id, - }, - }, - )); - } - BiosConfigJobAdvanceOutcome::RetryPlatformConfiguration { - retry_count: next_count, - } => HostPlatformConfigurationState::ConfigureBios { - bios_config_info: None, - retry_count: next_count, - }, - BiosConfigJobAdvanceOutcome::Wait(reason) => { - return Ok(StateHandlerOutcome::wait(reason)); - } - }; - return Ok(StateHandlerOutcome::transition( - ManagedHostState::Assigned { - instance_state: InstanceState::HostPlatformConfiguration { - platform_config_state: next_platform, - }, - }, - )); + .await; } HostPlatformConfigurationState::PollingBiosSetup { retry_count } => { - let next_instance_state = InstanceState::HostPlatformConfiguration { - platform_config_state: if should_skip_boot_order_remediation(mh_snapshot) { - HostPlatformConfigurationState::LockHost - } else { - HostPlatformConfigurationState::SetBootOrder { - set_boot_order_info: SetBootOrderInfo { - set_boot_order_jid: None, - set_boot_order_state: SetBootOrderState::SetBootOrder, - retry_count: 0, - }, - } - }, - }; - - let predictions = load_boot_predictions(ctx, &mh_snapshot.host_snapshot.id).await?; - match advance_polling_bios_setup( - redfish_client.as_ref(), + return handle_instance_host_boot_config_stage( + ctx, mh_snapshot, - retry_count, - &ctx.services.site_config.machine_state_controller, - &predictions, + reachability_params, + redfish_client.as_ref(), + HostBootConfigStage::PollingBiosSetup { retry_count }, ) - .await? - { - PollingBiosSetupOutcome::Verified => next_instance_state, - PollingBiosSetupOutcome::EnterRecovery(bios_config_info) => { - return Ok(StateHandlerOutcome::transition( - ManagedHostState::Assigned { - instance_state: InstanceState::HostPlatformConfiguration { - platform_config_state: - HostPlatformConfigurationState::WaitingForBiosJob { - bios_config_info, - }, - }, - }, - )); - } - PollingBiosSetupOutcome::Failed { failure } => { - return Ok(StateHandlerOutcome::transition( - ManagedHostState::Assigned { - instance_state: InstanceState::Failed { - details: FailureDetails { - cause: FailureCause::BiosSetupFailed { err: failure }, - failed_at: Utc::now(), - source: FailureSource::StateMachineArea( - StateMachineArea::AssignedInstance, - ), - }, - machine_id: mh_snapshot.host_snapshot.id, - }, - }, - )); - } - PollingBiosSetupOutcome::Wait(reason) => { - return Ok(StateHandlerOutcome::wait(reason)); - } - } + .await; } HostPlatformConfigurationState::SetBootOrder { set_boot_order_info, } => { - match set_host_boot_order( + return handle_instance_host_boot_config_stage( ctx, + mh_snapshot, reachability_params, redfish_client.as_ref(), - mh_snapshot, - set_boot_order_info, - ) - .await? - { - SetBootOrderOutcome::Continue(boot_order_info) => { - InstanceState::HostPlatformConfiguration { - platform_config_state: HostPlatformConfigurationState::SetBootOrder { - set_boot_order_info: boot_order_info, - }, - } - } - SetBootOrderOutcome::Done => InstanceState::HostPlatformConfiguration { - platform_config_state: HostPlatformConfigurationState::LockHost, + HostBootConfigStage::SetBootOrder { + set_boot_order_info, }, - SetBootOrderOutcome::WaitingForReboot(reason) => { - return Ok(StateHandlerOutcome::wait(reason)); - } - SetBootOrderOutcome::Wait(reason) => { - return Ok(StateHandlerOutcome::wait(reason)); - } - } + ) + .await; } HostPlatformConfigurationState::LockHost => { if mh_snapshot.host_snapshot.host_profile.disable_lockdown { @@ -11227,40 +10990,68 @@ async fn set_host_boot_order( // HostInband lease creates a real row, a zero-DPU/NIC-mode host // resolves via its predictions; it waits only when neither a real row // nor a usable prediction exists. - let boot_interface = match resolve_boot_interface(mh_snapshot, &predictions) { - BootInterfaceResolution::Ready(target) => target, - BootInterfaceResolution::AwaitingNic => { - return Ok(SetBootOrderOutcome::Wait(format!( - "Waiting for zero-DPU host {} to discover its boot NIC before setting boot order.", - mh_snapshot.host_snapshot.id - ))); - } - BootInterfaceResolution::Missing => { - return Err(StateHandlerError::GenericError(eyre::eyre!( - "Missing boot interface for host: {}", - mh_snapshot.host_snapshot.id - ))); - } + let boot_interface = match require_boot_interface( + mh_snapshot, + &predictions, + "setting boot order", + SetBootOrderOutcome::Wait, + )? { + RequiredBootInterface::Ready(target) => target, + RequiredBootInterface::Wait(outcome) => return Ok(outcome), }; - // Re-assert the platform BIOS config before reordering -- a NIC-mode - // reboot can de-enumerate the BlueField and revert HttpDev1 to the - // onboard default, and `set_boot_order_dpu_first` only reorders the - // boot options it finds, so it can't bring HttpDev1 back on its own. - // Re-running `machine_setup` (by interface id) restores the HTTP boot - // device for the reorder, the same recovery the BIOS-setup phase - // already does. Idempotent when it's already set, and applied by the - // existing RebootHost restart; the returned job id is dropped - // (zero-DPU hosts swallow it as `NoDpu`, and CheckBootOrder tracks - // the boot-order job below). - call_machine_setup_and_handle_no_dpu_error( - redfish_client, - Some(&boot_interface), - mh_snapshot.host_snapshot.associated_dpu_machine_ids().len(), - &ctx.services.site_config, - ) - .await - .map_err(|e| redfish_error("machine_setup", e))?; + // Don't re-apply a boot config that's already in place. On Dell two + // BIOS config writes cannot share one pending job: machine_setup can + // commit a job that collides with set_boot_order_dpu_first (iDRAC + // `SYS011`). If the HTTP device drifted since the owning lifecycle's + // inspection, return to the shared BIOS/job stages before attempting + // the order write. + let is_bios_setup = boot_interface + .run(|bi| redfish_client.is_bios_setup(Some(bi))) + .await + .map_err(|e| redfish_error("is_bios_setup", e))?; + + if !is_bios_setup { + // The HTTP-boot device drifted after the owning lifecycle's + // inspection. Route back through the shared BIOS driver rather + // than issuing machine_setup here: vendor jobs must be persisted + // and polled before boot-order work can safely continue. + return Ok(SetBootOrderOutcome::ConfigureBios); + } + + // Keep the vendor safety policy at the actuator boundary. This is + // intentionally after the BIOS check: validation reuses this state + // to restore a reverted HTTP boot device on Viking, where BIOS + // repair is supported but boot-order remediation is not. + if should_skip_boot_order_remediation(mh_snapshot) { + tracing::info!( + machine_id = %mh_snapshot.host_snapshot.id, + bmc_vendor = %mh_snapshot.host_snapshot.bmc_vendor(), + "Skipping boot order remediation on Viking (known FW/BMC issue)" + ); + return Ok(SetBootOrderOutcome::Done); + } + + // HTTP-boot device is already set, so `machine_setup` was not re-run + // this pass -- no pending BIOS job for the boot-order set to collide + // with. If the boot order is already set too, there's nothing to + // re-apply: go straight to verification. + let is_boot_order_setup = boot_interface + .run(|bi| redfish_client.is_boot_order_setup(bi)) + .await + .map_err(|e| redfish_error("is_boot_order_setup", e))?; + + if is_boot_order_setup { + tracing::info!( + machine_id = %mh_snapshot.host_snapshot.id, + "Boot config already in place at SetBootOrder; skipping re-apply and verifying", + ); + return Ok(SetBootOrderOutcome::Continue(SetBootOrderInfo { + set_boot_order_jid: None, + set_boot_order_state: SetBootOrderState::CheckBootOrder, + retry_count: set_boot_order_info.retry_count, + })); + } let jid = match set_boot_order_dpu_first_and_handle_no_dpu_error( redfish_client, @@ -11328,6 +11119,71 @@ async fn set_host_boot_order( retry_count: set_boot_order_info.retry_count, })) } + SetBootOrderState::WaitForHttpBootDeviceApplied => { + // Backward compatibility for hosts persisted in the former inline + // machine_setup flow. Once its wait window expires, migrate into the + // shared BIOS/job stages so any new vendor job is retained. + const HTTP_BOOT_DEVICE_APPLY_WAIT_MINUTES: i64 = 10; + const MAX_HTTP_BOOT_DEVICE_APPLY_RETRIES: u32 = 3; + + let boot_interface = match require_boot_interface( + mh_snapshot, + &predictions, + "verifying the re-asserted HTTP boot device", + SetBootOrderOutcome::Wait, + )? { + RequiredBootInterface::Ready(target) => target, + RequiredBootInterface::Wait(outcome) => return Ok(outcome), + }; + + let is_bios_setup = boot_interface + .run(|bi| redfish_client.is_bios_setup(Some(bi))) + .await + .map_err(|e| redfish_error("is_bios_setup", e))?; + + if is_bios_setup { + tracing::info!( + machine_id = %mh_snapshot.host_snapshot.id, + "Re-asserted HTTP boot device applied; returning to SetBootOrder to set the boot order", + ); + return Ok(SetBootOrderOutcome::Continue(SetBootOrderInfo { + set_boot_order_jid: None, + set_boot_order_state: SetBootOrderState::SetBootOrder, + retry_count: set_boot_order_info.retry_count, + })); + } + + let minutes_waiting = mh_snapshot + .host_snapshot + .state + .version + .since_state_change() + .num_minutes(); + + if minutes_waiting >= HTTP_BOOT_DEVICE_APPLY_WAIT_MINUTES { + // Preserve the terminal behavior of a legacy state whose old + // retry budget was already exhausted. Any nonterminal state + // carries its existing count into the shared convergence + // budget. + if set_boot_order_info.retry_count >= MAX_HTTP_BOOT_DEVICE_APPLY_RETRIES { + return Err(StateHandlerError::ManualInterventionRequired(format!( + "HTTP boot device on host {} still not applied after {} re-asserts; manual intervention required", + mh_snapshot.host_snapshot.id, MAX_HTTP_BOOT_DEVICE_APPLY_RETRIES + ))); + } + tracing::warn!( + machine_id = %mh_snapshot.host_snapshot.id, + minutes_waiting, + "Re-asserted HTTP boot device still not applied after the wait window; migrating to shared BIOS repair", + ); + return Ok(SetBootOrderOutcome::ConfigureBios); + } + + Ok(SetBootOrderOutcome::WaitingForReboot(format!( + "Waiting for the re-asserted HTTP boot device to apply on host {} ({minutes_waiting}m of {HTTP_BOOT_DEVICE_APPLY_WAIT_MINUTES}m)", + mh_snapshot.host_snapshot.id + ))) + } SetBootOrderState::WaitForSetBootOrderJobScheduled => { if let Some(job_id) = &set_boot_order_info.set_boot_order_jid { let job_state = redfish_client @@ -11350,7 +11206,8 @@ async fn set_host_boot_order( })) } SetBootOrderState::RebootHost => { - // Host needs to be rebooted to pick up the changes after calling machine_setup + // Reboot before final verification so any accepted boot-order + // change or vendor job can apply. handler_host_power_control(mh_snapshot, ctx, SystemPowerControl::ForceRestart).await?; Ok(SetBootOrderOutcome::Continue(SetBootOrderInfo { @@ -11534,27 +11391,55 @@ async fn set_host_boot_order( } } SetBootOrderState::CheckBootOrder => { - const MAX_BOOT_ORDER_RETRIES: u32 = 3; const CHECK_BOOT_ORDER_TIMEOUT_MINUTES: i64 = 30; let retry_count = set_boot_order_info.retry_count; + let max_retries = ctx + .services + .site_config + .machine_state_controller + .max_bios_config_retries; - let boot_interface = match resolve_boot_interface(mh_snapshot, &predictions) { - BootInterfaceResolution::Ready(target) => target, - BootInterfaceResolution::AwaitingNic => { - return Ok(SetBootOrderOutcome::Wait(format!( - "Waiting for zero-DPU host {} to discover its boot NIC before verifying boot order.", - mh_snapshot.host_snapshot.id - ))); - } - BootInterfaceResolution::Missing => { - return Err(StateHandlerError::GenericError(eyre::eyre!( - "Missing boot interface for host: {}", - mh_snapshot.host_snapshot.id - ))); - } + let boot_interface = match require_boot_interface( + mh_snapshot, + &predictions, + "verifying boot order", + SetBootOrderOutcome::Wait, + )? { + RequiredBootInterface::Ready(target) => target, + RequiredBootInterface::Wait(outcome) => return Ok(outcome), }; + // The boot-order job/recovery reboot can itself revert the HTTP + // device or another managed BIOS attribute. Verify BIOS first for + // every platform and return to the shared job-aware repair before + // declaring the overall boot configuration complete. + let is_bios_setup = boot_interface + .run(|bi| redfish_client.is_bios_setup(Some(bi))) + .await + .map_err(|e| redfish_error("is_bios_setup", e))?; + + if !is_bios_setup { + tracing::warn!( + machine_id = %mh_snapshot.host_snapshot.id, + bmc_vendor = %mh_snapshot.host_snapshot.bmc_vendor(), + "Host BIOS drifted during boot-order recovery; returning to shared BIOS repair" + ); + return Ok(SetBootOrderOutcome::ConfigureBios); + } + + // A controller upgrade can resume a Viking after an old boot-order + // job or power-recovery substate. Once the operation and BIOS + // verification are complete, skip the unsupported order read. + if should_skip_boot_order_remediation(mh_snapshot) { + tracing::info!( + machine_id = %mh_snapshot.host_snapshot.id, + bmc_vendor = %mh_snapshot.host_snapshot.bmc_vendor(), + "Skipping boot order verification on Viking (known FW/BMC issue)" + ); + return Ok(SetBootOrderOutcome::Done); + } + let boot_order_configured = boot_interface .run(|bi| redfish_client.is_boot_order_setup(bi)) .await @@ -11579,16 +11464,23 @@ async fn set_host_boot_order( time_since_state_change.num_minutes() ); - // If we've been stuck for 30+ minutes and haven't exhausted retries, retry SetBootOrder - if time_since_state_change.num_minutes() >= CHECK_BOOT_ORDER_TIMEOUT_MINUTES - && retry_count < MAX_BOOT_ORDER_RETRIES - { - tracing::info!( - "Boot order check timed out for {} after {} minutes, retrying SetBootOrder (retry {} of {})", + if time_since_state_change.num_minutes() < CHECK_BOOT_ORDER_TIMEOUT_MINUTES { + return Err(StateHandlerError::GenericError(eyre::eyre!( + "Boot order is not configured properly for host {} after SetBootOrder completed (retry_count: {}, max_retries: {}, time_in_state: {} minutes)", mh_snapshot.host_snapshot.id, - time_since_state_change.num_minutes(), - retry_count + 1, - MAX_BOOT_ORDER_RETRIES + retry_count, + max_retries, + time_since_state_change.num_minutes() + ))); + } + + if retry_count < max_retries { + tracing::info!( + machine_id = %mh_snapshot.host_snapshot.id, + time_in_state_minutes = time_since_state_change.num_minutes(), + retry_count = retry_count + 1, + max_retries, + "Boot order check timed out; retrying SetBootOrder" ); return Ok(SetBootOrderOutcome::Continue(SetBootOrderInfo { @@ -11598,11 +11490,13 @@ async fn set_host_boot_order( })); } - // Either still within timeout window or exhausted retries - return error - Err(StateHandlerError::GenericError(eyre::eyre!( - "Boot order is not configured properly for host {} after SetBootOrder completed (retry_count: {}, time_in_state: {} minutes)", + // Keep the state parked so an operator can repair the BMC and the + // next verification can complete without resetting the lifecycle. + Err(StateHandlerError::ManualInterventionRequired(format!( + "Boot order is not configured properly for host {} after SetBootOrder completed; automated boot-config convergence exhausted (retry_count: {}, max_retries: {}, time_in_state: {} minutes)", mh_snapshot.host_snapshot.id, retry_count, + max_retries, time_since_state_change.num_minutes() ))) } diff --git a/crates/machine-controller/src/handler/bios_config.rs b/crates/machine-controller/src/handler/bios_config.rs index b94aaa5992..b8644f809e 100644 --- a/crates/machine-controller/src/handler/bios_config.rs +++ b/crates/machine-controller/src/handler/bios_config.rs @@ -363,7 +363,7 @@ fn try_bios_recovery_attempt( ); return Ok(BiosRecoveryAttemptOutcome::Failed { failure: format!( - "{failure} (automated BIOS recovery exhausted after {} attempts)", + "{failure} (automated BIOS recovery exhausted after {} retries)", machine_controller_config.max_bios_config_retries ), }); diff --git a/crates/machine-controller/src/handler/host_boot_config.rs b/crates/machine-controller/src/handler/host_boot_config.rs new file mode 100644 index 0000000000..076f6a1527 --- /dev/null +++ b/crates/machine-controller/src/handler/host_boot_config.rs @@ -0,0 +1,542 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +//! Shared host boot-configuration convergence. +//! +//! The lifecycle state machines retain ownership of their persisted state +//! shapes. [`HostBootConfigStage`] is only a runtime projection of those +//! states, so this module can drive the common BIOS/job/poll/boot-order +//! sequence without introducing another serialized controller wrapper. +//! +//! [`run_host_boot_config_stage`] owns the lifecycle-neutral Redfish work. Its +//! four lifecycle adapters stay beside the state machines that own their +//! persisted state and terminal behavior: +//! - `handle_host_init_boot_config_stage` in `handler.rs`; +//! - `handle_instance_host_boot_config_stage` in `handler.rs`; +//! - `handle_dpu_reprovision_host_boot_config_stage` in `handler.rs`; +//! - `handle_validation_boot_config_stage` in `machine_validation.rs`. + +use carbide_redfish::boot_interface::BootInterfaceTarget; +use carbide_redfish::libredfish::error::state_handler_redfish_error as redfish_error; +use libredfish::Redfish; +use model::machine::{ + BiosConfigInfo, ManagedHostStateSnapshot, SetBootOrderInfo, SetBootOrderState, +}; +use state_controller::state_handler::{StateHandlerContext, StateHandlerError}; + +use super::bios_config::{ + BiosConfigJobAdvanceOutcome, BiosConfigOutcome, PollingBiosSetupOutcome, + advance_bios_config_job, advance_polling_bios_setup, configure_host_bios, +}; +use super::{ + ReachabilityParams, RequiredBootInterface, SetBootOrderOutcome, + are_dpus_up_trigger_reboot_if_needed, is_dpu_observed_since, load_boot_predictions, + log_host_config, require_boot_interface, set_host_boot_order, trigger_reboot_if_needed, +}; +use crate::context::MachineStateHandlerContextObjects; + +/// Runtime projection of the persisted BIOS and boot-order states shared by +/// HostInit, assigned platform configuration, DPU reprovision, and validation. +/// +/// This type must remain controller-internal: the owning lifecycle maps each +/// `Continue` outcome back to its existing serialized state variant. +#[derive(Debug, Clone, Eq, PartialEq)] +pub(super) enum HostBootConfigStage { + ConfigureBios { + retry_count: u32, + }, + WaitingForBiosJob { + bios_config_info: BiosConfigInfo, + }, + PollingBiosSetup { + retry_count: u32, + }, + SetBootOrder { + set_boot_order_info: SetBootOrderInfo, + }, +} + +/// Result of running one common boot-configuration stage. +#[derive(Debug, Clone, Eq, PartialEq)] +pub(super) enum HostBootConfigOutcome { + Continue(HostBootConfigStage), + Complete, + Wait(String), + Failed { failure: String }, +} + +/// The next remediation required after inspecting the desired and actual host +/// boot configuration. +#[derive(Debug, Clone, Eq, PartialEq)] +pub(super) enum HostBootConfigDecision { + /// The platform BIOS settings (including the HTTP boot device) drifted. + ConfigureBios, + /// BIOS is already correct; only boot order drifted. + SetBootOrder, + /// BIOS and every NICo-managed boot-order setting are already correct. + Complete, +} + +/// Result of pacing prerequisites and inspecting the host boot config. +#[derive(Debug, Clone, Eq, PartialEq)] +pub(super) enum HostBootConfigCheckOutcome { + Ready(HostBootConfigDecision), + /// The host cannot be inspected yet, for example while discovering a + /// zero-DPU host's boot NIC or waiting for a fresh DPU observation. + Wait(String), +} + +/// DPU observation freshness required before trusting Redfish boot state. +#[derive(Debug, Clone, Copy, Eq, PartialEq)] +pub(super) enum HostBootConfigDpuFreshness { + /// The caller already gated on healthy, synchronized DPUs. + AlreadyValidated, + /// Require observations associated with the current host lifecycle state. + CurrentHostState, + /// Require observations newer than the most recent host reboot request. + SinceLastHostRebootRequest, +} + +/// Actual host boot settings read from Redfish. +/// +/// `boot_order_setup` is `None` when BIOS drift already determines the next +/// action, or when boot order is intentionally not managed on this hardware. +#[derive(Debug, Clone, Copy, Eq, PartialEq)] +pub(super) struct HostBootConfigInspection { + pub is_bios_setup: bool, + pub is_boot_order_setup: Option, +} + +/// Convert an inspection into the smallest remediation that can converge it. +/// BIOS repair takes precedence because it may recreate the boot option whose +/// order would otherwise be inspected or changed. +pub(super) fn decide_host_boot_config( + inspection: HostBootConfigInspection, +) -> HostBootConfigDecision { + if !inspection.is_bios_setup { + HostBootConfigDecision::ConfigureBios + } else if inspection.is_boot_order_setup == Some(false) { + HostBootConfigDecision::SetBootOrder + } else { + HostBootConfigDecision::Complete + } +} + +/// Inspect a host whose boot interface has already been resolved. +/// +/// Keeping interface resolution outside this primitive lets validation retain +/// its existing distinction between a missing interface (hard error), an +/// undiscovered zero-DPU NIC (pace the reboot), and a Redfish read error (log +/// and retry). +pub(super) async fn inspect_host_boot_config( + redfish_client: &dyn Redfish, + mh_snapshot: &ManagedHostStateSnapshot, + boot_interface: &BootInterfaceTarget, +) -> Result { + let is_bios_setup = boot_interface + .run(|bi| redfish_client.is_bios_setup(Some(bi))) + .await + .map_err(|e| redfish_error("is_bios_setup", e))?; + + if !is_bios_setup || should_skip_boot_order_remediation(mh_snapshot) { + return Ok(HostBootConfigInspection { + is_bios_setup, + is_boot_order_setup: None, + }); + } + + let is_boot_order_setup = boot_interface + .run(|bi| redfish_client.is_boot_order_setup(bi)) + .await + .map_err(|e| redfish_error("is_boot_order_setup", e))?; + + Ok(HostBootConfigInspection { + is_bios_setup, + is_boot_order_setup: Some(is_boot_order_setup), + }) +} + +/// Check whether host BIOS or boot order needs remediation. +pub(super) async fn check_host_boot_config( + redfish_client: &dyn Redfish, + mh_snapshot: &ManagedHostStateSnapshot, + reachability_params: &ReachabilityParams, + dpu_freshness: HostBootConfigDpuFreshness, + ctx: &mut StateHandlerContext<'_, MachineStateHandlerContextObjects>, +) -> Result { + if should_wait_for_dpus_before_host_boot_config( + mh_snapshot, + reachability_params, + dpu_freshness, + ctx, + ) + .await + { + return Ok(HostBootConfigCheckOutcome::Wait( + "Waiting for DPUs to come up.".to_string(), + )); + } + + let predictions = load_boot_predictions(ctx, &mh_snapshot.host_snapshot.id).await?; + let boot_interface = match require_boot_interface( + mh_snapshot, + &predictions, + "configuring boot", + HostBootConfigCheckOutcome::Wait, + )? { + RequiredBootInterface::Ready(target) => target, + RequiredBootInterface::Wait(outcome) => return Ok(outcome), + }; + + log_host_config(redfish_client, mh_snapshot).await; + + let inspection = inspect_host_boot_config(redfish_client, mh_snapshot, &boot_interface).await?; + let decision = decide_host_boot_config(inspection); + let vendor = mh_snapshot.host_snapshot.bmc_vendor(); + + match &decision { + HostBootConfigDecision::ConfigureBios => tracing::warn!( + machine_id = %mh_snapshot.host_snapshot.id, + bmc_vendor = %vendor, + "Host BIOS setup is not configured properly" + ), + HostBootConfigDecision::SetBootOrder => tracing::warn!( + machine_id = %mh_snapshot.host_snapshot.id, + bmc_vendor = %vendor, + "Host BIOS setup is correct but boot order is not configured properly" + ), + HostBootConfigDecision::Complete if should_skip_boot_order_remediation(mh_snapshot) => { + tracing::info!( + machine_id = %mh_snapshot.host_snapshot.id, + bmc_vendor = %vendor, + "Host BIOS setup is configured; skipping boot order remediation on Viking" + ); + } + HostBootConfigDecision::Complete => tracing::info!( + machine_id = %mh_snapshot.host_snapshot.id, + bmc_vendor = %vendor, + "Host BIOS setup and boot order are configured properly" + ), + } + + Ok(HostBootConfigCheckOutcome::Ready(decision)) +} + +/// Runs one lifecycle-neutral host boot-configuration stage. +/// +/// Called by the host-init, assigned-instance, DPU-reprovision, and validation +/// adapters. This function performs the Redfish BIOS, vendor-job, polling, or +/// boot-order work for one [`HostBootConfigStage`] and returns a +/// [`HostBootConfigOutcome`]. The caller owns the persisted lifecycle state and +/// maps the outcome back into that state. +pub(super) async fn run_host_boot_config_stage( + ctx: &mut StateHandlerContext<'_, MachineStateHandlerContextObjects>, + reachability_params: &ReachabilityParams, + redfish_client: &dyn Redfish, + mh_snapshot: &ManagedHostStateSnapshot, + stage: HostBootConfigStage, +) -> Result { + match stage { + HostBootConfigStage::ConfigureBios { retry_count } => { + match configure_host_bios( + ctx, + reachability_params, + redfish_client, + mh_snapshot, + retry_count, + ) + .await? + { + BiosConfigOutcome::Done => Ok(HostBootConfigOutcome::Continue( + HostBootConfigStage::PollingBiosSetup { retry_count }, + )), + BiosConfigOutcome::WaitingForBiosJob(bios_config_info) => { + Ok(HostBootConfigOutcome::Continue( + HostBootConfigStage::WaitingForBiosJob { bios_config_info }, + )) + } + BiosConfigOutcome::WaitingForReboot(reason) => { + Ok(HostBootConfigOutcome::Wait(reason)) + } + } + } + HostBootConfigStage::WaitingForBiosJob { bios_config_info } => { + let retry_count = bios_config_info.retry_count; + match advance_bios_config_job(ctx, redfish_client, mh_snapshot, bios_config_info) + .await? + { + BiosConfigJobAdvanceOutcome::Continue(bios_config_info) => { + Ok(HostBootConfigOutcome::Continue( + HostBootConfigStage::WaitingForBiosJob { bios_config_info }, + )) + } + BiosConfigJobAdvanceOutcome::Done => Ok(HostBootConfigOutcome::Continue( + HostBootConfigStage::PollingBiosSetup { retry_count }, + )), + BiosConfigJobAdvanceOutcome::Failed { failure } => { + Ok(HostBootConfigOutcome::Failed { failure }) + } + BiosConfigJobAdvanceOutcome::Wait(reason) => { + Ok(HostBootConfigOutcome::Wait(reason)) + } + BiosConfigJobAdvanceOutcome::RetryPlatformConfiguration { retry_count } => { + Ok(HostBootConfigOutcome::Continue( + HostBootConfigStage::ConfigureBios { retry_count }, + )) + } + } + } + HostBootConfigStage::PollingBiosSetup { retry_count } => { + let predictions = load_boot_predictions(ctx, &mh_snapshot.host_snapshot.id).await?; + match advance_polling_bios_setup( + redfish_client, + mh_snapshot, + retry_count, + &ctx.services.site_config.machine_state_controller, + &predictions, + ) + .await? + { + PollingBiosSetupOutcome::Verified => { + if should_skip_boot_order_remediation(mh_snapshot) { + Ok(HostBootConfigOutcome::Complete) + } else { + Ok(HostBootConfigOutcome::Continue( + HostBootConfigStage::SetBootOrder { + set_boot_order_info: set_boot_order_info(retry_count), + }, + )) + } + } + PollingBiosSetupOutcome::Wait(reason) => Ok(HostBootConfigOutcome::Wait(reason)), + PollingBiosSetupOutcome::EnterRecovery(bios_config_info) => { + Ok(HostBootConfigOutcome::Continue( + HostBootConfigStage::WaitingForBiosJob { bios_config_info }, + )) + } + PollingBiosSetupOutcome::Failed { failure } => { + Ok(HostBootConfigOutcome::Failed { failure }) + } + } + } + HostBootConfigStage::SetBootOrder { + set_boot_order_info, + } => { + let retry_count = set_boot_order_info.retry_count; + match set_host_boot_order( + ctx, + reachability_params, + redfish_client, + mh_snapshot, + set_boot_order_info, + ) + .await? + { + SetBootOrderOutcome::Continue(set_boot_order_info) => Ok( + HostBootConfigOutcome::Continue(HostBootConfigStage::SetBootOrder { + set_boot_order_info, + }), + ), + SetBootOrderOutcome::ConfigureBios => { + let max_retries = ctx + .services + .site_config + .machine_state_controller + .max_bios_config_retries; + let Some(retry_count) = next_boot_config_retry_count(retry_count, max_retries) + else { + return Ok(HostBootConfigOutcome::Failed { + failure: format!( + "BIOS settings repeatedly drifted during boot-order repair; automated boot-config convergence exhausted after {max_retries} retries" + ), + }); + }; + + Ok(HostBootConfigOutcome::Continue( + HostBootConfigStage::ConfigureBios { retry_count }, + )) + } + SetBootOrderOutcome::Done => Ok(HostBootConfigOutcome::Complete), + SetBootOrderOutcome::WaitingForReboot(reason) + | SetBootOrderOutcome::Wait(reason) => Ok(HostBootConfigOutcome::Wait(reason)), + } + } + } +} + +pub(super) fn initial_set_boot_order_info() -> SetBootOrderInfo { + set_boot_order_info(0) +} + +fn set_boot_order_info(retry_count: u32) -> SetBootOrderInfo { + SetBootOrderInfo { + set_boot_order_jid: None, + set_boot_order_state: SetBootOrderState::SetBootOrder, + retry_count, + } +} + +/// Carry one bounded convergence budget when boot-order work discovers that +/// BIOS must be repaired again. Existing owner states already persist this +/// counter through both phases, so no serialized wrapper is required. +fn next_boot_config_retry_count(retry_count: u32, max_retries: u32) -> Option { + (retry_count < max_retries).then(|| retry_count + 1) +} + +/// Viking BMC firmware cannot safely run boot-order remediation; BIOS repair +/// still applies. +pub(super) fn should_skip_boot_order_remediation(mh_snapshot: &ManagedHostStateSnapshot) -> bool { + mh_snapshot + .host_snapshot + .hardware_info + .as_ref() + .is_some_and(|hw| hw.is_dgx_h100()) +} + +async fn should_wait_for_dpus_before_host_boot_config( + mh_snapshot: &ManagedHostStateSnapshot, + reachability_params: &ReachabilityParams, + dpu_freshness: HostBootConfigDpuFreshness, + ctx: &mut StateHandlerContext<'_, MachineStateHandlerContextObjects>, +) -> bool { + if !mh_snapshot.has_managed_dpus() { + return false; + } + + match dpu_freshness { + HostBootConfigDpuFreshness::AlreadyValidated => false, + HostBootConfigDpuFreshness::CurrentHostState => { + !are_dpus_up_trigger_reboot_if_needed(mh_snapshot, reachability_params, ctx).await + } + HostBootConfigDpuFreshness::SinceLastHostRebootRequest => { + let Some(last_reboot_requested) = mh_snapshot.host_snapshot.last_reboot_requested + else { + tracing::warn!( + machine_id = %mh_snapshot.host_snapshot.id, + "No host reboot request timestamp found before post-reboot host boot config check" + ); + return false; + }; + + for dpu_snapshot in &mh_snapshot.dpu_snapshots { + if !is_dpu_observed_since(dpu_snapshot, last_reboot_requested.time) { + match trigger_reboot_if_needed( + dpu_snapshot, + mh_snapshot, + None, + reachability_params, + ctx, + ) + .await + { + Ok(_) => {} + Err(e) => tracing::warn!( + dpu_id = %dpu_snapshot.id, + error = %e, + "Could not reboot DPU while waiting to check host boot config" + ), + } + return true; + } + } + + false + } + } +} + +#[cfg(test)] +mod tests { + use carbide_test_support::value_scenarios; + + use super::*; + + #[test] + fn boot_config_decision_chooses_the_smallest_remediation() { + value_scenarios!(decide_host_boot_config: + "BIOS drift takes precedence" { + HostBootConfigInspection { + is_bios_setup: false, + is_boot_order_setup: None, + } => HostBootConfigDecision::ConfigureBios, + HostBootConfigInspection { + is_bios_setup: false, + is_boot_order_setup: Some(false), + } => HostBootConfigDecision::ConfigureBios, + HostBootConfigInspection { + is_bios_setup: false, + is_boot_order_setup: Some(true), + } => HostBootConfigDecision::ConfigureBios, + } + + "configured BIOS chooses the narrowest remaining action" { + HostBootConfigInspection { + is_bios_setup: true, + is_boot_order_setup: Some(false), + } => HostBootConfigDecision::SetBootOrder, + HostBootConfigInspection { + is_bios_setup: true, + is_boot_order_setup: Some(true), + } => HostBootConfigDecision::Complete, + HostBootConfigInspection { + is_bios_setup: true, + is_boot_order_setup: None, + } => HostBootConfigDecision::Complete, + } + ); + } + + #[test] + fn boot_config_retry_count_is_bounded_across_bios_and_order() { + struct RetryCount { + retry_count: u32, + max_retries: u32, + } + + value_scenarios!( + run = |RetryCount { + retry_count, + max_retries, + }: RetryCount| next_boot_config_retry_count(retry_count, max_retries); + "retry budget remains" { + RetryCount { + retry_count: 0, + max_retries: 1, + } => Some(1), + RetryCount { + retry_count: 4, + max_retries: 5, + } => Some(5), + } + + "retry budget is unavailable" { + RetryCount { + retry_count: 1, + max_retries: 1, + } => None, + RetryCount { + retry_count: 0, + max_retries: 0, + } => None, + RetryCount { + retry_count: 4, + max_retries: 1, + } => None, + } + ); + } +} diff --git a/crates/machine-controller/src/handler/machine_validation.rs b/crates/machine-controller/src/handler/machine_validation.rs index 79f975d0cd..d29cfab079 100644 --- a/crates/machine-controller/src/handler/machine_validation.rs +++ b/crates/machine-controller/src/handler/machine_validation.rs @@ -15,10 +15,11 @@ * limitations under the License. */ use carbide_uuid::machine_validation::MachineValidationId; -use libredfish::SystemPowerControl; +use chrono::Utc; +use libredfish::{EnabledDisabled, RedfishError, SystemPowerControl}; use model::machine::{ - FailureCause, MachineState, MachineValidatingState, ManagedHostState, ManagedHostStateSnapshot, - ValidationState, + FailureCause, FailureDetails, FailureSource, MachineState, MachineValidatingState, + ManagedHostState, ManagedHostStateSnapshot, StateMachineArea, UnlockHostState, ValidationState, }; use model::machine_validation::{MachineValidationState, MachineValidationStatus}; use state_controller::state_handler::{ @@ -27,7 +28,123 @@ use state_controller::state_handler::{ use super::{HostHandlerParams, is_machine_validation_requested, machine_validation_completed}; use crate::context::{MachineStateHandlerContextObjects, MachineStateHandlerServices}; -use crate::handler::{handler_host_power_control, rebooted, trigger_reboot_if_needed}; +use crate::handler::host_boot_config::{ + HostBootConfigCheckOutcome, HostBootConfigDecision, HostBootConfigDpuFreshness, + HostBootConfigOutcome, HostBootConfigStage, check_host_boot_config, decide_host_boot_config, + initial_set_boot_order_info, inspect_host_boot_config, run_host_boot_config_stage, +}; +use crate::handler::{ + RequiredBootInterface, handler_host_power_control, host_power_control, load_boot_predictions, + rebooted, redfish_error, require_boot_interface, trigger_reboot_if_needed, wait, +}; + +/// The validation flavor of the managed-host state, so the repair arms can +/// transition between validation substates without repeating the wrapper. +fn validating(machine_validation: MachineValidatingState) -> ManagedHostState { + ManagedHostState::Validation { + validation_state: ValidationState::MachineValidation { machine_validation }, + } +} + +/// Handles a shared boot-configuration stage for machine validation. +/// +/// Called by `handle_machine_validation_state` for its BIOS configuration, +/// vendor-job, polling, and boot-order repair substates. This adapter preserves +/// the validation ID, maps continued stages back into `MachineValidatingState`, +/// continues to `LockAfterBootRepair` on completion, and closes the validation +/// run before reporting a terminal boot-configuration failure. +async fn handle_validation_boot_config_stage( + ctx: &mut StateHandlerContext<'_, MachineStateHandlerContextObjects>, + host_handler_params: &HostHandlerParams, + mh_snapshot: &ManagedHostStateSnapshot, + validation_id: MachineValidationId, + stage: HostBootConfigStage, +) -> Result, StateHandlerError> { + let redfish_client = ctx + .services + .create_redfish_client_from_machine(&mh_snapshot.host_snapshot) + .await?; + + match run_host_boot_config_stage( + ctx, + &host_handler_params.reachability_params, + redfish_client.as_ref(), + mh_snapshot, + stage, + ) + .await? + { + HostBootConfigOutcome::Continue(stage) => { + let machine_validation = match stage { + HostBootConfigStage::ConfigureBios { retry_count } => { + MachineValidatingState::ConfigureBootBios { + validation_id, + retry_count, + } + } + HostBootConfigStage::WaitingForBiosJob { bios_config_info } => { + MachineValidatingState::WaitingForBootBiosJob { + validation_id, + bios_config_info, + } + } + HostBootConfigStage::PollingBiosSetup { retry_count } => { + MachineValidatingState::PollingBootBiosSetup { + validation_id, + retry_count, + } + } + HostBootConfigStage::SetBootOrder { + set_boot_order_info, + } => MachineValidatingState::RepairBootConfig { + validation_id, + set_boot_order_info, + }, + }; + + Ok(StateHandlerOutcome::transition(validating( + machine_validation, + ))) + } + HostBootConfigOutcome::Complete => Ok(StateHandlerOutcome::transition(validating( + MachineValidatingState::LockAfterBootRepair { validation_id }, + ))), + HostBootConfigOutcome::Wait(reason) => Ok(StateHandlerOutcome::wait(reason)), + HostBootConfigOutcome::Failed { failure } => { + let machine_id = mh_snapshot.host_snapshot.id; + let mut txn = ctx.services.db_pool.begin().await?; + let completed = db::machine_validation::mark_machine_validation_complete( + txn.as_mut(), + &machine_id, + &validation_id, + MachineValidationStatus { + state: MachineValidationState::Failed, + ..MachineValidationStatus::default() + }, + ) + .await?; + + if !completed { + tracing::info!( + %machine_id, + %validation_id, + "Machine validation boot-config failure observed after the run was already terminal" + ); + } + + Ok(StateHandlerOutcome::transition(ManagedHostState::Failed { + details: FailureDetails { + cause: FailureCause::BiosSetupFailed { err: failure }, + failed_at: Utc::now(), + source: FailureSource::StateMachineArea(StateMachineArea::MainFlow), + }, + machine_id, + retry_count: 0, + }) + .with_txn(txn)) + } + } +} async fn skip_machine_validation( ctx: &mut StateHandlerContext<'_, MachineStateHandlerContextObjects>, @@ -122,6 +239,55 @@ pub(crate) async fn handle_machine_validation_state( is_enabled, ); if !rebooted(&mh_snapshot.host_snapshot) { + // Ensure the boot config is still what it should be while + // waiting. If it reads reverted -- changed externally, a BIOS + // quirk, or the boot NIC dropping off the BMC's inventory + // during the reboot's POST -- the host can't boot into the + // validation environment, and further reboots can't restore a + // setting that is gone. Correct it instead of pacing forever. + let predictions = load_boot_predictions(ctx, &mh_snapshot.host_snapshot.id).await?; + if let RequiredBootInterface::Ready(boot_interface) = require_boot_interface( + mh_snapshot, + &predictions, + "verifying the boot config during validation", + |message| message, + )? { + let redfish_client = ctx + .services + .create_redfish_client_from_machine(&mh_snapshot.host_snapshot) + .await?; + match inspect_host_boot_config( + redfish_client.as_ref(), + mh_snapshot, + &boot_interface, + ) + .await + { + Ok(inspection) + if matches!( + decide_host_boot_config(inspection), + HostBootConfigDecision::ConfigureBios + | HostBootConfigDecision::SetBootOrder + ) => + { + tracing::warn!( + machine_id = %mh_snapshot.host_snapshot.id, + "Boot config reads reverted while waiting for the validation reboot; correcting it via boot repair", + ); + return Ok(StateHandlerOutcome::transition(validating( + MachineValidatingState::PrepareBootRepair { validation_id: *id }, + ))); + } + Ok(_) => {} + Err(e) => { + tracing::warn!( + machine_id = %mh_snapshot.host_snapshot.id, + error = %e, + "Could not verify the boot config while waiting for the validation reboot; will retry", + ); + } + } + } let status = trigger_reboot_if_needed( &mh_snapshot.host_snapshot, mh_snapshot, @@ -174,6 +340,264 @@ pub(crate) async fn handle_machine_validation_state( } Ok(StateHandlerOutcome::do_nothing()) } + MachineValidatingState::PrepareBootRepair { validation_id } => { + // Boot repair writes BIOS settings, and lockdown was enabled + // earlier in HostInit -- writes bounce off a locked BMC. Check and + // unlock first, mirroring host boot repair. + let redfish_client = ctx + .services + .create_redfish_client_from_machine(&mh_snapshot.host_snapshot) + .await?; + + let next = match redfish_client.lockdown_status().await { + Err(RedfishError::NotSupported(_)) => { + tracing::info!( + machine_id = %mh_snapshot.host_snapshot.id, + "BMC vendor does not support checking lockdown status during validation boot repair", + ); + MachineValidatingState::CheckBootConfigForRepair { + validation_id: *validation_id, + } + } + Err(e) => { + tracing::warn!( + machine_id = %mh_snapshot.host_snapshot.id, + error = %e, + "Failed to fetch lockdown status during validation boot repair", + ); + return Ok(StateHandlerOutcome::wait(format!( + "Failed to fetch lockdown status: {e}" + ))); + } + Ok(lockdown_status) if !lockdown_status.is_fully_disabled() => { + tracing::info!( + machine_id = %mh_snapshot.host_snapshot.id, + "Lockdown is enabled during validation boot repair; disabling before boot config writes", + ); + MachineValidatingState::UnlockForBootRepair { + validation_id: *validation_id, + unlock_host_state: UnlockHostState::DisableLockdown, + } + } + Ok(_) => MachineValidatingState::CheckBootConfigForRepair { + validation_id: *validation_id, + }, + }; + + Ok(StateHandlerOutcome::transition(validating(next))) + } + MachineValidatingState::UnlockForBootRepair { + validation_id, + unlock_host_state, + } => { + // Mirror host boot repair's unlock choreography. + let redfish_client = ctx + .services + .create_redfish_client_from_machine(&mh_snapshot.host_snapshot) + .await?; + + let next = match unlock_host_state { + UnlockHostState::DisableLockdown => { + // Tolerate a vendor that reports lockdown status but does + // not support setting it, symmetric with the re-lock step. + match redfish_client.lockdown_bmc(EnabledDisabled::Disabled).await { + Ok(()) => {} + Err(RedfishError::NotSupported(_)) => { + tracing::info!( + machine_id = %mh_snapshot.host_snapshot.id, + "BMC vendor does not support disabling lockdown during validation boot repair", + ); + } + Err(e) => return Err(redfish_error("lockdown_bmc", e)), + } + + let vendor = mh_snapshot.host_snapshot.bmc_vendor(); + if vendor.is_supermicro() { + tracing::info!( + machine_id = %mh_snapshot.host_snapshot.id, + %vendor, + "BMC lockdown disabled; rebooting host so Redfish reflects actual boot state", + ); + MachineValidatingState::UnlockForBootRepair { + validation_id: *validation_id, + unlock_host_state: UnlockHostState::RebootHost, + } + } else { + MachineValidatingState::CheckBootConfigForRepair { + validation_id: *validation_id, + } + } + } + UnlockHostState::RebootHost => { + host_power_control( + redfish_client.as_ref(), + &mh_snapshot.host_snapshot, + SystemPowerControl::ForceRestart, + ctx, + ) + .await + .map_err(|e| { + StateHandlerError::GenericError(eyre::eyre!( + "failed to ForceRestart host after disabling BMC lockdown: {}", + e + )) + })?; + + MachineValidatingState::UnlockForBootRepair { + validation_id: *validation_id, + unlock_host_state: UnlockHostState::WaitForUefiBoot, + } + } + UnlockHostState::WaitForUefiBoot => { + let entered_at = mh_snapshot.host_snapshot.state.version.timestamp(); + if wait( + &entered_at, + host_handler_params.reachability_params.uefi_boot_wait, + ) { + return Ok(StateHandlerOutcome::wait(format!( + "Waiting for UEFI boot to complete on {} after post-unlock reboot", + mh_snapshot.host_snapshot.id + ))); + } + MachineValidatingState::CheckBootConfigForRepair { + validation_id: *validation_id, + } + } + }; + + Ok(StateHandlerOutcome::transition(validating(next))) + } + MachineValidatingState::CheckBootConfigForRepair { validation_id } => { + let redfish_client = ctx + .services + .create_redfish_client_from_machine(&mh_snapshot.host_snapshot) + .await?; + + let next = match check_host_boot_config( + redfish_client.as_ref(), + mh_snapshot, + &host_handler_params.reachability_params, + HostBootConfigDpuFreshness::CurrentHostState, + ctx, + ) + .await? + { + HostBootConfigCheckOutcome::Wait(reason) => { + return Ok(StateHandlerOutcome::wait(reason)); + } + HostBootConfigCheckOutcome::Ready(HostBootConfigDecision::ConfigureBios) => { + MachineValidatingState::ConfigureBootBios { + validation_id: *validation_id, + retry_count: 0, + } + } + HostBootConfigCheckOutcome::Ready(HostBootConfigDecision::SetBootOrder) => { + MachineValidatingState::RepairBootConfig { + validation_id: *validation_id, + set_boot_order_info: initial_set_boot_order_info(), + } + } + HostBootConfigCheckOutcome::Ready(HostBootConfigDecision::Complete) => { + MachineValidatingState::LockAfterBootRepair { + validation_id: *validation_id, + } + } + }; + + Ok(StateHandlerOutcome::transition(validating(next))) + } + MachineValidatingState::ConfigureBootBios { + validation_id, + retry_count, + } => { + handle_validation_boot_config_stage( + ctx, + host_handler_params, + mh_snapshot, + *validation_id, + HostBootConfigStage::ConfigureBios { + retry_count: *retry_count, + }, + ) + .await + } + MachineValidatingState::WaitingForBootBiosJob { + validation_id, + bios_config_info, + } => { + handle_validation_boot_config_stage( + ctx, + host_handler_params, + mh_snapshot, + *validation_id, + HostBootConfigStage::WaitingForBiosJob { + bios_config_info: bios_config_info.clone(), + }, + ) + .await + } + MachineValidatingState::PollingBootBiosSetup { + validation_id, + retry_count, + } => { + handle_validation_boot_config_stage( + ctx, + host_handler_params, + mh_snapshot, + *validation_id, + HostBootConfigStage::PollingBiosSetup { + retry_count: *retry_count, + }, + ) + .await + } + MachineValidatingState::RepairBootConfig { + validation_id, + set_boot_order_info, + } => { + handle_validation_boot_config_stage( + ctx, + host_handler_params, + mh_snapshot, + *validation_id, + HostBootConfigStage::SetBootOrder { + set_boot_order_info: set_boot_order_info.clone(), + }, + ) + .await + } + MachineValidatingState::LockAfterBootRepair { validation_id } => { + // Restore the lockdown that boot repair temporarily opened, then + // resume validation from its reboot step. + let redfish_client = ctx + .services + .create_redfish_client_from_machine(&mh_snapshot.host_snapshot) + .await?; + + if mh_snapshot.host_snapshot.host_profile.disable_lockdown { + tracing::info!( + machine_id = %mh_snapshot.host_snapshot.id, + "Skipping lockdown re-enable after validation boot repair per expected-machine config", + ); + } else { + match redfish_client.lockdown_bmc(EnabledDisabled::Enabled).await { + Ok(()) => {} + Err(RedfishError::NotSupported(_)) => { + tracing::info!( + machine_id = %mh_snapshot.host_snapshot.id, + "BMC vendor does not support re-enabling lockdown after validation boot repair", + ); + } + Err(e) => return Err(redfish_error("lockdown_bmc", e)), + } + } + + Ok(StateHandlerOutcome::transition(validating( + MachineValidatingState::RebootHost { + validation_id: *validation_id, + }, + ))) + } } } diff --git a/crates/machine-controller/src/io.rs b/crates/machine-controller/src/io.rs index d514116ff1..a80411c47b 100644 --- a/crates/machine-controller/src/io.rs +++ b/crates/machine-controller/src/io.rs @@ -264,6 +264,16 @@ impl StateControllerIO for MachineStateControllerIO { match validation_state { MachineValidatingState::MachineValidating { .. } => "machinevalidating", MachineValidatingState::RebootHost { .. } => "reboothost", + MachineValidatingState::PrepareBootRepair { .. } => "preparebootrepair", + MachineValidatingState::UnlockForBootRepair { .. } => "unlockforbootrepair", + MachineValidatingState::CheckBootConfigForRepair { .. } => { + "checkbootconfigforrepair" + } + MachineValidatingState::ConfigureBootBios { .. } => "configurebootbios", + MachineValidatingState::WaitingForBootBiosJob { .. } => "waitingforbootbiosjob", + MachineValidatingState::PollingBootBiosSetup { .. } => "pollingbootbiossetup", + MachineValidatingState::RepairBootConfig { .. } => "repairbootconfig", + MachineValidatingState::LockAfterBootRepair { .. } => "lockafterbootrepair", } } match state {