diff --git a/crates/api-core/src/cfg/README.md b/crates/api-core/src/cfg/README.md index 94dc2f9e1e..1e017e44e6 100644 --- a/crates/api-core/src/cfg/README.md +++ b/crates/api-core/src/cfg/README.md @@ -58,7 +58,7 @@ applicable. | `ib_partition_state_controller` | `IbPartitionStateControllerConfig` | *(see below)* | `hardware` | IB partition state controller timing. | | `dpa_interface_state_controller` | `DpaInterfaceStateControllerConfig` | *(see below)* | `networking` | DPA interface state controller timing. | | `rack_state_controller` | `RackStateControllerConfig` | *(see below)* | `hardware` | Rack state controller timing. | -| `power_shelf_state_controller` | `PowerShelfStateControllerConfig` | *(see below)* | `hardware` | Power shelf state controller timing. | +| `power_shelf_state_controller` | `PowerShelfStateControllerConfig` | *(see below)* | `hardware` | Power shelf state controller timing and optional rack firmware reprovisioning. | | `switch_state_controller` | `SwitchStateControllerConfig` | *(see below)* | `hardware` | Switch state controller timing. | | `spdm_state_controller` | `SpdmStateControllerConfig` | *(see below)* | `security` | SPDM state controller timing. | | `host_models` | `HashMap` | `{}` | `machines` | Maps host model identifiers to firmware definitions for BMC/UEFI/NIC upgrades. | diff --git a/crates/api-core/src/cfg/file.rs b/crates/api-core/src/cfg/file.rs index edb6b8dbc4..8554fc56d6 100644 --- a/crates/api-core/src/cfg/file.rs +++ b/crates/api-core/src/cfg/file.rs @@ -2309,6 +2309,22 @@ pub struct PowerShelfStateControllerConfig { /// Common state controller configs #[serde(default = "StateControllerConfig::default")] pub controller: StateControllerConfig, + + /// When `true`, the power shelf Ready handler accepts rack-level + /// `power_shelf_reprovisioning_requested` and enters + /// `ReProvisioning::WaitingForRackFirmwareUpgrade`. + /// + /// Defaults to `false` so power shelves stay out of rack firmware wait + /// unless explicitly enabled. + /// + /// Configured in `nico-api-config.toml`: + /// + /// ```toml + /// [power_shelf_state_controller] + /// rack_firmware_reprovisioning_enabled = true + /// ``` + #[serde(default)] + pub rack_firmware_reprovisioning_enabled: bool, } /// RackStateController related config diff --git a/crates/api-core/src/handlers/component_manager.rs b/crates/api-core/src/handlers/component_manager.rs index ede6ca557c..21dcbcc823 100644 --- a/crates/api-core/src/handlers/component_manager.rs +++ b/crates/api-core/src/handlers/component_manager.rs @@ -652,6 +652,7 @@ struct RackFirmwareMaintenanceTarget { rack_id: RackId, machine_ids: Vec, switch_ids: Vec, + power_shelf_ids: Vec, } fn push_rack_firmware_target( @@ -659,6 +660,7 @@ fn push_rack_firmware_target( rack_id: RackId, machine_id: Option, switch_id: Option, + power_shelf_id: Option, ) { let target = match targets.iter_mut().find(|target| target.rack_id == rack_id) { Some(target) => target, @@ -667,6 +669,7 @@ fn push_rack_firmware_target( rack_id, machine_ids: Vec::new(), switch_ids: Vec::new(), + power_shelf_ids: Vec::new(), }); targets.last_mut().expect("target was just pushed") } @@ -678,6 +681,9 @@ fn push_rack_firmware_target( if let Some(switch_id) = switch_id { target.switch_ids.push(switch_id); } + if let Some(power_shelf_id) = power_shelf_id { + target.power_shelf_ids.push(power_shelf_id); + } } async fn group_machine_ids_by_rack( @@ -706,7 +712,13 @@ async fn group_machine_ids_by_rack( "machine {machine_id} is not associated with a rack" )) })?; - push_rack_firmware_target(&mut targets, rack_id, Some(machine_id.to_string()), None); + push_rack_firmware_target( + &mut targets, + rack_id, + Some(machine_id.to_string()), + None, + None, + ); } Ok(targets) @@ -853,7 +865,57 @@ async fn group_switch_ids_by_rack( let rack_id = switch.rack_id.clone().ok_or_else(|| { Status::failed_precondition(format!("switch {switch_id} is not associated with a rack")) })?; - push_rack_firmware_target(&mut targets, rack_id, None, Some(switch_id.to_string())); + push_rack_firmware_target( + &mut targets, + rack_id, + None, + Some(switch_id.to_string()), + None, + ); + } + + Ok(targets) +} + +async fn group_power_shelf_ids_by_rack( + api: &Api, + power_shelf_ids: &[PowerShelfId], +) -> Result, Status> { + let mut txn = api + .database_connection + .begin() + .await + .map_err(|e| Status::internal(format!("failed to begin transaction: {e}")))?; + let power_shelves = db::power_shelf::find_by( + &mut txn, + db::ObjectColumnFilter::List(db::power_shelf::IdColumn, power_shelf_ids), + ) + .await + .map_err(|e| Status::internal(format!("failed to look up power shelves: {e}")))?; + drop(txn); + + let power_shelves_by_id: HashMap<_, _> = power_shelves + .into_iter() + .map(|power_shelf| (power_shelf.id, power_shelf)) + .collect(); + + let mut targets = Vec::new(); + for power_shelf_id in power_shelf_ids { + let power_shelf = power_shelves_by_id + .get(power_shelf_id) + .ok_or_else(|| Status::not_found(format!("power shelf {power_shelf_id} not found")))?; + let rack_id = power_shelf.rack_id.clone().ok_or_else(|| { + Status::failed_precondition(format!( + "power shelf {power_shelf_id} is not associated with a rack" + )) + })?; + push_rack_firmware_target( + &mut targets, + rack_id, + None, + None, + Some(power_shelf_id.to_string()), + ); } Ok(targets) @@ -881,6 +943,7 @@ async fn submit_rack_firmware_maintenance_requests( .machine_ids .iter() .chain(target.switch_ids.iter()) + .chain(target.power_shelf_ids.iter()) .cloned() .collect(); let maintenance_req = Request::new(rpc::RackMaintenanceOnDemandRequest { @@ -888,7 +951,7 @@ async fn submit_rack_firmware_maintenance_requests( scope: Some(rpc::RackMaintenanceScope { machine_ids: target.machine_ids, switch_ids: target.switch_ids, - power_shelf_ids: vec![], + power_shelf_ids: target.power_shelf_ids, activities: activities.clone(), }), }); @@ -2345,54 +2408,70 @@ pub(crate) async fn update_component_firmware( let route_through_state_controller = cm.power_shelf_use_state_controller && !bypass_state_controller; if route_through_state_controller { - // TODO: implement state controller path for power shelf firmware control - return Err(Status::unimplemented( - "power shelf firmware control through the state controller is not yet supported", - )); - } - - let options = if cm.power_shelf.supports_firmware_object_json() { - require_firmware_object_json_for_direct_rms( + let token = require_firmware_object_json_for_rack_maintenance( "power shelf", &access_token, &req.target_version, + )?; + let components = map_power_shelf_components(&t.components)?; + let component_names = components + .iter() + .map(|component| match component { + PowerShelfComponent::Pmc => "pmc".to_string(), + PowerShelfComponent::Psu => "psu".to_string(), + }) + .collect(); + maintenance_activities = vec![firmware_upgrade_activity( + req.target_version.clone(), + component_names, + Some(token), force_update, - )? + )]; + rack_maintenance_targets = group_power_shelf_ids_by_rack(api, &list.ids).await?; } else { - reject_power_shelf_firmware_object_json(&access_token)?; - FirmwareUpdateOptions { - force_update, - ..FirmwareUpdateOptions::default() - } - }; - let components = map_power_shelf_components(&t.components)?; - let endpoints = resolve_power_shelf_endpoints(api, &list.ids).await?; + let options = if cm.power_shelf.supports_firmware_object_json() { + require_firmware_object_json_for_direct_rms( + "power shelf", + &access_token, + &req.target_version, + force_update, + )? + } else { + reject_power_shelf_firmware_object_json(&access_token)?; + FirmwareUpdateOptions { + force_update, + ..FirmwareUpdateOptions::default() + } + }; + let components = map_power_shelf_components(&t.components)?; + let endpoints = resolve_power_shelf_endpoints(api, &list.ids).await?; - let mut results: Vec<_> = endpoints - .unresolved - .iter() - .map(|u| error_result(&u.id.to_string(), u.reason.clone())) - .collect(); + let mut results: Vec<_> = endpoints + .unresolved + .iter() + .map(|u| error_result(&u.id.to_string(), u.reason.clone())) + .collect(); - let backend_results = cm - .power_shelf - .update_firmware( - &endpoints.resolved.endpoints, - &req.target_version, - &components, - &options, - ) - .await - .map_err(component_manager_error_to_status)?; - results.extend(backend_results.into_iter().map(|r| { - let id = ps_mac_to_id_str(&r.pmc_mac, &endpoints.resolved.mac_to_id); - if r.success { - success_result(&id) - } else { - error_result(&id, r.error.unwrap_or_default()) - } - })); - power_shelf_results = Some(results); + let backend_results = cm + .power_shelf + .update_firmware( + &endpoints.resolved.endpoints, + &req.target_version, + &components, + &options, + ) + .await + .map_err(component_manager_error_to_status)?; + results.extend(backend_results.into_iter().map(|r| { + let id = ps_mac_to_id_str(&r.pmc_mac, &endpoints.resolved.mac_to_id); + if r.success { + success_result(&id) + } else { + error_result(&id, r.error.unwrap_or_default()) + } + })); + power_shelf_results = Some(results); + } } rpc::update_component_firmware_request::Target::Racks(t) => { if bypass_state_controller { @@ -2952,17 +3031,37 @@ mod tests { let rack_b = RackId::new("rack-b".to_string()); let mut targets = Vec::new(); - push_rack_firmware_target(&mut targets, rack_a.clone(), Some("machine-a".into()), None); - push_rack_firmware_target(&mut targets, rack_b.clone(), None, Some("switch-b".into())); - push_rack_firmware_target(&mut targets, rack_a.clone(), Some("machine-c".into()), None); + push_rack_firmware_target( + &mut targets, + rack_a.clone(), + Some("machine-a".into()), + None, + None, + ); + push_rack_firmware_target( + &mut targets, + rack_b.clone(), + None, + Some("switch-b".into()), + None, + ); + push_rack_firmware_target( + &mut targets, + rack_a.clone(), + Some("machine-c".into()), + None, + None, + ); assert_eq!(targets.len(), 2); assert_eq!(targets[0].rack_id, rack_a); assert_eq!(targets[0].machine_ids, vec!["machine-a", "machine-c"]); assert!(targets[0].switch_ids.is_empty()); + assert!(targets[0].power_shelf_ids.is_empty()); assert_eq!(targets[1].rack_id, rack_b); assert_eq!(targets[1].switch_ids, vec!["switch-b"]); assert!(targets[1].machine_ids.is_empty()); + assert!(targets[1].power_shelf_ids.is_empty()); } #[test] diff --git a/crates/api-core/src/setup.rs b/crates/api-core/src/setup.rs index 5d1760b76e..29d5709cb0 100644 --- a/crates/api-core/src/setup.rs +++ b/crates/api-core/src/setup.rs @@ -1444,6 +1444,9 @@ async fn initialize_and_start_controllers<'a>( component_manager: component_manager.clone().map(Arc::new), credential_manager: credential_manager.clone(), per_object_metrics_registry: per_object_metrics_registry.clone(), + rack_firmware_reprovisioning_enabled: carbide_config + .power_shelf_state_controller + .rack_firmware_reprovisioning_enabled, } .into(), ) diff --git a/crates/api-core/src/test_support/default_config.rs b/crates/api-core/src/test_support/default_config.rs index 41981c7b91..a614f9ce48 100644 --- a/crates/api-core/src/test_support/default_config.rs +++ b/crates/api-core/src/test_support/default_config.rs @@ -186,6 +186,7 @@ pub fn get() -> CarbideConfig { controller: StateControllerConfig::default(), }, power_shelf_state_controller: PowerShelfStateControllerConfig { + rack_firmware_reprovisioning_enabled: false, controller: StateControllerConfig::default(), }, rack_state_controller: RackStateControllerConfig { diff --git a/crates/api-core/src/tests/common/api_fixtures/mod.rs b/crates/api-core/src/tests/common/api_fixtures/mod.rs index 39ba50c9c0..081cb84c02 100644 --- a/crates/api-core/src/tests/common/api_fixtures/mod.rs +++ b/crates/api-core/src/tests/common/api_fixtures/mod.rs @@ -1655,6 +1655,7 @@ pub async fn create_test_env_with_overrides( component_manager: test_component_manager.clone(), credential_manager: credential_manager.clone(), per_object_metrics_registry: per_object_metrics_registry.clone(), + rack_firmware_reprovisioning_enabled: false, } .into(), ) diff --git a/crates/api-core/src/tests/power_shelf_state_controller/error_state.rs b/crates/api-core/src/tests/power_shelf_state_controller/error_state.rs index 3f58993089..53f60df299 100644 --- a/crates/api-core/src/tests/power_shelf_state_controller/error_state.rs +++ b/crates/api-core/src/tests/power_shelf_state_controller/error_state.rs @@ -66,6 +66,7 @@ async fn services( component_manager, credential_manager: Arc::new(TestCredentialManager::default()), per_object_metrics_registry: env.per_object_metrics_registry(), + rack_firmware_reprovisioning_enabled: false, } } diff --git a/crates/api-core/src/tests/power_shelf_state_controller/maintenance.rs b/crates/api-core/src/tests/power_shelf_state_controller/maintenance.rs index 0c80853c0b..89ecc541bc 100644 --- a/crates/api-core/src/tests/power_shelf_state_controller/maintenance.rs +++ b/crates/api-core/src/tests/power_shelf_state_controller/maintenance.rs @@ -144,6 +144,7 @@ fn services_with_component_manager( password: TEST_BMC_PASSWORD.into(), })), per_object_metrics_registry: env.per_object_metrics_registry(), + rack_firmware_reprovisioning_enabled: false, } } diff --git a/crates/api-core/src/tests/power_shelf_state_controller/mod.rs b/crates/api-core/src/tests/power_shelf_state_controller/mod.rs index 28bceeeafa..f6aea1a2cb 100644 --- a/crates/api-core/src/tests/power_shelf_state_controller/mod.rs +++ b/crates/api-core/src/tests/power_shelf_state_controller/mod.rs @@ -33,6 +33,7 @@ use crate::tests::common::api_fixtures::create_test_env; mod error_state; mod fixtures; mod maintenance; +mod reprovisioning; use carbide_secrets::test_support::credentials::TestCredentialManager; use fixtures::power_shelf::{mark_power_shelf_as_deleted, set_power_shelf_controller_state}; @@ -130,6 +131,7 @@ async fn test_power_shelf_deletion_with_state_controller( component_manager: None, credential_manager: credential_manager.clone(), per_object_metrics_registry: env.per_object_metrics_registry(), + rack_firmware_reprovisioning_enabled: false, } .into(), ) diff --git a/crates/api-core/src/tests/power_shelf_state_controller/reprovisioning.rs b/crates/api-core/src/tests/power_shelf_state_controller/reprovisioning.rs new file mode 100644 index 0000000000..4568f054e9 --- /dev/null +++ b/crates/api-core/src/tests/power_shelf_state_controller/reprovisioning.rs @@ -0,0 +1,226 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +//! Tests for power-shelf rack firmware reprovisioning. + +use std::sync::Arc; + +use carbide_power_shelf_controller::context::{ + PowerShelfStateHandlerContextObjects, PowerShelfStateHandlerServices, +}; +use carbide_power_shelf_controller::handler::PowerShelfStateHandler; +use carbide_power_shelf_controller::metrics::PowerShelfMetrics; +use carbide_secrets::test_support::credentials::TestCredentialManager; +use carbide_uuid::power_shelf::PowerShelfId; +use db::power_shelf as db_power_shelf; +use model::power_shelf::{PowerShelf, PowerShelfControllerState, ReProvisioningState}; +use model::rack::MaintenanceActivity; +use sqlx::PgConnection; +use state_controller::db_write_batch::DbWriteBatch; +use state_controller::state_handler::{StateHandler, StateHandlerContext, StateHandlerOutcome}; + +use crate::tests::common::api_fixtures::create_test_env; +use crate::tests::common::api_fixtures::site_explorer::new_power_shelf; +use crate::tests::power_shelf_state_controller::fixtures::power_shelf::set_power_shelf_controller_state; + +fn firmware_only_activities() -> Vec { + vec![MaintenanceActivity::FirmwareUpgrade { + firmware_version: None, + components: vec![], + force_update: false, + }] +} + +fn services( + env: &crate::tests::common::api_fixtures::TestEnv, + rack_firmware_reprovisioning_enabled: bool, +) -> PowerShelfStateHandlerServices { + PowerShelfStateHandlerServices { + db_pool: env.pool.clone(), + component_manager: None, + credential_manager: Arc::new(TestCredentialManager::default()), + per_object_metrics_registry: env.per_object_metrics_registry(), + rack_firmware_reprovisioning_enabled, + } +} + +async fn load_power_shelf(pool: &sqlx::PgPool, id: &PowerShelfId) -> PowerShelf { + let mut conn = pool.acquire().await.unwrap(); + db_power_shelf::find_by_id(conn.as_mut(), id) + .await + .unwrap() + .expect("power shelf should exist") +} + +async fn run_handler( + services: &mut PowerShelfStateHandlerServices, + state: &mut PowerShelf, +) -> StateHandlerOutcome { + let handler = PowerShelfStateHandler::default(); + let mut metrics = PowerShelfMetrics::default(); + let mut writes = DbWriteBatch::default(); + let mut ctx = StateHandlerContext:: { + services, + metrics: &mut metrics, + pending_db_writes: &mut writes, + }; + let controller_state = state.controller_state.value.clone(); + let power_shelf_id = state.id; + handler + .handle_object_state(&power_shelf_id, state, &controller_state, &mut ctx) + .await + .expect("state handler should not return an error result") +} + +async fn commit_outcome(mut outcome: StateHandlerOutcome) { + if let Some(txn) = outcome.take_transaction() { + txn.commit().await.unwrap(); + } +} + +async fn park_ready(txn: &mut PgConnection, power_shelf_id: &PowerShelfId) { + set_power_shelf_controller_state(txn, power_shelf_id, PowerShelfControllerState::Ready) + .await + .expect("set Ready"); +} + +#[crate::sqlx_test] +async fn test_ready_clears_reprovision_request_when_flag_disabled( + pool: sqlx::PgPool, +) -> Result<(), Box> { + let env = create_test_env(pool.clone()).await; + let power_shelf_id = new_power_shelf(&env, None, None, None, None).await?; + + let mut txn = pool.begin().await?; + park_ready(txn.as_mut(), &power_shelf_id).await; + db_power_shelf::set_power_shelf_reprovisioning_requested( + txn.as_mut(), + power_shelf_id, + "rack-test", + firmware_only_activities(), + ) + .await?; + txn.commit().await?; + + let mut state = load_power_shelf(&pool, &power_shelf_id).await; + let mut services = services(&env, false); + let outcome = run_handler(&mut services, &mut state).await; + assert!(matches!(outcome, StateHandlerOutcome::DoNothing { .. })); + commit_outcome(outcome).await; + + let state = load_power_shelf(&pool, &power_shelf_id).await; + assert!(state.power_shelf_reprovisioning_requested.is_none()); + assert!(matches!( + state.controller_state.value, + PowerShelfControllerState::Ready + )); + Ok(()) +} + +#[crate::sqlx_test] +async fn test_ready_enters_waiting_for_rack_firmware_when_flag_enabled( + pool: sqlx::PgPool, +) -> Result<(), Box> { + let env = create_test_env(pool.clone()).await; + let power_shelf_id = new_power_shelf(&env, None, None, None, None).await?; + + let mut txn = pool.begin().await?; + park_ready(txn.as_mut(), &power_shelf_id).await; + db_power_shelf::set_power_shelf_reprovisioning_requested( + txn.as_mut(), + power_shelf_id, + "rack-test", + firmware_only_activities(), + ) + .await?; + txn.commit().await?; + + let mut state = load_power_shelf(&pool, &power_shelf_id).await; + let mut services = services(&env, true); + let outcome = run_handler(&mut services, &mut state).await; + assert!(matches!( + outcome, + StateHandlerOutcome::Transition { + next_state: PowerShelfControllerState::ReProvisioning { + reprovisioning_state: ReProvisioningState::WaitingForRackFirmwareUpgrade, + }, + .. + } + )); + Ok(()) +} + +#[crate::sqlx_test] +async fn test_waiting_for_rack_firmware_completes_to_ready( + pool: sqlx::PgPool, +) -> Result<(), Box> { + let env = create_test_env(pool.clone()).await; + let power_shelf_id = new_power_shelf(&env, None, None, None, None).await?; + + let mut txn = pool.begin().await?; + db_power_shelf::set_power_shelf_reprovisioning_requested( + txn.as_mut(), + power_shelf_id, + "rack-test", + firmware_only_activities(), + ) + .await?; + let power_shelf = db_power_shelf::find_by_id(txn.as_mut(), &power_shelf_id) + .await? + .expect("power shelf should exist"); + let requested_at = power_shelf + .power_shelf_reprovisioning_requested + .as_ref() + .expect("request should exist") + .requested_at; + set_power_shelf_controller_state( + txn.as_mut(), + &power_shelf_id, + PowerShelfControllerState::ReProvisioning { + reprovisioning_state: ReProvisioningState::WaitingForRackFirmwareUpgrade, + }, + ) + .await?; + db_power_shelf::update_firmware_upgrade_status( + txn.as_mut(), + power_shelf_id, + Some(&model::rack::RackFirmwareUpgradeStatus { + task_id: "rack-job".to_string(), + status: model::rack::RackFirmwareUpgradeState::Completed, + started_at: Some(requested_at), + ended_at: Some(requested_at + chrono::Duration::seconds(1)), + }), + ) + .await?; + txn.commit().await?; + + let mut state = load_power_shelf(&pool, &power_shelf_id).await; + let mut services = services(&env, true); + let outcome = run_handler(&mut services, &mut state).await; + assert!(matches!( + outcome, + StateHandlerOutcome::Transition { + next_state: PowerShelfControllerState::Ready, + .. + } + )); + commit_outcome(outcome).await; + + let state = load_power_shelf(&pool, &power_shelf_id).await; + assert!(state.power_shelf_reprovisioning_requested.is_none()); + Ok(()) +} diff --git a/crates/api-core/src/tests/rack_state_controller/handler.rs b/crates/api-core/src/tests/rack_state_controller/handler.rs index 872f037c3d..0c79d04f84 100644 --- a/crates/api-core/src/tests/rack_state_controller/handler.rs +++ b/crates/api-core/src/tests/rack_state_controller/handler.rs @@ -207,6 +207,52 @@ async fn create_single_compute_rack( Ok((rack_id, host)) } +async fn set_machine_host_reprovision_state( + pool: &sqlx::PgPool, + machine_id: &MachineId, + reprovision_state: model::machine::HostReprovisionState, +) -> Result<(), Box> { + let mut txn = pool.begin().await?; + let machine = db::machine::find_one( + txn.as_mut(), + machine_id, + model::machine::machine_search_config::MachineSearchConfig::default(), + ) + .await? + .expect("machine should exist"); + db::machine::advance( + &machine, + txn.as_mut(), + &model::machine::ManagedHostState::HostReprovision { + reprovision_state, + retry_count: 0, + }, + None, + ) + .await?; + txn.commit().await?; + Ok(()) +} + +fn waiting_for_rack_firmware_upgrade_state() -> model::machine::HostReprovisionState { + model::machine::HostReprovisionState::WaitingForRackFirmwareUpgrade +} + +fn failed_rack_firmware_upgrade_state() -> model::machine::HostReprovisionState { + model::machine::HostReprovisionState::FailedFirmwareUpgrade { + firmware_type: model::firmware::FirmwareComponentType::Bmc, + report_time: Some(chrono::Utc::now()), + reason: Some("upgrade failed".to_string()), + } +} + +fn completed_rack_firmware_upgrade_state() -> model::machine::HostReprovisionState { + model::machine::HostReprovisionState::CheckingFirmwareRepeatV2 { + firmware_type: None, + firmware_number: None, + } +} + async fn create_two_compute_rack( env: &TestEnv, pool: &sqlx::PgPool, @@ -1583,8 +1629,9 @@ async fn test_firmware_upgrade_start_missing_profile_deletes_access_token( } /// test_firmware_upgrade_wait_for_complete_waits_while_jobs_running verifies -/// that WaitForComplete remains in a wait state while RMS child jobs are still -/// running and writes in-progress rack firmware status back to the machine. +/// that WaitForComplete remains in a wait state while machines are still in +/// WaitingForRackFirmwareUpgrade and writes in-progress rack firmware status +/// back to the machine from RMS. #[crate::sqlx_test] async fn test_firmware_upgrade_wait_for_complete_waits_while_jobs_running( pool: sqlx::PgPool, @@ -1598,6 +1645,12 @@ async fn test_firmware_upgrade_wait_for_complete_waits_while_jobs_running( ) .await; let (rack_id, host) = create_single_compute_rack(&env, &pool).await?; + set_machine_host_reprovision_state( + &pool, + &host.host_snapshot.id, + waiting_for_rack_firmware_upgrade_state(), + ) + .await?; env.rms_sim .set_firmware_job_status(librms::protos::rack_manager::GetFirmwareJobStatusResponse { status: librms::protos::rack_manager::ReturnCode::Success as i32, @@ -1651,7 +1704,7 @@ async fn test_firmware_upgrade_wait_for_complete_waits_while_jobs_running( assert!( matches!(outcome, StateHandlerOutcome::Wait { .. }), - "Expected Wait while RMS job is running" + "Expected Wait while machine controller is still WaitingForRackFirmwareUpgrade" ); let machine = db::machine::find_one( @@ -1671,8 +1724,8 @@ async fn test_firmware_upgrade_wait_for_complete_waits_while_jobs_running( } /// test_firmware_upgrade_wait_for_complete_transitions_to_error_on_job_failure -/// verifies that a failed RMS child job writes failed machine status and moves -/// the rack into Error. +/// verifies that a machine left WaitingForRackFirmwareUpgrade in +/// FailedFirmwareUpgrade moves the rack into Error. #[crate::sqlx_test] async fn test_firmware_upgrade_wait_for_complete_transitions_to_error_on_job_failure( pool: sqlx::PgPool, @@ -1686,6 +1739,12 @@ async fn test_firmware_upgrade_wait_for_complete_transitions_to_error_on_job_fai ) .await; let (rack_id, host) = create_single_compute_rack(&env, &pool).await?; + set_machine_host_reprovision_state( + &pool, + &host.host_snapshot.id, + failed_rack_firmware_upgrade_state(), + ) + .await?; env.rms_sim .set_firmware_job_status(librms::protos::rack_manager::GetFirmwareJobStatusResponse { status: librms::protos::rack_manager::ReturnCode::Success as i32, @@ -1772,9 +1831,9 @@ async fn test_firmware_upgrade_wait_for_complete_transitions_to_error_on_job_fai } /// test_firmware_upgrade_wait_for_complete_waits_for_all_nodes_to_be_terminal_before_error -/// verifies that the rack keeps polling when a mixed result contains both -/// failed and in-progress devices, then errors only after all tracked devices -/// reach a terminal state. +/// verifies that the rack keeps waiting while any tracked machine is still in +/// WaitingForRackFirmwareUpgrade, then errors only after every machine has left +/// that wait state and at least one failed. #[crate::sqlx_test] async fn test_firmware_upgrade_wait_for_complete_waits_for_all_nodes_to_be_terminal_before_error( pool: sqlx::PgPool, @@ -1788,6 +1847,18 @@ async fn test_firmware_upgrade_wait_for_complete_waits_for_all_nodes_to_be_termi ) .await; let (rack_id, host_a, host_b) = create_two_compute_rack(&env, &pool).await?; + set_machine_host_reprovision_state( + &pool, + &host_a.host_snapshot.id, + waiting_for_rack_firmware_upgrade_state(), + ) + .await?; + set_machine_host_reprovision_state( + &pool, + &host_b.host_snapshot.id, + waiting_for_rack_firmware_upgrade_state(), + ) + .await?; env.rms_sim .set_firmware_job_status(librms::protos::rack_manager::GetFirmwareJobStatusResponse { @@ -1864,7 +1935,7 @@ async fn test_firmware_upgrade_wait_for_complete_waits_for_all_nodes_to_be_termi assert!( matches!(outcome, StateHandlerOutcome::Wait { .. }), - "Expected Wait while some tracked devices are still non-terminal" + "Expected Wait while some tracked machines are still WaitingForRackFirmwareUpgrade" ); let machine_a = db::machine::find_one( @@ -1898,6 +1969,19 @@ async fn test_firmware_upgrade_wait_for_complete_waits_for_all_nodes_to_be_termi RackFirmwareUpgradeState::InProgress ); + set_machine_host_reprovision_state( + &pool, + &host_a.host_snapshot.id, + failed_rack_firmware_upgrade_state(), + ) + .await?; + set_machine_host_reprovision_state( + &pool, + &host_b.host_snapshot.id, + waiting_for_rack_firmware_upgrade_state(), + ) + .await?; + env.rms_sim .set_firmware_job_status(librms::protos::rack_manager::GetFirmwareJobStatusResponse { status: librms::protos::rack_manager::ReturnCode::Success as i32, @@ -1917,11 +2001,31 @@ async fn test_firmware_upgrade_wait_for_complete_waits_for_all_nodes_to_be_termi txn.commit().await?; } + assert!( + matches!(outcome, StateHandlerOutcome::Wait { .. }), + "Expected Wait while machine B is still WaitingForRackFirmwareUpgrade" + ); + + set_machine_host_reprovision_state( + &pool, + &host_b.host_snapshot.id, + completed_rack_firmware_upgrade_state(), + ) + .await?; + + let mut rack = get_db_rack(env.db_reader().as_mut(), &rack_id).await; + let mut outcome = handler_instance + .handle_object_state(&rack_id, &mut rack, &fw_state, &mut ctx) + .await?; + if let Some(txn) = outcome.take_transaction() { + txn.commit().await?; + } + match outcome { StateHandlerOutcome::Transition { next_state, .. } => { assert!( matches!(next_state, RackState::Error { .. }), - "Expected rack to transition to Error after all tracked devices are terminal, got {:?}", + "Expected rack to transition to Error after all tracked machines left firmware wait with a failure, got {:?}", next_state ); } @@ -1952,7 +2056,8 @@ async fn test_firmware_upgrade_wait_for_complete_waits_for_all_nodes_to_be_termi /// test_firmware_upgrade_wait_for_complete_retries_when_job_lookup_fails /// verifies that a response-level lookup failure from GetFirmwareJobStatus does -/// not mark the device failed and instead keeps the rack waiting. +/// not mark the device failed and instead keeps the rack waiting while the +/// machine remains in WaitingForRackFirmwareUpgrade. #[crate::sqlx_test] async fn test_firmware_upgrade_wait_for_complete_retries_when_job_lookup_fails( pool: sqlx::PgPool, @@ -1966,6 +2071,12 @@ async fn test_firmware_upgrade_wait_for_complete_retries_when_job_lookup_fails( ) .await; let (rack_id, host) = create_single_compute_rack(&env, &pool).await?; + set_machine_host_reprovision_state( + &pool, + &host.host_snapshot.id, + waiting_for_rack_firmware_upgrade_state(), + ) + .await?; env.rms_sim .set_firmware_job_status(librms::protos::rack_manager::GetFirmwareJobStatusResponse { status: librms::protos::rack_manager::ReturnCode::Failure as i32, @@ -2037,7 +2148,7 @@ async fn test_firmware_upgrade_wait_for_complete_retries_when_job_lookup_fails( /// test_firmware_upgrade_wait_for_complete_retries_on_transient_poll_error /// verifies that transport-level polling failures do not immediately fail the -/// rack upgrade. +/// rack upgrade while machines remain in WaitingForRackFirmwareUpgrade. #[crate::sqlx_test] async fn test_firmware_upgrade_wait_for_complete_retries_on_transient_poll_error( pool: sqlx::PgPool, @@ -2051,6 +2162,12 @@ async fn test_firmware_upgrade_wait_for_complete_retries_on_transient_poll_error ) .await; let (rack_id, host) = create_single_compute_rack(&env, &pool).await?; + set_machine_host_reprovision_state( + &pool, + &host.host_snapshot.id, + waiting_for_rack_firmware_upgrade_state(), + ) + .await?; env.rms_sim .set_firmware_job_error("child-job-1", "mock transport failure") .await; diff --git a/crates/api-core/src/tests/switch_state_controller/mod.rs b/crates/api-core/src/tests/switch_state_controller/mod.rs index c626996a9d..4c52e8f790 100644 --- a/crates/api-core/src/tests/switch_state_controller/mod.rs +++ b/crates/api-core/src/tests/switch_state_controller/mod.rs @@ -54,6 +54,28 @@ fn default_switch_mtls_services() -> Vec { ) } +fn firmware_only_activities() -> Vec { + vec![model::rack::MaintenanceActivity::FirmwareUpgrade { + firmware_version: None, + components: vec![], + force_update: false, + }] +} + +fn nvos_and_nmxc_activities() -> Vec { + vec![ + model::rack::MaintenanceActivity::FirmwareUpgrade { + firmware_version: None, + components: vec![], + force_update: false, + }, + model::rack::MaintenanceActivity::NvosUpdate { + config_json: String::new(), + }, + model::rack::MaintenanceActivity::ConfigureNmxCluster, + ] +} + async fn build_test_component_manager( env: &common::api_fixtures::TestEnv, rms_client: Option>, @@ -621,7 +643,13 @@ async fn test_switch_waiting_for_rack_firmware_upgrade_waits_for_terminal_status let switch_id = common::api_fixtures::site_explorer::new_switch(&env, None, None).await?; let mut txn = pool.begin().await?; - db_switch::set_switch_reprovisioning_requested(txn.as_mut(), switch_id, "rack-test").await?; + db_switch::set_switch_reprovisioning_requested( + txn.as_mut(), + switch_id, + "rack-test", + nvos_and_nmxc_activities(), + ) + .await?; let switch = db_switch::find_by_id(txn.as_mut(), &switch_id) .await? .expect("switch should exist"); @@ -678,7 +706,13 @@ async fn test_switch_waiting_for_rack_firmware_upgrade_transitions_to_waiting_fo let switch_id = common::api_fixtures::site_explorer::new_switch(&env, None, None).await?; let mut txn = pool.begin().await?; - db_switch::set_switch_reprovisioning_requested(txn.as_mut(), switch_id, "rack-test").await?; + db_switch::set_switch_reprovisioning_requested( + txn.as_mut(), + switch_id, + "rack-test", + nvos_and_nmxc_activities(), + ) + .await?; let switch = db_switch::find_by_id(txn.as_mut(), &switch_id) .await? .expect("switch should exist"); @@ -735,11 +769,11 @@ async fn test_switch_waiting_for_rack_firmware_upgrade_returns_ready_for_firmwar let switch_id = common::api_fixtures::site_explorer::new_switch(&env, None, None).await?; let mut txn = pool.begin().await?; - db_switch::set_switch_reprovisioning_requested_with_firmware_continuation( + db_switch::set_switch_reprovisioning_requested( txn.as_mut(), switch_id, "rack-test", - false, + firmware_only_activities(), ) .await?; let switch = db_switch::find_by_id(txn.as_mut(), &switch_id) @@ -796,7 +830,13 @@ async fn test_switch_waiting_for_rack_firmware_upgrade_accepts_completion_when_o let switch_id = common::api_fixtures::site_explorer::new_switch(&env, None, None).await?; let mut txn = pool.begin().await?; - db_switch::set_switch_reprovisioning_requested(txn.as_mut(), switch_id, "rack-test").await?; + db_switch::set_switch_reprovisioning_requested( + txn.as_mut(), + switch_id, + "rack-test", + nvos_and_nmxc_activities(), + ) + .await?; let switch = db_switch::find_by_id(txn.as_mut(), &switch_id) .await? .expect("switch should exist"); @@ -853,7 +893,13 @@ async fn test_switch_ready_routes_rack_requests_to_waiting_for_rack_firmware_upg let switch_id = common::api_fixtures::site_explorer::new_switch(&env, None, None).await?; let mut txn = pool.begin().await?; - db_switch::set_switch_reprovisioning_requested(txn.as_mut(), switch_id, "rack-test").await?; + db_switch::set_switch_reprovisioning_requested( + txn.as_mut(), + switch_id, + "rack-test", + nvos_and_nmxc_activities(), + ) + .await?; let switch = db_switch::find_by_id(txn.as_mut(), &switch_id) .await? .expect("switch should exist"); @@ -891,8 +937,13 @@ async fn test_switch_waiting_for_nvos_upgrade_transitions_to_waiting_for_nmxc_on let switch_id = common::api_fixtures::site_explorer::new_switch(&env, None, None).await?; let mut txn = pool.begin().await?; - db_switch::set_switch_reprovisioning_requested(txn.as_mut(), switch_id, "rack-nvos-test") - .await?; + db_switch::set_switch_reprovisioning_requested( + txn.as_mut(), + switch_id, + "rack-nvos-test", + nvos_and_nmxc_activities(), + ) + .await?; let switch = db_switch::find_by_id(txn.as_mut(), &switch_id) .await? .expect("switch should exist"); @@ -951,8 +1002,13 @@ async fn test_switch_waiting_for_nvos_upgrade_waits_for_current_cycle_status( let switch_id = common::api_fixtures::site_explorer::new_switch(&env, None, None).await?; let mut txn = pool.begin().await?; - db_switch::set_switch_reprovisioning_requested(txn.as_mut(), switch_id, "rack-nvos-test") - .await?; + db_switch::set_switch_reprovisioning_requested( + txn.as_mut(), + switch_id, + "rack-nvos-test", + nvos_and_nmxc_activities(), + ) + .await?; let switch = db_switch::find_by_id(txn.as_mut(), &switch_id) .await? .expect("switch should exist"); @@ -1011,8 +1067,13 @@ async fn test_switch_waiting_for_nvos_upgrade_transitions_to_error_on_failure( let switch_id = common::api_fixtures::site_explorer::new_switch(&env, None, None).await?; let mut txn = pool.begin().await?; - db_switch::set_switch_reprovisioning_requested(txn.as_mut(), switch_id, "rack-nvos-test") - .await?; + db_switch::set_switch_reprovisioning_requested( + txn.as_mut(), + switch_id, + "rack-nvos-test", + nvos_and_nmxc_activities(), + ) + .await?; let switch = db_switch::find_by_id(txn.as_mut(), &switch_id) .await? .expect("switch should exist"); @@ -1071,8 +1132,13 @@ async fn test_switch_waiting_for_nmxc_configure_returns_ready_when_fm_is_running let switch_id = common::api_fixtures::site_explorer::new_switch(&env, None, None).await?; let mut txn = pool.begin().await?; - db_switch::set_switch_reprovisioning_requested(txn.as_mut(), switch_id, "rack-nmxc-test") - .await?; + db_switch::set_switch_reprovisioning_requested( + txn.as_mut(), + switch_id, + "rack-nmxc-test", + nvos_and_nmxc_activities(), + ) + .await?; let switch = db_switch::find_by_id(txn.as_mut(), &switch_id) .await? .expect("switch should exist"); diff --git a/crates/api-db/migrations/20260724120000_power_shelf_reprovisioning_requested.sql b/crates/api-db/migrations/20260724120000_power_shelf_reprovisioning_requested.sql new file mode 100644 index 0000000000..eb98e580e7 --- /dev/null +++ b/crates/api-db/migrations/20260724120000_power_shelf_reprovisioning_requested.sql @@ -0,0 +1,7 @@ +-- Add power_shelf_reprovisioning_requested and firmware_upgrade_status columns +-- to power_shelves. Mirrors switches.switch_reprovisioning_requested / +-- firmware_upgrade_status for rack-level firmware upgrade wait phases. + +ALTER TABLE power_shelves + ADD COLUMN power_shelf_reprovisioning_requested JSONB, + ADD COLUMN firmware_upgrade_status JSONB; diff --git a/crates/api-db/src/power_shelf.rs b/crates/api-db/src/power_shelf.rs index dec1c5f74a..e0b904bd15 100644 --- a/crates/api-db/src/power_shelf.rs +++ b/crates/api-db/src/power_shelf.rs @@ -24,8 +24,9 @@ use model::controller_outcome::PersistentStateHandlerOutcome; use model::metadata::Metadata; use model::power_shelf::{ NewPowerShelf, PowerShelf, PowerShelfControllerState, PowerShelfMaintenanceOperation, - PowerShelfMaintenanceRequest, + PowerShelfMaintenanceRequest, PowerShelfReprovisionRequest, }; +use model::rack::{MaintenanceActivity, RackFirmwareUpgradeStatus}; use sqlx::PgConnection; use crate::db_read::DbReader; @@ -133,6 +134,8 @@ pub async fn create( version, rack_id: new_power_shelf.rack_id.clone(), power_shelf_maintenance_requested: None, + power_shelf_reprovisioning_requested: None, + firmware_upgrade_status: None, health_reports: Default::default(), }) } @@ -348,6 +351,65 @@ pub async fn clear_power_shelf_maintenance_requested( Ok(()) } +/// Sets `power_shelf_reprovisioning_requested` so the Ready handler can enter +/// `ReProvisioning` when rack-firmware reprovisioning is enabled. +/// +/// `activities` selects which rack maintenance phases the power shelf should wait +/// for. Empty means all activities. +pub async fn set_power_shelf_reprovisioning_requested( + txn: &mut PgConnection, + power_shelf_id: PowerShelfId, + initiator: &str, + activities: Vec, +) -> DatabaseResult<()> { + let req = PowerShelfReprovisionRequest { + requested_at: Utc::now(), + initiator: initiator.to_string(), + activities, + }; + let query = "UPDATE power_shelves SET power_shelf_reprovisioning_requested = $1 WHERE id = $2 RETURNING id"; + sqlx::query_as::<_, PowerShelfId>(query) + .bind(sqlx::types::Json(req)) + .bind(power_shelf_id) + .fetch_optional(txn) + .await + .map_err(|e| DatabaseError::new("set_power_shelf_reprovisioning_requested", e))?; + Ok(()) +} + +/// Clears `power_shelf_reprovisioning_requested`. Typically called when +/// reprovisioning completes or is cancelled. +pub async fn clear_power_shelf_reprovisioning_requested( + txn: &mut PgConnection, + power_shelf_id: PowerShelfId, +) -> DatabaseResult<()> { + let query = "UPDATE power_shelves SET power_shelf_reprovisioning_requested = NULL WHERE id = $1 RETURNING id"; + sqlx::query_as::<_, PowerShelfId>(query) + .bind(power_shelf_id) + .fetch_optional(txn) + .await + .map_err(|e| DatabaseError::new("clear_power_shelf_reprovisioning_requested", e))?; + Ok(()) +} + +/// Sets `firmware_upgrade_status` on the power shelf. Call from rack maintenance +/// to report upgrade progress. `WaitingForRackFirmwareUpgrade` reads this: +/// Completed → Ready, Failed → Error. +pub async fn update_firmware_upgrade_status( + txn: &mut PgConnection, + power_shelf_id: PowerShelfId, + status: Option<&RackFirmwareUpgradeStatus>, +) -> DatabaseResult<()> { + let query = "UPDATE power_shelves SET firmware_upgrade_status = $1 WHERE id = $2 RETURNING id"; + sqlx::query_as::<_, PowerShelfId>(query) + .bind(status.map(|s| sqlx::types::Json(s.clone()))) + .bind(power_shelf_id) + .fetch_optional(txn) + .await + .map_err(|e| DatabaseError::new("update_firmware_upgrade_status", e))?; + Ok(()) +} + pub async fn mark_as_deleted<'a>( power_shelf: &'a mut PowerShelf, txn: &mut PgConnection, diff --git a/crates/api-db/src/switch.rs b/crates/api-db/src/switch.rs index af23b2b1ca..e50435d435 100644 --- a/crates/api-db/src/switch.rs +++ b/crates/api-db/src/switch.rs @@ -368,25 +368,19 @@ pub async fn update_controller_state_outcome( /// Sets switch_reprovisioning_requested on the switch. Can be called from any state machine or /// service. When the switch is in Ready state, the switch state controller will observe the flag /// and transition to ReProvisioning::Start. +/// +/// `activities` selects which rack maintenance phases the switch should wait for. +/// Empty means all activities. pub async fn set_switch_reprovisioning_requested( txn: &mut PgConnection, switch_id: SwitchId, initiator: &str, -) -> DatabaseResult<()> { - set_switch_reprovisioning_requested_with_firmware_continuation(txn, switch_id, initiator, true) - .await -} - -pub async fn set_switch_reprovisioning_requested_with_firmware_continuation( - txn: &mut PgConnection, - switch_id: SwitchId, - initiator: &str, - continue_after_firmware_upgrade: bool, + activities: Vec, ) -> DatabaseResult<()> { let req = SwitchReprovisionRequest { requested_at: Utc::now(), initiator: initiator.to_string(), - continue_after_firmware_upgrade, + activities, }; let query = "UPDATE switches SET switch_reprovisioning_requested = $1 WHERE id = $2 RETURNING id"; diff --git a/crates/api-model/src/power_shelf/mod.rs b/crates/api-model/src/power_shelf/mod.rs index 9ec4a5250e..b20cee1770 100644 --- a/crates/api-model/src/power_shelf/mod.rs +++ b/crates/api-model/src/power_shelf/mod.rs @@ -86,6 +86,17 @@ pub struct PowerShelf { pub power_shelf_maintenance_requested: Option, + /// Set by rack maintenance to request power-shelf participation in a + /// rack-level firmware upgrade. When the power shelf is Ready and + /// rack-firmware reprovisioning is enabled on the controller, it + /// transitions to `ReProvisioning`. + pub power_shelf_reprovisioning_requested: Option, + + /// Per-device firmware upgrade status written by the rack state machine + /// during rack-level firmware upgrades. Read by + /// `ReProvisioning::WaitingForRackFirmwareUpgrade`. + pub firmware_upgrade_status: Option, + // Columns for these exist, but are unused in rust code // pub created: DateTime, // pub updated: DateTime, @@ -105,6 +116,12 @@ impl<'r> FromRow<'r, PgRow> for PowerShelf { let power_shelf_maintenance_requested: Option< sqlx::types::Json, > = row.try_get("power_shelf_maintenance_requested").ok(); + let power_shelf_reprovisioning_requested: Option< + sqlx::types::Json, + > = row.try_get("power_shelf_reprovisioning_requested").ok(); + let firmware_upgrade_status: Option< + sqlx::types::Json, + > = row.try_get("firmware_upgrade_status").ok(); let health_reports: HealthReportSources = row .try_get::, _>("health_reports") @@ -137,6 +154,8 @@ impl<'r> FromRow<'r, PgRow> for PowerShelf { version: row.try_get("version")?, rack_id: row.try_get("rack_id").ok().flatten(), power_shelf_maintenance_requested: power_shelf_maintenance_requested.map(|r| r.0), + power_shelf_reprovisioning_requested: power_shelf_reprovisioning_requested.map(|r| r.0), + firmware_upgrade_status: firmware_upgrade_status.map(|j| j.0), health_reports, }) } @@ -179,6 +198,30 @@ pub struct PowerShelfMaintenanceRequest { pub operation: PowerShelfMaintenanceOperation, } +/// Set by an external entity (typically rack maintenance) to request power-shelf +/// participation in rack-level reprovisioning. When the power shelf is Ready and +/// rack-firmware reprovisioning is enabled, the controller transitions to +/// `ReProvisioning`. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct PowerShelfReprovisionRequest { + pub requested_at: DateTime, + pub initiator: String, + /// Rack maintenance activities that initiated this request. The power shelf + /// controller uses these to decide whether to wait for firmware. Empty means + /// all activities. + #[serde(default)] + pub activities: Vec, +} + +/// Sub-state for PowerShelfControllerState::ReProvisioning +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[allow(clippy::enum_variant_names)] +pub enum ReProvisioningState { + /// Rack-level firmware upgrade in progress; the rack state machine manages the + /// upgrade and clears `power_shelf_reprovisioning_requested` when done. + WaitingForRackFirmwareUpgrade, +} + /// State of a PowerShelf as tracked by the controller #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] #[serde(tag = "state", rename_all = "lowercase")] @@ -195,6 +238,11 @@ pub enum PowerShelfControllerState { Maintenance { operation: PowerShelfMaintenanceOperation, }, + + /// Rack-driven firmware wait in progress. + ReProvisioning { + reprovisioning_state: ReProvisioningState, + }, /// There is error in PowerShelf; PowerShelf can not be used if it's in error. Error { cause: String }, /// The PowerShelf is in the process of deleting. @@ -226,6 +274,10 @@ pub fn state_sla(state: &PowerShelfControllerState, state_version: &ConfigVersio std::time::Duration::from_secs(slas::MAINTENANCE), time_in_state, ), + PowerShelfControllerState::ReProvisioning { .. } => StateSla::with_sla( + std::time::Duration::from_secs(slas::REPROVISIONING), + time_in_state, + ), PowerShelfControllerState::Error { .. } => StateSla::no_sla(), PowerShelfControllerState::Deleting => StateSla::with_sla( std::time::Duration::from_secs(slas::DELETING), @@ -331,6 +383,18 @@ mod tests { }, )), } + + "reprovisioning waiting for rack firmware" { + PowerShelfControllerState::ReProvisioning { + reprovisioning_state: ReProvisioningState::WaitingForRackFirmwareUpgrade, + } => Yields(( + r#"{"state":"reprovisioning","reprovisioning_state":"WaitingForRackFirmwareUpgrade"}"# + .to_string(), + PowerShelfControllerState::ReProvisioning { + reprovisioning_state: ReProvisioningState::WaitingForRackFirmwareUpgrade, + }, + )), + } ); } @@ -451,6 +515,14 @@ mod tests { }), } + "reprovisioning waiting for rack firmware" { + r#"{"state":"reprovisioning","reprovisioning_state":"WaitingForRackFirmwareUpgrade"}"# => Yields( + PowerShelfControllerState::ReProvisioning { + reprovisioning_state: ReProvisioningState::WaitingForRackFirmwareUpgrade, + }, + ), + } + "unknown tag is rejected" { r#"{"state":"running"}"# => Fails, } @@ -669,6 +741,12 @@ mod tests { } => (secs(slas::MAINTENANCE), true), } + "reprovisioning has the reprovisioning SLA" { + PowerShelfControllerState::ReProvisioning { + reprovisioning_state: ReProvisioningState::WaitingForRackFirmwareUpgrade, + } => (secs(slas::REPROVISIONING), true), + } + "ready carries no SLA" { PowerShelfControllerState::Ready => (None, false), } diff --git a/crates/api-model/src/power_shelf/slas.rs b/crates/api-model/src/power_shelf/slas.rs index b721cf770d..215516b577 100644 --- a/crates/api-model/src/power_shelf/slas.rs +++ b/crates/api-model/src/power_shelf/slas.rs @@ -35,3 +35,6 @@ pub const DELETING: u64 = 300; // 5 minutes /// SLA for PowerShelf maintenance (PowerOn / PowerOff) in seconds pub const MAINTENANCE: u64 = 300; // 5 minutes + +/// SLA for PowerShelf rack-level reprovisioning (firmware wait) in seconds +pub const REPROVISIONING: u64 = 3600; // 1 hour diff --git a/crates/api-model/src/switch/mod.rs b/crates/api-model/src/switch/mod.rs index 80b39f1be8..d76b40d34c 100644 --- a/crates/api-model/src/switch/mod.rs +++ b/crates/api-model/src/switch/mod.rs @@ -65,10 +65,6 @@ pub struct SwitchStatus { pub health_status: String, // "ok", "warning", "critical" } -fn default_continue_after_firmware_upgrade() -> bool { - true -} - #[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] #[serde(tag = "operation", rename_all = "lowercase")] #[allow(clippy::enum_variant_names)] @@ -96,9 +92,11 @@ pub struct SwitchMaintenanceRequest { pub struct SwitchReprovisionRequest { pub requested_at: DateTime, pub initiator: String, - /// Continue through rack-managed post-firmware phases such as NVOS/NMXC. - #[serde(default = "default_continue_after_firmware_upgrade")] - pub continue_after_firmware_upgrade: bool, + /// Rack maintenance activities that initiated this request. The switch + /// controller uses these to decide which ReProvisioning phases to wait for + /// (firmware / NVOS / NMXC). Empty means all activities. + #[serde(default)] + pub activities: Vec, } pub use crate::rack::{ @@ -863,24 +861,19 @@ mod tests { } #[test] - fn reprovision_request_defaults_continue_after_firmware_upgrade_to_true() { - scenarios!( - run = |json| { - serde_json::from_str::(json) - .map(|r| r.continue_after_firmware_upgrade) - .map_err(drop) - }; - "omitted flag defaults to true" { - r#"{"requested_at":"2026-01-01T00:00:00Z","initiator":"op"}"# => Yields(true), - } - - "explicit false is honored" { - r#"{"requested_at":"2026-01-01T00:00:00Z","initiator":"op","continue_after_firmware_upgrade":false}"# => Yields(false), - } + fn reprovision_request_defaults_activities_to_empty() { + let request: SwitchReprovisionRequest = + serde_json::from_str(r#"{"requested_at":"2026-01-01T00:00:00Z","initiator":"op"}"#) + .expect("SwitchReprovisionRequest should deserialize"); + assert!(request.activities.is_empty()); + } - "explicit true is honored" { - r#"{"requested_at":"2026-01-01T00:00:00Z","initiator":"op","continue_after_firmware_upgrade":true}"# => Yields(true), - } - ); + #[test] + fn reprovision_request_ignores_legacy_fields() { + let request: SwitchReprovisionRequest = serde_json::from_str( + r#"{"requested_at":"2026-01-01T00:00:00Z","initiator":"op","continue_after_firmware_upgrade":true,"scope":{"activities":[]}}"#, + ) + .expect("legacy continue_after_firmware_upgrade and scope fields should be ignored"); + assert!(request.activities.is_empty()); } } diff --git a/crates/power-shelf-controller/src/context.rs b/crates/power-shelf-controller/src/context.rs index 6fb18459c5..779a4707f9 100644 --- a/crates/power-shelf-controller/src/context.rs +++ b/crates/power-shelf-controller/src/context.rs @@ -34,6 +34,11 @@ pub struct PowerShelfStateHandlerServices { pub credential_manager: Arc, /// Shared registry backing the generic per-object health metrics. pub per_object_metrics_registry: Arc, + /// When `true`, Ready accepts rack-level `power_shelf_reprovisioning_requested` + /// and enters `ReProvisioning::WaitingForRackFirmwareUpgrade`. Defaults to + /// `false` so power shelves stay out of rack firmware wait unless explicitly + /// enabled. + pub rack_firmware_reprovisioning_enabled: bool, } impl StateHandlerContextObjects for PowerShelfStateHandlerContextObjects { diff --git a/crates/power-shelf-controller/src/handler.rs b/crates/power-shelf-controller/src/handler.rs index 71724c38c8..49ed324649 100644 --- a/crates/power-shelf-controller/src/handler.rs +++ b/crates/power-shelf-controller/src/handler.rs @@ -34,6 +34,7 @@ use crate::fetching_data::handle_fetching_data; use crate::initializing::handle_initializing; use crate::maintenance::handle_maintenance; use crate::ready::handle_ready; +use crate::reprovisioning::handle_reprovisioning; /// The actual PowerShelf State handler (structure mirrors SwitchStateHandler). #[derive(Debug, Default, Clone)] @@ -82,6 +83,9 @@ impl PowerShelfStateHandler { PowerShelfControllerState::Maintenance { .. } => { handle_maintenance(power_shelf_id, state, ctx).await } + PowerShelfControllerState::ReProvisioning { .. } => { + handle_reprovisioning(power_shelf_id, state, ctx).await + } PowerShelfControllerState::Deleting => { handle_deleting(power_shelf_id, state, ctx).await } diff --git a/crates/power-shelf-controller/src/io.rs b/crates/power-shelf-controller/src/io.rs index 0deb627444..f1ee2bbc2a 100644 --- a/crates/power-shelf-controller/src/io.rs +++ b/crates/power-shelf-controller/src/io.rs @@ -160,6 +160,16 @@ impl StateControllerIO for PowerShelfStateControllerIO { }; ("maintenance", op) } + PowerShelfControllerState::ReProvisioning { + reprovisioning_state, + } => { + let sub = match reprovisioning_state { + model::power_shelf::ReProvisioningState::WaitingForRackFirmwareUpgrade => { + "waiting_for_rack_firmware_upgrade" + } + }; + ("reprovisioning", sub) + } PowerShelfControllerState::Error { .. } => ("error", ""), PowerShelfControllerState::Deleting => ("deleting", ""), } diff --git a/crates/power-shelf-controller/src/lib.rs b/crates/power-shelf-controller/src/lib.rs index 0faefdddaa..d8502c6ab8 100644 --- a/crates/power-shelf-controller/src/lib.rs +++ b/crates/power-shelf-controller/src/lib.rs @@ -28,3 +28,4 @@ pub mod io; pub mod maintenance; pub mod metrics; pub mod ready; +pub mod reprovisioning; diff --git a/crates/power-shelf-controller/src/ready.rs b/crates/power-shelf-controller/src/ready.rs index 0b9e022e45..3ff73aa9c1 100644 --- a/crates/power-shelf-controller/src/ready.rs +++ b/crates/power-shelf-controller/src/ready.rs @@ -28,15 +28,17 @@ use state_controller::state_handler::{ use crate::context::PowerShelfStateHandlerContextObjects; use crate::maintenance::build_power_shelf_endpoint; +use crate::reprovisioning::first_reprovisioning_state; /// Handles the Ready state for a power shelf. /// /// If the power shelf is marked for deletion, transitions to `Deleting`. /// If a maintenance request has been posted via /// `power_shelf_maintenance_requested`, transitions to `Maintenance` with the -/// requested operation (PowerOn / PowerOff). Otherwise polls the configured -/// component manager backend for the current power state (best-effort -/// observation) and idles. +/// requested operation (PowerOn / PowerOff). If rack-level reprovisioning has +/// been requested and `rack_firmware_reprovisioning_enabled` is set, transitions +/// to `ReProvisioning`. Otherwise polls the configured component manager backend +/// for the current power state (best-effort observation) and idles. /// /// TODO: Implement PowerShelf monitoring (health checks, status updates, /// power consumption / efficiency tracking). @@ -64,10 +66,68 @@ pub async fn handle_ready( )); } + if let Some(req) = &state.power_shelf_reprovisioning_requested { + if !ctx.services.rack_firmware_reprovisioning_enabled { + tracing::info!( + power_shelf_id = %power_shelf_id, + initiator = %req.initiator, + "Rack reprovision request ignored; rack_firmware_reprovisioning_enabled is false" + ); + let mut txn = ctx.services.db_pool.begin().await?; + db_power_shelf::clear_power_shelf_reprovisioning_requested( + txn.as_mut(), + *power_shelf_id, + ) + .await?; + return Ok(StateHandlerOutcome::do_nothing().with_txn(txn)); + } + + if !req.initiator.starts_with("rack-") { + tracing::warn!( + initiator = %req.initiator, + "Unknown initiator for power shelf reprovisioning request", + ); + return Ok(StateHandlerOutcome::transition( + PowerShelfControllerState::Error { + cause: format!( + "unknown initiator for power shelf reprovisioning request: {}", + req.initiator + ), + }, + )); + } + + let Some(reprovisioning_state) = first_reprovisioning_state(req) else { + tracing::warn!( + power_shelf_id = %power_shelf_id, + initiator = %req.initiator, + "Rack reprovision request has no power-shelf-relevant activities; clearing request" + ); + let mut txn = ctx.services.db_pool.begin().await?; + db_power_shelf::clear_power_shelf_reprovisioning_requested( + txn.as_mut(), + *power_shelf_id, + ) + .await?; + return Ok(StateHandlerOutcome::do_nothing().with_txn(txn)); + }; + + tracing::info!( + ?reprovisioning_state, + "Rack-level reprovisioning requested — entering ReProvisioning" + ); + return Ok(StateHandlerOutcome::transition( + PowerShelfControllerState::ReProvisioning { + reprovisioning_state, + }, + )); + } + let txn = poll_power_state(power_shelf_id, state, ctx).await; Ok(StateHandlerOutcome::do_nothing().with_txn_opt(txn)) } + /// /// On a successful response, the observed power state for this power shelf is /// persisted to the `power_shelves.status` column and the in-memory `state` diff --git a/crates/power-shelf-controller/src/reprovisioning.rs b/crates/power-shelf-controller/src/reprovisioning.rs new file mode 100644 index 0000000000..63edd77757 --- /dev/null +++ b/crates/power-shelf-controller/src/reprovisioning.rs @@ -0,0 +1,183 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +//! Handler for PowerShelfControllerState::ReProvisioning. + +use carbide_uuid::power_shelf::PowerShelfId; +use db::db_read::PgPoolReader; +use db::{ObjectColumnFilter, power_shelf as db_power_shelf, rack as db_rack}; +use model::power_shelf::{ + PowerShelf, PowerShelfControllerState, PowerShelfReprovisionRequest, ReProvisioningState, +}; +use model::rack::{MaintenanceActivity, RackState}; +use state_controller::state_handler::{ + StateHandlerContext, StateHandlerError, StateHandlerOutcome, +}; + +use crate::context::PowerShelfStateHandlerContextObjects; + +fn is_rack_level_reprovisioning(state: &PowerShelf) -> bool { + state + .power_shelf_reprovisioning_requested + .as_ref() + .is_some_and(|req| req.initiator.starts_with("rack-")) +} + +fn should_run(activities: &[MaintenanceActivity], activity: &MaintenanceActivity) -> bool { + activities.is_empty() || activities.iter().any(|a| a.same_kind(activity)) +} + +fn firmware_upgrade_requested(activities: &[MaintenanceActivity]) -> bool { + should_run( + activities, + &MaintenanceActivity::FirmwareUpgrade { + firmware_version: None, + components: vec![], + force_update: false, + }, + ) +} + +/// First ReProvisioning sub-state to enter from Ready, based on the request +/// activities. Empty activities means all phases. Returns `None` when the +/// activities list has no power-shelf-relevant wait phases. +pub(crate) fn first_reprovisioning_state( + request: &PowerShelfReprovisionRequest, +) -> Option { + if firmware_upgrade_requested(&request.activities) { + return Some(ReProvisioningState::WaitingForRackFirmwareUpgrade); + } + None +} + +/// If the parent rack is in `RackState::Error`, clear +/// `power_shelf_reprovisioning_requested` and short-circuit to `Ready`. +async fn rack_failed_abort_outcome( + power_shelf_id: &PowerShelfId, + state: &PowerShelf, + ctx: &mut StateHandlerContext<'_, PowerShelfStateHandlerContextObjects>, +) -> Result>, StateHandlerError> { + if !is_rack_level_reprovisioning(state) { + return Ok(None); + } + + let Some(rack_id) = state.rack_id.as_ref() else { + return Ok(None); + }; + + let mut reader: PgPoolReader = ctx.services.db_pool.clone().into(); + let racks = db_rack::find_by( + reader.as_mut(), + ObjectColumnFilter::One(db_rack::IdColumn, rack_id), + ) + .await?; + let Some(rack) = racks.into_iter().next() else { + return Ok(None); + }; + if !matches!(rack.controller_state.value, RackState::Error { .. }) { + return Ok(None); + } + + tracing::info!( + power_shelf_id = %power_shelf_id, + rack_id = %rack_id, + "Rack is in Error; aborting power shelf ReProvisioning and returning to Ready", + ); + + let mut txn = ctx.services.db_pool.begin().await?; + db_power_shelf::clear_power_shelf_reprovisioning_requested(txn.as_mut(), *power_shelf_id) + .await?; + Ok(Some( + StateHandlerOutcome::transition(PowerShelfControllerState::Ready).with_txn(txn), + )) +} + +/// Handles the ReProvisioning state for a power shelf. +pub async fn handle_reprovisioning( + power_shelf_id: &PowerShelfId, + state: &mut PowerShelf, + ctx: &mut StateHandlerContext<'_, PowerShelfStateHandlerContextObjects>, +) -> Result, StateHandlerError> { + let reprovisioning_state = match &state.controller_state.value { + PowerShelfControllerState::ReProvisioning { + reprovisioning_state, + } => reprovisioning_state, + _ => unreachable!("handle_reprovisioning called with non-ReProvisioning state"), + }; + + if let Some(outcome) = rack_failed_abort_outcome(power_shelf_id, state, ctx).await? { + return Ok(outcome); + } + + match reprovisioning_state { + ReProvisioningState::WaitingForRackFirmwareUpgrade => { + let request = state + .power_shelf_reprovisioning_requested + .as_ref() + .expect("WaitingForRackFirmwareUpgrade requires a rack reprovision request"); + let requested_at = request.requested_at; + let Some(firmware_upgrade_status) = state.firmware_upgrade_status.as_ref() else { + return Ok(StateHandlerOutcome::wait( + "waiting for power shelf firmware upgrade status".into(), + )); + }; + if !firmware_upgrade_status.is_current_for(requested_at) { + return Ok(StateHandlerOutcome::wait( + "waiting for current rack firmware cycle".into(), + )); + } + if !firmware_upgrade_status.is_terminal() { + return Ok(StateHandlerOutcome::wait( + "waiting for power shelf firmware completion".into(), + )); + } + + match &firmware_upgrade_status.status { + model::rack::RackFirmwareUpgradeState::Completed => { + let mut txn = ctx.services.db_pool.begin().await?; + db_power_shelf::clear_power_shelf_reprovisioning_requested( + txn.as_mut(), + *power_shelf_id, + ) + .await?; + Ok( + StateHandlerOutcome::transition(PowerShelfControllerState::Ready) + .with_txn(txn), + ) + } + model::rack::RackFirmwareUpgradeState::Failed { cause } => { + let mut txn = ctx.services.db_pool.begin().await?; + db_power_shelf::clear_power_shelf_reprovisioning_requested( + txn.as_mut(), + *power_shelf_id, + ) + .await?; + Ok( + StateHandlerOutcome::transition(PowerShelfControllerState::Error { + cause: cause.clone(), + }) + .with_txn(txn), + ) + } + model::rack::RackFirmwareUpgradeState::Started + | model::rack::RackFirmwareUpgradeState::InProgress => Ok( + StateHandlerOutcome::wait("waiting for power shelf firmware completion".into()), + ), + } + } + } +} diff --git a/crates/rack-controller/src/maintenance.rs b/crates/rack-controller/src/maintenance.rs index 42c78e923e..ccb51ea624 100644 --- a/crates/rack-controller/src/maintenance.rs +++ b/crates/rack-controller/src/maintenance.rs @@ -43,7 +43,8 @@ use carbide_utils::none_if_empty::NoneIfEmpty; use carbide_uuid::rack::{RackId, RackProfileId}; use db::{ host_machine_update as db_host_machine_update, machine as db_machine, - machine_topology as db_machine_topology, rack as db_rack, switch as db_switch, + machine_topology as db_machine_topology, power_shelf as db_power_shelf, rack as db_rack, + switch as db_switch, }; use librms::protos::rack_manager as rms; use model::rack::{ @@ -96,7 +97,8 @@ async fn trigger_rack_firmware_reprovisioning_requests( rack_id: &RackId, machine_ids: &[carbide_uuid::machine::MachineId], switch_ids: &[carbide_uuid::switch::SwitchId], - continue_after_firmware_upgrade: bool, + power_shelf_ids: &[carbide_uuid::power_shelf::PowerShelfId], + activities: &[MaintenanceActivity], ) -> Result<(), StateHandlerError> { for machine_id in machine_ids { db_host_machine_update::trigger_host_reprovisioning_request( @@ -107,11 +109,20 @@ async fn trigger_rack_firmware_reprovisioning_requests( .await?; } for switch_id in switch_ids { - db_switch::set_switch_reprovisioning_requested_with_firmware_continuation( + db_switch::set_switch_reprovisioning_requested( txn, *switch_id, &format!("rack-{}", rack_id), - continue_after_firmware_upgrade, + activities.to_vec(), + ) + .await?; + } + for power_shelf_id in power_shelf_ids { + db_power_shelf::set_power_shelf_reprovisioning_requested( + txn, + *power_shelf_id, + &format!("rack-{}", rack_id), + activities.to_vec(), ) .await?; } @@ -122,6 +133,7 @@ async fn clear_rack_firmware_device_statuses( txn: &mut sqlx::PgConnection, machine_ids: &[carbide_uuid::machine::MachineId], switch_ids: &[carbide_uuid::switch::SwitchId], + power_shelf_ids: &[carbide_uuid::power_shelf::PowerShelfId], ) -> Result<(), StateHandlerError> { for machine_id in machine_ids { db_machine::update_rack_fw_details(txn, machine_id, None).await?; @@ -129,6 +141,9 @@ async fn clear_rack_firmware_device_statuses( for switch_id in switch_ids { db_switch::update_firmware_upgrade_status(txn, *switch_id, None).await?; } + for power_shelf_id in power_shelf_ids { + db_power_shelf::update_firmware_upgrade_status(txn, *power_shelf_id, None).await?; + } Ok(()) } @@ -142,6 +157,336 @@ async fn clear_nvos_update_statuses( Ok(()) } +/// Aggregated firmware progress for machines/switches participating in a rack +/// firmware job. Advancement out of `WaitForComplete` is based on machine and +/// switch controller states, not RMS job strings or `rack_fw_details`. +#[derive(Debug, Clone, PartialEq, Eq)] +enum DeviceFirmwareProgress { + Waiting { + pending: usize, + total: usize, + completed: usize, + failed: usize, + }, + Failed { + failed: usize, + total: usize, + }, + Completed { + completed: usize, + total: usize, + }, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum DeviceFirmwareOutcome { + Waiting, + Failed, + Completed, +} + +async fn resolve_machine_id_for_firmware_device( + txn: &mut sqlx::PgConnection, + device: &FirmwareUpgradeDeviceStatus, +) -> Result, StateHandlerError> { + if !device.node_id.is_empty() { + return Ok(device + .node_id + .parse::() + .ok()); + } + let mac: mac_address::MacAddress = match device.mac.parse() { + Ok(mac) => mac, + Err(_) => return Ok(None), + }; + Ok(db_machine_topology::find_machine_id_by_bmc_mac(txn, mac).await?) +} + +async fn resolve_switch_id_for_firmware_device( + txn: &mut sqlx::PgConnection, + rack_id: &RackId, + device: &FirmwareUpgradeDeviceStatus, +) -> Result, StateHandlerError> { + if !device.node_id.is_empty() { + return Ok(device + .node_id + .parse::() + .ok()); + } + let mac: mac_address::MacAddress = match device.mac.parse() { + Ok(mac) => mac, + Err(_) => return Ok(None), + }; + Ok(db_switch::find_ids( + txn, + model::switch::SwitchSearchFilter { + bmc_mac: Some(mac), + rack_id: Some(rack_id.clone()), + ..Default::default() + }, + ) + .await? + .first() + .copied()) +} + +async fn resolve_power_shelf_id_for_firmware_device( + txn: &mut sqlx::PgConnection, + rack_id: &RackId, + device: &FirmwareUpgradeDeviceStatus, +) -> Result, StateHandlerError> { + if !device.node_id.is_empty() { + return Ok(device + .node_id + .parse::() + .ok()); + } + let mac: mac_address::MacAddress = match device.mac.parse() { + Ok(mac) => mac, + Err(_) => return Ok(None), + }; + Ok(db_power_shelf::find_ids( + txn, + model::power_shelf::PowerShelfSearchFilter { + bmc_mac: Some(mac), + rack_id: Some(rack_id.clone()), + ..Default::default() + }, + ) + .await? + .first() + .copied()) +} + +fn machine_firmware_outcome(machine: &model::machine::Machine) -> DeviceFirmwareOutcome { + match &machine.state.value { + model::machine::ManagedHostState::HostReprovision { + reprovision_state: model::machine::HostReprovisionState::WaitingForRackFirmwareUpgrade, + .. + } => DeviceFirmwareOutcome::Waiting, + model::machine::ManagedHostState::HostReprovision { + reprovision_state: model::machine::HostReprovisionState::FailedFirmwareUpgrade { .. }, + .. + } + | model::machine::ManagedHostState::Failed { .. } => DeviceFirmwareOutcome::Failed, + // Machine has left WaitingForRackFirmwareUpgrade for a later + // HostReprovision sub-state (success path). + model::machine::ManagedHostState::HostReprovision { .. } => { + DeviceFirmwareOutcome::Completed + } + // Request posted but controller has not entered the wait state yet. + _ if machine.host_reprovision_requested.is_some() => DeviceFirmwareOutcome::Waiting, + _ => DeviceFirmwareOutcome::Completed, + } +} + +fn switch_firmware_outcome(switch: &model::switch::Switch) -> DeviceFirmwareOutcome { + match &switch.controller_state.value { + model::switch::SwitchControllerState::ReProvisioning { + reprovisioning_state: model::switch::ReProvisioningState::WaitingForRackFirmwareUpgrade, + } => DeviceFirmwareOutcome::Waiting, + model::switch::SwitchControllerState::ReProvisioning { + reprovisioning_state: + model::switch::ReProvisioningState::WaitingForNVOSUpgrade + | model::switch::ReProvisioningState::WaitingForNMXCConfigure, + } => DeviceFirmwareOutcome::Completed, + model::switch::SwitchControllerState::Error { .. } => DeviceFirmwareOutcome::Failed, + // Request posted but controller has not entered the wait state yet. + model::switch::SwitchControllerState::Ready + if switch.switch_reprovisioning_requested.is_some() => + { + DeviceFirmwareOutcome::Waiting + } + _ => DeviceFirmwareOutcome::Completed, + } +} + +fn power_shelf_firmware_outcome( + power_shelf: &model::power_shelf::PowerShelf, +) -> DeviceFirmwareOutcome { + match &power_shelf.controller_state.value { + model::power_shelf::PowerShelfControllerState::ReProvisioning { + reprovisioning_state: + model::power_shelf::ReProvisioningState::WaitingForRackFirmwareUpgrade, + } => DeviceFirmwareOutcome::Waiting, + model::power_shelf::PowerShelfControllerState::Error { .. } => { + DeviceFirmwareOutcome::Failed + } + // Request posted but controller has not entered the wait state yet + // (or is about to clear it when rack_firmware_reprovisioning_enabled + // is false). + model::power_shelf::PowerShelfControllerState::Ready + if power_shelf.power_shelf_reprovisioning_requested.is_some() => + { + DeviceFirmwareOutcome::Waiting + } + _ => DeviceFirmwareOutcome::Completed, + } +} + +fn summarize_firmware_outcomes(outcomes: &[DeviceFirmwareOutcome]) -> DeviceFirmwareProgress { + let total = outcomes.len(); + let completed = outcomes + .iter() + .filter(|outcome| matches!(outcome, DeviceFirmwareOutcome::Completed)) + .count(); + let failed = outcomes + .iter() + .filter(|outcome| matches!(outcome, DeviceFirmwareOutcome::Failed)) + .count(); + let pending = outcomes + .iter() + .filter(|outcome| matches!(outcome, DeviceFirmwareOutcome::Waiting)) + .count(); + if pending > 0 { + return DeviceFirmwareProgress::Waiting { + pending, + total, + completed, + failed, + }; + } + if failed > 0 { + return DeviceFirmwareProgress::Failed { failed, total }; + } + DeviceFirmwareProgress::Completed { completed, total } +} + +/// Reads machine and switch controller states for devices in `rack_id`, +/// filtered by `scope`, and decides whether firmware WaitForComplete can +/// advance. Device membership comes from the DB + scope, not the firmware job. +async fn evaluate_firmware_progress_from_devices( + txn: &mut sqlx::PgConnection, + rack_id: &RackId, + scope: &MaintenanceScope, +) -> Result { + let machine_ids = db_machine::find_machine_ids( + &mut *txn, + model::machine::machine_search_config::MachineSearchConfig { + rack_id: Some(rack_id.clone()), + ..Default::default() + }, + ) + .await?; + let machines = if machine_ids.is_empty() { + Vec::new() + } else { + db_machine::find( + &mut *txn, + db::ObjectFilter::List(&machine_ids), + model::machine::machine_search_config::MachineSearchConfig::default(), + ) + .await? + }; + let machines = filter_machines_by_scope(machines, scope); + + let switch_ids = db_switch::find_ids( + &mut *txn, + model::switch::SwitchSearchFilter { + rack_id: Some(rack_id.clone()), + deleted: model::DeletedFilter::Exclude, + ..Default::default() + }, + ) + .await?; + let switches = if switch_ids.is_empty() { + Vec::new() + } else { + db_switch::find_by( + txn, + db::ObjectColumnFilter::List(db_switch::IdColumn, &switch_ids), + ) + .await? + }; + let switches = filter_switches_by_scope(switches, scope); + + let power_shelf_ids = db_power_shelf::find_ids( + &mut *txn, + model::power_shelf::PowerShelfSearchFilter { + rack_id: Some(rack_id.clone()), + deleted: model::DeletedFilter::Exclude, + ..Default::default() + }, + ) + .await?; + let power_shelves = if power_shelf_ids.is_empty() { + Vec::new() + } else { + db_power_shelf::find_by( + txn, + db::ObjectColumnFilter::List(db_power_shelf::IdColumn, &power_shelf_ids), + ) + .await? + }; + let power_shelves = filter_power_shelves_by_scope(power_shelves, scope); + + let mut outcomes = Vec::with_capacity(machines.len() + switches.len() + power_shelves.len()); + outcomes.extend(machines.iter().map(machine_firmware_outcome)); + outcomes.extend(switches.iter().map(switch_firmware_outcome)); + outcomes.extend(power_shelves.iter().map(power_shelf_firmware_outcome)); + Ok(summarize_firmware_outcomes(&outcomes)) +} + +fn filter_machines_by_scope( + mut machines: Vec, + scope: &MaintenanceScope, +) -> Vec { + if scope.is_full_rack() { + return machines; + } + if scope.machine_ids.is_empty() { + return Vec::new(); + } + let allowed: std::collections::HashSet<_> = scope.machine_ids.iter().collect(); + machines.retain(|machine| allowed.contains(&machine.id)); + machines +} + +fn filter_switches_by_scope( + mut switches: Vec, + scope: &MaintenanceScope, +) -> Vec { + if scope.is_full_rack() { + return switches; + } + if scope.switch_ids.is_empty() { + return Vec::new(); + } + let allowed: std::collections::HashSet<_> = scope.switch_ids.iter().collect(); + switches.retain(|switch| allowed.contains(&switch.id)); + switches +} + +fn filter_power_shelves_by_scope( + mut power_shelves: Vec, + scope: &MaintenanceScope, +) -> Vec { + if scope.is_full_rack() { + return power_shelves; + } + if scope.power_shelf_ids.is_empty() { + return Vec::new(); + } + let allowed: std::collections::HashSet<_> = scope.power_shelf_ids.iter().collect(); + power_shelves.retain(|power_shelf| allowed.contains(&power_shelf.id)); + power_shelves +} + +fn filter_power_shelf_ids_by_scope( + mut power_shelf_ids: Vec, + scope: &MaintenanceScope, +) -> Vec { + if scope.is_full_rack() { + return power_shelf_ids; + } + if scope.power_shelf_ids.is_empty() { + return Vec::new(); + } + let allowed: std::collections::HashSet<_> = scope.power_shelf_ids.iter().collect(); + power_shelf_ids.retain(|id| allowed.contains(id)); + power_shelf_ids +} + fn skip_firmware_upgrade_outcome( rack_id: &RackId, reason: impl AsRef, @@ -361,6 +706,36 @@ pub(crate) fn first_maintenance_state(scope: &MaintenanceScope) -> RackMaintenan } } +/// Returns the state to advance to when the current maintenance state's +/// activity is not requested. Every [`RackMaintenanceState`] variant is +/// checked here so a persisted or stale state cannot execute outside the +/// current scope. +fn next_state_if_activity_not_requested( + maintenance_state: &RackMaintenanceState, + scope: &MaintenanceScope, +) -> Option { + match maintenance_state { + RackMaintenanceState::FirmwareUpgrade { .. } => { + (!scope.should_run(&MaintenanceActivity::FirmwareUpgrade { + firmware_version: None, + components: vec![], + force_update: false, + })) + .then(|| next_state_after_firmware(scope)) + } + RackMaintenanceState::NVOSUpdate { .. } => { + (!nvos_update_requested(scope)).then(|| next_state_after_nvos(scope)) + } + RackMaintenanceState::ConfigureNmxCluster { .. } => (!scope + .should_run(&MaintenanceActivity::ConfigureNmxCluster)) + .then(|| next_state_after_configure(scope)), + RackMaintenanceState::PowerSequence { .. } => (!scope + .should_run(&MaintenanceActivity::PowerSequence)) + .then_some(RackMaintenanceState::Completed), + RackMaintenanceState::Completed => None, + } +} + /// Filters a full-rack firmware inventory down to compute/switch devices listed /// in the maintenance scope. Power-shelf firmware-object JSON apply is not /// implemented yet. @@ -1304,6 +1679,18 @@ pub async fn handle_maintenance( .unwrap_or_default(); let scope = &scope; + if let Some(next) = next_state_if_activity_not_requested(maintenance_state, scope) { + tracing::info!( + rack_id = %id, + current_state = %maintenance_state, + next_state = %next, + "Skipping rack maintenance state not requested by scope activities" + ); + return Ok(StateHandlerOutcome::transition(RackState::Maintenance { + maintenance_state: next, + })); + } + match maintenance_state { RackMaintenanceState::FirmwareUpgrade { rack_firmware_upgrade, @@ -1462,19 +1849,30 @@ pub async fn handle_maintenance( }; let mut txn = ctx.services.db_pool.begin().await?; - let continue_after_firmware_upgrade = nvos_update_requested(scope); + let power_shelf_ids = db_power_shelf::find_ids( + txn.as_mut(), + model::power_shelf::PowerShelfSearchFilter { + rack_id: Some(id.clone()), + deleted: model::DeletedFilter::Exclude, + ..Default::default() + }, + ) + .await?; + let power_shelf_ids = filter_power_shelf_ids_by_scope(power_shelf_ids, scope); trigger_rack_firmware_reprovisioning_requests( txn.as_mut(), id, &inventory.machine_ids, &inventory.switch_ids, - continue_after_firmware_upgrade, + &power_shelf_ids, + &scope.activities, ) .await?; clear_rack_firmware_device_statuses( txn.as_mut(), &inventory.machine_ids, &inventory.switch_ids, + &power_shelf_ids, ) .await?; job.started_at = Some(chrono::Utc::now()); @@ -1509,18 +1907,6 @@ pub async fn handle_maintenance( let mut job = rms_get_firmware_upgrade_status(rms_client.as_ref(), current_job).await?; - let all: Vec<_> = job.all_devices().collect(); - let total = all.len(); - let completed = all.iter().filter(|d| d.status == "completed").count(); - let failed = all.iter().filter(|d| d.status == "failed").count(); - let terminal = completed + failed; - if failed > 0 && requested_nvos_config_json(scope).is_some() { - delete_rack_maintenance_access_token( - ctx.services.credential_manager.as_ref(), - id, - ) - .await; - } let mut txn = ctx.services.db_pool.begin().await?; let build_status = @@ -1551,19 +1937,9 @@ pub async fn handle_maintenance( }; for device in job.machines.iter() { - let machine_id = if !device.node_id.is_empty() { - device - .node_id - .parse::() - .ok() - } else { - let mac: mac_address::MacAddress = match device.mac.parse() { - Ok(mac) => mac, - Err(_) => continue, - }; - db_machine_topology::find_machine_id_by_bmc_mac(txn.as_mut(), mac).await? - }; - if let Some(machine_id) = machine_id { + if let Some(machine_id) = + resolve_machine_id_for_firmware_device(txn.as_mut(), device).await? + { let fw_status = build_status(device); db_machine::update_rack_fw_details( txn.as_mut(), @@ -1575,29 +1951,9 @@ pub async fn handle_maintenance( } for device in job.switches.iter() { - let switch_id = if !device.node_id.is_empty() { - device - .node_id - .parse::() - .ok() - } else { - let mac: mac_address::MacAddress = match device.mac.parse() { - Ok(mac) => mac, - Err(_) => continue, - }; - db_switch::find_ids( - txn.as_mut(), - model::switch::SwitchSearchFilter { - bmc_mac: Some(mac), - rack_id: Some(id.clone()), - ..Default::default() - }, - ) - .await? - .first() - .copied() - }; - if let Some(switch_id) = switch_id { + if let Some(switch_id) = + resolve_switch_id_for_firmware_device(txn.as_mut(), id, device).await? + { let fw_status = build_status(device); db_switch::update_firmware_upgrade_status( txn.as_mut(), @@ -1608,71 +1964,140 @@ pub async fn handle_maintenance( } } - if terminal < total { - db_rack::update_firmware_upgrade_job(txn.as_mut(), id, Some(&job)).await?; - state.firmware_upgrade_job = Some(job); - return Ok(StateHandlerOutcome::wait(format!( - "firmware upgrade: {}/{} devices terminal (completed={}, failed={})", - terminal, total, completed, failed - )) - .with_txn(txn)); - } - - if failed > 0 { - let now = chrono::Utc::now(); - job.status = Some("failed".into()); - if job.completed_at.is_none() { - job.completed_at = Some(now); - } - db_rack::update_firmware_upgrade_job(txn.as_mut(), id, Some(&job)).await?; - state.firmware_upgrade_job = Some(job); - if state.config.maintenance_requested.is_some() { - state.config.maintenance_requested = None; - db_rack::update(txn.as_mut(), id, &state.config).await?; + for device in job.power_shelves.iter() { + if let Some(power_shelf_id) = + resolve_power_shelf_id_for_firmware_device(txn.as_mut(), id, device).await? + { + let fw_status = build_status(device); + db_power_shelf::update_firmware_upgrade_status( + txn.as_mut(), + power_shelf_id, + Some(&fw_status), + ) + .await?; } - return Ok(StateHandlerOutcome::transition(RackState::Error { - cause: format!( - "firmware upgrade failed: {}/{} devices failed", - failed, total - ), - }) - .with_txn(txn)); } - let now = chrono::Utc::now(); - job.status = Some("completed".into()); - if job.completed_at.is_none() { - job.completed_at = Some(now); + // When RMS does not yet report power-shelf devices, stamp + // Completed for scoped shelves so enabled controllers can leave + // WaitingForRackFirmwareUpgrade without hanging. + if job.power_shelves.is_empty() { + let power_shelf_ids = db_power_shelf::find_ids( + txn.as_mut(), + model::power_shelf::PowerShelfSearchFilter { + rack_id: Some(id.clone()), + deleted: model::DeletedFilter::Exclude, + ..Default::default() + }, + ) + .await?; + let power_shelf_ids = filter_power_shelf_ids_by_scope(power_shelf_ids, scope); + if !power_shelf_ids.is_empty() { + let fw_status = RackFirmwareUpgradeStatus { + task_id: job.job_id.clone().unwrap_or_else(|| "unknown".to_string()), + status: RackFirmwareUpgradeState::Completed, + started_at: job.started_at, + ended_at: Some(chrono::Utc::now()), + }; + for power_shelf_id in power_shelf_ids { + db_power_shelf::update_firmware_upgrade_status( + txn.as_mut(), + power_shelf_id, + Some(&fw_status), + ) + .await?; + } + } } - db_rack::update_firmware_upgrade_job(txn.as_mut(), id, Some(&job)).await?; - state.firmware_upgrade_job = Some(job); - let next_maintenance_state = if nvos_update_requested(scope) { - let next = next_state_after_firmware(scope); - tracing::info!( - rack_id = %id, - completed_device_count = completed, - total_device_count = total, - next_state = %next, - "Rack firmware upgrade complete; advancing to explicitly requested next activity" - ); - next - } else { - let next = next_state_after_nvos(scope); - tracing::info!( - rack_id = %id, - completed_device_count = completed, - total_device_count = total, - next_state = %next, - "Rack firmware upgrade complete; no explicit NVOS update requested, advancing" - ); - next - }; + // Advancement is driven by machine/switch/power-shelf controller + // states for devices in this rack that are selected by the + // maintenance scope. + let progress = + evaluate_firmware_progress_from_devices(txn.as_mut(), id, scope).await?; + + match progress { + DeviceFirmwareProgress::Waiting { + pending: _, + total, + completed, + failed, + } => { + db_rack::update_firmware_upgrade_job(txn.as_mut(), id, Some(&job)).await?; + state.firmware_upgrade_job = Some(job); + return Ok(StateHandlerOutcome::wait(format!( + "firmware upgrade: waiting on machine/switch/power-shelf controller state ({}/{} past firmware wait, completed={}, failed={})", + completed + failed, + total, + completed, + failed + )) + .with_txn(txn)); + } + DeviceFirmwareProgress::Failed { failed, total } => { + if requested_nvos_config_json(scope).is_some() { + delete_rack_maintenance_access_token( + ctx.services.credential_manager.as_ref(), + id, + ) + .await; + } + let now = chrono::Utc::now(); + job.status = Some("failed".into()); + if job.completed_at.is_none() { + job.completed_at = Some(now); + } + db_rack::update_firmware_upgrade_job(txn.as_mut(), id, Some(&job)).await?; + state.firmware_upgrade_job = Some(job); + if state.config.maintenance_requested.is_some() { + state.config.maintenance_requested = None; + db_rack::update(txn.as_mut(), id, &state.config).await?; + } + return Ok(StateHandlerOutcome::transition(RackState::Error { + cause: format!( + "firmware upgrade failed: {}/{} devices failed", + failed, total + ), + }) + .with_txn(txn)); + } + DeviceFirmwareProgress::Completed { completed, total } => { + let now = chrono::Utc::now(); + job.status = Some("completed".into()); + if job.completed_at.is_none() { + job.completed_at = Some(now); + } + db_rack::update_firmware_upgrade_job(txn.as_mut(), id, Some(&job)).await?; + state.firmware_upgrade_job = Some(job); + + let next_maintenance_state = if nvos_update_requested(scope) { + let next = next_state_after_firmware(scope); + tracing::info!( + rack_id = %id, + completed_device_count = completed, + total_device_count = total, + next_state = %next, + "Rack firmware upgrade complete on machine/switch controllers; advancing to explicitly requested next activity" + ); + next + } else { + let next = next_state_after_nvos(scope); + tracing::info!( + rack_id = %id, + completed_device_count = completed, + total_device_count = total, + next_state = %next, + "Rack firmware upgrade complete on machine/switch controllers; no explicit NVOS update requested, advancing" + ); + next + }; - Ok(StateHandlerOutcome::transition(RackState::Maintenance { - maintenance_state: next_maintenance_state, - }) - .with_txn(txn)) + Ok(StateHandlerOutcome::transition(RackState::Maintenance { + maintenance_state: next_maintenance_state, + }) + .with_txn(txn)) + } + } } }, RackMaintenanceState::NVOSUpdate { nvos_update } => match nvos_update { @@ -2480,9 +2905,11 @@ mod tests { use model::rack_type::{RackHardwareType, RackProductFamily, RackProfile}; use super::{ - build_switch_device_info_request, filter_inventory_by_scope, firmware_device_status, - first_maintenance_state, next_state_after_configure, next_state_after_firmware, - next_state_after_nvos, profile_hardware_type_or_any, + DeviceFirmwareOutcome, DeviceFirmwareProgress, build_switch_device_info_request, + filter_inventory_by_scope, firmware_device_status, first_maintenance_state, + next_state_after_configure, next_state_after_firmware, next_state_after_nvos, + next_state_if_activity_not_requested, profile_hardware_type_or_any, + summarize_firmware_outcomes, }; fn test_machine_id(seed: u8) -> MachineId { @@ -2679,6 +3106,51 @@ mod tests { assert_eq!(status.error_message.as_deref(), Some("invalid SOT JSON")); } + #[test] + fn test_summarize_firmware_outcomes_from_controller_states() { + check_values( + [ + Check { + scenario: "any waiting keeps rack waiting", + input: vec![ + DeviceFirmwareOutcome::Completed, + DeviceFirmwareOutcome::Waiting, + DeviceFirmwareOutcome::Failed, + ], + expect: DeviceFirmwareProgress::Waiting { + pending: 1, + total: 3, + completed: 1, + failed: 1, + }, + }, + Check { + scenario: "all completed advances", + input: vec![ + DeviceFirmwareOutcome::Completed, + DeviceFirmwareOutcome::Completed, + ], + expect: DeviceFirmwareProgress::Completed { + completed: 2, + total: 2, + }, + }, + Check { + scenario: "any failed without waiting errors", + input: vec![ + DeviceFirmwareOutcome::Completed, + DeviceFirmwareOutcome::Failed, + ], + expect: DeviceFirmwareProgress::Failed { + failed: 1, + total: 2, + }, + }, + ], + |outcomes| summarize_firmware_outcomes(&outcomes), + ); + } + /// A firmware-upgrade activity with no version/components/force, the form /// used by the maintenance-state transition tables. fn firmware_upgrade() -> MaintenanceActivity { @@ -2726,6 +3198,82 @@ mod tests { } } + // ── current-state scope enforcement ───────────────────────────────── + + #[test] + fn test_next_state_if_activity_not_requested() { + check_values( + [ + Check { + scenario: "requested firmware continues", + input: (firmware_start(), scope_of(vec![firmware_upgrade()])), + expect: None, + }, + Check { + scenario: "unrequested firmware skips to configure", + input: ( + firmware_start(), + scope_of(vec![MaintenanceActivity::ConfigureNmxCluster]), + ), + expect: Some(configure_start()), + }, + Check { + scenario: "requested nvos continues", + input: (nvos_start(), scope_of(vec![nvos_update()])), + expect: None, + }, + Check { + scenario: "unrequested nvos skips to configure", + input: ( + nvos_start(), + scope_of(vec![MaintenanceActivity::ConfigureNmxCluster]), + ), + expect: Some(configure_start()), + }, + Check { + scenario: "requested configure continues", + input: ( + configure_start(), + scope_of(vec![MaintenanceActivity::ConfigureNmxCluster]), + ), + expect: None, + }, + Check { + scenario: "unrequested configure skips to power", + input: ( + configure_start(), + scope_of(vec![MaintenanceActivity::PowerSequence]), + ), + expect: Some(powering_on()), + }, + Check { + scenario: "requested power continues", + input: ( + powering_on(), + scope_of(vec![MaintenanceActivity::PowerSequence]), + ), + expect: None, + }, + Check { + scenario: "unrequested power skips to completed", + input: (powering_on(), scope_of(vec![firmware_upgrade()])), + expect: Some(RackMaintenanceState::Completed), + }, + Check { + scenario: "completed always continues", + input: ( + RackMaintenanceState::Completed, + scope_of(vec![firmware_upgrade()]), + ), + expect: None, + }, + ], + |(maintenance_state, scope)| { + next_state_if_activity_not_requested(&maintenance_state, &scope) + }, + ); + } + // ── first_maintenance_state ───────────────────────────────────────── #[test] diff --git a/crates/rpc/src/model/power_shelf.rs b/crates/rpc/src/model/power_shelf.rs index 234316d869..229f8ddbe7 100644 --- a/crates/rpc/src/model/power_shelf.rs +++ b/crates/rpc/src/model/power_shelf.rs @@ -186,6 +186,8 @@ mod tests { bmc_info: None, rack_id: None, power_shelf_maintenance_requested: None, + power_shelf_reprovisioning_requested: None, + firmware_upgrade_status: None, metadata: Metadata::default(), version: ConfigVersion::initial(), health_reports: Default::default(), diff --git a/crates/switch-controller/src/ready.rs b/crates/switch-controller/src/ready.rs index 06a3def884..a33828eccd 100644 --- a/crates/switch-controller/src/ready.rs +++ b/crates/switch-controller/src/ready.rs @@ -18,24 +18,26 @@ //! Handler for SwitchControllerState::Ready. use carbide_uuid::switch::SwitchId; -use model::switch::{ReProvisioningState, Switch, SwitchControllerState}; +use db::switch as db_switch; +use model::switch::{Switch, SwitchControllerState}; use state_controller::state_handler::{ StateHandlerContext, StateHandlerError, StateHandlerOutcome, }; use crate::context::SwitchStateHandlerContextObjects; +use crate::reprovisioning::first_reprovisioning_state; /// Handles the Ready state for a switch. /// /// If the switch is marked for deletion, transitions to `Deleting`. /// If a maintenance request has been posted via `switch_maintenance_requested`, /// transitions to `Maintenance` with the requested operation. If rack-level -/// reprovisioning has been requested, transitions to `ReProvisioning`. -/// Otherwise idles. +/// reprovisioning has been requested, transitions to the first +/// `ReProvisioning` sub-state selected by the request activities. Otherwise idles. pub async fn handle_ready( - _switch_id: &SwitchId, + switch_id: &SwitchId, state: &mut Switch, - _ctx: &mut StateHandlerContext<'_, SwitchStateHandlerContextObjects>, + ctx: &mut StateHandlerContext<'_, SwitchStateHandlerContextObjects>, ) -> Result, StateHandlerError> { if state.is_marked_as_deleted() { return Ok(StateHandlerOutcome::transition( @@ -55,27 +57,39 @@ pub async fn handle_ready( } if let Some(req) = &state.switch_reprovisioning_requested { - if req.initiator.starts_with("rack-") { - tracing::info!( - "Rack-level firmware upgrade requested — transitioning to WaitingForRackFirmwareUpgrade" + if !req.initiator.starts_with("rack-") { + tracing::warn!( + initiator = %req.initiator, + "Unknown initiator for switch reprovisioning request", ); return Ok(StateHandlerOutcome::transition( - SwitchControllerState::ReProvisioning { - reprovisioning_state: ReProvisioningState::WaitingForRackFirmwareUpgrade, + SwitchControllerState::Error { + cause: format!( + "unknown initiator for switch reprovisioning request: {}", + req.initiator + ), }, )); } - tracing::warn!( - initiator = %req.initiator, - "Unknown initiator for switch reprovisioning request", + let Some(reprovisioning_state) = first_reprovisioning_state(req) else { + tracing::warn!( + switch_id = %switch_id, + initiator = %req.initiator, + "Rack reprovision request has no switch-relevant activities; clearing request" + ); + let mut txn = ctx.services.db_pool.begin().await?; + db_switch::clear_switch_reprovisioning_requested(txn.as_mut(), *switch_id).await?; + return Ok(StateHandlerOutcome::do_nothing().with_txn(txn)); + }; + + tracing::info!( + ?reprovisioning_state, + "Rack-level reprovisioning requested — entering ReProvisioning" ); return Ok(StateHandlerOutcome::transition( - SwitchControllerState::Error { - cause: format!( - "unknown initiator for switch reprovisioning request: {}", - req.initiator - ), + SwitchControllerState::ReProvisioning { + reprovisioning_state, }, )); } diff --git a/crates/switch-controller/src/reprovisioning.rs b/crates/switch-controller/src/reprovisioning.rs index 0e1f5adfac..bdc795c9eb 100644 --- a/crates/switch-controller/src/reprovisioning.rs +++ b/crates/switch-controller/src/reprovisioning.rs @@ -20,8 +20,8 @@ use carbide_uuid::switch::SwitchId; use db::db_read::PgPoolReader; use db::{ObjectColumnFilter, rack as db_rack, switch as db_switch}; -use model::rack::RackState; -use model::switch::{ReProvisioningState, Switch, SwitchControllerState}; +use model::rack::{MaintenanceActivity, RackState}; +use model::switch::{ReProvisioningState, Switch, SwitchControllerState, SwitchReprovisionRequest}; use state_controller::state_handler::{ StateHandlerContext, StateHandlerError, StateHandlerOutcome, }; @@ -37,6 +37,68 @@ fn is_rack_level_reprovisioning(state: &Switch) -> bool { .is_some_and(|req| req.initiator.starts_with("rack-")) } +fn should_run(activities: &[MaintenanceActivity], activity: &MaintenanceActivity) -> bool { + activities.is_empty() || activities.iter().any(|a| a.same_kind(activity)) +} + +fn nvos_update_requested(activities: &[MaintenanceActivity]) -> bool { + activities + .iter() + .any(|activity| matches!(activity, MaintenanceActivity::NvosUpdate { .. })) +} + +fn configure_nmx_cluster_requested(activities: &[MaintenanceActivity]) -> bool { + should_run(activities, &MaintenanceActivity::ConfigureNmxCluster) +} + +fn firmware_upgrade_requested(activities: &[MaintenanceActivity]) -> bool { + should_run( + activities, + &MaintenanceActivity::FirmwareUpgrade { + firmware_version: None, + components: vec![], + force_update: false, + }, + ) +} + +/// First ReProvisioning sub-state to enter from Ready, based on the request +/// activities. Empty activities means all phases. Returns `None` when the +/// activities list has no switch-relevant wait phases. +pub(crate) fn first_reprovisioning_state( + request: &SwitchReprovisionRequest, +) -> Option { + let activities = &request.activities; + if firmware_upgrade_requested(activities) { + return Some(ReProvisioningState::WaitingForRackFirmwareUpgrade); + } + if nvos_update_requested(activities) { + return Some(ReProvisioningState::WaitingForNVOSUpgrade); + } + if configure_nmx_cluster_requested(activities) { + return Some(ReProvisioningState::WaitingForNMXCConfigure); + } + None +} + +/// Next ReProvisioning sub-state after firmware completes. +fn next_state_after_firmware(request: &SwitchReprovisionRequest) -> Option { + let activities = &request.activities; + if nvos_update_requested(activities) { + return Some(ReProvisioningState::WaitingForNVOSUpgrade); + } + if configure_nmx_cluster_requested(activities) { + return Some(ReProvisioningState::WaitingForNMXCConfigure); + } + None +} + +/// Next ReProvisioning sub-state after NVOS completes. +fn next_state_after_nvos(request: &SwitchReprovisionRequest) -> Option { + configure_nmx_cluster_requested(&request.activities) + .then_some(ReProvisioningState::WaitingForNMXCConfigure) +} + /// If the parent rack is in `RackState::Error`, clear /// `switch_reprovisioning_requested` and short-circuit to `Ready`. The /// rack will never advance the remaining `ReProvisioning` sub-states once @@ -107,7 +169,7 @@ pub async fn handle_reprovisioning( .as_ref() .expect("WaitingForRackFirmwareUpgrade requires a rack reprovision request"); let requested_at = request.requested_at; - let continue_after_firmware_upgrade = request.continue_after_firmware_upgrade; + let next_after_firmware = next_state_after_firmware(request); let Some(firmware_upgrade_status) = state.firmware_upgrade_status.as_ref() else { return Ok(StateHandlerOutcome::wait( "waiting for switch firmware upgrade status".into(), @@ -126,10 +188,10 @@ pub async fn handle_reprovisioning( match &firmware_upgrade_status.status { model::rack::RackFirmwareUpgradeState::Completed => { - if continue_after_firmware_upgrade { + if let Some(reprovisioning_state) = next_after_firmware { return Ok(StateHandlerOutcome::transition( SwitchControllerState::ReProvisioning { - reprovisioning_state: ReProvisioningState::WaitingForNVOSUpgrade, + reprovisioning_state, }, )); } @@ -157,11 +219,12 @@ pub async fn handle_reprovisioning( } } ReProvisioningState::WaitingForNVOSUpgrade => { - let requested_at = state + let request = state .switch_reprovisioning_requested .as_ref() - .map(|request| request.requested_at) .expect("WaitingForNVOSUpgrade requires a rack reprovision request"); + let requested_at = request.requested_at; + let next_after_nvos = next_state_after_nvos(request); let Some(nvos_update_status) = state.nvos_update_status.as_ref() else { return Ok(StateHandlerOutcome::wait( "waiting for switch NVOS update status".into(), @@ -179,11 +242,23 @@ pub async fn handle_reprovisioning( } match &nvos_update_status.status { - model::rack::SwitchNvosUpdateState::Completed => Ok( - StateHandlerOutcome::transition(SwitchControllerState::ReProvisioning { - reprovisioning_state: ReProvisioningState::WaitingForNMXCConfigure, - }), - ), + model::rack::SwitchNvosUpdateState::Completed => { + if let Some(reprovisioning_state) = next_after_nvos { + Ok(StateHandlerOutcome::transition( + SwitchControllerState::ReProvisioning { + reprovisioning_state, + }, + )) + } else { + let mut txn = ctx.services.db_pool.begin().await?; + db_switch::clear_switch_reprovisioning_requested(txn.as_mut(), *switch_id) + .await?; + Ok( + StateHandlerOutcome::transition(SwitchControllerState::Ready) + .with_txn(txn), + ) + } + } model::rack::SwitchNvosUpdateState::Failed { cause } => { let mut txn = ctx.services.db_pool.begin().await?; db_switch::clear_switch_reprovisioning_requested(txn.as_mut(), *switch_id)