From 748fa1d9e788a72df07d5d8be7d358844dcb6a17 Mon Sep 17 00:00:00 2001 From: Vinod Chitrali Date: Fri, 24 Jul 2026 08:51:35 +0000 Subject: [PATCH] feat(rack): coordinate firmware upgrades across rack components Route state-controller firmware requests for power shelves through rack maintenance, alongside machines and switches. Persist per-device reprovisioning requests and firmware status so rack orchestration can synchronize component progress and terminal outcomes. Add the power-shelf reprovisioning state machine behind an opt-in configuration flag, including completion, failure, stale-cycle, and rack-error handling. Tighten switch reprovisioning behavior and expand rack maintenance coordination to handle mixed component scopes, retries, and status propagation. Add the database migration, model and RPC wiring, controller configuration, and integration coverage for rack, switch, and power-shelf firmware workflows. --- crates/api-core/src/cfg/README.md | 2 +- crates/api-core/src/cfg/file.rs | 16 + .../src/handlers/component_manager.rs | 193 +++-- crates/api-core/src/setup.rs | 3 + .../src/test_support/default_config.rs | 1 + .../src/tests/common/api_fixtures/mod.rs | 1 + .../error_state.rs | 1 + .../maintenance.rs | 1 + .../tests/power_shelf_state_controller/mod.rs | 2 + .../reprovisioning.rs | 226 +++++ .../tests/rack_state_controller/handler.rs | 141 +++- .../src/tests/switch_state_controller/mod.rs | 94 ++- ...0_power_shelf_reprovisioning_requested.sql | 7 + crates/api-db/src/power_shelf.rs | 64 +- crates/api-db/src/switch.rs | 16 +- crates/api-model/src/power_shelf/mod.rs | 78 ++ crates/api-model/src/power_shelf/slas.rs | 3 + crates/api-model/src/switch/mod.rs | 43 +- crates/power-shelf-controller/src/context.rs | 5 + crates/power-shelf-controller/src/handler.rs | 4 + crates/power-shelf-controller/src/io.rs | 10 + crates/power-shelf-controller/src/lib.rs | 1 + crates/power-shelf-controller/src/ready.rs | 66 +- .../src/reprovisioning.rs | 183 ++++ crates/rack-controller/src/maintenance.rs | 780 +++++++++++++++--- crates/rpc/src/model/power_shelf.rs | 2 + crates/switch-controller/src/ready.rs | 50 +- .../switch-controller/src/reprovisioning.rs | 99 ++- 28 files changed, 1832 insertions(+), 260 deletions(-) create mode 100644 crates/api-core/src/tests/power_shelf_state_controller/reprovisioning.rs create mode 100644 crates/api-db/migrations/20260724120000_power_shelf_reprovisioning_requested.sql create mode 100644 crates/power-shelf-controller/src/reprovisioning.rs diff --git a/crates/api-core/src/cfg/README.md b/crates/api-core/src/cfg/README.md index 94dc2f9e1e..1e017e44e6 100644 --- a/crates/api-core/src/cfg/README.md +++ b/crates/api-core/src/cfg/README.md @@ -58,7 +58,7 @@ applicable. | `ib_partition_state_controller` | `IbPartitionStateControllerConfig` | *(see below)* | `hardware` | IB partition state controller timing. | | `dpa_interface_state_controller` | `DpaInterfaceStateControllerConfig` | *(see below)* | `networking` | DPA interface state controller timing. | | `rack_state_controller` | `RackStateControllerConfig` | *(see below)* | `hardware` | Rack state controller timing. | -| `power_shelf_state_controller` | `PowerShelfStateControllerConfig` | *(see below)* | `hardware` | Power shelf state controller timing. | +| `power_shelf_state_controller` | `PowerShelfStateControllerConfig` | *(see below)* | `hardware` | Power shelf state controller timing and optional rack firmware reprovisioning. | | `switch_state_controller` | `SwitchStateControllerConfig` | *(see below)* | `hardware` | Switch state controller timing. | | `spdm_state_controller` | `SpdmStateControllerConfig` | *(see below)* | `security` | SPDM state controller timing. | | `host_models` | `HashMap` | `{}` | `machines` | Maps host model identifiers to firmware definitions for BMC/UEFI/NIC upgrades. | diff --git a/crates/api-core/src/cfg/file.rs b/crates/api-core/src/cfg/file.rs index edb6b8dbc4..8554fc56d6 100644 --- a/crates/api-core/src/cfg/file.rs +++ b/crates/api-core/src/cfg/file.rs @@ -2309,6 +2309,22 @@ pub struct PowerShelfStateControllerConfig { /// Common state controller configs #[serde(default = "StateControllerConfig::default")] pub controller: StateControllerConfig, + + /// When `true`, the power shelf Ready handler accepts rack-level + /// `power_shelf_reprovisioning_requested` and enters + /// `ReProvisioning::WaitingForRackFirmwareUpgrade`. + /// + /// Defaults to `false` so power shelves stay out of rack firmware wait + /// unless explicitly enabled. + /// + /// Configured in `nico-api-config.toml`: + /// + /// ```toml + /// [power_shelf_state_controller] + /// rack_firmware_reprovisioning_enabled = true + /// ``` + #[serde(default)] + pub rack_firmware_reprovisioning_enabled: bool, } /// RackStateController related config diff --git a/crates/api-core/src/handlers/component_manager.rs b/crates/api-core/src/handlers/component_manager.rs index ede6ca557c..21dcbcc823 100644 --- a/crates/api-core/src/handlers/component_manager.rs +++ b/crates/api-core/src/handlers/component_manager.rs @@ -652,6 +652,7 @@ struct RackFirmwareMaintenanceTarget { rack_id: RackId, machine_ids: Vec, switch_ids: Vec, + power_shelf_ids: Vec, } fn push_rack_firmware_target( @@ -659,6 +660,7 @@ fn push_rack_firmware_target( rack_id: RackId, machine_id: Option, switch_id: Option, + power_shelf_id: Option, ) { let target = match targets.iter_mut().find(|target| target.rack_id == rack_id) { Some(target) => target, @@ -667,6 +669,7 @@ fn push_rack_firmware_target( rack_id, machine_ids: Vec::new(), switch_ids: Vec::new(), + power_shelf_ids: Vec::new(), }); targets.last_mut().expect("target was just pushed") } @@ -678,6 +681,9 @@ fn push_rack_firmware_target( if let Some(switch_id) = switch_id { target.switch_ids.push(switch_id); } + if let Some(power_shelf_id) = power_shelf_id { + target.power_shelf_ids.push(power_shelf_id); + } } async fn group_machine_ids_by_rack( @@ -706,7 +712,13 @@ async fn group_machine_ids_by_rack( "machine {machine_id} is not associated with a rack" )) })?; - push_rack_firmware_target(&mut targets, rack_id, Some(machine_id.to_string()), None); + push_rack_firmware_target( + &mut targets, + rack_id, + Some(machine_id.to_string()), + None, + None, + ); } Ok(targets) @@ -853,7 +865,57 @@ async fn group_switch_ids_by_rack( let rack_id = switch.rack_id.clone().ok_or_else(|| { Status::failed_precondition(format!("switch {switch_id} is not associated with a rack")) })?; - push_rack_firmware_target(&mut targets, rack_id, None, Some(switch_id.to_string())); + push_rack_firmware_target( + &mut targets, + rack_id, + None, + Some(switch_id.to_string()), + None, + ); + } + + Ok(targets) +} + +async fn group_power_shelf_ids_by_rack( + api: &Api, + power_shelf_ids: &[PowerShelfId], +) -> Result, Status> { + let mut txn = api + .database_connection + .begin() + .await + .map_err(|e| Status::internal(format!("failed to begin transaction: {e}")))?; + let power_shelves = db::power_shelf::find_by( + &mut txn, + db::ObjectColumnFilter::List(db::power_shelf::IdColumn, power_shelf_ids), + ) + .await + .map_err(|e| Status::internal(format!("failed to look up power shelves: {e}")))?; + drop(txn); + + let power_shelves_by_id: HashMap<_, _> = power_shelves + .into_iter() + .map(|power_shelf| (power_shelf.id, power_shelf)) + .collect(); + + let mut targets = Vec::new(); + for power_shelf_id in power_shelf_ids { + let power_shelf = power_shelves_by_id + .get(power_shelf_id) + .ok_or_else(|| Status::not_found(format!("power shelf {power_shelf_id} not found")))?; + let rack_id = power_shelf.rack_id.clone().ok_or_else(|| { + Status::failed_precondition(format!( + "power shelf {power_shelf_id} is not associated with a rack" + )) + })?; + push_rack_firmware_target( + &mut targets, + rack_id, + None, + None, + Some(power_shelf_id.to_string()), + ); } Ok(targets) @@ -881,6 +943,7 @@ async fn submit_rack_firmware_maintenance_requests( .machine_ids .iter() .chain(target.switch_ids.iter()) + .chain(target.power_shelf_ids.iter()) .cloned() .collect(); let maintenance_req = Request::new(rpc::RackMaintenanceOnDemandRequest { @@ -888,7 +951,7 @@ async fn submit_rack_firmware_maintenance_requests( scope: Some(rpc::RackMaintenanceScope { machine_ids: target.machine_ids, switch_ids: target.switch_ids, - power_shelf_ids: vec![], + power_shelf_ids: target.power_shelf_ids, activities: activities.clone(), }), }); @@ -2345,54 +2408,70 @@ pub(crate) async fn update_component_firmware( let route_through_state_controller = cm.power_shelf_use_state_controller && !bypass_state_controller; if route_through_state_controller { - // TODO: implement state controller path for power shelf firmware control - return Err(Status::unimplemented( - "power shelf firmware control through the state controller is not yet supported", - )); - } - - let options = if cm.power_shelf.supports_firmware_object_json() { - require_firmware_object_json_for_direct_rms( + let token = require_firmware_object_json_for_rack_maintenance( "power shelf", &access_token, &req.target_version, + )?; + let components = map_power_shelf_components(&t.components)?; + let component_names = components + .iter() + .map(|component| match component { + PowerShelfComponent::Pmc => "pmc".to_string(), + PowerShelfComponent::Psu => "psu".to_string(), + }) + .collect(); + maintenance_activities = vec![firmware_upgrade_activity( + req.target_version.clone(), + component_names, + Some(token), force_update, - )? + )]; + rack_maintenance_targets = group_power_shelf_ids_by_rack(api, &list.ids).await?; } else { - reject_power_shelf_firmware_object_json(&access_token)?; - FirmwareUpdateOptions { - force_update, - ..FirmwareUpdateOptions::default() - } - }; - let components = map_power_shelf_components(&t.components)?; - let endpoints = resolve_power_shelf_endpoints(api, &list.ids).await?; + let options = if cm.power_shelf.supports_firmware_object_json() { + require_firmware_object_json_for_direct_rms( + "power shelf", + &access_token, + &req.target_version, + force_update, + )? + } else { + reject_power_shelf_firmware_object_json(&access_token)?; + FirmwareUpdateOptions { + force_update, + ..FirmwareUpdateOptions::default() + } + }; + let components = map_power_shelf_components(&t.components)?; + let endpoints = resolve_power_shelf_endpoints(api, &list.ids).await?; - let mut results: Vec<_> = endpoints - .unresolved - .iter() - .map(|u| error_result(&u.id.to_string(), u.reason.clone())) - .collect(); + let mut results: Vec<_> = endpoints + .unresolved + .iter() + .map(|u| error_result(&u.id.to_string(), u.reason.clone())) + .collect(); - let backend_results = cm - .power_shelf - .update_firmware( - &endpoints.resolved.endpoints, - &req.target_version, - &components, - &options, - ) - .await - .map_err(component_manager_error_to_status)?; - results.extend(backend_results.into_iter().map(|r| { - let id = ps_mac_to_id_str(&r.pmc_mac, &endpoints.resolved.mac_to_id); - if r.success { - success_result(&id) - } else { - error_result(&id, r.error.unwrap_or_default()) - } - })); - power_shelf_results = Some(results); + let backend_results = cm + .power_shelf + .update_firmware( + &endpoints.resolved.endpoints, + &req.target_version, + &components, + &options, + ) + .await + .map_err(component_manager_error_to_status)?; + results.extend(backend_results.into_iter().map(|r| { + let id = ps_mac_to_id_str(&r.pmc_mac, &endpoints.resolved.mac_to_id); + if r.success { + success_result(&id) + } else { + error_result(&id, r.error.unwrap_or_default()) + } + })); + power_shelf_results = Some(results); + } } rpc::update_component_firmware_request::Target::Racks(t) => { if bypass_state_controller { @@ -2952,17 +3031,37 @@ mod tests { let rack_b = RackId::new("rack-b".to_string()); let mut targets = Vec::new(); - push_rack_firmware_target(&mut targets, rack_a.clone(), Some("machine-a".into()), None); - push_rack_firmware_target(&mut targets, rack_b.clone(), None, Some("switch-b".into())); - push_rack_firmware_target(&mut targets, rack_a.clone(), Some("machine-c".into()), None); + push_rack_firmware_target( + &mut targets, + rack_a.clone(), + Some("machine-a".into()), + None, + None, + ); + push_rack_firmware_target( + &mut targets, + rack_b.clone(), + None, + Some("switch-b".into()), + None, + ); + push_rack_firmware_target( + &mut targets, + rack_a.clone(), + Some("machine-c".into()), + None, + None, + ); assert_eq!(targets.len(), 2); assert_eq!(targets[0].rack_id, rack_a); assert_eq!(targets[0].machine_ids, vec!["machine-a", "machine-c"]); assert!(targets[0].switch_ids.is_empty()); + assert!(targets[0].power_shelf_ids.is_empty()); assert_eq!(targets[1].rack_id, rack_b); assert_eq!(targets[1].switch_ids, vec!["switch-b"]); assert!(targets[1].machine_ids.is_empty()); + assert!(targets[1].power_shelf_ids.is_empty()); } #[test] diff --git a/crates/api-core/src/setup.rs b/crates/api-core/src/setup.rs index 5d1760b76e..29d5709cb0 100644 --- a/crates/api-core/src/setup.rs +++ b/crates/api-core/src/setup.rs @@ -1444,6 +1444,9 @@ async fn initialize_and_start_controllers<'a>( component_manager: component_manager.clone().map(Arc::new), credential_manager: credential_manager.clone(), per_object_metrics_registry: per_object_metrics_registry.clone(), + rack_firmware_reprovisioning_enabled: carbide_config + .power_shelf_state_controller + .rack_firmware_reprovisioning_enabled, } .into(), ) diff --git a/crates/api-core/src/test_support/default_config.rs b/crates/api-core/src/test_support/default_config.rs index 41981c7b91..a614f9ce48 100644 --- a/crates/api-core/src/test_support/default_config.rs +++ b/crates/api-core/src/test_support/default_config.rs @@ -186,6 +186,7 @@ pub fn get() -> CarbideConfig { controller: StateControllerConfig::default(), }, power_shelf_state_controller: PowerShelfStateControllerConfig { + rack_firmware_reprovisioning_enabled: false, controller: StateControllerConfig::default(), }, rack_state_controller: RackStateControllerConfig { diff --git a/crates/api-core/src/tests/common/api_fixtures/mod.rs b/crates/api-core/src/tests/common/api_fixtures/mod.rs index 39ba50c9c0..081cb84c02 100644 --- a/crates/api-core/src/tests/common/api_fixtures/mod.rs +++ b/crates/api-core/src/tests/common/api_fixtures/mod.rs @@ -1655,6 +1655,7 @@ pub async fn create_test_env_with_overrides( component_manager: test_component_manager.clone(), credential_manager: credential_manager.clone(), per_object_metrics_registry: per_object_metrics_registry.clone(), + rack_firmware_reprovisioning_enabled: false, } .into(), ) diff --git a/crates/api-core/src/tests/power_shelf_state_controller/error_state.rs b/crates/api-core/src/tests/power_shelf_state_controller/error_state.rs index 3f58993089..53f60df299 100644 --- a/crates/api-core/src/tests/power_shelf_state_controller/error_state.rs +++ b/crates/api-core/src/tests/power_shelf_state_controller/error_state.rs @@ -66,6 +66,7 @@ async fn services( component_manager, credential_manager: Arc::new(TestCredentialManager::default()), per_object_metrics_registry: env.per_object_metrics_registry(), + rack_firmware_reprovisioning_enabled: false, } } diff --git a/crates/api-core/src/tests/power_shelf_state_controller/maintenance.rs b/crates/api-core/src/tests/power_shelf_state_controller/maintenance.rs index 0c80853c0b..89ecc541bc 100644 --- a/crates/api-core/src/tests/power_shelf_state_controller/maintenance.rs +++ b/crates/api-core/src/tests/power_shelf_state_controller/maintenance.rs @@ -144,6 +144,7 @@ fn services_with_component_manager( password: TEST_BMC_PASSWORD.into(), })), per_object_metrics_registry: env.per_object_metrics_registry(), + rack_firmware_reprovisioning_enabled: false, } } diff --git a/crates/api-core/src/tests/power_shelf_state_controller/mod.rs b/crates/api-core/src/tests/power_shelf_state_controller/mod.rs index 28bceeeafa..f6aea1a2cb 100644 --- a/crates/api-core/src/tests/power_shelf_state_controller/mod.rs +++ b/crates/api-core/src/tests/power_shelf_state_controller/mod.rs @@ -33,6 +33,7 @@ use crate::tests::common::api_fixtures::create_test_env; mod error_state; mod fixtures; mod maintenance; +mod reprovisioning; use carbide_secrets::test_support::credentials::TestCredentialManager; use fixtures::power_shelf::{mark_power_shelf_as_deleted, set_power_shelf_controller_state}; @@ -130,6 +131,7 @@ async fn test_power_shelf_deletion_with_state_controller( component_manager: None, credential_manager: credential_manager.clone(), per_object_metrics_registry: env.per_object_metrics_registry(), + rack_firmware_reprovisioning_enabled: false, } .into(), ) diff --git a/crates/api-core/src/tests/power_shelf_state_controller/reprovisioning.rs b/crates/api-core/src/tests/power_shelf_state_controller/reprovisioning.rs new file mode 100644 index 0000000000..4568f054e9 --- /dev/null +++ b/crates/api-core/src/tests/power_shelf_state_controller/reprovisioning.rs @@ -0,0 +1,226 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +//! Tests for power-shelf rack firmware reprovisioning. + +use std::sync::Arc; + +use carbide_power_shelf_controller::context::{ + PowerShelfStateHandlerContextObjects, PowerShelfStateHandlerServices, +}; +use carbide_power_shelf_controller::handler::PowerShelfStateHandler; +use carbide_power_shelf_controller::metrics::PowerShelfMetrics; +use carbide_secrets::test_support::credentials::TestCredentialManager; +use carbide_uuid::power_shelf::PowerShelfId; +use db::power_shelf as db_power_shelf; +use model::power_shelf::{PowerShelf, PowerShelfControllerState, ReProvisioningState}; +use model::rack::MaintenanceActivity; +use sqlx::PgConnection; +use state_controller::db_write_batch::DbWriteBatch; +use state_controller::state_handler::{StateHandler, StateHandlerContext, StateHandlerOutcome}; + +use crate::tests::common::api_fixtures::create_test_env; +use crate::tests::common::api_fixtures::site_explorer::new_power_shelf; +use crate::tests::power_shelf_state_controller::fixtures::power_shelf::set_power_shelf_controller_state; + +fn firmware_only_activities() -> Vec { + vec![MaintenanceActivity::FirmwareUpgrade { + firmware_version: None, + components: vec![], + force_update: false, + }] +} + +fn services( + env: &crate::tests::common::api_fixtures::TestEnv, + rack_firmware_reprovisioning_enabled: bool, +) -> PowerShelfStateHandlerServices { + PowerShelfStateHandlerServices { + db_pool: env.pool.clone(), + component_manager: None, + credential_manager: Arc::new(TestCredentialManager::default()), + per_object_metrics_registry: env.per_object_metrics_registry(), + rack_firmware_reprovisioning_enabled, + } +} + +async fn load_power_shelf(pool: &sqlx::PgPool, id: &PowerShelfId) -> PowerShelf { + let mut conn = pool.acquire().await.unwrap(); + db_power_shelf::find_by_id(conn.as_mut(), id) + .await + .unwrap() + .expect("power shelf should exist") +} + +async fn run_handler( + services: &mut PowerShelfStateHandlerServices, + state: &mut PowerShelf, +) -> StateHandlerOutcome { + let handler = PowerShelfStateHandler::default(); + let mut metrics = PowerShelfMetrics::default(); + let mut writes = DbWriteBatch::default(); + let mut ctx = StateHandlerContext:: { + services, + metrics: &mut metrics, + pending_db_writes: &mut writes, + }; + let controller_state = state.controller_state.value.clone(); + let power_shelf_id = state.id; + handler + .handle_object_state(&power_shelf_id, state, &controller_state, &mut ctx) + .await + .expect("state handler should not return an error result") +} + +async fn commit_outcome(mut outcome: StateHandlerOutcome) { + if let Some(txn) = outcome.take_transaction() { + txn.commit().await.unwrap(); + } +} + +async fn park_ready(txn: &mut PgConnection, power_shelf_id: &PowerShelfId) { + set_power_shelf_controller_state(txn, power_shelf_id, PowerShelfControllerState::Ready) + .await + .expect("set Ready"); +} + +#[crate::sqlx_test] +async fn test_ready_clears_reprovision_request_when_flag_disabled( + pool: sqlx::PgPool, +) -> Result<(), Box> { + let env = create_test_env(pool.clone()).await; + let power_shelf_id = new_power_shelf(&env, None, None, None, None).await?; + + let mut txn = pool.begin().await?; + park_ready(txn.as_mut(), &power_shelf_id).await; + db_power_shelf::set_power_shelf_reprovisioning_requested( + txn.as_mut(), + power_shelf_id, + "rack-test", + firmware_only_activities(), + ) + .await?; + txn.commit().await?; + + let mut state = load_power_shelf(&pool, &power_shelf_id).await; + let mut services = services(&env, false); + let outcome = run_handler(&mut services, &mut state).await; + assert!(matches!(outcome, StateHandlerOutcome::DoNothing { .. })); + commit_outcome(outcome).await; + + let state = load_power_shelf(&pool, &power_shelf_id).await; + assert!(state.power_shelf_reprovisioning_requested.is_none()); + assert!(matches!( + state.controller_state.value, + PowerShelfControllerState::Ready + )); + Ok(()) +} + +#[crate::sqlx_test] +async fn test_ready_enters_waiting_for_rack_firmware_when_flag_enabled( + pool: sqlx::PgPool, +) -> Result<(), Box> { + let env = create_test_env(pool.clone()).await; + let power_shelf_id = new_power_shelf(&env, None, None, None, None).await?; + + let mut txn = pool.begin().await?; + park_ready(txn.as_mut(), &power_shelf_id).await; + db_power_shelf::set_power_shelf_reprovisioning_requested( + txn.as_mut(), + power_shelf_id, + "rack-test", + firmware_only_activities(), + ) + .await?; + txn.commit().await?; + + let mut state = load_power_shelf(&pool, &power_shelf_id).await; + let mut services = services(&env, true); + let outcome = run_handler(&mut services, &mut state).await; + assert!(matches!( + outcome, + StateHandlerOutcome::Transition { + next_state: PowerShelfControllerState::ReProvisioning { + reprovisioning_state: ReProvisioningState::WaitingForRackFirmwareUpgrade, + }, + .. + } + )); + Ok(()) +} + +#[crate::sqlx_test] +async fn test_waiting_for_rack_firmware_completes_to_ready( + pool: sqlx::PgPool, +) -> Result<(), Box> { + let env = create_test_env(pool.clone()).await; + let power_shelf_id = new_power_shelf(&env, None, None, None, None).await?; + + let mut txn = pool.begin().await?; + db_power_shelf::set_power_shelf_reprovisioning_requested( + txn.as_mut(), + power_shelf_id, + "rack-test", + firmware_only_activities(), + ) + .await?; + let power_shelf = db_power_shelf::find_by_id(txn.as_mut(), &power_shelf_id) + .await? + .expect("power shelf should exist"); + let requested_at = power_shelf + .power_shelf_reprovisioning_requested + .as_ref() + .expect("request should exist") + .requested_at; + set_power_shelf_controller_state( + txn.as_mut(), + &power_shelf_id, + PowerShelfControllerState::ReProvisioning { + reprovisioning_state: ReProvisioningState::WaitingForRackFirmwareUpgrade, + }, + ) + .await?; + db_power_shelf::update_firmware_upgrade_status( + txn.as_mut(), + power_shelf_id, + Some(&model::rack::RackFirmwareUpgradeStatus { + task_id: "rack-job".to_string(), + status: model::rack::RackFirmwareUpgradeState::Completed, + started_at: Some(requested_at), + ended_at: Some(requested_at + chrono::Duration::seconds(1)), + }), + ) + .await?; + txn.commit().await?; + + let mut state = load_power_shelf(&pool, &power_shelf_id).await; + let mut services = services(&env, true); + let outcome = run_handler(&mut services, &mut state).await; + assert!(matches!( + outcome, + StateHandlerOutcome::Transition { + next_state: PowerShelfControllerState::Ready, + .. + } + )); + commit_outcome(outcome).await; + + let state = load_power_shelf(&pool, &power_shelf_id).await; + assert!(state.power_shelf_reprovisioning_requested.is_none()); + Ok(()) +} diff --git a/crates/api-core/src/tests/rack_state_controller/handler.rs b/crates/api-core/src/tests/rack_state_controller/handler.rs index 872f037c3d..0c79d04f84 100644 --- a/crates/api-core/src/tests/rack_state_controller/handler.rs +++ b/crates/api-core/src/tests/rack_state_controller/handler.rs @@ -207,6 +207,52 @@ async fn create_single_compute_rack( Ok((rack_id, host)) } +async fn set_machine_host_reprovision_state( + pool: &sqlx::PgPool, + machine_id: &MachineId, + reprovision_state: model::machine::HostReprovisionState, +) -> Result<(), Box> { + let mut txn = pool.begin().await?; + let machine = db::machine::find_one( + txn.as_mut(), + machine_id, + model::machine::machine_search_config::MachineSearchConfig::default(), + ) + .await? + .expect("machine should exist"); + db::machine::advance( + &machine, + txn.as_mut(), + &model::machine::ManagedHostState::HostReprovision { + reprovision_state, + retry_count: 0, + }, + None, + ) + .await?; + txn.commit().await?; + Ok(()) +} + +fn waiting_for_rack_firmware_upgrade_state() -> model::machine::HostReprovisionState { + model::machine::HostReprovisionState::WaitingForRackFirmwareUpgrade +} + +fn failed_rack_firmware_upgrade_state() -> model::machine::HostReprovisionState { + model::machine::HostReprovisionState::FailedFirmwareUpgrade { + firmware_type: model::firmware::FirmwareComponentType::Bmc, + report_time: Some(chrono::Utc::now()), + reason: Some("upgrade failed".to_string()), + } +} + +fn completed_rack_firmware_upgrade_state() -> model::machine::HostReprovisionState { + model::machine::HostReprovisionState::CheckingFirmwareRepeatV2 { + firmware_type: None, + firmware_number: None, + } +} + async fn create_two_compute_rack( env: &TestEnv, pool: &sqlx::PgPool, @@ -1583,8 +1629,9 @@ async fn test_firmware_upgrade_start_missing_profile_deletes_access_token( } /// test_firmware_upgrade_wait_for_complete_waits_while_jobs_running verifies -/// that WaitForComplete remains in a wait state while RMS child jobs are still -/// running and writes in-progress rack firmware status back to the machine. +/// that WaitForComplete remains in a wait state while machines are still in +/// WaitingForRackFirmwareUpgrade and writes in-progress rack firmware status +/// back to the machine from RMS. #[crate::sqlx_test] async fn test_firmware_upgrade_wait_for_complete_waits_while_jobs_running( pool: sqlx::PgPool, @@ -1598,6 +1645,12 @@ async fn test_firmware_upgrade_wait_for_complete_waits_while_jobs_running( ) .await; let (rack_id, host) = create_single_compute_rack(&env, &pool).await?; + set_machine_host_reprovision_state( + &pool, + &host.host_snapshot.id, + waiting_for_rack_firmware_upgrade_state(), + ) + .await?; env.rms_sim .set_firmware_job_status(librms::protos::rack_manager::GetFirmwareJobStatusResponse { status: librms::protos::rack_manager::ReturnCode::Success as i32, @@ -1651,7 +1704,7 @@ async fn test_firmware_upgrade_wait_for_complete_waits_while_jobs_running( assert!( matches!(outcome, StateHandlerOutcome::Wait { .. }), - "Expected Wait while RMS job is running" + "Expected Wait while machine controller is still WaitingForRackFirmwareUpgrade" ); let machine = db::machine::find_one( @@ -1671,8 +1724,8 @@ async fn test_firmware_upgrade_wait_for_complete_waits_while_jobs_running( } /// test_firmware_upgrade_wait_for_complete_transitions_to_error_on_job_failure -/// verifies that a failed RMS child job writes failed machine status and moves -/// the rack into Error. +/// verifies that a machine left WaitingForRackFirmwareUpgrade in +/// FailedFirmwareUpgrade moves the rack into Error. #[crate::sqlx_test] async fn test_firmware_upgrade_wait_for_complete_transitions_to_error_on_job_failure( pool: sqlx::PgPool, @@ -1686,6 +1739,12 @@ async fn test_firmware_upgrade_wait_for_complete_transitions_to_error_on_job_fai ) .await; let (rack_id, host) = create_single_compute_rack(&env, &pool).await?; + set_machine_host_reprovision_state( + &pool, + &host.host_snapshot.id, + failed_rack_firmware_upgrade_state(), + ) + .await?; env.rms_sim .set_firmware_job_status(librms::protos::rack_manager::GetFirmwareJobStatusResponse { status: librms::protos::rack_manager::ReturnCode::Success as i32, @@ -1772,9 +1831,9 @@ async fn test_firmware_upgrade_wait_for_complete_transitions_to_error_on_job_fai } /// test_firmware_upgrade_wait_for_complete_waits_for_all_nodes_to_be_terminal_before_error -/// verifies that the rack keeps polling when a mixed result contains both -/// failed and in-progress devices, then errors only after all tracked devices -/// reach a terminal state. +/// verifies that the rack keeps waiting while any tracked machine is still in +/// WaitingForRackFirmwareUpgrade, then errors only after every machine has left +/// that wait state and at least one failed. #[crate::sqlx_test] async fn test_firmware_upgrade_wait_for_complete_waits_for_all_nodes_to_be_terminal_before_error( pool: sqlx::PgPool, @@ -1788,6 +1847,18 @@ async fn test_firmware_upgrade_wait_for_complete_waits_for_all_nodes_to_be_termi ) .await; let (rack_id, host_a, host_b) = create_two_compute_rack(&env, &pool).await?; + set_machine_host_reprovision_state( + &pool, + &host_a.host_snapshot.id, + waiting_for_rack_firmware_upgrade_state(), + ) + .await?; + set_machine_host_reprovision_state( + &pool, + &host_b.host_snapshot.id, + waiting_for_rack_firmware_upgrade_state(), + ) + .await?; env.rms_sim .set_firmware_job_status(librms::protos::rack_manager::GetFirmwareJobStatusResponse { @@ -1864,7 +1935,7 @@ async fn test_firmware_upgrade_wait_for_complete_waits_for_all_nodes_to_be_termi assert!( matches!(outcome, StateHandlerOutcome::Wait { .. }), - "Expected Wait while some tracked devices are still non-terminal" + "Expected Wait while some tracked machines are still WaitingForRackFirmwareUpgrade" ); let machine_a = db::machine::find_one( @@ -1898,6 +1969,19 @@ async fn test_firmware_upgrade_wait_for_complete_waits_for_all_nodes_to_be_termi RackFirmwareUpgradeState::InProgress ); + set_machine_host_reprovision_state( + &pool, + &host_a.host_snapshot.id, + failed_rack_firmware_upgrade_state(), + ) + .await?; + set_machine_host_reprovision_state( + &pool, + &host_b.host_snapshot.id, + waiting_for_rack_firmware_upgrade_state(), + ) + .await?; + env.rms_sim .set_firmware_job_status(librms::protos::rack_manager::GetFirmwareJobStatusResponse { status: librms::protos::rack_manager::ReturnCode::Success as i32, @@ -1917,11 +2001,31 @@ async fn test_firmware_upgrade_wait_for_complete_waits_for_all_nodes_to_be_termi txn.commit().await?; } + assert!( + matches!(outcome, StateHandlerOutcome::Wait { .. }), + "Expected Wait while machine B is still WaitingForRackFirmwareUpgrade" + ); + + set_machine_host_reprovision_state( + &pool, + &host_b.host_snapshot.id, + completed_rack_firmware_upgrade_state(), + ) + .await?; + + let mut rack = get_db_rack(env.db_reader().as_mut(), &rack_id).await; + let mut outcome = handler_instance + .handle_object_state(&rack_id, &mut rack, &fw_state, &mut ctx) + .await?; + if let Some(txn) = outcome.take_transaction() { + txn.commit().await?; + } + match outcome { StateHandlerOutcome::Transition { next_state, .. } => { assert!( matches!(next_state, RackState::Error { .. }), - "Expected rack to transition to Error after all tracked devices are terminal, got {:?}", + "Expected rack to transition to Error after all tracked machines left firmware wait with a failure, got {:?}", next_state ); } @@ -1952,7 +2056,8 @@ async fn test_firmware_upgrade_wait_for_complete_waits_for_all_nodes_to_be_termi /// test_firmware_upgrade_wait_for_complete_retries_when_job_lookup_fails /// verifies that a response-level lookup failure from GetFirmwareJobStatus does -/// not mark the device failed and instead keeps the rack waiting. +/// not mark the device failed and instead keeps the rack waiting while the +/// machine remains in WaitingForRackFirmwareUpgrade. #[crate::sqlx_test] async fn test_firmware_upgrade_wait_for_complete_retries_when_job_lookup_fails( pool: sqlx::PgPool, @@ -1966,6 +2071,12 @@ async fn test_firmware_upgrade_wait_for_complete_retries_when_job_lookup_fails( ) .await; let (rack_id, host) = create_single_compute_rack(&env, &pool).await?; + set_machine_host_reprovision_state( + &pool, + &host.host_snapshot.id, + waiting_for_rack_firmware_upgrade_state(), + ) + .await?; env.rms_sim .set_firmware_job_status(librms::protos::rack_manager::GetFirmwareJobStatusResponse { status: librms::protos::rack_manager::ReturnCode::Failure as i32, @@ -2037,7 +2148,7 @@ async fn test_firmware_upgrade_wait_for_complete_retries_when_job_lookup_fails( /// test_firmware_upgrade_wait_for_complete_retries_on_transient_poll_error /// verifies that transport-level polling failures do not immediately fail the -/// rack upgrade. +/// rack upgrade while machines remain in WaitingForRackFirmwareUpgrade. #[crate::sqlx_test] async fn test_firmware_upgrade_wait_for_complete_retries_on_transient_poll_error( pool: sqlx::PgPool, @@ -2051,6 +2162,12 @@ async fn test_firmware_upgrade_wait_for_complete_retries_on_transient_poll_error ) .await; let (rack_id, host) = create_single_compute_rack(&env, &pool).await?; + set_machine_host_reprovision_state( + &pool, + &host.host_snapshot.id, + waiting_for_rack_firmware_upgrade_state(), + ) + .await?; env.rms_sim .set_firmware_job_error("child-job-1", "mock transport failure") .await; diff --git a/crates/api-core/src/tests/switch_state_controller/mod.rs b/crates/api-core/src/tests/switch_state_controller/mod.rs index c626996a9d..4c52e8f790 100644 --- a/crates/api-core/src/tests/switch_state_controller/mod.rs +++ b/crates/api-core/src/tests/switch_state_controller/mod.rs @@ -54,6 +54,28 @@ fn default_switch_mtls_services() -> Vec { ) } +fn firmware_only_activities() -> Vec { + vec![model::rack::MaintenanceActivity::FirmwareUpgrade { + firmware_version: None, + components: vec![], + force_update: false, + }] +} + +fn nvos_and_nmxc_activities() -> Vec { + vec![ + model::rack::MaintenanceActivity::FirmwareUpgrade { + firmware_version: None, + components: vec![], + force_update: false, + }, + model::rack::MaintenanceActivity::NvosUpdate { + config_json: String::new(), + }, + model::rack::MaintenanceActivity::ConfigureNmxCluster, + ] +} + async fn build_test_component_manager( env: &common::api_fixtures::TestEnv, rms_client: Option>, @@ -621,7 +643,13 @@ async fn test_switch_waiting_for_rack_firmware_upgrade_waits_for_terminal_status let switch_id = common::api_fixtures::site_explorer::new_switch(&env, None, None).await?; let mut txn = pool.begin().await?; - db_switch::set_switch_reprovisioning_requested(txn.as_mut(), switch_id, "rack-test").await?; + db_switch::set_switch_reprovisioning_requested( + txn.as_mut(), + switch_id, + "rack-test", + nvos_and_nmxc_activities(), + ) + .await?; let switch = db_switch::find_by_id(txn.as_mut(), &switch_id) .await? .expect("switch should exist"); @@ -678,7 +706,13 @@ async fn test_switch_waiting_for_rack_firmware_upgrade_transitions_to_waiting_fo let switch_id = common::api_fixtures::site_explorer::new_switch(&env, None, None).await?; let mut txn = pool.begin().await?; - db_switch::set_switch_reprovisioning_requested(txn.as_mut(), switch_id, "rack-test").await?; + db_switch::set_switch_reprovisioning_requested( + txn.as_mut(), + switch_id, + "rack-test", + nvos_and_nmxc_activities(), + ) + .await?; let switch = db_switch::find_by_id(txn.as_mut(), &switch_id) .await? .expect("switch should exist"); @@ -735,11 +769,11 @@ async fn test_switch_waiting_for_rack_firmware_upgrade_returns_ready_for_firmwar let switch_id = common::api_fixtures::site_explorer::new_switch(&env, None, None).await?; let mut txn = pool.begin().await?; - db_switch::set_switch_reprovisioning_requested_with_firmware_continuation( + db_switch::set_switch_reprovisioning_requested( txn.as_mut(), switch_id, "rack-test", - false, + firmware_only_activities(), ) .await?; let switch = db_switch::find_by_id(txn.as_mut(), &switch_id) @@ -796,7 +830,13 @@ async fn test_switch_waiting_for_rack_firmware_upgrade_accepts_completion_when_o let switch_id = common::api_fixtures::site_explorer::new_switch(&env, None, None).await?; let mut txn = pool.begin().await?; - db_switch::set_switch_reprovisioning_requested(txn.as_mut(), switch_id, "rack-test").await?; + db_switch::set_switch_reprovisioning_requested( + txn.as_mut(), + switch_id, + "rack-test", + nvos_and_nmxc_activities(), + ) + .await?; let switch = db_switch::find_by_id(txn.as_mut(), &switch_id) .await? .expect("switch should exist"); @@ -853,7 +893,13 @@ async fn test_switch_ready_routes_rack_requests_to_waiting_for_rack_firmware_upg let switch_id = common::api_fixtures::site_explorer::new_switch(&env, None, None).await?; let mut txn = pool.begin().await?; - db_switch::set_switch_reprovisioning_requested(txn.as_mut(), switch_id, "rack-test").await?; + db_switch::set_switch_reprovisioning_requested( + txn.as_mut(), + switch_id, + "rack-test", + nvos_and_nmxc_activities(), + ) + .await?; let switch = db_switch::find_by_id(txn.as_mut(), &switch_id) .await? .expect("switch should exist"); @@ -891,8 +937,13 @@ async fn test_switch_waiting_for_nvos_upgrade_transitions_to_waiting_for_nmxc_on let switch_id = common::api_fixtures::site_explorer::new_switch(&env, None, None).await?; let mut txn = pool.begin().await?; - db_switch::set_switch_reprovisioning_requested(txn.as_mut(), switch_id, "rack-nvos-test") - .await?; + db_switch::set_switch_reprovisioning_requested( + txn.as_mut(), + switch_id, + "rack-nvos-test", + nvos_and_nmxc_activities(), + ) + .await?; let switch = db_switch::find_by_id(txn.as_mut(), &switch_id) .await? .expect("switch should exist"); @@ -951,8 +1002,13 @@ async fn test_switch_waiting_for_nvos_upgrade_waits_for_current_cycle_status( let switch_id = common::api_fixtures::site_explorer::new_switch(&env, None, None).await?; let mut txn = pool.begin().await?; - db_switch::set_switch_reprovisioning_requested(txn.as_mut(), switch_id, "rack-nvos-test") - .await?; + db_switch::set_switch_reprovisioning_requested( + txn.as_mut(), + switch_id, + "rack-nvos-test", + nvos_and_nmxc_activities(), + ) + .await?; let switch = db_switch::find_by_id(txn.as_mut(), &switch_id) .await? .expect("switch should exist"); @@ -1011,8 +1067,13 @@ async fn test_switch_waiting_for_nvos_upgrade_transitions_to_error_on_failure( let switch_id = common::api_fixtures::site_explorer::new_switch(&env, None, None).await?; let mut txn = pool.begin().await?; - db_switch::set_switch_reprovisioning_requested(txn.as_mut(), switch_id, "rack-nvos-test") - .await?; + db_switch::set_switch_reprovisioning_requested( + txn.as_mut(), + switch_id, + "rack-nvos-test", + nvos_and_nmxc_activities(), + ) + .await?; let switch = db_switch::find_by_id(txn.as_mut(), &switch_id) .await? .expect("switch should exist"); @@ -1071,8 +1132,13 @@ async fn test_switch_waiting_for_nmxc_configure_returns_ready_when_fm_is_running let switch_id = common::api_fixtures::site_explorer::new_switch(&env, None, None).await?; let mut txn = pool.begin().await?; - db_switch::set_switch_reprovisioning_requested(txn.as_mut(), switch_id, "rack-nmxc-test") - .await?; + db_switch::set_switch_reprovisioning_requested( + txn.as_mut(), + switch_id, + "rack-nmxc-test", + nvos_and_nmxc_activities(), + ) + .await?; let switch = db_switch::find_by_id(txn.as_mut(), &switch_id) .await? .expect("switch should exist"); diff --git a/crates/api-db/migrations/20260724120000_power_shelf_reprovisioning_requested.sql b/crates/api-db/migrations/20260724120000_power_shelf_reprovisioning_requested.sql new file mode 100644 index 0000000000..eb98e580e7 --- /dev/null +++ b/crates/api-db/migrations/20260724120000_power_shelf_reprovisioning_requested.sql @@ -0,0 +1,7 @@ +-- Add power_shelf_reprovisioning_requested and firmware_upgrade_status columns +-- to power_shelves. Mirrors switches.switch_reprovisioning_requested / +-- firmware_upgrade_status for rack-level firmware upgrade wait phases. + +ALTER TABLE power_shelves + ADD COLUMN power_shelf_reprovisioning_requested JSONB, + ADD COLUMN firmware_upgrade_status JSONB; diff --git a/crates/api-db/src/power_shelf.rs b/crates/api-db/src/power_shelf.rs index dec1c5f74a..e0b904bd15 100644 --- a/crates/api-db/src/power_shelf.rs +++ b/crates/api-db/src/power_shelf.rs @@ -24,8 +24,9 @@ use model::controller_outcome::PersistentStateHandlerOutcome; use model::metadata::Metadata; use model::power_shelf::{ NewPowerShelf, PowerShelf, PowerShelfControllerState, PowerShelfMaintenanceOperation, - PowerShelfMaintenanceRequest, + PowerShelfMaintenanceRequest, PowerShelfReprovisionRequest, }; +use model::rack::{MaintenanceActivity, RackFirmwareUpgradeStatus}; use sqlx::PgConnection; use crate::db_read::DbReader; @@ -133,6 +134,8 @@ pub async fn create( version, rack_id: new_power_shelf.rack_id.clone(), power_shelf_maintenance_requested: None, + power_shelf_reprovisioning_requested: None, + firmware_upgrade_status: None, health_reports: Default::default(), }) } @@ -348,6 +351,65 @@ pub async fn clear_power_shelf_maintenance_requested( Ok(()) } +/// Sets `power_shelf_reprovisioning_requested` so the Ready handler can enter +/// `ReProvisioning` when rack-firmware reprovisioning is enabled. +/// +/// `activities` selects which rack maintenance phases the power shelf should wait +/// for. Empty means all activities. +pub async fn set_power_shelf_reprovisioning_requested( + txn: &mut PgConnection, + power_shelf_id: PowerShelfId, + initiator: &str, + activities: Vec, +) -> DatabaseResult<()> { + let req = PowerShelfReprovisionRequest { + requested_at: Utc::now(), + initiator: initiator.to_string(), + activities, + }; + let query = "UPDATE power_shelves SET power_shelf_reprovisioning_requested = $1 WHERE id = $2 RETURNING id"; + sqlx::query_as::<_, PowerShelfId>(query) + .bind(sqlx::types::Json(req)) + .bind(power_shelf_id) + .fetch_optional(txn) + .await + .map_err(|e| DatabaseError::new("set_power_shelf_reprovisioning_requested", e))?; + Ok(()) +} + +/// Clears `power_shelf_reprovisioning_requested`. Typically called when +/// reprovisioning completes or is cancelled. +pub async fn clear_power_shelf_reprovisioning_requested( + txn: &mut PgConnection, + power_shelf_id: PowerShelfId, +) -> DatabaseResult<()> { + let query = "UPDATE power_shelves SET power_shelf_reprovisioning_requested = NULL WHERE id = $1 RETURNING id"; + sqlx::query_as::<_, PowerShelfId>(query) + .bind(power_shelf_id) + .fetch_optional(txn) + .await + .map_err(|e| DatabaseError::new("clear_power_shelf_reprovisioning_requested", e))?; + Ok(()) +} + +/// Sets `firmware_upgrade_status` on the power shelf. Call from rack maintenance +/// to report upgrade progress. `WaitingForRackFirmwareUpgrade` reads this: +/// Completed → Ready, Failed → Error. +pub async fn update_firmware_upgrade_status( + txn: &mut PgConnection, + power_shelf_id: PowerShelfId, + status: Option<&RackFirmwareUpgradeStatus>, +) -> DatabaseResult<()> { + let query = "UPDATE power_shelves SET firmware_upgrade_status = $1 WHERE id = $2 RETURNING id"; + sqlx::query_as::<_, PowerShelfId>(query) + .bind(status.map(|s| sqlx::types::Json(s.clone()))) + .bind(power_shelf_id) + .fetch_optional(txn) + .await + .map_err(|e| DatabaseError::new("update_firmware_upgrade_status", e))?; + Ok(()) +} + pub async fn mark_as_deleted<'a>( power_shelf: &'a mut PowerShelf, txn: &mut PgConnection, diff --git a/crates/api-db/src/switch.rs b/crates/api-db/src/switch.rs index af23b2b1ca..e50435d435 100644 --- a/crates/api-db/src/switch.rs +++ b/crates/api-db/src/switch.rs @@ -368,25 +368,19 @@ pub async fn update_controller_state_outcome( /// Sets switch_reprovisioning_requested on the switch. Can be called from any state machine or /// service. When the switch is in Ready state, the switch state controller will observe the flag /// and transition to ReProvisioning::Start. +/// +/// `activities` selects which rack maintenance phases the switch should wait for. +/// Empty means all activities. pub async fn set_switch_reprovisioning_requested( txn: &mut PgConnection, switch_id: SwitchId, initiator: &str, -) -> DatabaseResult<()> { - set_switch_reprovisioning_requested_with_firmware_continuation(txn, switch_id, initiator, true) - .await -} - -pub async fn set_switch_reprovisioning_requested_with_firmware_continuation( - txn: &mut PgConnection, - switch_id: SwitchId, - initiator: &str, - continue_after_firmware_upgrade: bool, + activities: Vec, ) -> DatabaseResult<()> { let req = SwitchReprovisionRequest { requested_at: Utc::now(), initiator: initiator.to_string(), - continue_after_firmware_upgrade, + activities, }; let query = "UPDATE switches SET switch_reprovisioning_requested = $1 WHERE id = $2 RETURNING id"; diff --git a/crates/api-model/src/power_shelf/mod.rs b/crates/api-model/src/power_shelf/mod.rs index 9ec4a5250e..b20cee1770 100644 --- a/crates/api-model/src/power_shelf/mod.rs +++ b/crates/api-model/src/power_shelf/mod.rs @@ -86,6 +86,17 @@ pub struct PowerShelf { pub power_shelf_maintenance_requested: Option, + /// Set by rack maintenance to request power-shelf participation in a + /// rack-level firmware upgrade. When the power shelf is Ready and + /// rack-firmware reprovisioning is enabled on the controller, it + /// transitions to `ReProvisioning`. + pub power_shelf_reprovisioning_requested: Option, + + /// Per-device firmware upgrade status written by the rack state machine + /// during rack-level firmware upgrades. Read by + /// `ReProvisioning::WaitingForRackFirmwareUpgrade`. + pub firmware_upgrade_status: Option, + // Columns for these exist, but are unused in rust code // pub created: DateTime, // pub updated: DateTime, @@ -105,6 +116,12 @@ impl<'r> FromRow<'r, PgRow> for PowerShelf { let power_shelf_maintenance_requested: Option< sqlx::types::Json, > = row.try_get("power_shelf_maintenance_requested").ok(); + let power_shelf_reprovisioning_requested: Option< + sqlx::types::Json, + > = row.try_get("power_shelf_reprovisioning_requested").ok(); + let firmware_upgrade_status: Option< + sqlx::types::Json, + > = row.try_get("firmware_upgrade_status").ok(); let health_reports: HealthReportSources = row .try_get::, _>("health_reports") @@ -137,6 +154,8 @@ impl<'r> FromRow<'r, PgRow> for PowerShelf { version: row.try_get("version")?, rack_id: row.try_get("rack_id").ok().flatten(), power_shelf_maintenance_requested: power_shelf_maintenance_requested.map(|r| r.0), + power_shelf_reprovisioning_requested: power_shelf_reprovisioning_requested.map(|r| r.0), + firmware_upgrade_status: firmware_upgrade_status.map(|j| j.0), health_reports, }) } @@ -179,6 +198,30 @@ pub struct PowerShelfMaintenanceRequest { pub operation: PowerShelfMaintenanceOperation, } +/// Set by an external entity (typically rack maintenance) to request power-shelf +/// participation in rack-level reprovisioning. When the power shelf is Ready and +/// rack-firmware reprovisioning is enabled, the controller transitions to +/// `ReProvisioning`. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct PowerShelfReprovisionRequest { + pub requested_at: DateTime, + pub initiator: String, + /// Rack maintenance activities that initiated this request. The power shelf + /// controller uses these to decide whether to wait for firmware. Empty means + /// all activities. + #[serde(default)] + pub activities: Vec, +} + +/// Sub-state for PowerShelfControllerState::ReProvisioning +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[allow(clippy::enum_variant_names)] +pub enum ReProvisioningState { + /// Rack-level firmware upgrade in progress; the rack state machine manages the + /// upgrade and clears `power_shelf_reprovisioning_requested` when done. + WaitingForRackFirmwareUpgrade, +} + /// State of a PowerShelf as tracked by the controller #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] #[serde(tag = "state", rename_all = "lowercase")] @@ -195,6 +238,11 @@ pub enum PowerShelfControllerState { Maintenance { operation: PowerShelfMaintenanceOperation, }, + + /// Rack-driven firmware wait in progress. + ReProvisioning { + reprovisioning_state: ReProvisioningState, + }, /// There is error in PowerShelf; PowerShelf can not be used if it's in error. Error { cause: String }, /// The PowerShelf is in the process of deleting. @@ -226,6 +274,10 @@ pub fn state_sla(state: &PowerShelfControllerState, state_version: &ConfigVersio std::time::Duration::from_secs(slas::MAINTENANCE), time_in_state, ), + PowerShelfControllerState::ReProvisioning { .. } => StateSla::with_sla( + std::time::Duration::from_secs(slas::REPROVISIONING), + time_in_state, + ), PowerShelfControllerState::Error { .. } => StateSla::no_sla(), PowerShelfControllerState::Deleting => StateSla::with_sla( std::time::Duration::from_secs(slas::DELETING), @@ -331,6 +383,18 @@ mod tests { }, )), } + + "reprovisioning waiting for rack firmware" { + PowerShelfControllerState::ReProvisioning { + reprovisioning_state: ReProvisioningState::WaitingForRackFirmwareUpgrade, + } => Yields(( + r#"{"state":"reprovisioning","reprovisioning_state":"WaitingForRackFirmwareUpgrade"}"# + .to_string(), + PowerShelfControllerState::ReProvisioning { + reprovisioning_state: ReProvisioningState::WaitingForRackFirmwareUpgrade, + }, + )), + } ); } @@ -451,6 +515,14 @@ mod tests { }), } + "reprovisioning waiting for rack firmware" { + r#"{"state":"reprovisioning","reprovisioning_state":"WaitingForRackFirmwareUpgrade"}"# => Yields( + PowerShelfControllerState::ReProvisioning { + reprovisioning_state: ReProvisioningState::WaitingForRackFirmwareUpgrade, + }, + ), + } + "unknown tag is rejected" { r#"{"state":"running"}"# => Fails, } @@ -669,6 +741,12 @@ mod tests { } => (secs(slas::MAINTENANCE), true), } + "reprovisioning has the reprovisioning SLA" { + PowerShelfControllerState::ReProvisioning { + reprovisioning_state: ReProvisioningState::WaitingForRackFirmwareUpgrade, + } => (secs(slas::REPROVISIONING), true), + } + "ready carries no SLA" { PowerShelfControllerState::Ready => (None, false), } diff --git a/crates/api-model/src/power_shelf/slas.rs b/crates/api-model/src/power_shelf/slas.rs index b721cf770d..215516b577 100644 --- a/crates/api-model/src/power_shelf/slas.rs +++ b/crates/api-model/src/power_shelf/slas.rs @@ -35,3 +35,6 @@ pub const DELETING: u64 = 300; // 5 minutes /// SLA for PowerShelf maintenance (PowerOn / PowerOff) in seconds pub const MAINTENANCE: u64 = 300; // 5 minutes + +/// SLA for PowerShelf rack-level reprovisioning (firmware wait) in seconds +pub const REPROVISIONING: u64 = 3600; // 1 hour diff --git a/crates/api-model/src/switch/mod.rs b/crates/api-model/src/switch/mod.rs index 80b39f1be8..d76b40d34c 100644 --- a/crates/api-model/src/switch/mod.rs +++ b/crates/api-model/src/switch/mod.rs @@ -65,10 +65,6 @@ pub struct SwitchStatus { pub health_status: String, // "ok", "warning", "critical" } -fn default_continue_after_firmware_upgrade() -> bool { - true -} - #[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] #[serde(tag = "operation", rename_all = "lowercase")] #[allow(clippy::enum_variant_names)] @@ -96,9 +92,11 @@ pub struct SwitchMaintenanceRequest { pub struct SwitchReprovisionRequest { pub requested_at: DateTime, pub initiator: String, - /// Continue through rack-managed post-firmware phases such as NVOS/NMXC. - #[serde(default = "default_continue_after_firmware_upgrade")] - pub continue_after_firmware_upgrade: bool, + /// Rack maintenance activities that initiated this request. The switch + /// controller uses these to decide which ReProvisioning phases to wait for + /// (firmware / NVOS / NMXC). Empty means all activities. + #[serde(default)] + pub activities: Vec, } pub use crate::rack::{ @@ -863,24 +861,19 @@ mod tests { } #[test] - fn reprovision_request_defaults_continue_after_firmware_upgrade_to_true() { - scenarios!( - run = |json| { - serde_json::from_str::(json) - .map(|r| r.continue_after_firmware_upgrade) - .map_err(drop) - }; - "omitted flag defaults to true" { - r#"{"requested_at":"2026-01-01T00:00:00Z","initiator":"op"}"# => Yields(true), - } - - "explicit false is honored" { - r#"{"requested_at":"2026-01-01T00:00:00Z","initiator":"op","continue_after_firmware_upgrade":false}"# => Yields(false), - } + fn reprovision_request_defaults_activities_to_empty() { + let request: SwitchReprovisionRequest = + serde_json::from_str(r#"{"requested_at":"2026-01-01T00:00:00Z","initiator":"op"}"#) + .expect("SwitchReprovisionRequest should deserialize"); + assert!(request.activities.is_empty()); + } - "explicit true is honored" { - r#"{"requested_at":"2026-01-01T00:00:00Z","initiator":"op","continue_after_firmware_upgrade":true}"# => Yields(true), - } - ); + #[test] + fn reprovision_request_ignores_legacy_fields() { + let request: SwitchReprovisionRequest = serde_json::from_str( + r#"{"requested_at":"2026-01-01T00:00:00Z","initiator":"op","continue_after_firmware_upgrade":true,"scope":{"activities":[]}}"#, + ) + .expect("legacy continue_after_firmware_upgrade and scope fields should be ignored"); + assert!(request.activities.is_empty()); } } diff --git a/crates/power-shelf-controller/src/context.rs b/crates/power-shelf-controller/src/context.rs index 6fb18459c5..779a4707f9 100644 --- a/crates/power-shelf-controller/src/context.rs +++ b/crates/power-shelf-controller/src/context.rs @@ -34,6 +34,11 @@ pub struct PowerShelfStateHandlerServices { pub credential_manager: Arc, /// Shared registry backing the generic per-object health metrics. pub per_object_metrics_registry: Arc, + /// When `true`, Ready accepts rack-level `power_shelf_reprovisioning_requested` + /// and enters `ReProvisioning::WaitingForRackFirmwareUpgrade`. Defaults to + /// `false` so power shelves stay out of rack firmware wait unless explicitly + /// enabled. + pub rack_firmware_reprovisioning_enabled: bool, } impl StateHandlerContextObjects for PowerShelfStateHandlerContextObjects { diff --git a/crates/power-shelf-controller/src/handler.rs b/crates/power-shelf-controller/src/handler.rs index 71724c38c8..49ed324649 100644 --- a/crates/power-shelf-controller/src/handler.rs +++ b/crates/power-shelf-controller/src/handler.rs @@ -34,6 +34,7 @@ use crate::fetching_data::handle_fetching_data; use crate::initializing::handle_initializing; use crate::maintenance::handle_maintenance; use crate::ready::handle_ready; +use crate::reprovisioning::handle_reprovisioning; /// The actual PowerShelf State handler (structure mirrors SwitchStateHandler). #[derive(Debug, Default, Clone)] @@ -82,6 +83,9 @@ impl PowerShelfStateHandler { PowerShelfControllerState::Maintenance { .. } => { handle_maintenance(power_shelf_id, state, ctx).await } + PowerShelfControllerState::ReProvisioning { .. } => { + handle_reprovisioning(power_shelf_id, state, ctx).await + } PowerShelfControllerState::Deleting => { handle_deleting(power_shelf_id, state, ctx).await } diff --git a/crates/power-shelf-controller/src/io.rs b/crates/power-shelf-controller/src/io.rs index 0deb627444..f1ee2bbc2a 100644 --- a/crates/power-shelf-controller/src/io.rs +++ b/crates/power-shelf-controller/src/io.rs @@ -160,6 +160,16 @@ impl StateControllerIO for PowerShelfStateControllerIO { }; ("maintenance", op) } + PowerShelfControllerState::ReProvisioning { + reprovisioning_state, + } => { + let sub = match reprovisioning_state { + model::power_shelf::ReProvisioningState::WaitingForRackFirmwareUpgrade => { + "waiting_for_rack_firmware_upgrade" + } + }; + ("reprovisioning", sub) + } PowerShelfControllerState::Error { .. } => ("error", ""), PowerShelfControllerState::Deleting => ("deleting", ""), } diff --git a/crates/power-shelf-controller/src/lib.rs b/crates/power-shelf-controller/src/lib.rs index 0faefdddaa..d8502c6ab8 100644 --- a/crates/power-shelf-controller/src/lib.rs +++ b/crates/power-shelf-controller/src/lib.rs @@ -28,3 +28,4 @@ pub mod io; pub mod maintenance; pub mod metrics; pub mod ready; +pub mod reprovisioning; diff --git a/crates/power-shelf-controller/src/ready.rs b/crates/power-shelf-controller/src/ready.rs index 0b9e022e45..3ff73aa9c1 100644 --- a/crates/power-shelf-controller/src/ready.rs +++ b/crates/power-shelf-controller/src/ready.rs @@ -28,15 +28,17 @@ use state_controller::state_handler::{ use crate::context::PowerShelfStateHandlerContextObjects; use crate::maintenance::build_power_shelf_endpoint; +use crate::reprovisioning::first_reprovisioning_state; /// Handles the Ready state for a power shelf. /// /// If the power shelf is marked for deletion, transitions to `Deleting`. /// If a maintenance request has been posted via /// `power_shelf_maintenance_requested`, transitions to `Maintenance` with the -/// requested operation (PowerOn / PowerOff). Otherwise polls the configured -/// component manager backend for the current power state (best-effort -/// observation) and idles. +/// requested operation (PowerOn / PowerOff). If rack-level reprovisioning has +/// been requested and `rack_firmware_reprovisioning_enabled` is set, transitions +/// to `ReProvisioning`. Otherwise polls the configured component manager backend +/// for the current power state (best-effort observation) and idles. /// /// TODO: Implement PowerShelf monitoring (health checks, status updates, /// power consumption / efficiency tracking). @@ -64,10 +66,68 @@ pub async fn handle_ready( )); } + if let Some(req) = &state.power_shelf_reprovisioning_requested { + if !ctx.services.rack_firmware_reprovisioning_enabled { + tracing::info!( + power_shelf_id = %power_shelf_id, + initiator = %req.initiator, + "Rack reprovision request ignored; rack_firmware_reprovisioning_enabled is false" + ); + let mut txn = ctx.services.db_pool.begin().await?; + db_power_shelf::clear_power_shelf_reprovisioning_requested( + txn.as_mut(), + *power_shelf_id, + ) + .await?; + return Ok(StateHandlerOutcome::do_nothing().with_txn(txn)); + } + + if !req.initiator.starts_with("rack-") { + tracing::warn!( + initiator = %req.initiator, + "Unknown initiator for power shelf reprovisioning request", + ); + return Ok(StateHandlerOutcome::transition( + PowerShelfControllerState::Error { + cause: format!( + "unknown initiator for power shelf reprovisioning request: {}", + req.initiator + ), + }, + )); + } + + let Some(reprovisioning_state) = first_reprovisioning_state(req) else { + tracing::warn!( + power_shelf_id = %power_shelf_id, + initiator = %req.initiator, + "Rack reprovision request has no power-shelf-relevant activities; clearing request" + ); + let mut txn = ctx.services.db_pool.begin().await?; + db_power_shelf::clear_power_shelf_reprovisioning_requested( + txn.as_mut(), + *power_shelf_id, + ) + .await?; + return Ok(StateHandlerOutcome::do_nothing().with_txn(txn)); + }; + + tracing::info!( + ?reprovisioning_state, + "Rack-level reprovisioning requested — entering ReProvisioning" + ); + return Ok(StateHandlerOutcome::transition( + PowerShelfControllerState::ReProvisioning { + reprovisioning_state, + }, + )); + } + let txn = poll_power_state(power_shelf_id, state, ctx).await; Ok(StateHandlerOutcome::do_nothing().with_txn_opt(txn)) } + /// /// On a successful response, the observed power state for this power shelf is /// persisted to the `power_shelves.status` column and the in-memory `state` diff --git a/crates/power-shelf-controller/src/reprovisioning.rs b/crates/power-shelf-controller/src/reprovisioning.rs new file mode 100644 index 0000000000..63edd77757 --- /dev/null +++ b/crates/power-shelf-controller/src/reprovisioning.rs @@ -0,0 +1,183 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +//! Handler for PowerShelfControllerState::ReProvisioning. + +use carbide_uuid::power_shelf::PowerShelfId; +use db::db_read::PgPoolReader; +use db::{ObjectColumnFilter, power_shelf as db_power_shelf, rack as db_rack}; +use model::power_shelf::{ + PowerShelf, PowerShelfControllerState, PowerShelfReprovisionRequest, ReProvisioningState, +}; +use model::rack::{MaintenanceActivity, RackState}; +use state_controller::state_handler::{ + StateHandlerContext, StateHandlerError, StateHandlerOutcome, +}; + +use crate::context::PowerShelfStateHandlerContextObjects; + +fn is_rack_level_reprovisioning(state: &PowerShelf) -> bool { + state + .power_shelf_reprovisioning_requested + .as_ref() + .is_some_and(|req| req.initiator.starts_with("rack-")) +} + +fn should_run(activities: &[MaintenanceActivity], activity: &MaintenanceActivity) -> bool { + activities.is_empty() || activities.iter().any(|a| a.same_kind(activity)) +} + +fn firmware_upgrade_requested(activities: &[MaintenanceActivity]) -> bool { + should_run( + activities, + &MaintenanceActivity::FirmwareUpgrade { + firmware_version: None, + components: vec![], + force_update: false, + }, + ) +} + +/// First ReProvisioning sub-state to enter from Ready, based on the request +/// activities. Empty activities means all phases. Returns `None` when the +/// activities list has no power-shelf-relevant wait phases. +pub(crate) fn first_reprovisioning_state( + request: &PowerShelfReprovisionRequest, +) -> Option { + if firmware_upgrade_requested(&request.activities) { + return Some(ReProvisioningState::WaitingForRackFirmwareUpgrade); + } + None +} + +/// If the parent rack is in `RackState::Error`, clear +/// `power_shelf_reprovisioning_requested` and short-circuit to `Ready`. +async fn rack_failed_abort_outcome( + power_shelf_id: &PowerShelfId, + state: &PowerShelf, + ctx: &mut StateHandlerContext<'_, PowerShelfStateHandlerContextObjects>, +) -> Result>, StateHandlerError> { + if !is_rack_level_reprovisioning(state) { + return Ok(None); + } + + let Some(rack_id) = state.rack_id.as_ref() else { + return Ok(None); + }; + + let mut reader: PgPoolReader = ctx.services.db_pool.clone().into(); + let racks = db_rack::find_by( + reader.as_mut(), + ObjectColumnFilter::One(db_rack::IdColumn, rack_id), + ) + .await?; + let Some(rack) = racks.into_iter().next() else { + return Ok(None); + }; + if !matches!(rack.controller_state.value, RackState::Error { .. }) { + return Ok(None); + } + + tracing::info!( + power_shelf_id = %power_shelf_id, + rack_id = %rack_id, + "Rack is in Error; aborting power shelf ReProvisioning and returning to Ready", + ); + + let mut txn = ctx.services.db_pool.begin().await?; + db_power_shelf::clear_power_shelf_reprovisioning_requested(txn.as_mut(), *power_shelf_id) + .await?; + Ok(Some( + StateHandlerOutcome::transition(PowerShelfControllerState::Ready).with_txn(txn), + )) +} + +/// Handles the ReProvisioning state for a power shelf. +pub async fn handle_reprovisioning( + power_shelf_id: &PowerShelfId, + state: &mut PowerShelf, + ctx: &mut StateHandlerContext<'_, PowerShelfStateHandlerContextObjects>, +) -> Result, StateHandlerError> { + let reprovisioning_state = match &state.controller_state.value { + PowerShelfControllerState::ReProvisioning { + reprovisioning_state, + } => reprovisioning_state, + _ => unreachable!("handle_reprovisioning called with non-ReProvisioning state"), + }; + + if let Some(outcome) = rack_failed_abort_outcome(power_shelf_id, state, ctx).await? { + return Ok(outcome); + } + + match reprovisioning_state { + ReProvisioningState::WaitingForRackFirmwareUpgrade => { + let request = state + .power_shelf_reprovisioning_requested + .as_ref() + .expect("WaitingForRackFirmwareUpgrade requires a rack reprovision request"); + let requested_at = request.requested_at; + let Some(firmware_upgrade_status) = state.firmware_upgrade_status.as_ref() else { + return Ok(StateHandlerOutcome::wait( + "waiting for power shelf firmware upgrade status".into(), + )); + }; + if !firmware_upgrade_status.is_current_for(requested_at) { + return Ok(StateHandlerOutcome::wait( + "waiting for current rack firmware cycle".into(), + )); + } + if !firmware_upgrade_status.is_terminal() { + return Ok(StateHandlerOutcome::wait( + "waiting for power shelf firmware completion".into(), + )); + } + + match &firmware_upgrade_status.status { + model::rack::RackFirmwareUpgradeState::Completed => { + let mut txn = ctx.services.db_pool.begin().await?; + db_power_shelf::clear_power_shelf_reprovisioning_requested( + txn.as_mut(), + *power_shelf_id, + ) + .await?; + Ok( + StateHandlerOutcome::transition(PowerShelfControllerState::Ready) + .with_txn(txn), + ) + } + model::rack::RackFirmwareUpgradeState::Failed { cause } => { + let mut txn = ctx.services.db_pool.begin().await?; + db_power_shelf::clear_power_shelf_reprovisioning_requested( + txn.as_mut(), + *power_shelf_id, + ) + .await?; + Ok( + StateHandlerOutcome::transition(PowerShelfControllerState::Error { + cause: cause.clone(), + }) + .with_txn(txn), + ) + } + model::rack::RackFirmwareUpgradeState::Started + | model::rack::RackFirmwareUpgradeState::InProgress => Ok( + StateHandlerOutcome::wait("waiting for power shelf firmware completion".into()), + ), + } + } + } +} diff --git a/crates/rack-controller/src/maintenance.rs b/crates/rack-controller/src/maintenance.rs index 42c78e923e..ccb51ea624 100644 --- a/crates/rack-controller/src/maintenance.rs +++ b/crates/rack-controller/src/maintenance.rs @@ -43,7 +43,8 @@ use carbide_utils::none_if_empty::NoneIfEmpty; use carbide_uuid::rack::{RackId, RackProfileId}; use db::{ host_machine_update as db_host_machine_update, machine as db_machine, - machine_topology as db_machine_topology, rack as db_rack, switch as db_switch, + machine_topology as db_machine_topology, power_shelf as db_power_shelf, rack as db_rack, + switch as db_switch, }; use librms::protos::rack_manager as rms; use model::rack::{ @@ -96,7 +97,8 @@ async fn trigger_rack_firmware_reprovisioning_requests( rack_id: &RackId, machine_ids: &[carbide_uuid::machine::MachineId], switch_ids: &[carbide_uuid::switch::SwitchId], - continue_after_firmware_upgrade: bool, + power_shelf_ids: &[carbide_uuid::power_shelf::PowerShelfId], + activities: &[MaintenanceActivity], ) -> Result<(), StateHandlerError> { for machine_id in machine_ids { db_host_machine_update::trigger_host_reprovisioning_request( @@ -107,11 +109,20 @@ async fn trigger_rack_firmware_reprovisioning_requests( .await?; } for switch_id in switch_ids { - db_switch::set_switch_reprovisioning_requested_with_firmware_continuation( + db_switch::set_switch_reprovisioning_requested( txn, *switch_id, &format!("rack-{}", rack_id), - continue_after_firmware_upgrade, + activities.to_vec(), + ) + .await?; + } + for power_shelf_id in power_shelf_ids { + db_power_shelf::set_power_shelf_reprovisioning_requested( + txn, + *power_shelf_id, + &format!("rack-{}", rack_id), + activities.to_vec(), ) .await?; } @@ -122,6 +133,7 @@ async fn clear_rack_firmware_device_statuses( txn: &mut sqlx::PgConnection, machine_ids: &[carbide_uuid::machine::MachineId], switch_ids: &[carbide_uuid::switch::SwitchId], + power_shelf_ids: &[carbide_uuid::power_shelf::PowerShelfId], ) -> Result<(), StateHandlerError> { for machine_id in machine_ids { db_machine::update_rack_fw_details(txn, machine_id, None).await?; @@ -129,6 +141,9 @@ async fn clear_rack_firmware_device_statuses( for switch_id in switch_ids { db_switch::update_firmware_upgrade_status(txn, *switch_id, None).await?; } + for power_shelf_id in power_shelf_ids { + db_power_shelf::update_firmware_upgrade_status(txn, *power_shelf_id, None).await?; + } Ok(()) } @@ -142,6 +157,336 @@ async fn clear_nvos_update_statuses( Ok(()) } +/// Aggregated firmware progress for machines/switches participating in a rack +/// firmware job. Advancement out of `WaitForComplete` is based on machine and +/// switch controller states, not RMS job strings or `rack_fw_details`. +#[derive(Debug, Clone, PartialEq, Eq)] +enum DeviceFirmwareProgress { + Waiting { + pending: usize, + total: usize, + completed: usize, + failed: usize, + }, + Failed { + failed: usize, + total: usize, + }, + Completed { + completed: usize, + total: usize, + }, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum DeviceFirmwareOutcome { + Waiting, + Failed, + Completed, +} + +async fn resolve_machine_id_for_firmware_device( + txn: &mut sqlx::PgConnection, + device: &FirmwareUpgradeDeviceStatus, +) -> Result, StateHandlerError> { + if !device.node_id.is_empty() { + return Ok(device + .node_id + .parse::() + .ok()); + } + let mac: mac_address::MacAddress = match device.mac.parse() { + Ok(mac) => mac, + Err(_) => return Ok(None), + }; + Ok(db_machine_topology::find_machine_id_by_bmc_mac(txn, mac).await?) +} + +async fn resolve_switch_id_for_firmware_device( + txn: &mut sqlx::PgConnection, + rack_id: &RackId, + device: &FirmwareUpgradeDeviceStatus, +) -> Result, StateHandlerError> { + if !device.node_id.is_empty() { + return Ok(device + .node_id + .parse::() + .ok()); + } + let mac: mac_address::MacAddress = match device.mac.parse() { + Ok(mac) => mac, + Err(_) => return Ok(None), + }; + Ok(db_switch::find_ids( + txn, + model::switch::SwitchSearchFilter { + bmc_mac: Some(mac), + rack_id: Some(rack_id.clone()), + ..Default::default() + }, + ) + .await? + .first() + .copied()) +} + +async fn resolve_power_shelf_id_for_firmware_device( + txn: &mut sqlx::PgConnection, + rack_id: &RackId, + device: &FirmwareUpgradeDeviceStatus, +) -> Result, StateHandlerError> { + if !device.node_id.is_empty() { + return Ok(device + .node_id + .parse::() + .ok()); + } + let mac: mac_address::MacAddress = match device.mac.parse() { + Ok(mac) => mac, + Err(_) => return Ok(None), + }; + Ok(db_power_shelf::find_ids( + txn, + model::power_shelf::PowerShelfSearchFilter { + bmc_mac: Some(mac), + rack_id: Some(rack_id.clone()), + ..Default::default() + }, + ) + .await? + .first() + .copied()) +} + +fn machine_firmware_outcome(machine: &model::machine::Machine) -> DeviceFirmwareOutcome { + match &machine.state.value { + model::machine::ManagedHostState::HostReprovision { + reprovision_state: model::machine::HostReprovisionState::WaitingForRackFirmwareUpgrade, + .. + } => DeviceFirmwareOutcome::Waiting, + model::machine::ManagedHostState::HostReprovision { + reprovision_state: model::machine::HostReprovisionState::FailedFirmwareUpgrade { .. }, + .. + } + | model::machine::ManagedHostState::Failed { .. } => DeviceFirmwareOutcome::Failed, + // Machine has left WaitingForRackFirmwareUpgrade for a later + // HostReprovision sub-state (success path). + model::machine::ManagedHostState::HostReprovision { .. } => { + DeviceFirmwareOutcome::Completed + } + // Request posted but controller has not entered the wait state yet. + _ if machine.host_reprovision_requested.is_some() => DeviceFirmwareOutcome::Waiting, + _ => DeviceFirmwareOutcome::Completed, + } +} + +fn switch_firmware_outcome(switch: &model::switch::Switch) -> DeviceFirmwareOutcome { + match &switch.controller_state.value { + model::switch::SwitchControllerState::ReProvisioning { + reprovisioning_state: model::switch::ReProvisioningState::WaitingForRackFirmwareUpgrade, + } => DeviceFirmwareOutcome::Waiting, + model::switch::SwitchControllerState::ReProvisioning { + reprovisioning_state: + model::switch::ReProvisioningState::WaitingForNVOSUpgrade + | model::switch::ReProvisioningState::WaitingForNMXCConfigure, + } => DeviceFirmwareOutcome::Completed, + model::switch::SwitchControllerState::Error { .. } => DeviceFirmwareOutcome::Failed, + // Request posted but controller has not entered the wait state yet. + model::switch::SwitchControllerState::Ready + if switch.switch_reprovisioning_requested.is_some() => + { + DeviceFirmwareOutcome::Waiting + } + _ => DeviceFirmwareOutcome::Completed, + } +} + +fn power_shelf_firmware_outcome( + power_shelf: &model::power_shelf::PowerShelf, +) -> DeviceFirmwareOutcome { + match &power_shelf.controller_state.value { + model::power_shelf::PowerShelfControllerState::ReProvisioning { + reprovisioning_state: + model::power_shelf::ReProvisioningState::WaitingForRackFirmwareUpgrade, + } => DeviceFirmwareOutcome::Waiting, + model::power_shelf::PowerShelfControllerState::Error { .. } => { + DeviceFirmwareOutcome::Failed + } + // Request posted but controller has not entered the wait state yet + // (or is about to clear it when rack_firmware_reprovisioning_enabled + // is false). + model::power_shelf::PowerShelfControllerState::Ready + if power_shelf.power_shelf_reprovisioning_requested.is_some() => + { + DeviceFirmwareOutcome::Waiting + } + _ => DeviceFirmwareOutcome::Completed, + } +} + +fn summarize_firmware_outcomes(outcomes: &[DeviceFirmwareOutcome]) -> DeviceFirmwareProgress { + let total = outcomes.len(); + let completed = outcomes + .iter() + .filter(|outcome| matches!(outcome, DeviceFirmwareOutcome::Completed)) + .count(); + let failed = outcomes + .iter() + .filter(|outcome| matches!(outcome, DeviceFirmwareOutcome::Failed)) + .count(); + let pending = outcomes + .iter() + .filter(|outcome| matches!(outcome, DeviceFirmwareOutcome::Waiting)) + .count(); + if pending > 0 { + return DeviceFirmwareProgress::Waiting { + pending, + total, + completed, + failed, + }; + } + if failed > 0 { + return DeviceFirmwareProgress::Failed { failed, total }; + } + DeviceFirmwareProgress::Completed { completed, total } +} + +/// Reads machine and switch controller states for devices in `rack_id`, +/// filtered by `scope`, and decides whether firmware WaitForComplete can +/// advance. Device membership comes from the DB + scope, not the firmware job. +async fn evaluate_firmware_progress_from_devices( + txn: &mut sqlx::PgConnection, + rack_id: &RackId, + scope: &MaintenanceScope, +) -> Result { + let machine_ids = db_machine::find_machine_ids( + &mut *txn, + model::machine::machine_search_config::MachineSearchConfig { + rack_id: Some(rack_id.clone()), + ..Default::default() + }, + ) + .await?; + let machines = if machine_ids.is_empty() { + Vec::new() + } else { + db_machine::find( + &mut *txn, + db::ObjectFilter::List(&machine_ids), + model::machine::machine_search_config::MachineSearchConfig::default(), + ) + .await? + }; + let machines = filter_machines_by_scope(machines, scope); + + let switch_ids = db_switch::find_ids( + &mut *txn, + model::switch::SwitchSearchFilter { + rack_id: Some(rack_id.clone()), + deleted: model::DeletedFilter::Exclude, + ..Default::default() + }, + ) + .await?; + let switches = if switch_ids.is_empty() { + Vec::new() + } else { + db_switch::find_by( + txn, + db::ObjectColumnFilter::List(db_switch::IdColumn, &switch_ids), + ) + .await? + }; + let switches = filter_switches_by_scope(switches, scope); + + let power_shelf_ids = db_power_shelf::find_ids( + &mut *txn, + model::power_shelf::PowerShelfSearchFilter { + rack_id: Some(rack_id.clone()), + deleted: model::DeletedFilter::Exclude, + ..Default::default() + }, + ) + .await?; + let power_shelves = if power_shelf_ids.is_empty() { + Vec::new() + } else { + db_power_shelf::find_by( + txn, + db::ObjectColumnFilter::List(db_power_shelf::IdColumn, &power_shelf_ids), + ) + .await? + }; + let power_shelves = filter_power_shelves_by_scope(power_shelves, scope); + + let mut outcomes = Vec::with_capacity(machines.len() + switches.len() + power_shelves.len()); + outcomes.extend(machines.iter().map(machine_firmware_outcome)); + outcomes.extend(switches.iter().map(switch_firmware_outcome)); + outcomes.extend(power_shelves.iter().map(power_shelf_firmware_outcome)); + Ok(summarize_firmware_outcomes(&outcomes)) +} + +fn filter_machines_by_scope( + mut machines: Vec, + scope: &MaintenanceScope, +) -> Vec { + if scope.is_full_rack() { + return machines; + } + if scope.machine_ids.is_empty() { + return Vec::new(); + } + let allowed: std::collections::HashSet<_> = scope.machine_ids.iter().collect(); + machines.retain(|machine| allowed.contains(&machine.id)); + machines +} + +fn filter_switches_by_scope( + mut switches: Vec, + scope: &MaintenanceScope, +) -> Vec { + if scope.is_full_rack() { + return switches; + } + if scope.switch_ids.is_empty() { + return Vec::new(); + } + let allowed: std::collections::HashSet<_> = scope.switch_ids.iter().collect(); + switches.retain(|switch| allowed.contains(&switch.id)); + switches +} + +fn filter_power_shelves_by_scope( + mut power_shelves: Vec, + scope: &MaintenanceScope, +) -> Vec { + if scope.is_full_rack() { + return power_shelves; + } + if scope.power_shelf_ids.is_empty() { + return Vec::new(); + } + let allowed: std::collections::HashSet<_> = scope.power_shelf_ids.iter().collect(); + power_shelves.retain(|power_shelf| allowed.contains(&power_shelf.id)); + power_shelves +} + +fn filter_power_shelf_ids_by_scope( + mut power_shelf_ids: Vec, + scope: &MaintenanceScope, +) -> Vec { + if scope.is_full_rack() { + return power_shelf_ids; + } + if scope.power_shelf_ids.is_empty() { + return Vec::new(); + } + let allowed: std::collections::HashSet<_> = scope.power_shelf_ids.iter().collect(); + power_shelf_ids.retain(|id| allowed.contains(id)); + power_shelf_ids +} + fn skip_firmware_upgrade_outcome( rack_id: &RackId, reason: impl AsRef, @@ -361,6 +706,36 @@ pub(crate) fn first_maintenance_state(scope: &MaintenanceScope) -> RackMaintenan } } +/// Returns the state to advance to when the current maintenance state's +/// activity is not requested. Every [`RackMaintenanceState`] variant is +/// checked here so a persisted or stale state cannot execute outside the +/// current scope. +fn next_state_if_activity_not_requested( + maintenance_state: &RackMaintenanceState, + scope: &MaintenanceScope, +) -> Option { + match maintenance_state { + RackMaintenanceState::FirmwareUpgrade { .. } => { + (!scope.should_run(&MaintenanceActivity::FirmwareUpgrade { + firmware_version: None, + components: vec![], + force_update: false, + })) + .then(|| next_state_after_firmware(scope)) + } + RackMaintenanceState::NVOSUpdate { .. } => { + (!nvos_update_requested(scope)).then(|| next_state_after_nvos(scope)) + } + RackMaintenanceState::ConfigureNmxCluster { .. } => (!scope + .should_run(&MaintenanceActivity::ConfigureNmxCluster)) + .then(|| next_state_after_configure(scope)), + RackMaintenanceState::PowerSequence { .. } => (!scope + .should_run(&MaintenanceActivity::PowerSequence)) + .then_some(RackMaintenanceState::Completed), + RackMaintenanceState::Completed => None, + } +} + /// Filters a full-rack firmware inventory down to compute/switch devices listed /// in the maintenance scope. Power-shelf firmware-object JSON apply is not /// implemented yet. @@ -1304,6 +1679,18 @@ pub async fn handle_maintenance( .unwrap_or_default(); let scope = &scope; + if let Some(next) = next_state_if_activity_not_requested(maintenance_state, scope) { + tracing::info!( + rack_id = %id, + current_state = %maintenance_state, + next_state = %next, + "Skipping rack maintenance state not requested by scope activities" + ); + return Ok(StateHandlerOutcome::transition(RackState::Maintenance { + maintenance_state: next, + })); + } + match maintenance_state { RackMaintenanceState::FirmwareUpgrade { rack_firmware_upgrade, @@ -1462,19 +1849,30 @@ pub async fn handle_maintenance( }; let mut txn = ctx.services.db_pool.begin().await?; - let continue_after_firmware_upgrade = nvos_update_requested(scope); + let power_shelf_ids = db_power_shelf::find_ids( + txn.as_mut(), + model::power_shelf::PowerShelfSearchFilter { + rack_id: Some(id.clone()), + deleted: model::DeletedFilter::Exclude, + ..Default::default() + }, + ) + .await?; + let power_shelf_ids = filter_power_shelf_ids_by_scope(power_shelf_ids, scope); trigger_rack_firmware_reprovisioning_requests( txn.as_mut(), id, &inventory.machine_ids, &inventory.switch_ids, - continue_after_firmware_upgrade, + &power_shelf_ids, + &scope.activities, ) .await?; clear_rack_firmware_device_statuses( txn.as_mut(), &inventory.machine_ids, &inventory.switch_ids, + &power_shelf_ids, ) .await?; job.started_at = Some(chrono::Utc::now()); @@ -1509,18 +1907,6 @@ pub async fn handle_maintenance( let mut job = rms_get_firmware_upgrade_status(rms_client.as_ref(), current_job).await?; - let all: Vec<_> = job.all_devices().collect(); - let total = all.len(); - let completed = all.iter().filter(|d| d.status == "completed").count(); - let failed = all.iter().filter(|d| d.status == "failed").count(); - let terminal = completed + failed; - if failed > 0 && requested_nvos_config_json(scope).is_some() { - delete_rack_maintenance_access_token( - ctx.services.credential_manager.as_ref(), - id, - ) - .await; - } let mut txn = ctx.services.db_pool.begin().await?; let build_status = @@ -1551,19 +1937,9 @@ pub async fn handle_maintenance( }; for device in job.machines.iter() { - let machine_id = if !device.node_id.is_empty() { - device - .node_id - .parse::() - .ok() - } else { - let mac: mac_address::MacAddress = match device.mac.parse() { - Ok(mac) => mac, - Err(_) => continue, - }; - db_machine_topology::find_machine_id_by_bmc_mac(txn.as_mut(), mac).await? - }; - if let Some(machine_id) = machine_id { + if let Some(machine_id) = + resolve_machine_id_for_firmware_device(txn.as_mut(), device).await? + { let fw_status = build_status(device); db_machine::update_rack_fw_details( txn.as_mut(), @@ -1575,29 +1951,9 @@ pub async fn handle_maintenance( } for device in job.switches.iter() { - let switch_id = if !device.node_id.is_empty() { - device - .node_id - .parse::() - .ok() - } else { - let mac: mac_address::MacAddress = match device.mac.parse() { - Ok(mac) => mac, - Err(_) => continue, - }; - db_switch::find_ids( - txn.as_mut(), - model::switch::SwitchSearchFilter { - bmc_mac: Some(mac), - rack_id: Some(id.clone()), - ..Default::default() - }, - ) - .await? - .first() - .copied() - }; - if let Some(switch_id) = switch_id { + if let Some(switch_id) = + resolve_switch_id_for_firmware_device(txn.as_mut(), id, device).await? + { let fw_status = build_status(device); db_switch::update_firmware_upgrade_status( txn.as_mut(), @@ -1608,71 +1964,140 @@ pub async fn handle_maintenance( } } - if terminal < total { - db_rack::update_firmware_upgrade_job(txn.as_mut(), id, Some(&job)).await?; - state.firmware_upgrade_job = Some(job); - return Ok(StateHandlerOutcome::wait(format!( - "firmware upgrade: {}/{} devices terminal (completed={}, failed={})", - terminal, total, completed, failed - )) - .with_txn(txn)); - } - - if failed > 0 { - let now = chrono::Utc::now(); - job.status = Some("failed".into()); - if job.completed_at.is_none() { - job.completed_at = Some(now); - } - db_rack::update_firmware_upgrade_job(txn.as_mut(), id, Some(&job)).await?; - state.firmware_upgrade_job = Some(job); - if state.config.maintenance_requested.is_some() { - state.config.maintenance_requested = None; - db_rack::update(txn.as_mut(), id, &state.config).await?; + for device in job.power_shelves.iter() { + if let Some(power_shelf_id) = + resolve_power_shelf_id_for_firmware_device(txn.as_mut(), id, device).await? + { + let fw_status = build_status(device); + db_power_shelf::update_firmware_upgrade_status( + txn.as_mut(), + power_shelf_id, + Some(&fw_status), + ) + .await?; } - return Ok(StateHandlerOutcome::transition(RackState::Error { - cause: format!( - "firmware upgrade failed: {}/{} devices failed", - failed, total - ), - }) - .with_txn(txn)); } - let now = chrono::Utc::now(); - job.status = Some("completed".into()); - if job.completed_at.is_none() { - job.completed_at = Some(now); + // When RMS does not yet report power-shelf devices, stamp + // Completed for scoped shelves so enabled controllers can leave + // WaitingForRackFirmwareUpgrade without hanging. + if job.power_shelves.is_empty() { + let power_shelf_ids = db_power_shelf::find_ids( + txn.as_mut(), + model::power_shelf::PowerShelfSearchFilter { + rack_id: Some(id.clone()), + deleted: model::DeletedFilter::Exclude, + ..Default::default() + }, + ) + .await?; + let power_shelf_ids = filter_power_shelf_ids_by_scope(power_shelf_ids, scope); + if !power_shelf_ids.is_empty() { + let fw_status = RackFirmwareUpgradeStatus { + task_id: job.job_id.clone().unwrap_or_else(|| "unknown".to_string()), + status: RackFirmwareUpgradeState::Completed, + started_at: job.started_at, + ended_at: Some(chrono::Utc::now()), + }; + for power_shelf_id in power_shelf_ids { + db_power_shelf::update_firmware_upgrade_status( + txn.as_mut(), + power_shelf_id, + Some(&fw_status), + ) + .await?; + } + } } - db_rack::update_firmware_upgrade_job(txn.as_mut(), id, Some(&job)).await?; - state.firmware_upgrade_job = Some(job); - let next_maintenance_state = if nvos_update_requested(scope) { - let next = next_state_after_firmware(scope); - tracing::info!( - rack_id = %id, - completed_device_count = completed, - total_device_count = total, - next_state = %next, - "Rack firmware upgrade complete; advancing to explicitly requested next activity" - ); - next - } else { - let next = next_state_after_nvos(scope); - tracing::info!( - rack_id = %id, - completed_device_count = completed, - total_device_count = total, - next_state = %next, - "Rack firmware upgrade complete; no explicit NVOS update requested, advancing" - ); - next - }; + // Advancement is driven by machine/switch/power-shelf controller + // states for devices in this rack that are selected by the + // maintenance scope. + let progress = + evaluate_firmware_progress_from_devices(txn.as_mut(), id, scope).await?; + + match progress { + DeviceFirmwareProgress::Waiting { + pending: _, + total, + completed, + failed, + } => { + db_rack::update_firmware_upgrade_job(txn.as_mut(), id, Some(&job)).await?; + state.firmware_upgrade_job = Some(job); + return Ok(StateHandlerOutcome::wait(format!( + "firmware upgrade: waiting on machine/switch/power-shelf controller state ({}/{} past firmware wait, completed={}, failed={})", + completed + failed, + total, + completed, + failed + )) + .with_txn(txn)); + } + DeviceFirmwareProgress::Failed { failed, total } => { + if requested_nvos_config_json(scope).is_some() { + delete_rack_maintenance_access_token( + ctx.services.credential_manager.as_ref(), + id, + ) + .await; + } + let now = chrono::Utc::now(); + job.status = Some("failed".into()); + if job.completed_at.is_none() { + job.completed_at = Some(now); + } + db_rack::update_firmware_upgrade_job(txn.as_mut(), id, Some(&job)).await?; + state.firmware_upgrade_job = Some(job); + if state.config.maintenance_requested.is_some() { + state.config.maintenance_requested = None; + db_rack::update(txn.as_mut(), id, &state.config).await?; + } + return Ok(StateHandlerOutcome::transition(RackState::Error { + cause: format!( + "firmware upgrade failed: {}/{} devices failed", + failed, total + ), + }) + .with_txn(txn)); + } + DeviceFirmwareProgress::Completed { completed, total } => { + let now = chrono::Utc::now(); + job.status = Some("completed".into()); + if job.completed_at.is_none() { + job.completed_at = Some(now); + } + db_rack::update_firmware_upgrade_job(txn.as_mut(), id, Some(&job)).await?; + state.firmware_upgrade_job = Some(job); + + let next_maintenance_state = if nvos_update_requested(scope) { + let next = next_state_after_firmware(scope); + tracing::info!( + rack_id = %id, + completed_device_count = completed, + total_device_count = total, + next_state = %next, + "Rack firmware upgrade complete on machine/switch controllers; advancing to explicitly requested next activity" + ); + next + } else { + let next = next_state_after_nvos(scope); + tracing::info!( + rack_id = %id, + completed_device_count = completed, + total_device_count = total, + next_state = %next, + "Rack firmware upgrade complete on machine/switch controllers; no explicit NVOS update requested, advancing" + ); + next + }; - Ok(StateHandlerOutcome::transition(RackState::Maintenance { - maintenance_state: next_maintenance_state, - }) - .with_txn(txn)) + Ok(StateHandlerOutcome::transition(RackState::Maintenance { + maintenance_state: next_maintenance_state, + }) + .with_txn(txn)) + } + } } }, RackMaintenanceState::NVOSUpdate { nvos_update } => match nvos_update { @@ -2480,9 +2905,11 @@ mod tests { use model::rack_type::{RackHardwareType, RackProductFamily, RackProfile}; use super::{ - build_switch_device_info_request, filter_inventory_by_scope, firmware_device_status, - first_maintenance_state, next_state_after_configure, next_state_after_firmware, - next_state_after_nvos, profile_hardware_type_or_any, + DeviceFirmwareOutcome, DeviceFirmwareProgress, build_switch_device_info_request, + filter_inventory_by_scope, firmware_device_status, first_maintenance_state, + next_state_after_configure, next_state_after_firmware, next_state_after_nvos, + next_state_if_activity_not_requested, profile_hardware_type_or_any, + summarize_firmware_outcomes, }; fn test_machine_id(seed: u8) -> MachineId { @@ -2679,6 +3106,51 @@ mod tests { assert_eq!(status.error_message.as_deref(), Some("invalid SOT JSON")); } + #[test] + fn test_summarize_firmware_outcomes_from_controller_states() { + check_values( + [ + Check { + scenario: "any waiting keeps rack waiting", + input: vec![ + DeviceFirmwareOutcome::Completed, + DeviceFirmwareOutcome::Waiting, + DeviceFirmwareOutcome::Failed, + ], + expect: DeviceFirmwareProgress::Waiting { + pending: 1, + total: 3, + completed: 1, + failed: 1, + }, + }, + Check { + scenario: "all completed advances", + input: vec![ + DeviceFirmwareOutcome::Completed, + DeviceFirmwareOutcome::Completed, + ], + expect: DeviceFirmwareProgress::Completed { + completed: 2, + total: 2, + }, + }, + Check { + scenario: "any failed without waiting errors", + input: vec![ + DeviceFirmwareOutcome::Completed, + DeviceFirmwareOutcome::Failed, + ], + expect: DeviceFirmwareProgress::Failed { + failed: 1, + total: 2, + }, + }, + ], + |outcomes| summarize_firmware_outcomes(&outcomes), + ); + } + /// A firmware-upgrade activity with no version/components/force, the form /// used by the maintenance-state transition tables. fn firmware_upgrade() -> MaintenanceActivity { @@ -2726,6 +3198,82 @@ mod tests { } } + // ── current-state scope enforcement ───────────────────────────────── + + #[test] + fn test_next_state_if_activity_not_requested() { + check_values( + [ + Check { + scenario: "requested firmware continues", + input: (firmware_start(), scope_of(vec![firmware_upgrade()])), + expect: None, + }, + Check { + scenario: "unrequested firmware skips to configure", + input: ( + firmware_start(), + scope_of(vec![MaintenanceActivity::ConfigureNmxCluster]), + ), + expect: Some(configure_start()), + }, + Check { + scenario: "requested nvos continues", + input: (nvos_start(), scope_of(vec![nvos_update()])), + expect: None, + }, + Check { + scenario: "unrequested nvos skips to configure", + input: ( + nvos_start(), + scope_of(vec![MaintenanceActivity::ConfigureNmxCluster]), + ), + expect: Some(configure_start()), + }, + Check { + scenario: "requested configure continues", + input: ( + configure_start(), + scope_of(vec![MaintenanceActivity::ConfigureNmxCluster]), + ), + expect: None, + }, + Check { + scenario: "unrequested configure skips to power", + input: ( + configure_start(), + scope_of(vec![MaintenanceActivity::PowerSequence]), + ), + expect: Some(powering_on()), + }, + Check { + scenario: "requested power continues", + input: ( + powering_on(), + scope_of(vec![MaintenanceActivity::PowerSequence]), + ), + expect: None, + }, + Check { + scenario: "unrequested power skips to completed", + input: (powering_on(), scope_of(vec![firmware_upgrade()])), + expect: Some(RackMaintenanceState::Completed), + }, + Check { + scenario: "completed always continues", + input: ( + RackMaintenanceState::Completed, + scope_of(vec![firmware_upgrade()]), + ), + expect: None, + }, + ], + |(maintenance_state, scope)| { + next_state_if_activity_not_requested(&maintenance_state, &scope) + }, + ); + } + // ── first_maintenance_state ───────────────────────────────────────── #[test] diff --git a/crates/rpc/src/model/power_shelf.rs b/crates/rpc/src/model/power_shelf.rs index 234316d869..229f8ddbe7 100644 --- a/crates/rpc/src/model/power_shelf.rs +++ b/crates/rpc/src/model/power_shelf.rs @@ -186,6 +186,8 @@ mod tests { bmc_info: None, rack_id: None, power_shelf_maintenance_requested: None, + power_shelf_reprovisioning_requested: None, + firmware_upgrade_status: None, metadata: Metadata::default(), version: ConfigVersion::initial(), health_reports: Default::default(), diff --git a/crates/switch-controller/src/ready.rs b/crates/switch-controller/src/ready.rs index 06a3def884..a33828eccd 100644 --- a/crates/switch-controller/src/ready.rs +++ b/crates/switch-controller/src/ready.rs @@ -18,24 +18,26 @@ //! Handler for SwitchControllerState::Ready. use carbide_uuid::switch::SwitchId; -use model::switch::{ReProvisioningState, Switch, SwitchControllerState}; +use db::switch as db_switch; +use model::switch::{Switch, SwitchControllerState}; use state_controller::state_handler::{ StateHandlerContext, StateHandlerError, StateHandlerOutcome, }; use crate::context::SwitchStateHandlerContextObjects; +use crate::reprovisioning::first_reprovisioning_state; /// Handles the Ready state for a switch. /// /// If the switch is marked for deletion, transitions to `Deleting`. /// If a maintenance request has been posted via `switch_maintenance_requested`, /// transitions to `Maintenance` with the requested operation. If rack-level -/// reprovisioning has been requested, transitions to `ReProvisioning`. -/// Otherwise idles. +/// reprovisioning has been requested, transitions to the first +/// `ReProvisioning` sub-state selected by the request activities. Otherwise idles. pub async fn handle_ready( - _switch_id: &SwitchId, + switch_id: &SwitchId, state: &mut Switch, - _ctx: &mut StateHandlerContext<'_, SwitchStateHandlerContextObjects>, + ctx: &mut StateHandlerContext<'_, SwitchStateHandlerContextObjects>, ) -> Result, StateHandlerError> { if state.is_marked_as_deleted() { return Ok(StateHandlerOutcome::transition( @@ -55,27 +57,39 @@ pub async fn handle_ready( } if let Some(req) = &state.switch_reprovisioning_requested { - if req.initiator.starts_with("rack-") { - tracing::info!( - "Rack-level firmware upgrade requested — transitioning to WaitingForRackFirmwareUpgrade" + if !req.initiator.starts_with("rack-") { + tracing::warn!( + initiator = %req.initiator, + "Unknown initiator for switch reprovisioning request", ); return Ok(StateHandlerOutcome::transition( - SwitchControllerState::ReProvisioning { - reprovisioning_state: ReProvisioningState::WaitingForRackFirmwareUpgrade, + SwitchControllerState::Error { + cause: format!( + "unknown initiator for switch reprovisioning request: {}", + req.initiator + ), }, )); } - tracing::warn!( - initiator = %req.initiator, - "Unknown initiator for switch reprovisioning request", + let Some(reprovisioning_state) = first_reprovisioning_state(req) else { + tracing::warn!( + switch_id = %switch_id, + initiator = %req.initiator, + "Rack reprovision request has no switch-relevant activities; clearing request" + ); + let mut txn = ctx.services.db_pool.begin().await?; + db_switch::clear_switch_reprovisioning_requested(txn.as_mut(), *switch_id).await?; + return Ok(StateHandlerOutcome::do_nothing().with_txn(txn)); + }; + + tracing::info!( + ?reprovisioning_state, + "Rack-level reprovisioning requested — entering ReProvisioning" ); return Ok(StateHandlerOutcome::transition( - SwitchControllerState::Error { - cause: format!( - "unknown initiator for switch reprovisioning request: {}", - req.initiator - ), + SwitchControllerState::ReProvisioning { + reprovisioning_state, }, )); } diff --git a/crates/switch-controller/src/reprovisioning.rs b/crates/switch-controller/src/reprovisioning.rs index 0e1f5adfac..bdc795c9eb 100644 --- a/crates/switch-controller/src/reprovisioning.rs +++ b/crates/switch-controller/src/reprovisioning.rs @@ -20,8 +20,8 @@ use carbide_uuid::switch::SwitchId; use db::db_read::PgPoolReader; use db::{ObjectColumnFilter, rack as db_rack, switch as db_switch}; -use model::rack::RackState; -use model::switch::{ReProvisioningState, Switch, SwitchControllerState}; +use model::rack::{MaintenanceActivity, RackState}; +use model::switch::{ReProvisioningState, Switch, SwitchControllerState, SwitchReprovisionRequest}; use state_controller::state_handler::{ StateHandlerContext, StateHandlerError, StateHandlerOutcome, }; @@ -37,6 +37,68 @@ fn is_rack_level_reprovisioning(state: &Switch) -> bool { .is_some_and(|req| req.initiator.starts_with("rack-")) } +fn should_run(activities: &[MaintenanceActivity], activity: &MaintenanceActivity) -> bool { + activities.is_empty() || activities.iter().any(|a| a.same_kind(activity)) +} + +fn nvos_update_requested(activities: &[MaintenanceActivity]) -> bool { + activities + .iter() + .any(|activity| matches!(activity, MaintenanceActivity::NvosUpdate { .. })) +} + +fn configure_nmx_cluster_requested(activities: &[MaintenanceActivity]) -> bool { + should_run(activities, &MaintenanceActivity::ConfigureNmxCluster) +} + +fn firmware_upgrade_requested(activities: &[MaintenanceActivity]) -> bool { + should_run( + activities, + &MaintenanceActivity::FirmwareUpgrade { + firmware_version: None, + components: vec![], + force_update: false, + }, + ) +} + +/// First ReProvisioning sub-state to enter from Ready, based on the request +/// activities. Empty activities means all phases. Returns `None` when the +/// activities list has no switch-relevant wait phases. +pub(crate) fn first_reprovisioning_state( + request: &SwitchReprovisionRequest, +) -> Option { + let activities = &request.activities; + if firmware_upgrade_requested(activities) { + return Some(ReProvisioningState::WaitingForRackFirmwareUpgrade); + } + if nvos_update_requested(activities) { + return Some(ReProvisioningState::WaitingForNVOSUpgrade); + } + if configure_nmx_cluster_requested(activities) { + return Some(ReProvisioningState::WaitingForNMXCConfigure); + } + None +} + +/// Next ReProvisioning sub-state after firmware completes. +fn next_state_after_firmware(request: &SwitchReprovisionRequest) -> Option { + let activities = &request.activities; + if nvos_update_requested(activities) { + return Some(ReProvisioningState::WaitingForNVOSUpgrade); + } + if configure_nmx_cluster_requested(activities) { + return Some(ReProvisioningState::WaitingForNMXCConfigure); + } + None +} + +/// Next ReProvisioning sub-state after NVOS completes. +fn next_state_after_nvos(request: &SwitchReprovisionRequest) -> Option { + configure_nmx_cluster_requested(&request.activities) + .then_some(ReProvisioningState::WaitingForNMXCConfigure) +} + /// If the parent rack is in `RackState::Error`, clear /// `switch_reprovisioning_requested` and short-circuit to `Ready`. The /// rack will never advance the remaining `ReProvisioning` sub-states once @@ -107,7 +169,7 @@ pub async fn handle_reprovisioning( .as_ref() .expect("WaitingForRackFirmwareUpgrade requires a rack reprovision request"); let requested_at = request.requested_at; - let continue_after_firmware_upgrade = request.continue_after_firmware_upgrade; + let next_after_firmware = next_state_after_firmware(request); let Some(firmware_upgrade_status) = state.firmware_upgrade_status.as_ref() else { return Ok(StateHandlerOutcome::wait( "waiting for switch firmware upgrade status".into(), @@ -126,10 +188,10 @@ pub async fn handle_reprovisioning( match &firmware_upgrade_status.status { model::rack::RackFirmwareUpgradeState::Completed => { - if continue_after_firmware_upgrade { + if let Some(reprovisioning_state) = next_after_firmware { return Ok(StateHandlerOutcome::transition( SwitchControllerState::ReProvisioning { - reprovisioning_state: ReProvisioningState::WaitingForNVOSUpgrade, + reprovisioning_state, }, )); } @@ -157,11 +219,12 @@ pub async fn handle_reprovisioning( } } ReProvisioningState::WaitingForNVOSUpgrade => { - let requested_at = state + let request = state .switch_reprovisioning_requested .as_ref() - .map(|request| request.requested_at) .expect("WaitingForNVOSUpgrade requires a rack reprovision request"); + let requested_at = request.requested_at; + let next_after_nvos = next_state_after_nvos(request); let Some(nvos_update_status) = state.nvos_update_status.as_ref() else { return Ok(StateHandlerOutcome::wait( "waiting for switch NVOS update status".into(), @@ -179,11 +242,23 @@ pub async fn handle_reprovisioning( } match &nvos_update_status.status { - model::rack::SwitchNvosUpdateState::Completed => Ok( - StateHandlerOutcome::transition(SwitchControllerState::ReProvisioning { - reprovisioning_state: ReProvisioningState::WaitingForNMXCConfigure, - }), - ), + model::rack::SwitchNvosUpdateState::Completed => { + if let Some(reprovisioning_state) = next_after_nvos { + Ok(StateHandlerOutcome::transition( + SwitchControllerState::ReProvisioning { + reprovisioning_state, + }, + )) + } else { + let mut txn = ctx.services.db_pool.begin().await?; + db_switch::clear_switch_reprovisioning_requested(txn.as_mut(), *switch_id) + .await?; + Ok( + StateHandlerOutcome::transition(SwitchControllerState::Ready) + .with_txn(txn), + ) + } + } model::rack::SwitchNvosUpdateState::Failed { cause } => { let mut txn = ctx.services.db_pool.begin().await?; db_switch::clear_switch_reprovisioning_requested(txn.as_mut(), *switch_id)