diff --git a/vmm_core/state_unit/src/lib.rs b/vmm_core/state_unit/src/lib.rs index 87a9525e73d..ac43102b809 100644 --- a/vmm_core/state_unit/src/lib.rs +++ b/vmm_core/state_unit/src/lib.rs @@ -50,7 +50,6 @@ use parking_lot::Mutex; use std::collections::BTreeMap; use std::collections::BTreeSet; use std::collections::HashMap; -use std::collections::hash_map; use std::fmt::Debug; use std::fmt::Display; use std::future::Future; @@ -241,10 +240,39 @@ struct Unit { inspect_sensitivity: inspect::SensitivityLevel, } -/// An error returned when a state unit name is already in use. +/// An error returned when a state unit cannot be added to a set. +/// +/// A unit that cannot be added leaves the set unchanged. #[derive(Debug, Error)] -#[error("state unit name {0} is in use")] -pub struct NameInUse(Arc); +pub enum UnitBuildError { + /// Another unit in the set has the name. + #[error("state unit name {0} is in use")] + NameInUse(Arc), + /// A unit that the new unit depends on, or is a dependency of, has been + /// removed from the set. + #[error("state unit {unit} refers to removed state unit {removed}")] + RemovedUnit { + /// The name of the new unit. + unit: Arc, + /// The name of the removed unit. + removed: Arc, + }, + /// The new unit would depend, directly or through other units, on a unit + /// that it would also be a dependency of. No state change that orders the + /// units could complete with such a cycle. + #[error("state unit {unit} would form a dependency cycle with state unit {dependent}")] + DependencyCycle { + /// The name of the new unit. + unit: Arc, + /// The name of the unit that the new unit would both depend on and be a + /// dependency of. + dependent: Arc, + }, +} + +/// The error returned by [`UnitBuilder::build`] and [`UnitBuilder::spawn`], +/// under its name from when they only checked the unit name. +pub type NameInUse = UnitBuildError; #[derive(Debug, Error)] #[error("critical unit communication failure: {name}")] @@ -475,8 +503,8 @@ impl StateUnits { UnitBuilder { units: self, name: name.into(), - dependencies: BTreeSet::new(), - dependents: BTreeSet::new(), + dependencies: BTreeMap::new(), + dependents: BTreeMap::new(), inspect_sensitivity: inspect::SensitivityLevel::Unspecified, } } @@ -788,6 +816,53 @@ impl StateUnits { } impl Inner { + /// Checks that a unit named `name`, with `dependencies` and `dependents` + /// by ID and name, can be added to the set. + fn check_new_unit( + &self, + name: &Arc, + dependencies: &BTreeMap>, + dependents: &BTreeMap>, + ) -> Result<(), UnitBuildError> { + if self.names.contains_key(name) { + return Err(UnitBuildError::NameInUse(name.clone())); + } + if let Some(removed) = dependencies + .iter() + .chain(dependents) + .find_map(|(id, other)| (!self.units.contains_key(id)).then_some(other)) + { + return Err(UnitBuildError::RemovedUnit { + unit: name.clone(), + removed: removed.clone(), + }); + } + // The units in the set form no cycle, so a new cycle would run from the + // new unit to one of its dependencies, along existing dependency edges + // to one of its dependents, and back to the new unit. Without + // dependents, nothing leads back to the new unit. + if dependents.is_empty() { + return Ok(()); + } + let mut pending = dependencies.keys().copied().collect::>(); + let mut reached = BTreeSet::new(); + while let Some(id) = pending.pop() { + if let Some(dependent) = dependents.get(&id) { + return Err(UnitBuildError::DependencyCycle { + unit: name.clone(), + dependent: dependent.clone(), + }); + } + // A unit that is not in the set has no edges to follow. + if reached.insert(id) + && let Some(unit) = self.units.get(&id) + { + pending.extend(&unit.dependencies); + } + } + Ok(()) + } + fn ready_set(&self, unit_ids: Option<&[u64]>) -> ReadySet { let map = |id, unit: &Unit| { ( @@ -820,6 +895,11 @@ struct ReadyState { } impl ReadySet { + /// Waits, in turn, for each unit of `deps` in this set to finish its state + /// change, and returns false at the first one that failed. + /// + /// [`UnitBuilder::build`] rejects dependency cycles, so an operation's + /// waits all end once every unit answers its own request. async fn wait(&self, op: &str, id: u64, deps: &[u64]) -> bool { for dep in deps { if let Some(dep) = self.0.get(dep) { @@ -879,8 +959,10 @@ fn state_change) -> StateRe pub struct UnitBuilder<'a> { units: &'a StateUnits, name: Arc, - dependencies: BTreeSet, - dependents: BTreeSet, + // The units to order this one with, by ID, and their names, which a build + // reports if the unit has been removed by then. + dependencies: BTreeMap>, + dependents: BTreeMap>, inspect_sensitivity: inspect::SensitivityLevel, } @@ -890,7 +972,8 @@ impl UnitBuilder<'_> { /// Operations will be ordered to ensure that a dependency will stop after /// its dependants, and that it will reset or restore before its dependants. pub fn depends_on(mut self, handle: &UnitHandle) -> Self { - self.dependencies.insert(self.handle_id(handle)); + let UnitId { name, id } = self.unit_id(handle); + self.dependencies.insert(id, name); self } @@ -899,7 +982,8 @@ impl UnitBuilder<'_> { /// Operations will be ordered to ensure that a dependency will stop after /// its dependants, and that it will reset or restore before its dependants. pub fn dependency_of(mut self, handle: &UnitHandle) -> Self { - self.dependents.insert(self.handle_id(handle)); + let UnitId { name, id } = self.unit_id(handle); + self.dependents.insert(id, name); self } @@ -909,40 +993,53 @@ impl UnitBuilder<'_> { self } - fn handle_id(&self, handle: &UnitHandle) -> u64 { + fn unit_id(&self, handle: &UnitHandle) -> UnitId { // Ensure this handle is associated with this set of state units. assert_eq!( Weak::as_ptr(handle.inner.as_ref().unwrap()), Arc::as_ptr(&self.units.inner) ); - handle.id.id + handle.id.clone() } /// Adds a new state unit sending requests to `send`. - pub fn build(self, send: Sender) -> Result { + /// + /// Fails, and leaves the set unchanged, if another unit has the name, if + /// a unit passed to [`Self::depends_on`] or [`Self::dependency_of`] has + /// been removed since, or if the new unit would form a dependency cycle. + pub fn build(self, send: Sender) -> Result { let id = { let mut inner = self.units.inner.lock(); + // Check everything before changing the set, so that a unit that + // cannot be added leaves no name, edge, or ID behind. + inner.check_new_unit(&self.name, &self.dependencies, &self.dependents)?; let id = inner.next_id; - let entry = match inner.names.entry(self.name.clone()) { - hash_map::Entry::Occupied(_) => return Err(NameInUse(self.name)), - hash_map::Entry::Vacant(e) => e, - }; - entry.insert(id); + inner.names.insert(self.name.clone(), id); - for &dep in &self.dependencies { - inner.units.get_mut(&dep).unwrap().dependents.insert(id); + for &dep in self.dependencies.keys() { + inner + .units + .get_mut(&dep) + .expect("dependency exists") + .dependents + .insert(id); } - for &dep in &self.dependents { - inner.units.get_mut(&dep).unwrap().dependencies.insert(id); + for &dep in self.dependents.keys() { + inner + .units + .get_mut(&dep) + .expect("dependent exists") + .dependencies + .insert(id); } inner.units.insert( id, Unit { name: self.name.clone(), send, - dependencies: self.dependencies, - dependents: self.dependents, + dependencies: self.dependencies.into_keys().collect(), + dependents: self.dependents.into_keys().collect(), state: State::Stopped, inspect_sensitivity: self.inspect_sensitivity, }, @@ -970,7 +1067,7 @@ impl UnitBuilder<'_> { self, spawner: impl Spawn, f: F, - ) -> Result, NameInUse> + ) -> Result, UnitBuildError> where F: FnOnce(Receiver) -> Fut, Fut: 'static + Send + Future, @@ -1047,6 +1144,8 @@ impl Ready { #[cfg(test)] mod tests { + mod build; + use super::StateTransitionErrorKind; use super::StateUnit; use super::StateUnits; diff --git a/vmm_core/state_unit/src/tests/build.rs b/vmm_core/state_unit/src/tests/build.rs new file mode 100644 index 00000000000..7920dce5a97 --- /dev/null +++ b/vmm_core/state_unit/src/tests/build.rs @@ -0,0 +1,318 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +//! Tests for the checks that adding a state unit makes before it changes the +//! set. + +use super::SavedState; +use super::TestUnit; +use super::assert_no_dangling_edges; +use crate::SavedStateUnit; +use crate::StateTransitionErrorKind; +use crate::StateUnits; +use crate::UnitBuildError; +use crate::UnitHandle; +use crate::run_unit; +use pal_async::DefaultDriver; +use pal_async::async_test; +use std::collections::BTreeMap; +use std::collections::BTreeSet; +use std::sync::Arc; +use std::sync::Barrier; +use std::sync::atomic::AtomicBool; +use std::sync::atomic::Ordering; +use test_with_tracing::test; +use vmcore::save_restore::SavedStateBlob; + +/// The parts of a set that a failed build must leave unchanged. +#[derive(Debug, PartialEq)] +struct Inventory { + next_id: u64, + names: BTreeMap, + edges: BTreeMap, BTreeSet)>, +} + +impl Inventory { + fn of(units: &StateUnits) -> Self { + let inner = units.inner.lock(); + Self { + next_id: inner.next_id, + names: inner + .names + .iter() + .map(|(name, &id)| (name.to_string(), id)) + .collect(), + edges: inner + .units + .iter() + .map(|(&id, unit)| (id, (unit.dependencies.clone(), unit.dependents.clone()))) + .collect(), + } + } +} + +fn add(units: &StateUnits, name: &str) -> UnitHandle { + units.add(name).build(mesh::channel().0).unwrap() +} + +fn id(handle: &UnitHandle) -> u64 { + handle.id.id +} + +/// Removing a unit that a builder refers to makes the build fail, and the +/// failed build leaves no name, edge, or ID behind for a later unit. +#[test] +fn a_removed_unit_fails_the_build_without_changing_the_set() { + for removed_is_dependent in [false, true] { + let units = StateUnits::new(); + let a = add(&units, "a"); + let b = add(&units, "b"); + let builder = units.add("x").depends_on(&a); + let builder = if removed_is_dependent { + builder.dependency_of(&b) + } else { + builder.depends_on(&b) + }; + b.remove(); + let before = Inventory::of(&units); + + let err = builder.build(mesh::channel().0).unwrap_err(); + assert!( + matches!( + &err, + UnitBuildError::RemovedUnit { unit, removed } if &**unit == "x" && &**removed == "b" + ), + "{err}" + ); + assert_eq!(Inventory::of(&units), before); + + // The next unit takes the ID that x would have had, but none of its + // edges, and the name x is free. + let y = add(&units, "y"); + let x = units + .add("x") + .depends_on(&a) + .build(mesh::channel().0) + .unwrap(); + let inner = units.inner.lock(); + assert_eq!(inner.names["y"], id(&y)); + assert_eq!(inner.names["x"], id(&x)); + assert!(inner.units[&id(&y)].dependencies.is_empty()); + assert_eq!(inner.units[&id(&a)].dependents, BTreeSet::from([id(&x)])); + assert_no_dangling_edges(&inner); + } +} + +/// Restore matches saved state to units by name, so a failed build must not +/// leave its name pointing at the ID that a later unit receives. +#[async_test] +async fn a_failed_build_does_not_route_saved_state_to_a_later_unit(driver: DefaultDriver) { + let mut units = StateUnits::new(); + let a = units + .add("a") + .spawn(&driver, |recv| run_unit(TestUnit::default(), recv)) + .unwrap(); + let b = units + .add("b") + .spawn(&driver, |recv| run_unit(TestUnit::default(), recv)) + .unwrap(); + let builder = units.add("x").depends_on(a.handle()).depends_on(b.handle()); + b.remove().await; + builder.build(mesh::channel().0).unwrap_err(); + + let restored = Arc::new(AtomicBool::new(false)); + let _y = units + .add("y") + .spawn(&driver, |recv| { + run_unit( + TestUnit { + value: restored.clone(), + dep: None, + support_saved_state: true, + }, + recv, + ) + }) + .unwrap(); + let err = units + .restore(vec![SavedStateUnit { + name: "x".to_owned(), + state: SavedStateBlob::new(SavedState(true)), + }]) + .await + .unwrap_err(); + let StateTransitionErrorKind::Units(unit_errors) = err.error else { + panic!("expected named unit failures"); + }; + assert_eq!(unit_errors.0.len(), 1); + assert_eq!(unit_errors.0[0].0.as_ref(), "x"); + assert_eq!(unit_errors.0[0].1.to_string(), "unknown unit name"); + assert!(!restored.load(Ordering::Relaxed)); +} + +/// The set's lock orders a build and a concurrent removal of a unit that the +/// build refers to. If the build comes first, the removal prunes the new +/// unit's edge; otherwise the build fails without changing the set. +#[test] +fn a_removal_racing_a_build_leaves_the_set_consistent() { + for _ in 0..25 { + let units = StateUnits::new(); + let a = add(&units, "a"); + let b = add(&units, "b"); + let builder = units.add("x").depends_on(&a).dependency_of(&b); + let barrier = &Barrier::new(2); + let result = std::thread::scope(|scope| { + scope.spawn(move || { + barrier.wait(); + a.remove(); + }); + barrier.wait(); + builder.build(mesh::channel().0) + }); + + let inner = units.inner.lock(); + match &result { + Ok(x) => { + assert!(inner.units[&id(x)].dependencies.is_empty()); + assert_eq!(inner.units[&id(&b)].dependencies, BTreeSet::from([id(x)])); + } + Err(UnitBuildError::RemovedUnit { removed, .. }) if &**removed == "a" => { + assert!(!inner.names.contains_key("x")); + assert!(inner.units[&id(&b)].dependencies.is_empty()); + assert_eq!(inner.next_id, 2); + } + Err(err) => panic!("{err}"), + } + assert_eq!(inner.names.len(), inner.units.len()); + assert_no_dangling_edges(&inner); + } +} + +/// A build fails, without changing the set, if the new unit would depend, +/// directly or through other units, on a unit that it would also be a +/// dependency of, whether `depends_on` or `dependency_of` made the edges. +#[test] +fn dependency_cycles_fail_the_build_without_changing_the_set() { + let units = StateUnits::new(); + // The dependency edges, each from a unit to one that it depends on, are + // c -> b -> a -> e, where `e.dependency_of(a)` made the last one. + let a = add(&units, "a"); + let b = units + .add("b") + .depends_on(&a) + .build(mesh::channel().0) + .unwrap(); + let c = units + .add("c") + .depends_on(&b) + .build(mesh::channel().0) + .unwrap(); + let e = units + .add("e") + .dependency_of(&a) + .build(mesh::channel().0) + .unwrap(); + let before = Inventory::of(&units); + + let cycles = [ + // x -> a -> x + (units.add("x").depends_on(&a).dependency_of(&a), "a"), + // x -> c -> b -> a -> x + (units.add("x").depends_on(&c).dependency_of(&a), "a"), + // x -> a -> e -> x, through the edge that `dependency_of` made + (units.add("x").depends_on(&a).dependency_of(&e), "e"), + // x -> c -> b -> x, with each edge requested twice + ( + units + .add("x") + .depends_on(&c) + .depends_on(&c) + .dependency_of(&b) + .dependency_of(&b), + "b", + ), + ]; + for (builder, expected) in cycles { + let err = builder.build(mesh::channel().0).unwrap_err(); + let UnitBuildError::DependencyCycle { unit, dependent } = &err else { + panic!("{err}"); + }; + assert_eq!((&**unit, &**dependent), ("x", expected)); + assert_eq!(Inventory::of(&units), before); + } + + // Paths beside existing ones form no cycle: b -> x -> a beside b -> a, and + // c -> y -> b and c -> y -> e beside c -> b -> a -> e, where y reaches a + // and e along two paths each. + let x = units + .add("x") + .depends_on(&a) + .dependency_of(&b) + .build(mesh::channel().0) + .unwrap(); + let y = units + .add("y") + .depends_on(&b) + .depends_on(&e) + .dependency_of(&c) + .build(mesh::channel().0) + .unwrap(); + let inner = units.inner.lock(); + assert_eq!( + inner.units[&id(&b)].dependencies, + BTreeSet::from([id(&a), id(&x)]) + ); + assert_eq!( + inner.units[&id(&c)].dependencies, + BTreeSet::from([id(&b), id(&y)]) + ); + assert_eq!( + inner.units[&id(&y)].dependencies, + BTreeSet::from([id(&b), id(&e)]) + ); + assert_no_dangling_edges(&inner); +} + +/// Ordered state changes wait for a unit's dependencies, or for its dependents +/// when stopping, so the units of a cycle would wait for one another forever. +/// Adding a unit that would close a cycle fails, even while the units run, and +/// the set still stops and starts. +#[async_test] +async fn rejected_cycles_leave_ordered_state_changes_able_to_complete(driver: DefaultDriver) { + let mut units = StateUnits::new(); + let a = units + .add("a") + .spawn(&driver, |recv| run_unit(TestUnit::default(), recv)) + .unwrap(); + let b = units + .add("b") + .depends_on(a.handle()) + .spawn(&driver, |recv| run_unit(TestUnit::default(), recv)) + .unwrap(); + units.start().await; + + for builder in [ + units + .add("x") + .depends_on(a.handle()) + .dependency_of(a.handle()), + units + .add("x") + .depends_on(b.handle()) + .dependency_of(a.handle()), + ] { + let err = builder + .spawn(&driver, |recv| run_unit(TestUnit::default(), recv)) + .unwrap_err(); + assert!( + matches!(err, UnitBuildError::DependencyCycle { .. }), + "{err}" + ); + } + + units.start_stopped_units().await; + units.stop().await; + units.reset().await.unwrap(); + units.start().await; + units.stop().await; +} diff --git a/vmm_core/vmotherboard/src/chipset/backing/arc_mutex/services.rs b/vmm_core/vmotherboard/src/chipset/backing/arc_mutex/services.rs index b0e8498acae..771c3bcd67b 100644 --- a/vmm_core/vmotherboard/src/chipset/backing/arc_mutex/services.rs +++ b/vmm_core/vmotherboard/src/chipset/backing/arc_mutex/services.rs @@ -54,8 +54,8 @@ pub enum FinalizeError { EoiHandlerAlreadySet, #[error("pic handler already set")] PicHandlerAlreadySet, - #[error("name already in use")] - NameInUse(#[source] state_unit::NameInUse), + #[error("failed to add the device's state unit")] + AddUnit(#[source] state_unit::UnitBuildError), } impl ArcMutexChipsetServicesFinalize for ArcMutexChipsetServices<'_, '_> { @@ -144,7 +144,7 @@ impl ArcMutexChipsetServicesFinalize for ArcMutexChipset let driver = self.builder.driver_source.simple(); let unit = builder .spawn(driver.clone(), |recv| device_unit.run(recv)) - .map_err(FinalizeError::NameInUse)?; + .map_err(FinalizeError::AddUnit)?; inner .arc_mutex_device_units