diff --git a/vm/vmcore/src/vmtime.rs b/vm/vmcore/src/vmtime.rs index 26514141fe4..202f8575aac 100644 --- a/vm/vmcore/src/vmtime.rs +++ b/vm/vmcore/src/vmtime.rs @@ -31,6 +31,7 @@ use inspect::Inspect; use inspect::InspectMut; use mesh::MeshPayload; use mesh::payload::Protobuf; +use mesh::rpc::PendingRpc; use mesh::rpc::Rpc; use mesh::rpc::RpcSend; use pal_async::driver::Driver; @@ -335,6 +336,15 @@ impl TimerState { } /// A time keeper, which tracks the current time and all waiters. +/// +/// Dropping the future of one of its methods leaves the keeper consistent. +/// Once a method has asked the time keeping tasks to start, stop, or reset VM +/// time, that change happens even if the method's future is dropped before it +/// completes. A stop whose future is dropped stays pending until the next call +/// to [`stop`](Self::stop), [`start`](Self::start), [`reset`](Self::reset), or +/// [`restore`](Self::restore), which finishes it first; `stop` then returns +/// without stopping VM time again. [`save`](Self::save) cannot wait for a +/// pending stop, so await `stop` again before saving. #[derive(Debug, InspectMut)] pub struct VmTimeKeeper { #[inspect(skip)] @@ -345,6 +355,11 @@ pub struct VmTimeKeeper { builder: VmTimeSourceBuilder, #[inspect(skip)] time: TimeState, + /// The result of a stop that the primary keeper has not yet returned, + /// kept here rather than in the stop's future so that dropping that + /// future does not lose it. `time` stays started until it arrives. + #[inspect(skip)] + stopping: Option>, } // UNSAFETY: Needed to derive SavedStateRoot in the same crate it is declared @@ -461,6 +476,7 @@ impl VmTimeKeeper { }); Self { time, + stopping: None, req_send, builder: VmTimeSourceBuilder { new_send }, _task: task, @@ -468,7 +484,16 @@ impl VmTimeKeeper { } /// Saves the time state. + /// + /// VM time must be stopped. This method cannot wait for a pending stop, + /// whose future was dropped before it completed, so it panics instead of + /// saving a time that the stop has not reconciled. Await + /// [`stop`](Self::stop) again to finish the stop first. pub fn save(&self) -> SavedState { + assert!( + self.stopping.is_none(), + "VM time is still stopping; await `stop` before saving" + ); SavedState { vmtime: self.time.stop_time().expect("should be stopped"), } @@ -481,6 +506,7 @@ impl VmTimeKeeper { } async fn reset_to(&mut self, vmtime: VmTime) { + self.finish_stop().await; assert!(!self.time.is_started(), "should be stopped"); self.time = TimeState::Stopped(vmtime); self.req_send @@ -496,6 +522,7 @@ impl VmTimeKeeper { /// Starts the timer, so that the current time will increase. pub async fn start(&mut self) { + self.finish_stop().await; let vmtime = self.time.stop_time().expect("should be stopped"); let timestamp = Timestamp::new(vmtime, Instant::now()); self.time = TimeState::Started(timestamp); @@ -506,10 +533,28 @@ impl VmTimeKeeper { } /// Stops the timer, so that the current time will stop increasing. + /// + /// If a stop whose future was dropped is pending, finishes that stop + /// instead of stopping again. pub async fn stop(&mut self) { - assert!(self.time.is_started(), "should be running"); - let stop_time = self.req_send.call(KeeperRequest::Stop, ()).await.unwrap(); - self.time = TimeState::Stopped(stop_time); + if self.stopping.is_none() { + assert!(self.time.is_started(), "should be running"); + self.stopping = Some(self.req_send.call(KeeperRequest::Stop, ())); + } + self.finish_stop().await; + } + + /// Finishes a pending stop, if any, by adopting the VM time at which the + /// primary keeper stopped VM time. + /// + /// The stop's result stays in `self.stopping` until it arrives, so that + /// dropping this future leaves the stop pending instead of losing it. + async fn finish_stop(&mut self) { + if let Some(stopping) = &mut self.stopping { + let stop_time = stopping.await.unwrap(); + self.stopping = None; + self.time = TimeState::Stopped(stop_time); + } } /// Returns a time source builder, which can be used to spawn tasks that @@ -962,6 +1007,9 @@ impl VmTimerPeriodic { } } +#[cfg(test)] +mod cancel_tests; + #[cfg(test)] mod tests { use super::VmTime; diff --git a/vm/vmcore/src/vmtime/cancel_tests.rs b/vm/vmcore/src/vmtime/cancel_tests.rs new file mode 100644 index 00000000000..6ffaf9f2428 --- /dev/null +++ b/vm/vmcore/src/vmtime/cancel_tests.rs @@ -0,0 +1,267 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +//! Tests that drop the futures of [`VmTimeKeeper`]'s methods at their await +//! points, then check that later calls see a consistent keeper. +//! +//! The primary keeper waits for every secondary keeper to answer its +//! requests, so a manual keeper, which the test answers by hand, holds an +//! operation in flight while the test drops the future that started it. + +use super::KeeperRequest; +use super::NewKeeperRequest; +use super::SavedState; +use super::Timestamp; +use super::VmTime; +use super::VmTimeKeeper; +use futures::FutureExt; +use futures::StreamExt; +use futures::future::Either; +use futures::future::select; +use mesh::rpc::Rpc; +use mesh::rpc::RpcSend; +use pal_async::DefaultDriver; +use pal_async::async_test; +use std::pin::pin; +use test_with_tracing::test; + +/// A stop time that the manual keeper reports: an hour after the start, which +/// the other keepers have not reached, so it becomes the common stop time. +const AHEAD: VmTime = VmTime(36_000_000_000); + +/// A restored VM time. +const RESTORED: VmTime = VmTime(1_234_567); + +/// Creates a keeper, stopped at 0, with a manual keeper. +async fn new_keeper(driver: &DefaultDriver) -> (VmTimeKeeper, ManualKeeper) { + let keeper = VmTimeKeeper::new(driver, VmTime::from_100ns(0)); + let manual = ManualKeeper::new(&keeper).await; + (keeper, manual) +} + +/// Polls `op` until `until` completes, then drops `op` and returns `until`'s +/// output. +async fn drop_when(op: impl Future, until: impl Future) -> T { + match select(pin!(op), pin!(until)).await { + Either::Left(_) => panic!("the operation finished before it could be dropped"), + Either::Right((output, _)) => output, + } +} + +/// A secondary keeper that the test answers by hand. +struct ManualKeeper(mesh::Receiver); + +impl ManualKeeper { + /// Registers a manual keeper with `keeper`'s primary keeper. + async fn new(keeper: &VmTimeKeeper) -> Self { + let (send, recv) = mesh::channel(); + keeper + .builder() + .new_send + .call(NewKeeperRequest::New, send) + .await + .unwrap(); + Self(recv) + } + + /// Receives the primary keeper's next request. + async fn next(&mut self) -> KeeperRequest { + self.0.next().await.expect("the primary keeper is gone") + } + + /// Receives a start request, without answering it. + async fn hold_start(&mut self) -> Rpc { + match self.next().await { + KeeperRequest::Start(rpc) => rpc, + req => panic!("expected a start request, got a {} request", name(&req)), + } + } + + /// Receives a stop request, without answering it. + async fn hold_stop(&mut self) -> Rpc<(), VmTime> { + match self.next().await { + KeeperRequest::Stop(rpc) => rpc, + req => panic!("expected a stop request, got a {} request", name(&req)), + } + } + + /// Receives a reset request, without answering it. + async fn hold_reset(&mut self) -> Rpc { + match self.next().await { + KeeperRequest::Reset(rpc) => rpc, + req => panic!("expected a reset request, got a {} request", name(&req)), + } + } + + /// Answers a start request, returning the VM time that it starts from. + async fn answer_start(&mut self) -> VmTime { + let rpc = self.hold_start().await; + let vmtime = rpc.input().vmtime; + rpc.complete(()); + vmtime + } + + /// Answers a stop request as if this keeper stopped at `vmtime`. + async fn answer_stop(&mut self, vmtime: VmTime) { + self.hold_stop().await.complete(vmtime); + } + + /// Answers a reset request, returning the VM time that it resets to. + async fn answer_reset(&mut self) -> VmTime { + let rpc = self.hold_reset().await; + let vmtime = *rpc.input(); + rpc.complete(()); + vmtime + } + + /// Runs `op`, which must finish without sending this keeper a request. + async fn quietly(&mut self, op: impl Future) -> T { + match select(pin!(op), pin!(self.next())).await { + Either::Left((output, _)) => output, + Either::Right((req, _)) => panic!("unexpected {} request", name(&req)), + } + } +} + +/// Names the kind of `req`, for failure messages. +fn name(req: &KeeperRequest) -> &'static str { + match req { + KeeperRequest::Start(_) => "start", + KeeperRequest::Stop(_) => "stop", + KeeperRequest::Reset(_) => "reset", + KeeperRequest::Inspect(_) => "inspect", + } +} + +/// A stop whose future is dropped while the keepers stop still stops VM +/// time, and stopping again finishes it without stopping the keepers again. +#[async_test] +async fn dropped_stop_is_finished_by_stop(driver: DefaultDriver) { + let (mut keeper, mut manual) = new_keeper(&driver).await; + futures::join!(keeper.start(), manual.answer_start()); + + let stop = drop_when(keeper.stop(), manual.hold_stop()).await; + assert!(keeper.stop().now_or_never().is_none()); + stop.complete(AHEAD); + assert_eq!(manual.answer_reset().await, AHEAD); + + manual.quietly(keeper.stop()).await; + assert_eq!(keeper.save().vmtime, AHEAD); + + // The keepers see a start next, which starts from the stop time. + let ((), start) = futures::join!(keeper.start(), manual.answer_start()); + assert_eq!(start, AHEAD); +} + +/// A start finishes a stop whose future was dropped after the primary keeper +/// stopped, even when the start's own future is dropped while it waits for +/// the stop, and then starts VM time from the stop time. +#[async_test] +async fn dropped_stop_is_finished_by_start(driver: DefaultDriver) { + let (mut keeper, mut manual) = new_keeper(&driver).await; + let access = keeper + .builder() + .build(&driver) + .await + .unwrap() + .access("test"); + futures::join!(keeper.start(), manual.answer_start()); + + let reset = drop_when(keeper.stop(), async { + manual.answer_stop(AHEAD).await; + manual.hold_reset().await + }) + .await; + assert_eq!(*reset.input(), AHEAD); + assert!(keeper.start().now_or_never().is_none()); + reset.complete(()); + + let ((), start) = futures::join!(keeper.start(), manual.answer_start()); + assert_eq!(start, AHEAD); + assert!(!access.now().is_before(AHEAD)); + + let ((), stop_time) = futures::join!(keeper.stop(), async { + manual.answer_stop(AHEAD).await; + manual.answer_reset().await + }); + assert_eq!(keeper.save().vmtime, stop_time); + assert_eq!(access.now(), stop_time); +} + +/// A reset or restore finishes a stop whose future was dropped, even when +/// the reset's own future is dropped while it waits for the stop, and then +/// resets VM time. +#[async_test] +async fn dropped_stop_is_finished_by_reset_and_restore(driver: DefaultDriver) { + let (mut keeper, mut manual) = new_keeper(&driver).await; + let access = keeper + .builder() + .build(&driver) + .await + .unwrap() + .access("test"); + futures::join!(keeper.start(), manual.answer_start()); + + let stop = drop_when(keeper.stop(), manual.hold_stop()).await; + assert!(keeper.reset().now_or_never().is_none()); + stop.complete(AHEAD); + assert_eq!(manual.answer_reset().await, AHEAD); + + let ((), reset) = futures::join!( + keeper.restore(SavedState::from_vmtime(RESTORED)), + manual.answer_reset() + ); + assert_eq!(reset, RESTORED); + assert_eq!(keeper.save().vmtime, RESTORED); + assert_eq!(access.now(), RESTORED); +} + +/// Saving cannot wait for a stop whose future was dropped, so it panics +/// rather than save a time that the stop has not reconciled. +#[async_test] +#[should_panic(expected = "VM time is still stopping")] +async fn save_panics_while_a_dropped_stop_is_pending(driver: DefaultDriver) { + let (mut keeper, mut manual) = new_keeper(&driver).await; + futures::join!(keeper.start(), manual.answer_start()); + + let stop = drop_when(keeper.stop(), manual.hold_stop()).await; + stop.complete(AHEAD); + manual.answer_reset().await; + keeper.save(); +} + +/// A start whose future is dropped while the keepers start still starts VM +/// time, so that a stop queued behind it stops VM time. +#[async_test] +async fn dropped_start_still_starts(driver: DefaultDriver) { + let (mut keeper, mut manual) = new_keeper(&driver).await; + + let start = drop_when(keeper.start(), manual.hold_start()).await; + futures::join!(keeper.stop(), async { + start.complete(()); + manual.answer_stop(AHEAD).await; + assert_eq!(manual.answer_reset().await, AHEAD); + }); + assert_eq!(keeper.save().vmtime, AHEAD); +} + +/// A restore whose future is dropped while the keepers reset still restores +/// VM time, so that a start queued behind it starts from the restored time. +#[async_test] +async fn dropped_restore_still_restores(driver: DefaultDriver) { + let (mut keeper, mut manual) = new_keeper(&driver).await; + + let reset = drop_when( + keeper.restore(SavedState::from_vmtime(RESTORED)), + manual.hold_reset(), + ) + .await; + assert_eq!(*reset.input(), RESTORED); + assert_eq!(keeper.save().vmtime, RESTORED); + + let ((), start) = futures::join!(keeper.start(), async { + reset.complete(()); + manual.answer_start().await + }); + assert_eq!(start, RESTORED); +}