diff --git a/vm/devices/virtio/virtio/src/tests.rs b/vm/devices/virtio/virtio/src/tests.rs index 1c2f0bbe844..0ca85c3b71d 100644 --- a/vm/devices/virtio/virtio/src/tests.rs +++ b/vm/devices/virtio/virtio/src/tests.rs @@ -9,6 +9,7 @@ mod dma; mod interrupt; mod saved_state; +mod start; use crate::DeviceTraits; use crate::DynVirtioDevice; diff --git a/vm/devices/virtio/virtio/src/tests/start.rs b/vm/devices/virtio/virtio/src/tests/start.rs new file mode 100644 index 00000000000..912a5d09c75 --- /dev/null +++ b/vm/devices/virtio/virtio/src/tests/start.rs @@ -0,0 +1,200 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +//! Tests for restarting the queues of an active device when the VM starts: +//! a queue that fails to start, and the progress of the queues that started +//! before it. + +use super::*; +use test_with_tracing::test; + +/// The state that a [`ProgressDevice`] queue reports when it stops, having +/// completed one request since it started at `state`. +fn advanced(state: QueueState) -> QueueState { + QueueState { + avail_index: state.avail_index.wrapping_add(1), + used_index: state.used_index.wrapping_add(1), + } +} + +/// A two-queue device whose queues each complete one request while they run, +/// and whose queue start fails for the queue that the probe names. +#[derive(InspectMut)] +#[inspect(skip)] +struct ProgressDevice { + probe: ProgressProbe, + running: [Option; 2], +} + +#[derive(Clone, Default)] +struct ProgressProbe { + /// Per queue, the initial state of its last start. + last_start: Arc; 2]>>, + /// The queue whose start fails. + fail_at: Arc>>, + /// The queue whose stop reports no progress. + forget_at: Arc>>, +} + +impl ProgressProbe { + fn last_start(&self, idx: usize) -> Option { + self.last_start.lock()[idx] + } + + fn fail_at(&self, idx: Option) { + *self.fail_at.lock() = idx; + } + + fn forget_at(&self, idx: Option) { + *self.forget_at.lock() = idx; + } +} + +impl ProgressDevice { + fn new() -> (Self, ProgressProbe) { + let probe = ProgressProbe::default(); + ( + Self { + probe: probe.clone(), + running: [None; 2], + }, + probe, + ) + } +} + +impl VirtioDevice for ProgressDevice { + fn traits(&self) -> DeviceTraits { + DeviceTraits { + device_id: VirtioDeviceType::CONSOLE, + device_features: VirtioDeviceFeatures::new(), + max_queues: 2, + device_register_length: 0, + ..Default::default() + } + } + + async fn read_registers_u32(&mut self, _offset: u16) -> u32 { + 0 + } + + async fn write_registers_u32(&mut self, _offset: u16, _val: u32) {} + + async fn start_queue( + &mut self, + idx: u16, + _resources: QueueResources, + _features: &VirtioDeviceFeatures, + initial_state: Option, + ) -> anyhow::Result<()> { + anyhow::ensure!( + *self.probe.fail_at.lock() != Some(idx), + "intentional queue start failure" + ); + self.probe.last_start.lock()[usize::from(idx)] = initial_state; + self.running[usize::from(idx)] = Some(initial_state.unwrap_or_default()); + Ok(()) + } + + async fn stop_queue(&mut self, idx: u16) -> Option { + let state = self.running[usize::from(idx)].take().map(advanced); + // A device may report no progress, as virtio-net does for the queue + // pair that a failed start drops. + if *self.probe.forget_at.lock() == Some(idx) { + None + } else { + state + } + } +} + +/// Starts the device's queues, then resumes it twice, first with the second +/// queue failing to start, as `verify_failed_start_keeps_queue_progress` +/// does, but with the first queue reporting no progress when the failed start +/// stops it: the next start resumes that queue from the progress that the +/// transport saved before the failed start. +async fn verify_failed_start_keeps_saved_progress( + transport: &mut impl TestTransport, + probe: &ProgressProbe, +) { + transport.write_driver_ok(); + yield_and_poll(transport).await; + transport.stop().await; + + probe.fail_at(Some(1)); + probe.forget_at(Some(0)); + transport.start(); + transport.stop().await; + + probe.fail_at(None); + probe.forget_at(None); + transport.start(); + transport.stop().await; + assert_eq!( + probe.last_start(0), + Some(QueueState { + avail_index: 1, + used_index: 1 + }) + ); +} + +#[async_test] +async fn failed_start_keeps_saved_progress_mmio(driver: DefaultDriver) { + let (device, probe) = ProgressDevice::new(); + let mut transport = MmioTestTransport::new(Box::new(device), &driver, 2); + verify_failed_start_keeps_saved_progress(&mut transport, &probe).await; +} + +#[async_test] +async fn failed_start_keeps_saved_progress_pci(driver: DefaultDriver) { + let (device, probe) = ProgressDevice::new(); + let mut transport = PciTestTransport::new(Box::new(device), &driver, 2); + verify_failed_start_keeps_saved_progress(&mut transport, &probe).await; +} + +/// Starts the device's queues, then resumes it twice: first with the second +/// queue failing to start after the first queue has completed a request, +/// then successfully. +async fn verify_failed_start_keeps_queue_progress( + transport: &mut impl TestTransport, + probe: &ProgressProbe, +) { + let progress = |n| QueueState { + avail_index: n, + used_index: n, + }; + + transport.write_driver_ok(); + yield_and_poll(transport).await; + assert_ne!(transport.read_status() & VIRTIO_DRIVER_OK, 0); + transport.stop().await; + + probe.fail_at(Some(1)); + transport.start(); + transport.stop().await; + assert_eq!(probe.last_start(0), Some(progress(1))); + + // The next start resumes the first queue after the request that it + // completed before the failed start stopped it, rather than repeating + // that request. + probe.fail_at(None); + transport.start(); + transport.stop().await; + assert_eq!(probe.last_start(0), Some(progress(2))); + assert_eq!(probe.last_start(1), Some(progress(1))); +} + +#[async_test] +async fn failed_start_keeps_queue_progress_mmio(driver: DefaultDriver) { + let (device, probe) = ProgressDevice::new(); + let mut transport = MmioTestTransport::new(Box::new(device), &driver, 2); + verify_failed_start_keeps_queue_progress(&mut transport, &probe).await; +} + +#[async_test] +async fn failed_start_keeps_queue_progress_pci(driver: DefaultDriver) { + let (device, probe) = ProgressDevice::new(); + let mut transport = PciTestTransport::new(Box::new(device), &driver, 2); + verify_failed_start_keeps_queue_progress(&mut transport, &probe).await; +} diff --git a/vm/devices/virtio/virtio/src/transport/task.rs b/vm/devices/virtio/virtio/src/transport/task.rs index 4a2acf1dfcd..d54e7842caf 100644 --- a/vm/devices/virtio/virtio/src/transport/task.rs +++ b/vm/devices/virtio/virtio/src/transport/task.rs @@ -200,8 +200,12 @@ struct DeviceTask { max_queues: u16, /// The device type reported by restore lifecycle traces. device_type: u16, - /// Per queue, whether it is started. + /// Per queue, whether it started since the last stop, and so has progress + /// for the next stop to report. started: Vec, + /// Per queue, the progress of a started queue that a failed start stopped + /// again, which the next stop reports instead of stopping the queue. + stopped_progress: Vec>, } impl DeviceTask { @@ -262,14 +266,18 @@ impl DeviceTask { async fn stop(&mut self) -> restore::StopResult { let mut states = vec![None; self.max_queues as usize]; for idx in 0..self.max_queues { - states[idx as usize] = self.device.stop_queue(idx).await; + states[idx as usize] = match self.stopped_progress[idx as usize].take() { + Some(progress) => Some(progress), + None => self.device.stop_queue(idx).await, + }; } self.stop_result(states) } /// Restarts the queues of a running device. If one fails to start, stops - /// the queues that did, so that the device processes nothing until the - /// guest resets it. + /// the queues that did, so that the device processes nothing, and keeps + /// their progress for the next stop, so that a later start resumes them + /// where they stopped rather than repeating their work. async fn start(&mut self, params: StartParams) -> anyhow::Result<()> { for (idx, resources, initial_state) in params.queues { if let Err(err) = self @@ -287,7 +295,18 @@ impl DeviceTask { idx, "virtio device start_queue failed on resume" ); - self.stop_all_queues().await; + for queue in 0..self.max_queues { + if self.started[queue as usize] { + match self.device.stop_queue(queue).await { + Some(progress) => { + self.stopped_progress[queue as usize] = Some(progress); + } + // A queue without progress to report keeps the + // progress that the transport saved. + None => self.started[queue as usize] = false, + } + } + } return Err(err); } } @@ -303,6 +322,7 @@ impl DeviceTask { for idx in 0..self.max_queues { self.device.stop_queue(idx).await; self.started[idx as usize] = false; + self.stopped_progress[idx as usize] = None; } } } @@ -317,6 +337,7 @@ pub async fn run_device_task( max_queues: traits.max_queues, device_type: traits.device_id.0, started: vec![false; traits.max_queues as usize], + stopped_progress: vec![None; traits.max_queues as usize], device, };