Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions vm/devices/virtio/virtio/src/tests.rs
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@
mod dma;
mod interrupt;
mod saved_state;
mod start;

use crate::DeviceTraits;
use crate::DynVirtioDevice;
Expand Down
200 changes: 200 additions & 0 deletions vm/devices/virtio/virtio/src/tests/start.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1,200 @@
// Copyright (c) Microsoft Corporation.
// Licensed under the MIT License.

//! Tests for restarting the queues of an active device when the VM starts:
//! a queue that fails to start, and the progress of the queues that started
//! before it.

use super::*;
use test_with_tracing::test;

/// The state that a [`ProgressDevice`] queue reports when it stops, having
/// completed one request since it started at `state`.
fn advanced(state: QueueState) -> QueueState {
QueueState {
avail_index: state.avail_index.wrapping_add(1),
used_index: state.used_index.wrapping_add(1),
}
}

/// A two-queue device whose queues each complete one request while they run,
/// and whose queue start fails for the queue that the probe names.
#[derive(InspectMut)]
#[inspect(skip)]
struct ProgressDevice {
probe: ProgressProbe,
running: [Option<QueueState>; 2],
}

#[derive(Clone, Default)]
struct ProgressProbe {
/// Per queue, the initial state of its last start.
last_start: Arc<Mutex<[Option<QueueState>; 2]>>,
/// The queue whose start fails.
fail_at: Arc<Mutex<Option<u16>>>,
/// The queue whose stop reports no progress.
forget_at: Arc<Mutex<Option<u16>>>,
}

impl ProgressProbe {
fn last_start(&self, idx: usize) -> Option<QueueState> {
self.last_start.lock()[idx]
}

fn fail_at(&self, idx: Option<u16>) {
*self.fail_at.lock() = idx;
}

fn forget_at(&self, idx: Option<u16>) {
*self.forget_at.lock() = idx;
}
}

impl ProgressDevice {
fn new() -> (Self, ProgressProbe) {
let probe = ProgressProbe::default();
(
Self {
probe: probe.clone(),
running: [None; 2],
},
probe,
)
}
}

impl VirtioDevice for ProgressDevice {
fn traits(&self) -> DeviceTraits {
DeviceTraits {
device_id: VirtioDeviceType::CONSOLE,
device_features: VirtioDeviceFeatures::new(),
max_queues: 2,
device_register_length: 0,
..Default::default()
}
}

async fn read_registers_u32(&mut self, _offset: u16) -> u32 {
0
}

async fn write_registers_u32(&mut self, _offset: u16, _val: u32) {}

async fn start_queue(
&mut self,
idx: u16,
_resources: QueueResources,
_features: &VirtioDeviceFeatures,
initial_state: Option<QueueState>,
) -> anyhow::Result<()> {
anyhow::ensure!(
*self.probe.fail_at.lock() != Some(idx),
"intentional queue start failure"
);
self.probe.last_start.lock()[usize::from(idx)] = initial_state;
self.running[usize::from(idx)] = Some(initial_state.unwrap_or_default());
Ok(())
}

async fn stop_queue(&mut self, idx: u16) -> Option<QueueState> {
let state = self.running[usize::from(idx)].take().map(advanced);
// A device may report no progress, as virtio-net does for the queue
// pair that a failed start drops.
if *self.probe.forget_at.lock() == Some(idx) {
None
} else {
state
}
}
}

/// Starts the device's queues, then resumes it twice, first with the second
/// queue failing to start, as `verify_failed_start_keeps_queue_progress`
/// does, but with the first queue reporting no progress when the failed start
/// stops it: the next start resumes that queue from the progress that the
/// transport saved before the failed start.
async fn verify_failed_start_keeps_saved_progress(
transport: &mut impl TestTransport,
probe: &ProgressProbe,
) {
transport.write_driver_ok();
yield_and_poll(transport).await;
transport.stop().await;

probe.fail_at(Some(1));
probe.forget_at(Some(0));
transport.start();
transport.stop().await;

probe.fail_at(None);
probe.forget_at(None);
transport.start();
transport.stop().await;
assert_eq!(
probe.last_start(0),
Some(QueueState {
avail_index: 1,
used_index: 1
})
);
}

#[async_test]
async fn failed_start_keeps_saved_progress_mmio(driver: DefaultDriver) {
let (device, probe) = ProgressDevice::new();
let mut transport = MmioTestTransport::new(Box::new(device), &driver, 2);
verify_failed_start_keeps_saved_progress(&mut transport, &probe).await;
}

#[async_test]
async fn failed_start_keeps_saved_progress_pci(driver: DefaultDriver) {
let (device, probe) = ProgressDevice::new();
let mut transport = PciTestTransport::new(Box::new(device), &driver, 2);
verify_failed_start_keeps_saved_progress(&mut transport, &probe).await;
}

/// Starts the device's queues, then resumes it twice: first with the second
/// queue failing to start after the first queue has completed a request,
/// then successfully.
async fn verify_failed_start_keeps_queue_progress(
transport: &mut impl TestTransport,
probe: &ProgressProbe,
) {
let progress = |n| QueueState {
avail_index: n,
used_index: n,
};

transport.write_driver_ok();
yield_and_poll(transport).await;
assert_ne!(transport.read_status() & VIRTIO_DRIVER_OK, 0);
transport.stop().await;

probe.fail_at(Some(1));
transport.start();
transport.stop().await;
assert_eq!(probe.last_start(0), Some(progress(1)));

// The next start resumes the first queue after the request that it
// completed before the failed start stopped it, rather than repeating
// that request.
probe.fail_at(None);
transport.start();
transport.stop().await;
assert_eq!(probe.last_start(0), Some(progress(2)));
assert_eq!(probe.last_start(1), Some(progress(1)));
}

#[async_test]
async fn failed_start_keeps_queue_progress_mmio(driver: DefaultDriver) {
let (device, probe) = ProgressDevice::new();
let mut transport = MmioTestTransport::new(Box::new(device), &driver, 2);
verify_failed_start_keeps_queue_progress(&mut transport, &probe).await;
}

#[async_test]
async fn failed_start_keeps_queue_progress_pci(driver: DefaultDriver) {
let (device, probe) = ProgressDevice::new();
let mut transport = PciTestTransport::new(Box::new(device), &driver, 2);
verify_failed_start_keeps_queue_progress(&mut transport, &probe).await;
}
31 changes: 26 additions & 5 deletions vm/devices/virtio/virtio/src/transport/task.rs
Original file line number Diff line number Diff line change
Expand Up @@ -200,8 +200,12 @@ struct DeviceTask {
max_queues: u16,
/// The device type reported by restore lifecycle traces.
device_type: u16,
/// Per queue, whether it is started.
/// Per queue, whether it started since the last stop, and so has progress
/// for the next stop to report.
started: Vec<bool>,
/// Per queue, the progress of a started queue that a failed start stopped
/// again, which the next stop reports instead of stopping the queue.
stopped_progress: Vec<Option<QueueState>>,
}

impl DeviceTask {
Expand Down Expand Up @@ -262,14 +266,18 @@ impl DeviceTask {
async fn stop(&mut self) -> restore::StopResult {
let mut states = vec![None; self.max_queues as usize];
for idx in 0..self.max_queues {
states[idx as usize] = self.device.stop_queue(idx).await;
states[idx as usize] = match self.stopped_progress[idx as usize].take() {
Some(progress) => Some(progress),
None => self.device.stop_queue(idx).await,
};
}
self.stop_result(states)
}

/// Restarts the queues of a running device. If one fails to start, stops
/// the queues that did, so that the device processes nothing until the
/// guest resets it.
/// the queues that did, so that the device processes nothing, and keeps
/// their progress for the next stop, so that a later start resumes them
/// where they stopped rather than repeating their work.
async fn start(&mut self, params: StartParams) -> anyhow::Result<()> {
for (idx, resources, initial_state) in params.queues {
if let Err(err) = self
Expand All @@ -287,7 +295,18 @@ impl DeviceTask {
idx,
"virtio device start_queue failed on resume"
);
self.stop_all_queues().await;
for queue in 0..self.max_queues {
if self.started[queue as usize] {
match self.device.stop_queue(queue).await {
Some(progress) => {
self.stopped_progress[queue as usize] = Some(progress);
}
// A queue without progress to report keeps the
// progress that the transport saved.
None => self.started[queue as usize] = false,
}
}
}
return Err(err);
}
}
Expand All @@ -303,6 +322,7 @@ impl DeviceTask {
for idx in 0..self.max_queues {
self.device.stop_queue(idx).await;
self.started[idx as usize] = false;
self.stopped_progress[idx as usize] = None;
}
}
}
Expand All @@ -317,6 +337,7 @@ pub async fn run_device_task(
max_queues: traits.max_queues,
device_type: traits.device_id.0,
started: vec![false; traits.max_queues as usize],
stopped_progress: vec![None; traits.max_queues as usize],
device,
};

Expand Down
Loading