From d8e8f271c4081ad9aecd83549c6a2293c3669429 Mon Sep 17 00:00:00 2001 From: Pedro Henrique Penna Date: Fri, 6 Nov 2026 13:26:49 -0800 Subject: [PATCH] virtio: keep the progress of queues that a failed start stops When the VM starts an active virtio device, its device task starts each queue from the progress that the last stop saved. If a queue fails to start, the task stops the queues that had started, so that the device processes nothing, but it discarded the progress that they made in the meantime: the next stop reported them as never started, the transport kept their earlier progress, and a later start resumed them from it, repeating the requests that they had completed, such as a transmit or a file system operation. Keep that progress for the next stop to report, as it reports the progress of a queue that it stops itself, so that the next start resumes each queue where it stopped. A queue that the failed start stopped still counts as started until that stop, and a reset or a guest reset still discards it. A queue that reports no progress when the failed start stops it, as virtio-net's does for the queue pair that its failed start drops, no longer counts as started, so it keeps the progress that the transport saved, as before. This prepares the VM's start to stop a device whose start fails, and to start it again later, for microsoft/nvx#1232. Tests: on both transports, a device whose queues each complete a request while they run starts, stops, fails to start its second queue, stops, and starts again; the first queue must resume after the request that it completed before the failed start stopped it. Without the change, the tests fail: the queue resumes from its earlier progress. When the first queue reports no progress as the failed start stops it, it must resume from the progress that the transport saved; counting it as started would lose that progress, which the second pair of tests checks. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- vm/devices/virtio/virtio/src/tests.rs | 1 + vm/devices/virtio/virtio/src/tests/start.rs | 200 ++++++++++++++++++ .../virtio/virtio/src/transport/task.rs | 31 ++- 3 files changed, 227 insertions(+), 5 deletions(-) create mode 100644 vm/devices/virtio/virtio/src/tests/start.rs diff --git a/vm/devices/virtio/virtio/src/tests.rs b/vm/devices/virtio/virtio/src/tests.rs index 1c2f0bbe844..0ca85c3b71d 100644 --- a/vm/devices/virtio/virtio/src/tests.rs +++ b/vm/devices/virtio/virtio/src/tests.rs @@ -9,6 +9,7 @@ mod dma; mod interrupt; mod saved_state; +mod start; use crate::DeviceTraits; use crate::DynVirtioDevice; diff --git a/vm/devices/virtio/virtio/src/tests/start.rs b/vm/devices/virtio/virtio/src/tests/start.rs new file mode 100644 index 00000000000..912a5d09c75 --- /dev/null +++ b/vm/devices/virtio/virtio/src/tests/start.rs @@ -0,0 +1,200 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +//! Tests for restarting the queues of an active device when the VM starts: +//! a queue that fails to start, and the progress of the queues that started +//! before it. + +use super::*; +use test_with_tracing::test; + +/// The state that a [`ProgressDevice`] queue reports when it stops, having +/// completed one request since it started at `state`. +fn advanced(state: QueueState) -> QueueState { + QueueState { + avail_index: state.avail_index.wrapping_add(1), + used_index: state.used_index.wrapping_add(1), + } +} + +/// A two-queue device whose queues each complete one request while they run, +/// and whose queue start fails for the queue that the probe names. +#[derive(InspectMut)] +#[inspect(skip)] +struct ProgressDevice { + probe: ProgressProbe, + running: [Option; 2], +} + +#[derive(Clone, Default)] +struct ProgressProbe { + /// Per queue, the initial state of its last start. + last_start: Arc; 2]>>, + /// The queue whose start fails. + fail_at: Arc>>, + /// The queue whose stop reports no progress. + forget_at: Arc>>, +} + +impl ProgressProbe { + fn last_start(&self, idx: usize) -> Option { + self.last_start.lock()[idx] + } + + fn fail_at(&self, idx: Option) { + *self.fail_at.lock() = idx; + } + + fn forget_at(&self, idx: Option) { + *self.forget_at.lock() = idx; + } +} + +impl ProgressDevice { + fn new() -> (Self, ProgressProbe) { + let probe = ProgressProbe::default(); + ( + Self { + probe: probe.clone(), + running: [None; 2], + }, + probe, + ) + } +} + +impl VirtioDevice for ProgressDevice { + fn traits(&self) -> DeviceTraits { + DeviceTraits { + device_id: VirtioDeviceType::CONSOLE, + device_features: VirtioDeviceFeatures::new(), + max_queues: 2, + device_register_length: 0, + ..Default::default() + } + } + + async fn read_registers_u32(&mut self, _offset: u16) -> u32 { + 0 + } + + async fn write_registers_u32(&mut self, _offset: u16, _val: u32) {} + + async fn start_queue( + &mut self, + idx: u16, + _resources: QueueResources, + _features: &VirtioDeviceFeatures, + initial_state: Option, + ) -> anyhow::Result<()> { + anyhow::ensure!( + *self.probe.fail_at.lock() != Some(idx), + "intentional queue start failure" + ); + self.probe.last_start.lock()[usize::from(idx)] = initial_state; + self.running[usize::from(idx)] = Some(initial_state.unwrap_or_default()); + Ok(()) + } + + async fn stop_queue(&mut self, idx: u16) -> Option { + let state = self.running[usize::from(idx)].take().map(advanced); + // A device may report no progress, as virtio-net does for the queue + // pair that a failed start drops. + if *self.probe.forget_at.lock() == Some(idx) { + None + } else { + state + } + } +} + +/// Starts the device's queues, then resumes it twice, first with the second +/// queue failing to start, as `verify_failed_start_keeps_queue_progress` +/// does, but with the first queue reporting no progress when the failed start +/// stops it: the next start resumes that queue from the progress that the +/// transport saved before the failed start. +async fn verify_failed_start_keeps_saved_progress( + transport: &mut impl TestTransport, + probe: &ProgressProbe, +) { + transport.write_driver_ok(); + yield_and_poll(transport).await; + transport.stop().await; + + probe.fail_at(Some(1)); + probe.forget_at(Some(0)); + transport.start(); + transport.stop().await; + + probe.fail_at(None); + probe.forget_at(None); + transport.start(); + transport.stop().await; + assert_eq!( + probe.last_start(0), + Some(QueueState { + avail_index: 1, + used_index: 1 + }) + ); +} + +#[async_test] +async fn failed_start_keeps_saved_progress_mmio(driver: DefaultDriver) { + let (device, probe) = ProgressDevice::new(); + let mut transport = MmioTestTransport::new(Box::new(device), &driver, 2); + verify_failed_start_keeps_saved_progress(&mut transport, &probe).await; +} + +#[async_test] +async fn failed_start_keeps_saved_progress_pci(driver: DefaultDriver) { + let (device, probe) = ProgressDevice::new(); + let mut transport = PciTestTransport::new(Box::new(device), &driver, 2); + verify_failed_start_keeps_saved_progress(&mut transport, &probe).await; +} + +/// Starts the device's queues, then resumes it twice: first with the second +/// queue failing to start after the first queue has completed a request, +/// then successfully. +async fn verify_failed_start_keeps_queue_progress( + transport: &mut impl TestTransport, + probe: &ProgressProbe, +) { + let progress = |n| QueueState { + avail_index: n, + used_index: n, + }; + + transport.write_driver_ok(); + yield_and_poll(transport).await; + assert_ne!(transport.read_status() & VIRTIO_DRIVER_OK, 0); + transport.stop().await; + + probe.fail_at(Some(1)); + transport.start(); + transport.stop().await; + assert_eq!(probe.last_start(0), Some(progress(1))); + + // The next start resumes the first queue after the request that it + // completed before the failed start stopped it, rather than repeating + // that request. + probe.fail_at(None); + transport.start(); + transport.stop().await; + assert_eq!(probe.last_start(0), Some(progress(2))); + assert_eq!(probe.last_start(1), Some(progress(1))); +} + +#[async_test] +async fn failed_start_keeps_queue_progress_mmio(driver: DefaultDriver) { + let (device, probe) = ProgressDevice::new(); + let mut transport = MmioTestTransport::new(Box::new(device), &driver, 2); + verify_failed_start_keeps_queue_progress(&mut transport, &probe).await; +} + +#[async_test] +async fn failed_start_keeps_queue_progress_pci(driver: DefaultDriver) { + let (device, probe) = ProgressDevice::new(); + let mut transport = PciTestTransport::new(Box::new(device), &driver, 2); + verify_failed_start_keeps_queue_progress(&mut transport, &probe).await; +} diff --git a/vm/devices/virtio/virtio/src/transport/task.rs b/vm/devices/virtio/virtio/src/transport/task.rs index 4a2acf1dfcd..d54e7842caf 100644 --- a/vm/devices/virtio/virtio/src/transport/task.rs +++ b/vm/devices/virtio/virtio/src/transport/task.rs @@ -200,8 +200,12 @@ struct DeviceTask { max_queues: u16, /// The device type reported by restore lifecycle traces. device_type: u16, - /// Per queue, whether it is started. + /// Per queue, whether it started since the last stop, and so has progress + /// for the next stop to report. started: Vec, + /// Per queue, the progress of a started queue that a failed start stopped + /// again, which the next stop reports instead of stopping the queue. + stopped_progress: Vec>, } impl DeviceTask { @@ -262,14 +266,18 @@ impl DeviceTask { async fn stop(&mut self) -> restore::StopResult { let mut states = vec![None; self.max_queues as usize]; for idx in 0..self.max_queues { - states[idx as usize] = self.device.stop_queue(idx).await; + states[idx as usize] = match self.stopped_progress[idx as usize].take() { + Some(progress) => Some(progress), + None => self.device.stop_queue(idx).await, + }; } self.stop_result(states) } /// Restarts the queues of a running device. If one fails to start, stops - /// the queues that did, so that the device processes nothing until the - /// guest resets it. + /// the queues that did, so that the device processes nothing, and keeps + /// their progress for the next stop, so that a later start resumes them + /// where they stopped rather than repeating their work. async fn start(&mut self, params: StartParams) -> anyhow::Result<()> { for (idx, resources, initial_state) in params.queues { if let Err(err) = self @@ -287,7 +295,18 @@ impl DeviceTask { idx, "virtio device start_queue failed on resume" ); - self.stop_all_queues().await; + for queue in 0..self.max_queues { + if self.started[queue as usize] { + match self.device.stop_queue(queue).await { + Some(progress) => { + self.stopped_progress[queue as usize] = Some(progress); + } + // A queue without progress to report keeps the + // progress that the transport saved. + None => self.started[queue as usize] = false, + } + } + } return Err(err); } } @@ -303,6 +322,7 @@ impl DeviceTask { for idx in 0..self.max_queues { self.device.stop_queue(idx).await; self.started[idx as usize] = false; + self.stopped_progress[idx as usize] = None; } } } @@ -317,6 +337,7 @@ pub async fn run_device_task( max_queues: traits.max_queues, device_type: traits.device_id.0, started: vec![false; traits.max_queues as usize], + stopped_progress: vec![None; traits.max_queues as usize], device, };