From 39f3c9f9118343ee19c0b15298534d301f70bc69 Mon Sep 17 00:00:00 2001 From: Pedro Henrique Penna Date: Tue, 17 Nov 2026 05:18:13 -0800 Subject: [PATCH 1/6] virtio_net: keep the saved queue lifecycle through a failed start A device whose start fails must keep the state that it had before the start, so that the stop that follows leaves it as it was (ChangeDeviceState::confirm_start). A virtio-net NIC built for save and restore did not: when the VM's start of a restored or paused NIC failed, the NIC saved a queue-pair lifecycle that disagreed with the queues whose progress the transport kept, and the next restore of that snapshot failed with "virtio-net queue lifecycle does not match transport state" (microsoft/nvx#1232). Two steps of the start changed the lifecycle before the pair started. prepare_queue_start reset the stopped queue count of the pair to zero when its first queue started. And when the pair's second queue could not be created, the first one stayed half open, so the rollback's stop recorded a half-open pair of one queue and returned that queue's progress, although the transport had saved two. Only the failure of the endpoint's queue creation, which drops the pair, avoided the second. Keep the lifecycle that the last stop or restore recorded until a stop records it again: prepare_queue_start only checks the pair's state, and a stop of a started pair recomputes the count, as it did. And drop the half-open first queue, which has not run, when the second cannot be created, as the endpoint failure does, so that the rollback's stop reports no progress and the transport keeps the progress that it saved for both queues. Tests: a restored NIC whose pair fails to start, once because its endpoint cannot create its queues and once because its TX queue cannot be created, reports no progress when the transport stops its queues, saves a lifecycle that its restore accepts with the restored progress, and then starts and transmits the guest's next packet only. Without the first change, the save fails the validation; without the second, the stop of the RX queue reports the progress of a half-open pair. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- Guide/src/user_guide/openvmm/snapshots.md | 2 +- vm/devices/virtio/virtio/src/device.rs | 3 + vm/devices/virtio/virtio_net/src/lib.rs | 34 ++-- .../virtio/virtio_net/src/saved_state.rs | 15 +- .../virtio/virtio_net/src/tests/activation.rs | 154 ++++++++++++++++-- .../virtio/virtio_net/src/tests/harness.rs | 11 +- 6 files changed, 187 insertions(+), 32 deletions(-) diff --git a/Guide/src/user_guide/openvmm/snapshots.md b/Guide/src/user_guide/openvmm/snapshots.md index 165b2870b8a..fd89c4bee0e 100644 --- a/Guide/src/user_guide/openvmm/snapshots.md +++ b/Guide/src/user_guide/openvmm/snapshots.md @@ -256,7 +256,7 @@ fails to start then, such as one whose network backend cannot create its queues, fails the resume: OpenVMM stops the devices that started again and logs the failure, no vCPU runs, and the restore publishes no readiness event. The devices keep their restored state, so a later resume starts their queues -again. +again, and a save of the paused VM saves that state. A restore gives a KVM VP the NMIs that it held pending at capture, which can be two: a second NMI that reached the VP before it ran, which KVM injects once the diff --git a/vm/devices/virtio/virtio/src/device.rs b/vm/devices/virtio/virtio/src/device.rs index af13408f7fd..b31f122c7cd 100644 --- a/vm/devices/virtio/virtio/src/device.rs +++ b/vm/devices/virtio/virtio/src/device.rs @@ -93,6 +93,9 @@ pub trait VirtioDevice: InspectMut + Send { /// transport fails DRIVER_OK; when the VM starts, the transport fails the /// confirmation of the start, and the VM stops the device again before /// any vCPU runs, which keeps the progress of the queues that did start. + /// A device whose start fails keeps the private state that it saves as + /// it was before the start, so that a save of the stopped VM matches the + /// queue progress that the transport keeps. /// Acquire the host resources that restored state refers to in /// [`restore_device`](Self::restore_device) instead, so that a missing /// one fails the restore. diff --git a/vm/devices/virtio/virtio_net/src/lib.rs b/vm/devices/virtio/virtio_net/src/lib.rs index 0ea1dd8069b..1d2580979f9 100644 --- a/vm/devices/virtio/virtio_net/src/lib.rs +++ b/vm/devices/virtio/virtio_net/src/lib.rs @@ -405,17 +405,29 @@ impl VirtioDevice for Device { let guest_memory = resources.guest_memory.clone(); let queue_size = resources.params.size; - let queue_event = PolledWait::new(driver, resources.event) - .context("failed creating queue event")?; - let queue = VirtioQueue::new( - *features, - resources.params, - resources.guest_memory, - resources.notify, - queue_event, - initial_state, - ) - .context("failed creating virtio net queue")?; + let queue = PolledWait::new(driver, resources.event) + .context("failed creating queue event") + .and_then(|queue_event| { + VirtioQueue::new( + *features, + resources.params, + resources.guest_memory, + resources.notify, + queue_event, + initial_state, + ) + .context("failed creating virtio net queue") + }); + let queue = match queue { + Ok(queue) => queue, + Err(err) => { + // The pair does not start, so its first queue, which + // has not run, stops with it, and the transport keeps + // the progress that it saved for both. + self.pairs[pair_idx] = QueuePairState::Empty; + return Err(err); + } + }; // Both queues are successfully constructed; extract the first and form the pair. let first_pair = !self diff --git a/vm/devices/virtio/virtio_net/src/saved_state.rs b/vm/devices/virtio/virtio_net/src/saved_state.rs index 186c1ec080e..6090042fb12 100644 --- a/vm/devices/virtio/virtio_net/src/saved_state.rs +++ b/vm/devices/virtio/virtio_net/src/saved_state.rs @@ -89,19 +89,21 @@ impl NicBuilder { } impl Device { - /// Checks and resets the saved queue-pair state before `start_queue` - /// starts queue `pair_idx`/`is_rx`. + /// Checks the saved queue-pair state before `start_queue` starts queue + /// `pair_idx`/`is_rx`. + /// + /// The queue-pair lifecycle that the last stop or restore recorded stays + /// as it is: the next stop of a started pair records it again, and a start + /// that fails leaves it for the next save, as the transport keeps the + /// saved progress of the queues that do not start. pub(crate) fn prepare_queue_start( - &mut self, + &self, pair_idx: usize, is_rx: bool, features: &VirtioDeviceFeatures, ) -> anyhow::Result<()> { let feature_banks = [features.bank(0), features.bank(1)]; match self.pairs.get(pair_idx) { - Some(QueuePairState::Empty) => { - self.lifecycle.stopped_queue_count = 0; - } Some(QueuePairState::Stopped { rx, tx }) => { anyhow::ensure!( rx.is_none() && tx.is_none(), @@ -113,7 +115,6 @@ impl Device { "virtio-net negotiated features changed while restarting queues" ); } - self.lifecycle.stopped_queue_count = 0; } Some(QueuePairState::HalfOpen { is_rx: pending_is_rx, diff --git a/vm/devices/virtio/virtio_net/src/tests/activation.rs b/vm/devices/virtio/virtio_net/src/tests/activation.rs index df2df4cbf9d..ccdee77efe2 100644 --- a/vm/devices/virtio/virtio_net/src/tests/activation.rs +++ b/vm/devices/virtio/virtio_net/src/tests/activation.rs @@ -2,10 +2,16 @@ // Licensed under the MIT License. //! Activation tests: a queue pair whose endpoint cannot create its queues -//! fails its start, instead of leaving the device without endpoint queues. +//! fails its start, instead of leaving the device without endpoint queues, +//! and a restored device whose start fails keeps the state that the restore +//! gave it. use super::MockEndpoint; use super::QUEUE_SIZE; +use super::RX_AVAIL_ADDR; +use super::RX_DESC_ADDR; +use super::RX_USED_ADDR; +use super::TOTAL_MEM_SIZE; use super::TX_AVAIL_ADDR; use super::TX_DESC_ADDR; use super::TX_USED_ADDR; @@ -25,15 +31,17 @@ use pal_async::async_test; use test_with_tracing::test; use virtio::QueueResources; use virtio::VirtioDevice; +use virtio::device::saved_state::DeviceQueueState; use virtio::queue::QueueParams; +use virtio::queue::QueueState; use virtio::spec::VirtioDeviceFeatures; use vmcore::interrupt::Interrupt; use vmcore::vm_task::SingleDriverBackend; use vmcore::vm_task::VmTaskDriverSource; -/// A [`MockEndpoint`] that fails to create queues the first `failures` times. +/// An endpoint that fails to create queues the first `failures` times. struct FailingEndpoint { - endpoint: MockEndpoint, + endpoint: Box, failures: usize, } @@ -96,12 +104,12 @@ impl TestHarness { let (queue_config_tx, queue_config_rx) = mesh::channel(); let (endpoint_action_tx, endpoint_action_rx) = mesh::channel(); let endpoint = FailingEndpoint { - endpoint: MockEndpoint { + endpoint: Box::new(MockEndpoint { queue_tx, queue_config_tx, endpoint_action_rx, is_ordered: true, - }, + }), failures, }; let driver_source = VmTaskDriverSource::new(SingleDriverBackend::new(driver.clone())); @@ -115,15 +123,26 @@ impl TestHarness { harness } + /// Replaces the harness device with one built for save and restore whose + /// endpoint fails to create queues the first `failures` times. + fn new_save_restore_with_endpoint_failures(driver: &DefaultDriver, failures: usize) -> Self { + Self::new_save_restore_over(driver, |endpoint| { + Box::new(FailingEndpoint { endpoint, failures }) + }) + } + + fn rx_queue_resources(&self) -> QueueResources { + QueueResources { + params: rx_queue_params(), + notify: Interrupt::from_event(self.rx_interrupt_event.clone()), + event: self.rx_event.clone(), + guest_memory: self.mem.clone(), + } + } + fn tx_queue_resources(&self) -> QueueResources { QueueResources { - params: QueueParams { - size: QUEUE_SIZE, - enable: true, - desc_addr: TX_DESC_ADDR, - avail_addr: TX_AVAIL_ADDR, - used_addr: TX_USED_ADDR, - }, + params: tx_queue_params(), notify: Interrupt::from_event(self.tx_interrupt_event.clone()), event: self.tx_event.clone(), guest_memory: self.mem.clone(), @@ -131,6 +150,26 @@ impl TestHarness { } } +fn rx_queue_params() -> QueueParams { + QueueParams { + size: QUEUE_SIZE, + enable: true, + desc_addr: RX_DESC_ADDR, + avail_addr: RX_AVAIL_ADDR, + used_addr: RX_USED_ADDR, + } +} + +fn tx_queue_params() -> QueueParams { + QueueParams { + size: QUEUE_SIZE, + enable: true, + desc_addr: TX_DESC_ADDR, + avail_addr: TX_AVAIL_ADDR, + used_addr: TX_USED_ADDR, + } +} + #[async_test] async fn endpoint_queue_failure_fails_the_queue_start(driver: DefaultDriver) { let mut harness = TestHarness::new_with_endpoint_failures(&driver, 1); @@ -160,3 +199,94 @@ async fn endpoint_queue_failure_fails_the_queue_start(driver: DefaultDriver) { let (used_id, _) = harness.wait_for_used().await; assert_eq!(used_id, 0); } + +/// A restored NIC whose pair fails to start when the VM starts, because its +/// endpoint cannot create its queues or its TX queue cannot be created, keeps +/// the queue lifecycle that the restore gave it: a save of the stopped VM holds +/// the lifecycle of the queues whose progress the transport keeps, so that the +/// snapshot restores, and a later start resumes the queues where the restore +/// left them. +#[async_test] +async fn failed_start_of_a_restored_nic_keeps_its_queue_lifecycle(driver: DefaultDriver) { + let features = VirtioDeviceFeatures::new(); + let mut source = TestHarness::new_save_restore(&driver); + let _queue = source.enable_and_get_handle().await; + source.post_tx_and_signal(0, 64); + assert_eq!(source.wait_for_used().await.0, 0); + source.device.quiesce_input().await.unwrap(); + let rx = source.device.stop_queue(0).await.unwrap(); + let tx = source.device.stop_queue(1).await.unwrap(); + let saved = source.device.save_device().unwrap().unwrap(); + + let mut restored = TestHarness::new_save_restore_with_endpoint_failures(&driver, 1); + let mut memory = vec![0; TOTAL_MEM_SIZE]; + source.mem.read_at(0, &mut memory).unwrap(); + restored.mem.write_at(0, &memory).unwrap(); + restored.tx_avail_idx = source.tx_avail_idx; + restored.tx_used_idx = source.tx_used_idx; + restored.next_data_offset = source.next_data_offset; + restored.device.restore_device(Some(saved)).unwrap(); + + let restored_queues = [ + DeviceQueueState { + params: rx_queue_params(), + queue_state: Some(rx), + }, + DeviceQueueState { + params: tx_queue_params(), + queue_state: Some(tx), + }, + ]; + // TX progress that the queue refuses: more in flight than the ring holds. + let invalid_tx = QueueState { + avail_index: tx.avail_index, + used_index: tx.avail_index.wrapping_add(1), + }; + for (tx_state, failure) in [ + (tx, "injected endpoint queue failure"), + (invalid_tx, "failed creating virtio net queue"), + ] { + // The TX queue completes the pair, which fails to start. + restored + .device + .start_queue(0, restored.rx_queue_resources(), &features, Some(rx)) + .await + .unwrap(); + let error = restored + .device + .start_queue(1, restored.tx_queue_resources(), &features, Some(tx_state)) + .await + .unwrap_err(); + assert!(format!("{error:#}").contains(failure), "{error:#}"); + // The transport stops the queue that started, which reports no + // progress, so the transport keeps the restored progress of both. + assert!(restored.device.stop_queue(0).await.is_none()); + assert!(restored.device.stop_queue(1).await.is_none()); + + let resaved = restored.device.save_device().unwrap().unwrap(); + restored.device.device_state_validator()( + Some(&resaved), + &features, + &restored_queues, + &restored.mem, + ) + .unwrap(); + } + + // The next start resumes the queues: the device transmits the guest's + // next packet, and not the one that it transmitted before the snapshot. + restored + .device + .start_queue(0, restored.rx_queue_resources(), &features, Some(rx)) + .await + .unwrap(); + restored + .device + .start_queue(1, restored.tx_queue_resources(), &features, Some(tx)) + .await + .unwrap(); + let queue = restored.wait_for_recreated_queue().await; + restored.post_tx_and_signal(2, 64); + assert_eq!(restored.wait_for_used().await.0, 2); + assert_eq!(queue.take_tx_avail_log().len(), 1); +} diff --git a/vm/devices/virtio/virtio_net/src/tests/harness.rs b/vm/devices/virtio/virtio_net/src/tests/harness.rs index 852d744a7ab..44ac07d4e01 100644 --- a/vm/devices/virtio/virtio_net/src/tests/harness.rs +++ b/vm/devices/virtio/virtio_net/src/tests/harness.rs @@ -272,6 +272,15 @@ impl TestHarness { /// Replaces the harness device with one built for save and restore over a /// [`ForkMockEndpoint`]. pub(super) fn new_save_restore(driver: &DefaultDriver) -> Self { + Self::new_save_restore_over(driver, |endpoint| endpoint) + } + + /// Replaces the harness device with one built for save and restore over + /// the endpoint that `wrap` makes of a [`ForkMockEndpoint`]. + pub(super) fn new_save_restore_over( + driver: &DefaultDriver, + wrap: impl FnOnce(Box) -> Box, + ) -> Self { let mut harness = Self::new(driver); let (queue_tx, queue_handle_rx) = mesh::channel(); let (queue_config_tx, queue_config_rx) = mesh::channel(); @@ -297,7 +306,7 @@ impl TestHarness { }, (1 << 5) | (1 << 32), ) - .build(&driver_source, Box::new(endpoint), mac) + .build(&driver_source, wrap(Box::new(endpoint)), mac) .unwrap(); harness.queue_handle_rx = queue_handle_rx; harness.queue_config_rx = queue_config_rx; From ea2169dc71f2623cf476415a32af37f38ebe8cfa Mon Sep 17 00:00:00 2001 From: Pedro Henrique Penna Date: Tue, 17 Nov 2026 05:26:37 -0800 Subject: [PATCH 2/6] pci_core: program the delivery of restored MSI-X vectors A restored PCI device that uses MSI-X delivers none of its interrupts until the guest writes the vector table again, which a guest resumed mid-flight never does: its queues complete their work, and the guest never learns of it. MsixCapability::restore restored the vector table, but not the MSI interrupt behind each vector, which stays disabled, as a new device's are, so each delivery only latches a pending bit. The MSI capability's restore programs its interrupt; the MSI-X one did not. A save also lost the interrupt that a masked vector had received: the vector keeps it in its MSI interrupt, or on its route's event, until the guest reads the pending bits, and the save recorded only the pending bit. Program each restored vector's delivery as the guest's writes of the table did before the save: enable an unmasked vector of an enabled capability with its restored message, and disable the others. Deliver nothing during the restore, so that its interrupts reach the guest once the VM runs (microsoft/nvx#1232): a vector restored masked keeps its pending bit, and delivers it when the guest unmasks it, and an interrupt that the destination latched before the restore is dropped, as it is not part of the restored state. And have the save record the pending bit that the guest's read of the pending bits would return, counting the interrupt that a masked vector holds, but leave that interrupt where it is, so that the save changes nothing that the guest sees: moving it into the pending bit, as the read does, would have the vector deliver it twice if another interrupt reached its route before the guest unmasked it. Tests: a restored capability delivers nothing during the restore, then delivers an unmasked vector's interrupt to its restored message, and the interrupt that a masked vector received before the save, which the guest never read from the pending bits, when the guest unmasks it; a save records the interrupts that masked vectors hold on a route and in the MSI interrupt, and the saved capability then delivers the route's once, on the route, when the guest unmasks the vector, although another interrupt arrived after the save; a restored virtio-pci device delivers its queue's completion to the vector that the guest programmed before the save. Without the restore change no interrupt arrives, without the save change the masked vector's pending bit is lost, and a save that moves the interrupt into the pending bit delivers it twice. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../pci/pci_core/src/capabilities/msix.rs | 122 +++++++++++++++++- vm/devices/virtio/virtio/src/tests.rs | 1 + .../virtio/src/tests/restore_interrupt.rs | 59 +++++++++ 3 files changed, 181 insertions(+), 1 deletion(-) create mode 100644 vm/devices/virtio/virtio/src/tests/restore_interrupt.rs diff --git a/vm/devices/pci/pci_core/src/capabilities/msix.rs b/vm/devices/pci/pci_core/src/capabilities/msix.rs index 53b2524ede2..dc8a5938d66 100644 --- a/vm/devices/pci/pci_core/src/capabilities/msix.rs +++ b/vm/devices/pci/pci_core/src/capabilities/msix.rs @@ -216,6 +216,24 @@ impl MsiInterrupt { was_pending } + /// Returns whether an interrupt is pending, as + /// [`drain_pending`](Self::drain_pending) does, but leaves it pending. + pub fn holds_pending(&self) -> bool { + let state = self.0.lock(); + if state.pending { + return true; + } + match &state.route { + Some(route) if route.consume_pending() => { + // Put the interrupt back on the route's event, which the + // route delivers once it is enabled again. + route.event().signal(); + true + } + _ => false, + } + } + pub fn interrupt(&self) -> Interrupt { Interrupt::from_target(MsiInterruptTarget(self.0.clone())) } @@ -578,7 +596,14 @@ mod save_restore { address, data, control, - is_pending, + // An interrupt that a masked vector received + // stays in its MSI interrupt until the guest + // reads the pending bits. Save the bit that + // the read returns, and leave the interrupt + // where it is, so that the save changes + // nothing that the guest sees. + is_pending: is_pending + || (!vec.is_enabled(state.enabled) && vec.msi.holds_pending()), } }) .collect() @@ -606,6 +631,18 @@ mod save_restore { data: new_vec.data, control: new_vec.control, is_pending: new_vec.is_pending, + }; + // Program the vector's delivery as the guest's writes of the + // table did before the save, so that the restored vector + // delivers to its message. Deliver nothing now: a vector that + // the restore leaves masked keeps its pending bit for its + // unmask, and an interrupt that the destination latched before + // the restore is not part of the restored state. + vec.msi.drain_pending(); + if vec.is_enabled(enabled) { + vec.msi.enable(vec.state.address, vec.state.data, false); + } else { + vec.msi.disable(); } } @@ -903,4 +940,87 @@ mod tests { data: 0x99 })); } + + /// A restore programs each vector as the guest programmed it before the + /// save, without a write of the table, and delivers nothing itself: an + /// unmasked vector delivers its next interrupt to its message, and a + /// masked one keeps the interrupt that it received before the save until + /// the guest unmasks it, although the guest never read its pending bit. + #[test] + fn restore_programs_vectors_without_delivering() { + use vmcore::save_restore::ProtobufSaveRestore; + + let source_conn = MsiConnection::new(); + let (mut source, mut source_cap) = MsixEmulator::new(2, 2, &source_conn.target()); + source_conn.connect(TestPciInterruptController::new().signal_msi()); + write_cap_u32(&mut source_cap, 0, 0x80000000); + for vector in 0..2u64 { + source.write_u32(vector * 16, 0xFEE00000); + source.write_u32(vector * 16 + 8, 0x40 + vector as u32); + } + // Vector 0 is unmasked, and vector 1 stays masked with an interrupt + // pending. + source.write_u32(12, 0); + source.interrupt(1).unwrap().deliver(); + let saved = source_cap.save().unwrap(); + + let msi_conn = MsiConnection::new(); + let (mut msix, mut cap) = MsixEmulator::new(2, 2, &msi_conn.target()); + let msi_controller = TestPciInterruptController::new(); + msi_conn.connect(msi_controller.signal_msi()); + cap.restore(saved).unwrap(); + assert_eq!(msi_controller.get_next_interrupt(), None); + assert_eq!(msix.read_u32(32), 2); + + msix.interrupt(0).unwrap().deliver(); + assert_eq!( + msi_controller.get_next_interrupt(), + Some((0xFEE00000, 0x40)) + ); + msix.write_u32(16 + 12, 0); + assert_eq!( + msi_controller.get_next_interrupt(), + Some((0xFEE00000, 0x41)) + ); + assert_eq!(msi_controller.get_next_interrupt(), None); + assert_eq!(msix.read_u32(32), 0); + } + + /// A save records the interrupt that a masked vector holds, on its route's + /// event or in its MSI interrupt, and leaves it there, so that the save + /// changes nothing that the guest sees: the restored capability reports + /// both pending bits, and the saved one delivers its route's interrupt + /// once, on the route, when the guest unmasks the vector, although + /// another interrupt arrived after the save. + #[test] + fn save_leaves_masked_interrupts_pending() { + use vmcore::save_restore::ProtobufSaveRestore; + + let (irqfd, _calls) = mock_irqfd(1); + let msi_conn = MsiConnection::new(); + msi_conn.connect_irqfd(irqfd); + let (mut msix, mut cap) = MsixEmulator::new(2, 2, &msi_conn.target()); + let msi_controller = TestPciInterruptController::new(); + msi_conn.connect(msi_controller.signal_msi()); + // Vector 0 holds its interrupt on its route's event, and vector 1 in + // its MSI interrupt. + let event = msix.interrupt(0).unwrap().event().unwrap().clone(); + write_cap_u32(&mut cap, 0, 0x80000000); + msix.write_u32(0, 0xFEE00000); + msix.write_u32(8, 0x40); + event.signal(); + msix.interrupt(1).unwrap().deliver(); + let saved = cap.save().unwrap(); + + let restored_conn = MsiConnection::new(); + let (restored, mut restored_cap) = MsixEmulator::new(2, 2, &restored_conn.target()); + restored_conn.connect(TestPciInterruptController::new().signal_msi()); + restored_cap.restore(saved).unwrap(); + assert_eq!(restored.read_u32(32), 3); + + event.signal(); + msix.write_u32(12, 0); + assert_eq!(msi_controller.get_next_interrupt(), None); + assert!(event.try_wait()); + } } diff --git a/vm/devices/virtio/virtio/src/tests.rs b/vm/devices/virtio/virtio/src/tests.rs index aa5c9784dce..7c816e61772 100644 --- a/vm/devices/virtio/virtio/src/tests.rs +++ b/vm/devices/virtio/virtio/src/tests.rs @@ -9,6 +9,7 @@ mod dma; mod input_gate; mod interrupt; +mod restore_interrupt; mod saved_state; mod start; diff --git a/vm/devices/virtio/virtio/src/tests/restore_interrupt.rs b/vm/devices/virtio/virtio/src/tests/restore_interrupt.rs new file mode 100644 index 00000000000..033900e9f5e --- /dev/null +++ b/vm/devices/virtio/virtio/src/tests/restore_interrupt.rs @@ -0,0 +1,59 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +//! Tests of the interrupts of a restored transport: a restored PCI device +//! delivers its queues' MSI-X messages once it starts. + +use super::*; +use test_with_tracing::test; + +/// Waits for the next MSI that the test interrupt controller receives. +async fn next_msi(driver: &DefaultDriver, controller: &TestPciInterruptController) -> (u64, u32) { + let mut timer = PolledTimer::new(driver); + for _ in 0..500 { + if let Some(interrupt) = controller.get_next_interrupt() { + return interrupt; + } + timer.sleep(Duration::from_millis(10)).await; + } + panic!("no MSI arrived"); +} + +/// A restored PCI device delivers its queue's interrupts to the MSI-X vector +/// that the guest programmed before the save, although the guest does not +/// program it again, and the restore itself delivers nothing. +#[async_test] +async fn pci_restored_msix_vector_delivers_after_start(driver: DefaultDriver) { + use vmcore::device_state::ChangeDeviceState; + use vmcore::save_restore::AsyncSaveRestore; + + let test_mem = VirtioTestMemoryAccess::new(); + let mut guest = VirtioTestGuest::new_split(&driver, &test_mem, 1, 4, true); + let work: TestDeviceQueueWorkFn = Arc::new( + |_: u16, queue: &mut VirtioQueue, work: VirtioQueueCallbackWork| { + queue.complete(work, 7); + }, + ); + let mut source = VirtioPciTestDevice::new(&driver, 1, &test_mem, Some(work.clone())); + guest + .setup_pci_device(&mut source, guest.queue_features()) + .await; + source.pci_device.stop().await; + let saved = source.pci_device.save().await.unwrap(); + drop(source); + + let mut restored = VirtioPciTestDevice::new(&driver, 1, &test_mem, Some(work)); + restored.pci_device.restore(saved).await.unwrap(); + restored.pci_device.start(); + restored.pci_device.confirm_start().await.unwrap(); + assert_eq!(restored.test_intc.get_next_interrupt(), None); + + guest.add_to_avail_queue(0); + let address = pci_notification_address(&mut restored, 0); + restored.write_u32(address, 0); + // `setup_pci_device` gives queue 0 vector 1, whose message has address 1 + // and data 0. + assert_eq!(next_msi(&driver, &restored.test_intc).await, (1, 0)); + assert_eq!(guest.get_next_completed(0), Some((0, 7))); + restored.pci_device.stop().await; +} From f8c63a6c95e0aea56e963db31ac2d4e649456d18 Mon Sep 17 00:00:00 2001 From: Pedro Henrique Penna Date: Tue, 17 Nov 2026 05:31:04 -0800 Subject: [PATCH 3/6] virtio: hold restored interrupt lines until the device starts A restore must not deliver notifications before the VM starts (microsoft/nvx#1232). The virtio transports drove their interrupt lines while they restored: the MMIO transport lowered its line and raised it again to the restored status, and the PCI transport set its INTx line to the restored status. The interrupt controller that a line reaches is a chipset device that restores in the same pass, before or after the transport, so the restore either reached a controller whose state it was still restoring, where the restore of the controller overwrote it, or it raised the line of a restored controller from the low that the transport had just driven, an edge that can deliver the interrupt again to a processor whose state is not yet restored. In the shared-status mode of the microVM, the restore drove the line low twice. Hold the lines instead. A restore keeps the interrupt status that it restores, and drives no line. The start that follows drives a legacy MMIO line, or a PCI INTx line, to the restored status. A shared-status line, which only pulses as the device raises the status, is not driven at all, since the pulse of the status that the saved word reports was sent before the save, and a restore sends no MSI-X message: the queues' messages begin with the queues, which start with the device. A stop before the start keeps the hold, a reset ends it with the line low, and a later start, as from a pause, drives nothing. The saved state is unchanged. Tests: a restored MMIO device and a restored PCI device with INTx drive no line during their restore, although the restored status asserts it, drive it high when they start, and drive nothing at a second start; a save before the start keeps the status; a reset or a removal before the start leaves the line low. The MMIO interrupt state drives no legacy line while held, and its release drives the line once; a shared-status restore and its release drive the line not at all. The transport tests fail without the change. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- Guide/src/user_guide/openvmm/snapshots.md | 18 +- .../virtio/src/tests/restore_interrupt.rs | 162 +++++++++++++++++- .../virtio/virtio/src/transport/mmio.rs | 7 +- .../virtio/src/transport/mmio/interrupt.rs | 104 ++++++++++- vm/devices/virtio/virtio/src/transport/pci.rs | 20 ++- .../virtio/src/transport/pci/interrupt.rs | 35 ++++ 6 files changed, 322 insertions(+), 24 deletions(-) create mode 100644 vm/devices/virtio/virtio/src/transport/pci/interrupt.rs diff --git a/Guide/src/user_guide/openvmm/snapshots.md b/Guide/src/user_guide/openvmm/snapshots.md index fd89c4bee0e..7a93491add2 100644 --- a/Guide/src/user_guide/openvmm/snapshots.md +++ b/Guide/src/user_guide/openvmm/snapshots.md @@ -250,13 +250,17 @@ the complete frame. Restore completes the restoration of every device before any vCPU runs, even when the VM stays paused: invalid device state, or a host resource that a device's state refers to and that OpenVMM cannot reacquire, fails the restore -itself. Restoring starts no virtqueue. Each device starts its queues when the -VM first runs, and the resume waits for them before any vCPU runs. A queue that -fails to start then, such as one whose network backend cannot create its -queues, fails the resume: OpenVMM stops the devices that started again and -logs the failure, no vCPU runs, and the restore publishes no readiness event. -The devices keep their restored state, so a later resume starts their queues -again, and a save of the paused VM saves that state. +itself. Restoring starts no virtqueue, and no virtio device drives its interrupt +line: one whose line holds a level, as a legacy virtio-mmio device's line or a +PCI device's INTx does, drives the line to the status that it restored only when +the VM first runs, once every device and interrupt controller has restored its +state. Each device starts its queues when the VM first runs, and the resume +waits for them before any vCPU runs. A queue that fails to start then, such as +one whose network backend cannot create its queues, fails the resume: OpenVMM +stops the devices that started again and logs the failure, no vCPU runs, and the +restore publishes no readiness event. The devices keep their restored state, so +a later resume starts their queues again, and a save of the paused VM saves that +state. A restore gives a KVM VP the NMIs that it held pending at capture, which can be two: a second NMI that reached the VP before it ran, which KVM injects once the diff --git a/vm/devices/virtio/virtio/src/tests/restore_interrupt.rs b/vm/devices/virtio/virtio/src/tests/restore_interrupt.rs index 033900e9f5e..ea6b2d43e6c 100644 --- a/vm/devices/virtio/virtio/src/tests/restore_interrupt.rs +++ b/vm/devices/virtio/virtio/src/tests/restore_interrupt.rs @@ -1,11 +1,44 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT License. -//! Tests of the interrupts of a restored transport: a restored PCI device -//! delivers its queues' MSI-X messages once it starts. +//! Tests of the interrupts of a restored transport: a restore holds the +//! interrupt line of the MMIO transport and the INTx line of the PCI one until +//! the device starts, and a restored PCI device delivers its queues' MSI-X +//! messages once it starts. use super::*; use test_with_tracing::test; +use vmcore::line_interrupt::LineSetTarget; +use vmcore::save_restore::AsyncSaveRestore; +use vmcore::save_restore::SavedStateBlob; + +/// A line target that records each level that its line set gives it. +#[derive(Default)] +struct RecordingLineTarget(Mutex>); + +impl LineSetTarget for RecordingLineTarget { + fn set_irq(&self, _vector: u32, high: bool) { + self.0.lock().push(high); + } +} + +impl RecordingLineTarget { + /// Takes the levels recorded since the last call. + fn take(&self) -> Vec { + std::mem::take(&mut *self.0.lock()) + } +} + +fn console_traits(device_register_length: u32) -> DeviceTraits { + DeviceTraits { + device_id: VirtioDeviceType::CONSOLE, + device_features: VirtioDeviceFeatures::new() + .with_bank(0, 2 | VIRTIO_F_RING_INDIRECT_DESC | VIRTIO_F_RING_EVENT_IDX), + max_queues: 1, + device_register_length, + ..Default::default() + } +} /// Waits for the next MSI that the test interrupt controller receives. async fn next_msi(driver: &DefaultDriver, controller: &TestPciInterruptController) -> (u64, u32) { @@ -19,14 +52,133 @@ async fn next_msi(driver: &DefaultDriver, controller: &TestPciInterruptControlle panic!("no MSI arrived"); } +/// A restore of an active MMIO device drives no interrupt line, although its +/// restored used-buffer status asserts the line; the start drives the line to +/// that status. A save before the start keeps the status, and a reset or a +/// removal before it leaves the line low. +#[async_test] +async fn mmio_restore_holds_the_interrupt_line_until_start(driver: DefaultDriver) { + let test_mem = VirtioTestMemoryAccess::new(); + let mem = GuestMemory::new("test", test_mem.clone()); + let driver_source = VmTaskDriverSource::new(SingleDriverBackend::new(driver.clone())); + let guest = VirtioTestGuest::new_split(&driver, &test_mem, 1, 4, true); + let new_device = |interrupt| { + VirtioMmioDevice::new( + Box::new(TestDevice::new(&driver_source, console_traits(0), None)), + &driver_source.simple(), + mem.clone(), + interrupt, + None, + 0, + 1, + ) + .unwrap() + }; + + let mut source = new_device(LineInterrupt::detached()); + guest + .setup_chipset_device(&mut source, guest.queue_features()) + .await; + source.stop().await; + let mut saved = source.save().await.unwrap(); + saved.interrupt_status = VIRTIO_MMIO_INTERRUPT_STATUS_USED_BUFFER; + let saved = SavedStateBlob::new(saved); + drop(source); + + let target = Arc::new(RecordingLineTarget::default()); + let mut restored = new_device(LineInterrupt::new_with_target( + "restored", + target.clone(), + 0, + )); + restored + .restore(saved.clone().parse().unwrap()) + .await + .unwrap(); + assert!(target.take().is_empty()); + restored.stop().await; + assert_eq!( + restored.save().await.unwrap().interrupt_status, + VIRTIO_MMIO_INTERRUPT_STATUS_USED_BUFFER + ); + assert!(target.take().is_empty()); + restored.start(); + assert_eq!(target.take(), [true]); + restored.stop().await; + restored.start(); + assert!(target.take().is_empty()); + restored.stop().await; + + let target = Arc::new(RecordingLineTarget::default()); + let mut reset = new_device(LineInterrupt::new_with_target("reset", target.clone(), 0)); + reset.restore(saved.clone().parse().unwrap()).await.unwrap(); + reset.reset().await; + reset.start(); + assert!(!target.take().contains(&true)); + reset.stop().await; + + let target = Arc::new(RecordingLineTarget::default()); + let mut removed = new_device(LineInterrupt::new_with_target("removed", target.clone(), 0)); + removed.restore(saved.parse().unwrap()).await.unwrap(); + removed.teardown().await; + assert!(!target.take().contains(&true)); +} + +/// A restore of a PCI device drives no INTx line, although its restored +/// interrupt status asserts the line; the start drives the line to that +/// status, and a reset before it leaves the line low. +#[async_test] +async fn pci_restore_holds_the_intx_line_until_start(driver: DefaultDriver) { + let driver_source = VmTaskDriverSource::new(SingleDriverBackend::new(driver.clone())); + let new_device = |interrupt| { + VirtioPciDevice::new( + Box::new(TestDevice::new(&driver_source, console_traits(12), None)), + &driver, + GuestMemory::empty(), + PciInterruptModel::IntX(pci_core::PciInterruptPin::IntA, interrupt), + None, + &mut ExternallyManagedMmioIntercepts, + None, + ) + .unwrap() + }; + + let mut source = new_device(LineInterrupt::detached()); + source.stop().await; + let mut saved = source.save().await.unwrap(); + saved.interrupt_status = 1; + let saved = SavedStateBlob::new(saved); + drop(source); + + let target = Arc::new(RecordingLineTarget::default()); + let mut restored = new_device(LineInterrupt::new_with_target( + "restored", + target.clone(), + 0, + )); + restored + .restore(saved.clone().parse().unwrap()) + .await + .unwrap(); + assert!(target.take().is_empty()); + restored.start(); + assert_eq!(target.take(), [true]); + restored.stop().await; + + let target = Arc::new(RecordingLineTarget::default()); + let mut reset = new_device(LineInterrupt::new_with_target("reset", target.clone(), 0)); + reset.restore(saved.parse().unwrap()).await.unwrap(); + reset.reset().await; + reset.start(); + assert!(!target.take().contains(&true)); + reset.stop().await; +} + /// A restored PCI device delivers its queue's interrupts to the MSI-X vector /// that the guest programmed before the save, although the guest does not /// program it again, and the restore itself delivers nothing. #[async_test] async fn pci_restored_msix_vector_delivers_after_start(driver: DefaultDriver) { - use vmcore::device_state::ChangeDeviceState; - use vmcore::save_restore::AsyncSaveRestore; - let test_mem = VirtioTestMemoryAccess::new(); let mut guest = VirtioTestGuest::new_split(&driver, &test_mem, 1, 4, true); let work: TestDeviceQueueWorkFn = Arc::new( diff --git a/vm/devices/virtio/virtio/src/transport/mmio.rs b/vm/devices/virtio/virtio/src/transport/mmio.rs index 33288aeb058..339e23259ee 100644 --- a/vm/devices/virtio/virtio/src/transport/mmio.rs +++ b/vm/devices/virtio/virtio/src/transport/mmio.rs @@ -70,7 +70,7 @@ impl InterruptState { } else { self.status &= !bits; } - self.interrupt.set_level(self.status != 0); + self.delivery.set_level(&self.interrupt, self.status != 0); } } @@ -416,6 +416,7 @@ impl VirtioMmioDevice { impl ChangeDeviceState for VirtioMmioDevice { fn start(&mut self) { + self.mmio.release_held_interrupt(); self.core.start(&mut self.mmio); } @@ -515,7 +516,6 @@ mod saved_state { &mut self, state: Self::SavedState, ) -> Result<(), vmcore::save_restore::RestoreError> { - self.mmio.reset_interrupt_state(); let saved_queue_count = state.queues.len(); self.core.restore_common( &mut self.mmio, @@ -525,7 +525,8 @@ mod saved_state { saved_queue_count, )?; - // Restore MMIO-specific interrupt state. + // Restore MMIO-specific interrupt state, holding the interrupt + // line until the device starts. { let mut is = self.mmio.interrupt_state.lock(); is.restore_status(state.interrupt_status)?; diff --git a/vm/devices/virtio/virtio/src/transport/mmio/interrupt.rs b/vm/devices/virtio/virtio/src/transport/mmio/interrupt.rs index cc297d4681e..e95304dcb9d 100644 --- a/vm/devices/virtio/virtio/src/transport/mmio/interrupt.rs +++ b/vm/devices/virtio/virtio/src/transport/mmio/interrupt.rs @@ -11,6 +11,13 @@ //! read yet. In [`VirtioMmioInterruptMode::SharedStatus`] mode the status lives //! in a guest-memory word, and each zero-to-nonzero transition pulses the //! interrupt line. +//! +//! A restore holds a legacy interrupt line until the device starts: it restores +//! the status without driving the line, since the interrupt controller that the +//! line reaches may itself still be restoring, and the start then drives the +//! line to the restored level. A shared-status line only pulses as the device +//! raises the status, so a restore drives it not at all: the pulse of the +//! status that the saved word reports was sent before the save. use super::InterruptState; use super::MmioTransport; @@ -95,6 +102,9 @@ pub(super) struct StatusDelivery { used_buffer_generation: u64, observed_used_buffer_generation: Option, shared_status: Option, + /// Whether a restore holds the legacy interrupt line until the device + /// starts. + held: bool, } impl StatusDelivery { @@ -123,6 +133,13 @@ impl StatusDelivery { } false } + + /// Drives the level of a legacy interrupt line, unless a restore holds it. + pub(super) fn set_level(&self, interrupt: &LineInterrupt, high: bool) { + if !self.held { + interrupt.set_level(high); + } + } } #[derive(Inspect)] @@ -196,6 +213,7 @@ pub(super) fn setup( used_buffer_generation: 0, observed_used_buffer_generation: None, shared_status, + held: false, }, AckDoorbellSetup { #[cfg(target_os = "linux")] @@ -350,22 +368,30 @@ impl InterruptState { } } - /// Restores the saved interrupt status. + /// Restores the saved interrupt status without driving the line: a legacy + /// line keeps its level until [`Self::release_hold`] drives it to the + /// restored status. pub(super) fn restore_status(&mut self, status: u32) -> Result<(), RestoreError> { if let Some(shared_status) = &self.delivery.shared_status { shared_status .store(status) .map_err(|error| RestoreError::InvalidSavedState(error.into()))?; - self.interrupt.set_level(false); } else { self.status = status; self.delivery.used_buffer_generation = 0; self.delivery.observed_used_buffer_generation = (self.status & VIRTIO_MMIO_INTERRUPT_STATUS_USED_BUFFER != 0).then_some(0); - self.interrupt.set_level(self.status != 0); + self.delivery.held = true; } Ok(()) } + + /// Ends the hold of a restore: drives a legacy line to the restored level. + fn release_hold(&mut self) { + if std::mem::take(&mut self.delivery.held) { + self.interrupt.set_level(self.status != 0); + } + } } impl MmioTransport { @@ -379,6 +405,11 @@ impl MmioTransport { self.lock_interrupt_state().read_status() } + /// Releases the interrupt line that a restore holds, as the device starts. + pub(super) fn release_held_interrupt(&self) { + self.lock_interrupt_state().release_hold(); + } + pub(super) fn reset_interrupt_state(&self) { let mut state = self.lock_interrupt_state(); if let Some(shared_status) = &state.delivery.shared_status { @@ -389,6 +420,7 @@ impl MmioTransport { state.status = 0; state.delivery.used_buffer_generation = 0; state.delivery.observed_used_buffer_generation = None; + state.delivery.held = false; state.interrupt.set_level(false); } } @@ -417,10 +449,12 @@ mod tests { struct CountingInterruptTarget { high: AtomicBool, pulses: AtomicUsize, + calls: AtomicUsize, } impl LineSetTarget for CountingInterruptTarget { fn set_irq(&self, _vector: u32, high: bool) { + self.calls.fetch_add(1, Ordering::SeqCst); self.high.store(high, Ordering::SeqCst); if high { self.pulses.fetch_add(1, Ordering::SeqCst); @@ -442,10 +476,74 @@ mod tests { guest_memory: guest_memory.clone(), gpa: 0, }), + held: false, }, } } + fn legacy_interrupt_state(target: Arc) -> InterruptState { + InterruptState { + interrupt: LineInterrupt::new_with_target("legacy-test", target, 0), + status: 0, + delivery: StatusDelivery { + used_buffer_generation: 0, + observed_used_buffer_generation: None, + shared_status: None, + held: false, + }, + } + } + + /// A restore drives no legacy line, even when its status or a change + /// during the hold asserts the interrupt; the release drives the line to + /// the status's level, once. + #[test] + fn restore_holds_the_legacy_line_until_release() { + let target = Arc::new(CountingInterruptTarget::default()); + let mut state = legacy_interrupt_state(target.clone()); + + state + .restore_status(VIRTIO_MMIO_INTERRUPT_STATUS_USED_BUFFER) + .unwrap(); + state.update(true, VIRTIO_MMIO_INTERRUPT_STATUS_CONFIG_CHANGE); + assert_eq!(target.calls.load(Ordering::SeqCst), 0); + assert_eq!(state.read_status(), 3); + + state.release_hold(); + assert_eq!(target.calls.load(Ordering::SeqCst), 1); + assert!(target.high.load(Ordering::SeqCst)); + state.release_hold(); + assert_eq!(target.calls.load(Ordering::SeqCst), 1); + + // A restored status of zero leaves the line low. + let target = Arc::new(CountingInterruptTarget::default()); + let mut state = legacy_interrupt_state(target.clone()); + state.restore_status(0).unwrap(); + state.release_hold(); + assert!(!target.high.load(Ordering::SeqCst)); + assert_eq!(target.pulses.load(Ordering::SeqCst), 0); + } + + /// A restore drives a shared-status line not at all, and neither does the + /// release, although the restored word reports a status: its pulse was + /// sent before the save. + #[test] + fn restore_drives_no_shared_status_line() { + let guest_memory = GuestMemory::allocate(0x1000); + let target = Arc::new(CountingInterruptTarget::default()); + let mut state = shared_interrupt_state(&guest_memory, target.clone()); + + state + .restore_status(VIRTIO_MMIO_INTERRUPT_STATUS_CONFIG_CHANGE) + .unwrap(); + assert_eq!( + state.read_status(), + VIRTIO_MMIO_INTERRUPT_STATUS_CONFIG_CHANGE + ); + state.release_hold(); + assert_eq!(target.calls.load(Ordering::SeqCst), 0); + } + #[test] fn shared_status_coalesces_bits_and_pulses_on_zero_transition() { let guest_memory = GuestMemory::allocate(0x1000); diff --git a/vm/devices/virtio/virtio/src/transport/pci.rs b/vm/devices/virtio/virtio/src/transport/pci.rs index f1565c8eda8..09a6766d908 100644 --- a/vm/devices/virtio/virtio/src/transport/pci.rs +++ b/vm/devices/virtio/virtio/src/transport/pci.rs @@ -3,6 +3,8 @@ //! PCI transport for virtio devices +mod interrupt; + use self::capabilities::*; use super::StalledIo; use super::core::QueueDoorbell; @@ -154,6 +156,8 @@ struct PciTransport { #[inspect(hex)] notify_offset: u32, notify_length: u32, + /// Whether a restore holds the INTx line until the device starts. + held: bool, } impl TransportOps for PciTransport { @@ -186,6 +190,9 @@ impl TransportOps for PciTransport { interrupt.deliver(); } InterruptKind::Msix(_) => (), + // A restore holds the line until the device starts, which drives + // it to the status. + InterruptKind::IntX(_) if self.held => (), InterruptKind::IntX(line) => line.set_level(true), } } @@ -196,6 +203,7 @@ impl TransportOps for PciTransport { line.set_level(false); } self.msix_config_vector = 0; + self.held = false; } fn queue_doorbell(&mut self, index: u16) -> Option { @@ -391,6 +399,7 @@ impl VirtioPciDevice { device_register_length: traits.device_register_length, notify_offset, notify_length, + held: false, }, }) } @@ -834,6 +843,7 @@ impl VirtioPciDevice { impl ChangeDeviceState for VirtioPciDevice { fn start(&mut self) { + self.pci.release_held_interrupts(); self.core.start(&mut self.pci); } @@ -994,12 +1004,10 @@ mod saved_state { saved_queue_count, )?; - // Restore PCI-specific interrupt state. - *self.pci.interrupt_status.lock() = state.interrupt_status; - if let InterruptKind::IntX(line) = &self.pci.interrupt_kind { - line.set_level(state.interrupt_status != 0); - } - self.pci.msix_config_vector = state.msix_config_vector; + // Restore PCI-specific interrupt state, holding the interrupt + // outputs until the device starts. + self.pci + .restore_interrupts(state.interrupt_status, state.msix_config_vector); self.core.restore_device_state(state.device_state).await } diff --git a/vm/devices/virtio/virtio/src/transport/pci/interrupt.rs b/vm/devices/virtio/virtio/src/transport/pci/interrupt.rs new file mode 100644 index 00000000000..dbdcec0fcb6 --- /dev/null +++ b/vm/devices/virtio/virtio/src/transport/pci/interrupt.rs @@ -0,0 +1,35 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +//! The interrupt outputs of the PCI transport across a restore. +//! +//! A restore holds the INTx line until the device starts: it restores the +//! interrupt status without driving the line, since the interrupt controller +//! that the line reaches may itself still be restoring, and the start then +//! drives the line to the restored status. A restore sends no MSI-X message: +//! the queues' messages begin with the queues, which start only with the +//! device, and the vectors keep the messages that they held pending at the +//! save in their pending bits. + +use super::InterruptKind; +use super::PciTransport; + +impl PciTransport { + /// Restores the interrupt status and the MSI-X configuration vector, and + /// holds the INTx line until [`Self::release_held_interrupts`]. + pub(super) fn restore_interrupts(&mut self, status: u32, msix_config_vector: u16) { + *self.interrupt_status.lock() = status; + self.msix_config_vector = msix_config_vector; + self.held = matches!(self.interrupt_kind, InterruptKind::IntX(_)); + } + + /// Ends the hold of a restore, as the device starts: drives the INTx line + /// to the restored status. + pub(super) fn release_held_interrupts(&mut self) { + if std::mem::take(&mut self.held) + && let InterruptKind::IntX(line) = &self.interrupt_kind + { + line.set_level(*self.interrupt_status.lock() != 0); + } + } +} From 3a6a29318d672c809381ffc4f2dfc025d4fff774 Mon Sep 17 00:00:00 2001 From: Pedro Henrique Penna Date: Tue, 17 Nov 2026 05:34:44 -0800 Subject: [PATCH 4/6] virtiofs: test a transport restore of a file gone from the host A restore of a microVM virtio-fs reopens the host files of the handles and the inodes that the guest held, so that a file that is gone from the host fails the restore itself, before any vCPU runs, even for a VM restored paused (microsoft/nvx#1232). The filesystem's tests checked that its restore fails then, and the device's tests that the transport reports an invalid device-private state, but no test sent a missing host file through the transport's awaited restore. Add that test: a device whose guest holds a file open is saved through the MMIO transport, the file is removed from the host, and the restore of a new transport fails with the reopen error, before any queue starts. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../virtio/virtiofs/src/microvm/device.rs | 56 +++++++++++++++++++ 1 file changed, 56 insertions(+) diff --git a/vm/devices/virtio/virtiofs/src/microvm/device.rs b/vm/devices/virtio/virtiofs/src/microvm/device.rs index 786d28346f4..3767bc02a59 100644 --- a/vm/devices/virtio/virtiofs/src/microvm/device.rs +++ b/vm/devices/virtio/virtiofs/src/microvm/device.rs @@ -710,4 +710,60 @@ mod tests { transport.stop().await; transport.save().await.unwrap(); } + + /// The restore of the transport reopens the host files of the handles + /// that the guest held open, so a file that is gone from the host fails + /// the restore itself, before any queue starts, rather than the start + /// that follows it. + #[async_test] + async fn transport_restore_fails_when_an_open_file_is_gone(driver: DefaultDriver) { + let root = tempfile::tempdir().unwrap(); + std::fs::write(root.path().join("data"), b"data").unwrap(); + let driver_source = VmTaskDriverSource::new(SingleDriverBackend::new(driver.clone())); + let device = VirtioFsDevice::new_microvm_hostfs( + &driver_source, + MICROVM_ATTACHMENT_ID.to_owned(), + microvm_root_identity(root.path()).unwrap(), + false, + Vec::new(), + Vec::new(), + Vec::new(), + MicroVmOwnerMode::Vmm, + root.path(), + None, + ) + .unwrap(); + { + let fs = device.stateful_fs.as_ref().unwrap(); + let (data, _) = fs + .get_inode(fuse::protocol::FUSE_ROOT_ID) + .unwrap() + .lookup_child(lx::LxStr::from_bytes(b"data")) + .unwrap(); + let (data, _) = fs.insert_inode(data).unwrap(); + fs.insert_file(data.open(lx::O_RDONLY).unwrap()).unwrap(); + } + let mut source = VirtioMmioDevice::new( + Box::new(device), + &driver, + GuestMemory::empty(), + LineInterrupt::detached(), + None, + 0, + 0x1000, + ) + .unwrap(); + source.quiesce_input().await.unwrap(); + source.stop().await; + let saved = source.save().await.unwrap(); + source.teardown().await; + + std::fs::remove_file(root.path().join("data")).unwrap(); + let mut destination = microvm_transport(&driver, root.path()); + let error = destination.restore(saved).await.unwrap_err(); + assert!( + format!("{:#}", anyhow::Error::from(error)).contains("cannot be reopened"), + "the restore failed for another reason" + ); + } } From 5dac44c33efdcecb824067e6d7bbf7e1cb0cdb5f Mon Sep 17 00:00:00 2001 From: Pedro Henrique Penna Date: Tue, 17 Nov 2026 05:34:44 -0800 Subject: [PATCH 5/6] chipset_device_worker: test requests to an ended device worker The proxy of a remote chipset device sends a start without waiting for it, as the device-state contract expects, and reports the outcome from confirm_start, which the state units await before the VM runs. A start must not turn the failure of a worker that has ended into a success (microsoft/nvx#1232). The tests checked a confirmation that the remote device fails, and a save or restore after the worker ends, but not a start or an input-gate change after it. Add that test: once the worker has ended, the confirmation of a start and both input-gate changes fail. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- workers/chipset_device_worker/src/proxy.rs | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/workers/chipset_device_worker/src/proxy.rs b/workers/chipset_device_worker/src/proxy.rs index c365a188927..f02575980c5 100644 --- a/workers/chipset_device_worker/src/proxy.rs +++ b/workers/chipset_device_worker/src/proxy.rs @@ -686,4 +686,19 @@ mod tests { proxy.stop().await; stop_worker(&mut proxy).await; } + + /// A device whose worker has ended does not start: its start and the + /// changes of its input gate fail, rather than succeed without the + /// device. + #[async_test] + async fn requests_to_an_ended_worker_fail(driver: DefaultDriver) { + let vmtime = VmTimeKeeper::new(&driver, VmTime::from_100ns(0)); + let mut proxy = new_proxy(&vmtime, &driver).await; + stop_worker(&mut proxy).await; + + proxy.start(); + assert!(proxy.confirm_start().await.is_err()); + assert!(proxy.quiesce_input().await.is_err()); + assert!(proxy.resume_input().await.is_err()); + } } From 8bbe042b1322023e95cce03ef6d4e1ae8e18a6be Mon Sep 17 00:00:00 2001 From: Pedro Henrique Penna Date: Tue, 17 Nov 2026 13:35:49 -0800 Subject: [PATCH 6/6] vmcore, virtio: say that a failed start keeps its queues' progress ChangeDeviceState::confirm_start said that a device whose start fails keeps the state that it had before the start, so that stopping it leaves it as it was, and VirtioDevice::start_queue and the Guide's snapshots page said that such a device keeps the private state that it saves as it was before the start, and its restored state. A virtio device whose second queue fails to start can have started its first one, which completes the requests that were pending in it before the VM stops the device again, and the stop keeps that queue's progress, as the virtio test failed_confirmation_keeps_queue_progress requires. So a failed start leaves the state that its stop records, with that progress, and a save of the stopped VM saves that state; the confirmed start keeps the vCPUs from running, but does not undo the device's I/O. Say so in the three places. No code changes. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- Guide/src/user_guide/openvmm/snapshots.md | 7 ++++--- vm/devices/virtio/virtio/src/device.rs | 7 ++++--- vm/vmcore/src/device_state.rs | 7 +++++-- 3 files changed, 13 insertions(+), 8 deletions(-) diff --git a/Guide/src/user_guide/openvmm/snapshots.md b/Guide/src/user_guide/openvmm/snapshots.md index 7a93491add2..45b602aec97 100644 --- a/Guide/src/user_guide/openvmm/snapshots.md +++ b/Guide/src/user_guide/openvmm/snapshots.md @@ -258,9 +258,10 @@ state. Each device starts its queues when the VM first runs, and the resume waits for them before any vCPU runs. A queue that fails to start then, such as one whose network backend cannot create its queues, fails the resume: OpenVMM stops the devices that started again and logs the failure, no vCPU runs, and the -restore publishes no readiness event. The devices keep their restored state, so -a later resume starts their queues again, and a save of the paused VM saves that -state. +restore publishes no readiness event. The devices keep the state that this stop +leaves, which includes the progress of a queue that started and completed +pending requests before another failed, so a later resume starts their queues +again where they stopped, and a save of the paused VM saves that state. A restore gives a KVM VP the NMIs that it held pending at capture, which can be two: a second NMI that reached the VP before it ran, which KVM injects once the diff --git a/vm/devices/virtio/virtio/src/device.rs b/vm/devices/virtio/virtio/src/device.rs index b31f122c7cd..2cf72840ff5 100644 --- a/vm/devices/virtio/virtio/src/device.rs +++ b/vm/devices/virtio/virtio/src/device.rs @@ -93,9 +93,10 @@ pub trait VirtioDevice: InspectMut + Send { /// transport fails DRIVER_OK; when the VM starts, the transport fails the /// confirmation of the start, and the VM stops the device again before /// any vCPU runs, which keeps the progress of the queues that did start. - /// A device whose start fails keeps the private state that it saves as - /// it was before the start, so that a save of the stopped VM matches the - /// queue progress that the transport keeps. + /// A device whose start fails keeps the private state that it saves in + /// step with the queue progress that the transport keeps, which includes + /// the requests that a queue that did start completed before the stop, so + /// that a save of the stopped VM restores. /// Acquire the host resources that restored state refers to in /// [`restore_device`](Self::restore_device) instead, so that a missing /// one fails the restore. diff --git a/vm/vmcore/src/device_state.rs b/vm/vmcore/src/device_state.rs index c80fd6f0671..673b235d917 100644 --- a/vm/vmcore/src/device_state.rs +++ b/vm/vmcore/src/device_state.rs @@ -75,8 +75,11 @@ pub trait ChangeDeviceState { /// /// Callers call this after each [`Self::start`], before any other state /// change, and stop the device again if it fails. A device that fails - /// keeps the state that it had before the start, so that stopping it - /// leaves it as it was and a later start can try again. + /// keeps the state that it had before the start, apart from the progress + /// of a part of it that did start, such as a queue that completed pending + /// requests before another queue failed, so that stopping it records a + /// consistent state, which a save keeps and from which a later start can + /// try again. /// /// The default reports success, for devices whose start cannot fail. fn confirm_start(&mut self) -> impl Send + Future> {