From 1d1ac833d8a09b3f6eadb42385da4231b9337e6b Mon Sep 17 00:00:00 2001 From: Pedro Henrique Penna Date: Thu, 5 Nov 2026 17:14:14 -0800 Subject: [PATCH] virtio_net: fail the queue start when the endpoint cannot create queues The coordinator created the endpoint queues only after the device task had started it. When the endpoint failed to create them, the coordinator logged "failed to restart queues" and started workers that wait forever for an endpoint queue: the guest's rings stalled, the transport never marked the device as needing reset, and a resume after a snapshot restore reported success. Create the endpoint queues in `start_queue` when it completes the first queue pair, before the coordinator starts. A failure now fails the queue start like the device's other start failures: on activation the transport refuses DRIVER_OK and resets the device, and on resume it marks the device DEVICE_NEEDS_RESET. The coordinator still re-creates the queues when the endpoint requests a restart, and still re-establishes a latched input gate after creating them. The new test fails the endpoint's first queue creation: the start fails and leaves no queue started, and after a reset the pair starts again and transmits. It fails at the base revision, whose start succeeds. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- vm/devices/virtio/virtio_net/src/lib.rs | 28 ++- vm/devices/virtio/virtio_net/src/tests.rs | 1 + .../virtio/virtio_net/src/tests/activation.rs | 162 ++++++++++++++++++ 3 files changed, 190 insertions(+), 1 deletion(-) create mode 100644 vm/devices/virtio/virtio_net/src/tests/activation.rs diff --git a/vm/devices/virtio/virtio_net/src/lib.rs b/vm/devices/virtio/virtio_net/src/lib.rs index 82aeca71adc..0ea1dd8069b 100644 --- a/vm/devices/virtio/virtio_net/src/lib.rs +++ b/vm/devices/virtio/virtio_net/src/lib.rs @@ -466,6 +466,11 @@ impl VirtioDevice for Device { ); if first_pair { + if let Err(err) = self.create_endpoint_queues().await { + self.coordinator.remove(); + self.pairs[pair_idx] = QueuePairState::Empty; + return Err(err); + } self.coordinator.start(); } } @@ -748,11 +753,27 @@ impl Device { num_queues, enabled_rx_offloads: negotiated_rx_offload_support(negotiated_features), restart: true, + endpoint_queues_created: false, input_quiesced: self.lifecycle.input_quiesced, }, ); } + /// Creates the endpoint queues of the coordinator that `start_queue` has + /// just inserted, before the coordinator starts, so that an endpoint that + /// cannot provide them fails the queue start instead of leaving the device + /// without endpoint queues. + async fn create_endpoint_queues(&mut self) -> anyhow::Result<()> { + let (state, coordinator) = self.coordinator.get_mut(); + let coordinator = coordinator.context("virtio-net coordinator state is unavailable")?; + coordinator + .restart_queues(state) + .await + .context("failed to create the network endpoint queues")?; + coordinator.endpoint_queues_created = true; + Ok(()) + } + /// Allocates and inserts a worker. /// /// The coordinator must be stopped. @@ -789,6 +810,9 @@ struct Coordinator { num_queues: u16, enabled_rx_offloads: RxOffloadSupport, restart: bool, + /// Whether `start_queue` has created the endpoint queues that the first + /// restart would otherwise create. + endpoint_queues_created: bool, input_quiesced: bool, } @@ -844,7 +868,9 @@ impl Coordinator { stop.until_stopped(self.stop_workers()).await?; // The queue restart operation is not restartable, so do not // poll on `stop` here. - if let Err(err) = self.restart_queues(state).await { + if !std::mem::take(&mut self.endpoint_queues_created) + && let Err(err) = self.restart_queues(state).await + { tracing::error!( error = &err as &dyn std::error::Error, "failed to restart queues" diff --git a/vm/devices/virtio/virtio_net/src/tests.rs b/vm/devices/virtio/virtio_net/src/tests.rs index d107600c4d6..1fbb436fd39 100644 --- a/vm/devices/virtio/virtio_net/src/tests.rs +++ b/vm/devices/virtio/virtio_net/src/tests.rs @@ -1,6 +1,7 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT License. +mod activation; mod egress; mod harness; mod quiesce; diff --git a/vm/devices/virtio/virtio_net/src/tests/activation.rs b/vm/devices/virtio/virtio_net/src/tests/activation.rs new file mode 100644 index 00000000000..df2df4cbf9d --- /dev/null +++ b/vm/devices/virtio/virtio_net/src/tests/activation.rs @@ -0,0 +1,162 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +//! Activation tests: a queue pair whose endpoint cannot create its queues +//! fails its start, instead of leaving the device without endpoint queues. + +use super::MockEndpoint; +use super::QUEUE_SIZE; +use super::TX_AVAIL_ADDR; +use super::TX_DESC_ADDR; +use super::TX_USED_ADDR; +use super::TestHarness; +use crate::Device; +use async_trait::async_trait; +use inspect::InspectMut; +use net_backend::Endpoint; +use net_backend::EndpointAction; +use net_backend::MultiQueueSupport; +use net_backend::QueueConfig; +use net_backend::RssConfig; +use net_backend::TxOffloadSupport; +use net_backend_resources::mac_address::MacAddress; +use pal_async::DefaultDriver; +use pal_async::async_test; +use test_with_tracing::test; +use virtio::QueueResources; +use virtio::VirtioDevice; +use virtio::queue::QueueParams; +use virtio::spec::VirtioDeviceFeatures; +use vmcore::interrupt::Interrupt; +use vmcore::vm_task::SingleDriverBackend; +use vmcore::vm_task::VmTaskDriverSource; + +/// A [`MockEndpoint`] that fails to create queues the first `failures` times. +struct FailingEndpoint { + endpoint: MockEndpoint, + failures: usize, +} + +impl InspectMut for FailingEndpoint { + fn inspect_mut(&mut self, req: inspect::Request<'_>) { + self.endpoint.inspect_mut(req); + } +} + +#[async_trait] +impl Endpoint for FailingEndpoint { + fn endpoint_type(&self) -> &'static str { + self.endpoint.endpoint_type() + } + + async fn get_queues( + &mut self, + config: Vec, + rss: Option<&RssConfig<'_>>, + queues: &mut Vec>, + ) -> anyhow::Result<()> { + if self.failures > 0 { + self.failures -= 1; + anyhow::bail!("injected endpoint queue failure"); + } + self.endpoint.get_queues(config, rss, queues).await + } + + async fn stop(&mut self) { + self.endpoint.stop().await + } + + fn is_ordered(&self) -> bool { + self.endpoint.is_ordered() + } + + fn tx_offload_support(&self) -> TxOffloadSupport { + self.endpoint.tx_offload_support() + } + + fn multiqueue_support(&self) -> MultiQueueSupport { + self.endpoint.multiqueue_support() + } + + fn tx_fast_completions(&self) -> bool { + self.endpoint.tx_fast_completions() + } + + async fn wait_for_endpoint_action(&mut self) -> EndpointAction { + self.endpoint.wait_for_endpoint_action().await + } +} + +impl TestHarness { + /// Replaces the harness device with one whose endpoint fails to create + /// queues the first `failures` times. + fn new_with_endpoint_failures(driver: &DefaultDriver, failures: usize) -> Self { + let mut harness = Self::new(driver); + let (queue_tx, queue_handle_rx) = mesh::channel(); + let (queue_config_tx, queue_config_rx) = mesh::channel(); + let (endpoint_action_tx, endpoint_action_rx) = mesh::channel(); + let endpoint = FailingEndpoint { + endpoint: MockEndpoint { + queue_tx, + queue_config_tx, + endpoint_action_rx, + is_ordered: true, + }, + failures, + }; + let driver_source = VmTaskDriverSource::new(SingleDriverBackend::new(driver.clone())); + let mac = MacAddress::new([0x00, 0x15, 0x5d, 0xaa, 0xbb, 0xcc]); + harness.device = Device::builder() + .build(&driver_source, Box::new(endpoint), mac) + .unwrap(); + harness.queue_handle_rx = queue_handle_rx; + harness.queue_config_rx = queue_config_rx; + harness.endpoint_action_tx = endpoint_action_tx; + harness + } + + fn tx_queue_resources(&self) -> QueueResources { + QueueResources { + params: QueueParams { + size: QUEUE_SIZE, + enable: true, + desc_addr: TX_DESC_ADDR, + avail_addr: TX_AVAIL_ADDR, + used_addr: TX_USED_ADDR, + }, + notify: Interrupt::from_event(self.tx_interrupt_event.clone()), + event: self.tx_event.clone(), + guest_memory: self.mem.clone(), + } + } +} + +#[async_test] +async fn endpoint_queue_failure_fails_the_queue_start(driver: DefaultDriver) { + let mut harness = TestHarness::new_with_endpoint_failures(&driver, 1); + let features = VirtioDeviceFeatures::new(); + + // The TX queue completes the pair, whose endpoint queues fail. + harness.start_rx_queue(&features).await; + let error = harness + .device + .start_queue(1, harness.tx_queue_resources(), &features, None) + .await + .unwrap_err(); + assert!( + format!("{error:#}").contains("injected endpoint queue failure"), + "{error:#}" + ); + + // No queue of the failed pair remains started. + assert!(harness.device.stop_queue(0).await.is_none()); + assert!(harness.device.stop_queue(1).await.is_none()); + + // After the reset that follows a failed start, the pair starts again and + // transmits. + harness.device.reset().await; + let _handle = harness.enable_and_get_handle().await; + harness.post_tx_and_signal(0, 64); + let (used_id, _) = harness.wait_for_used().await; + assert_eq!(used_id, 0); +}