Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 9 additions & 0 deletions Guide/src/reference/devices/firmware/linux_direct.md
Original file line number Diff line number Diff line change
Expand Up @@ -70,6 +70,15 @@ hole and the live shared-status page in e820, and leaves the fixed 3-to-4-GiB
MMIO aperture out of RAM. The loader enters the ELF kernel in long mode with
`RSI=0x2000`, `CR3=0x4000`, and paging enabled.

The microVM virtio devices may reach only the RAM ranges of this e820 map with
DMA, which after a restore include the RAM that the restore adds. A device
refuses a queue whose rings, or a descriptor whose buffer, lies even partly
outside them, such as in the shared-status page, in the ISA hole, or across the
aperture: it reads and writes none of it, and its transport sets
`DEVICE_NEEDS_RESET` in the device status, with a configuration-change
notification if the driver has set `DRIVER_OK`. A refused ring fails
`DRIVER_OK`. Resetting the device clears the mark.

## AArch64 Boot Flow

On AArch64, OpenVMM supports two modes for presenting hardware descriptions to
Expand Down
2 changes: 1 addition & 1 deletion openvmm/openvmm_core/src/worker/dispatch.rs
Original file line number Diff line number Diff line change
Expand Up @@ -3290,7 +3290,7 @@ impl InitializedVm {
let mut pci_device_number = 10;
let mut virtio_mmio_index = 0;
let mut microvm_virtio_slots =
microvm::VirtioMmioSlots::new(&cfg.microvm.sandbox_blocks, chipset_mmio);
microvm::VirtioMmioSlots::new(&cfg.microvm.sandbox_blocks, chipset_mmio, &mem_layout);

// Avoid an ISA interrupt to avoid conflicts and to avoid needing to
// configure the line as level-triggered in the MADT (necessary for
Expand Down
55 changes: 48 additions & 7 deletions openvmm/openvmm_core/src/worker/dispatch/microvm.rs
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,7 @@ use virtio::VirtioMmioInterruptMode;
use virtio::resolve::ResolvedVirtioDevice;
#[cfg(guest_arch = "x86_64")]
use vm_loader::InitialLoad;
use vm_topology::memory::MemoryLayout;
use vmcore::vm_task::VmTaskDriverSource;
use vmm_core::partition_unit::StopGuard;

Expand Down Expand Up @@ -172,11 +173,7 @@ pub(super) fn load_linux_x86_mptable(
.vps_arch()
.map(|vp| vp.apic_id)
.collect::<Vec<_>>();
let reserved_memory_ranges = [MemoryRange::new(
openvmm_defs::microvm::MICROVM_SHARED_STATUS_PAGE_GPA
..openvmm_defs::microvm::MICROVM_SHARED_STATUS_PAGE_GPA
+ openvmm_defs::microvm::MICROVM_SHARED_STATUS_PAGE_SIZE,
)];
let reserved_memory_ranges = reserved_memory_ranges();
let kernel_config = crate::worker::vm_loaders::linux::KernelConfig {
kernel,
initrd,
Expand Down Expand Up @@ -539,29 +536,63 @@ fn virtio_mmio_config(
Ok((start, len, irq, disabled_features, interrupt_mode))
}

/// The guest memory that the microVM e820 map reserves inside RAM besides the
/// ISA hole: the shared virtio interrupt-status page.
fn reserved_memory_ranges() -> [MemoryRange; 1] {
[MemoryRange::new(
openvmm_defs::microvm::MICROVM_SHARED_STATUS_PAGE_GPA
..openvmm_defs::microvm::MICROVM_SHARED_STATUS_PAGE_GPA
+ openvmm_defs::microvm::MICROVM_SHARED_STATUS_PAGE_SIZE,
)]
}

/// Assigns the fixed virtio-mmio slots of the microVM profile to devices, in
/// device order.
pub(super) struct VirtioMmioSlots<'a> {
sandbox_blocks: &'a [openvmm_defs::microvm::MicrovmSandboxBlockConfig],
sandbox_block_index: usize,
filesystem_index: usize,
chipset_mmio: ChipsetMmioRanges,
mem_layout: &'a MemoryLayout,
/// The RAM ranges of the e820 map, computed for the first device.
dma_ranges: Option<Vec<MemoryRange>>,
}

impl<'a> VirtioMmioSlots<'a> {
pub(super) fn new(
sandbox_blocks: &'a [openvmm_defs::microvm::MicrovmSandboxBlockConfig],
chipset_mmio: ChipsetMmioRanges,
mem_layout: &'a MemoryLayout,
) -> Self {
Self {
sandbox_blocks,
sandbox_block_index: 0,
filesystem_index: 0,
chipset_mmio,
mem_layout,
dma_ranges: None,
}
}

/// Returns the guest memory that a microVM device may access with DMA:
/// the RAM ranges of the e820 map, which leave out the shared
/// interrupt-status page, the ISA hole, and the MMIO aperture. After a
/// restore, they include the RAM that the restore adds.
fn dma_ranges(&mut self) -> anyhow::Result<&[MemoryRange]> {
if self.dma_ranges.is_none() {
self.dma_ranges = Some(
loader::linux::microvm::mptable_ram_ranges(
self.mem_layout,
&reserved_memory_ranges(),
)
.context("failed to find the RAM ranges of the microVM e820 map")?,
);
}
Ok(self.dma_ranges.as_deref().unwrap_or_default())
}

/// Adds a virtio-mmio device at its fixed microVM slot.
/// Adds a virtio-mmio device at its fixed microVM slot, restricting its
/// DMA to the RAM ranges of the e820 map.
pub(super) fn add_device(
&mut self,
chipset_builder: &vmotherboard::ChipsetBuilder<'_>,
Expand All @@ -578,10 +609,15 @@ impl<'a> VirtioMmioSlots<'a> {
&mut self.filesystem_index,
self.chipset_mmio,
)?;
let dma_ranges = self
.dma_ranges()?
.iter()
.map(|range| range.start()..range.end())
.collect::<Vec<_>>();
let id = format!("{id}-{mmio_start}");
let gm = gm.clone();
chipset_builder.arc_mutex_device(id).try_add(|services| {
VirtioMmioDevice::new_with_disabled_features_and_interrupt_mode(
let mut device = VirtioMmioDevice::new_with_disabled_features_and_interrupt_mode(
device.0,
&driver_source.simple(),
gm,
Expand All @@ -592,6 +628,11 @@ impl<'a> VirtioMmioSlots<'a> {
disabled_features,
interrupt_mode,
)
.context("failed to create the virtio-mmio transport")?;
device
.restrict_dma(dma_ranges)
.context("failed to restrict device DMA to RAM")?;
anyhow::Ok(device)
})?;
Ok(())
}
Expand Down
14 changes: 14 additions & 0 deletions vm/devices/virtio/virtio/src/queue.rs
Original file line number Diff line number Diff line change
Expand Up @@ -103,6 +103,8 @@ pub struct QueueState {
pub enum QueueError {
#[error("error accessing queue memory")]
Memory(#[source] GuestMemoryError),
#[error("a descriptor addresses memory that the device may not access")]
BufferNotAdmitted(#[source] GuestMemoryError),
#[error("an indirect descriptor had the indirect flag set")]
DoubleIndirect,
#[error("a descriptor chain is too long or has a cycle")]
Expand Down Expand Up @@ -539,6 +541,15 @@ impl QueueCoreGetWork {
Ok(descriptor)
}

/// Fails unless the device memory admits the whole buffer of
/// `descriptor`, so that the device reads and writes none of a buffer that
/// lies even partly outside it.
fn check_admitted(&self, descriptor: &QueueDescriptor) -> Result<(), QueueError> {
self.mem
.check_admitted(descriptor.address, descriptor.length.into())
.map_err(QueueError::BufferNotAdmitted)
}

fn size(&self) -> u16 {
self.queue_size
}
Expand Down Expand Up @@ -696,6 +707,7 @@ impl<'a> DescriptorChain<'a> {
if self.indirect_queue.is_some() {
return Err(QueueError::DoubleIndirect);
}
self.queue.check_admitted(&descriptor)?;
let indirect_queue = self.indirect_queue.insert(
self.queue
.mem
Expand All @@ -708,6 +720,8 @@ impl<'a> DescriptorChain<'a> {
self.queue
.descriptor(indirect_queue, 0, Some(indirect_len))?
};
// Refuse the chain before the device reads or writes any of it.
self.queue.check_admitted(&descriptor)?;

self.num_read += 1;
self.descriptor_index = descriptor.next;
Expand Down
3 changes: 2 additions & 1 deletion vm/devices/virtio/virtio/src/tests.rs
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@
#![expect(unsafe_code)]
#![cfg(test)]

mod dma;
mod interrupt;
mod saved_state;

Expand Down Expand Up @@ -82,7 +83,7 @@ const VIRTIO_ACKNOWLEDGE: u32 = 1;
const VIRTIO_DRIVER: u32 = 2;
const VIRTIO_DRIVER_OK: u32 = 4;
const VIRTIO_FEATURES_OK: u32 = 8;
const _VIRTIO_DEVICE_NEEDS_RESET: u32 = 0x40;
const VIRTIO_DEVICE_NEEDS_RESET: u32 = 0x40;
const _VIRTIO_FAILED: u32 = 0x80;

async fn must_recv_in_timeout<T: 'static + Send>(
Expand Down
Loading
Loading