From f10eaca613f6eef28add5d69f4fbf573ca6ed156 Mon Sep 17 00:00:00 2001 From: Aaron Paterson Date: Sat, 1 Aug 2026 18:48:34 +0000 Subject: [PATCH 1/8] Resolve test directory paths before reaping per-test files The reaper that runs at a concurrency of one deletes everything in the test directory except the unpacked rootfs it means to preserve. It held that rootfs path as FileManager reports it, under /var, and compared it against the entries of contentsOfDirectory, which reports them under /private/var, so the preserved path never matched any entry and the rootfs was deleted along with the per-test files. The unpack coordinator still held it as unpacked, so the next test opened a rootfs that was no longer there and failed with a missing file error. Resolve both sides with resolvingSymlinksInPathWithPrivate, which exists for this difference between the two views of the same directory. --- Sources/Integration/Suite.swift | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/Sources/Integration/Suite.swift b/Sources/Integration/Suite.swift index 9dae8435..e43cf372 100644 --- a/Sources/Integration/Suite.swift +++ b/Sources/Integration/Suite.swift @@ -259,12 +259,16 @@ struct IntegrationSuite: AsyncParsableCommand { // a ~2GB rootfs and a ~512MB initfs, so without reaping the dev // container fills its CoW layer in ~10 tests. if self.maxConcurrency == 1 { - let preserve = fsPath.absolutePath() + // contentsOfDirectory reports paths under /private, while testDir is + // built from FileManager's /var view of the same directory, so both + // sides are resolved before comparing. + let preserve = fsPath.resolvingSymlinksInPathWithPrivate().absolutePath() if let entries = try? FileManager.default.contentsOfDirectory( at: Self.testDir, includingPropertiesForKeys: nil ) { - for url in entries where url.absolutePath() != preserve { + for url in entries + where url.resolvingSymlinksInPathWithPrivate().absolutePath() != preserve { try? FileManager.default.removeItem(at: url) } } From 6499f7dfa7fff64c903642db690094d7e6c64ec8 Mon Sep 17 00:00:00 2001 From: Aaron Paterson Date: Sun, 2 Aug 2026 00:51:10 +0000 Subject: [PATCH 2/8] Say how to run the tests that macOS skips Tests guarded for Linux are compiled out on macOS, so `make test` reports success without having run them, and nothing says so. The target that does run them is not mentioned anywhere outside the makefile. --- README.md | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/README.md b/README.md index f658b08d..372a872a 100644 --- a/README.md +++ b/README.md @@ -130,6 +130,16 @@ After building, run basic and integration tests: make test integration ``` +Tests that only apply to Linux are compiled out on macOS, so `make test` passes +without running them. Run those with: + +```bash +make linux-test +``` + +which runs the same tests inside the Linux dev container, as the Linux build +workflow does. + A kernel is required to run integration tests. If you do not have a kernel locally, a default kernel can be fetched using the `make fetch-default-kernel` target. From bf4d7dcf8bbe64c745e9ef344a8c173d2a42a3af Mon Sep 17 00:00:00 2001 From: Aaron Paterson Date: Sun, 2 Aug 2026 01:20:29 +0000 Subject: [PATCH 3/8] Test against the kernel release the shipped tool installs The default kernel the tests fetch had drifted from the one users run. container installs kata 3.28.0 and its 6.18.15 kernel, while these tests fetched 3.17.0, so the suite exercised a guest with a different feature set to the one it is meant to represent. Nested runtimes are the visible case: 3.17.0 was built without nf_tables, so a docker daemon inside a container fails there and works on what ships. Kata moved from xz to zstd between those releases, so the archive is no longer named for its compression and tar is left to recognise it rather than being told, which also holds if the format changes again. --- Makefile | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Makefile b/Makefile index c79d6995..3d01b93a 100644 --- a/Makefile +++ b/Makefile @@ -64,7 +64,7 @@ LIBARCHIVE_UPSTREAM_REPO := https://github.com/libarchive/libarchive LIBARCHIVE_UPSTREAM_VERSION := v3.7.7 LIBARCHIVE_LOCAL_DIR := workdir/libarchive -KATA_BINARY_PACKAGE := https://github.com/kata-containers/kata-containers/releases/download/3.17.0/kata-static-3.17.0-arm64.tar.xz +KATA_BINARY_PACKAGE := https://github.com/kata-containers/kata-containers/releases/download/3.28.0/kata-static-3.28.0-arm64.tar.zst CLOUD_HYPERVISOR_URL := https://github.com/cloud-hypervisor/cloud-hypervisor/releases/download/v52.0/cloud-hypervisor-static-aarch64 # SHA256 of the v52.0 aarch64 static binary (verified locally from the # upstream release artifact). Bump alongside CLOUD_HYPERVISOR_URL. @@ -410,11 +410,11 @@ integration: .PHONY: fetch-default-kernel fetch-default-kernel: @mkdir -p .local/ bin/ -ifeq (,$(wildcard .local/kata.tar.gz)) - @curl -SsL -o .local/kata.tar.gz ${KATA_BINARY_PACKAGE} +ifeq (,$(wildcard .local/kata.tar)) + @curl -SsL -o .local/kata.tar ${KATA_BINARY_PACKAGE} endif ifeq (,$(wildcard .local/vmlinux-$(KERNEL_ARCH))) - @tar -zxf .local/kata.tar.gz -C .local/ --strip-components=1 + @tar -xf .local/kata.tar -C .local/ --strip-components=1 @cp -L .local/opt/kata/share/kata-containers/vmlinux.container .local/vmlinux-$(KERNEL_ARCH) endif ifeq (,$(wildcard bin/vmlinux-$(KERNEL_ARCH))) From 21f1f16eaaacc051bbc5be6566814cb606de5698 Mon Sep 17 00:00:00 2001 From: Aaron Paterson Date: Wed, 12 Aug 2026 05:25:08 +0000 Subject: [PATCH 4/8] Run each integration test in its own scratch directory A test's rootfs clones, init block, and writable layers land in a directory the runner creates for it and deletes when it finishes, so a test holds disk only while it runs at any concurrency. The run-shared directory holds the unpacked images every bootstrap clones from, and is removed when the run ends. --- Sources/Integration/PodTests.swift | 7 +++-- Sources/Integration/Suite.swift | 43 +++++++++++++----------------- 2 files changed, 21 insertions(+), 29 deletions(-) diff --git a/Sources/Integration/PodTests.swift b/Sources/Integration/PodTests.swift index ae1caec8..7dff20fa 100644 --- a/Sources/Integration/PodTests.swift +++ b/Sources/Integration/PodTests.swift @@ -2228,10 +2228,9 @@ extension IntegrationSuite { /// (directory-share) rootfs. Assumes a single-layer image (the alpine /// image used by the suite) so no OCI whiteout processing is required. /// - /// The extracted dir lives under `Self.testDir`; do NOT `defer`-remove it - /// here — virtiofsd shares it for the whole test. It is swept by - /// `bootstrap`'s `maxConcurrency == 1` reaper on the next test and by the - /// suite-end `removeItem(at: Self.testDir)`. + /// The extracted dir lives under the test's scratch directory; do NOT + /// `defer`-remove it here — virtiofsd shares it for the whole test. The + /// runner removes the scratch directory when the test finishes. private func unpackRootfsDirectory(_ image: Containerization.Image, testID: String) async throws -> Containerization.Mount { let dir = Self.testDir.appending(component: "\(testID)-rootfs-dir") try? FileManager.default.removeItem(at: dir) diff --git a/Sources/Integration/Suite.swift b/Sources/Integration/Suite.swift index e43cf372..84580dba 100644 --- a/Sources/Integration/Suite.swift +++ b/Sources/Integration/Suite.swift @@ -135,7 +135,19 @@ struct IntegrationSuite: AsyncParsableCommand { FileManager.default.uniqueTemporaryDirectory(create: true) }() + /// The scratch directory of the test running on this task. The runner + /// creates one per test and deletes it when the test finishes, so a + /// test's clones and layers hold disk only while it runs. + @TaskLocal static var currentTestDir: URL? + static var testDir: URL { + currentTestDir ?? _testDir + } + + /// The run-shared directory holding unpacked images, which every test's + /// bootstrap clones from. It lives for the whole run and is removed at + /// the end. + static var imageCacheDir: URL { _testDir } @@ -233,7 +245,7 @@ struct IntegrationSuite: AsyncParsableCommand { let platform = Platform(arch: "arm64", os: "linux", variant: "v8") // Unpack to shared location with coordination to prevent concurrent unpacks - let fsPath = Self.testDir.appending(component: image.digest) + let fsPath = Self.imageCacheDir.appending(component: image.digest) let fs = try await Self.unpackCoordinator.unpack(key: fsPath.absolutePath()) { do { let unpacker = EXT4Unpacker(capacityInBytes: 2.gib()) @@ -251,29 +263,6 @@ struct IntegrationSuite: AsyncParsableCommand { } } - // Reap any per-test artifacts left over from prior tests. With - // `--max-concurrency 1` (linux-integration default) this runs after - // the previous test has fully completed, so it's race-free; on - // macOS where tests can run in parallel we just keep all files — - // disk usage isn't a concern there. Each per-test bootstrap clones - // a ~2GB rootfs and a ~512MB initfs, so without reaping the dev - // container fills its CoW layer in ~10 tests. - if self.maxConcurrency == 1 { - // contentsOfDirectory reports paths under /private, while testDir is - // built from FileManager's /var view of the same directory, so both - // sides are resolved before comparing. - let preserve = fsPath.resolvingSymlinksInPathWithPrivate().absolutePath() - if let entries = try? FileManager.default.contentsOfDirectory( - at: Self.testDir, - includingPropertiesForKeys: nil - ) { - for url in entries - where url.resolvingSymlinksInPathWithPrivate().absolutePath() != preserve { - try? FileManager.default.removeItem(at: url) - } - } - } - // Clone to test-specific path let clPath = Self.testDir.appending(component: "\(testID).ext4").absolutePath() try? FileManager.default.removeItem(atPath: clPath) @@ -658,11 +647,14 @@ struct IntegrationSuite: AsyncParsableCommand { for _ in 0.. Date: Wed, 12 Aug 2026 15:00:44 +0000 Subject: [PATCH 5/8] Give the machine's storage the shape its containers have A container's storage is its rootfs, an optional writable layer, and its mounts; a machine's is its containers' plus the volumes they share. One generic shape describes both the Mount values a machine is configured with and the AttachedFilesystem values it reports once attached, so converting between the two is a map over the structure and the roles cannot drift between configuration and attachment. Device addresses are allocated walking the same sorted order the devices are created in, on both backends. Consumers read roles instead of list positions: the spec builders take a container's mounts without prefix arithmetic, pod volumes mount by name, and a cloud-hypervisor volume's virtiofsd is held by a machine-lifetime reference alongside its containers' reference counts. --- .../Containerization/CHHotplugProvider.swift | 79 ++++++----- .../CHVirtualMachineInstance.swift | 127 ++++++++++-------- .../CHVirtualMachineManager.swift | 2 +- .../Containerization/ContainerStorage.swift | 107 +++++++++++++++ .../Containerization/HotplugProvider.swift | 2 +- Sources/Containerization/LinuxContainer.swift | 51 +++---- Sources/Containerization/LinuxPod.swift | 98 +++++++------- .../Containerization/VMConfiguration.swift | 10 +- .../VZVirtualMachineInstance.swift | 100 +++++++------- .../VZVirtualMachineManager.swift | 2 +- .../VirtualMachineInstance.swift | 5 +- 11 files changed, 351 insertions(+), 232 deletions(-) create mode 100644 Sources/Containerization/ContainerStorage.swift diff --git a/Sources/Containerization/CHHotplugProvider.swift b/Sources/Containerization/CHHotplugProvider.swift index 870fb1ff..380cba76 100644 --- a/Sources/Containerization/CHHotplugProvider.swift +++ b/Sources/Containerization/CHHotplugProvider.swift @@ -27,8 +27,8 @@ import Synchronization /// /// Handles both block (`vm.add-disk`) and virtiofs (`vm.add-fs`, with one /// `virtiofsd` per unique source-hash tag) hotplug, plus the matching -/// `vm.remove-device` teardown. Owns the per-VM mount registry so -/// `CHVirtualMachineInstance.mounts` can forward to it. +/// `vm.remove-device` teardown. Owns the machine's storage so +/// `CHVirtualMachineInstance.storage` can forward to it. final class CHHotplugProvider: HotplugProvider { struct HotplugRecord: Sendable { let chDeviceId: String @@ -50,7 +50,7 @@ final class CHHotplugProvider: HotplugProvider { private let workDir: URL private let virtiofsdBinaryOverride: URL? private let allocator: any AddressAllocator - private let _mounts: Mutex<[String: [AttachedFilesystem]]> + private let _storage: Mutex private let _records: Mutex<[String: [HotplugRecord]]> private let _tags: Mutex<[String: VirtiofsdTagState]> /// Serializes per-tag virtiofsd spawn so a concurrent hotplug for the @@ -65,14 +65,14 @@ final class CHHotplugProvider: HotplugProvider { workDir: URL, virtiofsdBinary: URL?, allocator: any AddressAllocator, - initialMounts: [String: [AttachedFilesystem]], + initialStorage: MachineAttachments, logger: Logger? ) { self.client = client self.workDir = workDir self.virtiofsdBinaryOverride = virtiofsdBinary self.allocator = allocator - self._mounts = Mutex(initialMounts) + self._storage = Mutex(initialStorage) self._records = Mutex([:]) self._tags = Mutex([:]) self.spawnLock = AsyncLock() @@ -81,14 +81,14 @@ final class CHHotplugProvider: HotplugProvider { // MARK: - Read accessors - var mounts: [String: [AttachedFilesystem]] { - _mounts.withLock { $0 } + var storage: MachineAttachments { + _storage.withLock { $0 } } - func withMountRegistry( - _ body: (inout sending [String: [AttachedFilesystem]]) throws -> sending T + func withStorage( + _ body: (inout sending MachineAttachments) throws -> sending T ) rethrows -> T { - try _mounts.withLock(body) + try _storage.withLock(body) } // MARK: - HotplugProvider conformance @@ -151,12 +151,13 @@ final class CHHotplugProvider: HotplugProvider { } func registerMounts(id: String, rootfs: AttachedFilesystem, additionalMounts: [Mount]) throws { - var attached: [AttachedFilesystem] = [rootfs] + var mounts: [AttachedFilesystem] = [] for mount in additionalMounts { - attached.append(try AttachedFilesystem(mount: mount, allocator: allocator)) + mounts.append(try AttachedFilesystem(mount: mount, allocator: allocator)) } - _mounts.withLock { - $0[id, default: []].append(contentsOf: attached) + let container = ContainerAttachments(rootfs: rootfs, mounts: mounts) + _storage.withLock { + $0.containers[id] = container } } @@ -190,19 +191,10 @@ final class CHHotplugProvider: HotplugProvider { } } - // Drop block-derived AttachedFilesystem entries for `id`. Block entries - // are the ones whose source was rewritten to "/dev/vd" by - // `hotplug(_:)` (or by AttachedFilesystem(mount:allocator:) for an - // additionalMount of type virtio-blk). - _mounts.withLock { state in - guard var perID = state[id] else { return } - perID.removeAll { $0.source.hasPrefix("/dev/vd") } - if perID.isEmpty { - state.removeValue(forKey: id) - } else { - state[id] = perID - } - } + // The container's devices are gone, so the container leaves the + // registry with them; its shares are released separately and their + // processes reference-counted through `_tags`. + _ = _storage.withLock { $0.containers.removeValue(forKey: id) } } func hotplugVirtioFS(_ mounts: [Mount], id: String) async throws { @@ -226,7 +218,7 @@ final class CHHotplugProvider: HotplugProvider { let chDeviceId = try await ensureVirtiofsDevice(tag: tag, source: source, readonly: readonly) // Record once per tag for this container. The AttachedFilesystem // entries for these mounts are written by registerMounts (the sole - // _mounts writer), so we do NOT touch _mounts here. + // registry writer), so we do NOT touch the storage here. _records.withLock { $0[id, default: []].append(HotplugRecord(chDeviceId: chDeviceId, kind: .virtiofs(tag: tag))) } @@ -338,16 +330,17 @@ final class CHHotplugProvider: HotplugProvider { try? FileManager.default.removeItem(at: socket) } - // Drop virtiofs AttachedFilesystem entries for `id`. AttachedFilesystem - // sets `type = mount.type` which for a `.virtiofs` mount is "virtiofs". - _mounts.withLock { state in - guard var perID = state[id] else { return } - perID.removeAll { $0.type == "virtiofs" } - if perID.isEmpty { - state.removeValue(forKey: id) - } else { - state[id] = perID + // Drop the container's virtiofs entries. A container whose rootfs is + // itself a share leaves the registry whole; one that keeps block + // devices keeps its entry with the share entries dropped. + _storage.withLock { state in + guard var container = state.containers[id] else { return } + if container.rootfs.type == "virtiofs" { + state.containers.removeValue(forKey: id) + return } + container.mounts.removeAll { $0.type == "virtiofs" } + state.containers[id] = container } } @@ -358,15 +351,21 @@ final class CHHotplugProvider: HotplugProvider { /// is the user-supplied `FsConfig.id` (which `vm.remove-device` keys on). /// `ownerIds` are the container ids that count toward this tag's refcount; /// each gets a `HotplugRecord` so `releaseVirtioFS(id:)` walks them - /// uniformly. + /// uniformly. `machineHeld` adds one reference nothing releases, for a + /// share the machine itself owns (a volume). func recordBootTimeVirtiofs( tag: String, process: VirtiofsdProcess, chDeviceId: String, - ownerIds: [String] + ownerIds: [String], + machineHeld: Bool ) { _tags.withLock { - $0[tag] = VirtiofsdTagState(process: process, refcount: ownerIds.count, chDeviceId: chDeviceId) + $0[tag] = VirtiofsdTagState( + process: process, + refcount: ownerIds.count + (machineHeld ? 1 : 0), + chDeviceId: chDeviceId + ) } _records.withLock { records in for id in ownerIds { diff --git a/Sources/Containerization/CHVirtualMachineInstance.swift b/Sources/Containerization/CHVirtualMachineInstance.swift index 122f592a..e363d5ca 100644 --- a/Sources/Containerization/CHVirtualMachineInstance.swift +++ b/Sources/Containerization/CHVirtualMachineInstance.swift @@ -46,7 +46,7 @@ public final class CHVirtualMachineInstance: Sendable { public struct Configuration: Sendable { public var cpus: Int public var memoryInBytes: UInt64 - public var mountsByID: [String: [Mount]] + public var storage: MachineMounts public var interfaces: [any Interface] public var kernel: Kernel? public var initialFilesystem: Mount? @@ -56,7 +56,7 @@ public final class CHVirtualMachineInstance: Sendable { public init() { self.cpus = 4 self.memoryInBytes = 1024 * 1024 * 1024 - self.mountsByID = [:] + self.storage = MachineMounts() self.interfaces = [] } } @@ -65,8 +65,8 @@ public final class CHVirtualMachineInstance: Sendable { /// `start()`'s `VmConfig.disks` ordering matches the allocator letters. struct BootDisk: Sendable { let mount: Mount - let containerId: String? // nil for rootfs - let letter: Character + let chId: String + let readonly: Bool } // MARK: - State @@ -76,8 +76,8 @@ public final class CHVirtualMachineInstance: Sendable { _state.withLock { $0 } } - public var mounts: [String: [AttachedFilesystem]] { - hotplug.mounts + public var storage: MachineAttachments { + hotplug.storage } /// Cloud-hypervisor exposes one virtio-fs device per source-hash tag, so @@ -167,9 +167,9 @@ public final class CHVirtualMachineInstance: Sendable { ) self.workDir = workDir - // 2. Block allocator + boot inventory. Walks rootfs first, then - // mountsByID sorted by container id, allocating disk letters in - // that order. The same allocator is later handed to the hotplug + // 2. Block allocator + boot inventory. Walks rootfs first, then the + // machine's storage in its device order, allocating disk letters + // as it goes. The same allocator is later handed to the hotplug // provider so runtime add-disk picks up where boot wiring left off. let allocator = Character.blockDeviceTagAllocator() let inventory = try config.bootInventory(allocator: allocator) @@ -201,14 +201,14 @@ public final class CHVirtualMachineInstance: Sendable { logger: logger ?? Logger(label: "CloudHypervisor.Client") ) - // 5. Hotplug provider — owns the mount registry, seeded with the + // 5. Hotplug provider — owns the machine's storage, seeded with the // boot inventory so registerMounts can append to it. self.hotplug = CHHotplugProvider( client: self.client, workDir: workDir, virtiofsdBinary: virtiofsdBinary, allocator: allocator, - initialMounts: inventory.attachments, + initialStorage: inventory.attachments, logger: logger ) @@ -222,11 +222,11 @@ public final class CHVirtualMachineInstance: Sendable { self._stdioPool = Mutex([:]) } - /// Mutate the mount registry. Forwards to the hotplug provider, which + /// Mutate the storage registry. Forwards to the hotplug provider, which /// owns the registry. Kept on the instance for parity with the macOS - /// path's `withMountRegistry` API. - func withMountRegistry(_ body: (inout sending [String: [AttachedFilesystem]]) throws -> sending T) rethrows -> T { - try hotplug.withMountRegistry(body) + /// path's `withStorage` API. + func withStorage(_ body: (inout sending MachineAttachments) throws -> sending T) rethrows -> T { + try hotplug.withStorage(body) } } @@ -510,34 +510,46 @@ extension CHVirtualMachineInstance { throw ContainerizationError(.invalidArgument, message: "initialFilesystem is required for cloud-hypervisor backend") } - // Disks: rootfs forced read-only at the device level; container disks + // Disks: rootfs forced read-only at the device level; other disks // honor their `ro` option through chDiskConfig. var disks: [CloudHypervisor.DiskConfig] = [] for bd in bootDisks { - let chId = bd.containerId.map { "blk-\($0)-\(bd.letter)" } ?? "rootfs" - if var disk = bd.mount.chDiskConfig(id: chId) { - if bd.containerId == nil { + if var disk = bd.mount.chDiskConfig(id: bd.chId) { + if bd.readonly { disk.readonly = true } disks.append(disk) } } - // Virtiofs: group all .virtiofs mounts in mountsByID by source-hash - // tag, spawn one virtiofsd per tag, build matching FsConfigs. - var byTag: [String: (mounts: [Mount], owners: [String])] = [:] - for cid in config.mountsByID.keys.sorted() { - guard let mounts = config.mountsByID[cid] else { continue } - for mount in mounts { - guard case .virtiofs = mount.runtimeOptions else { continue } - let tag = try hashFilePath(path: mount.source) - var entry = byTag[tag] ?? (mounts: [], owners: []) - entry.mounts.append(mount) - if !entry.owners.contains(cid) { - entry.owners.append(cid) + // Virtiofs: group all virtiofs mounts in the machine's storage by + // source-hash tag, spawn one virtiofsd per tag, build matching + // FsConfigs. A volume's tag is the machine's own, held for its + // lifetime. + var byTag: [String: (mounts: [Mount], owners: [String], machineHeld: Bool)] = [:] + func groupVirtiofs(_ mount: Mount, owner: String?) throws { + guard case .virtiofs = mount.runtimeOptions else { return } + let tag = try hashFilePath(path: mount.source) + var entry = byTag[tag] ?? (mounts: [], owners: [], machineHeld: false) + entry.mounts.append(mount) + if let owner { + if !entry.owners.contains(owner) { + entry.owners.append(owner) } - byTag[tag] = entry + } else { + entry.machineHeld = true } + byTag[tag] = entry + } + for cid in config.storage.containers.keys.sorted() { + guard let container = config.storage.containers[cid] else { continue } + for mount in container.all { + try groupVirtiofs(mount, owner: cid) + } + } + for name in config.storage.volumes.keys.sorted() { + guard let mount = config.storage.volumes[name] else { continue } + try groupVirtiofs(mount, owner: nil) } var fsConfigs: [CloudHypervisor.FsConfig] = [] @@ -569,7 +581,8 @@ extension CHVirtualMachineInstance { tag: tag, process: process, chDeviceId: chDeviceId, - ownerIds: entry.owners + ownerIds: entry.owners, + machineHeld: entry.machineHeld ) fsConfigs.append( @@ -714,40 +727,46 @@ extension CHVirtualMachineInstance { // MARK: - Boot inventory extension CHVirtualMachineInstance.Configuration { - /// Walks boot-time mounts in deterministic order (rootfs first, then - /// `mountsByID` sorted by container id, then each container's mounts in - /// input order), allocating disk letters for virtio-blk mounts and seeding - /// the per-container `AttachedFilesystem` registry. + /// Walks boot-time storage in the machine's device order (rootfs first, + /// then containers sorted by id, each in role order, then volumes sorted + /// by name), allocating disk letters for virtio-blk mounts and seeding + /// the machine's `AttachedFilesystem` registry. /// /// The allocator is shared with the runtime hotplug provider, so block /// hotplug picks up at the next free letter after boot. func bootInventory( allocator: any AddressAllocator - ) throws -> (attachments: [String: [AttachedFilesystem]], bootDisks: [CHVirtualMachineInstance.BootDisk]) { + ) throws -> (attachments: MachineAttachments, bootDisks: [CHVirtualMachineInstance.BootDisk]) { var bootDisks: [CHVirtualMachineInstance.BootDisk] = [] - var attachments: [String: [AttachedFilesystem]] = [:] - // Rootfs is not part of mountsByID. If it's a block device, it claims - // the first letter (vda) so the kernel cmdline `root=/dev/vda` is right. + // The rootfs is the machine's own, not part of its containers' + // storage. If it's a block device, it claims the first letter (vda) + // so the kernel cmdline `root=/dev/vda` is right. if let rootfs = self.initialFilesystem, rootfs.isBlock { - let letter = try allocator.allocate() - bootDisks.append(.init(mount: rootfs, containerId: nil, letter: letter)) + _ = try allocator.allocate() + bootDisks.append(.init(mount: rootfs, chId: "rootfs", readonly: true)) } - for cid in self.mountsByID.keys.sorted() { - guard let mounts = self.mountsByID[cid] else { continue } - var perContainer: [AttachedFilesystem] = [] - for mount in mounts { - let attached = try AttachedFilesystem(mount: mount, allocator: allocator) - if mount.isBlock, let letter = attached.source.last { - bootDisks.append(.init(mount: mount, containerId: cid, letter: letter)) - } - perContainer.append(attached) + func attach(_ mount: Mount, chId: (Character) -> String) throws -> AttachedFilesystem { + let attached = try AttachedFilesystem(mount: mount, allocator: allocator) + if mount.isBlock, let letter = attached.source.last { + bootDisks.append(.init(mount: mount, chId: chId(letter), readonly: false)) } - attachments[cid] = perContainer + return attached + } + + var containers: [String: ContainerAttachments] = [:] + for cid in self.storage.containers.keys.sorted() { + guard let container = self.storage.containers[cid] else { continue } + containers[cid] = try container.map { try attach($0, chId: { "blk-\(cid)-\($0)" }) } + } + var volumes: [String: AttachedFilesystem] = [:] + for name in self.storage.volumes.keys.sorted() { + guard let mount = self.storage.volumes[name] else { continue } + volumes[name] = try attach(mount, chId: { "vol-\(name)-\($0)" }) } - return (attachments, bootDisks) + return (MachineAttachments(containers: containers, volumes: volumes), bootDisks) } } #endif diff --git a/Sources/Containerization/CHVirtualMachineManager.swift b/Sources/Containerization/CHVirtualMachineManager.swift index ba20b5d1..c444620d 100644 --- a/Sources/Containerization/CHVirtualMachineManager.swift +++ b/Sources/Containerization/CHVirtualMachineManager.swift @@ -99,7 +99,7 @@ public struct CHVirtualMachineManager: VirtualMachineManager { instanceConfig.cpus = vmConfig.cpus instanceConfig.memoryInBytes = vmConfig.memoryInBytes instanceConfig.interfaces = vmConfig.interfaces - instanceConfig.mountsByID = vmConfig.mountsByID + instanceConfig.storage = vmConfig.storage instanceConfig.bootLog = vmConfig.bootLog instanceConfig.extensions = vmConfig.extensions instanceConfig.kernel = kernel diff --git a/Sources/Containerization/ContainerStorage.swift b/Sources/Containerization/ContainerStorage.swift new file mode 100644 index 00000000..4d8de2ed --- /dev/null +++ b/Sources/Containerization/ContainerStorage.swift @@ -0,0 +1,107 @@ +//===----------------------------------------------------------------------===// +// Copyright © 2026 Apple Inc. and the Containerization project authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//===----------------------------------------------------------------------===// + +/// A container's storage, by role. +/// +/// The same shape describes storage at both stages of its life: as the +/// `Mount` values a container is configured with, and as the +/// `AttachedFilesystem` values the machine reports once they are attached. +/// Converting between the two is a `map` over the structure, so the roles +/// cannot drift apart between configuration and attachment. +public struct ContainerStorage: Sendable { + /// The container's root filesystem. + public var rootfs: Value + + /// The writable layer mounted over the rootfs as the upper side of an + /// overlay, when the container has one. + public var writableLayer: Value? + + /// The container's remaining mounts, in configuration order. + public var mounts: [Value] + + public init(rootfs: Value, writableLayer: Value? = nil, mounts: [Value] = []) { + self.rootfs = rootfs + self.writableLayer = writableLayer + self.mounts = mounts + } + + /// Every value in the structure: the rootfs, the writable layer when + /// present, then the mounts, in that order. + public var all: [Value] { + var values = [rootfs] + if let writableLayer { + values.append(writableLayer) + } + values.append(contentsOf: mounts) + return values + } + + /// The storage with `transform` applied to every value, each keeping + /// its role. + public func map(_ transform: (Value) throws -> U) rethrows -> ContainerStorage { + ContainerStorage( + rootfs: try transform(rootfs), + writableLayer: try writableLayer.map(transform), + mounts: try mounts.map(transform) + ) + } +} + +/// A container's storage as configured, before its machine exists. +public typealias ContainerMounts = ContainerStorage + +/// A container's storage as attached to its machine. +public typealias ContainerAttachments = ContainerStorage + +/// The storage a machine carries: each container's, and the resources its +/// containers share. +public struct MachineStorage: Sendable { + /// Each container's storage, by container ID. + public var containers: [String: ContainerStorage] + + /// Volumes shared by the machine's containers, by volume name. + public var volumes: [String: Value] + + public init(containers: [String: ContainerStorage] = [:], volumes: [String: Value] = [:]) { + self.containers = containers + self.volumes = volumes + } + + /// Every value the machine carries: containers sorted by ID, each in + /// role order, then volumes sorted by name. Walks that allocate device + /// addresses and walks that create the devices use this one order, so an + /// address always names the device it was handed out for. + public var ordered: [Value] { + containers.keys.sorted().flatMap { containers[$0]?.all ?? [] } + + volumes.keys.sorted().compactMap { volumes[$0] } + } + + /// The storage with `transform` applied to every value, each keeping its + /// place. The transform runs in no particular order, so an allocating + /// conversion walks the fields sorted itself. + public func map(_ transform: (Value) throws -> U) rethrows -> MachineStorage { + MachineStorage( + containers: try containers.mapValues { try $0.map(transform) }, + volumes: try volumes.mapValues(transform) + ) + } +} + +/// A machine's storage as configured, before it exists. +public typealias MachineMounts = MachineStorage + +/// A machine's storage as attached. +public typealias MachineAttachments = MachineStorage diff --git a/Sources/Containerization/HotplugProvider.swift b/Sources/Containerization/HotplugProvider.swift index f535ce3f..0835077f 100644 --- a/Sources/Containerization/HotplugProvider.swift +++ b/Sources/Containerization/HotplugProvider.swift @@ -26,7 +26,7 @@ public protocol HotplugProvider: Sendable { /// - Returns: The attached filesystem with the device path in the guest func hotplug(_ block: Mount, id: String) async throws -> AttachedFilesystem - /// Register mounts for a container in the VM's mount registry. + /// Register mounts for a container in the machine's storage. /// - Parameters: /// - id: The container ID /// - rootfs: The rootfs attachment from hotplug diff --git a/Sources/Containerization/LinuxContainer.swift b/Sources/Containerization/LinuxContainer.swift index 34964fcd..55bbb39a 100644 --- a/Sources/Containerization/LinuxContainer.swift +++ b/Sources/Containerization/LinuxContainer.swift @@ -537,24 +537,14 @@ extension LinuxContainer { } private func mountRootfs( - attachments: [AttachedFilesystem], + attached: ContainerAttachments, rootfsPath: String, agent: VirtualMachineAgent ) async throws { - guard let rootfsAttachment = attachments.first else { - throw ContainerizationError(.notFound, message: "rootfs mount not found") - } + let rootfsAttachment = attached.rootfs - if self.writableLayer != nil { + if let writableAttachment = attached.writableLayer { // Set up overlayfs with image as lower layer and writable layer as upper. - guard attachments.count >= 2 else { - throw ContainerizationError( - .notFound, - message: "writable layer mount not found" - ) - } - let writableAttachment = attachments[1] - let lowerPath = "/run/container/\(self.id)/lower" let upperMountPath = "/run/container/\(self.id)/upper" let upperPath = "/run/container/\(self.id)/upper/diff" @@ -623,17 +613,18 @@ extension LinuxContainer { // This is dumb, but alas. let fileMountContextHolder = Mutex(fileMountContext) - // Build the list of mounts to attach to the VM. - var containerMounts = [modifiedRootfs] + fileMountContext.transformedMounts - if let writableLayer = self.writableLayer { - containerMounts.insert(writableLayer, at: 1) - } + // Build the container's storage to attach to the VM. + let containerStorage = ContainerMounts( + rootfs: modifiedRootfs, + writableLayer: self.writableLayer, + mounts: fileMountContext.transformedMounts + ) let vmConfig = VMConfiguration( cpus: vmCpus, memoryInBytes: vmMemory, interfaces: self.interfaces, - mountsByID: [self.id: containerMounts], + storage: MachineMounts(containers: [self.id: containerStorage]), bootLog: self.config.bootLog, nestedVirtualization: self.config.virtualization ) @@ -643,7 +634,7 @@ extension LinuxContainer { do { try await vm.start() - let mountsForAgent = containerMounts + let storageForAgent = containerStorage try await vm.withAgent { agent in try await agent.standardSetup() @@ -656,7 +647,7 @@ extension LinuxContainer { // with zero shares), but the cloud-hypervisor backend // only spawns virtiofsd when shares exist; mounting an // unbacked tag fails with EINVAL. - let hasVirtiofsMount = mountsForAgent.contains { mount in + let hasVirtiofsMount = storageForAgent.all.contains { mount in if case .virtiofs = mount.runtimeOptions { return true } return false } @@ -671,7 +662,7 @@ extension LinuxContainer { // gets populated. if vm.virtiofsLayout == .perTag { try await agent.mkdir(path: "/run/virtiofs", all: true, perms: 0o755) - let virtiofsAttachments = (vm.mounts[self.id] ?? []).filter { $0.type == "virtiofs" } + let virtiofsAttachments = (vm.storage.containers[self.id]?.all ?? []).filter { $0.type == "virtiofs" } let uniqueTags = Set(virtiofsAttachments.map(\.source)) for tag in uniqueTags { let dest = "/run/virtiofs/\(tag)" @@ -695,18 +686,17 @@ extension LinuxContainer { } } - guard let attachments = vm.mounts[self.id] else { + guard let attached = vm.storage.containers[self.id] else { throw ContainerizationError(.notFound, message: "rootfs mount not found") } let rootfsPath = Self.guestRootfsPath(self.id) - try await self.mountRootfs(attachments: attachments, rootfsPath: rootfsPath, agent: agent) + try await self.mountRootfs(attached: attached, rootfsPath: rootfsPath, agent: agent) // Mount file mount holding directories under /run. if fileMountContext.hasFileMounts { - let containerMounts = vm.mounts[self.id] ?? [] var ctx = fileMountContextHolder.withLock { $0 } try await ctx.mountHoldingDirectories( - vmMounts: containerMounts, + vmMounts: attached.mounts, agent: agent ) fileMountContextHolder.withLock { $0 = ctx } @@ -764,15 +754,12 @@ extension LinuxContainer { let agent = try await createdState.vm.dialAgent() do { var spec = self.generateRuntimeSpec() - // We don't need the rootfs (or writable layer), nor do OCI runtimes want it included. - // Also filter out file mount holding directories. We'll mount those separately under /run. + // Filter out file mount holding directories. We'll mount those separately under /run. // Transform virtiofs mounts to bind mounts from /run/virtiofs/{tag} - let containerMounts = createdState.vm.mounts[self.id] ?? [] + let containerMounts = createdState.vm.storage.containers[self.id]?.mounts ?? [] let holdingTags = createdState.fileMountContext.holdingDirectoryTags - // Drop rootfs, and writable layer if present. - let mountsToSkip = self.writableLayer != nil ? 2 : 1 var mounts: [ContainerizationOCI.Mount] = - containerMounts.dropFirst(mountsToSkip) + containerMounts .filter { !holdingTags.contains($0.source) } .map { attached -> ContainerizationOCI.Mount in if attached.type == "virtiofs" { diff --git a/Sources/Containerization/LinuxPod.swift b/Sources/Containerization/LinuxPod.swift index 6275a4d4..fa4d1785 100644 --- a/Sources/Containerization/LinuxPod.swift +++ b/Sources/Containerization/LinuxPod.swift @@ -446,24 +446,28 @@ extension LinuxPod { // the container's bind mounts from /run/virtiofs/ fail // with ENOENT. // - // Derive the tags from the additional mounts directly rather - // than from vm.mounts[id], so this is independent of the - // rootfs (which may be virtiofs or virtio-blk) and of mount - // ordering. The rootfs is mounted at /run/container//rootfs - // and is never consumed from /run/virtiofs. + // Derive the new tags from the additional mounts being added; + // the machine's storage names what is already shared. let newVirtiofsTags = try virtioFSMounts.map { try hashFilePath(path: $0.source) } if !newVirtiofsTags.isEmpty { try await agent.mkdir(path: "/run/virtiofs", all: true, perms: 0o755) if vm.virtiofsLayout == .perTag { // Tags already mounted in the guest at boot or by a - // prior hotplug (i.e. present on another container). - let alreadyMounted = Set( - vm.mounts - .filter { $0.key != id } - .values.flatMap { $0 } - .filter { $0.type == "virtiofs" } - .map { $0.source } - ) + // prior hotplug (i.e. present on another container or + // a machine volume). + let alreadyMounted: Set = { + var mounted = Set( + vm.storage.containers + .filter { $0.key != id } + .values.flatMap { $0.all } + .filter { $0.type == "virtiofs" } + .map { $0.source }) + mounted.formUnion( + vm.storage.volumes.values + .filter { $0.type == "virtiofs" } + .map { $0.source }) + return mounted + }() var seen: Set = [] for tag in newVirtiofsTags where !alreadyMounted.contains(tag) && seen.insert(tag).inserted { @@ -493,7 +497,7 @@ extension LinuxPod { } if fileMountContext.hasFileMounts { - let containerMounts = vm.mounts[id] ?? [] + let containerMounts = vm.storage.containers[id]?.mounts ?? [] try await updatedFileMountContext.mountHoldingDirectories( vmMounts: containerMounts, agent: agent @@ -557,20 +561,23 @@ extension LinuxPod { try await self.state.withLock { state in try state.phase.validateForCreate() - // Build mountsByID for all containers. + // Build the machine's storage from its containers. // Strip "ro" from rootfs options - we handle readonly via the OCI spec's // root.readonly field and remount in vmexec after setup is complete. // Use transformedMounts from fileMountContext (file mounts become directory shares). - var mountsByID: [String: [Mount]] = [:] + var machineStorage = MachineMounts() for (id, container) in state.containers { var modifiedRootfs = container.rootfs modifiedRootfs.options.removeAll(where: { $0 == "ro" }) - // Filter out shared mounts — those are handled separately as pod volume bind mounts. + // Shared mounts are handled separately as pod volume bind mounts. let containerMounts = container.fileMountContext.transformedMounts.filter { if case .shared = $0.runtimeOptions { return false } return true } - mountsByID[id] = [modifiedRootfs] + containerMounts + machineStorage.containers[id] = ContainerMounts( + rootfs: modifiedRootfs, + mounts: containerMounts + ) } // Validate pod volume names are unique. @@ -597,9 +604,8 @@ extension LinuxPod { } } } - let podVolumeMounts = self.config.volumes.map { $0.toMount() } - if !podVolumeMounts.isEmpty { - mountsByID[self.id] = podVolumeMounts + for volume in self.config.volumes { + machineStorage.volumes[volume.name] = volume.toMount() } // Capture into an immutable `let` so the value is safely usable @@ -608,18 +614,16 @@ extension LinuxPod { // only attaches a virtiofs device when shares are configured, // so mounting an unbacked /run/virtiofs would fail with EINVAL // on the CH backend. - let hasVirtiofsMount = mountsByID.values.contains { mounts in - mounts.contains { mount in - if case .virtiofs = mount.runtimeOptions { return true } - return false - } + let hasVirtiofsMount = machineStorage.ordered.contains { mount in + if case .virtiofs = mount.runtimeOptions { return true } + return false } var vmConfig = VMConfiguration( cpus: self.config.cpus, memoryInBytes: self.config.memoryInBytes, interfaces: self.config.interfaces, - mountsByID: mountsByID, + storage: machineStorage, bootLog: self.config.bootLog, nestedVirtualization: self.config.virtualization ) @@ -649,19 +653,17 @@ extension LinuxPod { // /run/virtiofs/. See LinuxContainer for the // VZ vs. CH model split. var seenTags: Set = [] - for (_, attached) in vm.mounts { - for entry in attached where entry.type == "virtiofs" { - guard seenTags.insert(entry.source).inserted else { continue } - let dest = "/run/virtiofs/\(entry.source)" - try await agent.mkdir(path: dest, all: true, perms: 0o755) - try await agent.mount( - ContainerizationOCI.Mount( - type: "virtiofs", - source: entry.source, - destination: dest, - options: [] - )) - } + for entry in vm.storage.ordered where entry.type == "virtiofs" { + guard seenTags.insert(entry.source).inserted else { continue } + let dest = "/run/virtiofs/\(entry.source)" + try await agent.mkdir(path: dest, all: true, perms: 0o755) + try await agent.mount( + ContainerizationOCI.Mount( + type: "virtiofs", + source: entry.source, + destination: dest, + options: [] + )) } } else { try await agent.mount( @@ -728,10 +730,10 @@ extension LinuxPod { // Mount all container rootfs for (_, container) in containers { - guard let attachments = vm.mounts[container.id], let rootfsAttachment = attachments.first else { + guard let attached = vm.storage.containers[container.id] else { throw ContainerizationError(.notFound, message: "rootfs mount not found for container \(container.id)") } - var rootfs = rootfsAttachment.to + var rootfs = attached.rootfs.to rootfs.destination = Self.guestRootfsPath(container.id) try await agent.mount(rootfs) } @@ -740,7 +742,7 @@ extension LinuxPod { for (id, container) in containers { if container.fileMountContext.hasFileMounts { var ctx = container.fileMountContext - let containerMounts = vm.mounts[id] ?? [] + let containerMounts = vm.storage.containers[id]?.mounts ?? [] try await ctx.mountHoldingDirectories( vmMounts: containerMounts, agent: agent @@ -750,15 +752,13 @@ extension LinuxPod { } // Mount pod-level volumes. - let podVolumeAttachments = vm.mounts[self.id] ?? [] - for (index, volume) in self.config.volumes.enumerated() { - guard index < podVolumeAttachments.count else { + for volume in self.config.volumes { + guard let attachment = vm.storage.volumes[volume.name] else { throw ContainerizationError( .notFound, message: "attached filesystem not found for pod volume \"\(volume.name)\"" ) } - let attachment = podVolumeAttachments[index] let guestPath = Self.guestVolumePath(volume.name) try await agent.mount( ContainerizationOCI.Mount( @@ -864,10 +864,10 @@ extension LinuxPod { // We don't need the rootfs, nor do OCI runtimes want it included. // Also filter out file mount holding directories - we mount those separately under /run. // Transform virtiofs mounts to bind mounts from /run/virtiofs/{tag} - let containerMounts = createdState.vm.mounts[containerID] ?? [] + let containerMounts = createdState.vm.storage.containers[containerID]?.mounts ?? [] let holdingTags = container.fileMountContext.holdingDirectoryTags var mounts: [ContainerizationOCI.Mount] = - containerMounts.dropFirst() + containerMounts .filter { !holdingTags.contains($0.source) } .map { attached -> ContainerizationOCI.Mount in if attached.type == "virtiofs" { diff --git a/Sources/Containerization/VMConfiguration.swift b/Sources/Containerization/VMConfiguration.swift index 30faebc4..7a5e8e2c 100644 --- a/Sources/Containerization/VMConfiguration.swift +++ b/Sources/Containerization/VMConfiguration.swift @@ -72,9 +72,9 @@ public struct VMConfiguration: Sendable { public var memoryInBytes: UInt64 /// The network interfaces to attach. public var interfaces: [any Interface] - /// Mounts organized by metadata ID (e.g. container ID). - /// Each ID maps to an array of mounts for that workload. - public var mountsByID: [String: [Mount]] + /// The storage the machine carries: each container's mounts by role, + /// and the volumes its containers share. + public var storage: MachineMounts /// Optional destination for serial boot logs. public var bootLog: BootLog? /// Enable nested virtualization support. If the VirtualMachineManager @@ -89,14 +89,14 @@ public struct VMConfiguration: Sendable { cpus: Int = 4, memoryInBytes: UInt64 = 1024 * 1024 * 1024, interfaces: [any Interface] = [], - mountsByID: [String: [Mount]] = [:], + storage: MachineMounts = MachineMounts(), bootLog: BootLog? = nil, nestedVirtualization: Bool = false ) { self.cpus = cpus self.memoryInBytes = memoryInBytes self.interfaces = interfaces - self.mountsByID = mountsByID + self.storage = storage self.bootLog = bootLog self.nestedVirtualization = nestedVirtualization } diff --git a/Sources/Containerization/VZVirtualMachineInstance.swift b/Sources/Containerization/VZVirtualMachineInstance.swift index 160c5026..670ab512 100644 --- a/Sources/Containerization/VZVirtualMachineInstance.swift +++ b/Sources/Containerization/VZVirtualMachineInstance.swift @@ -28,10 +28,10 @@ import Synchronization public final class VZVirtualMachineInstance: Sendable { public typealias Agent = Vminitd - /// Attached mounts on the virtual machine, organized by metadata ID. - private let _mounts: Mutex<[String: [AttachedFilesystem]]> - public var mounts: [String: [AttachedFilesystem]] { - _mounts.withLock { $0 } + /// The machine's attached storage. + private let _storage: Mutex + public var storage: MachineAttachments { + _storage.withLock { $0 } } /// The underlying Virtualization framework virtual machine. @@ -40,9 +40,9 @@ public final class VZVirtualMachineInstance: Sendable { /// The dispatch queue used for VZ operations. public var vmQueue: DispatchQueue { queue } - /// Mutate the mount registry. - public func withMountRegistry(_ body: (inout sending [String: [AttachedFilesystem]]) throws -> sending T) rethrows -> T { - try _mounts.withLock(body) + /// Mutate the storage registry. + public func withStorage(_ body: (inout sending MachineAttachments) throws -> sending T) rethrows -> T { + try _storage.withLock(body) } /// Serialize VM operations with the instance lock. @@ -73,8 +73,9 @@ public final class VZVirtualMachineInstance: Sendable { public var rosetta: Bool /// Toggle nested virtualization support. public var nestedVirtualization: Bool - /// Mount attachments organized by metadata ID. - public var mountsByID: [String: [Mount]] + /// The machine's storage: each container's mounts by role, and the + /// volumes its containers share. + public var storage: MachineMounts /// Network interface attachments. public var interfaces: [any Interface] /// Kernel image. @@ -91,7 +92,7 @@ public final class VZVirtualMachineInstance: Sendable { self.memoryInBytes = 1024.mib() self.rosetta = false self.nestedVirtualization = false - self.mountsByID = [:] + self.storage = MachineMounts() self.interfaces = [] } } @@ -132,7 +133,7 @@ public final class VZVirtualMachineInstance: Sendable { let allocator = Character.blockDeviceTagAllocator() let (mountAttachments, _) = try config.mountAttachments(allocator: allocator) - self._mounts = Mutex(mountAttachments) + self._storage = Mutex(mountAttachments) self.vm = VZVirtualMachine( configuration: try config.toVZ(allocator: allocator), @@ -153,7 +154,7 @@ public protocol VZInstanceExtension: Sendable { _ config: inout VZVirtualMachineConfiguration, allocator: any AddressAllocator, storageDeviceCount: Int, - mountsByID: [String: [Mount]] + storage: MachineMounts ) throws /// Called after the VZVirtualMachine is created but before start. @@ -168,7 +169,7 @@ extension VZInstanceExtension { _ config: inout VZVirtualMachineConfiguration, allocator: any AddressAllocator, storageDeviceCount: Int, - mountsByID: [String: [Mount]] + storage: MachineMounts ) throws {} public func didCreate(_ instance: VZVirtualMachineInstance) throws {} @@ -477,35 +478,33 @@ extension VZVirtualMachineInstance.Configuration { // Track used virtiofs tags to avoid creating duplicate VZ devices. // The same source directory mounted to multiple destinations shares one device. + // The walk is the machine's device order, matching the addresses + // `mountAttachments` hands out walking the same way. var usedVirtioFSTags: Set = [] - for (_, mounts) in self.mountsByID { - for mount in mounts { - if case .virtiofs = mount.runtimeOptions { - let tag = try hashFilePath(path: mount.source) - if usedVirtioFSTags.contains(tag) { - continue - } - usedVirtioFSTags.insert(tag) + for mount in self.storage.ordered { + if case .virtiofs = mount.runtimeOptions { + let tag = try hashFilePath(path: mount.source) + if usedVirtioFSTags.contains(tag) { + continue } - try mount.configure(config: &config) + usedVirtioFSTags.insert(tag) } + try mount.configure(config: &config) } // Create the unified virtiofs device with VZMultipleDirectoryShare // This device hosts all virtiofs shares and supports runtime updates var directories: [String: VZSharedDirectory] = [:] - for (_, mounts) in self.mountsByID { - for mount in mounts { - guard case .virtiofs(_) = mount.runtimeOptions else { continue } - guard FileManager.default.fileExists(atPath: mount.source) else { - throw ContainerizationError(.notFound, message: "directory \(mount.source) does not exist") - } - let name = try hashFilePath(path: mount.source) - directories[name] = VZSharedDirectory( - url: URL(fileURLWithPath: mount.source), - readOnly: mount.options.contains("ro") - ) + for mount in self.storage.ordered { + guard case .virtiofs(_) = mount.runtimeOptions else { continue } + guard FileManager.default.fileExists(atPath: mount.source) else { + throw ContainerizationError(.notFound, message: "directory \(mount.source) does not exist") } + let name = try hashFilePath(path: mount.source) + directories[name] = VZSharedDirectory( + url: URL(fileURLWithPath: mount.source), + readOnly: mount.options.contains("ro") + ) } let multiShare = VZMultipleDirectoryShare(directories: directories) let virtiofsDevice = VZVirtioFileSystemDeviceConfiguration(tag: "virtiofs") @@ -527,7 +526,7 @@ extension VZVirtualMachineInstance.Configuration { config.platform = platform for ext in self.extensions.compactMap({ $0 as? any VZInstanceExtension }) { - try ext.configureVZ(&config, allocator: allocator, storageDeviceCount: storageDeviceCount, mountsByID: self.mountsByID) + try ext.configureVZ(&config, allocator: allocator, storageDeviceCount: storageDeviceCount, storage: self.storage) } try config.validate() @@ -535,7 +534,7 @@ extension VZVirtualMachineInstance.Configuration { } func mountAttachments(allocator: any AddressAllocator) throws -> ( - attachments: [String: [AttachedFilesystem]], storageDeviceCount: Int + attachments: MachineAttachments, storageDeviceCount: Int ) { var storageDeviceCount = 0 @@ -548,21 +547,28 @@ extension VZVirtualMachineInstance.Configuration { } } - var attachmentsByID: [String: [AttachedFilesystem]] = [:] - - for (id, mounts) in self.mountsByID { - var attachments: [AttachedFilesystem] = [] - for mount in mounts { - let attached = try AttachedFilesystem(mount: mount, allocator: allocator) - attachments.append(attached) - if mount.isBlock { - storageDeviceCount += 1 - } + // The machine's device order: addresses are handed out in the same + // walk `makeConfiguration` creates the devices in. + func attach(_ mount: Mount) throws -> AttachedFilesystem { + let attached = try AttachedFilesystem(mount: mount, allocator: allocator) + if mount.isBlock { + storageDeviceCount += 1 } - attachmentsByID[id] = attachments + return attached + } + + var containers: [String: ContainerAttachments] = [:] + for id in self.storage.containers.keys.sorted() { + guard let container = self.storage.containers[id] else { continue } + containers[id] = try container.map(attach) + } + var volumes: [String: AttachedFilesystem] = [:] + for name in self.storage.volumes.keys.sorted() { + guard let mount = self.storage.volumes[name] else { continue } + volumes[name] = try attach(mount) } - return (attachmentsByID, storageDeviceCount) + return (MachineAttachments(containers: containers, volumes: volumes), storageDeviceCount) } } diff --git a/Sources/Containerization/VZVirtualMachineManager.swift b/Sources/Containerization/VZVirtualMachineManager.swift index 4959bee4..d0888447 100644 --- a/Sources/Containerization/VZVirtualMachineManager.swift +++ b/Sources/Containerization/VZVirtualMachineManager.swift @@ -76,7 +76,7 @@ public struct VZVirtualMachineManager: VirtualMachineManager { instanceConfig.rosetta = self.rosetta instanceConfig.nestedVirtualization = useNestedVirtualization - instanceConfig.mountsByID = vmConfig.mountsByID + instanceConfig.storage = vmConfig.storage instanceConfig.extensions = vmConfig.extensions }) } diff --git a/Sources/Containerization/VirtualMachineInstance.swift b/Sources/Containerization/VirtualMachineInstance.swift index 302e97ae..b9340612 100644 --- a/Sources/Containerization/VirtualMachineInstance.swift +++ b/Sources/Containerization/VirtualMachineInstance.swift @@ -44,7 +44,8 @@ public protocol VirtualMachineInstance: Sendable { // The state of the virtual machine. var state: VirtualMachineInstanceState { get } - var mounts: [String: [AttachedFilesystem]] { get } + /// The machine's attached storage. + var storage: MachineAttachments { get } /// How this VMM exposes virtiofs devices to the guest. Defaults to /// `.unified` (the VZ-shaped behavior); CH overrides to `.perTag`. @@ -73,7 +74,7 @@ public protocol VirtualMachineInstance: Sendable { func hotplug(_ block: Mount, id: String) async throws -> AttachedFilesystem /// Register mounts for a container after hotplug. - /// This is used to add the rootfs and additional mounts to the VM's mount registry + /// This is used to add the rootfs and additional mounts to the machine's storage /// so they can be found when building the container's OCI spec. /// - Parameter id: The container ID /// - Parameter rootfs: The rootfs attachment from hotplug From f17e501244d2bc11ec9057c530916e60c5172dde Mon Sep 17 00:00:00 2001 From: Aaron Paterson Date: Sun, 2 Aug 2026 23:19:33 +0000 Subject: [PATCH 6/8] Keep a test's temporary directory with the rest of the run's A test that wants a directory of its own asks for a temporary one, and none of the thirty four places that do take it away again, so a run left one behind for every test that asked and every run added more. The suite already removes its own directory when it finishes, so pointing the temporary directory at that one puts them somewhere already reached, and takes any a test adds later with them. --- Sources/Integration/Suite.swift | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/Sources/Integration/Suite.swift b/Sources/Integration/Suite.swift index 84580dba..39618af3 100644 --- a/Sources/Integration/Suite.swift +++ b/Sources/Integration/Suite.swift @@ -408,6 +408,13 @@ struct IntegrationSuite: AsyncParsableCommand { // Hopefully this improves over time. func run() async throws { try Self.adjustLimits() + + // Tests reach for a temporary directory of their own freely and none of + // them take it away again. Pointing the temporary directory at the + // suite's own puts whatever they make where the removal at the end of + // the run already reaches, and takes any a test adds later with it. + setenv("TMPDIR", Self.testDir.absolutePath(), 1) + let suiteStarted = Date().timeIntervalSinceReferenceDate log.info("starting integration suite\n") From 41bcd8f27ca619c0835aa656644d57ca93bc5bcd Mon Sep 17 00:00:00 2001 From: Aaron Paterson Date: Mon, 3 Aug 2026 01:23:04 +0000 Subject: [PATCH 7/8] Gather a run's files where they can be swept A run took a name of its own among the system's temporary files, so anything it left behind could only be found by knowing which run had left it, and nothing in the project could clean it. Runs gather under one directory now, which a make target sweeps the way the content one sweeps the image store. A run takes its own files away as it goes, so the target is for what an interrupted run, or one asked to keep them, has left. --- Makefile | 8 ++++++++ Sources/Integration/Suite.swift | 14 +++++++++++++- 2 files changed, 21 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 3d01b93a..456768b4 100644 --- a/Makefile +++ b/Makefile @@ -491,6 +491,14 @@ cleancontent: @echo Cleaning the content... @rm -rf ~/Library/Application\ Support/com.apple.containerization +# The integration suite takes its files away as it goes, so this is for what a +# run that was interrupted, or one asked to keep them, has left behind. The +# directory is the one IntegrationSuite.testRootName names. +.PHONY: cleantests +cleantests: + @echo Cleaning the integration test files... + @rm -rf "$${TMPDIR:-/tmp}/containerization-integration" + .PHONY: examples examples: @echo Building examples... diff --git a/Sources/Integration/Suite.swift b/Sources/Integration/Suite.swift index 39618af3..85e623c9 100644 --- a/Sources/Integration/Suite.swift +++ b/Sources/Integration/Suite.swift @@ -131,8 +131,20 @@ struct IntegrationSuite: AsyncParsableCommand { ) }() + /// Where a run keeps everything it makes. + /// + /// Runs gather under one directory rather than each taking a name of its + /// own among the system's temporary files, so that what a run leaves behind + /// can be swept without knowing which run left it. `make cleantests` sweeps + /// this same directory, so the name is shared between the two. + static let testRootName = "containerization-integration" + static let _testDir: URL = { - FileManager.default.uniqueTemporaryDirectory(create: true) + let root = FileManager.default.temporaryDirectory + .appendingPathComponent(testRootName) + let run = root.appendingPathComponent(UUID().uuidString) + try? FileManager.default.createDirectory(at: run, withIntermediateDirectories: true) + return run }() /// The scratch directory of the test running on this task. The runner From 3e7dda46581dee3c2cdab1cd6b3db68700bac087 Mon Sep 17 00:00:00 2001 From: Aaron Paterson Date: Mon, 3 Aug 2026 07:41:46 +0000 Subject: [PATCH 8/8] Clean the content the dev container keeps as well The Linux dev container has no home directory that survives a run, so the imageStore it fills sits beside the repository and is bind mounted into place. That makes it the same content the macOS side keeps under its own home, held somewhere else only because the container cannot keep it. Cleaning the content took away one of them and left the other, so a tree asked to give up its content still held a store of images. --- Makefile | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/Makefile b/Makefile index 456768b4..c5645fb6 100644 --- a/Makefile +++ b/Makefile @@ -486,10 +486,14 @@ docs: @rm -rf _site @scripts/make-docs.sh _site containerization +# The Linux dev container keeps its content beside the repository rather than +# under a home directory of its own, which does not survive a `container run`. +# It is the same store either way, so cleaning the content means both of them. .PHONY: cleancontent cleancontent: @echo Cleaning the content... @rm -rf ~/Library/Application\ Support/com.apple.containerization + @rm -rf $(ROOT_DIR)/.local/integration-cache # The integration suite takes its files away as it goes, so this is for what a # run that was interrupted, or one asked to keep them, has left behind. The