From f10eaca613f6eef28add5d69f4fbf573ca6ed156 Mon Sep 17 00:00:00 2001 From: Aaron Paterson Date: Sat, 1 Aug 2026 18:48:34 +0000 Subject: [PATCH 01/25] Resolve test directory paths before reaping per-test files The reaper that runs at a concurrency of one deletes everything in the test directory except the unpacked rootfs it means to preserve. It held that rootfs path as FileManager reports it, under /var, and compared it against the entries of contentsOfDirectory, which reports them under /private/var, so the preserved path never matched any entry and the rootfs was deleted along with the per-test files. The unpack coordinator still held it as unpacked, so the next test opened a rootfs that was no longer there and failed with a missing file error. Resolve both sides with resolvingSymlinksInPathWithPrivate, which exists for this difference between the two views of the same directory. --- Sources/Integration/Suite.swift | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/Sources/Integration/Suite.swift b/Sources/Integration/Suite.swift index 9dae8435..e43cf372 100644 --- a/Sources/Integration/Suite.swift +++ b/Sources/Integration/Suite.swift @@ -259,12 +259,16 @@ struct IntegrationSuite: AsyncParsableCommand { // a ~2GB rootfs and a ~512MB initfs, so without reaping the dev // container fills its CoW layer in ~10 tests. if self.maxConcurrency == 1 { - let preserve = fsPath.absolutePath() + // contentsOfDirectory reports paths under /private, while testDir is + // built from FileManager's /var view of the same directory, so both + // sides are resolved before comparing. + let preserve = fsPath.resolvingSymlinksInPathWithPrivate().absolutePath() if let entries = try? FileManager.default.contentsOfDirectory( at: Self.testDir, includingPropertiesForKeys: nil ) { - for url in entries where url.absolutePath() != preserve { + for url in entries + where url.resolvingSymlinksInPathWithPrivate().absolutePath() != preserve { try? FileManager.default.removeItem(at: url) } } From 6499f7dfa7fff64c903642db690094d7e6c64ec8 Mon Sep 17 00:00:00 2001 From: Aaron Paterson Date: Sun, 2 Aug 2026 00:51:10 +0000 Subject: [PATCH 02/25] Say how to run the tests that macOS skips Tests guarded for Linux are compiled out on macOS, so `make test` reports success without having run them, and nothing says so. The target that does run them is not mentioned anywhere outside the makefile. --- README.md | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/README.md b/README.md index f658b08d..372a872a 100644 --- a/README.md +++ b/README.md @@ -130,6 +130,16 @@ After building, run basic and integration tests: make test integration ``` +Tests that only apply to Linux are compiled out on macOS, so `make test` passes +without running them. Run those with: + +```bash +make linux-test +``` + +which runs the same tests inside the Linux dev container, as the Linux build +workflow does. + A kernel is required to run integration tests. If you do not have a kernel locally, a default kernel can be fetched using the `make fetch-default-kernel` target. From bf4d7dcf8bbe64c745e9ef344a8c173d2a42a3af Mon Sep 17 00:00:00 2001 From: Aaron Paterson Date: Sun, 2 Aug 2026 01:20:29 +0000 Subject: [PATCH 03/25] Test against the kernel release the shipped tool installs The default kernel the tests fetch had drifted from the one users run. container installs kata 3.28.0 and its 6.18.15 kernel, while these tests fetched 3.17.0, so the suite exercised a guest with a different feature set to the one it is meant to represent. Nested runtimes are the visible case: 3.17.0 was built without nf_tables, so a docker daemon inside a container fails there and works on what ships. Kata moved from xz to zstd between those releases, so the archive is no longer named for its compression and tar is left to recognise it rather than being told, which also holds if the format changes again. --- Makefile | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Makefile b/Makefile index c79d6995..3d01b93a 100644 --- a/Makefile +++ b/Makefile @@ -64,7 +64,7 @@ LIBARCHIVE_UPSTREAM_REPO := https://github.com/libarchive/libarchive LIBARCHIVE_UPSTREAM_VERSION := v3.7.7 LIBARCHIVE_LOCAL_DIR := workdir/libarchive -KATA_BINARY_PACKAGE := https://github.com/kata-containers/kata-containers/releases/download/3.17.0/kata-static-3.17.0-arm64.tar.xz +KATA_BINARY_PACKAGE := https://github.com/kata-containers/kata-containers/releases/download/3.28.0/kata-static-3.28.0-arm64.tar.zst CLOUD_HYPERVISOR_URL := https://github.com/cloud-hypervisor/cloud-hypervisor/releases/download/v52.0/cloud-hypervisor-static-aarch64 # SHA256 of the v52.0 aarch64 static binary (verified locally from the # upstream release artifact). Bump alongside CLOUD_HYPERVISOR_URL. @@ -410,11 +410,11 @@ integration: .PHONY: fetch-default-kernel fetch-default-kernel: @mkdir -p .local/ bin/ -ifeq (,$(wildcard .local/kata.tar.gz)) - @curl -SsL -o .local/kata.tar.gz ${KATA_BINARY_PACKAGE} +ifeq (,$(wildcard .local/kata.tar)) + @curl -SsL -o .local/kata.tar ${KATA_BINARY_PACKAGE} endif ifeq (,$(wildcard .local/vmlinux-$(KERNEL_ARCH))) - @tar -zxf .local/kata.tar.gz -C .local/ --strip-components=1 + @tar -xf .local/kata.tar -C .local/ --strip-components=1 @cp -L .local/opt/kata/share/kata-containers/vmlinux.container .local/vmlinux-$(KERNEL_ARCH) endif ifeq (,$(wildcard bin/vmlinux-$(KERNEL_ARCH))) From 21f1f16eaaacc051bbc5be6566814cb606de5698 Mon Sep 17 00:00:00 2001 From: Aaron Paterson Date: Wed, 12 Aug 2026 05:25:08 +0000 Subject: [PATCH 04/25] Run each integration test in its own scratch directory A test's rootfs clones, init block, and writable layers land in a directory the runner creates for it and deletes when it finishes, so a test holds disk only while it runs at any concurrency. The run-shared directory holds the unpacked images every bootstrap clones from, and is removed when the run ends. --- Sources/Integration/PodTests.swift | 7 +++-- Sources/Integration/Suite.swift | 43 +++++++++++++----------------- 2 files changed, 21 insertions(+), 29 deletions(-) diff --git a/Sources/Integration/PodTests.swift b/Sources/Integration/PodTests.swift index ae1caec8..7dff20fa 100644 --- a/Sources/Integration/PodTests.swift +++ b/Sources/Integration/PodTests.swift @@ -2228,10 +2228,9 @@ extension IntegrationSuite { /// (directory-share) rootfs. Assumes a single-layer image (the alpine /// image used by the suite) so no OCI whiteout processing is required. /// - /// The extracted dir lives under `Self.testDir`; do NOT `defer`-remove it - /// here — virtiofsd shares it for the whole test. It is swept by - /// `bootstrap`'s `maxConcurrency == 1` reaper on the next test and by the - /// suite-end `removeItem(at: Self.testDir)`. + /// The extracted dir lives under the test's scratch directory; do NOT + /// `defer`-remove it here — virtiofsd shares it for the whole test. The + /// runner removes the scratch directory when the test finishes. private func unpackRootfsDirectory(_ image: Containerization.Image, testID: String) async throws -> Containerization.Mount { let dir = Self.testDir.appending(component: "\(testID)-rootfs-dir") try? FileManager.default.removeItem(at: dir) diff --git a/Sources/Integration/Suite.swift b/Sources/Integration/Suite.swift index e43cf372..84580dba 100644 --- a/Sources/Integration/Suite.swift +++ b/Sources/Integration/Suite.swift @@ -135,7 +135,19 @@ struct IntegrationSuite: AsyncParsableCommand { FileManager.default.uniqueTemporaryDirectory(create: true) }() + /// The scratch directory of the test running on this task. The runner + /// creates one per test and deletes it when the test finishes, so a + /// test's clones and layers hold disk only while it runs. + @TaskLocal static var currentTestDir: URL? + static var testDir: URL { + currentTestDir ?? _testDir + } + + /// The run-shared directory holding unpacked images, which every test's + /// bootstrap clones from. It lives for the whole run and is removed at + /// the end. + static var imageCacheDir: URL { _testDir } @@ -233,7 +245,7 @@ struct IntegrationSuite: AsyncParsableCommand { let platform = Platform(arch: "arm64", os: "linux", variant: "v8") // Unpack to shared location with coordination to prevent concurrent unpacks - let fsPath = Self.testDir.appending(component: image.digest) + let fsPath = Self.imageCacheDir.appending(component: image.digest) let fs = try await Self.unpackCoordinator.unpack(key: fsPath.absolutePath()) { do { let unpacker = EXT4Unpacker(capacityInBytes: 2.gib()) @@ -251,29 +263,6 @@ struct IntegrationSuite: AsyncParsableCommand { } } - // Reap any per-test artifacts left over from prior tests. With - // `--max-concurrency 1` (linux-integration default) this runs after - // the previous test has fully completed, so it's race-free; on - // macOS where tests can run in parallel we just keep all files — - // disk usage isn't a concern there. Each per-test bootstrap clones - // a ~2GB rootfs and a ~512MB initfs, so without reaping the dev - // container fills its CoW layer in ~10 tests. - if self.maxConcurrency == 1 { - // contentsOfDirectory reports paths under /private, while testDir is - // built from FileManager's /var view of the same directory, so both - // sides are resolved before comparing. - let preserve = fsPath.resolvingSymlinksInPathWithPrivate().absolutePath() - if let entries = try? FileManager.default.contentsOfDirectory( - at: Self.testDir, - includingPropertiesForKeys: nil - ) { - for url in entries - where url.resolvingSymlinksInPathWithPrivate().absolutePath() != preserve { - try? FileManager.default.removeItem(at: url) - } - } - } - // Clone to test-specific path let clPath = Self.testDir.appending(component: "\(testID).ext4").absolutePath() try? FileManager.default.removeItem(atPath: clPath) @@ -658,11 +647,14 @@ struct IntegrationSuite: AsyncParsableCommand { for _ in 0.. Date: Wed, 12 Aug 2026 15:00:44 +0000 Subject: [PATCH 05/25] Give the machine's storage the shape its containers have A container's storage is its rootfs, an optional writable layer, and its mounts; a machine's is its containers' plus the volumes they share. One generic shape describes both the Mount values a machine is configured with and the AttachedFilesystem values it reports once attached, so converting between the two is a map over the structure and the roles cannot drift between configuration and attachment. Device addresses are allocated walking the same sorted order the devices are created in, on both backends. Consumers read roles instead of list positions: the spec builders take a container's mounts without prefix arithmetic, pod volumes mount by name, and a cloud-hypervisor volume's virtiofsd is held by a machine-lifetime reference alongside its containers' reference counts. --- .../Containerization/CHHotplugProvider.swift | 79 ++++++----- .../CHVirtualMachineInstance.swift | 127 ++++++++++-------- .../CHVirtualMachineManager.swift | 2 +- .../Containerization/ContainerStorage.swift | 107 +++++++++++++++ .../Containerization/HotplugProvider.swift | 2 +- Sources/Containerization/LinuxContainer.swift | 51 +++---- Sources/Containerization/LinuxPod.swift | 98 +++++++------- .../Containerization/VMConfiguration.swift | 10 +- .../VZVirtualMachineInstance.swift | 100 +++++++------- .../VZVirtualMachineManager.swift | 2 +- .../VirtualMachineInstance.swift | 5 +- 11 files changed, 351 insertions(+), 232 deletions(-) create mode 100644 Sources/Containerization/ContainerStorage.swift diff --git a/Sources/Containerization/CHHotplugProvider.swift b/Sources/Containerization/CHHotplugProvider.swift index 870fb1ff..380cba76 100644 --- a/Sources/Containerization/CHHotplugProvider.swift +++ b/Sources/Containerization/CHHotplugProvider.swift @@ -27,8 +27,8 @@ import Synchronization /// /// Handles both block (`vm.add-disk`) and virtiofs (`vm.add-fs`, with one /// `virtiofsd` per unique source-hash tag) hotplug, plus the matching -/// `vm.remove-device` teardown. Owns the per-VM mount registry so -/// `CHVirtualMachineInstance.mounts` can forward to it. +/// `vm.remove-device` teardown. Owns the machine's storage so +/// `CHVirtualMachineInstance.storage` can forward to it. final class CHHotplugProvider: HotplugProvider { struct HotplugRecord: Sendable { let chDeviceId: String @@ -50,7 +50,7 @@ final class CHHotplugProvider: HotplugProvider { private let workDir: URL private let virtiofsdBinaryOverride: URL? private let allocator: any AddressAllocator - private let _mounts: Mutex<[String: [AttachedFilesystem]]> + private let _storage: Mutex private let _records: Mutex<[String: [HotplugRecord]]> private let _tags: Mutex<[String: VirtiofsdTagState]> /// Serializes per-tag virtiofsd spawn so a concurrent hotplug for the @@ -65,14 +65,14 @@ final class CHHotplugProvider: HotplugProvider { workDir: URL, virtiofsdBinary: URL?, allocator: any AddressAllocator, - initialMounts: [String: [AttachedFilesystem]], + initialStorage: MachineAttachments, logger: Logger? ) { self.client = client self.workDir = workDir self.virtiofsdBinaryOverride = virtiofsdBinary self.allocator = allocator - self._mounts = Mutex(initialMounts) + self._storage = Mutex(initialStorage) self._records = Mutex([:]) self._tags = Mutex([:]) self.spawnLock = AsyncLock() @@ -81,14 +81,14 @@ final class CHHotplugProvider: HotplugProvider { // MARK: - Read accessors - var mounts: [String: [AttachedFilesystem]] { - _mounts.withLock { $0 } + var storage: MachineAttachments { + _storage.withLock { $0 } } - func withMountRegistry( - _ body: (inout sending [String: [AttachedFilesystem]]) throws -> sending T + func withStorage( + _ body: (inout sending MachineAttachments) throws -> sending T ) rethrows -> T { - try _mounts.withLock(body) + try _storage.withLock(body) } // MARK: - HotplugProvider conformance @@ -151,12 +151,13 @@ final class CHHotplugProvider: HotplugProvider { } func registerMounts(id: String, rootfs: AttachedFilesystem, additionalMounts: [Mount]) throws { - var attached: [AttachedFilesystem] = [rootfs] + var mounts: [AttachedFilesystem] = [] for mount in additionalMounts { - attached.append(try AttachedFilesystem(mount: mount, allocator: allocator)) + mounts.append(try AttachedFilesystem(mount: mount, allocator: allocator)) } - _mounts.withLock { - $0[id, default: []].append(contentsOf: attached) + let container = ContainerAttachments(rootfs: rootfs, mounts: mounts) + _storage.withLock { + $0.containers[id] = container } } @@ -190,19 +191,10 @@ final class CHHotplugProvider: HotplugProvider { } } - // Drop block-derived AttachedFilesystem entries for `id`. Block entries - // are the ones whose source was rewritten to "/dev/vd" by - // `hotplug(_:)` (or by AttachedFilesystem(mount:allocator:) for an - // additionalMount of type virtio-blk). - _mounts.withLock { state in - guard var perID = state[id] else { return } - perID.removeAll { $0.source.hasPrefix("/dev/vd") } - if perID.isEmpty { - state.removeValue(forKey: id) - } else { - state[id] = perID - } - } + // The container's devices are gone, so the container leaves the + // registry with them; its shares are released separately and their + // processes reference-counted through `_tags`. + _ = _storage.withLock { $0.containers.removeValue(forKey: id) } } func hotplugVirtioFS(_ mounts: [Mount], id: String) async throws { @@ -226,7 +218,7 @@ final class CHHotplugProvider: HotplugProvider { let chDeviceId = try await ensureVirtiofsDevice(tag: tag, source: source, readonly: readonly) // Record once per tag for this container. The AttachedFilesystem // entries for these mounts are written by registerMounts (the sole - // _mounts writer), so we do NOT touch _mounts here. + // registry writer), so we do NOT touch the storage here. _records.withLock { $0[id, default: []].append(HotplugRecord(chDeviceId: chDeviceId, kind: .virtiofs(tag: tag))) } @@ -338,16 +330,17 @@ final class CHHotplugProvider: HotplugProvider { try? FileManager.default.removeItem(at: socket) } - // Drop virtiofs AttachedFilesystem entries for `id`. AttachedFilesystem - // sets `type = mount.type` which for a `.virtiofs` mount is "virtiofs". - _mounts.withLock { state in - guard var perID = state[id] else { return } - perID.removeAll { $0.type == "virtiofs" } - if perID.isEmpty { - state.removeValue(forKey: id) - } else { - state[id] = perID + // Drop the container's virtiofs entries. A container whose rootfs is + // itself a share leaves the registry whole; one that keeps block + // devices keeps its entry with the share entries dropped. + _storage.withLock { state in + guard var container = state.containers[id] else { return } + if container.rootfs.type == "virtiofs" { + state.containers.removeValue(forKey: id) + return } + container.mounts.removeAll { $0.type == "virtiofs" } + state.containers[id] = container } } @@ -358,15 +351,21 @@ final class CHHotplugProvider: HotplugProvider { /// is the user-supplied `FsConfig.id` (which `vm.remove-device` keys on). /// `ownerIds` are the container ids that count toward this tag's refcount; /// each gets a `HotplugRecord` so `releaseVirtioFS(id:)` walks them - /// uniformly. + /// uniformly. `machineHeld` adds one reference nothing releases, for a + /// share the machine itself owns (a volume). func recordBootTimeVirtiofs( tag: String, process: VirtiofsdProcess, chDeviceId: String, - ownerIds: [String] + ownerIds: [String], + machineHeld: Bool ) { _tags.withLock { - $0[tag] = VirtiofsdTagState(process: process, refcount: ownerIds.count, chDeviceId: chDeviceId) + $0[tag] = VirtiofsdTagState( + process: process, + refcount: ownerIds.count + (machineHeld ? 1 : 0), + chDeviceId: chDeviceId + ) } _records.withLock { records in for id in ownerIds { diff --git a/Sources/Containerization/CHVirtualMachineInstance.swift b/Sources/Containerization/CHVirtualMachineInstance.swift index 122f592a..e363d5ca 100644 --- a/Sources/Containerization/CHVirtualMachineInstance.swift +++ b/Sources/Containerization/CHVirtualMachineInstance.swift @@ -46,7 +46,7 @@ public final class CHVirtualMachineInstance: Sendable { public struct Configuration: Sendable { public var cpus: Int public var memoryInBytes: UInt64 - public var mountsByID: [String: [Mount]] + public var storage: MachineMounts public var interfaces: [any Interface] public var kernel: Kernel? public var initialFilesystem: Mount? @@ -56,7 +56,7 @@ public final class CHVirtualMachineInstance: Sendable { public init() { self.cpus = 4 self.memoryInBytes = 1024 * 1024 * 1024 - self.mountsByID = [:] + self.storage = MachineMounts() self.interfaces = [] } } @@ -65,8 +65,8 @@ public final class CHVirtualMachineInstance: Sendable { /// `start()`'s `VmConfig.disks` ordering matches the allocator letters. struct BootDisk: Sendable { let mount: Mount - let containerId: String? // nil for rootfs - let letter: Character + let chId: String + let readonly: Bool } // MARK: - State @@ -76,8 +76,8 @@ public final class CHVirtualMachineInstance: Sendable { _state.withLock { $0 } } - public var mounts: [String: [AttachedFilesystem]] { - hotplug.mounts + public var storage: MachineAttachments { + hotplug.storage } /// Cloud-hypervisor exposes one virtio-fs device per source-hash tag, so @@ -167,9 +167,9 @@ public final class CHVirtualMachineInstance: Sendable { ) self.workDir = workDir - // 2. Block allocator + boot inventory. Walks rootfs first, then - // mountsByID sorted by container id, allocating disk letters in - // that order. The same allocator is later handed to the hotplug + // 2. Block allocator + boot inventory. Walks rootfs first, then the + // machine's storage in its device order, allocating disk letters + // as it goes. The same allocator is later handed to the hotplug // provider so runtime add-disk picks up where boot wiring left off. let allocator = Character.blockDeviceTagAllocator() let inventory = try config.bootInventory(allocator: allocator) @@ -201,14 +201,14 @@ public final class CHVirtualMachineInstance: Sendable { logger: logger ?? Logger(label: "CloudHypervisor.Client") ) - // 5. Hotplug provider — owns the mount registry, seeded with the + // 5. Hotplug provider — owns the machine's storage, seeded with the // boot inventory so registerMounts can append to it. self.hotplug = CHHotplugProvider( client: self.client, workDir: workDir, virtiofsdBinary: virtiofsdBinary, allocator: allocator, - initialMounts: inventory.attachments, + initialStorage: inventory.attachments, logger: logger ) @@ -222,11 +222,11 @@ public final class CHVirtualMachineInstance: Sendable { self._stdioPool = Mutex([:]) } - /// Mutate the mount registry. Forwards to the hotplug provider, which + /// Mutate the storage registry. Forwards to the hotplug provider, which /// owns the registry. Kept on the instance for parity with the macOS - /// path's `withMountRegistry` API. - func withMountRegistry(_ body: (inout sending [String: [AttachedFilesystem]]) throws -> sending T) rethrows -> T { - try hotplug.withMountRegistry(body) + /// path's `withStorage` API. + func withStorage(_ body: (inout sending MachineAttachments) throws -> sending T) rethrows -> T { + try hotplug.withStorage(body) } } @@ -510,34 +510,46 @@ extension CHVirtualMachineInstance { throw ContainerizationError(.invalidArgument, message: "initialFilesystem is required for cloud-hypervisor backend") } - // Disks: rootfs forced read-only at the device level; container disks + // Disks: rootfs forced read-only at the device level; other disks // honor their `ro` option through chDiskConfig. var disks: [CloudHypervisor.DiskConfig] = [] for bd in bootDisks { - let chId = bd.containerId.map { "blk-\($0)-\(bd.letter)" } ?? "rootfs" - if var disk = bd.mount.chDiskConfig(id: chId) { - if bd.containerId == nil { + if var disk = bd.mount.chDiskConfig(id: bd.chId) { + if bd.readonly { disk.readonly = true } disks.append(disk) } } - // Virtiofs: group all .virtiofs mounts in mountsByID by source-hash - // tag, spawn one virtiofsd per tag, build matching FsConfigs. - var byTag: [String: (mounts: [Mount], owners: [String])] = [:] - for cid in config.mountsByID.keys.sorted() { - guard let mounts = config.mountsByID[cid] else { continue } - for mount in mounts { - guard case .virtiofs = mount.runtimeOptions else { continue } - let tag = try hashFilePath(path: mount.source) - var entry = byTag[tag] ?? (mounts: [], owners: []) - entry.mounts.append(mount) - if !entry.owners.contains(cid) { - entry.owners.append(cid) + // Virtiofs: group all virtiofs mounts in the machine's storage by + // source-hash tag, spawn one virtiofsd per tag, build matching + // FsConfigs. A volume's tag is the machine's own, held for its + // lifetime. + var byTag: [String: (mounts: [Mount], owners: [String], machineHeld: Bool)] = [:] + func groupVirtiofs(_ mount: Mount, owner: String?) throws { + guard case .virtiofs = mount.runtimeOptions else { return } + let tag = try hashFilePath(path: mount.source) + var entry = byTag[tag] ?? (mounts: [], owners: [], machineHeld: false) + entry.mounts.append(mount) + if let owner { + if !entry.owners.contains(owner) { + entry.owners.append(owner) } - byTag[tag] = entry + } else { + entry.machineHeld = true } + byTag[tag] = entry + } + for cid in config.storage.containers.keys.sorted() { + guard let container = config.storage.containers[cid] else { continue } + for mount in container.all { + try groupVirtiofs(mount, owner: cid) + } + } + for name in config.storage.volumes.keys.sorted() { + guard let mount = config.storage.volumes[name] else { continue } + try groupVirtiofs(mount, owner: nil) } var fsConfigs: [CloudHypervisor.FsConfig] = [] @@ -569,7 +581,8 @@ extension CHVirtualMachineInstance { tag: tag, process: process, chDeviceId: chDeviceId, - ownerIds: entry.owners + ownerIds: entry.owners, + machineHeld: entry.machineHeld ) fsConfigs.append( @@ -714,40 +727,46 @@ extension CHVirtualMachineInstance { // MARK: - Boot inventory extension CHVirtualMachineInstance.Configuration { - /// Walks boot-time mounts in deterministic order (rootfs first, then - /// `mountsByID` sorted by container id, then each container's mounts in - /// input order), allocating disk letters for virtio-blk mounts and seeding - /// the per-container `AttachedFilesystem` registry. + /// Walks boot-time storage in the machine's device order (rootfs first, + /// then containers sorted by id, each in role order, then volumes sorted + /// by name), allocating disk letters for virtio-blk mounts and seeding + /// the machine's `AttachedFilesystem` registry. /// /// The allocator is shared with the runtime hotplug provider, so block /// hotplug picks up at the next free letter after boot. func bootInventory( allocator: any AddressAllocator - ) throws -> (attachments: [String: [AttachedFilesystem]], bootDisks: [CHVirtualMachineInstance.BootDisk]) { + ) throws -> (attachments: MachineAttachments, bootDisks: [CHVirtualMachineInstance.BootDisk]) { var bootDisks: [CHVirtualMachineInstance.BootDisk] = [] - var attachments: [String: [AttachedFilesystem]] = [:] - // Rootfs is not part of mountsByID. If it's a block device, it claims - // the first letter (vda) so the kernel cmdline `root=/dev/vda` is right. + // The rootfs is the machine's own, not part of its containers' + // storage. If it's a block device, it claims the first letter (vda) + // so the kernel cmdline `root=/dev/vda` is right. if let rootfs = self.initialFilesystem, rootfs.isBlock { - let letter = try allocator.allocate() - bootDisks.append(.init(mount: rootfs, containerId: nil, letter: letter)) + _ = try allocator.allocate() + bootDisks.append(.init(mount: rootfs, chId: "rootfs", readonly: true)) } - for cid in self.mountsByID.keys.sorted() { - guard let mounts = self.mountsByID[cid] else { continue } - var perContainer: [AttachedFilesystem] = [] - for mount in mounts { - let attached = try AttachedFilesystem(mount: mount, allocator: allocator) - if mount.isBlock, let letter = attached.source.last { - bootDisks.append(.init(mount: mount, containerId: cid, letter: letter)) - } - perContainer.append(attached) + func attach(_ mount: Mount, chId: (Character) -> String) throws -> AttachedFilesystem { + let attached = try AttachedFilesystem(mount: mount, allocator: allocator) + if mount.isBlock, let letter = attached.source.last { + bootDisks.append(.init(mount: mount, chId: chId(letter), readonly: false)) } - attachments[cid] = perContainer + return attached + } + + var containers: [String: ContainerAttachments] = [:] + for cid in self.storage.containers.keys.sorted() { + guard let container = self.storage.containers[cid] else { continue } + containers[cid] = try container.map { try attach($0, chId: { "blk-\(cid)-\($0)" }) } + } + var volumes: [String: AttachedFilesystem] = [:] + for name in self.storage.volumes.keys.sorted() { + guard let mount = self.storage.volumes[name] else { continue } + volumes[name] = try attach(mount, chId: { "vol-\(name)-\($0)" }) } - return (attachments, bootDisks) + return (MachineAttachments(containers: containers, volumes: volumes), bootDisks) } } #endif diff --git a/Sources/Containerization/CHVirtualMachineManager.swift b/Sources/Containerization/CHVirtualMachineManager.swift index ba20b5d1..c444620d 100644 --- a/Sources/Containerization/CHVirtualMachineManager.swift +++ b/Sources/Containerization/CHVirtualMachineManager.swift @@ -99,7 +99,7 @@ public struct CHVirtualMachineManager: VirtualMachineManager { instanceConfig.cpus = vmConfig.cpus instanceConfig.memoryInBytes = vmConfig.memoryInBytes instanceConfig.interfaces = vmConfig.interfaces - instanceConfig.mountsByID = vmConfig.mountsByID + instanceConfig.storage = vmConfig.storage instanceConfig.bootLog = vmConfig.bootLog instanceConfig.extensions = vmConfig.extensions instanceConfig.kernel = kernel diff --git a/Sources/Containerization/ContainerStorage.swift b/Sources/Containerization/ContainerStorage.swift new file mode 100644 index 00000000..4d8de2ed --- /dev/null +++ b/Sources/Containerization/ContainerStorage.swift @@ -0,0 +1,107 @@ +//===----------------------------------------------------------------------===// +// Copyright © 2026 Apple Inc. and the Containerization project authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//===----------------------------------------------------------------------===// + +/// A container's storage, by role. +/// +/// The same shape describes storage at both stages of its life: as the +/// `Mount` values a container is configured with, and as the +/// `AttachedFilesystem` values the machine reports once they are attached. +/// Converting between the two is a `map` over the structure, so the roles +/// cannot drift apart between configuration and attachment. +public struct ContainerStorage: Sendable { + /// The container's root filesystem. + public var rootfs: Value + + /// The writable layer mounted over the rootfs as the upper side of an + /// overlay, when the container has one. + public var writableLayer: Value? + + /// The container's remaining mounts, in configuration order. + public var mounts: [Value] + + public init(rootfs: Value, writableLayer: Value? = nil, mounts: [Value] = []) { + self.rootfs = rootfs + self.writableLayer = writableLayer + self.mounts = mounts + } + + /// Every value in the structure: the rootfs, the writable layer when + /// present, then the mounts, in that order. + public var all: [Value] { + var values = [rootfs] + if let writableLayer { + values.append(writableLayer) + } + values.append(contentsOf: mounts) + return values + } + + /// The storage with `transform` applied to every value, each keeping + /// its role. + public func map(_ transform: (Value) throws -> U) rethrows -> ContainerStorage { + ContainerStorage( + rootfs: try transform(rootfs), + writableLayer: try writableLayer.map(transform), + mounts: try mounts.map(transform) + ) + } +} + +/// A container's storage as configured, before its machine exists. +public typealias ContainerMounts = ContainerStorage + +/// A container's storage as attached to its machine. +public typealias ContainerAttachments = ContainerStorage + +/// The storage a machine carries: each container's, and the resources its +/// containers share. +public struct MachineStorage: Sendable { + /// Each container's storage, by container ID. + public var containers: [String: ContainerStorage] + + /// Volumes shared by the machine's containers, by volume name. + public var volumes: [String: Value] + + public init(containers: [String: ContainerStorage] = [:], volumes: [String: Value] = [:]) { + self.containers = containers + self.volumes = volumes + } + + /// Every value the machine carries: containers sorted by ID, each in + /// role order, then volumes sorted by name. Walks that allocate device + /// addresses and walks that create the devices use this one order, so an + /// address always names the device it was handed out for. + public var ordered: [Value] { + containers.keys.sorted().flatMap { containers[$0]?.all ?? [] } + + volumes.keys.sorted().compactMap { volumes[$0] } + } + + /// The storage with `transform` applied to every value, each keeping its + /// place. The transform runs in no particular order, so an allocating + /// conversion walks the fields sorted itself. + public func map(_ transform: (Value) throws -> U) rethrows -> MachineStorage { + MachineStorage( + containers: try containers.mapValues { try $0.map(transform) }, + volumes: try volumes.mapValues(transform) + ) + } +} + +/// A machine's storage as configured, before it exists. +public typealias MachineMounts = MachineStorage + +/// A machine's storage as attached. +public typealias MachineAttachments = MachineStorage diff --git a/Sources/Containerization/HotplugProvider.swift b/Sources/Containerization/HotplugProvider.swift index f535ce3f..0835077f 100644 --- a/Sources/Containerization/HotplugProvider.swift +++ b/Sources/Containerization/HotplugProvider.swift @@ -26,7 +26,7 @@ public protocol HotplugProvider: Sendable { /// - Returns: The attached filesystem with the device path in the guest func hotplug(_ block: Mount, id: String) async throws -> AttachedFilesystem - /// Register mounts for a container in the VM's mount registry. + /// Register mounts for a container in the machine's storage. /// - Parameters: /// - id: The container ID /// - rootfs: The rootfs attachment from hotplug diff --git a/Sources/Containerization/LinuxContainer.swift b/Sources/Containerization/LinuxContainer.swift index 34964fcd..55bbb39a 100644 --- a/Sources/Containerization/LinuxContainer.swift +++ b/Sources/Containerization/LinuxContainer.swift @@ -537,24 +537,14 @@ extension LinuxContainer { } private func mountRootfs( - attachments: [AttachedFilesystem], + attached: ContainerAttachments, rootfsPath: String, agent: VirtualMachineAgent ) async throws { - guard let rootfsAttachment = attachments.first else { - throw ContainerizationError(.notFound, message: "rootfs mount not found") - } + let rootfsAttachment = attached.rootfs - if self.writableLayer != nil { + if let writableAttachment = attached.writableLayer { // Set up overlayfs with image as lower layer and writable layer as upper. - guard attachments.count >= 2 else { - throw ContainerizationError( - .notFound, - message: "writable layer mount not found" - ) - } - let writableAttachment = attachments[1] - let lowerPath = "/run/container/\(self.id)/lower" let upperMountPath = "/run/container/\(self.id)/upper" let upperPath = "/run/container/\(self.id)/upper/diff" @@ -623,17 +613,18 @@ extension LinuxContainer { // This is dumb, but alas. let fileMountContextHolder = Mutex(fileMountContext) - // Build the list of mounts to attach to the VM. - var containerMounts = [modifiedRootfs] + fileMountContext.transformedMounts - if let writableLayer = self.writableLayer { - containerMounts.insert(writableLayer, at: 1) - } + // Build the container's storage to attach to the VM. + let containerStorage = ContainerMounts( + rootfs: modifiedRootfs, + writableLayer: self.writableLayer, + mounts: fileMountContext.transformedMounts + ) let vmConfig = VMConfiguration( cpus: vmCpus, memoryInBytes: vmMemory, interfaces: self.interfaces, - mountsByID: [self.id: containerMounts], + storage: MachineMounts(containers: [self.id: containerStorage]), bootLog: self.config.bootLog, nestedVirtualization: self.config.virtualization ) @@ -643,7 +634,7 @@ extension LinuxContainer { do { try await vm.start() - let mountsForAgent = containerMounts + let storageForAgent = containerStorage try await vm.withAgent { agent in try await agent.standardSetup() @@ -656,7 +647,7 @@ extension LinuxContainer { // with zero shares), but the cloud-hypervisor backend // only spawns virtiofsd when shares exist; mounting an // unbacked tag fails with EINVAL. - let hasVirtiofsMount = mountsForAgent.contains { mount in + let hasVirtiofsMount = storageForAgent.all.contains { mount in if case .virtiofs = mount.runtimeOptions { return true } return false } @@ -671,7 +662,7 @@ extension LinuxContainer { // gets populated. if vm.virtiofsLayout == .perTag { try await agent.mkdir(path: "/run/virtiofs", all: true, perms: 0o755) - let virtiofsAttachments = (vm.mounts[self.id] ?? []).filter { $0.type == "virtiofs" } + let virtiofsAttachments = (vm.storage.containers[self.id]?.all ?? []).filter { $0.type == "virtiofs" } let uniqueTags = Set(virtiofsAttachments.map(\.source)) for tag in uniqueTags { let dest = "/run/virtiofs/\(tag)" @@ -695,18 +686,17 @@ extension LinuxContainer { } } - guard let attachments = vm.mounts[self.id] else { + guard let attached = vm.storage.containers[self.id] else { throw ContainerizationError(.notFound, message: "rootfs mount not found") } let rootfsPath = Self.guestRootfsPath(self.id) - try await self.mountRootfs(attachments: attachments, rootfsPath: rootfsPath, agent: agent) + try await self.mountRootfs(attached: attached, rootfsPath: rootfsPath, agent: agent) // Mount file mount holding directories under /run. if fileMountContext.hasFileMounts { - let containerMounts = vm.mounts[self.id] ?? [] var ctx = fileMountContextHolder.withLock { $0 } try await ctx.mountHoldingDirectories( - vmMounts: containerMounts, + vmMounts: attached.mounts, agent: agent ) fileMountContextHolder.withLock { $0 = ctx } @@ -764,15 +754,12 @@ extension LinuxContainer { let agent = try await createdState.vm.dialAgent() do { var spec = self.generateRuntimeSpec() - // We don't need the rootfs (or writable layer), nor do OCI runtimes want it included. - // Also filter out file mount holding directories. We'll mount those separately under /run. + // Filter out file mount holding directories. We'll mount those separately under /run. // Transform virtiofs mounts to bind mounts from /run/virtiofs/{tag} - let containerMounts = createdState.vm.mounts[self.id] ?? [] + let containerMounts = createdState.vm.storage.containers[self.id]?.mounts ?? [] let holdingTags = createdState.fileMountContext.holdingDirectoryTags - // Drop rootfs, and writable layer if present. - let mountsToSkip = self.writableLayer != nil ? 2 : 1 var mounts: [ContainerizationOCI.Mount] = - containerMounts.dropFirst(mountsToSkip) + containerMounts .filter { !holdingTags.contains($0.source) } .map { attached -> ContainerizationOCI.Mount in if attached.type == "virtiofs" { diff --git a/Sources/Containerization/LinuxPod.swift b/Sources/Containerization/LinuxPod.swift index 6275a4d4..fa4d1785 100644 --- a/Sources/Containerization/LinuxPod.swift +++ b/Sources/Containerization/LinuxPod.swift @@ -446,24 +446,28 @@ extension LinuxPod { // the container's bind mounts from /run/virtiofs/ fail // with ENOENT. // - // Derive the tags from the additional mounts directly rather - // than from vm.mounts[id], so this is independent of the - // rootfs (which may be virtiofs or virtio-blk) and of mount - // ordering. The rootfs is mounted at /run/container//rootfs - // and is never consumed from /run/virtiofs. + // Derive the new tags from the additional mounts being added; + // the machine's storage names what is already shared. let newVirtiofsTags = try virtioFSMounts.map { try hashFilePath(path: $0.source) } if !newVirtiofsTags.isEmpty { try await agent.mkdir(path: "/run/virtiofs", all: true, perms: 0o755) if vm.virtiofsLayout == .perTag { // Tags already mounted in the guest at boot or by a - // prior hotplug (i.e. present on another container). - let alreadyMounted = Set( - vm.mounts - .filter { $0.key != id } - .values.flatMap { $0 } - .filter { $0.type == "virtiofs" } - .map { $0.source } - ) + // prior hotplug (i.e. present on another container or + // a machine volume). + let alreadyMounted: Set = { + var mounted = Set( + vm.storage.containers + .filter { $0.key != id } + .values.flatMap { $0.all } + .filter { $0.type == "virtiofs" } + .map { $0.source }) + mounted.formUnion( + vm.storage.volumes.values + .filter { $0.type == "virtiofs" } + .map { $0.source }) + return mounted + }() var seen: Set = [] for tag in newVirtiofsTags where !alreadyMounted.contains(tag) && seen.insert(tag).inserted { @@ -493,7 +497,7 @@ extension LinuxPod { } if fileMountContext.hasFileMounts { - let containerMounts = vm.mounts[id] ?? [] + let containerMounts = vm.storage.containers[id]?.mounts ?? [] try await updatedFileMountContext.mountHoldingDirectories( vmMounts: containerMounts, agent: agent @@ -557,20 +561,23 @@ extension LinuxPod { try await self.state.withLock { state in try state.phase.validateForCreate() - // Build mountsByID for all containers. + // Build the machine's storage from its containers. // Strip "ro" from rootfs options - we handle readonly via the OCI spec's // root.readonly field and remount in vmexec after setup is complete. // Use transformedMounts from fileMountContext (file mounts become directory shares). - var mountsByID: [String: [Mount]] = [:] + var machineStorage = MachineMounts() for (id, container) in state.containers { var modifiedRootfs = container.rootfs modifiedRootfs.options.removeAll(where: { $0 == "ro" }) - // Filter out shared mounts — those are handled separately as pod volume bind mounts. + // Shared mounts are handled separately as pod volume bind mounts. let containerMounts = container.fileMountContext.transformedMounts.filter { if case .shared = $0.runtimeOptions { return false } return true } - mountsByID[id] = [modifiedRootfs] + containerMounts + machineStorage.containers[id] = ContainerMounts( + rootfs: modifiedRootfs, + mounts: containerMounts + ) } // Validate pod volume names are unique. @@ -597,9 +604,8 @@ extension LinuxPod { } } } - let podVolumeMounts = self.config.volumes.map { $0.toMount() } - if !podVolumeMounts.isEmpty { - mountsByID[self.id] = podVolumeMounts + for volume in self.config.volumes { + machineStorage.volumes[volume.name] = volume.toMount() } // Capture into an immutable `let` so the value is safely usable @@ -608,18 +614,16 @@ extension LinuxPod { // only attaches a virtiofs device when shares are configured, // so mounting an unbacked /run/virtiofs would fail with EINVAL // on the CH backend. - let hasVirtiofsMount = mountsByID.values.contains { mounts in - mounts.contains { mount in - if case .virtiofs = mount.runtimeOptions { return true } - return false - } + let hasVirtiofsMount = machineStorage.ordered.contains { mount in + if case .virtiofs = mount.runtimeOptions { return true } + return false } var vmConfig = VMConfiguration( cpus: self.config.cpus, memoryInBytes: self.config.memoryInBytes, interfaces: self.config.interfaces, - mountsByID: mountsByID, + storage: machineStorage, bootLog: self.config.bootLog, nestedVirtualization: self.config.virtualization ) @@ -649,19 +653,17 @@ extension LinuxPod { // /run/virtiofs/. See LinuxContainer for the // VZ vs. CH model split. var seenTags: Set = [] - for (_, attached) in vm.mounts { - for entry in attached where entry.type == "virtiofs" { - guard seenTags.insert(entry.source).inserted else { continue } - let dest = "/run/virtiofs/\(entry.source)" - try await agent.mkdir(path: dest, all: true, perms: 0o755) - try await agent.mount( - ContainerizationOCI.Mount( - type: "virtiofs", - source: entry.source, - destination: dest, - options: [] - )) - } + for entry in vm.storage.ordered where entry.type == "virtiofs" { + guard seenTags.insert(entry.source).inserted else { continue } + let dest = "/run/virtiofs/\(entry.source)" + try await agent.mkdir(path: dest, all: true, perms: 0o755) + try await agent.mount( + ContainerizationOCI.Mount( + type: "virtiofs", + source: entry.source, + destination: dest, + options: [] + )) } } else { try await agent.mount( @@ -728,10 +730,10 @@ extension LinuxPod { // Mount all container rootfs for (_, container) in containers { - guard let attachments = vm.mounts[container.id], let rootfsAttachment = attachments.first else { + guard let attached = vm.storage.containers[container.id] else { throw ContainerizationError(.notFound, message: "rootfs mount not found for container \(container.id)") } - var rootfs = rootfsAttachment.to + var rootfs = attached.rootfs.to rootfs.destination = Self.guestRootfsPath(container.id) try await agent.mount(rootfs) } @@ -740,7 +742,7 @@ extension LinuxPod { for (id, container) in containers { if container.fileMountContext.hasFileMounts { var ctx = container.fileMountContext - let containerMounts = vm.mounts[id] ?? [] + let containerMounts = vm.storage.containers[id]?.mounts ?? [] try await ctx.mountHoldingDirectories( vmMounts: containerMounts, agent: agent @@ -750,15 +752,13 @@ extension LinuxPod { } // Mount pod-level volumes. - let podVolumeAttachments = vm.mounts[self.id] ?? [] - for (index, volume) in self.config.volumes.enumerated() { - guard index < podVolumeAttachments.count else { + for volume in self.config.volumes { + guard let attachment = vm.storage.volumes[volume.name] else { throw ContainerizationError( .notFound, message: "attached filesystem not found for pod volume \"\(volume.name)\"" ) } - let attachment = podVolumeAttachments[index] let guestPath = Self.guestVolumePath(volume.name) try await agent.mount( ContainerizationOCI.Mount( @@ -864,10 +864,10 @@ extension LinuxPod { // We don't need the rootfs, nor do OCI runtimes want it included. // Also filter out file mount holding directories - we mount those separately under /run. // Transform virtiofs mounts to bind mounts from /run/virtiofs/{tag} - let containerMounts = createdState.vm.mounts[containerID] ?? [] + let containerMounts = createdState.vm.storage.containers[containerID]?.mounts ?? [] let holdingTags = container.fileMountContext.holdingDirectoryTags var mounts: [ContainerizationOCI.Mount] = - containerMounts.dropFirst() + containerMounts .filter { !holdingTags.contains($0.source) } .map { attached -> ContainerizationOCI.Mount in if attached.type == "virtiofs" { diff --git a/Sources/Containerization/VMConfiguration.swift b/Sources/Containerization/VMConfiguration.swift index 30faebc4..7a5e8e2c 100644 --- a/Sources/Containerization/VMConfiguration.swift +++ b/Sources/Containerization/VMConfiguration.swift @@ -72,9 +72,9 @@ public struct VMConfiguration: Sendable { public var memoryInBytes: UInt64 /// The network interfaces to attach. public var interfaces: [any Interface] - /// Mounts organized by metadata ID (e.g. container ID). - /// Each ID maps to an array of mounts for that workload. - public var mountsByID: [String: [Mount]] + /// The storage the machine carries: each container's mounts by role, + /// and the volumes its containers share. + public var storage: MachineMounts /// Optional destination for serial boot logs. public var bootLog: BootLog? /// Enable nested virtualization support. If the VirtualMachineManager @@ -89,14 +89,14 @@ public struct VMConfiguration: Sendable { cpus: Int = 4, memoryInBytes: UInt64 = 1024 * 1024 * 1024, interfaces: [any Interface] = [], - mountsByID: [String: [Mount]] = [:], + storage: MachineMounts = MachineMounts(), bootLog: BootLog? = nil, nestedVirtualization: Bool = false ) { self.cpus = cpus self.memoryInBytes = memoryInBytes self.interfaces = interfaces - self.mountsByID = mountsByID + self.storage = storage self.bootLog = bootLog self.nestedVirtualization = nestedVirtualization } diff --git a/Sources/Containerization/VZVirtualMachineInstance.swift b/Sources/Containerization/VZVirtualMachineInstance.swift index 160c5026..670ab512 100644 --- a/Sources/Containerization/VZVirtualMachineInstance.swift +++ b/Sources/Containerization/VZVirtualMachineInstance.swift @@ -28,10 +28,10 @@ import Synchronization public final class VZVirtualMachineInstance: Sendable { public typealias Agent = Vminitd - /// Attached mounts on the virtual machine, organized by metadata ID. - private let _mounts: Mutex<[String: [AttachedFilesystem]]> - public var mounts: [String: [AttachedFilesystem]] { - _mounts.withLock { $0 } + /// The machine's attached storage. + private let _storage: Mutex + public var storage: MachineAttachments { + _storage.withLock { $0 } } /// The underlying Virtualization framework virtual machine. @@ -40,9 +40,9 @@ public final class VZVirtualMachineInstance: Sendable { /// The dispatch queue used for VZ operations. public var vmQueue: DispatchQueue { queue } - /// Mutate the mount registry. - public func withMountRegistry(_ body: (inout sending [String: [AttachedFilesystem]]) throws -> sending T) rethrows -> T { - try _mounts.withLock(body) + /// Mutate the storage registry. + public func withStorage(_ body: (inout sending MachineAttachments) throws -> sending T) rethrows -> T { + try _storage.withLock(body) } /// Serialize VM operations with the instance lock. @@ -73,8 +73,9 @@ public final class VZVirtualMachineInstance: Sendable { public var rosetta: Bool /// Toggle nested virtualization support. public var nestedVirtualization: Bool - /// Mount attachments organized by metadata ID. - public var mountsByID: [String: [Mount]] + /// The machine's storage: each container's mounts by role, and the + /// volumes its containers share. + public var storage: MachineMounts /// Network interface attachments. public var interfaces: [any Interface] /// Kernel image. @@ -91,7 +92,7 @@ public final class VZVirtualMachineInstance: Sendable { self.memoryInBytes = 1024.mib() self.rosetta = false self.nestedVirtualization = false - self.mountsByID = [:] + self.storage = MachineMounts() self.interfaces = [] } } @@ -132,7 +133,7 @@ public final class VZVirtualMachineInstance: Sendable { let allocator = Character.blockDeviceTagAllocator() let (mountAttachments, _) = try config.mountAttachments(allocator: allocator) - self._mounts = Mutex(mountAttachments) + self._storage = Mutex(mountAttachments) self.vm = VZVirtualMachine( configuration: try config.toVZ(allocator: allocator), @@ -153,7 +154,7 @@ public protocol VZInstanceExtension: Sendable { _ config: inout VZVirtualMachineConfiguration, allocator: any AddressAllocator, storageDeviceCount: Int, - mountsByID: [String: [Mount]] + storage: MachineMounts ) throws /// Called after the VZVirtualMachine is created but before start. @@ -168,7 +169,7 @@ extension VZInstanceExtension { _ config: inout VZVirtualMachineConfiguration, allocator: any AddressAllocator, storageDeviceCount: Int, - mountsByID: [String: [Mount]] + storage: MachineMounts ) throws {} public func didCreate(_ instance: VZVirtualMachineInstance) throws {} @@ -477,35 +478,33 @@ extension VZVirtualMachineInstance.Configuration { // Track used virtiofs tags to avoid creating duplicate VZ devices. // The same source directory mounted to multiple destinations shares one device. + // The walk is the machine's device order, matching the addresses + // `mountAttachments` hands out walking the same way. var usedVirtioFSTags: Set = [] - for (_, mounts) in self.mountsByID { - for mount in mounts { - if case .virtiofs = mount.runtimeOptions { - let tag = try hashFilePath(path: mount.source) - if usedVirtioFSTags.contains(tag) { - continue - } - usedVirtioFSTags.insert(tag) + for mount in self.storage.ordered { + if case .virtiofs = mount.runtimeOptions { + let tag = try hashFilePath(path: mount.source) + if usedVirtioFSTags.contains(tag) { + continue } - try mount.configure(config: &config) + usedVirtioFSTags.insert(tag) } + try mount.configure(config: &config) } // Create the unified virtiofs device with VZMultipleDirectoryShare // This device hosts all virtiofs shares and supports runtime updates var directories: [String: VZSharedDirectory] = [:] - for (_, mounts) in self.mountsByID { - for mount in mounts { - guard case .virtiofs(_) = mount.runtimeOptions else { continue } - guard FileManager.default.fileExists(atPath: mount.source) else { - throw ContainerizationError(.notFound, message: "directory \(mount.source) does not exist") - } - let name = try hashFilePath(path: mount.source) - directories[name] = VZSharedDirectory( - url: URL(fileURLWithPath: mount.source), - readOnly: mount.options.contains("ro") - ) + for mount in self.storage.ordered { + guard case .virtiofs(_) = mount.runtimeOptions else { continue } + guard FileManager.default.fileExists(atPath: mount.source) else { + throw ContainerizationError(.notFound, message: "directory \(mount.source) does not exist") } + let name = try hashFilePath(path: mount.source) + directories[name] = VZSharedDirectory( + url: URL(fileURLWithPath: mount.source), + readOnly: mount.options.contains("ro") + ) } let multiShare = VZMultipleDirectoryShare(directories: directories) let virtiofsDevice = VZVirtioFileSystemDeviceConfiguration(tag: "virtiofs") @@ -527,7 +526,7 @@ extension VZVirtualMachineInstance.Configuration { config.platform = platform for ext in self.extensions.compactMap({ $0 as? any VZInstanceExtension }) { - try ext.configureVZ(&config, allocator: allocator, storageDeviceCount: storageDeviceCount, mountsByID: self.mountsByID) + try ext.configureVZ(&config, allocator: allocator, storageDeviceCount: storageDeviceCount, storage: self.storage) } try config.validate() @@ -535,7 +534,7 @@ extension VZVirtualMachineInstance.Configuration { } func mountAttachments(allocator: any AddressAllocator) throws -> ( - attachments: [String: [AttachedFilesystem]], storageDeviceCount: Int + attachments: MachineAttachments, storageDeviceCount: Int ) { var storageDeviceCount = 0 @@ -548,21 +547,28 @@ extension VZVirtualMachineInstance.Configuration { } } - var attachmentsByID: [String: [AttachedFilesystem]] = [:] - - for (id, mounts) in self.mountsByID { - var attachments: [AttachedFilesystem] = [] - for mount in mounts { - let attached = try AttachedFilesystem(mount: mount, allocator: allocator) - attachments.append(attached) - if mount.isBlock { - storageDeviceCount += 1 - } + // The machine's device order: addresses are handed out in the same + // walk `makeConfiguration` creates the devices in. + func attach(_ mount: Mount) throws -> AttachedFilesystem { + let attached = try AttachedFilesystem(mount: mount, allocator: allocator) + if mount.isBlock { + storageDeviceCount += 1 } - attachmentsByID[id] = attachments + return attached + } + + var containers: [String: ContainerAttachments] = [:] + for id in self.storage.containers.keys.sorted() { + guard let container = self.storage.containers[id] else { continue } + containers[id] = try container.map(attach) + } + var volumes: [String: AttachedFilesystem] = [:] + for name in self.storage.volumes.keys.sorted() { + guard let mount = self.storage.volumes[name] else { continue } + volumes[name] = try attach(mount) } - return (attachmentsByID, storageDeviceCount) + return (MachineAttachments(containers: containers, volumes: volumes), storageDeviceCount) } } diff --git a/Sources/Containerization/VZVirtualMachineManager.swift b/Sources/Containerization/VZVirtualMachineManager.swift index 4959bee4..d0888447 100644 --- a/Sources/Containerization/VZVirtualMachineManager.swift +++ b/Sources/Containerization/VZVirtualMachineManager.swift @@ -76,7 +76,7 @@ public struct VZVirtualMachineManager: VirtualMachineManager { instanceConfig.rosetta = self.rosetta instanceConfig.nestedVirtualization = useNestedVirtualization - instanceConfig.mountsByID = vmConfig.mountsByID + instanceConfig.storage = vmConfig.storage instanceConfig.extensions = vmConfig.extensions }) } diff --git a/Sources/Containerization/VirtualMachineInstance.swift b/Sources/Containerization/VirtualMachineInstance.swift index 302e97ae..b9340612 100644 --- a/Sources/Containerization/VirtualMachineInstance.swift +++ b/Sources/Containerization/VirtualMachineInstance.swift @@ -44,7 +44,8 @@ public protocol VirtualMachineInstance: Sendable { // The state of the virtual machine. var state: VirtualMachineInstanceState { get } - var mounts: [String: [AttachedFilesystem]] { get } + /// The machine's attached storage. + var storage: MachineAttachments { get } /// How this VMM exposes virtiofs devices to the guest. Defaults to /// `.unified` (the VZ-shaped behavior); CH overrides to `.perTag`. @@ -73,7 +74,7 @@ public protocol VirtualMachineInstance: Sendable { func hotplug(_ block: Mount, id: String) async throws -> AttachedFilesystem /// Register mounts for a container after hotplug. - /// This is used to add the rootfs and additional mounts to the VM's mount registry + /// This is used to add the rootfs and additional mounts to the machine's storage /// so they can be found when building the container's OCI spec. /// - Parameter id: The container ID /// - Parameter rootfs: The rootfs attachment from hotplug From b3d72088b9deaeda3c02e4dcf377c2831c1473a0 Mon Sep 17 00:00:00 2001 From: Aaron Paterson Date: Sat, 1 Aug 2026 19:06:18 +0000 Subject: [PATCH 06/25] Add guest swap support A container can be given a swap area, so a workload whose memory exceeds its limit is reclaimed rather than killed. The area is a raw file on the host attached as a block device, which is how a guest gets swap it can write to when its own root is read only, and vminitd formats and enables it while mounting the container's filesystems. The area is enabled with discard, and the device backing it is marked non rotational first. virtio block devices are rotational by default, and the kernel only tracks a swap area in clusters when its device is non rotational, so without that the discard flags are accepted and no discard is ever issued. Together they let the sparse file that backs the area release the blocks the guest stops using, rather than holding the area's high water mark for as long as the container runs. --- Sources/CShim/include/swap.h | 36 ++++ Sources/CShim/swap.c | 23 +++ .../Containerization/ContainerManager.swift | 57 +++++- .../Containerization/ContainerStorage.swift | 14 +- Sources/Containerization/LinuxContainer.swift | 40 +++- Sources/ContainerizationOS/Linux/Swap.swift | 176 +++++++++++++++++ Sources/Integration/ContainerTests.swift | 178 ++++++++++++++++++ Sources/Integration/Suite.swift | 26 +++ Sources/cctl/RunCommand.swift | 4 + Tests/ContainerizationTests/SwapTests.swift | 73 +++++++ vminitd/Sources/VminitdCore/Server+GRPC.swift | 15 ++ 11 files changed, 635 insertions(+), 7 deletions(-) create mode 100644 Sources/CShim/include/swap.h create mode 100644 Sources/CShim/swap.c create mode 100644 Sources/ContainerizationOS/Linux/Swap.swift create mode 100644 Tests/ContainerizationTests/SwapTests.swift diff --git a/Sources/CShim/include/swap.h b/Sources/CShim/include/swap.h new file mode 100644 index 00000000..f16000ee --- /dev/null +++ b/Sources/CShim/include/swap.h @@ -0,0 +1,36 @@ +/* + * Copyright © 2026 Apple Inc. and the Containerization project authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef __SWAP_H +#define __SWAP_H + +#if defined(__linux__) + +// swapon(2) lives in , which Swift's glibc modulemap does not +// carry, so it is reachable from Swift only through a wrapper. + +// Discard the whole area when it is enabled, and each page as it is freed. +// These come from the same header, and SWAP_FLAG_DISCARD_PAGES has no UAPI +// header of its own at all, so every consumer declares it; see util-linux +// sys-utils/swapon.c. +#define CZ_SWAP_DISCARD 0x10000 +#define CZ_SWAP_DISCARD_PAGES 0x40000 + +int CZ_swapon(const char *path, int flags); + +#endif + +#endif diff --git a/Sources/CShim/swap.c b/Sources/CShim/swap.c new file mode 100644 index 00000000..2a66461d --- /dev/null +++ b/Sources/CShim/swap.c @@ -0,0 +1,23 @@ +/* + * Copyright © 2026 Apple Inc. and the Containerization project authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifdef __linux__ +#include + +#include "swap.h" + +int CZ_swapon(const char *path, int flags) { return swapon(path, flags); } +#endif diff --git a/Sources/Containerization/ContainerManager.swift b/Sources/Containerization/ContainerManager.swift index 27e9fbe4..a1b082e1 100644 --- a/Sources/Containerization/ContainerManager.swift +++ b/Sources/Containerization/ContainerManager.swift @@ -195,6 +195,9 @@ public struct ContainerManager: Sendable { /// - writableLayerSizeInBytes: Optional size for a separate writable layer. When provided, /// the rootfs becomes read-only and an overlayfs is used with a separate writable layer of this size. /// - readOnly: Whether to mount the root filesystem as read-only. + /// - swapSizeInBytes: Optional size for a swap area. When provided, a raw block + /// device of this size is created and the guest enables it as swap, so a + /// workload exceeding its memory limit reclaims rather than being killed. /// - networking: Whether to create a network interface for this container. Defaults to `true`. /// When `false`, no network resources are allocated and `releaseNetwork`/`delete` remain safe to call. /// - progress: Optional handler for tracking rootfs unpacking progress. @@ -203,6 +206,7 @@ public struct ContainerManager: Sendable { reference: String, rootfsSizeInBytes: UInt64 = 8.gib(), writableLayerSizeInBytes: UInt64? = nil, + swapSizeInBytes: UInt64? = nil, readOnly: Bool = false, networking: Bool = true, progress: ProgressHandler? = nil, @@ -214,6 +218,7 @@ public struct ContainerManager: Sendable { image: image, rootfsSizeInBytes: rootfsSizeInBytes, writableLayerSizeInBytes: writableLayerSizeInBytes, + swapSizeInBytes: swapSizeInBytes, readOnly: readOnly, networking: networking, progress: progress, @@ -229,6 +234,9 @@ public struct ContainerManager: Sendable { /// - writableLayerSizeInBytes: Optional size for a separate writable layer. When provided, /// the rootfs becomes read-only and an overlayfs is used with a separate writable layer of this size. /// - readOnly: Whether to mount the root filesystem as read-only. + /// - swapSizeInBytes: Optional size for a swap area. When provided, a raw block + /// device of this size is created and the guest enables it as swap, so a + /// workload exceeding its memory limit reclaims rather than being killed. /// - networking: Whether to create a network interface for this container. Defaults to `true`. /// When `false`, no network resources are allocated and `releaseNetwork`/`delete` remain safe to call. /// - progress: Optional handler for tracking rootfs unpacking progress. @@ -237,6 +245,7 @@ public struct ContainerManager: Sendable { image: Image, rootfsSizeInBytes: UInt64 = 8.gib(), writableLayerSizeInBytes: UInt64? = nil, + swapSizeInBytes: UInt64? = nil, readOnly: Bool = false, networking: Bool = true, progress: ProgressHandler? = nil, @@ -263,14 +272,25 @@ public struct ContainerManager: Sendable { ) } + // Create the swap device if a size is specified. + var swapLayer: Mount? = nil + if let swapSize = swapSizeInBytes, swapSize > 0 { + swapLayer = try createSwapDevice( + at: path.appendingPathComponent("swap.raw"), + size: swapSize + ) + } + return try await create( id, image: image, rootfs: rootfs, writableLayer: writableLayer, - networking: networking, - configuration: configuration - ) + networking: networking + ) { config in + config.swapLayer = swapLayer + try configuration(&config) + } } /// Returns a new container from the provided image and root filesystem mount. @@ -357,6 +377,37 @@ public struct ContainerManager: Sendable { } } + /// Create a raw block file to back a container's swap area. + /// + /// It carries no filesystem: the agent writes the swap header to the device + /// and enables it. The file is sparse, so it costs the host only the pages + /// the guest has actually swapped out, and gives them back on discard. A + /// swap area held in a file has to be free of holes, since the kernel walks + /// its extents; the guest reaches this one as a block device, which the + /// kernel takes as a single extent without consulting the host's layout. + /// https://github.com/torvalds/linux/blob/master/mm/swapfile.c + private func createSwapDevice(at destination: URL, size: UInt64) throws -> Mount { + let path = destination.absolutePath() + guard !FileManager.default.fileExists(atPath: path) else { + throw ContainerizationError(.exists, message: "swap device already exists at \(path)") + } + guard FileManager.default.createFile(atPath: path, contents: nil) else { + throw ContainerizationError(.internalError, message: "failed to create swap device at \(path)") + } + let handle = try FileHandle(forWritingTo: destination) + defer { try? handle.close() } + try handle.truncate(atOffset: size) + // A swap area holds nothing that outlives the container, so the host + // has no reason to synchronize it to permanent storage. + return .block( + format: Swap.mountType, + source: path, + destination: "", + options: [], + runtimeOptions: ["vzDiskImageSynchronizationMode=none"] + ) + } + private func createEmptyFilesystem(at destination: URL, size: UInt64) throws -> Mount { let path = destination.absolutePath() guard !FileManager.default.fileExists(atPath: path) else { diff --git a/Sources/Containerization/ContainerStorage.swift b/Sources/Containerization/ContainerStorage.swift index 4d8de2ed..49fae6d4 100644 --- a/Sources/Containerization/ContainerStorage.swift +++ b/Sources/Containerization/ContainerStorage.swift @@ -29,22 +29,29 @@ public struct ContainerStorage: Sendable { /// overlay, when the container has one. public var writableLayer: Value? + /// The swap area enabled for the container, when it has one. + public var swap: Value? + /// The container's remaining mounts, in configuration order. public var mounts: [Value] - public init(rootfs: Value, writableLayer: Value? = nil, mounts: [Value] = []) { + public init(rootfs: Value, writableLayer: Value? = nil, swap: Value? = nil, mounts: [Value] = []) { self.rootfs = rootfs self.writableLayer = writableLayer + self.swap = swap self.mounts = mounts } - /// Every value in the structure: the rootfs, the writable layer when - /// present, then the mounts, in that order. + /// Every value in the structure: the rootfs, the writable layer and swap + /// when present, then the mounts, in that order. public var all: [Value] { var values = [rootfs] if let writableLayer { values.append(writableLayer) } + if let swap { + values.append(swap) + } values.append(contentsOf: mounts) return values } @@ -55,6 +62,7 @@ public struct ContainerStorage: Sendable { ContainerStorage( rootfs: try transform(rootfs), writableLayer: try writableLayer.map(transform), + swap: try swap.map(transform), mounts: try mounts.map(transform) ) } diff --git a/Sources/Containerization/LinuxContainer.swift b/Sources/Containerization/LinuxContainer.swift index 55bbb39a..4d5289a7 100644 --- a/Sources/Containerization/LinuxContainer.swift +++ b/Sources/Containerization/LinuxContainer.swift @@ -99,6 +99,16 @@ public final class LinuxContainer: Container, Sendable { /// The total is aligned to a 1 MiB boundary. public var memoryOverhead: UInt64 = 128.mib() + /// Optional swap area for the container, as a block device mount. + /// + /// Swap lets a workload whose resident set exceeds `memoryInBytes` + /// reclaim rather than meet the out of memory killer. The area is a + /// block device rather than a file in the guest, because no filesystem + /// the agent can write to exists when the sandbox starts: its root is + /// mounted read only. The `destination` field is ignored, as the area + /// is enabled rather than mounted. + public var swapLayer: Mount? = nil + public init() {} public init( @@ -536,6 +546,30 @@ extension LinuxContainer { config.interfaces } + /// Enable the container's swap area, if it has one. + /// + /// The device is attached with the container's other block devices, so the + /// agent is told the guest path the VMM allocated it. It is enabled rather + /// than mounted, which is why it travels as a mount of type `swap`. + private func enableSwap( + attached: ContainerAttachments, + agent: VirtualMachineAgent + ) async throws { + guard self.config.swapLayer != nil else { + return + } + guard let swap = attached.swap else { + throw ContainerizationError(.notFound, message: "swap mount not found") + } + try await agent.mount( + ContainerizationOCI.Mount( + type: Swap.mountType, + source: swap.source, + destination: "", + options: swap.options + )) + } + private func mountRootfs( attached: ContainerAttachments, rootfsPath: String, @@ -613,10 +647,13 @@ extension LinuxContainer { // This is dumb, but alas. let fileMountContextHolder = Mutex(fileMountContext) - // Build the container's storage to attach to the VM. + // Build the container's storage to attach to the VM. The swap + // device is attached with the container's other block devices so + // the guest is told the /dev path the VMM allocates it. let containerStorage = ContainerMounts( rootfs: modifiedRootfs, writableLayer: self.writableLayer, + swap: self.config.swapLayer, mounts: fileMountContext.transformedMounts ) @@ -691,6 +728,7 @@ extension LinuxContainer { } let rootfsPath = Self.guestRootfsPath(self.id) try await self.mountRootfs(attached: attached, rootfsPath: rootfsPath, agent: agent) + try await self.enableSwap(attached: attached, agent: agent) // Mount file mount holding directories under /run. if fileMountContext.hasFileMounts { diff --git a/Sources/ContainerizationOS/Linux/Swap.swift b/Sources/ContainerizationOS/Linux/Swap.swift new file mode 100644 index 00000000..82006738 --- /dev/null +++ b/Sources/ContainerizationOS/Linux/Swap.swift @@ -0,0 +1,176 @@ +//===----------------------------------------------------------------------===// +// Copyright © 2026 Apple Inc. and the Containerization project authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//===----------------------------------------------------------------------===// + +import CShim +import ContainerizationError + +#if canImport(FoundationEssentials) +import FoundationEssentials +#else +import Foundation +#endif + +#if canImport(Musl) +import Musl +#elseif canImport(Glibc) +import Glibc +#endif + +/// `Swap` is a utility type that contains static helpers for creating a swap +/// file and enabling it as swap space for the guest. +/// +/// The swap area header layout is the kernel's on-disk format, version 1: +/// a page of header whose last 10 bytes are the magic `SWAPSPACE2`, with the +/// version, the last usable page and the page count at fixed offsets from the +/// start of the second kilobyte. +/// https://docs.kernel.org/admin-guide/mm/concepts.html +/// https://github.com/torvalds/linux/blob/master/include/linux/swap.h +public struct Swap: Sendable { + /// Offset of the version field within the swap header, and of the last + /// usable page field that follows it. + private static let versionOffset = 1024 + private static let lastPageOffset = 1028 + /// The magic that marks the last bytes of the first page of a swap area. + private static let magic = "SWAPSPACE2" + /// The queue attributes the kernel exposes for each block device. + public static let blockPath = "/sys/block" + + /// Mount type that marks a block device as a container's swap area, which + /// the agent enables rather than mounts. It travels in the `type` field of + /// the mount the host sends, so both sides read it from here. + /// + /// Kata gives its guest swap the same shape, the host attaching a raw file + /// as a block device and the agent calling `swapon` on it, and carries the + /// request on an RPC of its own (`AddSwap`). Here it travels as a mount so + /// the device path the VMM allocates reaches the guest the way every other + /// attached device's does. + /// https://github.com/kata-containers/kata-containers/blob/main/src/agent/src/rpc.rs + public static let mountType = "swap" + + #if os(Linux) + /// Format the swap area at `path` and enable it. + /// + /// `path` may be a block device or a file that already has its final size, + /// which is what `size` describes; `create` makes such a file. + public static func enable(path: String, size: UInt64, pageSize: Int = 4096) throws { + try format(path: path, size: size, pageSize: pageSize) + try on(path: path) + } + + /// Write the swap area header the kernel expects to an existing block + /// device or fully allocated file. + /// + /// This is what `mkswap` writes, and kata has its host run `mkswap` before + /// attaching the device. That is not open to a host which is not Linux, so + /// the header is written here instead, from the guest that is about to + /// enable it. + /// https://github.com/kata-containers/kata-containers/blob/main/src/runtime-rs/crates/resource/src/cpu_mem/swap.rs + public static func format(path: String, size: UInt64, pageSize: Int = 4096) throws { + let pages = size / UInt64(pageSize) + guard pages > 1 else { + throw ContainerizationError( + .invalidArgument, + message: "swap size \(size) is smaller than the two pages a swap area needs" + ) + } + // The header carries the last page number in 32 bits. + guard let lastPage = UInt32(exactly: pages - 1) else { + throw ContainerizationError( + .invalidArgument, + message: "swap size \(size) exceeds the \(UInt64(UInt32.max) + 1) pages a swap header can carry" + ) + } + + let fd = open(path, O_WRONLY) + guard fd >= 0 else { + throw POSIXError.fromErrno() + } + defer { close(fd) } + + var header = [UInt8](repeating: 0, count: pageSize) + header.replaceSubrange(versionOffset..<(versionOffset + 4), with: littleEndianBytes(1)) + header.replaceSubrange( + lastPageOffset..<(lastPageOffset + 4), + with: littleEndianBytes(lastPage) + ) + // The magic occupies the last bytes of the header page. + header.replaceSubrange((pageSize - magic.count).. UInt64 { + let fd = open(path, O_RDONLY) + guard fd >= 0 else { + throw POSIXError.fromErrno() + } + defer { close(fd) } + let end = lseek(fd, 0, SEEK_END) + guard end > 0 else { + throw POSIXError.fromErrno() + } + return UInt64(end) + } + + /// Enable the swap area at `path`, asking the kernel to discard the blocks + /// it stops using so the file backing the area does not keep them. + /// + /// Ours: kata enables its swap with no flags, which suits an area on a + /// host disk that was sized once and stays that size. + /// https://github.com/kata-containers/kata-containers/blob/main/src/agent/src/rpc.rs + public static func on(path: String) throws { + try markSolidState(devicePath: path) + guard CZ_swapon(path, CZ_SWAP_DISCARD | CZ_SWAP_DISCARD_PAGES) == 0 else { + throw POSIXError.fromErrno() + } + } + + /// Mark the block device backing the swap area as non rotational. + /// + /// Ours: no other runtime does this, because no other runtime needs the + /// area to give its blocks back. Kata calls `swapon` with no flags at all. + /// + /// The kernel only tracks a swap area in clusters when its device is non + /// rotational, and freeing a cluster is the only thing that schedules a + /// discard. A virtio block device reports as rotational, so without this + /// the area is scanned rather than clustered and the discard flags above + /// never take effect, leaving the file that backs the area holding every + /// block the guest has ever swapped to. + /// https://github.com/torvalds/linux/blob/master/mm/swapfile.c + static func markSolidState(devicePath: String) throws { + let device = URL(fileURLWithPath: devicePath).lastPathComponent + try "0".write( + to: URL(fileURLWithPath: Self.blockPath) + .appendingPathComponent(device) + .appendingPathComponent("queue") + .appendingPathComponent("rotational"), + atomically: false, + encoding: .ascii + ) + } + + private static func littleEndianBytes(_ value: UInt32) -> [UInt8] { + withUnsafeBytes(of: value.littleEndian) { Array($0) } + } + + #endif // os(Linux) +} diff --git a/Sources/Integration/ContainerTests.swift b/Sources/Integration/ContainerTests.swift index dd2a9165..9980cf21 100644 --- a/Sources/Integration/ContainerTests.swift +++ b/Sources/Integration/ContainerTests.swift @@ -137,6 +137,184 @@ extension IntegrationSuite { } } + func testContainerSwap() async throws { + let id = "test-container-swap" + let bs = try await bootstrap(id) + + let swapPath = Self.binPath(name: "\(id)-swap.raw") + let swap = try Self.makeSwapDevice(at: swapPath, size: 64.mib()) + defer { try? FileManager.default.removeItem(at: swapPath) } + + let buffer = BufferWriter() + let container = try LinuxContainer(id, rootfs: bs.rootfs, vmm: bs.vmm) { config in + config.process.arguments = ["/bin/cat", "/proc/swaps"] + config.process.stdout = buffer + config.swapLayer = swap + config.bootLog = bs.bootLog + } + + do { + try await container.create() + try await container.start() + + let status = try await container.wait() + try await container.stop() + + guard status.exitCode == 0 else { + throw IntegrationError.assert(msg: "process status \(status) != 0") + } + + // The guest lists the area it enabled, so the device is present and + // the kernel accepted the header the agent wrote to it. + let swaps = String(data: buffer.data, encoding: .utf8) ?? "" + guard swaps.contains("partition") else { + throw IntegrationError.assert(msg: "swap not enabled in guest: '\(swaps)'") + } + } catch { + try? await container.stop() + throw error + } + } + + /// The point of swap: a workload whose pages exceed the memory limit + /// reclaims instead of meeting the out of memory killer. The tmpfs is + /// sized explicitly because its default is half of RAM, which cannot + /// exceed the limit and so would never drive a page out. + func testContainerSwapUnderPressure() async throws { + let id = "test-container-swap-pressure" + let bs = try await bootstrap(id) + + let swapPath = Self.binPath(name: "\(id)-swap.raw") + let swap = try Self.makeSwapDevice(at: swapPath, size: 512.mib()) + defer { try? FileManager.default.removeItem(at: swapPath) } + + let buffer = BufferWriter() + let container = try LinuxContainer(id, rootfs: bs.rootfs, vmm: bs.vmm) { config in + // A shell variable is anonymous memory, which is what the kernel + // reclaims to a swap area, and asking the container's cgroup to + // reclaim puts it there rather than leaving it to the pressure the + // allocation happens to produce. Reading the value back afterwards + // is what shows it made the round trip intact, so the length is + // checked after the area has been measured. + config.process.arguments = [ + "/bin/sh", "-c", + "fill=$(head -c 340000000 /dev/zero | tr '\\0' 'a'); " + + "echo 340M > /sys/fs/cgroup/memory.reclaim; " + + "awk '/SwapTotal|SwapFree/ { print $1, $2 }' /proc/meminfo; " + + "test ${#fill} -eq 340000000", + ] + config.process.stdout = buffer + config.memoryInBytes = 256.mib() + config.swapLayer = swap + config.bootLog = bs.bootLog + } + + do { + try await container.create() + try await container.start() + + let status = try await container.wait() + try await container.stop() + + // Survival is the first half of the claim: without swap this + // workload is killed rather than reclaimed. + guard status.exitCode == 0 else { + throw IntegrationError.assert(msg: "workload did not survive memory pressure: \(status)") + } + + // And the second half: pages actually reached the swap device. + let out = String(data: buffer.data, encoding: .utf8) ?? "" + let values = out.split(separator: "\n").reduce(into: [String: Int]()) { acc, line in + let parts = line.split(separator: " ") + if parts.count == 2 { acc[String(parts[0])] = Int(parts[1]) } + } + guard let total = values["SwapTotal:"], let free = values["SwapFree:"], total > 0 else { + throw IntegrationError.assert(msg: "guest reported no swap: '\(out)'") + } + guard UInt64(total - free) * 1024 > 64.mib() else { + throw IntegrationError.assert(msg: "little or nothing was swapped out: '\(out)'") + } + } catch { + try? await container.stop() + throw error + } + } + + func testContainerSwapReclaimsFreedBlocks() async throws { + let id = "test-container-swap-reclaim" + let bs = try await bootstrap(id) + + let swapPath = Self.binPath(name: "\(id)-swap.raw") + let swap = try Self.makeSwapDevice(at: swapPath, size: 512.mib()) + defer { try? FileManager.default.removeItem(at: swapPath) } + + let container = try LinuxContainer(id, rootfs: bs.rootfs, vmm: bs.vmm) { config in + // The allocation is held by a child that exits, because the swap + // slots are only freed once the pages themselves are, and a shell + // that drops a variable keeps the pages in its own allocator. + config.process.arguments = [ + "/bin/sh", "-c", + "sh -c 'fill=$(head -c 200000000 /dev/zero | tr \"\\0\" a); " + + "echo 200M > /sys/fs/cgroup/memory.reclaim; " + + "test ${#fill} -eq 200000000' && sleep 30", + ] + config.memoryInBytes = 256.mib() + config.swapLayer = swap + config.bootLog = bs.bootLog + } + + // The blocks the file holds, rather than the size it reports, because a + // sparse file only ever reports the whole area. URL resource values + // cache after their first read, which a sampler cannot use. + func allocatedBytes() -> UInt64 { + var info = stat() + guard stat(swapPath.absolutePath(), &info) == 0 else { + return 0 + } + return UInt64(info.st_blocks) * 512 + } + + do { + try await container.create() + try await container.start() + + // Watching the file while the workload runs is what separates an + // area that released its blocks from one that never held any. + let peak = Task { + var high: UInt64 = 0 + while !Task.isCancelled { + high = max(high, allocatedBytes()) + try? await Task.sleep(nanoseconds: 200_000_000) + } + return high + } + + let status = try await container.wait() + peak.cancel() + let highWater = await peak.value + try await container.stop() + + guard status.exitCode == 0 else { + throw IntegrationError.assert(msg: "workload did not complete: \(status)") + } + guard highWater > 100.mib() else { + throw IntegrationError.assert( + msg: "workload never filled the swap area, it held \(highWater) bytes") + } + + // Having been filled, the area releases what the guest stopped + // using, so the file backing it does not hold its high water mark. + let settled = allocatedBytes() + guard settled < highWater / 4 else { + throw IntegrationError.assert( + msg: "swap area kept \(settled) of \(highWater) bytes after the guest freed it") + } + } catch { + try? await container.stop() + throw error + } + } + func testProcessEchoHi() async throws { let id = "test-process-echo-hi" let bs = try await bootstrap(id) diff --git a/Sources/Integration/Suite.swift b/Sources/Integration/Suite.swift index 84580dba..6087aafe 100644 --- a/Sources/Integration/Suite.swift +++ b/Sources/Integration/Suite.swift @@ -207,6 +207,27 @@ struct IntegrationSuite: AsyncParsableCommand { static let eventLoop = MultiThreadedEventLoopGroup(numberOfThreads: System.coreCount) + /// The swap area a test hands to a container. A suite that builds its own + /// containers owns the file behind them, the way `ContainerManager` owns + /// the ones it makes for callers that do not. + static func makeSwapDevice(at path: URL, size: UInt64) throws -> Containerization.Mount { + guard FileManager.default.createFile(atPath: path.absolutePath(), contents: nil) else { + throw IntegrationError.assert(msg: "failed to create swap device at \(path.absolutePath())") + } + let handle = try FileHandle(forWritingTo: path) + defer { try? handle.close() } + try handle.truncate(atOffset: size) + // A swap area holds nothing that outlives the container, so the host + // has no reason to synchronize it to permanent storage. + return .block( + format: Swap.mountType, + source: path.absolutePath(), + destination: "", + options: [], + runtimeOptions: ["vzDiskImageSynchronizationMode=none"] + ) + } + func bootstrap(_ testID: String) async throws -> (rootfs: Containerization.Mount, vmm: VirtualMachineManager, image: Containerization.Image, bootLog: BootLog) { let reference = "ghcr.io/linuxcontainers/alpine:3.20" let store = Self.imageStore @@ -614,6 +635,11 @@ struct IntegrationSuite: AsyncParsableCommand { Test("pod filesystem operation", testPodFilesystemOperation), Test("pod shared disk image volume", testPodSharedDiskImageVolume), Test("pod shared tmpfs volume", testPodSharedTmpfsVolume), + + // Swap + Test("container swap", testContainerSwap), + Test("container swap under pressure", testContainerSwapUnderPressure), + Test("container swap reclaims freed blocks", testContainerSwapReclaimsFreedBlocks), ] + macOS26Tests() let tests: [Test] = crossPlatformTests + macOSOnlyTests #else diff --git a/Sources/cctl/RunCommand.swift b/Sources/cctl/RunCommand.swift index fceff172..21f13a3e 100644 --- a/Sources/cctl/RunCommand.swift +++ b/Sources/cctl/RunCommand.swift @@ -48,6 +48,9 @@ extension Application { @Flag(name: .customLong("rosetta"), help: "Enable rosetta x64 emulation") var rosetta = false + @Option(name: .customLong("swap"), help: "Amount of swap in megabytes to create in the guest") + var swap: UInt64 = 0 + @Option(name: .customLong("mount"), help: "Directory to share into the container (Example: /foo:/bar)") var mounts: [String] = [] @@ -106,6 +109,7 @@ extension Application { id, reference: imageReference, rootfsSizeInBytes: fsSizeInMB.mib(), + swapSizeInBytes: swap.mib(), readOnly: readOnly, networking: true ) { config in diff --git a/Tests/ContainerizationTests/SwapTests.swift b/Tests/ContainerizationTests/SwapTests.swift new file mode 100644 index 00000000..1b8aa8a9 --- /dev/null +++ b/Tests/ContainerizationTests/SwapTests.swift @@ -0,0 +1,73 @@ +//===----------------------------------------------------------------------===// +// Copyright © 2026 Apple Inc. and the Containerization project authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//===----------------------------------------------------------------------===// + +#if os(Linux) +import ContainerizationOS +import Foundation +import Testing + +struct SwapTests { + private func makeArea(pages: UInt64, pageSize: Int) throws -> String { + let path = FileManager.default.temporaryDirectory + .appendingPathComponent("swap-\(UUID().uuidString)").path + try #require( + FileManager.default.createFile( + atPath: path, + contents: Data(count: Int(pages) * pageSize) + ) + ) + return path + } + + /// The header the kernel reads back is the one `mkswap` writes: version 1 + /// and the last usable page at the documented offsets, and the magic in the + /// final bytes of the first page. + @Test func formatWritesTheSwapAreaHeader() throws { + let pageSize = 4096 + let pages: UInt64 = 32 + let path = try makeArea(pages: pages, pageSize: pageSize) + defer { try? FileManager.default.removeItem(atPath: path) } + + try Swap.format(path: path, size: pages * UInt64(pageSize), pageSize: pageSize) + + let header = try Data(contentsOf: URL(fileURLWithPath: path))[0.. Date: Sun, 2 Aug 2026 00:03:43 +0000 Subject: [PATCH 07/25] Give a pod a swap area its containers share A pod's containers share one virtual machine, so they can share one swap area as well, with the guest kernel deciding whose pages are reclaimed to it rather than each container carrying its own. The area is attached with the pod's other mounts and enabled once, after the agent comes up. A container may cap how much of the area it uses. That cap counts swap alone, while the runtime spec carries memory and swap as a single total, so the container's memory limit is added to it when the spec is built. A cap without a memory limit is rejected when the container is added, because the total cannot be worked out without one. Leaving the cap unset lets a container use the whole area, which is what containers sharing a pool generally want. --- .../CHVirtualMachineInstance.swift | 7 +- .../Containerization/ContainerStorage.swift | 18 +++-- Sources/Containerization/LinuxPod.swift | 66 ++++++++++++++++++- .../Containerization/VMConfiguration.swift | 2 +- .../VZVirtualMachineInstance.swift | 5 +- Sources/Integration/PodTests.swift | 60 +++++++++++++++++ Sources/Integration/Suite.swift | 1 + 7 files changed, 147 insertions(+), 12 deletions(-) diff --git a/Sources/Containerization/CHVirtualMachineInstance.swift b/Sources/Containerization/CHVirtualMachineInstance.swift index e363d5ca..1704ce92 100644 --- a/Sources/Containerization/CHVirtualMachineInstance.swift +++ b/Sources/Containerization/CHVirtualMachineInstance.swift @@ -729,8 +729,8 @@ extension CHVirtualMachineInstance { extension CHVirtualMachineInstance.Configuration { /// Walks boot-time storage in the machine's device order (rootfs first, /// then containers sorted by id, each in role order, then volumes sorted - /// by name), allocating disk letters for virtio-blk mounts and seeding - /// the machine's `AttachedFilesystem` registry. + /// by name, then swap), allocating disk letters for virtio-blk mounts + /// and seeding the machine's `AttachedFilesystem` registry. /// /// The allocator is shared with the runtime hotplug provider, so block /// hotplug picks up at the next free letter after boot. @@ -765,8 +765,9 @@ extension CHVirtualMachineInstance.Configuration { guard let mount = self.storage.volumes[name] else { continue } volumes[name] = try attach(mount, chId: { "vol-\(name)-\($0)" }) } + let swap = try self.storage.swap.map { try attach($0, chId: { "swap-\($0)" }) } - return (MachineAttachments(containers: containers, volumes: volumes), bootDisks) + return (MachineAttachments(containers: containers, volumes: volumes, swap: swap), bootDisks) } } #endif diff --git a/Sources/Containerization/ContainerStorage.swift b/Sources/Containerization/ContainerStorage.swift index 49fae6d4..c98b8558 100644 --- a/Sources/Containerization/ContainerStorage.swift +++ b/Sources/Containerization/ContainerStorage.swift @@ -83,18 +83,25 @@ public struct MachineStorage: Sendable { /// Volumes shared by the machine's containers, by volume name. public var volumes: [String: Value] - public init(containers: [String: ContainerStorage] = [:], volumes: [String: Value] = [:]) { + /// The swap area shared by every container in the machine, when it has + /// one. + public var swap: Value? + + public init(containers: [String: ContainerStorage] = [:], volumes: [String: Value] = [:], swap: Value? = nil) { self.containers = containers self.volumes = volumes + self.swap = swap } /// Every value the machine carries: containers sorted by ID, each in - /// role order, then volumes sorted by name. Walks that allocate device - /// addresses and walks that create the devices use this one order, so an - /// address always names the device it was handed out for. + /// role order, then volumes sorted by name, then swap. Walks that + /// allocate device addresses and walks that create the devices use this + /// one order, so an address always names the device it was handed out + /// for. public var ordered: [Value] { containers.keys.sorted().flatMap { containers[$0]?.all ?? [] } + volumes.keys.sorted().compactMap { volumes[$0] } + + (swap.map { [$0] } ?? []) } /// The storage with `transform` applied to every value, each keeping its @@ -103,7 +110,8 @@ public struct MachineStorage: Sendable { public func map(_ transform: (Value) throws -> U) rethrows -> MachineStorage { MachineStorage( containers: try containers.mapValues { try $0.map(transform) }, - volumes: try volumes.mapValues(transform) + volumes: try volumes.mapValues(transform), + swap: try swap.map(transform) ) } } diff --git a/Sources/Containerization/LinuxPod.swift b/Sources/Containerization/LinuxPod.swift index fa4d1785..140f01ab 100644 --- a/Sources/Containerization/LinuxPod.swift +++ b/Sources/Containerization/LinuxPod.swift @@ -21,6 +21,7 @@ import Foundation import Logging import Synchronization +import struct ContainerizationOS.Swap import struct ContainerizationOS.Terminal /// NOTE: Experimental API @@ -43,6 +44,15 @@ public final class LinuxPod: Sendable { public var cpus: Int = 4 /// The memory in bytes to give to the pod's VM. public var memoryInBytes: UInt64 = 1024.mib() + /// Optional swap area shared by every container in the pod, as a block + /// device mount. + /// + /// The area belongs to the pod rather than to any one container, so the + /// guest kernel decides which container's pages are reclaimed to it. + /// Containers are free to use all of it unless they carry a limit of + /// their own. The `destination` field is ignored, as the area is + /// enabled rather than mounted. + public var swapLayer: Mount? = nil /// The network interfaces for the pod. public var interfaces: [any Interface] = [] /// Whether nested virtualization should be turned on for the pod. @@ -77,6 +87,24 @@ public final class LinuxPod: Sendable { public var cpus: Int? /// Optional per-container memory limit in bytes (can exceed pod total for oversubscription). public var memoryInBytes: UInt64? + /// Optional cap on how much of the pod's swap area this container may + /// use, in bytes. Leaving it unset lets the container use the whole + /// area, which is what containers sharing a pool usually want. + /// + /// This counts swap alone. The runtime spec carries memory and swap + /// combined, so `memoryInBytes` is added to it when the spec is built, + /// and a swap limit without a memory limit is rejected because the + /// combined figure cannot be worked out without one. + /// + /// Kata and docker spell the same limit as the combined figure the spec + /// carries, subtracting the memory limit to size the area, so a + /// container asking there for 2 GiB against a 1 GiB memory limit is + /// asking for 1 GiB of swap and here for 2 GiB. That figure suits a + /// container whose swap is sized for it alone; this one is a share of + /// an area the pod owns, and how much of that share a container may + /// take is what the number says. + /// https://github.com/kata-containers/kata-containers/blob/main/docs/how-to/how-to-setup-swap-devices-in-guest-kernel.md + public var swapInBytes: UInt64? /// The hostname for the container. public var hostname: String? /// The system control options for the container. @@ -326,8 +354,15 @@ public final class LinuxPod: Sendable { ) } if let memoryInBytes = config.memoryInBytes, memoryInBytes > 0 { + // The runtime spec's `swap` is the memory and swap total, not the + // swap alone, so the container's memory limit is folded in here. + var swapTotal: Int64? = nil + if let swapInBytes = config.swapInBytes { + swapTotal = Int64(memoryInBytes + swapInBytes) + } spec.linux?.resources?.memory = LinuxMemory( - limit: Int64(memoryInBytes) + limit: Int64(memoryInBytes), + swap: swapTotal ) } @@ -390,6 +425,15 @@ extension LinuxPod { var config = ContainerConfiguration() try configuration(&config) + // The runtime spec carries memory and swap as one total, so a swap + // limit cannot be expressed without a memory limit to add it to. + if config.swapInBytes != nil, config.memoryInBytes == nil { + throw ContainerizationError( + .invalidArgument, + message: "container \(id) sets a swap limit without a memory limit" + ) + } + let fileMountContext = try FileMountContext.prepare(mounts: config.mounts) switch state.phase { @@ -607,6 +651,9 @@ extension LinuxPod { for volume in self.config.volumes { machineStorage.volumes[volume.name] = volume.toMount() } + // The swap area is attached with the machine's own storage so the + // guest is told the /dev path the VMM allocates it. + machineStorage.swap = self.config.swapLayer // Capture into an immutable `let` so the value is safely usable // from the concurrent `withAgent` closure below. The container @@ -638,10 +685,27 @@ extension LinuxPod { let shareProcessNamespace = self.config.shareProcessNamespace let pauseProcessHolder = Mutex(nil) let fileMountContextUpdates = Mutex<[String: FileMountContext]>([:]) + let hasSwapLayer = self.config.swapLayer != nil try await vm.withAgent { agent in try await agent.standardSetup() + // The swap area belongs to the pod rather than to any one + // container, so it is enabled once here and every container + // reclaims to it through the guest's own memory management. + if hasSwapLayer { + guard let swap = vm.storage.swap else { + throw ContainerizationError(.notFound, message: "swap mount not found") + } + try await agent.mount( + ContainerizationOCI.Mount( + type: Swap.mountType, + source: swap.source, + destination: "", + options: swap.options + )) + } + // Mount the unified virtiofs share at /run/virtiofs only // when at least one container has a virtiofs mount. VZ // tolerates the unbacked mount; CH does not. diff --git a/Sources/Containerization/VMConfiguration.swift b/Sources/Containerization/VMConfiguration.swift index 7a5e8e2c..f00e3e04 100644 --- a/Sources/Containerization/VMConfiguration.swift +++ b/Sources/Containerization/VMConfiguration.swift @@ -73,7 +73,7 @@ public struct VMConfiguration: Sendable { /// The network interfaces to attach. public var interfaces: [any Interface] /// The storage the machine carries: each container's mounts by role, - /// and the volumes its containers share. + /// and the volumes and swap its containers share. public var storage: MachineMounts /// Optional destination for serial boot logs. public var bootLog: BootLog? diff --git a/Sources/Containerization/VZVirtualMachineInstance.swift b/Sources/Containerization/VZVirtualMachineInstance.swift index 670ab512..69828ec9 100644 --- a/Sources/Containerization/VZVirtualMachineInstance.swift +++ b/Sources/Containerization/VZVirtualMachineInstance.swift @@ -74,7 +74,7 @@ public final class VZVirtualMachineInstance: Sendable { /// Toggle nested virtualization support. public var nestedVirtualization: Bool /// The machine's storage: each container's mounts by role, and the - /// volumes its containers share. + /// volumes and swap its containers share. public var storage: MachineMounts /// Network interface attachments. public var interfaces: [any Interface] @@ -567,8 +567,9 @@ extension VZVirtualMachineInstance.Configuration { guard let mount = self.storage.volumes[name] else { continue } volumes[name] = try attach(mount) } + let swap = try self.storage.swap.map(attach) - return (MachineAttachments(containers: containers, volumes: volumes), storageDeviceCount) + return (MachineAttachments(containers: containers, volumes: volumes, swap: swap), storageDeviceCount) } } diff --git a/Sources/Integration/PodTests.swift b/Sources/Integration/PodTests.swift index 7dff20fa..93b0b7c9 100644 --- a/Sources/Integration/PodTests.swift +++ b/Sources/Integration/PodTests.swift @@ -96,6 +96,66 @@ extension IntegrationSuite { } } + func testPodSharedSwap() async throws { + let id = "test-pod-shared-swap" + + let bs = try await bootstrap(id) + let swapPath = Self.binPath(name: "\(id)-swap.raw") + let swap = try Self.makeSwapDevice(at: swapPath, size: 512.mib()) + defer { try? FileManager.default.removeItem(at: swapPath) } + + let pod = try LinuxPod(id, vmm: bs.vmm) { config in + config.cpus = 2 + config.memoryInBytes = 512.mib() + config.bootLog = bs.bootLog + config.swapLayer = swap + } + + // Both containers report the same area, because the pod owns it and + // the guest kernel decides whose pages are reclaimed to it. + let names = ["swap1", "swap2"] + let buffers = [names[0]: BufferWriter(), names[1]: BufferWriter()] + for name in names { + let buffer = buffers[name]! + try await pod.addContainer( + name, + rootfs: try cloneRootfs(bs.rootfs, testID: id, containerID: name) + ) { config in + config.process.arguments = [ + "/bin/sh", "-c", + "awk '/SwapTotal/ { print $2 }' /proc/meminfo", + ] + config.process.stdout = buffer + } + } + + try await pod.create() + + var totals: [UInt64] = [] + for name in names { + try await pod.startContainer(name) + let status = try await pod.waitContainer(name) + guard status.exitCode == 0 else { + throw IntegrationError.assert(msg: "\(name) status \(status) != 0") + } + let out = String(data: buffers[name]!.data, encoding: .utf8) ?? "" + guard let total = UInt64(out.trimmingCharacters(in: .whitespacesAndNewlines)) else { + throw IntegrationError.assert(msg: "\(name) reported no swap total: '\(out)'") + } + totals.append(total) + } + + try await pod.stop() + + guard totals[0] > 0 else { + throw IntegrationError.assert(msg: "pod swap area was not enabled: \(totals)") + } + guard totals[0] == totals[1] else { + throw IntegrationError.assert( + msg: "containers saw different swap areas: \(totals)") + } + } + func testPodContainerOutput() async throws { let id = "test-pod-container-output" diff --git a/Sources/Integration/Suite.swift b/Sources/Integration/Suite.swift index 6087aafe..f74321fc 100644 --- a/Sources/Integration/Suite.swift +++ b/Sources/Integration/Suite.swift @@ -409,6 +409,7 @@ struct IntegrationSuite: AsyncParsableCommand { Test("container IPv6 only default route", testIPv6OnlyDefaultRoute), Test("container IPv6 only gateway outside subnet", testIPv6OnlyGatewayOutsideSubnet), Test("container IPv6 dual stack", testIPv6DualStack), + Test("pod shared swap", testPodSharedSwap), Test("pod IPv6 address", testPodIPv6AddressAdd), ] } From c0d0220cc589741b22efed6ba2fd84558c4889df Mon Sep 17 00:00:00 2001 From: Aaron Paterson Date: Sun, 2 Aug 2026 20:09:40 +0000 Subject: [PATCH 08/25] Hold a container to the swap limit it was given A pod works out the memory and swap total its containers are capped at, the way the runtime spec asks for it, and sends that along with the rest of the resources. The guest wrote the memory limit and dropped the rest, so a container drew on the whole of the pod's swap area whatever it had been given. Cgroup v2 wants the swap on its own rather than the total, so the memory limit comes back out of it before the limit is written, and the spec's unlimited and unset values are carried across as the controller spells them. Reading the limit back from the kernel is what the test does, since a limit the guest never applied leaves nothing else to see. https://github.com/opencontainers/cgroups/blob/main/utils.go --- Sources/Integration/PodTests.swift | 55 +++++++++++++++++++++ Sources/Integration/Suite.swift | 1 + vminitd/Sources/Cgroup/Cgroup2Manager.swift | 34 +++++++++++++ 3 files changed, 90 insertions(+) diff --git a/Sources/Integration/PodTests.swift b/Sources/Integration/PodTests.swift index 93b0b7c9..19854194 100644 --- a/Sources/Integration/PodTests.swift +++ b/Sources/Integration/PodTests.swift @@ -156,6 +156,61 @@ extension IntegrationSuite { } } + /// A container's cap names the swap alone while the runtime spec carries the + /// memory and swap total, so the guest has to take the memory back out of it + /// before the kernel will hold the container to it. Read the cap back from + /// the kernel, because a spec the guest ignores leaves the container drawing + /// on the whole pod area with nothing to show it. + func testPodContainerSwapLimit() async throws { + let id = "test-pod-container-swap-limit" + + let bs = try await bootstrap(id) + let swapPath = Self.binPath(name: "\(id)-swap.raw") + let swap = try Self.makeSwapDevice(at: swapPath, size: 512.mib()) + defer { try? FileManager.default.removeItem(at: swapPath) } + + let pod = try LinuxPod(id, vmm: bs.vmm) { config in + config.cpus = 2 + config.memoryInBytes = 512.mib() + config.bootLog = bs.bootLog + config.swapLayer = swap + } + + let capped: UInt64 = 64.mib() + let buffer = BufferWriter() + try await pod.addContainer( + "capped", + rootfs: try cloneRootfs(bs.rootfs, testID: id, containerID: "capped") + ) { config in + config.memoryInBytes = 128.mib() + config.swapInBytes = capped + config.process.arguments = [ + "/bin/sh", "-c", "cat /sys/fs/cgroup/memory.swap.max", + ] + config.process.stdout = buffer + } + + try await pod.create() + try await pod.startContainer("capped") + let status = try await pod.waitContainer("capped") + try await pod.stop() + guard status.exitCode == 0 else { + throw IntegrationError.assert(msg: "capped status \(status) != 0") + } + + let reported = + String(data: buffer.data, encoding: .utf8)? + .trimmingCharacters(in: .whitespacesAndNewlines) ?? "" + guard let limit = UInt64(reported) else { + throw IntegrationError.assert( + msg: "swap cap never reached the kernel, memory.swap.max is '\(reported)'") + } + guard limit == capped else { + throw IntegrationError.assert( + msg: "expected a \(capped) byte swap cap, kernel holds \(limit)") + } + } + func testPodContainerOutput() async throws { let id = "test-pod-container-output" diff --git a/Sources/Integration/Suite.swift b/Sources/Integration/Suite.swift index f74321fc..bd4ec656 100644 --- a/Sources/Integration/Suite.swift +++ b/Sources/Integration/Suite.swift @@ -410,6 +410,7 @@ struct IntegrationSuite: AsyncParsableCommand { Test("container IPv6 only gateway outside subnet", testIPv6OnlyGatewayOutsideSubnet), Test("container IPv6 dual stack", testIPv6DualStack), Test("pod shared swap", testPodSharedSwap), + Test("pod container swap limit", testPodContainerSwapLimit), Test("pod IPv6 address", testPodIPv6AddressAdd), ] } diff --git a/vminitd/Sources/Cgroup/Cgroup2Manager.swift b/vminitd/Sources/Cgroup/Cgroup2Manager.swift index 62fa78c7..9ab812bc 100644 --- a/vminitd/Sources/Cgroup/Cgroup2Manager.swift +++ b/vminitd/Sources/Cgroup/Cgroup2Manager.swift @@ -216,6 +216,37 @@ public struct Cgroup2Manager: Sendable { ) } + // The runtime spec's `swap` is the memory and swap total, the way cgroup + // v1 took it, while cgroup v2 wants the swap on its own. A zero is the + // spec's "unset", which leaves the limit at max. + // https://github.com/opencontainers/cgroups/blob/main/utils.go + if let memory = resources.memory, let swap = memory.swap, swap != 0 { + let value: String + if swap < 0 { + value = "max" + } else { + guard let limit = memory.limit, limit != 0 else { + throw Error.invalidResource( + message: "unable to set swap limit without memory limit") + } + if limit < 0 { + // Memory is unlimited, so the total that contains it is too. + value = "max" + } else { + guard swap >= limit else { + throw Error.invalidResource( + message: "memory and swap limit \(swap) is below the memory limit \(limit)") + } + value = String(swap - limit) + } + } + try Self.writeValue( + path: self.path, + value: value, + fileName: "memory.swap.max" + ) + } + if let cpu = resources.cpu, let quota = cpu.quota, let period = cpu.period { // cpu.max format is "quota period" let value = "\(quota) \(period)" @@ -751,6 +782,7 @@ extension Cgroup2Manager { case cgroup1 case errno(errno: Int32, message: String) case notExist(path: String) + case invalidResource(message: String) package var description: String { switch self { @@ -762,6 +794,8 @@ extension Cgroup2Manager { return "tried to load a cgroup v1 path" case .notCgroup: return "path is not a cgroup mountpoint" + case .invalidResource(let message): + return message } } } From 2cbb72321404631d517f957d57ebb98dd48abec1 Mon Sep 17 00:00:00 2001 From: Aaron Paterson Date: Sun, 2 Aug 2026 20:28:50 +0000 Subject: [PATCH 09/25] Take the swap total as given when memory is unlimited An unlimited memory limit leaves nothing to subtract from the memory and swap total, so the swap stands as it was given rather than becoming unlimited alongside it. A memory limit that is negative without being the unlimited sentinel is not a limit at all, and is refused rather than read as one. https://github.com/opencontainers/cgroups/blob/main/utils.go --- vminitd/Sources/Cgroup/Cgroup2Manager.swift | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/vminitd/Sources/Cgroup/Cgroup2Manager.swift b/vminitd/Sources/Cgroup/Cgroup2Manager.swift index 9ab812bc..a9dcd97d 100644 --- a/vminitd/Sources/Cgroup/Cgroup2Manager.swift +++ b/vminitd/Sources/Cgroup/Cgroup2Manager.swift @@ -229,9 +229,12 @@ public struct Cgroup2Manager: Sendable { throw Error.invalidResource( message: "unable to set swap limit without memory limit") } - if limit < 0 { - // Memory is unlimited, so the total that contains it is too. - value = "max" + if limit == -1 { + // Unlimited memory leaves nothing to take out of the total, + // so the swap stands as it was given. + value = String(swap) + } else if limit < 0 { + throw Error.invalidResource(message: "invalid memory value: \(limit)") } else { guard swap >= limit else { throw Error.invalidResource( From be4f9cca1aafdd58cde8d3bfaa75aa516ebbb9ba Mon Sep 17 00:00:00 2001 From: Aaron Paterson Date: Fri, 7 Aug 2026 17:31:13 +0000 Subject: [PATCH 10/25] Let a pod's containers copy files in and out Copying a file between the host and a container is reachable only for a container in a machine of its own; the same container placed in a pod has no way to be copied into or out of, so a tool that offers `cp` loses it the moment a container joins a pod. What the transfer needs is the machine, the path the container's filesystem sits at in the guest, and a vsock port, and only the path differs between the two cases. It moves to a type that takes those, which both a standalone container and a pod's containers hand it. The round trip is covered against a running pod, including that a file copied into one container reaches that container alone. --- .../Containerization/GuestFileTransfer.swift | 303 ++++++++++++++++++ Sources/Containerization/LinuxContainer.swift | 264 ++------------- Sources/Containerization/LinuxPod.swift | 79 +++++ Sources/Integration/PodTests.swift | 84 +++++ Sources/Integration/Suite.swift | 1 + 5 files changed, 492 insertions(+), 239 deletions(-) create mode 100644 Sources/Containerization/GuestFileTransfer.swift diff --git a/Sources/Containerization/GuestFileTransfer.swift b/Sources/Containerization/GuestFileTransfer.swift new file mode 100644 index 00000000..d508c571 --- /dev/null +++ b/Sources/Containerization/GuestFileTransfer.swift @@ -0,0 +1,303 @@ +//===----------------------------------------------------------------------===// +// Copyright © 2026 Apple Inc. and the container project authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//===----------------------------------------------------------------------===// + +import ContainerizationArchive +import ContainerizationError +import ContainerizationOS +import Foundation + +/// Moves files between the host and a container's filesystem inside a guest. +/// +/// The transfer runs over a dedicated vsock connection on a port the caller +/// allocates. A container in a machine of its own and a container among +/// several in a pod differ only in where their filesystem sits in the guest, +/// so that path is what this is given. +struct GuestFileTransfer: Sendable { + /// Default chunk size for file transfers (1MiB). + static let defaultChunkSize = 1024 * 1024 + + /// The machine holding the container's filesystem. + let vm: any VirtualMachineInstance + /// Where the container's filesystem sits in the guest. + let guestRoot: String + /// The vsock port the data travels over. + let port: UInt32 + /// Where the blocking read and write work runs. + let queue: DispatchQueue + + /// Copy a file or directory from the host into the container. + /// + /// For directories, the source is archived as tar+gzip and streamed + /// directly through vsock without intermediate temp files. + func copyIn( + from source: URL, + to destination: URL, + mode: UInt32, + createParents: Bool, + chunkSize: Int + ) async throws { + var isDirectory: ObjCBool = false + guard FileManager.default.fileExists(atPath: source.path, isDirectory: &isDirectory) else { + throw ContainerizationError(.notFound, message: "copyIn: source not found '\(source.path)'") + } + let isArchive = isDirectory.boolValue + + let guestPath: URL = try await vm.withAgent { agent in + guard let vminitd = agent as? Vminitd else { + throw ContainerizationError(.unsupported, message: "copyIn requires Vminitd agent") + } + + return try await self.resolveCopyInGuestPath( + from: source, + to: destination, + sourceIsDirectory: isArchive, + using: vminitd + ) + } + + let listener = try vm.listen(port) + + try await withThrowingTaskGroup(of: Void.self) { group in + group.addTask { + try await self.vm.withAgent { agent in + guard let vminitd = agent as? Vminitd else { + throw ContainerizationError(.unsupported, message: "copyIn requires Vminitd agent") + } + try await vminitd.copy( + direction: .copyIn, + guestPath: guestPath, + vsockPort: self.port, + mode: mode, + createParents: createParents, + isArchive: isArchive + ) + } + } + + group.addTask { + guard let conn = await listener.first(where: { _ in true }) else { + throw ContainerizationError(.internalError, message: "copyIn: vsock connection not established") + } + try listener.finish() + + try await withCheckedThrowingContinuation { (continuation: CheckedContinuation) in + self.queue.async { + do { + defer { conn.closeFile() } + + if isArchive { + let writer = try ArchiveWriter(configuration: .init(format: .pax, filter: .gzip)) + try writer.open(fileDescriptor: conn.fileDescriptor) + try writer.archiveDirectory(source) + try writer.finishEncoding() + } else { + let srcFd = open(source.path, O_RDONLY) + guard srcFd != -1 else { + throw ContainerizationError( + .internalError, + message: "copyIn: failed to open '\(source.path)': \(String(cString: strerror(errno)))" + ) + } + defer { close(srcFd) } + + var buf = [UInt8](repeating: 0, count: chunkSize) + while true { + let n = read(srcFd, &buf, buf.count) + if n == 0 { break } + guard n > 0 else { + throw ContainerizationError( + .internalError, + message: "copyIn: read error: \(String(cString: strerror(errno)))" + ) + } + var written = 0 + while written < n { + let w = buf.withUnsafeBytes { ptr in + write(conn.fileDescriptor, ptr.baseAddress! + written, n - written) + } + guard w > 0 else { + throw ContainerizationError( + .internalError, + message: "copyIn: vsock write error: \(String(cString: strerror(errno)))" + ) + } + written += w + } + } + } + continuation.resume() + } catch { + continuation.resume(throwing: error) + } + } + } + } + + try await group.waitForAll() + } + } + + /// Copy a file or directory from the container to the host. + /// + /// For directories, the guest archives the source as tar+gzip and streams + /// it directly through vsock. The host extracts the archive without + /// intermediate temp files. + func copyOut( + from source: URL, + to destination: URL, + createParents: Bool, + chunkSize: Int + ) async throws { + if createParents { + let parentDir = destination.deletingLastPathComponent() + try FileManager.default.createDirectory(at: parentDir, withIntermediateDirectories: true) + } + + let guestPath = URL(filePath: guestRoot).appending(path: source.path) + let listener = try vm.listen(port) + + let (metadataStream, metadataCont) = AsyncStream.makeStream(of: Vminitd.CopyMetadata.self) + + try await withThrowingTaskGroup(of: Void.self) { group in + group.addTask { + defer { metadataCont.finish() } + try await self.vm.withAgent { agent in + guard let vminitd = agent as? Vminitd else { + throw ContainerizationError(.unsupported, message: "copyOut requires Vminitd agent") + } + try await vminitd.copy( + direction: .copyOut, + guestPath: guestPath, + vsockPort: self.port, + onMetadata: { meta in + metadataCont.yield(meta) + metadataCont.finish() + } + ) + } + } + + group.addTask { + guard let metadata = await metadataStream.first(where: { _ in true }) else { + throw ContainerizationError(.internalError, message: "copyOut: no metadata received") + } + + guard let conn = await listener.first(where: { _ in true }) else { + throw ContainerizationError(.internalError, message: "copyOut: vsock connection not established") + } + try listener.finish() + + try await withCheckedThrowingContinuation { (continuation: CheckedContinuation) in + self.queue.async { + do { + defer { conn.closeFile() } + + if metadata.isArchive { + try FileManager.default.createDirectory(at: destination, withIntermediateDirectories: true) + let fh = FileHandle(fileDescriptor: dup(conn.fileDescriptor), closeOnDealloc: true) + let reader = try ArchiveReader(format: .pax, filter: .gzip, fileHandle: fh) + _ = try reader.extractContents(to: destination) + } else { + let destFd = open(destination.path, O_WRONLY | O_CREAT | O_TRUNC, 0o644) + guard destFd != -1 else { + throw ContainerizationError( + .internalError, + message: "copyOut: failed to open '\(destination.path)': \(String(cString: strerror(errno)))" + ) + } + defer { close(destFd) } + + var buf = [UInt8](repeating: 0, count: chunkSize) + while true { + let n = read(conn.fileDescriptor, &buf, buf.count) + if n == 0 { break } + guard n > 0 else { + throw ContainerizationError( + .internalError, + message: "copyOut: vsock read error: \(String(cString: strerror(errno)))" + ) + } + var written = 0 + while written < n { + let w = buf.withUnsafeBytes { ptr in + write(destFd, ptr.baseAddress! + written, n - written) + } + guard w > 0 else { + throw ContainerizationError( + .internalError, + message: "copyOut: write error: \(String(cString: strerror(errno)))" + ) + } + written += w + } + } + } + continuation.resume() + } catch { + continuation.resume(throwing: error) + } + } + } + } + + try await group.waitForAll() + } + } + + /// Where a copy lands in the guest, given what the destination already is. + /// + /// A destination that names an existing directory receives the source + /// under its own name, the way `cp` behaves. + private func resolveCopyInGuestPath( + from source: URL, + to destination: URL, + sourceIsDirectory: Bool, + using vminitd: Vminitd + ) async throws -> URL { + let guestDestination = URL(filePath: guestRoot).appending(path: destination.path) + + let stat: ContainerizationOS.Stat? + do { + stat = try await vminitd.stat(path: guestDestination) + } catch let error as ContainerizationError where error.code == .notFound { + stat = nil + } + // Any other error propagates so transport and permission failures are visible. + + guard let stat else { + if destination.hasDirectoryPath && !sourceIsDirectory { + throw ContainerizationError( + .invalidArgument, + message: "destination directory does not exist: \(destination.path)" + ) + } + return guestDestination + } + + let destinationIsDirectory = (stat.mode & UInt32(S_IFMT)) == UInt32(S_IFDIR) + guard destinationIsDirectory else { + if sourceIsDirectory { + throw ContainerizationError( + .invalidArgument, + message: "cannot copy directory over existing file: \(destination.path)" + ) + } + return guestDestination + } + + return guestDestination.appendingPathComponent(source.lastPathComponent) + } +} diff --git a/Sources/Containerization/LinuxContainer.swift b/Sources/Containerization/LinuxContainer.swift index 4d5289a7..eca34a1a 100644 --- a/Sources/Containerization/LinuxContainer.swift +++ b/Sources/Containerization/LinuxContainer.swift @@ -669,8 +669,8 @@ extension LinuxContainer { let vm = try await self.vmm.create(config: creationConfig) let relayManager = UnixSocketRelayManager(vm: vm, log: self.logger) + try await vm.start() do { - try await vm.start() let storageForAgent = containerStorage try await vm.withAgent { agent in try await agent.standardSetup() @@ -1194,7 +1194,7 @@ extension LinuxContainer { } /// Default chunk size for file transfers (1MiB). - public static let defaultCopyChunkSize = 1024 * 1024 + public static let defaultCopyChunkSize = GuestFileTransfer.defaultChunkSize /// Copy a file or directory from the host into the container. /// @@ -1210,148 +1210,14 @@ extension LinuxContainer { ) async throws { try await self.state.withLock { let state = try $0.startedState("copyIn") - - var isDirectory: ObjCBool = false - guard FileManager.default.fileExists(atPath: source.path, isDirectory: &isDirectory) else { - throw ContainerizationError(.notFound, message: "copyIn: source not found '\(source.path)'") - } - let isArchive = isDirectory.boolValue - - let guestPath: URL = try await state.vm.withAgent { agent in - guard let vminitd = agent as? Vminitd else { - throw ContainerizationError(.unsupported, message: "copyIn requires Vminitd agent") - } - - return try await self.resolveCopyInGuestPath( - from: source, - to: destination, - sourceIsDirectory: isArchive, - using: vminitd - ) - } - - let port = self.hostVsockPorts.wrappingAdd(1, ordering: .relaxed).oldValue - let listener = try state.vm.listen(port) - - try await withThrowingTaskGroup(of: Void.self) { group in - group.addTask { - try await state.vm.withAgent { agent in - guard let vminitd = agent as? Vminitd else { - throw ContainerizationError(.unsupported, message: "copyIn requires Vminitd agent") - } - try await vminitd.copy( - direction: .copyIn, - guestPath: guestPath, - vsockPort: port, - mode: mode, - createParents: createParents, - isArchive: isArchive - ) - } - } - - group.addTask { - guard let conn = await listener.first(where: { _ in true }) else { - throw ContainerizationError(.internalError, message: "copyIn: vsock connection not established") - } - try listener.finish() - - try await withCheckedThrowingContinuation { (continuation: CheckedContinuation) in - self.copyQueue.async { - do { - defer { conn.closeFile() } - - if isArchive { - let writer = try ArchiveWriter(configuration: .init(format: .pax, filter: .gzip)) - try writer.open(fileDescriptor: conn.fileDescriptor) - try writer.archiveDirectory(source) - try writer.finishEncoding() - } else { - let srcFd = open(source.path, O_RDONLY) - guard srcFd != -1 else { - throw ContainerizationError( - .internalError, - message: "copyIn: failed to open '\(source.path)': \(String(cString: strerror(errno)))" - ) - } - defer { close(srcFd) } - - var buf = [UInt8](repeating: 0, count: chunkSize) - while true { - let n = read(srcFd, &buf, buf.count) - if n == 0 { break } - guard n > 0 else { - throw ContainerizationError( - .internalError, - message: "copyIn: read error: \(String(cString: strerror(errno)))" - ) - } - var written = 0 - while written < n { - let w = buf.withUnsafeBytes { ptr in - write(conn.fileDescriptor, ptr.baseAddress! + written, n - written) - } - guard w > 0 else { - throw ContainerizationError( - .internalError, - message: "copyIn: vsock write error: \(String(cString: strerror(errno)))" - ) - } - written += w - } - } - } - continuation.resume() - } catch { - continuation.resume(throwing: error) - } - } - } - } - - try await group.waitForAll() - } - } - } - - private func resolveCopyInGuestPath( - from source: URL, - to destination: URL, - sourceIsDirectory: Bool, - using vminitd: Vminitd - ) async throws -> URL { - let guestDestination = URL(filePath: self.root).appending(path: destination.path) - - let stat: ContainerizationOS.Stat? - do { - stat = try await vminitd.stat(path: guestDestination) - } catch let error as ContainerizationError where error.code == .notFound { - stat = nil - } - // Any other error propagates so transport and permission failures are visible. - - guard let stat else { - if destination.hasDirectoryPath && !sourceIsDirectory { - throw ContainerizationError( - .invalidArgument, - message: "destination directory does not exist: \(destination.path)" - ) - } - return guestDestination - } - - let destinationIsDirectory = (stat.mode & UInt32(S_IFMT)) == UInt32(S_IFDIR) - guard destinationIsDirectory else { - if sourceIsDirectory { - throw ContainerizationError( - .invalidArgument, - message: "cannot copy directory over existing file: \(destination.path)" - ) - } - return guestDestination + try await self.transfer(vm: state.vm).copyIn( + from: source, + to: destination, + mode: mode, + createParents: createParents, + chunkSize: chunkSize + ) } - - return guestDestination.appendingPathComponent(source.lastPathComponent) } /// Copy a file or directory from the container to the host. @@ -1367,104 +1233,24 @@ extension LinuxContainer { ) async throws { try await self.state.withLock { let state = try $0.startedState("copyOut") - - if createParents { - let parentDir = destination.deletingLastPathComponent() - try FileManager.default.createDirectory(at: parentDir, withIntermediateDirectories: true) - } - - let guestPath = URL(filePath: self.root).appending(path: source.path) - let port = self.hostVsockPorts.wrappingAdd(1, ordering: .relaxed).oldValue - let listener = try state.vm.listen(port) - - let (metadataStream, metadataCont) = AsyncStream.makeStream(of: Vminitd.CopyMetadata.self) - - try await withThrowingTaskGroup(of: Void.self) { group in - group.addTask { - defer { metadataCont.finish() } - try await state.vm.withAgent { agent in - guard let vminitd = agent as? Vminitd else { - throw ContainerizationError(.unsupported, message: "copyOut requires Vminitd agent") - } - try await vminitd.copy( - direction: .copyOut, - guestPath: guestPath, - vsockPort: port, - onMetadata: { meta in - metadataCont.yield(meta) - metadataCont.finish() - } - ) - } - } - - group.addTask { - guard let metadata = await metadataStream.first(where: { _ in true }) else { - throw ContainerizationError(.internalError, message: "copyOut: no metadata received") - } - - guard let conn = await listener.first(where: { _ in true }) else { - throw ContainerizationError(.internalError, message: "copyOut: vsock connection not established") - } - try listener.finish() - - try await withCheckedThrowingContinuation { (continuation: CheckedContinuation) in - self.copyQueue.async { - do { - defer { conn.closeFile() } - - if metadata.isArchive { - try FileManager.default.createDirectory(at: destination, withIntermediateDirectories: true) - let fh = FileHandle(fileDescriptor: dup(conn.fileDescriptor), closeOnDealloc: true) - let reader = try ArchiveReader(format: .pax, filter: .gzip, fileHandle: fh) - _ = try reader.extractContents(to: destination) - } else { - let destFd = open(destination.path, O_WRONLY | O_CREAT | O_TRUNC, 0o644) - guard destFd != -1 else { - throw ContainerizationError( - .internalError, - message: "copyOut: failed to open '\(destination.path)': \(String(cString: strerror(errno)))" - ) - } - defer { close(destFd) } - - var buf = [UInt8](repeating: 0, count: chunkSize) - while true { - let n = read(conn.fileDescriptor, &buf, buf.count) - if n == 0 { break } - guard n > 0 else { - throw ContainerizationError( - .internalError, - message: "copyOut: vsock read error: \(String(cString: strerror(errno)))" - ) - } - var written = 0 - while written < n { - let w = buf.withUnsafeBytes { ptr in - write(destFd, ptr.baseAddress! + written, n - written) - } - guard w > 0 else { - throw ContainerizationError( - .internalError, - message: "copyOut: write error: \(String(cString: strerror(errno)))" - ) - } - written += w - } - } - } - continuation.resume() - } catch { - continuation.resume(throwing: error) - } - } - } - } - - try await group.waitForAll() - } + try await self.transfer(vm: state.vm).copyOut( + from: source, + to: destination, + createParents: createParents, + chunkSize: chunkSize + ) } } + + /// A transfer against this container's filesystem, on a port of its own. + private func transfer(vm: any VirtualMachineInstance) -> GuestFileTransfer { + GuestFileTransfer( + vm: vm, + guestRoot: self.root, + port: self.hostVsockPorts.wrappingAdd(1, ordering: .relaxed).oldValue, + queue: self.copyQueue + ) + } } extension VirtualMachineInstance { diff --git a/Sources/Containerization/LinuxPod.swift b/Sources/Containerization/LinuxPod.swift index 140f01ab..197f29bc 100644 --- a/Sources/Containerization/LinuxPod.swift +++ b/Sources/Containerization/LinuxPod.swift @@ -219,6 +219,9 @@ public final class LinuxPod: Sendable { // the host. private let guestVsockPorts: Atomic + // Where the blocking reads and writes of a file transfer run. + private let copyQueue = DispatchQueue(label: "com.apple.containerization.copy") + private struct State: Sendable { var phase: Phase var containers: [String: PodContainer] @@ -1355,6 +1358,82 @@ extension LinuxPod { } } + /// Default chunk size for file transfers (1MiB). + public static let defaultCopyChunkSize = GuestFileTransfer.defaultChunkSize + + /// Copy a file or directory from the host into a container in the pod. + /// + /// Data transfer happens over a dedicated vsock connection. For + /// directories, the source is archived as tar+gzip and streamed directly + /// through vsock without intermediate temp files. + public func copyIn( + _ containerID: String, + from source: URL, + to destination: URL, + mode: UInt32 = 0o644, + createParents: Bool = true, + chunkSize: Int = defaultCopyChunkSize + ) async throws { + try await self.state.withLock { state in + try await self.transfer(containerID, state: state, operation: "copyIn").copyIn( + from: source, + to: destination, + mode: mode, + createParents: createParents, + chunkSize: chunkSize + ) + } + } + + /// Copy a file or directory from a container in the pod to the host. + /// + /// Data transfer happens over a dedicated vsock connection. For + /// directories, the guest archives the source as tar+gzip and streams it + /// directly through vsock. The host extracts the archive without + /// intermediate temp files. + public func copyOut( + _ containerID: String, + from source: URL, + to destination: URL, + createParents: Bool = true, + chunkSize: Int = defaultCopyChunkSize + ) async throws { + try await self.state.withLock { state in + try await self.transfer(containerID, state: state, operation: "copyOut").copyOut( + from: source, + to: destination, + createParents: createParents, + chunkSize: chunkSize + ) + } + } + + /// A transfer against one container's filesystem, on a port of its own. + private func transfer(_ containerID: String, state: State, operation: String) throws -> GuestFileTransfer { + let createdState = try state.phase.createdState(operation) + + guard let container = state.containers[containerID] else { + throw ContainerizationError( + .notFound, + message: "container \(containerID) not found in pod" + ) + } + + guard container.state == .started else { + throw ContainerizationError( + .invalidState, + message: "container \(containerID) must be started to copy files" + ) + } + + return GuestFileTransfer( + vm: createdState.vm, + guestRoot: Self.guestRootfsPath(containerID), + port: self.hostVsockPorts.wrappingAdd(1, ordering: .relaxed).oldValue, + queue: self.copyQueue + ) + } + /// Close a container's standard input to signal no more input is arriving. public func closeContainerStdin(_ containerID: String) async throws { try await self.state.withLock { state in diff --git a/Sources/Integration/PodTests.swift b/Sources/Integration/PodTests.swift index 19854194..5703bde1 100644 --- a/Sources/Integration/PodTests.swift +++ b/Sources/Integration/PodTests.swift @@ -2477,4 +2477,88 @@ extension IntegrationSuite { } } #endif + + /// A file copied into one container in a pod arrives in that container and + /// nowhere else, and comes back out with what it held. + func testPodCopyRoundTrip() async throws { + let id = "test-pod-copy-round-trip" + + let bs = try await bootstrap(id) + let pod = try LinuxPod(id, vmm: bs.vmm) { config in + config.cpus = 4 + config.memoryInBytes = 1024.mib() + config.bootLog = bs.bootLog + } + + let testContent = "Hello from the host! This is a pod copy test." + let hostDirectory = FileManager.default.uniqueTemporaryDirectory(create: true) + let hostFile = hostDirectory.appendingPathComponent("test-input.txt") + try testContent.write(to: hostFile, atomically: true, encoding: .utf8) + + try await pod.addContainer("holder", rootfs: try cloneRootfs(bs.rootfs, testID: id, containerID: "holder")) { config in + config.process.arguments = ["sleep", "100"] + } + try await pod.addContainer("bystander", rootfs: try cloneRootfs(bs.rootfs, testID: id, containerID: "bystander")) { config in + config.process.arguments = ["sleep", "100"] + } + + do { + try await pod.create() + try await pod.startContainer("holder") + try await pod.startContainer("bystander") + + try await pod.copyIn( + "holder", + from: hostFile, + to: URL(filePath: "/tmp/copied-file.txt") + ) + + let buffer = BufferWriter() + let exec = try await pod.execInContainer("holder", processID: "verify-copy") { config in + config.arguments = ["cat", "/tmp/copied-file.txt"] + config.stdout = buffer + } + try await exec.start() + let status = try await exec.wait() + try await exec.delete() + + guard status.exitCode == 0 else { + throw IntegrationError.assert(msg: "cat in the container copied into failed with status \(status)") + } + guard String(data: buffer.data, encoding: .utf8) == testContent else { + throw IntegrationError.assert( + msg: "expected '\(testContent)', got '\(String(data: buffer.data, encoding: .utf8) ?? "nil")'") + } + + // The containers share a machine and not a filesystem, so the copy + // reached one of them alone. + let bystander = try await pod.execInContainer("bystander", processID: "verify-absent") { config in + config.arguments = ["test", "-e", "/tmp/copied-file.txt"] + } + try await bystander.start() + let bystanderStatus = try await bystander.wait() + try await bystander.delete() + + guard bystanderStatus.exitCode != 0 else { + throw IntegrationError.assert(msg: "the copy reached a container it was not addressed to") + } + + let returned = hostDirectory.appendingPathComponent("test-output.txt") + try await pod.copyOut( + "holder", + from: URL(filePath: "/tmp/copied-file.txt"), + to: returned + ) + + let returnedContent = try String(contentsOf: returned, encoding: .utf8) + guard returnedContent == testContent else { + throw IntegrationError.assert(msg: "expected '\(testContent)' back, got '\(returnedContent)'") + } + + try await pod.stop() + } catch { + try? await pod.stop() + throw error + } + } } diff --git a/Sources/Integration/Suite.swift b/Sources/Integration/Suite.swift index bd4ec656..769ccc12 100644 --- a/Sources/Integration/Suite.swift +++ b/Sources/Integration/Suite.swift @@ -559,6 +559,7 @@ struct IntegrationSuite: AsyncParsableCommand { Test("pod memory events OOM kill", testPodMemoryEventsOOMKill), Test("pod container resource limits", testPodContainerResourceLimits), Test("pod container filesystem isolation", testPodContainerFilesystemIsolation), + Test("pod copy round trip", testPodCopyRoundTrip), Test("pod container PID namespace isolation", testPodContainerPIDNamespaceIsolation), Test("pod container independent resource limits", testPodContainerIndependentResourceLimits), Test("pod shared PID namespace", testPodSharedPIDNamespace), From 8f7a007c5d83298f79b27a824e4882f835f8b1a0 Mon Sep 17 00:00:00 2001 From: Aaron Paterson Date: Fri, 7 Aug 2026 20:42:42 +0000 Subject: [PATCH 11/25] Give a pod the machine it asked for A container with a machine of its own is given the size it asked for plus the guest agent's, so a container that asks for a gibibyte has a gibibyte and the agent runs beside it. A pod hands its size to the machine as it stands, so the agent comes out of what the pod asked for and its containers are left with less: a pod given twelve gibibytes reports eleven and a half. The pod's size and the agent's are added the same way, and what a pod was given is what its containers have. --- Sources/Containerization/LinuxPod.swift | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/Sources/Containerization/LinuxPod.swift b/Sources/Containerization/LinuxPod.swift index 197f29bc..fab44f0f 100644 --- a/Sources/Containerization/LinuxPod.swift +++ b/Sources/Containerization/LinuxPod.swift @@ -57,6 +57,13 @@ public final class LinuxPod: Sendable { public var interfaces: [any Interface] = [] /// Whether nested virtualization should be turned on for the pod. public var virtualization: Bool = false + /// Additional CPU cores to allocate for the virtual machine on top of + /// the pod's configured `cpus`, so what the pod was given is what its + /// containers have rather than what the guest agent leaves of it. + public var cpuOverhead: Int = 1 + /// Additional memory in bytes to allocate for the virtual machine on + /// top of the pod's configured `memoryInBytes`. + public var memoryOverhead: UInt64 = 128.mib() /// Optional file path to store serial boot logs. public var bootLog: BootLog? /// Whether containers in the pod should share a PID namespace. @@ -669,9 +676,12 @@ extension LinuxPod { return false } + // The machine carries the guest agent as well as the containers, + // so it is given the pod's size and the agent's on top; what the + // pod was given is then what its containers have. var vmConfig = VMConfiguration( - cpus: self.config.cpus, - memoryInBytes: self.config.memoryInBytes, + cpus: self.config.cpus + self.config.cpuOverhead, + memoryInBytes: self.config.memoryInBytes + self.config.memoryOverhead, interfaces: self.config.interfaces, storage: machineStorage, bootLog: self.config.bootLog, @@ -681,9 +691,8 @@ extension LinuxPod { let creationConfig = StandardVMConfig(configuration: vmConfig) let vm = try await self.vmm.create(config: creationConfig) let relayManager = UnixSocketRelayManager(vm: vm) - try await vm.start() - do { + try await vm.start() let containers = state.containers let shareProcessNamespace = self.config.shareProcessNamespace let pauseProcessHolder = Mutex(nil) From ea2a9a221eaeacda38c6b9dc14375873d56e0b8e Mon Sep 17 00:00:00 2001 From: Aaron Paterson Date: Fri, 7 Aug 2026 21:10:11 +0000 Subject: [PATCH 12/25] Let a pod's containers be given a writable layer A container with a machine of its own can be given a block device to write into, with its image mounted read-only beneath, so the image is left as it is and every write lands in the layer. The same container placed in a pod has nowhere to be given one. The layer is attached with the container's other block devices, mounted in the guest as the upper layer of an overlay over the image, dropped from the mounts the runtime spec carries, and unmounted with the container. A pod's containers take one whether they were placed before the machine booted or hotplugged into it afterwards. --- .../Containerization/CHHotplugProvider.swift | 4 +- .../CHVirtualMachineInstance.swift | 4 +- .../Containerization/HotplugProvider.swift | 4 +- Sources/Containerization/LinuxContainer.swift | 43 +------ Sources/Containerization/LinuxPod.swift | 76 ++++++++++-- Sources/Containerization/OverlayRootfs.swift | 64 ++++++++++ .../VZVirtualMachineInstance.swift | 4 +- .../VirtualMachineInstance.swift | 6 +- Sources/Integration/PodTests.swift | 117 ++++++++++++++++++ Sources/Integration/Suite.swift | 2 + 10 files changed, 268 insertions(+), 56 deletions(-) create mode 100644 Sources/Containerization/OverlayRootfs.swift diff --git a/Sources/Containerization/CHHotplugProvider.swift b/Sources/Containerization/CHHotplugProvider.swift index 380cba76..e6a33d38 100644 --- a/Sources/Containerization/CHHotplugProvider.swift +++ b/Sources/Containerization/CHHotplugProvider.swift @@ -150,12 +150,12 @@ final class CHHotplugProvider: HotplugProvider { } } - func registerMounts(id: String, rootfs: AttachedFilesystem, additionalMounts: [Mount]) throws { + func registerMounts(id: String, rootfs: AttachedFilesystem, writableLayer: AttachedFilesystem?, additionalMounts: [Mount]) throws { var mounts: [AttachedFilesystem] = [] for mount in additionalMounts { mounts.append(try AttachedFilesystem(mount: mount, allocator: allocator)) } - let container = ContainerAttachments(rootfs: rootfs, mounts: mounts) + let container = ContainerAttachments(rootfs: rootfs, writableLayer: writableLayer, mounts: mounts) _storage.withLock { $0.containers[id] = container } diff --git a/Sources/Containerization/CHVirtualMachineInstance.swift b/Sources/Containerization/CHVirtualMachineInstance.swift index 1704ce92..11d7e0a3 100644 --- a/Sources/Containerization/CHVirtualMachineInstance.swift +++ b/Sources/Containerization/CHVirtualMachineInstance.swift @@ -490,8 +490,8 @@ extension CHVirtualMachineInstance: VirtualMachineInstance { try await hotplug.releaseVirtioFS(id: id) } - public func registerMounts(id: String, rootfs: AttachedFilesystem, additionalMounts: [Mount]) throws { - try hotplug.registerMounts(id: id, rootfs: rootfs, additionalMounts: additionalMounts) + public func registerMounts(id: String, rootfs: AttachedFilesystem, writableLayer: AttachedFilesystem?, additionalMounts: [Mount]) throws { + try hotplug.registerMounts(id: id, rootfs: rootfs, writableLayer: writableLayer, additionalMounts: additionalMounts) } } diff --git a/Sources/Containerization/HotplugProvider.swift b/Sources/Containerization/HotplugProvider.swift index 0835077f..94fad4b4 100644 --- a/Sources/Containerization/HotplugProvider.swift +++ b/Sources/Containerization/HotplugProvider.swift @@ -30,8 +30,10 @@ public protocol HotplugProvider: Sendable { /// - Parameters: /// - id: The container ID /// - rootfs: The rootfs attachment from hotplug + /// - writableLayer: The container's writable layer attachment when it + /// has one /// - additionalMounts: Additional mounts to register - func registerMounts(id: String, rootfs: AttachedFilesystem, additionalMounts: [Mount]) throws + func registerMounts(id: String, rootfs: AttachedFilesystem, writableLayer: AttachedFilesystem?, additionalMounts: [Mount]) throws /// Release a hotplug device. /// - Parameter id: The container ID who should be released diff --git a/Sources/Containerization/LinuxContainer.swift b/Sources/Containerization/LinuxContainer.swift index eca34a1a..ce57488f 100644 --- a/Sources/Containerization/LinuxContainer.swift +++ b/Sources/Containerization/LinuxContainer.swift @@ -575,47 +575,16 @@ extension LinuxContainer { rootfsPath: String, agent: VirtualMachineAgent ) async throws { - let rootfsAttachment = attached.rootfs - if let writableAttachment = attached.writableLayer { - // Set up overlayfs with image as lower layer and writable layer as upper. - let lowerPath = "/run/container/\(self.id)/lower" - let upperMountPath = "/run/container/\(self.id)/upper" - let upperPath = "/run/container/\(self.id)/upper/diff" - let workPath = "/run/container/\(self.id)/upper/work" - - // Mount the image (lower layer) as read-only. - var lowerMount = rootfsAttachment.to - lowerMount.destination = lowerPath - if !lowerMount.options.contains("ro") { - lowerMount.options.append("ro") - } - try await agent.mount(lowerMount) - - // Mount the writable layer. - var upperMount = writableAttachment.to - upperMount.destination = upperMountPath - try await agent.mount(upperMount) - - // Create the upper and work directories inside the writable layer. - try await agent.mkdir(path: upperPath, all: true, perms: 0o755) - try await agent.mkdir(path: workPath, all: true, perms: 0o755) - - // Mount the overlay. - let overlayMount = ContainerizationOCI.Mount( - type: "overlay", - source: "overlay", - destination: rootfsPath, - options: [ - "lowerdir=\(lowerPath)", - "upperdir=\(upperPath)", - "workdir=\(workPath)", - ] + try await agent.mountOverlayRootfs( + containerID: self.id, + rootfsAttachment: attached.rootfs, + writableAttachment: writableAttachment, + rootfsPath: rootfsPath ) - try await agent.mount(overlayMount) } else { // No writable layer. Mount rootfs directly. - var rootfs = rootfsAttachment.to + var rootfs = attached.rootfs.to rootfs.destination = rootfsPath try await agent.mount(rootfs) } diff --git a/Sources/Containerization/LinuxPod.swift b/Sources/Containerization/LinuxPod.swift index fab44f0f..76967821 100644 --- a/Sources/Containerization/LinuxPod.swift +++ b/Sources/Containerization/LinuxPod.swift @@ -202,6 +202,7 @@ public final class LinuxPod: Sendable { private struct PodContainer: Sendable { let id: String let rootfs: Mount + let writableLayer: Mount? let config: ContainerConfiguration var state: ContainerState var process: LinuxProcess? @@ -329,7 +330,7 @@ public final class LinuxPod: Sendable { ) } - private func generateRuntimeSpec(containerID: String, config: ContainerConfiguration, rootfs: Mount) -> Spec { + private func generateRuntimeSpec(containerID: String, config: ContainerConfiguration, rootfs: Mount, writableLayer: Mount? = nil) -> Spec { var spec = Self.createDefaultRuntimeSpec(containerID, podID: self.id) // Process configuration @@ -354,7 +355,7 @@ public final class LinuxPod: Sendable { // If the rootfs was requested as read-only, set it in the OCI spec. // We let the OCI runtime remount as ro, instead of doing it originally. - spec.root?.readonly = rootfs.options.contains("ro") + spec.root?.readonly = rootfs.options.contains("ro") && writableLayer == nil // Resource limits (if specified) if let cpus = config.cpus, cpus > 0 { @@ -413,9 +414,14 @@ extension LinuxPod { /// When called before `create()`, the container is registered for setup during VM creation. /// When called after `create()`, the container is hotplugged into the running VM. /// If the underlying VMM does not support hotplug, an error is thrown. + /// - Parameters: + /// - writableLayer: Optional writable layer mount. When provided, an overlayfs is used with + /// the container's rootfs as the lower layer and this as the upper layer, so all writes + /// go to this layer instead of the rootfs. public func addContainer( _ id: String, rootfs: Mount, + writableLayer: Mount? = nil, configuration: @Sendable @escaping (inout ContainerConfiguration) throws -> Void ) async throws { guard id.count <= Self.maxIDLength else { @@ -424,6 +430,14 @@ extension LinuxPod { message: "container id length \(id.count) exceeds maximum of \(Self.maxIDLength) characters" ) } + if let writableLayer { + guard writableLayer.isBlock else { + throw ContainerizationError( + .invalidArgument, + message: "writableLayer must be a block device" + ) + } + } try await self.state.withLock { state in guard state.containers[id] == nil else { throw ContainerizationError( @@ -451,6 +465,7 @@ extension LinuxPod { state.containers[id] = PodContainer( id: id, rootfs: rootfs, + writableLayer: writableLayer, config: config, state: .registered, process: nil, @@ -460,6 +475,9 @@ extension LinuxPod { case .created(let createdState): let vm = createdState.vm + // Strip "ro" as create() does: readonly is expressed through + // the OCI spec's root.readonly field and a remount in vmexec + // after setup completes, so the device attaches writable. var modifiedRootfs = rootfs modifiedRootfs.options.removeAll(where: { $0 == "ro" }) @@ -467,6 +485,13 @@ extension LinuxPod { var updatedFileMountContext = fileMountContext do { + // The writable layer is a block device like the rootfs, + // attached alongside it so the overlay has both layers. + var writableAttachment: AttachedFilesystem? + if let writableLayer { + writableAttachment = try await vm.hotplug(writableLayer, id: id) + } + let virtioFSMounts = fileMountContext.transformedMounts.filter { if case .virtiofs(_) = $0.runtimeOptions { return true } return false @@ -477,13 +502,23 @@ extension LinuxPod { let agent = try await vm.dialAgent() do { - var mount = attachment.to - mount.destination = Self.guestRootfsPath(id) - try await agent.mount(mount) + if let writableAttachment { + try await agent.mountOverlayRootfs( + containerID: id, + rootfsAttachment: attachment, + writableAttachment: writableAttachment, + rootfsPath: Self.guestRootfsPath(id) + ) + } else { + var mount = attachment.to + mount.destination = Self.guestRootfsPath(id) + try await agent.mount(mount) + } - // Filter out shared mounts — those are handled separately as - // pod volume bind mounts. Without it here, a container added to an - // already-created would add a duplicated mount into the shared VM. + // Shared mounts are handled separately as pod volume + // bind mounts; without the filter here, a container + // added to an already-created pod would add a + // duplicated mount into the shared VM. let nonSharedMounts = fileMountContext.transformedMounts.filter { if case .shared = $0.runtimeOptions { return false } return true @@ -491,6 +526,7 @@ extension LinuxPod { try vm.registerMounts( id: id, rootfs: attachment, + writableLayer: writableAttachment, additionalMounts: nonSharedMounts ) @@ -591,6 +627,7 @@ extension LinuxPod { state.containers[id] = PodContainer( id: id, rootfs: rootfs, + writableLayer: writableLayer, config: config, state: .created, process: nil, @@ -630,6 +667,7 @@ extension LinuxPod { } machineStorage.containers[id] = ContainerMounts( rootfs: modifiedRootfs, + writableLayer: container.writableLayer, mounts: containerMounts ) } @@ -809,6 +847,15 @@ extension LinuxPod { guard let attached = vm.storage.containers[container.id] else { throw ContainerizationError(.notFound, message: "rootfs mount not found for container \(container.id)") } + if let writableAttachment = attached.writableLayer { + try await agent.mountOverlayRootfs( + containerID: container.id, + rootfsAttachment: attached.rootfs, + writableAttachment: writableAttachment, + rootfsPath: Self.guestRootfsPath(container.id) + ) + continue + } var rootfs = attached.rootfs.to rootfs.destination = Self.guestRootfsPath(container.id) try await agent.mount(rootfs) @@ -936,7 +983,7 @@ extension LinuxPod { let agent = try await createdState.vm.dialAgent() do { - var spec = self.generateRuntimeSpec(containerID: containerID, config: container.config, rootfs: container.rootfs) + var spec = self.generateRuntimeSpec(containerID: containerID, config: container.config, rootfs: container.rootfs, writableLayer: container.writableLayer) // We don't need the rootfs, nor do OCI runtimes want it included. // Also filter out file mount holding directories - we mount those separately under /run. // Transform virtiofs mounts to bind mounts from /run/virtiofs/{tag} @@ -1101,12 +1148,21 @@ extension LinuxPod { try await process.kill(.kill) try await process.wait(timeoutInSeconds: 3) + let hasWritableLayer = container.writableLayer != nil try await createdState.vm.withAgent { agent in // Unmount the rootfs try await agent.umount( path: Self.guestRootfsPath(containerID), flags: 0 ) + + // If we have a writable layer, we also need to unmount the lower and upper layers. + if hasWritableLayer { + let upperPath = "/run/container/\(containerID)/upper" + let lowerPath = "/run/container/\(containerID)/lower" + try await agent.umount(path: upperPath, flags: 0) + try await agent.umount(path: lowerPath, flags: 0) + } } // Release the hotplug device and virtiofs shares so they can be reused by new containers @@ -1261,7 +1317,7 @@ extension LinuxPod { ) } - var spec = self.generateRuntimeSpec(containerID: containerID, config: container.config, rootfs: container.rootfs) + var spec = self.generateRuntimeSpec(containerID: containerID, config: container.config, rootfs: container.rootfs, writableLayer: container.writableLayer) // Inherit environment variables, working directory, user, capabilities, rlimits from container process. // Reset: process arguments, terminal, stdio as these are not supposed to be inherited. var config = container.config.process diff --git a/Sources/Containerization/OverlayRootfs.swift b/Sources/Containerization/OverlayRootfs.swift new file mode 100644 index 00000000..9af76763 --- /dev/null +++ b/Sources/Containerization/OverlayRootfs.swift @@ -0,0 +1,64 @@ +//===----------------------------------------------------------------------===// +// Copyright © 2026 Apple Inc. and the Containerization project authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//===----------------------------------------------------------------------===// + +import ContainerizationOCI + +extension VirtualMachineAgent { + /// Mount a container's rootfs as an overlay, with the image as the lower + /// layer and the container's writable layer as the upper, so that writes + /// land in the layer and the image stays as it is. + func mountOverlayRootfs( + containerID: String, + rootfsAttachment: AttachedFilesystem, + writableAttachment: AttachedFilesystem, + rootfsPath: String + ) async throws { + let lowerPath = "/run/container/\(containerID)/lower" + let upperMountPath = "/run/container/\(containerID)/upper" + let upperPath = "/run/container/\(containerID)/upper/diff" + let workPath = "/run/container/\(containerID)/upper/work" + + // Mount the image (lower layer) as read-only. + var lowerMount = rootfsAttachment.to + lowerMount.destination = lowerPath + if !lowerMount.options.contains("ro") { + lowerMount.options.append("ro") + } + try await self.mount(lowerMount) + + // Mount the writable layer. + var upperMount = writableAttachment.to + upperMount.destination = upperMountPath + try await self.mount(upperMount) + + // Create the upper and work directories inside the writable layer. + try await self.mkdir(path: upperPath, all: true, perms: 0o755) + try await self.mkdir(path: workPath, all: true, perms: 0o755) + + // Mount the overlay. + let overlayMount = ContainerizationOCI.Mount( + type: "overlay", + source: "overlay", + destination: rootfsPath, + options: [ + "lowerdir=\(lowerPath)", + "upperdir=\(upperPath)", + "workdir=\(workPath)", + ] + ) + try await self.mount(overlayMount) + } +} diff --git a/Sources/Containerization/VZVirtualMachineInstance.swift b/Sources/Containerization/VZVirtualMachineInstance.swift index 69828ec9..135879e0 100644 --- a/Sources/Containerization/VZVirtualMachineInstance.swift +++ b/Sources/Containerization/VZVirtualMachineInstance.swift @@ -325,9 +325,9 @@ extension VZVirtualMachineInstance: VirtualMachineInstance { return try await hotplugProvider.hotplug(block, id: id) } - public func registerMounts(id: String, rootfs: AttachedFilesystem, additionalMounts: [Mount]) throws { + public func registerMounts(id: String, rootfs: AttachedFilesystem, writableLayer: AttachedFilesystem?, additionalMounts: [Mount]) throws { guard let hotplugProvider else { return } - try hotplugProvider.registerMounts(id: id, rootfs: rootfs, additionalMounts: additionalMounts) + try hotplugProvider.registerMounts(id: id, rootfs: rootfs, writableLayer: writableLayer, additionalMounts: additionalMounts) } public func releaseHotplug(id: String) async throws { diff --git a/Sources/Containerization/VirtualMachineInstance.swift b/Sources/Containerization/VirtualMachineInstance.swift index b9340612..da7935d5 100644 --- a/Sources/Containerization/VirtualMachineInstance.swift +++ b/Sources/Containerization/VirtualMachineInstance.swift @@ -78,8 +78,10 @@ public protocol VirtualMachineInstance: Sendable { /// so they can be found when building the container's OCI spec. /// - Parameter id: The container ID /// - Parameter rootfs: The rootfs attachment from hotplug + /// - Parameter writableLayer: The container's writable layer attachment when it + /// has one /// - Parameter additionalMounts: Additional mounts (like /proc, /sys) to register - func registerMounts(id: String, rootfs: AttachedFilesystem, additionalMounts: [Mount]) throws + func registerMounts(id: String, rootfs: AttachedFilesystem, writableLayer: AttachedFilesystem?, additionalMounts: [Mount]) throws /// Release a hotplug device. /// This should be called when a hotplugged container is stopped or fails to start. @@ -107,7 +109,7 @@ extension VirtualMachineInstance { public func hotplug(_ block: Mount, id: String) async throws -> AttachedFilesystem { throw ContainerizationError(.unsupported, message: "hotplug not supported") } - public func registerMounts(id: String, rootfs: AttachedFilesystem, additionalMounts: [Mount]) throws { + public func registerMounts(id: String, rootfs: AttachedFilesystem, writableLayer: AttachedFilesystem?, additionalMounts: [Mount]) throws { // no-op default } public func releaseHotplug(id: String) async throws { diff --git a/Sources/Integration/PodTests.swift b/Sources/Integration/PodTests.swift index 5703bde1..2c0dffc2 100644 --- a/Sources/Integration/PodTests.swift +++ b/Sources/Integration/PodTests.swift @@ -2478,6 +2478,123 @@ extension IntegrationSuite { } #endif + /// A container in a pod given a writable layer writes into it, and the + /// image it was built from is left as it is for the pod's others. + /// Add a container with a writable layer to a pod whose machine is + /// already running: the overlay assembles from the two disks attached + /// while it runs, and writes land in the layer. + func testPodHotplugWritableLayer() async throws { + let id = "test-pod-hotplug-writable-layer" + let bs = try await bootstrap(id) + + let pod = try LinuxPod(id, vmm: bs.vmm) { config in + config.cpus = 4 + config.memoryInBytes = 1024.mib() + config.bootLog = bs.bootLog + } + + try await pod.addContainer("seed", rootfs: try cloneRootfs(bs.rootfs, testID: id, containerID: "seed")) { config in + config.process.arguments = ["/bin/sleep", "infinity"] + } + + try await pod.create() + + let writableLayerPath = Self.testDir.appending(component: "\(id)-writable.ext4") + try? FileManager.default.removeItem(at: writableLayerPath) + let filesystem = try EXT4.Formatter(FilePath(writableLayerPath.absolutePath()), minDiskSize: 512.mib()) + try filesystem.close() + let writableLayer = Mount.block( + format: "ext4", + source: writableLayerPath.absolutePath(), + destination: "/", + options: [] + ) + + let buffer = BufferWriter() + try await pod.addContainer( + "hot", + rootfs: try cloneRootfs(bs.rootfs, testID: id, containerID: "hot"), + writableLayer: writableLayer + ) { config in + config.process.arguments = ["/bin/sh", "-c", "echo 'written into a layer added while running' > /written && cat /written"] + config.process.stdout = buffer + } + + do { + try await pod.startContainer("hot") + let status = try await pod.waitContainer("hot") + + try await pod.stopContainer("hot") + try await pod.stop() + + guard status.exitCode == 0 else { + throw IntegrationError.assert(msg: "hot container status \(status) != 0") + } + let expected = "written into a layer added while running" + guard let output = String(data: buffer.data, encoding: .utf8), + output.trimmingCharacters(in: .whitespacesAndNewlines) == expected + else { + throw IntegrationError.assert( + msg: "expected '\(expected)', got '\(String(data: buffer.data, encoding: .utf8) ?? "nil")'") + } + } catch { + try? await pod.stop() + throw error + } + } + + func testPodWritableLayer() async throws { + let id = "test-pod-writable-layer" + + let bs = try await bootstrap(id) + let pod = try LinuxPod(id, vmm: bs.vmm) { config in + config.cpus = 4 + config.memoryInBytes = 1024.mib() + config.bootLog = bs.bootLog + } + + let writableLayerPath = Self.testDir.appending(component: "\(id)-writable.ext4") + try? FileManager.default.removeItem(at: writableLayerPath) + let filesystem = try EXT4.Formatter(FilePath(writableLayerPath.absolutePath()), minDiskSize: 512.mib()) + try filesystem.close() + let writableLayer = Mount.block( + format: "ext4", + source: writableLayerPath.absolutePath(), + destination: "/", + options: [] + ) + + let buffer = BufferWriter() + try await pod.addContainer( + "layered", + rootfs: try cloneRootfs(bs.rootfs, testID: id, containerID: "layered"), + writableLayer: writableLayer + ) { config in + config.process.arguments = ["/bin/sh", "-c", "echo 'writable layer test' > /tmp/testfile && cat /tmp/testfile"] + config.process.stdout = buffer + } + + do { + try await pod.create() + try await pod.startContainer("layered") + let status = try await pod.waitContainer("layered") + try await pod.stop() + + guard status.exitCode == 0 else { + throw IntegrationError.assert(msg: "process failed with status \(status)") + } + guard let output = String(data: buffer.data, encoding: .utf8) else { + throw IntegrationError.assert(msg: "failed to convert stdout to UTF8") + } + guard output.trimmingCharacters(in: .whitespacesAndNewlines) == "writable layer test" else { + throw IntegrationError.assert(msg: "unexpected output: \(output)") + } + } catch { + try? await pod.stop() + throw error + } + } + /// A file copied into one container in a pod arrives in that container and /// nowhere else, and comes back out with what it held. func testPodCopyRoundTrip() async throws { diff --git a/Sources/Integration/Suite.swift b/Sources/Integration/Suite.swift index 769ccc12..002e30a4 100644 --- a/Sources/Integration/Suite.swift +++ b/Sources/Integration/Suite.swift @@ -560,6 +560,7 @@ struct IntegrationSuite: AsyncParsableCommand { Test("pod container resource limits", testPodContainerResourceLimits), Test("pod container filesystem isolation", testPodContainerFilesystemIsolation), Test("pod copy round trip", testPodCopyRoundTrip), + Test("pod writable layer", testPodWritableLayer), Test("pod container PID namespace isolation", testPodContainerPIDNamespaceIsolation), Test("pod container independent resource limits", testPodContainerIndependentResourceLimits), Test("pod shared PID namespace", testPodSharedPIDNamespace), @@ -651,6 +652,7 @@ struct IntegrationSuite: AsyncParsableCommand { let linuxOnlyTests: [Test] = [ Test("pod hotplug block rootfs", testPodHotplugBlockRootfs), Test("pod hotplug virtiofs rootfs", testPodHotplugVirtiofsRootfs), + Test("pod hotplug writable layer", testPodHotplugWritableLayer), ] let tests: [Test] = crossPlatformTests + linuxOnlyTests #endif From 9adf57c175cc721fae870d0f3659523b0fe2e415 Mon Sep 17 00:00:00 2001 From: Aaron Paterson Date: Sat, 8 Aug 2026 02:01:02 +0000 Subject: [PATCH 13/25] Name the project the guest file transfer belongs to The file carried the sibling repository's copyright line, so the license header check found no header it recognized. --- Sources/Containerization/GuestFileTransfer.swift | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Sources/Containerization/GuestFileTransfer.swift b/Sources/Containerization/GuestFileTransfer.swift index d508c571..5eeb039d 100644 --- a/Sources/Containerization/GuestFileTransfer.swift +++ b/Sources/Containerization/GuestFileTransfer.swift @@ -1,5 +1,5 @@ //===----------------------------------------------------------------------===// -// Copyright © 2026 Apple Inc. and the container project authors. +// Copyright © 2026 Apple Inc. and the Containerization project authors. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. From bf57208a9871a0079fa6803f3f88c484f2e7b2a9 Mon Sep 17 00:00:00 2001 From: Aaron Paterson Date: Mon, 10 Aug 2026 05:40:19 +0000 Subject: [PATCH 14/25] Let a pod give a container's place back Stopping a container tears down what it was running and keeps its place: the name still answers for it, and placing another container under it is refused. Removal is the separate act the runtime specification names for giving the place up, taken once the container has stopped, so a name can run again in a machine that outlives what it last ran. https://github.com/kubernetes/cri-api/blob/master/pkg/apis/runtime/v1/api.proto --- Sources/Containerization/LinuxPod.swift | 29 +++++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/Sources/Containerization/LinuxPod.swift b/Sources/Containerization/LinuxPod.swift index 76967821..9d98d79f 100644 --- a/Sources/Containerization/LinuxPod.swift +++ b/Sources/Containerization/LinuxPod.swift @@ -1189,6 +1189,35 @@ extension LinuxPod { } } + /// Take a container out of the pod, so its name is free to place again. + /// + /// Stopping a container tears down what it was running and keeps its + /// place; the name still answers for it, and placing another container + /// under it is refused. Removal is the separate act the runtime + /// specification names for giving the place up, taken once the container + /// has stopped. A container that is running keeps its place and this + /// call refuses it. + /// https://github.com/kubernetes/cri-api/blob/master/pkg/apis/runtime/v1/api.proto + public func removeContainer(_ containerID: String) async throws { + try await self.state.withLock { state in + guard let container = state.containers[containerID] else { + throw ContainerizationError( + .notFound, + message: "container \(containerID) not found in pod" + ) + } + switch container.state { + case .registered, .stopped, .errored: + state.containers[containerID] = nil + default: + throw ContainerizationError( + .invalidState, + message: "container \(containerID) must stop before it is removed" + ) + } + } + } + /// Stop the pod's VM and all containers. public func stop() async throws { try await self.state.withLock { state in From a72170e0fdd7d1f7db2000fdf6a98ef7e0dc8ce1 Mon Sep 17 00:00:00 2001 From: Aaron Paterson Date: Sun, 2 Aug 2026 01:22:56 +0000 Subject: [PATCH 15/25] Let a caller set the runtime spec's hooks The runtime specification's hooks have a field on the spec type and no way to reach it: the configuration has nowhere to put them and the spec the container is built from never carries any. They travel intact once set, and the bundle's config.json contains them. Whether anything runs them depends on the runtime the container is launched under, which the field's documentation says. --- Sources/Containerization/LinuxContainer.swift | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/Sources/Containerization/LinuxContainer.swift b/Sources/Containerization/LinuxContainer.swift index ce57488f..0d0c7745 100644 --- a/Sources/Containerization/LinuxContainer.swift +++ b/Sources/Containerization/LinuxContainer.swift @@ -91,6 +91,16 @@ public final class LinuxContainer: Container, Sendable { /// Run the container with a minimal init process that handles signal /// forwarding and zombie reaping. public var useInit: Bool = false + /// Programs the runtime runs at points in the container's lifecycle, + /// as described by the runtime specification. + /// + /// The paths are resolved inside the guest, because that is where the + /// runtime that would run them lives. + /// + /// NOTE: these reach the bundle's config.json but are only acted on + /// when the container runs under an external OCI runtime. The default + /// launcher is not one, so they have no effect on that path. + public var hooks: ContainerizationOCI.Hooks? = nil /// Additional CPU cores to allocate for the virtual machine on top /// of the container's configured `cpus` value. public var cpuOverhead: Int = 1 @@ -405,6 +415,8 @@ public final class LinuxContainer: Container, Sendable { // Process toggles. spec.process = config.process.toOCI() + spec.hooks = config.hooks + // Wrap with init process if requested. if config.useInit { let originalArgs = spec.process?.args ?? [] From 12bf1f6840db7987e81dbd4565f806c9c66db38f Mon Sep 17 00:00:00 2001 From: Aaron Paterson Date: Sun, 2 Aug 2026 01:22:56 +0000 Subject: [PATCH 16/25] Let a pod's containers reach an OCI runtime and its hooks A container in a pod is spawned with the runtime path hardcoded to nil, so the runtime a standalone container reaches through `ociRuntimePath` is out of reach for the same container placed in a pod, and the spec it is built from carries no hooks for that runtime to run. Nothing about a pod prevents either: the runtime path is passed per process, and the pause process is the only one with a reason to stay on the default. The configuration carries both. The container's own runtime path is used when its process starts and when a process is executed in it, and its hooks travel on the spec, as they do for a standalone container. --- Sources/Containerization/LinuxPod.swift | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/Sources/Containerization/LinuxPod.swift b/Sources/Containerization/LinuxPod.swift index 9d98d79f..1dd65e07 100644 --- a/Sources/Containerization/LinuxPod.swift +++ b/Sources/Containerization/LinuxPod.swift @@ -137,6 +137,13 @@ public final class LinuxPod: Sendable { /// Run the container with a minimal init process that handles signal /// forwarding and zombie reaping. public var useInit: Bool = false + /// EXPERIMENTAL: Path in the root filesystem for the virtual + /// machine where the OCI runtime used to spawn the container lives. + public var ociRuntimePath: String? + /// Hooks the runtime specification carries for the container. An OCI + /// runtime is what runs them, so they take effect for a container + /// given an `ociRuntimePath`. + public var hooks: ContainerizationOCI.Hooks? = nil public init() {} } @@ -347,6 +354,7 @@ public final class LinuxPod: Sendable { if let hostname = config.hostname ?? self.config.hostname { spec.hostname = hostname } + spec.hooks = config.hooks // Linux toggles spec.linux?.sysctl = config.sysctl @@ -1085,7 +1093,7 @@ extension LinuxPod { containerID: containerID, spec: spec, io: stdio, - ociRuntimePath: nil, + ociRuntimePath: container.config.ociRuntimePath, agent: agent, vm: createdState.vm, logger: self.logger @@ -1370,7 +1378,7 @@ extension LinuxPod { containerID: containerID, spec: spec, io: stdio, - ociRuntimePath: nil, + ociRuntimePath: container.config.ociRuntimePath, agent: agent, vm: createdState.vm, logger: self.logger From 13a720fb157b9be53e50c63add504803dbf4e0de Mon Sep 17 00:00:00 2001 From: Aaron Paterson Date: Sun, 2 Aug 2026 02:35:37 +0000 Subject: [PATCH 17/25] Create the devices a container's runtime spec asks for vmexec builds the container's root without consulting the spec's device list, so a device arrives only if the guest kernel already exposed it in devtmpfs, carrying the kernel's permissions. Those are stricter than what a machine running udev shows: /dev/net/tun and /dev/fuse are 0600 root here and 0666 on any systemd host, which its udev rules set. A container process that is not root therefore cannot open them, which is what a nested rootless runtime needs to do. Create each device the spec names, and set the permissions it asks for on one that is already present, so the spec says what the container sees. LinuxContainer gains the field to populate the list with. --- Sources/Containerization/LinuxContainer.swift | 8 +++ Sources/Integration/ContainerTests.swift | 43 ++++++++++++ Sources/Integration/Suite.swift | 1 + vminitd/Sources/LCShim/include/syscall.h | 3 + vminitd/Sources/LCShim/syscall.c | 5 ++ vminitd/Sources/vmexec/RunCommand.swift | 67 ++++++++++++++++++- 6 files changed, 125 insertions(+), 2 deletions(-) diff --git a/Sources/Containerization/LinuxContainer.swift b/Sources/Containerization/LinuxContainer.swift index 0d0c7745..72ddced4 100644 --- a/Sources/Containerization/LinuxContainer.swift +++ b/Sources/Containerization/LinuxContainer.swift @@ -101,6 +101,13 @@ public final class LinuxContainer: Container, Sendable { /// when the container runs under an external OCI runtime. The default /// launcher is not one, so they have no effect on that path. public var hooks: ContainerizationOCI.Hooks? = nil + /// Devices the container should be given, with the permissions it + /// should see them under. + /// + /// A device already present in the container's /dev arrives with the + /// kernel's permissions, which are stricter than the ones a machine + /// running udev would show. Naming it here is how those are asked for. + public var devices: [ContainerizationOCI.LinuxDevice] = [] /// Additional CPU cores to allocate for the virtual machine on top /// of the container's configured `cpus` value. public var cpuOverhead: Int = 1 @@ -416,6 +423,7 @@ public final class LinuxContainer: Container, Sendable { spec.process = config.process.toOCI() spec.hooks = config.hooks + spec.linux?.devices = config.devices // Wrap with init process if requested. if config.useInit { diff --git a/Sources/Integration/ContainerTests.swift b/Sources/Integration/ContainerTests.swift index 9980cf21..cf534a91 100644 --- a/Sources/Integration/ContainerTests.swift +++ b/Sources/Integration/ContainerTests.swift @@ -315,6 +315,49 @@ extension IntegrationSuite { } } + func testContainerDeclaredDevices() async throws { + let id = "test-container-declared-devices" + let bs = try await bootstrap(id) + + let buffer = BufferWriter() + let container = try LinuxContainer(id, rootfs: bs.rootfs, vmm: bs.vmm) { config in + // The permissions a device arrives with are the kernel's; the ones + // it should have are the spec's. These two are what unprivileged + // container runtimes reach for, and udev grants them the same way. + config.devices = [ + ContainerizationOCI.LinuxDevice( + path: "/dev/net/tun", type: "c", major: 10, minor: 200, + fileMode: 0o666, uid: 0, gid: 0), + ContainerizationOCI.LinuxDevice( + path: "/dev/fuse", type: "c", major: 10, minor: 229, + fileMode: 0o666, uid: 0, gid: 0), + ] + config.process.user = ContainerizationOCI.User(uid: 1000, gid: 1000) + config.process.arguments = [ + "/bin/sh", "-c", + "ls -l /dev/net/tun /dev/fuse; " + + "if : < /dev/net/tun; then echo TUN_OPEN; else echo TUN_DENIED; fi; " + + "if : < /dev/fuse; then echo FUSE_OPEN; else echo FUSE_DENIED; fi", + ] + config.process.stdout = buffer + config.bootLog = bs.bootLog + } + try await container.create() + try await container.start() + _ = try await container.wait() + try await container.stop() + + let out = String(data: buffer.data, encoding: .utf8) ?? "" + // Both are readable by a process that is not root, which is what they + // are for, and which the kernel's own permissions would not allow. + guard out.contains("TUN_OPEN"), out.contains("FUSE_OPEN") else { + throw IntegrationError.assert(msg: "declared devices were not usable: '\(out)'") + } + guard out.contains("crw-rw-rw-") else { + throw IntegrationError.assert(msg: "devices did not take the mode asked for: '\(out)'") + } + } + func testProcessEchoHi() async throws { let id = "test-process-echo-hi" let bs = try await bootstrap(id) diff --git a/Sources/Integration/Suite.swift b/Sources/Integration/Suite.swift index 002e30a4..cb4fdf78 100644 --- a/Sources/Integration/Suite.swift +++ b/Sources/Integration/Suite.swift @@ -438,6 +438,7 @@ struct IntegrationSuite: AsyncParsableCommand { // Process basics Test("process true", testProcessTrue), Test("process false", testProcessFalse), + Test("declared devices", testContainerDeclaredDevices), Test("process echo hi", testProcessEchoHi), Test("process no executable", testProcessNoExecutable), Test("process user", testProcessUser), diff --git a/vminitd/Sources/LCShim/include/syscall.h b/vminitd/Sources/LCShim/include/syscall.h index 815dd476..55acb99b 100644 --- a/vminitd/Sources/LCShim/include/syscall.h +++ b/vminitd/Sources/LCShim/include/syscall.h @@ -99,6 +99,9 @@ int CZ_pidfd_open(pid_t pid, unsigned int flags); #endif int CZ_pidfd_getfd(int pidfd, int targetfd, unsigned int flags); +// makedev(3) is a macro, so Swift cannot call it. +dev_t CZ_makedev(unsigned int major, unsigned int minor); + int CZ_prctl_set_no_new_privs(); #endif diff --git a/vminitd/Sources/LCShim/syscall.c b/vminitd/Sources/LCShim/syscall.c index 094f6c61..153fa263 100644 --- a/vminitd/Sources/LCShim/syscall.c +++ b/vminitd/Sources/LCShim/syscall.c @@ -18,6 +18,7 @@ #include #include #include +#include #include #include "syscall.h" @@ -38,6 +39,10 @@ int CZ_pidfd_getfd(int pidfd, int targetfd, unsigned int flags) { return syscall(SYS_pidfd_getfd, pidfd, targetfd, flags); } +dev_t CZ_makedev(unsigned int major, unsigned int minor) { + return makedev(major, minor); +} + int CZ_prctl_set_no_new_privs() { return prctl(PR_SET_NO_NEW_PRIVS, 1, 0, 0, 0); } diff --git a/vminitd/Sources/vmexec/RunCommand.swift b/vminitd/Sources/vmexec/RunCommand.swift index e20b86ad..b880cdfd 100644 --- a/vminitd/Sources/vmexec/RunCommand.swift +++ b/vminitd/Sources/vmexec/RunCommand.swift @@ -55,11 +55,13 @@ struct RunCommand: ParsableCommand { private func childRootSetup( rootfs: ContainerizationOCI.Root, - mounts: [ContainerizationOCI.Mount] + mounts: [ContainerizationOCI.Mount], + devices: [ContainerizationOCI.LinuxDevice] ) throws { // setup rootfs try prepareRoot(rootfs: rootfs.path) try mountRootfs(rootfs: rootfs.path, mounts: mounts) + try createDevices(rootfs: rootfs.path, devices: devices) try setDevSymlinks(rootfs: rootfs.path) try pivotRoot(rootfs: rootfs.path) @@ -190,7 +192,7 @@ struct RunCommand: ParsableCommand { throw App.Errno(stage: "setsid()") } - try childRootSetup(rootfs: root, mounts: spec.mounts) + try childRootSetup(rootfs: root, mounts: spec.mounts, devices: spec.linux?.devices ?? []) if process.terminal { let pty = try Console() @@ -382,6 +384,67 @@ struct RunCommand: ParsableCommand { } } + /// Give the container the devices its runtime spec asks for, as runc does + /// from the same field. + /// https://github.com/opencontainers/runc/blob/main/libcontainer/rootfs_linux.go + /// + /// A device the spec names may already be present, because the container's + /// /dev can be a devtmpfs carrying everything the guest kernel exposes, in + /// which case it arrives with the kernel's own permissions rather than the + /// ones the spec asks for: the kernel gives a misc device no mode of its + /// own, so it lands at 0600 root. A machine running udev is what usually + /// says otherwise, `/dev/net/tun` and `/dev/fuse` being 0666 there because + /// its rules say so; there is no udev here, so the spec is what says. + /// https://github.com/systemd/systemd/blob/main/rules.d/50-udev-default.rules.in + private func createDevices(rootfs: String, devices: [ContainerizationOCI.LinuxDevice]) throws { + let rootfsURL = URL(fileURLWithPath: rootfs) + for device in devices { + let path = rootfsURL.appendingPathComponent(device.path).path + let mode = device.fileMode ?? 0o600 + + let kind: mode_t + switch device.type { + case "c", "u": + kind = S_IFCHR + case "b": + kind = S_IFBLK + case "p": + kind = S_IFIFO + default: + throw App.Failure(message: "unknown device type \(device.type) for \(device.path)") + } + + let parent = URL(fileURLWithPath: path).deletingLastPathComponent().path + try FileManager.default.createDirectory( + atPath: parent, withIntermediateDirectories: true) + + // Truncating as runc does, since the spec types these wider than + // any device number the kernel will encode. + // https://github.com/opencontainers/runc/blob/main/libcontainer/rootfs_linux.go + let id = CZ_makedev( + UInt32(truncatingIfNeeded: device.major), + UInt32(truncatingIfNeeded: device.minor)) + + if mknod(path, kind | mode_t(mode), id) != 0 { + guard errno == EEXIST else { + throw App.Errno(stage: "mknod(\(device.path))") + } + // Already there, so only its permissions are ours to set. + guard chmod(path, mode_t(mode)) == 0 else { + throw App.Errno(stage: "chmod(\(device.path))") + } + } + + if device.uid != nil || device.gid != nil { + let uid = device.uid ?? 0 + let gid = device.gid ?? 0 + guard chown(path, uid, gid) == 0 else { + throw App.Errno(stage: "chown(\(device.path))") + } + } + } + } + private func setDevSymlinks(rootfs: String) throws { let links: [(src: String, dst: String)] = [ ("/proc/self/fd", "/dev/fd"), From 3bb548c24ccc3da279aae238c9469a5b5565cf55 Mon Sep 17 00:00:00 2001 From: Aaron Paterson Date: Fri, 7 Aug 2026 20:40:46 +0000 Subject: [PATCH 18/25] Let a pod's containers be given devices A container is given devices with the permissions it should see them under, and the same container placed in a pod has nowhere to name them, so a device that a machine running udev would show relaxed arrives with the kernel's stricter permissions instead. The configuration carries them and they reach the runtime specification, as they do for a container with a machine of its own. --- Sources/Containerization/LinuxPod.swift | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/Sources/Containerization/LinuxPod.swift b/Sources/Containerization/LinuxPod.swift index 1dd65e07..f4ee7e00 100644 --- a/Sources/Containerization/LinuxPod.swift +++ b/Sources/Containerization/LinuxPod.swift @@ -137,6 +137,13 @@ public final class LinuxPod: Sendable { /// Run the container with a minimal init process that handles signal /// forwarding and zombie reaping. public var useInit: Bool = false + /// Devices the container should be given, with the permissions it + /// should see them under. + /// + /// A device already present in the container's /dev arrives with the + /// kernel's permissions, which are stricter than the ones a machine + /// running udev would show. Naming it here is how those are asked for. + public var devices: [ContainerizationOCI.LinuxDevice] = [] /// EXPERIMENTAL: Path in the root filesystem for the virtual /// machine where the OCI runtime used to spawn the container lives. public var ociRuntimePath: String? @@ -358,6 +365,7 @@ public final class LinuxPod: Sendable { // Linux toggles spec.linux?.sysctl = config.sysctl + spec.linux?.devices = config.devices spec.linux?.maskedPaths = config.maskedPaths spec.linux?.readonlyPaths = config.readonlyPaths From 5feab027e93ad31e7e123b149b5e968810a03dd9 Mon Sep 17 00:00:00 2001 From: Aaron Paterson Date: Fri, 7 Aug 2026 21:24:12 +0000 Subject: [PATCH 19/25] Let Virtualization take a disk into a running machine A machine's storage devices are fixed once it boots, so a container added to a pod that is already running has nowhere for its filesystem to arrive and the attempt is refused outright. The machine's USB controller takes devices while it runs, and mass storage is one of them. The machine is configured with a controller so one exists to attach to, and a provider attaches a disk through it, names it apart from the disks the machine booted with, and detaches it when the container is done. A directory share stays fixed at boot, which is said rather than pretended. --- .../Containerization/VZHotplugProvider.swift | 190 ++++++++++++++++++ .../VZVirtualMachineInstance.swift | 18 ++ Sources/Integration/PodTests.swift | 9 +- Sources/Integration/Suite.swift | 5 +- 4 files changed, 216 insertions(+), 6 deletions(-) create mode 100644 Sources/Containerization/VZHotplugProvider.swift diff --git a/Sources/Containerization/VZHotplugProvider.swift b/Sources/Containerization/VZHotplugProvider.swift new file mode 100644 index 00000000..92389e19 --- /dev/null +++ b/Sources/Containerization/VZHotplugProvider.swift @@ -0,0 +1,190 @@ +//===----------------------------------------------------------------------===// +// Copyright © 2026 Apple Inc. and the Containerization project authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//===----------------------------------------------------------------------===// + +#if os(macOS) + +import ContainerizationError +import ContainerizationExtras +import Foundation +import Logging +import Synchronization +import Virtualization + +/// Attaches block devices to a running virtual machine. +/// +/// Virtualization's storage devices are fixed once a machine boots, but its USB +/// controller takes devices while it runs, and mass storage is one of the +/// devices it takes. A block device attached this way appears to the guest as a +/// SCSI disk, so it is named from a separate run of letters to the virtio-blk +/// devices the machine booted with. +/// https://developer.apple.com/documentation/virtualization/vzusbcontroller +@available(macOS 15.0, *) +final class VZHotplugProvider: HotplugProvider, @unchecked Sendable { + /// A disk attached to the running machine, kept so it can be detached. + /// + /// The device is a Virtualization object, which is safe to touch on the + /// machine's own queue and nowhere else. Every use of it here is inside a + /// block dispatched to that queue. + private struct HotplugRecord: @unchecked Sendable { + let device: VZUSBMassStorageDevice + let letter: Character + } + + private let vm: VZVirtualMachine + private let queue: DispatchQueue + /// Names for the disks attached while the machine runs, which the guest + /// numbers apart from the ones it booted with. + private let allocator: any AddressAllocator + private let _mounts: Mutex<[String: [AttachedFilesystem]]> + private let _records: Mutex<[String: [HotplugRecord]]> + private let logger: Logger? + + init( + vm: VZVirtualMachine, + queue: DispatchQueue, + initialMounts: [String: [AttachedFilesystem]], + logger: Logger? + ) { + self.vm = vm + self.queue = queue + self.allocator = Character.blockDeviceTagAllocator() + self._mounts = Mutex(initialMounts) + self._records = Mutex([:]) + self.logger = logger + } + + var mounts: [String: [AttachedFilesystem]] { + _mounts.withLock { $0 } + } + + func withMountRegistry( + _ body: (inout sending [String: [AttachedFilesystem]]) throws -> sending T + ) rethrows -> T { + try _mounts.withLock(body) + } + + // MARK: - HotplugProvider conformance + + func hotplug(_ block: Mount, id: String) async throws -> AttachedFilesystem { + guard block.isBlock else { + throw ContainerizationError( + .invalidArgument, + message: "only a block device can be attached to a running machine" + ) + } + guard let controller = vm.usbControllers.first else { + throw ContainerizationError( + .unsupported, + message: "the machine has no USB controller to attach to" + ) + } + + let letter = try allocator.allocate() + do { + let attachment = try VZDiskImageStorageDeviceAttachment( + url: URL(filePath: block.source), + readOnly: block.options.contains("ro") + ) + let device = VZUSBMassStorageDevice( + configuration: VZUSBMassStorageDeviceConfiguration(attachment: attachment) + ) + + try await withCheckedThrowingContinuation { (continuation: CheckedContinuation) in + queue.async { + controller.attach(device: device) { error in + if let error { + continuation.resume(throwing: error) + return + } + continuation.resume() + } + } + } + + _records.withLock { + $0[id, default: []].append(HotplugRecord(device: device, letter: letter)) + } + + return AttachedFilesystem( + type: block.type, + source: "/dev/sd\(letter)", + destination: block.destination, + options: block.options + ) + } catch { + try? allocator.release(letter) + throw error + } + } + + func registerMounts(id: String, rootfs: AttachedFilesystem, additionalMounts: [Mount]) throws { + var attached: [AttachedFilesystem] = [rootfs] + for mount in additionalMounts { + attached.append(try AttachedFilesystem(mount: mount, allocator: allocator)) + } + _mounts.withLock { + $0[id, default: []].append(contentsOf: attached) + } + } + + func releaseHotplug(id: String) async throws { + let popped: [HotplugRecord] = _records.withLock { records in + defer { records.removeValue(forKey: id) } + return records[id] ?? [] + } + guard let controller = vm.usbControllers.first else { + return + } + + for record in popped { + do { + try await withCheckedThrowingContinuation { (continuation: CheckedContinuation) in + queue.async { + controller.detach(device: record.device) { error in + if let error { + continuation.resume(throwing: error) + return + } + continuation.resume() + } + } + } + } catch { + logger?.error( + "failed to detach a disk from the running machine", + metadata: ["id": "\(id)", "error": "\(error)"] + ) + } + try? allocator.release(record.letter) + } + + _mounts.withLock { $0.removeValue(forKey: id) } + } + + /// Virtualization shares directories through devices fixed at boot, so a + /// share cannot be added to a machine that is already running. + func hotplugVirtioFS(_ mounts: [Mount], id: String) async throws { + guard !mounts.isEmpty else { return } + throw ContainerizationError( + .unsupported, + message: "a directory share cannot be added to a running machine" + ) + } + + func releaseVirtioFS(id: String) async throws {} +} + +#endif diff --git a/Sources/Containerization/VZVirtualMachineInstance.swift b/Sources/Containerization/VZVirtualMachineInstance.swift index 135879e0..7ac3ff62 100644 --- a/Sources/Containerization/VZVirtualMachineInstance.swift +++ b/Sources/Containerization/VZVirtualMachineInstance.swift @@ -140,6 +140,17 @@ public final class VZVirtualMachineInstance: Sendable { queue: self.queue ) + // A disk can be attached while the machine runs, so the machine has + // somewhere to send the request. + if #available(macOS 15.0, *) { + self.hotplugProvider = VZHotplugProvider( + vm: self.vm, + queue: self.queue, + initialMounts: mountAttachments, + logger: logger + ) + } + for ext in config.extensions.compactMap({ $0 as? any VZInstanceExtension }) { try ext.didCreate(self) } @@ -525,6 +536,13 @@ extension VZVirtualMachineInstance.Configuration { platform.isNestedVirtualizationEnabled = self.nestedVirtualization config.platform = platform + // The machine's storage devices are fixed once it boots, so a disk + // that arrives later arrives over USB. The controller has to be in + // the configuration for one to exist to attach to. + if #available(macOS 15.0, *) { + config.usbControllers = [VZXHCIControllerConfiguration()] + } + for ext in self.extensions.compactMap({ $0 as? any VZInstanceExtension }) { try ext.configureVZ(&config, allocator: allocator, storageDeviceCount: storageDeviceCount, storage: self.storage) } diff --git a/Sources/Integration/PodTests.swift b/Sources/Integration/PodTests.swift index 2c0dffc2..a7d9e547 100644 --- a/Sources/Integration/PodTests.swift +++ b/Sources/Integration/PodTests.swift @@ -2432,9 +2432,11 @@ extension IntegrationSuite { } } - /// Hotplug a container with a block rootfs into a running pod VM. Guards - /// the existing block hotplug path against the registry-consolidation - /// change. CH-only. + #endif + + /// Add a container with a block rootfs to a pod whose machine is already + /// running. Both backends attach a disk to a running machine, so both are + /// held to it. func testPodHotplugBlockRootfs() async throws { let id = "test-pod-hotplug-block-rootfs" let bs = try await bootstrap(id) @@ -2476,7 +2478,6 @@ extension IntegrationSuite { throw error } } - #endif /// A container in a pod given a writable layer writes into it, and the /// image it was built from is left as it is for the pod's others. diff --git a/Sources/Integration/Suite.swift b/Sources/Integration/Suite.swift index cb4fdf78..457c2afe 100644 --- a/Sources/Integration/Suite.swift +++ b/Sources/Integration/Suite.swift @@ -648,8 +648,9 @@ struct IntegrationSuite: AsyncParsableCommand { ] + macOS26Tests() let tests: [Test] = crossPlatformTests + macOSOnlyTests #else - // Hotplug into a running pod VM is CH-only (VZ has no runtime hotplug), - // and no pod test elsewhere exercises addContainer-after-create. + // VZ takes a disk into a running machine over its USB controller, but + // the guest's naming of it is not yet settled, so the block case is + // held here alongside the directory share, which VZ fixes at boot. let linuxOnlyTests: [Test] = [ Test("pod hotplug block rootfs", testPodHotplugBlockRootfs), Test("pod hotplug virtiofs rootfs", testPodHotplugVirtiofsRootfs), From 80ac5dad21cde8608cef6e275938da6961048295 Mon Sep 17 00:00:00 2001 From: Aaron Paterson Date: Fri, 7 Aug 2026 21:46:30 +0000 Subject: [PATCH 20/25] Give the guest kernel the drivers a hotplugged disk needs A disk that arrives over the machine's USB controller needs a guest that can see it, and the kernel is built with no USB support at all and no SCSI disk driver, so the disk attaches to a guest with nothing to enumerate it. The controller, mass storage and the SCSI disk it presents as are built in. --- .../Containerization/VZHotplugProvider.swift | 17 +++++++++++------ Sources/Integration/Suite.swift | 11 +++++------ kernel/config-arm64 | 11 +++++++++-- 3 files changed, 25 insertions(+), 14 deletions(-) diff --git a/Sources/Containerization/VZHotplugProvider.swift b/Sources/Containerization/VZHotplugProvider.swift index 92389e19..8dad1bc3 100644 --- a/Sources/Containerization/VZHotplugProvider.swift +++ b/Sources/Containerization/VZHotplugProvider.swift @@ -21,7 +21,7 @@ import ContainerizationExtras import Foundation import Logging import Synchronization -import Virtualization +@preconcurrency import Virtualization /// Attaches block devices to a running virtual machine. /// @@ -85,12 +85,15 @@ final class VZHotplugProvider: HotplugProvider, @unchecked Sendable { message: "only a block device can be attached to a running machine" ) } - guard let controller = vm.usbControllers.first else { + // The controller and the device are Virtualization objects, touched + // only inside a block dispatched to the machine's own queue. + guard let first = vm.usbControllers.first else { throw ContainerizationError( .unsupported, message: "the machine has no USB controller to attach to" ) } + nonisolated(unsafe) let controller = first let letter = try allocator.allocate() do { @@ -98,7 +101,7 @@ final class VZHotplugProvider: HotplugProvider, @unchecked Sendable { url: URL(filePath: block.source), readOnly: block.options.contains("ro") ) - let device = VZUSBMassStorageDevice( + nonisolated(unsafe) let device = VZUSBMassStorageDevice( configuration: VZUSBMassStorageDeviceConfiguration(attachment: attachment) ) @@ -145,15 +148,17 @@ final class VZHotplugProvider: HotplugProvider, @unchecked Sendable { defer { records.removeValue(forKey: id) } return records[id] ?? [] } - guard let controller = vm.usbControllers.first else { + guard let first = vm.usbControllers.first else { return } + nonisolated(unsafe) let controller = first for record in popped { + nonisolated(unsafe) let device = record.device do { try await withCheckedThrowingContinuation { (continuation: CheckedContinuation) in queue.async { - controller.detach(device: record.device) { error in + controller.detach(device: device) { error in if let error { continuation.resume(throwing: error) return @@ -171,7 +176,7 @@ final class VZHotplugProvider: HotplugProvider, @unchecked Sendable { try? allocator.release(record.letter) } - _mounts.withLock { $0.removeValue(forKey: id) } + _ = _mounts.withLock { $0.removeValue(forKey: id) } } /// Virtualization shares directories through devices fixed at boot, so a diff --git a/Sources/Integration/Suite.swift b/Sources/Integration/Suite.swift index 457c2afe..959140ea 100644 --- a/Sources/Integration/Suite.swift +++ b/Sources/Integration/Suite.swift @@ -562,6 +562,8 @@ struct IntegrationSuite: AsyncParsableCommand { Test("pod container filesystem isolation", testPodContainerFilesystemIsolation), Test("pod copy round trip", testPodCopyRoundTrip), Test("pod writable layer", testPodWritableLayer), + Test("pod hotplug block rootfs", testPodHotplugBlockRootfs), + Test("pod hotplug writable layer", testPodHotplugWritableLayer), Test("pod container PID namespace isolation", testPodContainerPIDNamespaceIsolation), Test("pod container independent resource limits", testPodContainerIndependentResourceLimits), Test("pod shared PID namespace", testPodSharedPIDNamespace), @@ -648,13 +650,10 @@ struct IntegrationSuite: AsyncParsableCommand { ] + macOS26Tests() let tests: [Test] = crossPlatformTests + macOSOnlyTests #else - // VZ takes a disk into a running machine over its USB controller, but - // the guest's naming of it is not yet settled, so the block case is - // held here alongside the directory share, which VZ fixes at boot. + // A directory share is fixed at boot on VZ, so only the backend that + // adds one to a running machine is held to it. let linuxOnlyTests: [Test] = [ - Test("pod hotplug block rootfs", testPodHotplugBlockRootfs), - Test("pod hotplug virtiofs rootfs", testPodHotplugVirtiofsRootfs), - Test("pod hotplug writable layer", testPodHotplugWritableLayer), + Test("pod hotplug virtiofs rootfs", testPodHotplugVirtiofsRootfs) ] let tests: [Test] = crossPlatformTests + linuxOnlyTests #endif diff --git a/kernel/config-arm64 b/kernel/config-arm64 index 429535b9..547f5382 100644 --- a/kernel/config-arm64 +++ b/kernel/config-arm64 @@ -1896,7 +1896,9 @@ CONFIG_VIRTIO_BLK=y # CONFIG_SCSI_MOD=y # CONFIG_RAID_ATTRS is not set -# CONFIG_SCSI is not set +CONFIG_SCSI=y +CONFIG_SCSI_COMMON=y +CONFIG_BLK_DEV_SD=y # end of SCSI device support # CONFIG_ATA is not set @@ -2859,7 +2861,12 @@ CONFIG_HID_REDRAGON=y # end of HID support CONFIG_USB_OHCI_LITTLE_ENDIAN=y -# CONFIG_USB_SUPPORT is not set +CONFIG_USB_SUPPORT=y +CONFIG_USB=y +CONFIG_USB_PCI=y +CONFIG_USB_XHCI_HCD=y +CONFIG_USB_XHCI_PCI=y +CONFIG_USB_STORAGE=y # CONFIG_MMC is not set # CONFIG_MEMSTICK is not set # CONFIG_NEW_LEDS is not set From 85ba74b156cf755356dbbe46ae27564c697d8919 Mon Sep 17 00:00:00 2001 From: Aaron Paterson Date: Sat, 8 Aug 2026 02:03:48 +0000 Subject: [PATCH 21/25] Let a disk taken onto a bus without a rescan wait to be enumerated The guest already retries a mount that races a device the machine has taken while running, forcing a PCI rescan between attempts, but it did that only for a virtio source. A USB mass storage disk arrives as a SCSI disk on its own bus, which needs no rescan and does need the wait, so it went straight to the failure instead. The rescan is left where it is and the sources it is reached for now include the SCSI disks, so one mechanism covers a device however it arrives. --- Sources/Integration/PodTests.swift | 7 ++++--- vminitd/Sources/VminitdCore/Server+GRPC.swift | 21 ++++++++++++------- 2 files changed, 17 insertions(+), 11 deletions(-) diff --git a/Sources/Integration/PodTests.swift b/Sources/Integration/PodTests.swift index a7d9e547..77a887ed 100644 --- a/Sources/Integration/PodTests.swift +++ b/Sources/Integration/PodTests.swift @@ -2455,7 +2455,7 @@ extension IntegrationSuite { let buffer = BufferWriter() try await pod.addContainer("hot", rootfs: try cloneRootfs(bs.rootfs, testID: id, containerID: "hot")) { config in - config.process.arguments = ["/bin/echo", "hello from block rootfs"] + config.process.arguments = ["/bin/echo", "hello from a disk added while running"] config.process.stdout = buffer } @@ -2469,9 +2469,10 @@ extension IntegrationSuite { guard status.exitCode == 0 else { throw IntegrationError.assert(msg: "hot container status \(status) != 0") } - guard String(data: buffer.data, encoding: .utf8) == "hello from block rootfs\n" else { + let expected = "hello from a disk added while running\n" + guard String(data: buffer.data, encoding: .utf8) == expected else { throw IntegrationError.assert( - msg: "expected 'hello from block rootfs', got '\(String(data: buffer.data, encoding: .utf8) ?? "nil")'") + msg: "expected '\(expected)', got '\(String(data: buffer.data, encoding: .utf8) ?? "nil")'") } } catch { try? await pod.stop() diff --git a/vminitd/Sources/VminitdCore/Server+GRPC.swift b/vminitd/Sources/VminitdCore/Server+GRPC.swift index 7c6b7e13..b9b40b45 100644 --- a/vminitd/Sources/VminitdCore/Server+GRPC.swift +++ b/vminitd/Sources/VminitdCore/Server+GRPC.swift @@ -681,14 +681,19 @@ extension Initd: Com_Apple_Containerization_Sandbox_V3_SandboxContext.SimpleServ do { try mnt.mount(createWithPerms: 0o755) } catch { - // A hot-plugged virtio device (virtio-blk / virtio-fs) may not - // be enumerated by the guest yet when the host issues this - // mount immediately after vm.add-disk / vm.add-fs: cloud- - // hypervisor places the device on the PCI bus but the guest - // does not always auto-probe it. Force a PCI rescan and retry - // with a bounded wait. Scoped to hot-plug-candidate sources so - // an ordinary mount failure isn't delayed. - let hotplugCandidate = request.type == "virtiofs" || request.source.hasPrefix("/dev/vd") + // A device attached to a running machine may not be enumerated + // by the guest yet when the host issues this mount straight + // after attaching it, so the mount races the guest and finds + // nothing. A virtio device (virtio-blk / virtio-fs) is placed + // on the PCI bus, which the guest does not always auto-probe, + // so the rescan below prompts it; a USB mass storage disk + // (/dev/sd*) enumerates on its own bus and simply needs the + // retry to wait for it. Scoped to the sources that can arrive + // this way so an ordinary mount failure isn't delayed. + let hotplugCandidate = + request.type == "virtiofs" + || request.source.hasPrefix("/dev/vd") + || request.source.hasPrefix("/dev/sd") guard hotplugCandidate else { throw error } if let rescan = FileHandle(forWritingAtPath: "/sys/bus/pci/rescan") { From ba26b76d94b282a9ae5a5fb6a37df30baac945c4 Mon Sep 17 00:00:00 2001 From: Aaron Paterson Date: Sat, 8 Aug 2026 02:03:56 +0000 Subject: [PATCH 22/25] Read a machine's mounts from the provider that registers them A hotplug provider holds the machine's mount registry, seeded with what the machine booted with, so that a container added while the machine runs is registered alongside the ones that were there from the start. Cloud hypervisor's machine reads it back through the provider for exactly that reason. Virtualization's machine kept a second registry of its own and answered from that one, so every registration the provider took landed somewhere nothing read. A container added to a running machine was then built with no mounts at all, and its process died reaching for /dev/null after the pivot rather than anywhere near the registry that was missing. The registry belongs to whichever provider is installed, which the protocol now says, so the machine has one to forward to and answers from its own copy only where no provider exists. --- .../Containerization/HotplugProvider.swift | 10 +++++++ .../Containerization/VZHotplugProvider.swift | 29 ++++++++++--------- .../VZVirtualMachineInstance.swift | 16 ++++++++-- 3 files changed, 38 insertions(+), 17 deletions(-) diff --git a/Sources/Containerization/HotplugProvider.swift b/Sources/Containerization/HotplugProvider.swift index 94fad4b4..509de9f0 100644 --- a/Sources/Containerization/HotplugProvider.swift +++ b/Sources/Containerization/HotplugProvider.swift @@ -19,6 +19,16 @@ /// Conforming types implement the mechanics of hotplugging block devices and /// virtiofs shares into a running VM. public protocol HotplugProvider: Sendable { + /// The machine's attached storage. + /// + /// A provider holds the machine's whole registry, seeded with what the + /// machine booted with, so a device taken while it runs is registered + /// alongside the rest and a container reads one registry either way. + var storage: MachineAttachments { get } + + /// Mutate the storage registry. + func withStorage(_ body: (inout sending MachineAttachments) throws -> sending T) rethrows -> T + /// Hotplug a block device into the running VM. /// - Parameters: /// - block: The mount configuration for the block device diff --git a/Sources/Containerization/VZHotplugProvider.swift b/Sources/Containerization/VZHotplugProvider.swift index 8dad1bc3..d6db5c1a 100644 --- a/Sources/Containerization/VZHotplugProvider.swift +++ b/Sources/Containerization/VZHotplugProvider.swift @@ -48,32 +48,32 @@ final class VZHotplugProvider: HotplugProvider, @unchecked Sendable { /// Names for the disks attached while the machine runs, which the guest /// numbers apart from the ones it booted with. private let allocator: any AddressAllocator - private let _mounts: Mutex<[String: [AttachedFilesystem]]> + private let _storage: Mutex private let _records: Mutex<[String: [HotplugRecord]]> private let logger: Logger? init( vm: VZVirtualMachine, queue: DispatchQueue, - initialMounts: [String: [AttachedFilesystem]], + initialStorage: MachineAttachments, logger: Logger? ) { self.vm = vm self.queue = queue self.allocator = Character.blockDeviceTagAllocator() - self._mounts = Mutex(initialMounts) + self._storage = Mutex(initialStorage) self._records = Mutex([:]) self.logger = logger } - var mounts: [String: [AttachedFilesystem]] { - _mounts.withLock { $0 } + var storage: MachineAttachments { + _storage.withLock { $0 } } - func withMountRegistry( - _ body: (inout sending [String: [AttachedFilesystem]]) throws -> sending T + func withStorage( + _ body: (inout sending MachineAttachments) throws -> sending T ) rethrows -> T { - try _mounts.withLock(body) + try _storage.withLock(body) } // MARK: - HotplugProvider conformance @@ -133,13 +133,14 @@ final class VZHotplugProvider: HotplugProvider, @unchecked Sendable { } } - func registerMounts(id: String, rootfs: AttachedFilesystem, additionalMounts: [Mount]) throws { - var attached: [AttachedFilesystem] = [rootfs] + func registerMounts(id: String, rootfs: AttachedFilesystem, writableLayer: AttachedFilesystem?, additionalMounts: [Mount]) throws { + var mounts: [AttachedFilesystem] = [] for mount in additionalMounts { - attached.append(try AttachedFilesystem(mount: mount, allocator: allocator)) + mounts.append(try AttachedFilesystem(mount: mount, allocator: allocator)) } - _mounts.withLock { - $0[id, default: []].append(contentsOf: attached) + let container = ContainerAttachments(rootfs: rootfs, writableLayer: writableLayer, mounts: mounts) + _storage.withLock { + $0.containers[id] = container } } @@ -176,7 +177,7 @@ final class VZHotplugProvider: HotplugProvider, @unchecked Sendable { try? allocator.release(record.letter) } - _ = _mounts.withLock { $0.removeValue(forKey: id) } + _ = _storage.withLock { $0.containers.removeValue(forKey: id) } } /// Virtualization shares directories through devices fixed at boot, so a diff --git a/Sources/Containerization/VZVirtualMachineInstance.swift b/Sources/Containerization/VZVirtualMachineInstance.swift index 7ac3ff62..b7cf894e 100644 --- a/Sources/Containerization/VZVirtualMachineInstance.swift +++ b/Sources/Containerization/VZVirtualMachineInstance.swift @@ -29,9 +29,16 @@ public final class VZVirtualMachineInstance: Sendable { public typealias Agent = Vminitd /// The machine's attached storage. + /// + /// Where a hotplug provider is installed it holds the registry, so a disk + /// taken while the machine runs is registered with the ones it booted + /// with. The machine's own copy answers only where there is no provider. private let _storage: Mutex public var storage: MachineAttachments { - _storage.withLock { $0 } + if let hotplugProvider { + return hotplugProvider.storage + } + return _storage.withLock { $0 } } /// The underlying Virtualization framework virtual machine. @@ -42,7 +49,10 @@ public final class VZVirtualMachineInstance: Sendable { /// Mutate the storage registry. public func withStorage(_ body: (inout sending MachineAttachments) throws -> sending T) rethrows -> T { - try _storage.withLock(body) + if let hotplugProvider { + return try hotplugProvider.withStorage(body) + } + return try _storage.withLock(body) } /// Serialize VM operations with the instance lock. @@ -146,7 +156,7 @@ public final class VZVirtualMachineInstance: Sendable { self.hotplugProvider = VZHotplugProvider( vm: self.vm, queue: self.queue, - initialMounts: mountAttachments, + initialStorage: mountAttachments, logger: logger ) } From 98293e191a4e74a9679342334efd4e590da93854 Mon Sep 17 00:00:00 2001 From: Aaron Paterson Date: Wed, 12 Aug 2026 07:13:26 +0000 Subject: [PATCH 23/25] Export directory shares to a running machine The virtiofs device takes a replacement share while the machine runs, so hotplugVirtioFS exports a directory by setting a share carrying it alongside the ones already exported, and releaseVirtioFS withdraws what no container references anymore. --- .../Containerization/VZHotplugProvider.swift | 158 ++++++++++++++- Sources/Integration/PodTests.swift | 184 +++++++++++++++++- Sources/Integration/Suite.swift | 7 +- 3 files changed, 334 insertions(+), 15 deletions(-) diff --git a/Sources/Containerization/VZHotplugProvider.swift b/Sources/Containerization/VZHotplugProvider.swift index d6db5c1a..6bc3a35d 100644 --- a/Sources/Containerization/VZHotplugProvider.swift +++ b/Sources/Containerization/VZHotplugProvider.swift @@ -23,7 +23,7 @@ import Logging import Synchronization @preconcurrency import Virtualization -/// Attaches block devices to a running virtual machine. +/// Attaches block devices and directory shares to a running virtual machine. /// /// Virtualization's storage devices are fixed once a machine boots, but its USB /// controller takes devices while it runs, and mass storage is one of the @@ -31,6 +31,11 @@ import Synchronization /// SCSI disk, so it is named from a separate run of letters to the virtio-blk /// devices the machine booted with. /// https://developer.apple.com/documentation/virtualization/vzusbcontroller +/// +/// Directory shares ride the machine's one virtiofs device, whose share is +/// replaceable while the machine runs, so a directory is exported by setting a +/// share that carries it alongside the ones already exported. +/// https://developer.apple.com/documentation/virtualization/vzvirtiofilesystemdevice/share @available(macOS 15.0, *) final class VZHotplugProvider: HotplugProvider, @unchecked Sendable { /// A disk attached to the running machine, kept so it can be detached. @@ -50,6 +55,10 @@ final class VZHotplugProvider: HotplugProvider, @unchecked Sendable { private let allocator: any AddressAllocator private let _storage: Mutex private let _records: Mutex<[String: [HotplugRecord]]> + /// The virtiofs tags exported for each container while the machine runs. + /// The machine keeps what it booted with; what these record leaves with + /// the containers that asked for it. + private let _shareRecords: Mutex<[String: Set]> private let logger: Logger? init( @@ -63,6 +72,7 @@ final class VZHotplugProvider: HotplugProvider, @unchecked Sendable { self.allocator = Character.blockDeviceTagAllocator() self._storage = Mutex(initialStorage) self._records = Mutex([:]) + self._shareRecords = Mutex([:]) self.logger = logger } @@ -180,17 +190,147 @@ final class VZHotplugProvider: HotplugProvider, @unchecked Sendable { _ = _storage.withLock { $0.containers.removeValue(forKey: id) } } - /// Virtualization shares directories through devices fixed at boot, so a - /// share cannot be added to a machine that is already running. + /// Export directories to the running machine. + /// + /// The machine's virtiofs device takes a replacement share while it runs, + /// so a directory is exported by setting a share that carries it alongside + /// the ones already exported. A directory the machine already exports is + /// taken as it is, the way a booted share is. + /// https://developer.apple.com/documentation/virtualization/vzvirtiofilesystemdevice/share func hotplugVirtioFS(_ mounts: [Mount], id: String) async throws { - guard !mounts.isEmpty else { return } - throw ContainerizationError( - .unsupported, - message: "a directory share cannot be added to a running machine" - ) + let virtiofs = mounts.filter { + if case .virtiofs = $0.runtimeOptions { return true } + return false + } + guard !virtiofs.isEmpty else { return } + + // Group by tag: several mounts of one source directory share an export. + var additions: [String: DirectoryExport] = [:] + for mount in virtiofs { + guard FileManager.default.fileExists(atPath: mount.source) else { + throw ContainerizationError(.notFound, message: "directory \(mount.source) does not exist") + } + let tag = try hashFilePath(path: mount.source) + if additions[tag] == nil { + additions[tag] = DirectoryExport( + path: mount.source, + readOnly: mount.options.contains("ro") + ) + } + } + + try await mergeShare(additions) + + _shareRecords.withLock { $0[id, default: []].formUnion(additions.keys) } + } + + /// Withdraw the directories exported for a container while the machine + /// runs, keeping every directory another container still references and + /// everything the machine booted with. + func releaseVirtioFS(id: String) async throws { + let dropped: Set = _shareRecords.withLock { records in + records.removeValue(forKey: id) ?? [] + } + guard !dropped.isEmpty else { return } + + let heldByRecords: Set = _shareRecords.withLock { Set($0.values.flatMap { $0 }) } + let heldByRegistry: Set = _storage.withLock { storage in + var held = Set( + storage.containers.filter { $0.key != id } + .values.flatMap { $0.all } + .filter { $0.type == "virtiofs" } + .map { $0.source }) + held.formUnion( + storage.volumes.values + .filter { $0.type == "virtiofs" } + .map { $0.source }) + return held + } + let removable = dropped.subtracting(heldByRecords).subtracting(heldByRegistry) + guard !removable.isEmpty else { return } + + do { + try await withdrawShare(removable) + } catch { + logger?.error( + "failed to withdraw directory shares from the running machine", + metadata: ["id": "\(id)", "error": "\(error)"] + ) + } + } + + /// What a directory export is made from, carried onto the machine's queue + /// where the Virtualization objects for it are built. + private struct DirectoryExport: Sendable { + let path: String + let readOnly: Bool + } + + /// The machine's virtiofs device, which every share rides. The device is + /// a Virtualization object, so this is callable only on the machine's own + /// queue. + private static func shareDevice(of vm: VZVirtualMachine) -> VZVirtioFileSystemDevice? { + vm.directorySharingDevices + .compactMap { $0 as? VZVirtioFileSystemDevice } + .first { $0.tag == "virtiofs" } + } + + /// Set a share on the machine's virtiofs device carrying the current + /// directories plus `additions`, leaving an already-exported tag as it is. + /// The device is a Virtualization object, touched only inside a block + /// dispatched to the machine's own queue. + private func mergeShare(_ additions: [String: DirectoryExport]) async throws { + nonisolated(unsafe) let vm = self.vm + try await withCheckedThrowingContinuation { (continuation: CheckedContinuation) in + queue.async { + guard let device = Self.shareDevice(of: vm) else { + continuation.resume( + throwing: ContainerizationError( + .unsupported, + message: "the machine has no directory sharing device to export through" + )) + return + } + let current = (device.share as? VZMultipleDirectoryShare)?.directories ?? [:] + var updated = current + for (tag, export) in additions where updated[tag] == nil { + updated[tag] = VZSharedDirectory( + url: URL(fileURLWithPath: export.path), + readOnly: export.readOnly + ) + } + if updated.count != current.count { + device.share = VZMultipleDirectoryShare(directories: updated) + } + continuation.resume() + } + } } - func releaseVirtioFS(id: String) async throws {} + /// Set a share on the machine's virtiofs device carrying the current + /// directories minus `tags`. The device is a Virtualization object, + /// touched only inside a block dispatched to the machine's own queue. + private func withdrawShare(_ tags: Set) async throws { + nonisolated(unsafe) let vm = self.vm + try await withCheckedThrowingContinuation { (continuation: CheckedContinuation) in + queue.async { + guard let device = Self.shareDevice(of: vm) else { + continuation.resume( + throwing: ContainerizationError( + .unsupported, + message: "the machine has no directory sharing device to withdraw from" + )) + return + } + let current = (device.share as? VZMultipleDirectoryShare)?.directories ?? [:] + let updated = current.filter { !tags.contains($0.key) } + if updated.count != current.count { + device.share = VZMultipleDirectoryShare(directories: updated) + } + continuation.resume() + } + } + } } #endif diff --git a/Sources/Integration/PodTests.swift b/Sources/Integration/PodTests.swift index 77a887ed..f6335a6c 100644 --- a/Sources/Integration/PodTests.swift +++ b/Sources/Integration/PodTests.swift @@ -2374,8 +2374,9 @@ extension IntegrationSuite { } /// Hotplug a container with a virtiofs (directory-share) rootfs into a - /// running pod VM, plus an additional virtiofs file-mount. CH-only: VZ has - /// no runtime hotplug. + /// running pod VM, plus an additional virtiofs file-mount. CH-only: a + /// virtiofs rootfs rides its own device, which cloud-hypervisor alone + /// adds to a running machine. func testPodHotplugVirtiofsRootfs() async throws { let id = "test-pod-hotplug-virtiofs-rootfs" let bs = try await bootstrap(id) @@ -2480,8 +2481,6 @@ extension IntegrationSuite { } } - /// A container in a pod given a writable layer writes into it, and the - /// image it was built from is left as it is for the pod's others. /// Add a container with a writable layer to a pod whose machine is /// already running: the overlay assembles from the two disks attached /// while it runs, and writes land in the layer. @@ -2545,6 +2544,183 @@ extension IntegrationSuite { } } + /// Add a container with a directory-share mount to a pod whose machine is + /// already running. Both backends export a directory to a running machine, + /// so both are held to it. + func testPodHotplugVirtiofsShare() async throws { + let id = "test-pod-hotplug-virtiofs-share" + let bs = try await bootstrap(id) + + let pod = try LinuxPod(id, vmm: bs.vmm) { config in + config.cpus = 4 + config.memoryInBytes = 1024.mib() + config.bootLog = bs.bootLog + } + + try await pod.addContainer("seed", rootfs: try cloneRootfs(bs.rootfs, testID: id, containerID: "seed")) { config in + config.process.arguments = ["/bin/sleep", "infinity"] + } + + try await pod.create() + + let content = "hello from a directory exported while running" + let hostDir = FileManager.default.uniqueTemporaryDirectory(create: true) + try content.write(to: hostDir.appendingPathComponent("hot.txt"), atomically: true, encoding: .utf8) + + let buffer = BufferWriter() + try await pod.addContainer("hot", rootfs: try cloneRootfs(bs.rootfs, testID: id, containerID: "hot")) { config in + config.process.arguments = ["/bin/cat", "/shared/hot.txt"] + config.mounts.append(.share(source: hostDir.absolutePath(), destination: "/shared")) + config.process.stdout = buffer + } + + do { + try await pod.startContainer("hot") + let status = try await pod.waitContainer("hot") + + try await pod.stopContainer("hot") + try await pod.stop() + + guard status.exitCode == 0 else { + throw IntegrationError.assert(msg: "hot container status \(status) != 0") + } + guard String(data: buffer.data, encoding: .utf8) == content else { + throw IntegrationError.assert( + msg: "expected '\(content)', got '\(String(data: buffer.data, encoding: .utf8) ?? "nil")'") + } + } catch { + try? await pod.stop() + throw error + } + } + + /// Add a container sharing the host directory a booted container already + /// mounts. The machine exports a directory once, so the added container + /// takes the export the machine booted with. + func testPodHotplugVirtiofsSameShare() async throws { + let id = "test-pod-hotplug-virtiofs-same-share" + let bs = try await bootstrap(id) + + let pod = try LinuxPod(id, vmm: bs.vmm) { config in + config.cpus = 4 + config.memoryInBytes = 1024.mib() + config.bootLog = bs.bootLog + } + + let content = "hello from a directory the machine booted with" + let hostDir = FileManager.default.uniqueTemporaryDirectory(create: true) + try content.write(to: hostDir.appendingPathComponent("seed.txt"), atomically: true, encoding: .utf8) + + try await pod.addContainer("seed", rootfs: try cloneRootfs(bs.rootfs, testID: id, containerID: "seed")) { config in + config.process.arguments = ["/bin/sleep", "infinity"] + config.mounts.append(.share(source: hostDir.absolutePath(), destination: "/shared")) + } + + try await pod.create() + + let buffer = BufferWriter() + try await pod.addContainer("hot", rootfs: try cloneRootfs(bs.rootfs, testID: id, containerID: "hot")) { config in + config.process.arguments = ["/bin/cat", "/shared/seed.txt"] + config.mounts.append(.share(source: hostDir.absolutePath(), destination: "/shared")) + config.process.stdout = buffer + } + + do { + try await pod.startContainer("hot") + let status = try await pod.waitContainer("hot") + + try await pod.stopContainer("hot") + try await pod.stop() + + guard status.exitCode == 0 else { + throw IntegrationError.assert(msg: "hot container status \(status) != 0") + } + guard String(data: buffer.data, encoding: .utf8) == content else { + throw IntegrationError.assert( + msg: "expected '\(content)', got '\(String(data: buffer.data, encoding: .utf8) ?? "nil")'") + } + } catch { + try? await pod.stop() + throw error + } + } + + /// A booted container keeps its directory share through another + /// container's whole hotplug lifecycle: a second container takes the + /// same export, stops (releasing its shares), the booted container + /// still reads the directory, and a third container takes the export + /// again. + func testPodHotplugVirtiofsShareLifecycle() async throws { + let id = "test-pod-hotplug-virtiofs-share-lifecycle" + let bs = try await bootstrap(id) + + let pod = try LinuxPod(id, vmm: bs.vmm) { config in + config.cpus = 4 + config.memoryInBytes = 1024.mib() + config.bootLog = bs.bootLog + } + + let content = "hello from a directory shared across lifecycles" + let hostDir = FileManager.default.uniqueTemporaryDirectory(create: true) + try content.write(to: hostDir.appendingPathComponent("data.txt"), atomically: true, encoding: .utf8) + + try await pod.addContainer("seed", rootfs: try cloneRootfs(bs.rootfs, testID: id, containerID: "seed")) { config in + config.process.arguments = ["/bin/sleep", "infinity"] + config.mounts.append(.share(source: hostDir.absolutePath(), destination: "/shared")) + } + + try await pod.create() + try await pod.startContainer("seed") + + do { + for hot in ["hot1", "hot2"] { + let buffer = BufferWriter() + try await pod.addContainer(hot, rootfs: try cloneRootfs(bs.rootfs, testID: id, containerID: hot)) { config in + config.process.arguments = ["/bin/cat", "/shared/data.txt"] + config.mounts.append(.share(source: hostDir.absolutePath(), destination: "/shared")) + config.process.stdout = buffer + } + try await pod.startContainer(hot) + let status = try await pod.waitContainer(hot) + try await pod.stopContainer(hot) + guard status.exitCode == 0 else { + throw IntegrationError.assert(msg: "\(hot) container status \(status) != 0") + } + guard String(data: buffer.data, encoding: .utf8) == content else { + throw IntegrationError.assert( + msg: "\(hot): expected '\(content)', got '\(String(data: buffer.data, encoding: .utf8) ?? "nil")'") + } + + // The stopped container's shares were released; the booted + // container's export stays. + let execBuffer = BufferWriter() + let exec = try await pod.execInContainer("seed", processID: "check-\(hot)") { config in + config.arguments = ["/bin/cat", "/shared/data.txt"] + config.stdout = execBuffer + } + try await exec.start() + let execStatus = try await exec.wait() + try await exec.delete() + guard execStatus.exitCode == 0 else { + throw IntegrationError.assert(msg: "seed read after \(hot) stop: status \(execStatus) != 0") + } + guard String(data: execBuffer.data, encoding: .utf8) == content else { + throw IntegrationError.assert( + msg: "seed after \(hot) stop: expected '\(content)', got '\(String(data: execBuffer.data, encoding: .utf8) ?? "nil")'") + } + } + + try await pod.killContainer("seed", signal: .kill) + try await pod.waitContainer("seed") + try await pod.stop() + } catch { + try? await pod.stop() + throw error + } + } + + /// A container in a pod given a writable layer writes into it, and the + /// image it was built from is left as it is for the pod's others. func testPodWritableLayer() async throws { let id = "test-pod-writable-layer" diff --git a/Sources/Integration/Suite.swift b/Sources/Integration/Suite.swift index 959140ea..4580b677 100644 --- a/Sources/Integration/Suite.swift +++ b/Sources/Integration/Suite.swift @@ -564,6 +564,9 @@ struct IntegrationSuite: AsyncParsableCommand { Test("pod writable layer", testPodWritableLayer), Test("pod hotplug block rootfs", testPodHotplugBlockRootfs), Test("pod hotplug writable layer", testPodHotplugWritableLayer), + Test("pod hotplug virtiofs share", testPodHotplugVirtiofsShare), + Test("pod hotplug virtiofs same share", testPodHotplugVirtiofsSameShare), + Test("pod hotplug virtiofs share lifecycle", testPodHotplugVirtiofsShareLifecycle), Test("pod container PID namespace isolation", testPodContainerPIDNamespaceIsolation), Test("pod container independent resource limits", testPodContainerIndependentResourceLimits), Test("pod shared PID namespace", testPodSharedPIDNamespace), @@ -650,8 +653,8 @@ struct IntegrationSuite: AsyncParsableCommand { ] + macOS26Tests() let tests: [Test] = crossPlatformTests + macOSOnlyTests #else - // A directory share is fixed at boot on VZ, so only the backend that - // adds one to a running machine is held to it. + // A virtiofs rootfs rides its own device, which cloud-hypervisor alone + // adds to a running machine. let linuxOnlyTests: [Test] = [ Test("pod hotplug virtiofs rootfs", testPodHotplugVirtiofsRootfs) ] From bf3177ad1405401db6259b6945e8bb3db501e445 Mon Sep 17 00:00:00 2001 From: Aaron Paterson Date: Tue, 11 Aug 2026 13:05:45 +0000 Subject: [PATCH 24/25] Restart a stopped pod member by re-mounting its rootfs startContainer guarded the created state, so a member that ran and then stopped could only be brought back by rebooting the whole machine, taking every sibling with it. Bringing it back in place is cheap: a pod member's block devices are attached when the machine boots and stay attached while it runs, since they cannot be detached from a running machine, so a stopped member still has its storage; only its guest rootfs mount and its process were torn down. Follow the lifecycle the runtime specification names, the one removeContainer already cites: stopping keeps the member's place, removal gives it up. stopContainer tears down the process and unmounts the rootfs but leaves the block devices attached and the mount registry entry intact; removeContainer detaches the devices and clears the entry. startContainer accepts a stopped member and re-mounts its rootfs before starting a fresh process on it, reusing the block, image, and shares unchanged. Re-mounting leaves the member created, exactly as a freshly placed one, so a failure before the process starts is cleaned up by the same stopContainer path. --- Sources/Containerization/LinuxPod.swift | 102 ++++++++++++++++++------ 1 file changed, 76 insertions(+), 26 deletions(-) diff --git a/Sources/Containerization/LinuxPod.swift b/Sources/Containerization/LinuxPod.swift index f4ee7e00..536a8182 100644 --- a/Sources/Containerization/LinuxPod.swift +++ b/Sources/Containerization/LinuxPod.swift @@ -396,6 +396,37 @@ public final class LinuxPod: Sendable { return spec } + /// Re-mount a stopped member's rootfs so a fresh process can run on it. The + /// member kept its block devices across the stop, stopContainer leaves them + /// attached and the storage entry intact, so only the guest rootfs + /// mount was torn down and re-establishing it is all a restart needs. The + /// block, its image, and its shares are unchanged. + private static func remountRootfs( + containerID: String, + container: PodContainer, + vm: any VirtualMachineInstance, + agent: any VirtualMachineAgent + ) async throws { + guard let attached = vm.storage.containers[containerID] else { + throw ContainerizationError( + .invalidState, + message: "container \(containerID) has no registered rootfs to re-mount" + ) + } + if let writableAttachment = attached.writableLayer { + try await agent.mountOverlayRootfs( + containerID: containerID, + rootfsAttachment: attached.rootfs, + writableAttachment: writableAttachment, + rootfsPath: Self.guestRootfsPath(containerID) + ) + } else { + var mount = attached.rootfs.to + mount.destination = Self.guestRootfsPath(containerID) + try await agent.mount(mount) + } + } + static func guestRootfsPath(_ containerID: String) -> String { "/run/container/\(containerID)/rootfs" } @@ -990,15 +1021,33 @@ extension LinuxPod { ) } - guard container.state == .created else { + guard container.state == .created || container.state == .stopped else { throw ContainerizationError( .invalidState, - message: "container \(containerID) must be in created state to start" + message: "container \(containerID) must be in created or stopped state to start" ) } let agent = try await createdState.vm.dialAgent() do { + // A member that ran and then stopped kept its place: its block + // devices are still attached and registered, but stopContainer + // unmounted its rootfs. Re-mount it the way boot did before + // starting a fresh process on it, the runtime's "start a new task + // on the container, reusing its rootfs" for a pod member. + // Re-mounting leaves the member created, exactly as a freshly + // placed one, so a failure before the process starts is cleaned + // up by the same stopContainer path. + if container.state == .stopped { + try await Self.remountRootfs( + containerID: containerID, + container: container, + vm: createdState.vm, + agent: agent + ) + state.containers[containerID]?.state = .created + } + var spec = self.generateRuntimeSpec(containerID: containerID, config: container.config, rootfs: container.rootfs, writableLayer: container.writableLayer) // We don't need the rootfs, nor do OCI runtimes want it included. // Also filter out file mount holding directories - we mount those separately under /run. @@ -1135,34 +1184,33 @@ extension LinuxPod { return } - // Handle containers that were hotplugged but never started - if container.state == .created { - // Release the hotplug device and virtiofs shares - try? await createdState.vm.releaseHotplug(id: containerID) - try? await createdState.vm.releaseVirtioFS(id: containerID) - - container.state = .stopped - state.containers[containerID] = container - return - } - - guard container.state == .started, let process = container.process else { + guard container.state == .created || container.state == .started else { throw ContainerizationError( .invalidState, - message: "container \(containerID) must be in started state to stop" + message: "container \(containerID) must be in created or started state to stop" ) } do { // Check if the vm is even still running if createdState.vm.state == .stopped { + container.process = nil container.state = .stopped state.containers[containerID] = container return } - try await process.kill(.kill) - try await process.wait(timeoutInSeconds: 3) + // Stopping keeps the member's place: the process is torn down and + // the rootfs unmounted, but the block devices stay attached and + // the storage entry is kept, so the member can be started + // again by re-mounting. Detaching the devices is removeContainer's + // job, the separate act the runtime specification names for giving + // the place up. + // https://github.com/kubernetes/cri-api/blob/master/pkg/apis/runtime/v1/api.proto + if let process = container.process { + try await process.kill(.kill) + try await process.wait(timeoutInSeconds: 3) + } let hasWritableLayer = container.writableLayer != nil try await createdState.vm.withAgent { agent in @@ -1181,21 +1229,15 @@ extension LinuxPod { } } - // Release the hotplug device and virtiofs shares so they can be reused by new containers - try await createdState.vm.releaseHotplug(id: containerID) - try await createdState.vm.releaseVirtioFS(id: containerID) - // Clean up the process resources - try await process.delete() + if let process = container.process { + try await process.delete() + } container.process = nil container.state = .stopped state.containers[containerID] = container } catch { - // Try to release the hotplug device and virtiofs shares even on error - try? await createdState.vm.releaseHotplug(id: containerID) - try? await createdState.vm.releaseVirtioFS(id: containerID) - container.state = .errored container.process = nil state.containers[containerID] = container @@ -1224,6 +1266,14 @@ extension LinuxPod { } switch container.state { case .registered, .stopped, .errored: + // Giving the place up detaches the member's block devices and + // clears its storage entry, the resources stopContainer + // keeps so a stopped member can start again. A member removed + // before the machine booted never took a device. + if case .created(let createdState) = state.phase { + try? await createdState.vm.releaseHotplug(id: containerID) + try? await createdState.vm.releaseVirtioFS(id: containerID) + } state.containers[containerID] = nil default: throw ContainerizationError( From 2daf68f0e837db71c02dd79a8428ea0bbcc23f7d Mon Sep 17 00:00:00 2001 From: Aaron Paterson Date: Tue, 11 Aug 2026 13:05:45 +0000 Subject: [PATCH 25/25] Cover restarting a stopped pod member with an integration test Two sleeping members share a pod; one is stopped while the other holds the machine up, then started again and exec'd into to prove it came back on its reused rootfs. The member keeps its block devices across the stop, so the restart is a plain re-mount of the guest rootfs. --- Sources/Integration/PodTests.swift | 74 ++++++++++++++++++++++++++++++ Sources/Integration/Suite.swift | 1 + 2 files changed, 75 insertions(+) diff --git a/Sources/Integration/PodTests.swift b/Sources/Integration/PodTests.swift index f6335a6c..f20b8493 100644 --- a/Sources/Integration/PodTests.swift +++ b/Sources/Integration/PodTests.swift @@ -96,6 +96,80 @@ extension IntegrationSuite { } } + func testPodRestartStoppedContainer() async throws { + let id = "test-pod-restart-stopped-container" + + let bs = try await bootstrap(id) + let pod = try LinuxPod(id, vmm: bs.vmm) { config in + config.cpus = 4 + config.memoryInBytes = 1024.mib() + config.bootLog = bs.bootLog + } + + // The holder keeps the machine up while the other member is stopped and + // restarted, the way a sibling holds a live pod up. + try await pod.addContainer("holder", rootfs: try cloneRootfs(bs.rootfs, testID: id, containerID: "holder")) { config in + config.process.arguments = ["/bin/sleep", "600"] + } + try await pod.addContainer("restarted", rootfs: try cloneRootfs(bs.rootfs, testID: id, containerID: "restarted")) { config in + config.process.arguments = ["/bin/sleep", "600"] + } + + try await pod.create() + try await pod.startContainer("holder") + try await pod.startContainer("restarted") + + // Prove the restart reuses the member's rootfs rather than attaching a + // fresh one: a file written before the stop is read back after. + let marker = try await pod.execInContainer("restarted", processID: "mark") { config in + config.arguments = ["/bin/sh", "-c", "echo kept > /marker"] + } + try await marker.start() + let markerStatus = try await marker.wait() + try await marker.delete() + guard markerStatus.exitCode == 0 else { + throw IntegrationError.assert(msg: "marker write status \(markerStatus) != 0") + } + + // Stop one member. Its process is torn down and its guest rootfs + // unmounted, while its block devices stay attached and its mount + // registry entry stays: stopping keeps the member's place, removal is + // what gives it up. The machine stays up because the holder is still + // running. + try await pod.stopContainer("restarted") + + // Start it again into the running machine. This is the resume path: the + // rootfs is re-attached and the mounts re-registered, reusing the shares + // that persisted, and a fresh process is started on the reused rootfs. + try await pod.startContainer("restarted") + + // The restarted member is alive, and the marker written before the stop + // is still there: the same rootfs came back. + let buffer = BufferWriter() + let exec = try await pod.execInContainer("restarted", processID: "exec1") { config in + config.arguments = ["/bin/cat", "/marker"] + config.stdout = buffer + } + try await exec.start() + let status = try await exec.wait() + try await exec.delete() + + try await pod.killContainer("restarted", signal: .kill) + try await pod.waitContainer("restarted") + try await pod.killContainer("holder", signal: .kill) + try await pod.waitContainer("holder") + try await pod.stop() + + guard status.exitCode == 0 else { + throw IntegrationError.assert(msg: "exec after restart status \(status) != 0") + } + + guard String(data: buffer.data, encoding: .utf8) == "kept\n" else { + throw IntegrationError.assert( + msg: "marker after restart should have read 'kept' != '\(String(data: buffer.data, encoding: .utf8) ?? "")'") + } + } + func testPodSharedSwap() async throws { let id = "test-pod-shared-swap" diff --git a/Sources/Integration/Suite.swift b/Sources/Integration/Suite.swift index 4580b677..07c8e9cd 100644 --- a/Sources/Integration/Suite.swift +++ b/Sources/Integration/Suite.swift @@ -548,6 +548,7 @@ struct IntegrationSuite: AsyncParsableCommand { // Pods Test("pod single container", testPodSingleContainer), Test("pod multiple containers", testPodMultipleContainers), + Test("pod restart stopped container", testPodRestartStoppedContainer), Test("pod container output", testPodContainerOutput), Test("pod concurrent containers", testPodConcurrentContainers), Test("pod exec in container", testPodExecInContainer),