From f10eaca613f6eef28add5d69f4fbf573ca6ed156 Mon Sep 17 00:00:00 2001 From: Aaron Paterson Date: Sat, 1 Aug 2026 18:48:34 +0000 Subject: [PATCH 01/16] Resolve test directory paths before reaping per-test files The reaper that runs at a concurrency of one deletes everything in the test directory except the unpacked rootfs it means to preserve. It held that rootfs path as FileManager reports it, under /var, and compared it against the entries of contentsOfDirectory, which reports them under /private/var, so the preserved path never matched any entry and the rootfs was deleted along with the per-test files. The unpack coordinator still held it as unpacked, so the next test opened a rootfs that was no longer there and failed with a missing file error. Resolve both sides with resolvingSymlinksInPathWithPrivate, which exists for this difference between the two views of the same directory. --- Sources/Integration/Suite.swift | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/Sources/Integration/Suite.swift b/Sources/Integration/Suite.swift index 9dae8435..e43cf372 100644 --- a/Sources/Integration/Suite.swift +++ b/Sources/Integration/Suite.swift @@ -259,12 +259,16 @@ struct IntegrationSuite: AsyncParsableCommand { // a ~2GB rootfs and a ~512MB initfs, so without reaping the dev // container fills its CoW layer in ~10 tests. if self.maxConcurrency == 1 { - let preserve = fsPath.absolutePath() + // contentsOfDirectory reports paths under /private, while testDir is + // built from FileManager's /var view of the same directory, so both + // sides are resolved before comparing. + let preserve = fsPath.resolvingSymlinksInPathWithPrivate().absolutePath() if let entries = try? FileManager.default.contentsOfDirectory( at: Self.testDir, includingPropertiesForKeys: nil ) { - for url in entries where url.absolutePath() != preserve { + for url in entries + where url.resolvingSymlinksInPathWithPrivate().absolutePath() != preserve { try? FileManager.default.removeItem(at: url) } } From 6499f7dfa7fff64c903642db690094d7e6c64ec8 Mon Sep 17 00:00:00 2001 From: Aaron Paterson Date: Sun, 2 Aug 2026 00:51:10 +0000 Subject: [PATCH 02/16] Say how to run the tests that macOS skips Tests guarded for Linux are compiled out on macOS, so `make test` reports success without having run them, and nothing says so. The target that does run them is not mentioned anywhere outside the makefile. --- README.md | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/README.md b/README.md index f658b08d..372a872a 100644 --- a/README.md +++ b/README.md @@ -130,6 +130,16 @@ After building, run basic and integration tests: make test integration ``` +Tests that only apply to Linux are compiled out on macOS, so `make test` passes +without running them. Run those with: + +```bash +make linux-test +``` + +which runs the same tests inside the Linux dev container, as the Linux build +workflow does. + A kernel is required to run integration tests. If you do not have a kernel locally, a default kernel can be fetched using the `make fetch-default-kernel` target. From bf4d7dcf8bbe64c745e9ef344a8c173d2a42a3af Mon Sep 17 00:00:00 2001 From: Aaron Paterson Date: Sun, 2 Aug 2026 01:20:29 +0000 Subject: [PATCH 03/16] Test against the kernel release the shipped tool installs The default kernel the tests fetch had drifted from the one users run. container installs kata 3.28.0 and its 6.18.15 kernel, while these tests fetched 3.17.0, so the suite exercised a guest with a different feature set to the one it is meant to represent. Nested runtimes are the visible case: 3.17.0 was built without nf_tables, so a docker daemon inside a container fails there and works on what ships. Kata moved from xz to zstd between those releases, so the archive is no longer named for its compression and tar is left to recognise it rather than being told, which also holds if the format changes again. --- Makefile | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Makefile b/Makefile index c79d6995..3d01b93a 100644 --- a/Makefile +++ b/Makefile @@ -64,7 +64,7 @@ LIBARCHIVE_UPSTREAM_REPO := https://github.com/libarchive/libarchive LIBARCHIVE_UPSTREAM_VERSION := v3.7.7 LIBARCHIVE_LOCAL_DIR := workdir/libarchive -KATA_BINARY_PACKAGE := https://github.com/kata-containers/kata-containers/releases/download/3.17.0/kata-static-3.17.0-arm64.tar.xz +KATA_BINARY_PACKAGE := https://github.com/kata-containers/kata-containers/releases/download/3.28.0/kata-static-3.28.0-arm64.tar.zst CLOUD_HYPERVISOR_URL := https://github.com/cloud-hypervisor/cloud-hypervisor/releases/download/v52.0/cloud-hypervisor-static-aarch64 # SHA256 of the v52.0 aarch64 static binary (verified locally from the # upstream release artifact). Bump alongside CLOUD_HYPERVISOR_URL. @@ -410,11 +410,11 @@ integration: .PHONY: fetch-default-kernel fetch-default-kernel: @mkdir -p .local/ bin/ -ifeq (,$(wildcard .local/kata.tar.gz)) - @curl -SsL -o .local/kata.tar.gz ${KATA_BINARY_PACKAGE} +ifeq (,$(wildcard .local/kata.tar)) + @curl -SsL -o .local/kata.tar ${KATA_BINARY_PACKAGE} endif ifeq (,$(wildcard .local/vmlinux-$(KERNEL_ARCH))) - @tar -zxf .local/kata.tar.gz -C .local/ --strip-components=1 + @tar -xf .local/kata.tar -C .local/ --strip-components=1 @cp -L .local/opt/kata/share/kata-containers/vmlinux.container .local/vmlinux-$(KERNEL_ARCH) endif ifeq (,$(wildcard bin/vmlinux-$(KERNEL_ARCH))) From 21f1f16eaaacc051bbc5be6566814cb606de5698 Mon Sep 17 00:00:00 2001 From: Aaron Paterson Date: Wed, 12 Aug 2026 05:25:08 +0000 Subject: [PATCH 04/16] Run each integration test in its own scratch directory A test's rootfs clones, init block, and writable layers land in a directory the runner creates for it and deletes when it finishes, so a test holds disk only while it runs at any concurrency. The run-shared directory holds the unpacked images every bootstrap clones from, and is removed when the run ends. --- Sources/Integration/PodTests.swift | 7 +++-- Sources/Integration/Suite.swift | 43 +++++++++++++----------------- 2 files changed, 21 insertions(+), 29 deletions(-) diff --git a/Sources/Integration/PodTests.swift b/Sources/Integration/PodTests.swift index ae1caec8..7dff20fa 100644 --- a/Sources/Integration/PodTests.swift +++ b/Sources/Integration/PodTests.swift @@ -2228,10 +2228,9 @@ extension IntegrationSuite { /// (directory-share) rootfs. Assumes a single-layer image (the alpine /// image used by the suite) so no OCI whiteout processing is required. /// - /// The extracted dir lives under `Self.testDir`; do NOT `defer`-remove it - /// here — virtiofsd shares it for the whole test. It is swept by - /// `bootstrap`'s `maxConcurrency == 1` reaper on the next test and by the - /// suite-end `removeItem(at: Self.testDir)`. + /// The extracted dir lives under the test's scratch directory; do NOT + /// `defer`-remove it here — virtiofsd shares it for the whole test. The + /// runner removes the scratch directory when the test finishes. private func unpackRootfsDirectory(_ image: Containerization.Image, testID: String) async throws -> Containerization.Mount { let dir = Self.testDir.appending(component: "\(testID)-rootfs-dir") try? FileManager.default.removeItem(at: dir) diff --git a/Sources/Integration/Suite.swift b/Sources/Integration/Suite.swift index e43cf372..84580dba 100644 --- a/Sources/Integration/Suite.swift +++ b/Sources/Integration/Suite.swift @@ -135,7 +135,19 @@ struct IntegrationSuite: AsyncParsableCommand { FileManager.default.uniqueTemporaryDirectory(create: true) }() + /// The scratch directory of the test running on this task. The runner + /// creates one per test and deletes it when the test finishes, so a + /// test's clones and layers hold disk only while it runs. + @TaskLocal static var currentTestDir: URL? + static var testDir: URL { + currentTestDir ?? _testDir + } + + /// The run-shared directory holding unpacked images, which every test's + /// bootstrap clones from. It lives for the whole run and is removed at + /// the end. + static var imageCacheDir: URL { _testDir } @@ -233,7 +245,7 @@ struct IntegrationSuite: AsyncParsableCommand { let platform = Platform(arch: "arm64", os: "linux", variant: "v8") // Unpack to shared location with coordination to prevent concurrent unpacks - let fsPath = Self.testDir.appending(component: image.digest) + let fsPath = Self.imageCacheDir.appending(component: image.digest) let fs = try await Self.unpackCoordinator.unpack(key: fsPath.absolutePath()) { do { let unpacker = EXT4Unpacker(capacityInBytes: 2.gib()) @@ -251,29 +263,6 @@ struct IntegrationSuite: AsyncParsableCommand { } } - // Reap any per-test artifacts left over from prior tests. With - // `--max-concurrency 1` (linux-integration default) this runs after - // the previous test has fully completed, so it's race-free; on - // macOS where tests can run in parallel we just keep all files — - // disk usage isn't a concern there. Each per-test bootstrap clones - // a ~2GB rootfs and a ~512MB initfs, so without reaping the dev - // container fills its CoW layer in ~10 tests. - if self.maxConcurrency == 1 { - // contentsOfDirectory reports paths under /private, while testDir is - // built from FileManager's /var view of the same directory, so both - // sides are resolved before comparing. - let preserve = fsPath.resolvingSymlinksInPathWithPrivate().absolutePath() - if let entries = try? FileManager.default.contentsOfDirectory( - at: Self.testDir, - includingPropertiesForKeys: nil - ) { - for url in entries - where url.resolvingSymlinksInPathWithPrivate().absolutePath() != preserve { - try? FileManager.default.removeItem(at: url) - } - } - } - // Clone to test-specific path let clPath = Self.testDir.appending(component: "\(testID).ext4").absolutePath() try? FileManager.default.removeItem(atPath: clPath) @@ -658,11 +647,14 @@ struct IntegrationSuite: AsyncParsableCommand { for _ in 0.. Date: Wed, 12 Aug 2026 15:00:44 +0000 Subject: [PATCH 05/16] Give the machine's storage the shape its containers have A container's storage is its rootfs, an optional writable layer, and its mounts; a machine's is its containers' plus the volumes they share. One generic shape describes both the Mount values a machine is configured with and the AttachedFilesystem values it reports once attached, so converting between the two is a map over the structure and the roles cannot drift between configuration and attachment. Device addresses are allocated walking the same sorted order the devices are created in, on both backends. Consumers read roles instead of list positions: the spec builders take a container's mounts without prefix arithmetic, pod volumes mount by name, and a cloud-hypervisor volume's virtiofsd is held by a machine-lifetime reference alongside its containers' reference counts. --- .../Containerization/CHHotplugProvider.swift | 79 ++++++----- .../CHVirtualMachineInstance.swift | 127 ++++++++++-------- .../CHVirtualMachineManager.swift | 2 +- .../Containerization/ContainerStorage.swift | 107 +++++++++++++++ .../Containerization/HotplugProvider.swift | 2 +- Sources/Containerization/LinuxContainer.swift | 51 +++---- Sources/Containerization/LinuxPod.swift | 98 +++++++------- .../Containerization/VMConfiguration.swift | 10 +- .../VZVirtualMachineInstance.swift | 100 +++++++------- .../VZVirtualMachineManager.swift | 2 +- .../VirtualMachineInstance.swift | 5 +- 11 files changed, 351 insertions(+), 232 deletions(-) create mode 100644 Sources/Containerization/ContainerStorage.swift diff --git a/Sources/Containerization/CHHotplugProvider.swift b/Sources/Containerization/CHHotplugProvider.swift index 870fb1ff..380cba76 100644 --- a/Sources/Containerization/CHHotplugProvider.swift +++ b/Sources/Containerization/CHHotplugProvider.swift @@ -27,8 +27,8 @@ import Synchronization /// /// Handles both block (`vm.add-disk`) and virtiofs (`vm.add-fs`, with one /// `virtiofsd` per unique source-hash tag) hotplug, plus the matching -/// `vm.remove-device` teardown. Owns the per-VM mount registry so -/// `CHVirtualMachineInstance.mounts` can forward to it. +/// `vm.remove-device` teardown. Owns the machine's storage so +/// `CHVirtualMachineInstance.storage` can forward to it. final class CHHotplugProvider: HotplugProvider { struct HotplugRecord: Sendable { let chDeviceId: String @@ -50,7 +50,7 @@ final class CHHotplugProvider: HotplugProvider { private let workDir: URL private let virtiofsdBinaryOverride: URL? private let allocator: any AddressAllocator - private let _mounts: Mutex<[String: [AttachedFilesystem]]> + private let _storage: Mutex private let _records: Mutex<[String: [HotplugRecord]]> private let _tags: Mutex<[String: VirtiofsdTagState]> /// Serializes per-tag virtiofsd spawn so a concurrent hotplug for the @@ -65,14 +65,14 @@ final class CHHotplugProvider: HotplugProvider { workDir: URL, virtiofsdBinary: URL?, allocator: any AddressAllocator, - initialMounts: [String: [AttachedFilesystem]], + initialStorage: MachineAttachments, logger: Logger? ) { self.client = client self.workDir = workDir self.virtiofsdBinaryOverride = virtiofsdBinary self.allocator = allocator - self._mounts = Mutex(initialMounts) + self._storage = Mutex(initialStorage) self._records = Mutex([:]) self._tags = Mutex([:]) self.spawnLock = AsyncLock() @@ -81,14 +81,14 @@ final class CHHotplugProvider: HotplugProvider { // MARK: - Read accessors - var mounts: [String: [AttachedFilesystem]] { - _mounts.withLock { $0 } + var storage: MachineAttachments { + _storage.withLock { $0 } } - func withMountRegistry( - _ body: (inout sending [String: [AttachedFilesystem]]) throws -> sending T + func withStorage( + _ body: (inout sending MachineAttachments) throws -> sending T ) rethrows -> T { - try _mounts.withLock(body) + try _storage.withLock(body) } // MARK: - HotplugProvider conformance @@ -151,12 +151,13 @@ final class CHHotplugProvider: HotplugProvider { } func registerMounts(id: String, rootfs: AttachedFilesystem, additionalMounts: [Mount]) throws { - var attached: [AttachedFilesystem] = [rootfs] + var mounts: [AttachedFilesystem] = [] for mount in additionalMounts { - attached.append(try AttachedFilesystem(mount: mount, allocator: allocator)) + mounts.append(try AttachedFilesystem(mount: mount, allocator: allocator)) } - _mounts.withLock { - $0[id, default: []].append(contentsOf: attached) + let container = ContainerAttachments(rootfs: rootfs, mounts: mounts) + _storage.withLock { + $0.containers[id] = container } } @@ -190,19 +191,10 @@ final class CHHotplugProvider: HotplugProvider { } } - // Drop block-derived AttachedFilesystem entries for `id`. Block entries - // are the ones whose source was rewritten to "/dev/vd" by - // `hotplug(_:)` (or by AttachedFilesystem(mount:allocator:) for an - // additionalMount of type virtio-blk). - _mounts.withLock { state in - guard var perID = state[id] else { return } - perID.removeAll { $0.source.hasPrefix("/dev/vd") } - if perID.isEmpty { - state.removeValue(forKey: id) - } else { - state[id] = perID - } - } + // The container's devices are gone, so the container leaves the + // registry with them; its shares are released separately and their + // processes reference-counted through `_tags`. + _ = _storage.withLock { $0.containers.removeValue(forKey: id) } } func hotplugVirtioFS(_ mounts: [Mount], id: String) async throws { @@ -226,7 +218,7 @@ final class CHHotplugProvider: HotplugProvider { let chDeviceId = try await ensureVirtiofsDevice(tag: tag, source: source, readonly: readonly) // Record once per tag for this container. The AttachedFilesystem // entries for these mounts are written by registerMounts (the sole - // _mounts writer), so we do NOT touch _mounts here. + // registry writer), so we do NOT touch the storage here. _records.withLock { $0[id, default: []].append(HotplugRecord(chDeviceId: chDeviceId, kind: .virtiofs(tag: tag))) } @@ -338,16 +330,17 @@ final class CHHotplugProvider: HotplugProvider { try? FileManager.default.removeItem(at: socket) } - // Drop virtiofs AttachedFilesystem entries for `id`. AttachedFilesystem - // sets `type = mount.type` which for a `.virtiofs` mount is "virtiofs". - _mounts.withLock { state in - guard var perID = state[id] else { return } - perID.removeAll { $0.type == "virtiofs" } - if perID.isEmpty { - state.removeValue(forKey: id) - } else { - state[id] = perID + // Drop the container's virtiofs entries. A container whose rootfs is + // itself a share leaves the registry whole; one that keeps block + // devices keeps its entry with the share entries dropped. + _storage.withLock { state in + guard var container = state.containers[id] else { return } + if container.rootfs.type == "virtiofs" { + state.containers.removeValue(forKey: id) + return } + container.mounts.removeAll { $0.type == "virtiofs" } + state.containers[id] = container } } @@ -358,15 +351,21 @@ final class CHHotplugProvider: HotplugProvider { /// is the user-supplied `FsConfig.id` (which `vm.remove-device` keys on). /// `ownerIds` are the container ids that count toward this tag's refcount; /// each gets a `HotplugRecord` so `releaseVirtioFS(id:)` walks them - /// uniformly. + /// uniformly. `machineHeld` adds one reference nothing releases, for a + /// share the machine itself owns (a volume). func recordBootTimeVirtiofs( tag: String, process: VirtiofsdProcess, chDeviceId: String, - ownerIds: [String] + ownerIds: [String], + machineHeld: Bool ) { _tags.withLock { - $0[tag] = VirtiofsdTagState(process: process, refcount: ownerIds.count, chDeviceId: chDeviceId) + $0[tag] = VirtiofsdTagState( + process: process, + refcount: ownerIds.count + (machineHeld ? 1 : 0), + chDeviceId: chDeviceId + ) } _records.withLock { records in for id in ownerIds { diff --git a/Sources/Containerization/CHVirtualMachineInstance.swift b/Sources/Containerization/CHVirtualMachineInstance.swift index 122f592a..e363d5ca 100644 --- a/Sources/Containerization/CHVirtualMachineInstance.swift +++ b/Sources/Containerization/CHVirtualMachineInstance.swift @@ -46,7 +46,7 @@ public final class CHVirtualMachineInstance: Sendable { public struct Configuration: Sendable { public var cpus: Int public var memoryInBytes: UInt64 - public var mountsByID: [String: [Mount]] + public var storage: MachineMounts public var interfaces: [any Interface] public var kernel: Kernel? public var initialFilesystem: Mount? @@ -56,7 +56,7 @@ public final class CHVirtualMachineInstance: Sendable { public init() { self.cpus = 4 self.memoryInBytes = 1024 * 1024 * 1024 - self.mountsByID = [:] + self.storage = MachineMounts() self.interfaces = [] } } @@ -65,8 +65,8 @@ public final class CHVirtualMachineInstance: Sendable { /// `start()`'s `VmConfig.disks` ordering matches the allocator letters. struct BootDisk: Sendable { let mount: Mount - let containerId: String? // nil for rootfs - let letter: Character + let chId: String + let readonly: Bool } // MARK: - State @@ -76,8 +76,8 @@ public final class CHVirtualMachineInstance: Sendable { _state.withLock { $0 } } - public var mounts: [String: [AttachedFilesystem]] { - hotplug.mounts + public var storage: MachineAttachments { + hotplug.storage } /// Cloud-hypervisor exposes one virtio-fs device per source-hash tag, so @@ -167,9 +167,9 @@ public final class CHVirtualMachineInstance: Sendable { ) self.workDir = workDir - // 2. Block allocator + boot inventory. Walks rootfs first, then - // mountsByID sorted by container id, allocating disk letters in - // that order. The same allocator is later handed to the hotplug + // 2. Block allocator + boot inventory. Walks rootfs first, then the + // machine's storage in its device order, allocating disk letters + // as it goes. The same allocator is later handed to the hotplug // provider so runtime add-disk picks up where boot wiring left off. let allocator = Character.blockDeviceTagAllocator() let inventory = try config.bootInventory(allocator: allocator) @@ -201,14 +201,14 @@ public final class CHVirtualMachineInstance: Sendable { logger: logger ?? Logger(label: "CloudHypervisor.Client") ) - // 5. Hotplug provider — owns the mount registry, seeded with the + // 5. Hotplug provider — owns the machine's storage, seeded with the // boot inventory so registerMounts can append to it. self.hotplug = CHHotplugProvider( client: self.client, workDir: workDir, virtiofsdBinary: virtiofsdBinary, allocator: allocator, - initialMounts: inventory.attachments, + initialStorage: inventory.attachments, logger: logger ) @@ -222,11 +222,11 @@ public final class CHVirtualMachineInstance: Sendable { self._stdioPool = Mutex([:]) } - /// Mutate the mount registry. Forwards to the hotplug provider, which + /// Mutate the storage registry. Forwards to the hotplug provider, which /// owns the registry. Kept on the instance for parity with the macOS - /// path's `withMountRegistry` API. - func withMountRegistry(_ body: (inout sending [String: [AttachedFilesystem]]) throws -> sending T) rethrows -> T { - try hotplug.withMountRegistry(body) + /// path's `withStorage` API. + func withStorage(_ body: (inout sending MachineAttachments) throws -> sending T) rethrows -> T { + try hotplug.withStorage(body) } } @@ -510,34 +510,46 @@ extension CHVirtualMachineInstance { throw ContainerizationError(.invalidArgument, message: "initialFilesystem is required for cloud-hypervisor backend") } - // Disks: rootfs forced read-only at the device level; container disks + // Disks: rootfs forced read-only at the device level; other disks // honor their `ro` option through chDiskConfig. var disks: [CloudHypervisor.DiskConfig] = [] for bd in bootDisks { - let chId = bd.containerId.map { "blk-\($0)-\(bd.letter)" } ?? "rootfs" - if var disk = bd.mount.chDiskConfig(id: chId) { - if bd.containerId == nil { + if var disk = bd.mount.chDiskConfig(id: bd.chId) { + if bd.readonly { disk.readonly = true } disks.append(disk) } } - // Virtiofs: group all .virtiofs mounts in mountsByID by source-hash - // tag, spawn one virtiofsd per tag, build matching FsConfigs. - var byTag: [String: (mounts: [Mount], owners: [String])] = [:] - for cid in config.mountsByID.keys.sorted() { - guard let mounts = config.mountsByID[cid] else { continue } - for mount in mounts { - guard case .virtiofs = mount.runtimeOptions else { continue } - let tag = try hashFilePath(path: mount.source) - var entry = byTag[tag] ?? (mounts: [], owners: []) - entry.mounts.append(mount) - if !entry.owners.contains(cid) { - entry.owners.append(cid) + // Virtiofs: group all virtiofs mounts in the machine's storage by + // source-hash tag, spawn one virtiofsd per tag, build matching + // FsConfigs. A volume's tag is the machine's own, held for its + // lifetime. + var byTag: [String: (mounts: [Mount], owners: [String], machineHeld: Bool)] = [:] + func groupVirtiofs(_ mount: Mount, owner: String?) throws { + guard case .virtiofs = mount.runtimeOptions else { return } + let tag = try hashFilePath(path: mount.source) + var entry = byTag[tag] ?? (mounts: [], owners: [], machineHeld: false) + entry.mounts.append(mount) + if let owner { + if !entry.owners.contains(owner) { + entry.owners.append(owner) } - byTag[tag] = entry + } else { + entry.machineHeld = true } + byTag[tag] = entry + } + for cid in config.storage.containers.keys.sorted() { + guard let container = config.storage.containers[cid] else { continue } + for mount in container.all { + try groupVirtiofs(mount, owner: cid) + } + } + for name in config.storage.volumes.keys.sorted() { + guard let mount = config.storage.volumes[name] else { continue } + try groupVirtiofs(mount, owner: nil) } var fsConfigs: [CloudHypervisor.FsConfig] = [] @@ -569,7 +581,8 @@ extension CHVirtualMachineInstance { tag: tag, process: process, chDeviceId: chDeviceId, - ownerIds: entry.owners + ownerIds: entry.owners, + machineHeld: entry.machineHeld ) fsConfigs.append( @@ -714,40 +727,46 @@ extension CHVirtualMachineInstance { // MARK: - Boot inventory extension CHVirtualMachineInstance.Configuration { - /// Walks boot-time mounts in deterministic order (rootfs first, then - /// `mountsByID` sorted by container id, then each container's mounts in - /// input order), allocating disk letters for virtio-blk mounts and seeding - /// the per-container `AttachedFilesystem` registry. + /// Walks boot-time storage in the machine's device order (rootfs first, + /// then containers sorted by id, each in role order, then volumes sorted + /// by name), allocating disk letters for virtio-blk mounts and seeding + /// the machine's `AttachedFilesystem` registry. /// /// The allocator is shared with the runtime hotplug provider, so block /// hotplug picks up at the next free letter after boot. func bootInventory( allocator: any AddressAllocator - ) throws -> (attachments: [String: [AttachedFilesystem]], bootDisks: [CHVirtualMachineInstance.BootDisk]) { + ) throws -> (attachments: MachineAttachments, bootDisks: [CHVirtualMachineInstance.BootDisk]) { var bootDisks: [CHVirtualMachineInstance.BootDisk] = [] - var attachments: [String: [AttachedFilesystem]] = [:] - // Rootfs is not part of mountsByID. If it's a block device, it claims - // the first letter (vda) so the kernel cmdline `root=/dev/vda` is right. + // The rootfs is the machine's own, not part of its containers' + // storage. If it's a block device, it claims the first letter (vda) + // so the kernel cmdline `root=/dev/vda` is right. if let rootfs = self.initialFilesystem, rootfs.isBlock { - let letter = try allocator.allocate() - bootDisks.append(.init(mount: rootfs, containerId: nil, letter: letter)) + _ = try allocator.allocate() + bootDisks.append(.init(mount: rootfs, chId: "rootfs", readonly: true)) } - for cid in self.mountsByID.keys.sorted() { - guard let mounts = self.mountsByID[cid] else { continue } - var perContainer: [AttachedFilesystem] = [] - for mount in mounts { - let attached = try AttachedFilesystem(mount: mount, allocator: allocator) - if mount.isBlock, let letter = attached.source.last { - bootDisks.append(.init(mount: mount, containerId: cid, letter: letter)) - } - perContainer.append(attached) + func attach(_ mount: Mount, chId: (Character) -> String) throws -> AttachedFilesystem { + let attached = try AttachedFilesystem(mount: mount, allocator: allocator) + if mount.isBlock, let letter = attached.source.last { + bootDisks.append(.init(mount: mount, chId: chId(letter), readonly: false)) } - attachments[cid] = perContainer + return attached + } + + var containers: [String: ContainerAttachments] = [:] + for cid in self.storage.containers.keys.sorted() { + guard let container = self.storage.containers[cid] else { continue } + containers[cid] = try container.map { try attach($0, chId: { "blk-\(cid)-\($0)" }) } + } + var volumes: [String: AttachedFilesystem] = [:] + for name in self.storage.volumes.keys.sorted() { + guard let mount = self.storage.volumes[name] else { continue } + volumes[name] = try attach(mount, chId: { "vol-\(name)-\($0)" }) } - return (attachments, bootDisks) + return (MachineAttachments(containers: containers, volumes: volumes), bootDisks) } } #endif diff --git a/Sources/Containerization/CHVirtualMachineManager.swift b/Sources/Containerization/CHVirtualMachineManager.swift index ba20b5d1..c444620d 100644 --- a/Sources/Containerization/CHVirtualMachineManager.swift +++ b/Sources/Containerization/CHVirtualMachineManager.swift @@ -99,7 +99,7 @@ public struct CHVirtualMachineManager: VirtualMachineManager { instanceConfig.cpus = vmConfig.cpus instanceConfig.memoryInBytes = vmConfig.memoryInBytes instanceConfig.interfaces = vmConfig.interfaces - instanceConfig.mountsByID = vmConfig.mountsByID + instanceConfig.storage = vmConfig.storage instanceConfig.bootLog = vmConfig.bootLog instanceConfig.extensions = vmConfig.extensions instanceConfig.kernel = kernel diff --git a/Sources/Containerization/ContainerStorage.swift b/Sources/Containerization/ContainerStorage.swift new file mode 100644 index 00000000..4d8de2ed --- /dev/null +++ b/Sources/Containerization/ContainerStorage.swift @@ -0,0 +1,107 @@ +//===----------------------------------------------------------------------===// +// Copyright © 2026 Apple Inc. and the Containerization project authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//===----------------------------------------------------------------------===// + +/// A container's storage, by role. +/// +/// The same shape describes storage at both stages of its life: as the +/// `Mount` values a container is configured with, and as the +/// `AttachedFilesystem` values the machine reports once they are attached. +/// Converting between the two is a `map` over the structure, so the roles +/// cannot drift apart between configuration and attachment. +public struct ContainerStorage: Sendable { + /// The container's root filesystem. + public var rootfs: Value + + /// The writable layer mounted over the rootfs as the upper side of an + /// overlay, when the container has one. + public var writableLayer: Value? + + /// The container's remaining mounts, in configuration order. + public var mounts: [Value] + + public init(rootfs: Value, writableLayer: Value? = nil, mounts: [Value] = []) { + self.rootfs = rootfs + self.writableLayer = writableLayer + self.mounts = mounts + } + + /// Every value in the structure: the rootfs, the writable layer when + /// present, then the mounts, in that order. + public var all: [Value] { + var values = [rootfs] + if let writableLayer { + values.append(writableLayer) + } + values.append(contentsOf: mounts) + return values + } + + /// The storage with `transform` applied to every value, each keeping + /// its role. + public func map(_ transform: (Value) throws -> U) rethrows -> ContainerStorage { + ContainerStorage( + rootfs: try transform(rootfs), + writableLayer: try writableLayer.map(transform), + mounts: try mounts.map(transform) + ) + } +} + +/// A container's storage as configured, before its machine exists. +public typealias ContainerMounts = ContainerStorage + +/// A container's storage as attached to its machine. +public typealias ContainerAttachments = ContainerStorage + +/// The storage a machine carries: each container's, and the resources its +/// containers share. +public struct MachineStorage: Sendable { + /// Each container's storage, by container ID. + public var containers: [String: ContainerStorage] + + /// Volumes shared by the machine's containers, by volume name. + public var volumes: [String: Value] + + public init(containers: [String: ContainerStorage] = [:], volumes: [String: Value] = [:]) { + self.containers = containers + self.volumes = volumes + } + + /// Every value the machine carries: containers sorted by ID, each in + /// role order, then volumes sorted by name. Walks that allocate device + /// addresses and walks that create the devices use this one order, so an + /// address always names the device it was handed out for. + public var ordered: [Value] { + containers.keys.sorted().flatMap { containers[$0]?.all ?? [] } + + volumes.keys.sorted().compactMap { volumes[$0] } + } + + /// The storage with `transform` applied to every value, each keeping its + /// place. The transform runs in no particular order, so an allocating + /// conversion walks the fields sorted itself. + public func map(_ transform: (Value) throws -> U) rethrows -> MachineStorage { + MachineStorage( + containers: try containers.mapValues { try $0.map(transform) }, + volumes: try volumes.mapValues(transform) + ) + } +} + +/// A machine's storage as configured, before it exists. +public typealias MachineMounts = MachineStorage + +/// A machine's storage as attached. +public typealias MachineAttachments = MachineStorage diff --git a/Sources/Containerization/HotplugProvider.swift b/Sources/Containerization/HotplugProvider.swift index f535ce3f..0835077f 100644 --- a/Sources/Containerization/HotplugProvider.swift +++ b/Sources/Containerization/HotplugProvider.swift @@ -26,7 +26,7 @@ public protocol HotplugProvider: Sendable { /// - Returns: The attached filesystem with the device path in the guest func hotplug(_ block: Mount, id: String) async throws -> AttachedFilesystem - /// Register mounts for a container in the VM's mount registry. + /// Register mounts for a container in the machine's storage. /// - Parameters: /// - id: The container ID /// - rootfs: The rootfs attachment from hotplug diff --git a/Sources/Containerization/LinuxContainer.swift b/Sources/Containerization/LinuxContainer.swift index 34964fcd..55bbb39a 100644 --- a/Sources/Containerization/LinuxContainer.swift +++ b/Sources/Containerization/LinuxContainer.swift @@ -537,24 +537,14 @@ extension LinuxContainer { } private func mountRootfs( - attachments: [AttachedFilesystem], + attached: ContainerAttachments, rootfsPath: String, agent: VirtualMachineAgent ) async throws { - guard let rootfsAttachment = attachments.first else { - throw ContainerizationError(.notFound, message: "rootfs mount not found") - } + let rootfsAttachment = attached.rootfs - if self.writableLayer != nil { + if let writableAttachment = attached.writableLayer { // Set up overlayfs with image as lower layer and writable layer as upper. - guard attachments.count >= 2 else { - throw ContainerizationError( - .notFound, - message: "writable layer mount not found" - ) - } - let writableAttachment = attachments[1] - let lowerPath = "/run/container/\(self.id)/lower" let upperMountPath = "/run/container/\(self.id)/upper" let upperPath = "/run/container/\(self.id)/upper/diff" @@ -623,17 +613,18 @@ extension LinuxContainer { // This is dumb, but alas. let fileMountContextHolder = Mutex(fileMountContext) - // Build the list of mounts to attach to the VM. - var containerMounts = [modifiedRootfs] + fileMountContext.transformedMounts - if let writableLayer = self.writableLayer { - containerMounts.insert(writableLayer, at: 1) - } + // Build the container's storage to attach to the VM. + let containerStorage = ContainerMounts( + rootfs: modifiedRootfs, + writableLayer: self.writableLayer, + mounts: fileMountContext.transformedMounts + ) let vmConfig = VMConfiguration( cpus: vmCpus, memoryInBytes: vmMemory, interfaces: self.interfaces, - mountsByID: [self.id: containerMounts], + storage: MachineMounts(containers: [self.id: containerStorage]), bootLog: self.config.bootLog, nestedVirtualization: self.config.virtualization ) @@ -643,7 +634,7 @@ extension LinuxContainer { do { try await vm.start() - let mountsForAgent = containerMounts + let storageForAgent = containerStorage try await vm.withAgent { agent in try await agent.standardSetup() @@ -656,7 +647,7 @@ extension LinuxContainer { // with zero shares), but the cloud-hypervisor backend // only spawns virtiofsd when shares exist; mounting an // unbacked tag fails with EINVAL. - let hasVirtiofsMount = mountsForAgent.contains { mount in + let hasVirtiofsMount = storageForAgent.all.contains { mount in if case .virtiofs = mount.runtimeOptions { return true } return false } @@ -671,7 +662,7 @@ extension LinuxContainer { // gets populated. if vm.virtiofsLayout == .perTag { try await agent.mkdir(path: "/run/virtiofs", all: true, perms: 0o755) - let virtiofsAttachments = (vm.mounts[self.id] ?? []).filter { $0.type == "virtiofs" } + let virtiofsAttachments = (vm.storage.containers[self.id]?.all ?? []).filter { $0.type == "virtiofs" } let uniqueTags = Set(virtiofsAttachments.map(\.source)) for tag in uniqueTags { let dest = "/run/virtiofs/\(tag)" @@ -695,18 +686,17 @@ extension LinuxContainer { } } - guard let attachments = vm.mounts[self.id] else { + guard let attached = vm.storage.containers[self.id] else { throw ContainerizationError(.notFound, message: "rootfs mount not found") } let rootfsPath = Self.guestRootfsPath(self.id) - try await self.mountRootfs(attachments: attachments, rootfsPath: rootfsPath, agent: agent) + try await self.mountRootfs(attached: attached, rootfsPath: rootfsPath, agent: agent) // Mount file mount holding directories under /run. if fileMountContext.hasFileMounts { - let containerMounts = vm.mounts[self.id] ?? [] var ctx = fileMountContextHolder.withLock { $0 } try await ctx.mountHoldingDirectories( - vmMounts: containerMounts, + vmMounts: attached.mounts, agent: agent ) fileMountContextHolder.withLock { $0 = ctx } @@ -764,15 +754,12 @@ extension LinuxContainer { let agent = try await createdState.vm.dialAgent() do { var spec = self.generateRuntimeSpec() - // We don't need the rootfs (or writable layer), nor do OCI runtimes want it included. - // Also filter out file mount holding directories. We'll mount those separately under /run. + // Filter out file mount holding directories. We'll mount those separately under /run. // Transform virtiofs mounts to bind mounts from /run/virtiofs/{tag} - let containerMounts = createdState.vm.mounts[self.id] ?? [] + let containerMounts = createdState.vm.storage.containers[self.id]?.mounts ?? [] let holdingTags = createdState.fileMountContext.holdingDirectoryTags - // Drop rootfs, and writable layer if present. - let mountsToSkip = self.writableLayer != nil ? 2 : 1 var mounts: [ContainerizationOCI.Mount] = - containerMounts.dropFirst(mountsToSkip) + containerMounts .filter { !holdingTags.contains($0.source) } .map { attached -> ContainerizationOCI.Mount in if attached.type == "virtiofs" { diff --git a/Sources/Containerization/LinuxPod.swift b/Sources/Containerization/LinuxPod.swift index 6275a4d4..fa4d1785 100644 --- a/Sources/Containerization/LinuxPod.swift +++ b/Sources/Containerization/LinuxPod.swift @@ -446,24 +446,28 @@ extension LinuxPod { // the container's bind mounts from /run/virtiofs/ fail // with ENOENT. // - // Derive the tags from the additional mounts directly rather - // than from vm.mounts[id], so this is independent of the - // rootfs (which may be virtiofs or virtio-blk) and of mount - // ordering. The rootfs is mounted at /run/container//rootfs - // and is never consumed from /run/virtiofs. + // Derive the new tags from the additional mounts being added; + // the machine's storage names what is already shared. let newVirtiofsTags = try virtioFSMounts.map { try hashFilePath(path: $0.source) } if !newVirtiofsTags.isEmpty { try await agent.mkdir(path: "/run/virtiofs", all: true, perms: 0o755) if vm.virtiofsLayout == .perTag { // Tags already mounted in the guest at boot or by a - // prior hotplug (i.e. present on another container). - let alreadyMounted = Set( - vm.mounts - .filter { $0.key != id } - .values.flatMap { $0 } - .filter { $0.type == "virtiofs" } - .map { $0.source } - ) + // prior hotplug (i.e. present on another container or + // a machine volume). + let alreadyMounted: Set = { + var mounted = Set( + vm.storage.containers + .filter { $0.key != id } + .values.flatMap { $0.all } + .filter { $0.type == "virtiofs" } + .map { $0.source }) + mounted.formUnion( + vm.storage.volumes.values + .filter { $0.type == "virtiofs" } + .map { $0.source }) + return mounted + }() var seen: Set = [] for tag in newVirtiofsTags where !alreadyMounted.contains(tag) && seen.insert(tag).inserted { @@ -493,7 +497,7 @@ extension LinuxPod { } if fileMountContext.hasFileMounts { - let containerMounts = vm.mounts[id] ?? [] + let containerMounts = vm.storage.containers[id]?.mounts ?? [] try await updatedFileMountContext.mountHoldingDirectories( vmMounts: containerMounts, agent: agent @@ -557,20 +561,23 @@ extension LinuxPod { try await self.state.withLock { state in try state.phase.validateForCreate() - // Build mountsByID for all containers. + // Build the machine's storage from its containers. // Strip "ro" from rootfs options - we handle readonly via the OCI spec's // root.readonly field and remount in vmexec after setup is complete. // Use transformedMounts from fileMountContext (file mounts become directory shares). - var mountsByID: [String: [Mount]] = [:] + var machineStorage = MachineMounts() for (id, container) in state.containers { var modifiedRootfs = container.rootfs modifiedRootfs.options.removeAll(where: { $0 == "ro" }) - // Filter out shared mounts — those are handled separately as pod volume bind mounts. + // Shared mounts are handled separately as pod volume bind mounts. let containerMounts = container.fileMountContext.transformedMounts.filter { if case .shared = $0.runtimeOptions { return false } return true } - mountsByID[id] = [modifiedRootfs] + containerMounts + machineStorage.containers[id] = ContainerMounts( + rootfs: modifiedRootfs, + mounts: containerMounts + ) } // Validate pod volume names are unique. @@ -597,9 +604,8 @@ extension LinuxPod { } } } - let podVolumeMounts = self.config.volumes.map { $0.toMount() } - if !podVolumeMounts.isEmpty { - mountsByID[self.id] = podVolumeMounts + for volume in self.config.volumes { + machineStorage.volumes[volume.name] = volume.toMount() } // Capture into an immutable `let` so the value is safely usable @@ -608,18 +614,16 @@ extension LinuxPod { // only attaches a virtiofs device when shares are configured, // so mounting an unbacked /run/virtiofs would fail with EINVAL // on the CH backend. - let hasVirtiofsMount = mountsByID.values.contains { mounts in - mounts.contains { mount in - if case .virtiofs = mount.runtimeOptions { return true } - return false - } + let hasVirtiofsMount = machineStorage.ordered.contains { mount in + if case .virtiofs = mount.runtimeOptions { return true } + return false } var vmConfig = VMConfiguration( cpus: self.config.cpus, memoryInBytes: self.config.memoryInBytes, interfaces: self.config.interfaces, - mountsByID: mountsByID, + storage: machineStorage, bootLog: self.config.bootLog, nestedVirtualization: self.config.virtualization ) @@ -649,19 +653,17 @@ extension LinuxPod { // /run/virtiofs/. See LinuxContainer for the // VZ vs. CH model split. var seenTags: Set = [] - for (_, attached) in vm.mounts { - for entry in attached where entry.type == "virtiofs" { - guard seenTags.insert(entry.source).inserted else { continue } - let dest = "/run/virtiofs/\(entry.source)" - try await agent.mkdir(path: dest, all: true, perms: 0o755) - try await agent.mount( - ContainerizationOCI.Mount( - type: "virtiofs", - source: entry.source, - destination: dest, - options: [] - )) - } + for entry in vm.storage.ordered where entry.type == "virtiofs" { + guard seenTags.insert(entry.source).inserted else { continue } + let dest = "/run/virtiofs/\(entry.source)" + try await agent.mkdir(path: dest, all: true, perms: 0o755) + try await agent.mount( + ContainerizationOCI.Mount( + type: "virtiofs", + source: entry.source, + destination: dest, + options: [] + )) } } else { try await agent.mount( @@ -728,10 +730,10 @@ extension LinuxPod { // Mount all container rootfs for (_, container) in containers { - guard let attachments = vm.mounts[container.id], let rootfsAttachment = attachments.first else { + guard let attached = vm.storage.containers[container.id] else { throw ContainerizationError(.notFound, message: "rootfs mount not found for container \(container.id)") } - var rootfs = rootfsAttachment.to + var rootfs = attached.rootfs.to rootfs.destination = Self.guestRootfsPath(container.id) try await agent.mount(rootfs) } @@ -740,7 +742,7 @@ extension LinuxPod { for (id, container) in containers { if container.fileMountContext.hasFileMounts { var ctx = container.fileMountContext - let containerMounts = vm.mounts[id] ?? [] + let containerMounts = vm.storage.containers[id]?.mounts ?? [] try await ctx.mountHoldingDirectories( vmMounts: containerMounts, agent: agent @@ -750,15 +752,13 @@ extension LinuxPod { } // Mount pod-level volumes. - let podVolumeAttachments = vm.mounts[self.id] ?? [] - for (index, volume) in self.config.volumes.enumerated() { - guard index < podVolumeAttachments.count else { + for volume in self.config.volumes { + guard let attachment = vm.storage.volumes[volume.name] else { throw ContainerizationError( .notFound, message: "attached filesystem not found for pod volume \"\(volume.name)\"" ) } - let attachment = podVolumeAttachments[index] let guestPath = Self.guestVolumePath(volume.name) try await agent.mount( ContainerizationOCI.Mount( @@ -864,10 +864,10 @@ extension LinuxPod { // We don't need the rootfs, nor do OCI runtimes want it included. // Also filter out file mount holding directories - we mount those separately under /run. // Transform virtiofs mounts to bind mounts from /run/virtiofs/{tag} - let containerMounts = createdState.vm.mounts[containerID] ?? [] + let containerMounts = createdState.vm.storage.containers[containerID]?.mounts ?? [] let holdingTags = container.fileMountContext.holdingDirectoryTags var mounts: [ContainerizationOCI.Mount] = - containerMounts.dropFirst() + containerMounts .filter { !holdingTags.contains($0.source) } .map { attached -> ContainerizationOCI.Mount in if attached.type == "virtiofs" { diff --git a/Sources/Containerization/VMConfiguration.swift b/Sources/Containerization/VMConfiguration.swift index 30faebc4..7a5e8e2c 100644 --- a/Sources/Containerization/VMConfiguration.swift +++ b/Sources/Containerization/VMConfiguration.swift @@ -72,9 +72,9 @@ public struct VMConfiguration: Sendable { public var memoryInBytes: UInt64 /// The network interfaces to attach. public var interfaces: [any Interface] - /// Mounts organized by metadata ID (e.g. container ID). - /// Each ID maps to an array of mounts for that workload. - public var mountsByID: [String: [Mount]] + /// The storage the machine carries: each container's mounts by role, + /// and the volumes its containers share. + public var storage: MachineMounts /// Optional destination for serial boot logs. public var bootLog: BootLog? /// Enable nested virtualization support. If the VirtualMachineManager @@ -89,14 +89,14 @@ public struct VMConfiguration: Sendable { cpus: Int = 4, memoryInBytes: UInt64 = 1024 * 1024 * 1024, interfaces: [any Interface] = [], - mountsByID: [String: [Mount]] = [:], + storage: MachineMounts = MachineMounts(), bootLog: BootLog? = nil, nestedVirtualization: Bool = false ) { self.cpus = cpus self.memoryInBytes = memoryInBytes self.interfaces = interfaces - self.mountsByID = mountsByID + self.storage = storage self.bootLog = bootLog self.nestedVirtualization = nestedVirtualization } diff --git a/Sources/Containerization/VZVirtualMachineInstance.swift b/Sources/Containerization/VZVirtualMachineInstance.swift index 160c5026..670ab512 100644 --- a/Sources/Containerization/VZVirtualMachineInstance.swift +++ b/Sources/Containerization/VZVirtualMachineInstance.swift @@ -28,10 +28,10 @@ import Synchronization public final class VZVirtualMachineInstance: Sendable { public typealias Agent = Vminitd - /// Attached mounts on the virtual machine, organized by metadata ID. - private let _mounts: Mutex<[String: [AttachedFilesystem]]> - public var mounts: [String: [AttachedFilesystem]] { - _mounts.withLock { $0 } + /// The machine's attached storage. + private let _storage: Mutex + public var storage: MachineAttachments { + _storage.withLock { $0 } } /// The underlying Virtualization framework virtual machine. @@ -40,9 +40,9 @@ public final class VZVirtualMachineInstance: Sendable { /// The dispatch queue used for VZ operations. public var vmQueue: DispatchQueue { queue } - /// Mutate the mount registry. - public func withMountRegistry(_ body: (inout sending [String: [AttachedFilesystem]]) throws -> sending T) rethrows -> T { - try _mounts.withLock(body) + /// Mutate the storage registry. + public func withStorage(_ body: (inout sending MachineAttachments) throws -> sending T) rethrows -> T { + try _storage.withLock(body) } /// Serialize VM operations with the instance lock. @@ -73,8 +73,9 @@ public final class VZVirtualMachineInstance: Sendable { public var rosetta: Bool /// Toggle nested virtualization support. public var nestedVirtualization: Bool - /// Mount attachments organized by metadata ID. - public var mountsByID: [String: [Mount]] + /// The machine's storage: each container's mounts by role, and the + /// volumes its containers share. + public var storage: MachineMounts /// Network interface attachments. public var interfaces: [any Interface] /// Kernel image. @@ -91,7 +92,7 @@ public final class VZVirtualMachineInstance: Sendable { self.memoryInBytes = 1024.mib() self.rosetta = false self.nestedVirtualization = false - self.mountsByID = [:] + self.storage = MachineMounts() self.interfaces = [] } } @@ -132,7 +133,7 @@ public final class VZVirtualMachineInstance: Sendable { let allocator = Character.blockDeviceTagAllocator() let (mountAttachments, _) = try config.mountAttachments(allocator: allocator) - self._mounts = Mutex(mountAttachments) + self._storage = Mutex(mountAttachments) self.vm = VZVirtualMachine( configuration: try config.toVZ(allocator: allocator), @@ -153,7 +154,7 @@ public protocol VZInstanceExtension: Sendable { _ config: inout VZVirtualMachineConfiguration, allocator: any AddressAllocator, storageDeviceCount: Int, - mountsByID: [String: [Mount]] + storage: MachineMounts ) throws /// Called after the VZVirtualMachine is created but before start. @@ -168,7 +169,7 @@ extension VZInstanceExtension { _ config: inout VZVirtualMachineConfiguration, allocator: any AddressAllocator, storageDeviceCount: Int, - mountsByID: [String: [Mount]] + storage: MachineMounts ) throws {} public func didCreate(_ instance: VZVirtualMachineInstance) throws {} @@ -477,35 +478,33 @@ extension VZVirtualMachineInstance.Configuration { // Track used virtiofs tags to avoid creating duplicate VZ devices. // The same source directory mounted to multiple destinations shares one device. + // The walk is the machine's device order, matching the addresses + // `mountAttachments` hands out walking the same way. var usedVirtioFSTags: Set = [] - for (_, mounts) in self.mountsByID { - for mount in mounts { - if case .virtiofs = mount.runtimeOptions { - let tag = try hashFilePath(path: mount.source) - if usedVirtioFSTags.contains(tag) { - continue - } - usedVirtioFSTags.insert(tag) + for mount in self.storage.ordered { + if case .virtiofs = mount.runtimeOptions { + let tag = try hashFilePath(path: mount.source) + if usedVirtioFSTags.contains(tag) { + continue } - try mount.configure(config: &config) + usedVirtioFSTags.insert(tag) } + try mount.configure(config: &config) } // Create the unified virtiofs device with VZMultipleDirectoryShare // This device hosts all virtiofs shares and supports runtime updates var directories: [String: VZSharedDirectory] = [:] - for (_, mounts) in self.mountsByID { - for mount in mounts { - guard case .virtiofs(_) = mount.runtimeOptions else { continue } - guard FileManager.default.fileExists(atPath: mount.source) else { - throw ContainerizationError(.notFound, message: "directory \(mount.source) does not exist") - } - let name = try hashFilePath(path: mount.source) - directories[name] = VZSharedDirectory( - url: URL(fileURLWithPath: mount.source), - readOnly: mount.options.contains("ro") - ) + for mount in self.storage.ordered { + guard case .virtiofs(_) = mount.runtimeOptions else { continue } + guard FileManager.default.fileExists(atPath: mount.source) else { + throw ContainerizationError(.notFound, message: "directory \(mount.source) does not exist") } + let name = try hashFilePath(path: mount.source) + directories[name] = VZSharedDirectory( + url: URL(fileURLWithPath: mount.source), + readOnly: mount.options.contains("ro") + ) } let multiShare = VZMultipleDirectoryShare(directories: directories) let virtiofsDevice = VZVirtioFileSystemDeviceConfiguration(tag: "virtiofs") @@ -527,7 +526,7 @@ extension VZVirtualMachineInstance.Configuration { config.platform = platform for ext in self.extensions.compactMap({ $0 as? any VZInstanceExtension }) { - try ext.configureVZ(&config, allocator: allocator, storageDeviceCount: storageDeviceCount, mountsByID: self.mountsByID) + try ext.configureVZ(&config, allocator: allocator, storageDeviceCount: storageDeviceCount, storage: self.storage) } try config.validate() @@ -535,7 +534,7 @@ extension VZVirtualMachineInstance.Configuration { } func mountAttachments(allocator: any AddressAllocator) throws -> ( - attachments: [String: [AttachedFilesystem]], storageDeviceCount: Int + attachments: MachineAttachments, storageDeviceCount: Int ) { var storageDeviceCount = 0 @@ -548,21 +547,28 @@ extension VZVirtualMachineInstance.Configuration { } } - var attachmentsByID: [String: [AttachedFilesystem]] = [:] - - for (id, mounts) in self.mountsByID { - var attachments: [AttachedFilesystem] = [] - for mount in mounts { - let attached = try AttachedFilesystem(mount: mount, allocator: allocator) - attachments.append(attached) - if mount.isBlock { - storageDeviceCount += 1 - } + // The machine's device order: addresses are handed out in the same + // walk `makeConfiguration` creates the devices in. + func attach(_ mount: Mount) throws -> AttachedFilesystem { + let attached = try AttachedFilesystem(mount: mount, allocator: allocator) + if mount.isBlock { + storageDeviceCount += 1 } - attachmentsByID[id] = attachments + return attached + } + + var containers: [String: ContainerAttachments] = [:] + for id in self.storage.containers.keys.sorted() { + guard let container = self.storage.containers[id] else { continue } + containers[id] = try container.map(attach) + } + var volumes: [String: AttachedFilesystem] = [:] + for name in self.storage.volumes.keys.sorted() { + guard let mount = self.storage.volumes[name] else { continue } + volumes[name] = try attach(mount) } - return (attachmentsByID, storageDeviceCount) + return (MachineAttachments(containers: containers, volumes: volumes), storageDeviceCount) } } diff --git a/Sources/Containerization/VZVirtualMachineManager.swift b/Sources/Containerization/VZVirtualMachineManager.swift index 4959bee4..d0888447 100644 --- a/Sources/Containerization/VZVirtualMachineManager.swift +++ b/Sources/Containerization/VZVirtualMachineManager.swift @@ -76,7 +76,7 @@ public struct VZVirtualMachineManager: VirtualMachineManager { instanceConfig.rosetta = self.rosetta instanceConfig.nestedVirtualization = useNestedVirtualization - instanceConfig.mountsByID = vmConfig.mountsByID + instanceConfig.storage = vmConfig.storage instanceConfig.extensions = vmConfig.extensions }) } diff --git a/Sources/Containerization/VirtualMachineInstance.swift b/Sources/Containerization/VirtualMachineInstance.swift index 302e97ae..b9340612 100644 --- a/Sources/Containerization/VirtualMachineInstance.swift +++ b/Sources/Containerization/VirtualMachineInstance.swift @@ -44,7 +44,8 @@ public protocol VirtualMachineInstance: Sendable { // The state of the virtual machine. var state: VirtualMachineInstanceState { get } - var mounts: [String: [AttachedFilesystem]] { get } + /// The machine's attached storage. + var storage: MachineAttachments { get } /// How this VMM exposes virtiofs devices to the guest. Defaults to /// `.unified` (the VZ-shaped behavior); CH overrides to `.perTag`. @@ -73,7 +74,7 @@ public protocol VirtualMachineInstance: Sendable { func hotplug(_ block: Mount, id: String) async throws -> AttachedFilesystem /// Register mounts for a container after hotplug. - /// This is used to add the rootfs and additional mounts to the VM's mount registry + /// This is used to add the rootfs and additional mounts to the machine's storage /// so they can be found when building the container's OCI spec. /// - Parameter id: The container ID /// - Parameter rootfs: The rootfs attachment from hotplug From b3d72088b9deaeda3c02e4dcf377c2831c1473a0 Mon Sep 17 00:00:00 2001 From: Aaron Paterson Date: Sat, 1 Aug 2026 19:06:18 +0000 Subject: [PATCH 06/16] Add guest swap support A container can be given a swap area, so a workload whose memory exceeds its limit is reclaimed rather than killed. The area is a raw file on the host attached as a block device, which is how a guest gets swap it can write to when its own root is read only, and vminitd formats and enables it while mounting the container's filesystems. The area is enabled with discard, and the device backing it is marked non rotational first. virtio block devices are rotational by default, and the kernel only tracks a swap area in clusters when its device is non rotational, so without that the discard flags are accepted and no discard is ever issued. Together they let the sparse file that backs the area release the blocks the guest stops using, rather than holding the area's high water mark for as long as the container runs. --- Sources/CShim/include/swap.h | 36 ++++ Sources/CShim/swap.c | 23 +++ .../Containerization/ContainerManager.swift | 57 +++++- .../Containerization/ContainerStorage.swift | 14 +- Sources/Containerization/LinuxContainer.swift | 40 +++- Sources/ContainerizationOS/Linux/Swap.swift | 176 +++++++++++++++++ Sources/Integration/ContainerTests.swift | 178 ++++++++++++++++++ Sources/Integration/Suite.swift | 26 +++ Sources/cctl/RunCommand.swift | 4 + Tests/ContainerizationTests/SwapTests.swift | 73 +++++++ vminitd/Sources/VminitdCore/Server+GRPC.swift | 15 ++ 11 files changed, 635 insertions(+), 7 deletions(-) create mode 100644 Sources/CShim/include/swap.h create mode 100644 Sources/CShim/swap.c create mode 100644 Sources/ContainerizationOS/Linux/Swap.swift create mode 100644 Tests/ContainerizationTests/SwapTests.swift diff --git a/Sources/CShim/include/swap.h b/Sources/CShim/include/swap.h new file mode 100644 index 00000000..f16000ee --- /dev/null +++ b/Sources/CShim/include/swap.h @@ -0,0 +1,36 @@ +/* + * Copyright © 2026 Apple Inc. and the Containerization project authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef __SWAP_H +#define __SWAP_H + +#if defined(__linux__) + +// swapon(2) lives in , which Swift's glibc modulemap does not +// carry, so it is reachable from Swift only through a wrapper. + +// Discard the whole area when it is enabled, and each page as it is freed. +// These come from the same header, and SWAP_FLAG_DISCARD_PAGES has no UAPI +// header of its own at all, so every consumer declares it; see util-linux +// sys-utils/swapon.c. +#define CZ_SWAP_DISCARD 0x10000 +#define CZ_SWAP_DISCARD_PAGES 0x40000 + +int CZ_swapon(const char *path, int flags); + +#endif + +#endif diff --git a/Sources/CShim/swap.c b/Sources/CShim/swap.c new file mode 100644 index 00000000..2a66461d --- /dev/null +++ b/Sources/CShim/swap.c @@ -0,0 +1,23 @@ +/* + * Copyright © 2026 Apple Inc. and the Containerization project authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifdef __linux__ +#include + +#include "swap.h" + +int CZ_swapon(const char *path, int flags) { return swapon(path, flags); } +#endif diff --git a/Sources/Containerization/ContainerManager.swift b/Sources/Containerization/ContainerManager.swift index 27e9fbe4..a1b082e1 100644 --- a/Sources/Containerization/ContainerManager.swift +++ b/Sources/Containerization/ContainerManager.swift @@ -195,6 +195,9 @@ public struct ContainerManager: Sendable { /// - writableLayerSizeInBytes: Optional size for a separate writable layer. When provided, /// the rootfs becomes read-only and an overlayfs is used with a separate writable layer of this size. /// - readOnly: Whether to mount the root filesystem as read-only. + /// - swapSizeInBytes: Optional size for a swap area. When provided, a raw block + /// device of this size is created and the guest enables it as swap, so a + /// workload exceeding its memory limit reclaims rather than being killed. /// - networking: Whether to create a network interface for this container. Defaults to `true`. /// When `false`, no network resources are allocated and `releaseNetwork`/`delete` remain safe to call. /// - progress: Optional handler for tracking rootfs unpacking progress. @@ -203,6 +206,7 @@ public struct ContainerManager: Sendable { reference: String, rootfsSizeInBytes: UInt64 = 8.gib(), writableLayerSizeInBytes: UInt64? = nil, + swapSizeInBytes: UInt64? = nil, readOnly: Bool = false, networking: Bool = true, progress: ProgressHandler? = nil, @@ -214,6 +218,7 @@ public struct ContainerManager: Sendable { image: image, rootfsSizeInBytes: rootfsSizeInBytes, writableLayerSizeInBytes: writableLayerSizeInBytes, + swapSizeInBytes: swapSizeInBytes, readOnly: readOnly, networking: networking, progress: progress, @@ -229,6 +234,9 @@ public struct ContainerManager: Sendable { /// - writableLayerSizeInBytes: Optional size for a separate writable layer. When provided, /// the rootfs becomes read-only and an overlayfs is used with a separate writable layer of this size. /// - readOnly: Whether to mount the root filesystem as read-only. + /// - swapSizeInBytes: Optional size for a swap area. When provided, a raw block + /// device of this size is created and the guest enables it as swap, so a + /// workload exceeding its memory limit reclaims rather than being killed. /// - networking: Whether to create a network interface for this container. Defaults to `true`. /// When `false`, no network resources are allocated and `releaseNetwork`/`delete` remain safe to call. /// - progress: Optional handler for tracking rootfs unpacking progress. @@ -237,6 +245,7 @@ public struct ContainerManager: Sendable { image: Image, rootfsSizeInBytes: UInt64 = 8.gib(), writableLayerSizeInBytes: UInt64? = nil, + swapSizeInBytes: UInt64? = nil, readOnly: Bool = false, networking: Bool = true, progress: ProgressHandler? = nil, @@ -263,14 +272,25 @@ public struct ContainerManager: Sendable { ) } + // Create the swap device if a size is specified. + var swapLayer: Mount? = nil + if let swapSize = swapSizeInBytes, swapSize > 0 { + swapLayer = try createSwapDevice( + at: path.appendingPathComponent("swap.raw"), + size: swapSize + ) + } + return try await create( id, image: image, rootfs: rootfs, writableLayer: writableLayer, - networking: networking, - configuration: configuration - ) + networking: networking + ) { config in + config.swapLayer = swapLayer + try configuration(&config) + } } /// Returns a new container from the provided image and root filesystem mount. @@ -357,6 +377,37 @@ public struct ContainerManager: Sendable { } } + /// Create a raw block file to back a container's swap area. + /// + /// It carries no filesystem: the agent writes the swap header to the device + /// and enables it. The file is sparse, so it costs the host only the pages + /// the guest has actually swapped out, and gives them back on discard. A + /// swap area held in a file has to be free of holes, since the kernel walks + /// its extents; the guest reaches this one as a block device, which the + /// kernel takes as a single extent without consulting the host's layout. + /// https://github.com/torvalds/linux/blob/master/mm/swapfile.c + private func createSwapDevice(at destination: URL, size: UInt64) throws -> Mount { + let path = destination.absolutePath() + guard !FileManager.default.fileExists(atPath: path) else { + throw ContainerizationError(.exists, message: "swap device already exists at \(path)") + } + guard FileManager.default.createFile(atPath: path, contents: nil) else { + throw ContainerizationError(.internalError, message: "failed to create swap device at \(path)") + } + let handle = try FileHandle(forWritingTo: destination) + defer { try? handle.close() } + try handle.truncate(atOffset: size) + // A swap area holds nothing that outlives the container, so the host + // has no reason to synchronize it to permanent storage. + return .block( + format: Swap.mountType, + source: path, + destination: "", + options: [], + runtimeOptions: ["vzDiskImageSynchronizationMode=none"] + ) + } + private func createEmptyFilesystem(at destination: URL, size: UInt64) throws -> Mount { let path = destination.absolutePath() guard !FileManager.default.fileExists(atPath: path) else { diff --git a/Sources/Containerization/ContainerStorage.swift b/Sources/Containerization/ContainerStorage.swift index 4d8de2ed..49fae6d4 100644 --- a/Sources/Containerization/ContainerStorage.swift +++ b/Sources/Containerization/ContainerStorage.swift @@ -29,22 +29,29 @@ public struct ContainerStorage: Sendable { /// overlay, when the container has one. public var writableLayer: Value? + /// The swap area enabled for the container, when it has one. + public var swap: Value? + /// The container's remaining mounts, in configuration order. public var mounts: [Value] - public init(rootfs: Value, writableLayer: Value? = nil, mounts: [Value] = []) { + public init(rootfs: Value, writableLayer: Value? = nil, swap: Value? = nil, mounts: [Value] = []) { self.rootfs = rootfs self.writableLayer = writableLayer + self.swap = swap self.mounts = mounts } - /// Every value in the structure: the rootfs, the writable layer when - /// present, then the mounts, in that order. + /// Every value in the structure: the rootfs, the writable layer and swap + /// when present, then the mounts, in that order. public var all: [Value] { var values = [rootfs] if let writableLayer { values.append(writableLayer) } + if let swap { + values.append(swap) + } values.append(contentsOf: mounts) return values } @@ -55,6 +62,7 @@ public struct ContainerStorage: Sendable { ContainerStorage( rootfs: try transform(rootfs), writableLayer: try writableLayer.map(transform), + swap: try swap.map(transform), mounts: try mounts.map(transform) ) } diff --git a/Sources/Containerization/LinuxContainer.swift b/Sources/Containerization/LinuxContainer.swift index 55bbb39a..4d5289a7 100644 --- a/Sources/Containerization/LinuxContainer.swift +++ b/Sources/Containerization/LinuxContainer.swift @@ -99,6 +99,16 @@ public final class LinuxContainer: Container, Sendable { /// The total is aligned to a 1 MiB boundary. public var memoryOverhead: UInt64 = 128.mib() + /// Optional swap area for the container, as a block device mount. + /// + /// Swap lets a workload whose resident set exceeds `memoryInBytes` + /// reclaim rather than meet the out of memory killer. The area is a + /// block device rather than a file in the guest, because no filesystem + /// the agent can write to exists when the sandbox starts: its root is + /// mounted read only. The `destination` field is ignored, as the area + /// is enabled rather than mounted. + public var swapLayer: Mount? = nil + public init() {} public init( @@ -536,6 +546,30 @@ extension LinuxContainer { config.interfaces } + /// Enable the container's swap area, if it has one. + /// + /// The device is attached with the container's other block devices, so the + /// agent is told the guest path the VMM allocated it. It is enabled rather + /// than mounted, which is why it travels as a mount of type `swap`. + private func enableSwap( + attached: ContainerAttachments, + agent: VirtualMachineAgent + ) async throws { + guard self.config.swapLayer != nil else { + return + } + guard let swap = attached.swap else { + throw ContainerizationError(.notFound, message: "swap mount not found") + } + try await agent.mount( + ContainerizationOCI.Mount( + type: Swap.mountType, + source: swap.source, + destination: "", + options: swap.options + )) + } + private func mountRootfs( attached: ContainerAttachments, rootfsPath: String, @@ -613,10 +647,13 @@ extension LinuxContainer { // This is dumb, but alas. let fileMountContextHolder = Mutex(fileMountContext) - // Build the container's storage to attach to the VM. + // Build the container's storage to attach to the VM. The swap + // device is attached with the container's other block devices so + // the guest is told the /dev path the VMM allocates it. let containerStorage = ContainerMounts( rootfs: modifiedRootfs, writableLayer: self.writableLayer, + swap: self.config.swapLayer, mounts: fileMountContext.transformedMounts ) @@ -691,6 +728,7 @@ extension LinuxContainer { } let rootfsPath = Self.guestRootfsPath(self.id) try await self.mountRootfs(attached: attached, rootfsPath: rootfsPath, agent: agent) + try await self.enableSwap(attached: attached, agent: agent) // Mount file mount holding directories under /run. if fileMountContext.hasFileMounts { diff --git a/Sources/ContainerizationOS/Linux/Swap.swift b/Sources/ContainerizationOS/Linux/Swap.swift new file mode 100644 index 00000000..82006738 --- /dev/null +++ b/Sources/ContainerizationOS/Linux/Swap.swift @@ -0,0 +1,176 @@ +//===----------------------------------------------------------------------===// +// Copyright © 2026 Apple Inc. and the Containerization project authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//===----------------------------------------------------------------------===// + +import CShim +import ContainerizationError + +#if canImport(FoundationEssentials) +import FoundationEssentials +#else +import Foundation +#endif + +#if canImport(Musl) +import Musl +#elseif canImport(Glibc) +import Glibc +#endif + +/// `Swap` is a utility type that contains static helpers for creating a swap +/// file and enabling it as swap space for the guest. +/// +/// The swap area header layout is the kernel's on-disk format, version 1: +/// a page of header whose last 10 bytes are the magic `SWAPSPACE2`, with the +/// version, the last usable page and the page count at fixed offsets from the +/// start of the second kilobyte. +/// https://docs.kernel.org/admin-guide/mm/concepts.html +/// https://github.com/torvalds/linux/blob/master/include/linux/swap.h +public struct Swap: Sendable { + /// Offset of the version field within the swap header, and of the last + /// usable page field that follows it. + private static let versionOffset = 1024 + private static let lastPageOffset = 1028 + /// The magic that marks the last bytes of the first page of a swap area. + private static let magic = "SWAPSPACE2" + /// The queue attributes the kernel exposes for each block device. + public static let blockPath = "/sys/block" + + /// Mount type that marks a block device as a container's swap area, which + /// the agent enables rather than mounts. It travels in the `type` field of + /// the mount the host sends, so both sides read it from here. + /// + /// Kata gives its guest swap the same shape, the host attaching a raw file + /// as a block device and the agent calling `swapon` on it, and carries the + /// request on an RPC of its own (`AddSwap`). Here it travels as a mount so + /// the device path the VMM allocates reaches the guest the way every other + /// attached device's does. + /// https://github.com/kata-containers/kata-containers/blob/main/src/agent/src/rpc.rs + public static let mountType = "swap" + + #if os(Linux) + /// Format the swap area at `path` and enable it. + /// + /// `path` may be a block device or a file that already has its final size, + /// which is what `size` describes; `create` makes such a file. + public static func enable(path: String, size: UInt64, pageSize: Int = 4096) throws { + try format(path: path, size: size, pageSize: pageSize) + try on(path: path) + } + + /// Write the swap area header the kernel expects to an existing block + /// device or fully allocated file. + /// + /// This is what `mkswap` writes, and kata has its host run `mkswap` before + /// attaching the device. That is not open to a host which is not Linux, so + /// the header is written here instead, from the guest that is about to + /// enable it. + /// https://github.com/kata-containers/kata-containers/blob/main/src/runtime-rs/crates/resource/src/cpu_mem/swap.rs + public static func format(path: String, size: UInt64, pageSize: Int = 4096) throws { + let pages = size / UInt64(pageSize) + guard pages > 1 else { + throw ContainerizationError( + .invalidArgument, + message: "swap size \(size) is smaller than the two pages a swap area needs" + ) + } + // The header carries the last page number in 32 bits. + guard let lastPage = UInt32(exactly: pages - 1) else { + throw ContainerizationError( + .invalidArgument, + message: "swap size \(size) exceeds the \(UInt64(UInt32.max) + 1) pages a swap header can carry" + ) + } + + let fd = open(path, O_WRONLY) + guard fd >= 0 else { + throw POSIXError.fromErrno() + } + defer { close(fd) } + + var header = [UInt8](repeating: 0, count: pageSize) + header.replaceSubrange(versionOffset..<(versionOffset + 4), with: littleEndianBytes(1)) + header.replaceSubrange( + lastPageOffset..<(lastPageOffset + 4), + with: littleEndianBytes(lastPage) + ) + // The magic occupies the last bytes of the header page. + header.replaceSubrange((pageSize - magic.count).. UInt64 { + let fd = open(path, O_RDONLY) + guard fd >= 0 else { + throw POSIXError.fromErrno() + } + defer { close(fd) } + let end = lseek(fd, 0, SEEK_END) + guard end > 0 else { + throw POSIXError.fromErrno() + } + return UInt64(end) + } + + /// Enable the swap area at `path`, asking the kernel to discard the blocks + /// it stops using so the file backing the area does not keep them. + /// + /// Ours: kata enables its swap with no flags, which suits an area on a + /// host disk that was sized once and stays that size. + /// https://github.com/kata-containers/kata-containers/blob/main/src/agent/src/rpc.rs + public static func on(path: String) throws { + try markSolidState(devicePath: path) + guard CZ_swapon(path, CZ_SWAP_DISCARD | CZ_SWAP_DISCARD_PAGES) == 0 else { + throw POSIXError.fromErrno() + } + } + + /// Mark the block device backing the swap area as non rotational. + /// + /// Ours: no other runtime does this, because no other runtime needs the + /// area to give its blocks back. Kata calls `swapon` with no flags at all. + /// + /// The kernel only tracks a swap area in clusters when its device is non + /// rotational, and freeing a cluster is the only thing that schedules a + /// discard. A virtio block device reports as rotational, so without this + /// the area is scanned rather than clustered and the discard flags above + /// never take effect, leaving the file that backs the area holding every + /// block the guest has ever swapped to. + /// https://github.com/torvalds/linux/blob/master/mm/swapfile.c + static func markSolidState(devicePath: String) throws { + let device = URL(fileURLWithPath: devicePath).lastPathComponent + try "0".write( + to: URL(fileURLWithPath: Self.blockPath) + .appendingPathComponent(device) + .appendingPathComponent("queue") + .appendingPathComponent("rotational"), + atomically: false, + encoding: .ascii + ) + } + + private static func littleEndianBytes(_ value: UInt32) -> [UInt8] { + withUnsafeBytes(of: value.littleEndian) { Array($0) } + } + + #endif // os(Linux) +} diff --git a/Sources/Integration/ContainerTests.swift b/Sources/Integration/ContainerTests.swift index dd2a9165..9980cf21 100644 --- a/Sources/Integration/ContainerTests.swift +++ b/Sources/Integration/ContainerTests.swift @@ -137,6 +137,184 @@ extension IntegrationSuite { } } + func testContainerSwap() async throws { + let id = "test-container-swap" + let bs = try await bootstrap(id) + + let swapPath = Self.binPath(name: "\(id)-swap.raw") + let swap = try Self.makeSwapDevice(at: swapPath, size: 64.mib()) + defer { try? FileManager.default.removeItem(at: swapPath) } + + let buffer = BufferWriter() + let container = try LinuxContainer(id, rootfs: bs.rootfs, vmm: bs.vmm) { config in + config.process.arguments = ["/bin/cat", "/proc/swaps"] + config.process.stdout = buffer + config.swapLayer = swap + config.bootLog = bs.bootLog + } + + do { + try await container.create() + try await container.start() + + let status = try await container.wait() + try await container.stop() + + guard status.exitCode == 0 else { + throw IntegrationError.assert(msg: "process status \(status) != 0") + } + + // The guest lists the area it enabled, so the device is present and + // the kernel accepted the header the agent wrote to it. + let swaps = String(data: buffer.data, encoding: .utf8) ?? "" + guard swaps.contains("partition") else { + throw IntegrationError.assert(msg: "swap not enabled in guest: '\(swaps)'") + } + } catch { + try? await container.stop() + throw error + } + } + + /// The point of swap: a workload whose pages exceed the memory limit + /// reclaims instead of meeting the out of memory killer. The tmpfs is + /// sized explicitly because its default is half of RAM, which cannot + /// exceed the limit and so would never drive a page out. + func testContainerSwapUnderPressure() async throws { + let id = "test-container-swap-pressure" + let bs = try await bootstrap(id) + + let swapPath = Self.binPath(name: "\(id)-swap.raw") + let swap = try Self.makeSwapDevice(at: swapPath, size: 512.mib()) + defer { try? FileManager.default.removeItem(at: swapPath) } + + let buffer = BufferWriter() + let container = try LinuxContainer(id, rootfs: bs.rootfs, vmm: bs.vmm) { config in + // A shell variable is anonymous memory, which is what the kernel + // reclaims to a swap area, and asking the container's cgroup to + // reclaim puts it there rather than leaving it to the pressure the + // allocation happens to produce. Reading the value back afterwards + // is what shows it made the round trip intact, so the length is + // checked after the area has been measured. + config.process.arguments = [ + "/bin/sh", "-c", + "fill=$(head -c 340000000 /dev/zero | tr '\\0' 'a'); " + + "echo 340M > /sys/fs/cgroup/memory.reclaim; " + + "awk '/SwapTotal|SwapFree/ { print $1, $2 }' /proc/meminfo; " + + "test ${#fill} -eq 340000000", + ] + config.process.stdout = buffer + config.memoryInBytes = 256.mib() + config.swapLayer = swap + config.bootLog = bs.bootLog + } + + do { + try await container.create() + try await container.start() + + let status = try await container.wait() + try await container.stop() + + // Survival is the first half of the claim: without swap this + // workload is killed rather than reclaimed. + guard status.exitCode == 0 else { + throw IntegrationError.assert(msg: "workload did not survive memory pressure: \(status)") + } + + // And the second half: pages actually reached the swap device. + let out = String(data: buffer.data, encoding: .utf8) ?? "" + let values = out.split(separator: "\n").reduce(into: [String: Int]()) { acc, line in + let parts = line.split(separator: " ") + if parts.count == 2 { acc[String(parts[0])] = Int(parts[1]) } + } + guard let total = values["SwapTotal:"], let free = values["SwapFree:"], total > 0 else { + throw IntegrationError.assert(msg: "guest reported no swap: '\(out)'") + } + guard UInt64(total - free) * 1024 > 64.mib() else { + throw IntegrationError.assert(msg: "little or nothing was swapped out: '\(out)'") + } + } catch { + try? await container.stop() + throw error + } + } + + func testContainerSwapReclaimsFreedBlocks() async throws { + let id = "test-container-swap-reclaim" + let bs = try await bootstrap(id) + + let swapPath = Self.binPath(name: "\(id)-swap.raw") + let swap = try Self.makeSwapDevice(at: swapPath, size: 512.mib()) + defer { try? FileManager.default.removeItem(at: swapPath) } + + let container = try LinuxContainer(id, rootfs: bs.rootfs, vmm: bs.vmm) { config in + // The allocation is held by a child that exits, because the swap + // slots are only freed once the pages themselves are, and a shell + // that drops a variable keeps the pages in its own allocator. + config.process.arguments = [ + "/bin/sh", "-c", + "sh -c 'fill=$(head -c 200000000 /dev/zero | tr \"\\0\" a); " + + "echo 200M > /sys/fs/cgroup/memory.reclaim; " + + "test ${#fill} -eq 200000000' && sleep 30", + ] + config.memoryInBytes = 256.mib() + config.swapLayer = swap + config.bootLog = bs.bootLog + } + + // The blocks the file holds, rather than the size it reports, because a + // sparse file only ever reports the whole area. URL resource values + // cache after their first read, which a sampler cannot use. + func allocatedBytes() -> UInt64 { + var info = stat() + guard stat(swapPath.absolutePath(), &info) == 0 else { + return 0 + } + return UInt64(info.st_blocks) * 512 + } + + do { + try await container.create() + try await container.start() + + // Watching the file while the workload runs is what separates an + // area that released its blocks from one that never held any. + let peak = Task { + var high: UInt64 = 0 + while !Task.isCancelled { + high = max(high, allocatedBytes()) + try? await Task.sleep(nanoseconds: 200_000_000) + } + return high + } + + let status = try await container.wait() + peak.cancel() + let highWater = await peak.value + try await container.stop() + + guard status.exitCode == 0 else { + throw IntegrationError.assert(msg: "workload did not complete: \(status)") + } + guard highWater > 100.mib() else { + throw IntegrationError.assert( + msg: "workload never filled the swap area, it held \(highWater) bytes") + } + + // Having been filled, the area releases what the guest stopped + // using, so the file backing it does not hold its high water mark. + let settled = allocatedBytes() + guard settled < highWater / 4 else { + throw IntegrationError.assert( + msg: "swap area kept \(settled) of \(highWater) bytes after the guest freed it") + } + } catch { + try? await container.stop() + throw error + } + } + func testProcessEchoHi() async throws { let id = "test-process-echo-hi" let bs = try await bootstrap(id) diff --git a/Sources/Integration/Suite.swift b/Sources/Integration/Suite.swift index 84580dba..6087aafe 100644 --- a/Sources/Integration/Suite.swift +++ b/Sources/Integration/Suite.swift @@ -207,6 +207,27 @@ struct IntegrationSuite: AsyncParsableCommand { static let eventLoop = MultiThreadedEventLoopGroup(numberOfThreads: System.coreCount) + /// The swap area a test hands to a container. A suite that builds its own + /// containers owns the file behind them, the way `ContainerManager` owns + /// the ones it makes for callers that do not. + static func makeSwapDevice(at path: URL, size: UInt64) throws -> Containerization.Mount { + guard FileManager.default.createFile(atPath: path.absolutePath(), contents: nil) else { + throw IntegrationError.assert(msg: "failed to create swap device at \(path.absolutePath())") + } + let handle = try FileHandle(forWritingTo: path) + defer { try? handle.close() } + try handle.truncate(atOffset: size) + // A swap area holds nothing that outlives the container, so the host + // has no reason to synchronize it to permanent storage. + return .block( + format: Swap.mountType, + source: path.absolutePath(), + destination: "", + options: [], + runtimeOptions: ["vzDiskImageSynchronizationMode=none"] + ) + } + func bootstrap(_ testID: String) async throws -> (rootfs: Containerization.Mount, vmm: VirtualMachineManager, image: Containerization.Image, bootLog: BootLog) { let reference = "ghcr.io/linuxcontainers/alpine:3.20" let store = Self.imageStore @@ -614,6 +635,11 @@ struct IntegrationSuite: AsyncParsableCommand { Test("pod filesystem operation", testPodFilesystemOperation), Test("pod shared disk image volume", testPodSharedDiskImageVolume), Test("pod shared tmpfs volume", testPodSharedTmpfsVolume), + + // Swap + Test("container swap", testContainerSwap), + Test("container swap under pressure", testContainerSwapUnderPressure), + Test("container swap reclaims freed blocks", testContainerSwapReclaimsFreedBlocks), ] + macOS26Tests() let tests: [Test] = crossPlatformTests + macOSOnlyTests #else diff --git a/Sources/cctl/RunCommand.swift b/Sources/cctl/RunCommand.swift index fceff172..21f13a3e 100644 --- a/Sources/cctl/RunCommand.swift +++ b/Sources/cctl/RunCommand.swift @@ -48,6 +48,9 @@ extension Application { @Flag(name: .customLong("rosetta"), help: "Enable rosetta x64 emulation") var rosetta = false + @Option(name: .customLong("swap"), help: "Amount of swap in megabytes to create in the guest") + var swap: UInt64 = 0 + @Option(name: .customLong("mount"), help: "Directory to share into the container (Example: /foo:/bar)") var mounts: [String] = [] @@ -106,6 +109,7 @@ extension Application { id, reference: imageReference, rootfsSizeInBytes: fsSizeInMB.mib(), + swapSizeInBytes: swap.mib(), readOnly: readOnly, networking: true ) { config in diff --git a/Tests/ContainerizationTests/SwapTests.swift b/Tests/ContainerizationTests/SwapTests.swift new file mode 100644 index 00000000..1b8aa8a9 --- /dev/null +++ b/Tests/ContainerizationTests/SwapTests.swift @@ -0,0 +1,73 @@ +//===----------------------------------------------------------------------===// +// Copyright © 2026 Apple Inc. and the Containerization project authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//===----------------------------------------------------------------------===// + +#if os(Linux) +import ContainerizationOS +import Foundation +import Testing + +struct SwapTests { + private func makeArea(pages: UInt64, pageSize: Int) throws -> String { + let path = FileManager.default.temporaryDirectory + .appendingPathComponent("swap-\(UUID().uuidString)").path + try #require( + FileManager.default.createFile( + atPath: path, + contents: Data(count: Int(pages) * pageSize) + ) + ) + return path + } + + /// The header the kernel reads back is the one `mkswap` writes: version 1 + /// and the last usable page at the documented offsets, and the magic in the + /// final bytes of the first page. + @Test func formatWritesTheSwapAreaHeader() throws { + let pageSize = 4096 + let pages: UInt64 = 32 + let path = try makeArea(pages: pages, pageSize: pageSize) + defer { try? FileManager.default.removeItem(atPath: path) } + + try Swap.format(path: path, size: pages * UInt64(pageSize), pageSize: pageSize) + + let header = try Data(contentsOf: URL(fileURLWithPath: path))[0.. Date: Sun, 2 Aug 2026 00:03:43 +0000 Subject: [PATCH 07/16] Give a pod a swap area its containers share A pod's containers share one virtual machine, so they can share one swap area as well, with the guest kernel deciding whose pages are reclaimed to it rather than each container carrying its own. The area is attached with the pod's other mounts and enabled once, after the agent comes up. A container may cap how much of the area it uses. That cap counts swap alone, while the runtime spec carries memory and swap as a single total, so the container's memory limit is added to it when the spec is built. A cap without a memory limit is rejected when the container is added, because the total cannot be worked out without one. Leaving the cap unset lets a container use the whole area, which is what containers sharing a pool generally want. --- .../CHVirtualMachineInstance.swift | 7 +- .../Containerization/ContainerStorage.swift | 18 +++-- Sources/Containerization/LinuxPod.swift | 66 ++++++++++++++++++- .../Containerization/VMConfiguration.swift | 2 +- .../VZVirtualMachineInstance.swift | 5 +- Sources/Integration/PodTests.swift | 60 +++++++++++++++++ Sources/Integration/Suite.swift | 1 + 7 files changed, 147 insertions(+), 12 deletions(-) diff --git a/Sources/Containerization/CHVirtualMachineInstance.swift b/Sources/Containerization/CHVirtualMachineInstance.swift index e363d5ca..1704ce92 100644 --- a/Sources/Containerization/CHVirtualMachineInstance.swift +++ b/Sources/Containerization/CHVirtualMachineInstance.swift @@ -729,8 +729,8 @@ extension CHVirtualMachineInstance { extension CHVirtualMachineInstance.Configuration { /// Walks boot-time storage in the machine's device order (rootfs first, /// then containers sorted by id, each in role order, then volumes sorted - /// by name), allocating disk letters for virtio-blk mounts and seeding - /// the machine's `AttachedFilesystem` registry. + /// by name, then swap), allocating disk letters for virtio-blk mounts + /// and seeding the machine's `AttachedFilesystem` registry. /// /// The allocator is shared with the runtime hotplug provider, so block /// hotplug picks up at the next free letter after boot. @@ -765,8 +765,9 @@ extension CHVirtualMachineInstance.Configuration { guard let mount = self.storage.volumes[name] else { continue } volumes[name] = try attach(mount, chId: { "vol-\(name)-\($0)" }) } + let swap = try self.storage.swap.map { try attach($0, chId: { "swap-\($0)" }) } - return (MachineAttachments(containers: containers, volumes: volumes), bootDisks) + return (MachineAttachments(containers: containers, volumes: volumes, swap: swap), bootDisks) } } #endif diff --git a/Sources/Containerization/ContainerStorage.swift b/Sources/Containerization/ContainerStorage.swift index 49fae6d4..c98b8558 100644 --- a/Sources/Containerization/ContainerStorage.swift +++ b/Sources/Containerization/ContainerStorage.swift @@ -83,18 +83,25 @@ public struct MachineStorage: Sendable { /// Volumes shared by the machine's containers, by volume name. public var volumes: [String: Value] - public init(containers: [String: ContainerStorage] = [:], volumes: [String: Value] = [:]) { + /// The swap area shared by every container in the machine, when it has + /// one. + public var swap: Value? + + public init(containers: [String: ContainerStorage] = [:], volumes: [String: Value] = [:], swap: Value? = nil) { self.containers = containers self.volumes = volumes + self.swap = swap } /// Every value the machine carries: containers sorted by ID, each in - /// role order, then volumes sorted by name. Walks that allocate device - /// addresses and walks that create the devices use this one order, so an - /// address always names the device it was handed out for. + /// role order, then volumes sorted by name, then swap. Walks that + /// allocate device addresses and walks that create the devices use this + /// one order, so an address always names the device it was handed out + /// for. public var ordered: [Value] { containers.keys.sorted().flatMap { containers[$0]?.all ?? [] } + volumes.keys.sorted().compactMap { volumes[$0] } + + (swap.map { [$0] } ?? []) } /// The storage with `transform` applied to every value, each keeping its @@ -103,7 +110,8 @@ public struct MachineStorage: Sendable { public func map(_ transform: (Value) throws -> U) rethrows -> MachineStorage { MachineStorage( containers: try containers.mapValues { try $0.map(transform) }, - volumes: try volumes.mapValues(transform) + volumes: try volumes.mapValues(transform), + swap: try swap.map(transform) ) } } diff --git a/Sources/Containerization/LinuxPod.swift b/Sources/Containerization/LinuxPod.swift index fa4d1785..140f01ab 100644 --- a/Sources/Containerization/LinuxPod.swift +++ b/Sources/Containerization/LinuxPod.swift @@ -21,6 +21,7 @@ import Foundation import Logging import Synchronization +import struct ContainerizationOS.Swap import struct ContainerizationOS.Terminal /// NOTE: Experimental API @@ -43,6 +44,15 @@ public final class LinuxPod: Sendable { public var cpus: Int = 4 /// The memory in bytes to give to the pod's VM. public var memoryInBytes: UInt64 = 1024.mib() + /// Optional swap area shared by every container in the pod, as a block + /// device mount. + /// + /// The area belongs to the pod rather than to any one container, so the + /// guest kernel decides which container's pages are reclaimed to it. + /// Containers are free to use all of it unless they carry a limit of + /// their own. The `destination` field is ignored, as the area is + /// enabled rather than mounted. + public var swapLayer: Mount? = nil /// The network interfaces for the pod. public var interfaces: [any Interface] = [] /// Whether nested virtualization should be turned on for the pod. @@ -77,6 +87,24 @@ public final class LinuxPod: Sendable { public var cpus: Int? /// Optional per-container memory limit in bytes (can exceed pod total for oversubscription). public var memoryInBytes: UInt64? + /// Optional cap on how much of the pod's swap area this container may + /// use, in bytes. Leaving it unset lets the container use the whole + /// area, which is what containers sharing a pool usually want. + /// + /// This counts swap alone. The runtime spec carries memory and swap + /// combined, so `memoryInBytes` is added to it when the spec is built, + /// and a swap limit without a memory limit is rejected because the + /// combined figure cannot be worked out without one. + /// + /// Kata and docker spell the same limit as the combined figure the spec + /// carries, subtracting the memory limit to size the area, so a + /// container asking there for 2 GiB against a 1 GiB memory limit is + /// asking for 1 GiB of swap and here for 2 GiB. That figure suits a + /// container whose swap is sized for it alone; this one is a share of + /// an area the pod owns, and how much of that share a container may + /// take is what the number says. + /// https://github.com/kata-containers/kata-containers/blob/main/docs/how-to/how-to-setup-swap-devices-in-guest-kernel.md + public var swapInBytes: UInt64? /// The hostname for the container. public var hostname: String? /// The system control options for the container. @@ -326,8 +354,15 @@ public final class LinuxPod: Sendable { ) } if let memoryInBytes = config.memoryInBytes, memoryInBytes > 0 { + // The runtime spec's `swap` is the memory and swap total, not the + // swap alone, so the container's memory limit is folded in here. + var swapTotal: Int64? = nil + if let swapInBytes = config.swapInBytes { + swapTotal = Int64(memoryInBytes + swapInBytes) + } spec.linux?.resources?.memory = LinuxMemory( - limit: Int64(memoryInBytes) + limit: Int64(memoryInBytes), + swap: swapTotal ) } @@ -390,6 +425,15 @@ extension LinuxPod { var config = ContainerConfiguration() try configuration(&config) + // The runtime spec carries memory and swap as one total, so a swap + // limit cannot be expressed without a memory limit to add it to. + if config.swapInBytes != nil, config.memoryInBytes == nil { + throw ContainerizationError( + .invalidArgument, + message: "container \(id) sets a swap limit without a memory limit" + ) + } + let fileMountContext = try FileMountContext.prepare(mounts: config.mounts) switch state.phase { @@ -607,6 +651,9 @@ extension LinuxPod { for volume in self.config.volumes { machineStorage.volumes[volume.name] = volume.toMount() } + // The swap area is attached with the machine's own storage so the + // guest is told the /dev path the VMM allocates it. + machineStorage.swap = self.config.swapLayer // Capture into an immutable `let` so the value is safely usable // from the concurrent `withAgent` closure below. The container @@ -638,10 +685,27 @@ extension LinuxPod { let shareProcessNamespace = self.config.shareProcessNamespace let pauseProcessHolder = Mutex(nil) let fileMountContextUpdates = Mutex<[String: FileMountContext]>([:]) + let hasSwapLayer = self.config.swapLayer != nil try await vm.withAgent { agent in try await agent.standardSetup() + // The swap area belongs to the pod rather than to any one + // container, so it is enabled once here and every container + // reclaims to it through the guest's own memory management. + if hasSwapLayer { + guard let swap = vm.storage.swap else { + throw ContainerizationError(.notFound, message: "swap mount not found") + } + try await agent.mount( + ContainerizationOCI.Mount( + type: Swap.mountType, + source: swap.source, + destination: "", + options: swap.options + )) + } + // Mount the unified virtiofs share at /run/virtiofs only // when at least one container has a virtiofs mount. VZ // tolerates the unbacked mount; CH does not. diff --git a/Sources/Containerization/VMConfiguration.swift b/Sources/Containerization/VMConfiguration.swift index 7a5e8e2c..f00e3e04 100644 --- a/Sources/Containerization/VMConfiguration.swift +++ b/Sources/Containerization/VMConfiguration.swift @@ -73,7 +73,7 @@ public struct VMConfiguration: Sendable { /// The network interfaces to attach. public var interfaces: [any Interface] /// The storage the machine carries: each container's mounts by role, - /// and the volumes its containers share. + /// and the volumes and swap its containers share. public var storage: MachineMounts /// Optional destination for serial boot logs. public var bootLog: BootLog? diff --git a/Sources/Containerization/VZVirtualMachineInstance.swift b/Sources/Containerization/VZVirtualMachineInstance.swift index 670ab512..69828ec9 100644 --- a/Sources/Containerization/VZVirtualMachineInstance.swift +++ b/Sources/Containerization/VZVirtualMachineInstance.swift @@ -74,7 +74,7 @@ public final class VZVirtualMachineInstance: Sendable { /// Toggle nested virtualization support. public var nestedVirtualization: Bool /// The machine's storage: each container's mounts by role, and the - /// volumes its containers share. + /// volumes and swap its containers share. public var storage: MachineMounts /// Network interface attachments. public var interfaces: [any Interface] @@ -567,8 +567,9 @@ extension VZVirtualMachineInstance.Configuration { guard let mount = self.storage.volumes[name] else { continue } volumes[name] = try attach(mount) } + let swap = try self.storage.swap.map(attach) - return (MachineAttachments(containers: containers, volumes: volumes), storageDeviceCount) + return (MachineAttachments(containers: containers, volumes: volumes, swap: swap), storageDeviceCount) } } diff --git a/Sources/Integration/PodTests.swift b/Sources/Integration/PodTests.swift index 7dff20fa..93b0b7c9 100644 --- a/Sources/Integration/PodTests.swift +++ b/Sources/Integration/PodTests.swift @@ -96,6 +96,66 @@ extension IntegrationSuite { } } + func testPodSharedSwap() async throws { + let id = "test-pod-shared-swap" + + let bs = try await bootstrap(id) + let swapPath = Self.binPath(name: "\(id)-swap.raw") + let swap = try Self.makeSwapDevice(at: swapPath, size: 512.mib()) + defer { try? FileManager.default.removeItem(at: swapPath) } + + let pod = try LinuxPod(id, vmm: bs.vmm) { config in + config.cpus = 2 + config.memoryInBytes = 512.mib() + config.bootLog = bs.bootLog + config.swapLayer = swap + } + + // Both containers report the same area, because the pod owns it and + // the guest kernel decides whose pages are reclaimed to it. + let names = ["swap1", "swap2"] + let buffers = [names[0]: BufferWriter(), names[1]: BufferWriter()] + for name in names { + let buffer = buffers[name]! + try await pod.addContainer( + name, + rootfs: try cloneRootfs(bs.rootfs, testID: id, containerID: name) + ) { config in + config.process.arguments = [ + "/bin/sh", "-c", + "awk '/SwapTotal/ { print $2 }' /proc/meminfo", + ] + config.process.stdout = buffer + } + } + + try await pod.create() + + var totals: [UInt64] = [] + for name in names { + try await pod.startContainer(name) + let status = try await pod.waitContainer(name) + guard status.exitCode == 0 else { + throw IntegrationError.assert(msg: "\(name) status \(status) != 0") + } + let out = String(data: buffers[name]!.data, encoding: .utf8) ?? "" + guard let total = UInt64(out.trimmingCharacters(in: .whitespacesAndNewlines)) else { + throw IntegrationError.assert(msg: "\(name) reported no swap total: '\(out)'") + } + totals.append(total) + } + + try await pod.stop() + + guard totals[0] > 0 else { + throw IntegrationError.assert(msg: "pod swap area was not enabled: \(totals)") + } + guard totals[0] == totals[1] else { + throw IntegrationError.assert( + msg: "containers saw different swap areas: \(totals)") + } + } + func testPodContainerOutput() async throws { let id = "test-pod-container-output" diff --git a/Sources/Integration/Suite.swift b/Sources/Integration/Suite.swift index 6087aafe..f74321fc 100644 --- a/Sources/Integration/Suite.swift +++ b/Sources/Integration/Suite.swift @@ -409,6 +409,7 @@ struct IntegrationSuite: AsyncParsableCommand { Test("container IPv6 only default route", testIPv6OnlyDefaultRoute), Test("container IPv6 only gateway outside subnet", testIPv6OnlyGatewayOutsideSubnet), Test("container IPv6 dual stack", testIPv6DualStack), + Test("pod shared swap", testPodSharedSwap), Test("pod IPv6 address", testPodIPv6AddressAdd), ] } From c0d0220cc589741b22efed6ba2fd84558c4889df Mon Sep 17 00:00:00 2001 From: Aaron Paterson Date: Sun, 2 Aug 2026 20:09:40 +0000 Subject: [PATCH 08/16] Hold a container to the swap limit it was given A pod works out the memory and swap total its containers are capped at, the way the runtime spec asks for it, and sends that along with the rest of the resources. The guest wrote the memory limit and dropped the rest, so a container drew on the whole of the pod's swap area whatever it had been given. Cgroup v2 wants the swap on its own rather than the total, so the memory limit comes back out of it before the limit is written, and the spec's unlimited and unset values are carried across as the controller spells them. Reading the limit back from the kernel is what the test does, since a limit the guest never applied leaves nothing else to see. https://github.com/opencontainers/cgroups/blob/main/utils.go --- Sources/Integration/PodTests.swift | 55 +++++++++++++++++++++ Sources/Integration/Suite.swift | 1 + vminitd/Sources/Cgroup/Cgroup2Manager.swift | 34 +++++++++++++ 3 files changed, 90 insertions(+) diff --git a/Sources/Integration/PodTests.swift b/Sources/Integration/PodTests.swift index 93b0b7c9..19854194 100644 --- a/Sources/Integration/PodTests.swift +++ b/Sources/Integration/PodTests.swift @@ -156,6 +156,61 @@ extension IntegrationSuite { } } + /// A container's cap names the swap alone while the runtime spec carries the + /// memory and swap total, so the guest has to take the memory back out of it + /// before the kernel will hold the container to it. Read the cap back from + /// the kernel, because a spec the guest ignores leaves the container drawing + /// on the whole pod area with nothing to show it. + func testPodContainerSwapLimit() async throws { + let id = "test-pod-container-swap-limit" + + let bs = try await bootstrap(id) + let swapPath = Self.binPath(name: "\(id)-swap.raw") + let swap = try Self.makeSwapDevice(at: swapPath, size: 512.mib()) + defer { try? FileManager.default.removeItem(at: swapPath) } + + let pod = try LinuxPod(id, vmm: bs.vmm) { config in + config.cpus = 2 + config.memoryInBytes = 512.mib() + config.bootLog = bs.bootLog + config.swapLayer = swap + } + + let capped: UInt64 = 64.mib() + let buffer = BufferWriter() + try await pod.addContainer( + "capped", + rootfs: try cloneRootfs(bs.rootfs, testID: id, containerID: "capped") + ) { config in + config.memoryInBytes = 128.mib() + config.swapInBytes = capped + config.process.arguments = [ + "/bin/sh", "-c", "cat /sys/fs/cgroup/memory.swap.max", + ] + config.process.stdout = buffer + } + + try await pod.create() + try await pod.startContainer("capped") + let status = try await pod.waitContainer("capped") + try await pod.stop() + guard status.exitCode == 0 else { + throw IntegrationError.assert(msg: "capped status \(status) != 0") + } + + let reported = + String(data: buffer.data, encoding: .utf8)? + .trimmingCharacters(in: .whitespacesAndNewlines) ?? "" + guard let limit = UInt64(reported) else { + throw IntegrationError.assert( + msg: "swap cap never reached the kernel, memory.swap.max is '\(reported)'") + } + guard limit == capped else { + throw IntegrationError.assert( + msg: "expected a \(capped) byte swap cap, kernel holds \(limit)") + } + } + func testPodContainerOutput() async throws { let id = "test-pod-container-output" diff --git a/Sources/Integration/Suite.swift b/Sources/Integration/Suite.swift index f74321fc..bd4ec656 100644 --- a/Sources/Integration/Suite.swift +++ b/Sources/Integration/Suite.swift @@ -410,6 +410,7 @@ struct IntegrationSuite: AsyncParsableCommand { Test("container IPv6 only gateway outside subnet", testIPv6OnlyGatewayOutsideSubnet), Test("container IPv6 dual stack", testIPv6DualStack), Test("pod shared swap", testPodSharedSwap), + Test("pod container swap limit", testPodContainerSwapLimit), Test("pod IPv6 address", testPodIPv6AddressAdd), ] } diff --git a/vminitd/Sources/Cgroup/Cgroup2Manager.swift b/vminitd/Sources/Cgroup/Cgroup2Manager.swift index 62fa78c7..9ab812bc 100644 --- a/vminitd/Sources/Cgroup/Cgroup2Manager.swift +++ b/vminitd/Sources/Cgroup/Cgroup2Manager.swift @@ -216,6 +216,37 @@ public struct Cgroup2Manager: Sendable { ) } + // The runtime spec's `swap` is the memory and swap total, the way cgroup + // v1 took it, while cgroup v2 wants the swap on its own. A zero is the + // spec's "unset", which leaves the limit at max. + // https://github.com/opencontainers/cgroups/blob/main/utils.go + if let memory = resources.memory, let swap = memory.swap, swap != 0 { + let value: String + if swap < 0 { + value = "max" + } else { + guard let limit = memory.limit, limit != 0 else { + throw Error.invalidResource( + message: "unable to set swap limit without memory limit") + } + if limit < 0 { + // Memory is unlimited, so the total that contains it is too. + value = "max" + } else { + guard swap >= limit else { + throw Error.invalidResource( + message: "memory and swap limit \(swap) is below the memory limit \(limit)") + } + value = String(swap - limit) + } + } + try Self.writeValue( + path: self.path, + value: value, + fileName: "memory.swap.max" + ) + } + if let cpu = resources.cpu, let quota = cpu.quota, let period = cpu.period { // cpu.max format is "quota period" let value = "\(quota) \(period)" @@ -751,6 +782,7 @@ extension Cgroup2Manager { case cgroup1 case errno(errno: Int32, message: String) case notExist(path: String) + case invalidResource(message: String) package var description: String { switch self { @@ -762,6 +794,8 @@ extension Cgroup2Manager { return "tried to load a cgroup v1 path" case .notCgroup: return "path is not a cgroup mountpoint" + case .invalidResource(let message): + return message } } } From 2cbb72321404631d517f957d57ebb98dd48abec1 Mon Sep 17 00:00:00 2001 From: Aaron Paterson Date: Sun, 2 Aug 2026 20:28:50 +0000 Subject: [PATCH 09/16] Take the swap total as given when memory is unlimited An unlimited memory limit leaves nothing to subtract from the memory and swap total, so the swap stands as it was given rather than becoming unlimited alongside it. A memory limit that is negative without being the unlimited sentinel is not a limit at all, and is refused rather than read as one. https://github.com/opencontainers/cgroups/blob/main/utils.go --- vminitd/Sources/Cgroup/Cgroup2Manager.swift | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/vminitd/Sources/Cgroup/Cgroup2Manager.swift b/vminitd/Sources/Cgroup/Cgroup2Manager.swift index 9ab812bc..a9dcd97d 100644 --- a/vminitd/Sources/Cgroup/Cgroup2Manager.swift +++ b/vminitd/Sources/Cgroup/Cgroup2Manager.swift @@ -229,9 +229,12 @@ public struct Cgroup2Manager: Sendable { throw Error.invalidResource( message: "unable to set swap limit without memory limit") } - if limit < 0 { - // Memory is unlimited, so the total that contains it is too. - value = "max" + if limit == -1 { + // Unlimited memory leaves nothing to take out of the total, + // so the swap stands as it was given. + value = String(swap) + } else if limit < 0 { + throw Error.invalidResource(message: "invalid memory value: \(limit)") } else { guard swap >= limit else { throw Error.invalidResource( From be4f9cca1aafdd58cde8d3bfaa75aa516ebbb9ba Mon Sep 17 00:00:00 2001 From: Aaron Paterson Date: Fri, 7 Aug 2026 17:31:13 +0000 Subject: [PATCH 10/16] Let a pod's containers copy files in and out Copying a file between the host and a container is reachable only for a container in a machine of its own; the same container placed in a pod has no way to be copied into or out of, so a tool that offers `cp` loses it the moment a container joins a pod. What the transfer needs is the machine, the path the container's filesystem sits at in the guest, and a vsock port, and only the path differs between the two cases. It moves to a type that takes those, which both a standalone container and a pod's containers hand it. The round trip is covered against a running pod, including that a file copied into one container reaches that container alone. --- .../Containerization/GuestFileTransfer.swift | 303 ++++++++++++++++++ Sources/Containerization/LinuxContainer.swift | 264 ++------------- Sources/Containerization/LinuxPod.swift | 79 +++++ Sources/Integration/PodTests.swift | 84 +++++ Sources/Integration/Suite.swift | 1 + 5 files changed, 492 insertions(+), 239 deletions(-) create mode 100644 Sources/Containerization/GuestFileTransfer.swift diff --git a/Sources/Containerization/GuestFileTransfer.swift b/Sources/Containerization/GuestFileTransfer.swift new file mode 100644 index 00000000..d508c571 --- /dev/null +++ b/Sources/Containerization/GuestFileTransfer.swift @@ -0,0 +1,303 @@ +//===----------------------------------------------------------------------===// +// Copyright © 2026 Apple Inc. and the container project authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//===----------------------------------------------------------------------===// + +import ContainerizationArchive +import ContainerizationError +import ContainerizationOS +import Foundation + +/// Moves files between the host and a container's filesystem inside a guest. +/// +/// The transfer runs over a dedicated vsock connection on a port the caller +/// allocates. A container in a machine of its own and a container among +/// several in a pod differ only in where their filesystem sits in the guest, +/// so that path is what this is given. +struct GuestFileTransfer: Sendable { + /// Default chunk size for file transfers (1MiB). + static let defaultChunkSize = 1024 * 1024 + + /// The machine holding the container's filesystem. + let vm: any VirtualMachineInstance + /// Where the container's filesystem sits in the guest. + let guestRoot: String + /// The vsock port the data travels over. + let port: UInt32 + /// Where the blocking read and write work runs. + let queue: DispatchQueue + + /// Copy a file or directory from the host into the container. + /// + /// For directories, the source is archived as tar+gzip and streamed + /// directly through vsock without intermediate temp files. + func copyIn( + from source: URL, + to destination: URL, + mode: UInt32, + createParents: Bool, + chunkSize: Int + ) async throws { + var isDirectory: ObjCBool = false + guard FileManager.default.fileExists(atPath: source.path, isDirectory: &isDirectory) else { + throw ContainerizationError(.notFound, message: "copyIn: source not found '\(source.path)'") + } + let isArchive = isDirectory.boolValue + + let guestPath: URL = try await vm.withAgent { agent in + guard let vminitd = agent as? Vminitd else { + throw ContainerizationError(.unsupported, message: "copyIn requires Vminitd agent") + } + + return try await self.resolveCopyInGuestPath( + from: source, + to: destination, + sourceIsDirectory: isArchive, + using: vminitd + ) + } + + let listener = try vm.listen(port) + + try await withThrowingTaskGroup(of: Void.self) { group in + group.addTask { + try await self.vm.withAgent { agent in + guard let vminitd = agent as? Vminitd else { + throw ContainerizationError(.unsupported, message: "copyIn requires Vminitd agent") + } + try await vminitd.copy( + direction: .copyIn, + guestPath: guestPath, + vsockPort: self.port, + mode: mode, + createParents: createParents, + isArchive: isArchive + ) + } + } + + group.addTask { + guard let conn = await listener.first(where: { _ in true }) else { + throw ContainerizationError(.internalError, message: "copyIn: vsock connection not established") + } + try listener.finish() + + try await withCheckedThrowingContinuation { (continuation: CheckedContinuation) in + self.queue.async { + do { + defer { conn.closeFile() } + + if isArchive { + let writer = try ArchiveWriter(configuration: .init(format: .pax, filter: .gzip)) + try writer.open(fileDescriptor: conn.fileDescriptor) + try writer.archiveDirectory(source) + try writer.finishEncoding() + } else { + let srcFd = open(source.path, O_RDONLY) + guard srcFd != -1 else { + throw ContainerizationError( + .internalError, + message: "copyIn: failed to open '\(source.path)': \(String(cString: strerror(errno)))" + ) + } + defer { close(srcFd) } + + var buf = [UInt8](repeating: 0, count: chunkSize) + while true { + let n = read(srcFd, &buf, buf.count) + if n == 0 { break } + guard n > 0 else { + throw ContainerizationError( + .internalError, + message: "copyIn: read error: \(String(cString: strerror(errno)))" + ) + } + var written = 0 + while written < n { + let w = buf.withUnsafeBytes { ptr in + write(conn.fileDescriptor, ptr.baseAddress! + written, n - written) + } + guard w > 0 else { + throw ContainerizationError( + .internalError, + message: "copyIn: vsock write error: \(String(cString: strerror(errno)))" + ) + } + written += w + } + } + } + continuation.resume() + } catch { + continuation.resume(throwing: error) + } + } + } + } + + try await group.waitForAll() + } + } + + /// Copy a file or directory from the container to the host. + /// + /// For directories, the guest archives the source as tar+gzip and streams + /// it directly through vsock. The host extracts the archive without + /// intermediate temp files. + func copyOut( + from source: URL, + to destination: URL, + createParents: Bool, + chunkSize: Int + ) async throws { + if createParents { + let parentDir = destination.deletingLastPathComponent() + try FileManager.default.createDirectory(at: parentDir, withIntermediateDirectories: true) + } + + let guestPath = URL(filePath: guestRoot).appending(path: source.path) + let listener = try vm.listen(port) + + let (metadataStream, metadataCont) = AsyncStream.makeStream(of: Vminitd.CopyMetadata.self) + + try await withThrowingTaskGroup(of: Void.self) { group in + group.addTask { + defer { metadataCont.finish() } + try await self.vm.withAgent { agent in + guard let vminitd = agent as? Vminitd else { + throw ContainerizationError(.unsupported, message: "copyOut requires Vminitd agent") + } + try await vminitd.copy( + direction: .copyOut, + guestPath: guestPath, + vsockPort: self.port, + onMetadata: { meta in + metadataCont.yield(meta) + metadataCont.finish() + } + ) + } + } + + group.addTask { + guard let metadata = await metadataStream.first(where: { _ in true }) else { + throw ContainerizationError(.internalError, message: "copyOut: no metadata received") + } + + guard let conn = await listener.first(where: { _ in true }) else { + throw ContainerizationError(.internalError, message: "copyOut: vsock connection not established") + } + try listener.finish() + + try await withCheckedThrowingContinuation { (continuation: CheckedContinuation) in + self.queue.async { + do { + defer { conn.closeFile() } + + if metadata.isArchive { + try FileManager.default.createDirectory(at: destination, withIntermediateDirectories: true) + let fh = FileHandle(fileDescriptor: dup(conn.fileDescriptor), closeOnDealloc: true) + let reader = try ArchiveReader(format: .pax, filter: .gzip, fileHandle: fh) + _ = try reader.extractContents(to: destination) + } else { + let destFd = open(destination.path, O_WRONLY | O_CREAT | O_TRUNC, 0o644) + guard destFd != -1 else { + throw ContainerizationError( + .internalError, + message: "copyOut: failed to open '\(destination.path)': \(String(cString: strerror(errno)))" + ) + } + defer { close(destFd) } + + var buf = [UInt8](repeating: 0, count: chunkSize) + while true { + let n = read(conn.fileDescriptor, &buf, buf.count) + if n == 0 { break } + guard n > 0 else { + throw ContainerizationError( + .internalError, + message: "copyOut: vsock read error: \(String(cString: strerror(errno)))" + ) + } + var written = 0 + while written < n { + let w = buf.withUnsafeBytes { ptr in + write(destFd, ptr.baseAddress! + written, n - written) + } + guard w > 0 else { + throw ContainerizationError( + .internalError, + message: "copyOut: write error: \(String(cString: strerror(errno)))" + ) + } + written += w + } + } + } + continuation.resume() + } catch { + continuation.resume(throwing: error) + } + } + } + } + + try await group.waitForAll() + } + } + + /// Where a copy lands in the guest, given what the destination already is. + /// + /// A destination that names an existing directory receives the source + /// under its own name, the way `cp` behaves. + private func resolveCopyInGuestPath( + from source: URL, + to destination: URL, + sourceIsDirectory: Bool, + using vminitd: Vminitd + ) async throws -> URL { + let guestDestination = URL(filePath: guestRoot).appending(path: destination.path) + + let stat: ContainerizationOS.Stat? + do { + stat = try await vminitd.stat(path: guestDestination) + } catch let error as ContainerizationError where error.code == .notFound { + stat = nil + } + // Any other error propagates so transport and permission failures are visible. + + guard let stat else { + if destination.hasDirectoryPath && !sourceIsDirectory { + throw ContainerizationError( + .invalidArgument, + message: "destination directory does not exist: \(destination.path)" + ) + } + return guestDestination + } + + let destinationIsDirectory = (stat.mode & UInt32(S_IFMT)) == UInt32(S_IFDIR) + guard destinationIsDirectory else { + if sourceIsDirectory { + throw ContainerizationError( + .invalidArgument, + message: "cannot copy directory over existing file: \(destination.path)" + ) + } + return guestDestination + } + + return guestDestination.appendingPathComponent(source.lastPathComponent) + } +} diff --git a/Sources/Containerization/LinuxContainer.swift b/Sources/Containerization/LinuxContainer.swift index 4d5289a7..eca34a1a 100644 --- a/Sources/Containerization/LinuxContainer.swift +++ b/Sources/Containerization/LinuxContainer.swift @@ -669,8 +669,8 @@ extension LinuxContainer { let vm = try await self.vmm.create(config: creationConfig) let relayManager = UnixSocketRelayManager(vm: vm, log: self.logger) + try await vm.start() do { - try await vm.start() let storageForAgent = containerStorage try await vm.withAgent { agent in try await agent.standardSetup() @@ -1194,7 +1194,7 @@ extension LinuxContainer { } /// Default chunk size for file transfers (1MiB). - public static let defaultCopyChunkSize = 1024 * 1024 + public static let defaultCopyChunkSize = GuestFileTransfer.defaultChunkSize /// Copy a file or directory from the host into the container. /// @@ -1210,148 +1210,14 @@ extension LinuxContainer { ) async throws { try await self.state.withLock { let state = try $0.startedState("copyIn") - - var isDirectory: ObjCBool = false - guard FileManager.default.fileExists(atPath: source.path, isDirectory: &isDirectory) else { - throw ContainerizationError(.notFound, message: "copyIn: source not found '\(source.path)'") - } - let isArchive = isDirectory.boolValue - - let guestPath: URL = try await state.vm.withAgent { agent in - guard let vminitd = agent as? Vminitd else { - throw ContainerizationError(.unsupported, message: "copyIn requires Vminitd agent") - } - - return try await self.resolveCopyInGuestPath( - from: source, - to: destination, - sourceIsDirectory: isArchive, - using: vminitd - ) - } - - let port = self.hostVsockPorts.wrappingAdd(1, ordering: .relaxed).oldValue - let listener = try state.vm.listen(port) - - try await withThrowingTaskGroup(of: Void.self) { group in - group.addTask { - try await state.vm.withAgent { agent in - guard let vminitd = agent as? Vminitd else { - throw ContainerizationError(.unsupported, message: "copyIn requires Vminitd agent") - } - try await vminitd.copy( - direction: .copyIn, - guestPath: guestPath, - vsockPort: port, - mode: mode, - createParents: createParents, - isArchive: isArchive - ) - } - } - - group.addTask { - guard let conn = await listener.first(where: { _ in true }) else { - throw ContainerizationError(.internalError, message: "copyIn: vsock connection not established") - } - try listener.finish() - - try await withCheckedThrowingContinuation { (continuation: CheckedContinuation) in - self.copyQueue.async { - do { - defer { conn.closeFile() } - - if isArchive { - let writer = try ArchiveWriter(configuration: .init(format: .pax, filter: .gzip)) - try writer.open(fileDescriptor: conn.fileDescriptor) - try writer.archiveDirectory(source) - try writer.finishEncoding() - } else { - let srcFd = open(source.path, O_RDONLY) - guard srcFd != -1 else { - throw ContainerizationError( - .internalError, - message: "copyIn: failed to open '\(source.path)': \(String(cString: strerror(errno)))" - ) - } - defer { close(srcFd) } - - var buf = [UInt8](repeating: 0, count: chunkSize) - while true { - let n = read(srcFd, &buf, buf.count) - if n == 0 { break } - guard n > 0 else { - throw ContainerizationError( - .internalError, - message: "copyIn: read error: \(String(cString: strerror(errno)))" - ) - } - var written = 0 - while written < n { - let w = buf.withUnsafeBytes { ptr in - write(conn.fileDescriptor, ptr.baseAddress! + written, n - written) - } - guard w > 0 else { - throw ContainerizationError( - .internalError, - message: "copyIn: vsock write error: \(String(cString: strerror(errno)))" - ) - } - written += w - } - } - } - continuation.resume() - } catch { - continuation.resume(throwing: error) - } - } - } - } - - try await group.waitForAll() - } - } - } - - private func resolveCopyInGuestPath( - from source: URL, - to destination: URL, - sourceIsDirectory: Bool, - using vminitd: Vminitd - ) async throws -> URL { - let guestDestination = URL(filePath: self.root).appending(path: destination.path) - - let stat: ContainerizationOS.Stat? - do { - stat = try await vminitd.stat(path: guestDestination) - } catch let error as ContainerizationError where error.code == .notFound { - stat = nil - } - // Any other error propagates so transport and permission failures are visible. - - guard let stat else { - if destination.hasDirectoryPath && !sourceIsDirectory { - throw ContainerizationError( - .invalidArgument, - message: "destination directory does not exist: \(destination.path)" - ) - } - return guestDestination - } - - let destinationIsDirectory = (stat.mode & UInt32(S_IFMT)) == UInt32(S_IFDIR) - guard destinationIsDirectory else { - if sourceIsDirectory { - throw ContainerizationError( - .invalidArgument, - message: "cannot copy directory over existing file: \(destination.path)" - ) - } - return guestDestination + try await self.transfer(vm: state.vm).copyIn( + from: source, + to: destination, + mode: mode, + createParents: createParents, + chunkSize: chunkSize + ) } - - return guestDestination.appendingPathComponent(source.lastPathComponent) } /// Copy a file or directory from the container to the host. @@ -1367,104 +1233,24 @@ extension LinuxContainer { ) async throws { try await self.state.withLock { let state = try $0.startedState("copyOut") - - if createParents { - let parentDir = destination.deletingLastPathComponent() - try FileManager.default.createDirectory(at: parentDir, withIntermediateDirectories: true) - } - - let guestPath = URL(filePath: self.root).appending(path: source.path) - let port = self.hostVsockPorts.wrappingAdd(1, ordering: .relaxed).oldValue - let listener = try state.vm.listen(port) - - let (metadataStream, metadataCont) = AsyncStream.makeStream(of: Vminitd.CopyMetadata.self) - - try await withThrowingTaskGroup(of: Void.self) { group in - group.addTask { - defer { metadataCont.finish() } - try await state.vm.withAgent { agent in - guard let vminitd = agent as? Vminitd else { - throw ContainerizationError(.unsupported, message: "copyOut requires Vminitd agent") - } - try await vminitd.copy( - direction: .copyOut, - guestPath: guestPath, - vsockPort: port, - onMetadata: { meta in - metadataCont.yield(meta) - metadataCont.finish() - } - ) - } - } - - group.addTask { - guard let metadata = await metadataStream.first(where: { _ in true }) else { - throw ContainerizationError(.internalError, message: "copyOut: no metadata received") - } - - guard let conn = await listener.first(where: { _ in true }) else { - throw ContainerizationError(.internalError, message: "copyOut: vsock connection not established") - } - try listener.finish() - - try await withCheckedThrowingContinuation { (continuation: CheckedContinuation) in - self.copyQueue.async { - do { - defer { conn.closeFile() } - - if metadata.isArchive { - try FileManager.default.createDirectory(at: destination, withIntermediateDirectories: true) - let fh = FileHandle(fileDescriptor: dup(conn.fileDescriptor), closeOnDealloc: true) - let reader = try ArchiveReader(format: .pax, filter: .gzip, fileHandle: fh) - _ = try reader.extractContents(to: destination) - } else { - let destFd = open(destination.path, O_WRONLY | O_CREAT | O_TRUNC, 0o644) - guard destFd != -1 else { - throw ContainerizationError( - .internalError, - message: "copyOut: failed to open '\(destination.path)': \(String(cString: strerror(errno)))" - ) - } - defer { close(destFd) } - - var buf = [UInt8](repeating: 0, count: chunkSize) - while true { - let n = read(conn.fileDescriptor, &buf, buf.count) - if n == 0 { break } - guard n > 0 else { - throw ContainerizationError( - .internalError, - message: "copyOut: vsock read error: \(String(cString: strerror(errno)))" - ) - } - var written = 0 - while written < n { - let w = buf.withUnsafeBytes { ptr in - write(destFd, ptr.baseAddress! + written, n - written) - } - guard w > 0 else { - throw ContainerizationError( - .internalError, - message: "copyOut: write error: \(String(cString: strerror(errno)))" - ) - } - written += w - } - } - } - continuation.resume() - } catch { - continuation.resume(throwing: error) - } - } - } - } - - try await group.waitForAll() - } + try await self.transfer(vm: state.vm).copyOut( + from: source, + to: destination, + createParents: createParents, + chunkSize: chunkSize + ) } } + + /// A transfer against this container's filesystem, on a port of its own. + private func transfer(vm: any VirtualMachineInstance) -> GuestFileTransfer { + GuestFileTransfer( + vm: vm, + guestRoot: self.root, + port: self.hostVsockPorts.wrappingAdd(1, ordering: .relaxed).oldValue, + queue: self.copyQueue + ) + } } extension VirtualMachineInstance { diff --git a/Sources/Containerization/LinuxPod.swift b/Sources/Containerization/LinuxPod.swift index 140f01ab..197f29bc 100644 --- a/Sources/Containerization/LinuxPod.swift +++ b/Sources/Containerization/LinuxPod.swift @@ -219,6 +219,9 @@ public final class LinuxPod: Sendable { // the host. private let guestVsockPorts: Atomic + // Where the blocking reads and writes of a file transfer run. + private let copyQueue = DispatchQueue(label: "com.apple.containerization.copy") + private struct State: Sendable { var phase: Phase var containers: [String: PodContainer] @@ -1355,6 +1358,82 @@ extension LinuxPod { } } + /// Default chunk size for file transfers (1MiB). + public static let defaultCopyChunkSize = GuestFileTransfer.defaultChunkSize + + /// Copy a file or directory from the host into a container in the pod. + /// + /// Data transfer happens over a dedicated vsock connection. For + /// directories, the source is archived as tar+gzip and streamed directly + /// through vsock without intermediate temp files. + public func copyIn( + _ containerID: String, + from source: URL, + to destination: URL, + mode: UInt32 = 0o644, + createParents: Bool = true, + chunkSize: Int = defaultCopyChunkSize + ) async throws { + try await self.state.withLock { state in + try await self.transfer(containerID, state: state, operation: "copyIn").copyIn( + from: source, + to: destination, + mode: mode, + createParents: createParents, + chunkSize: chunkSize + ) + } + } + + /// Copy a file or directory from a container in the pod to the host. + /// + /// Data transfer happens over a dedicated vsock connection. For + /// directories, the guest archives the source as tar+gzip and streams it + /// directly through vsock. The host extracts the archive without + /// intermediate temp files. + public func copyOut( + _ containerID: String, + from source: URL, + to destination: URL, + createParents: Bool = true, + chunkSize: Int = defaultCopyChunkSize + ) async throws { + try await self.state.withLock { state in + try await self.transfer(containerID, state: state, operation: "copyOut").copyOut( + from: source, + to: destination, + createParents: createParents, + chunkSize: chunkSize + ) + } + } + + /// A transfer against one container's filesystem, on a port of its own. + private func transfer(_ containerID: String, state: State, operation: String) throws -> GuestFileTransfer { + let createdState = try state.phase.createdState(operation) + + guard let container = state.containers[containerID] else { + throw ContainerizationError( + .notFound, + message: "container \(containerID) not found in pod" + ) + } + + guard container.state == .started else { + throw ContainerizationError( + .invalidState, + message: "container \(containerID) must be started to copy files" + ) + } + + return GuestFileTransfer( + vm: createdState.vm, + guestRoot: Self.guestRootfsPath(containerID), + port: self.hostVsockPorts.wrappingAdd(1, ordering: .relaxed).oldValue, + queue: self.copyQueue + ) + } + /// Close a container's standard input to signal no more input is arriving. public func closeContainerStdin(_ containerID: String) async throws { try await self.state.withLock { state in diff --git a/Sources/Integration/PodTests.swift b/Sources/Integration/PodTests.swift index 19854194..5703bde1 100644 --- a/Sources/Integration/PodTests.swift +++ b/Sources/Integration/PodTests.swift @@ -2477,4 +2477,88 @@ extension IntegrationSuite { } } #endif + + /// A file copied into one container in a pod arrives in that container and + /// nowhere else, and comes back out with what it held. + func testPodCopyRoundTrip() async throws { + let id = "test-pod-copy-round-trip" + + let bs = try await bootstrap(id) + let pod = try LinuxPod(id, vmm: bs.vmm) { config in + config.cpus = 4 + config.memoryInBytes = 1024.mib() + config.bootLog = bs.bootLog + } + + let testContent = "Hello from the host! This is a pod copy test." + let hostDirectory = FileManager.default.uniqueTemporaryDirectory(create: true) + let hostFile = hostDirectory.appendingPathComponent("test-input.txt") + try testContent.write(to: hostFile, atomically: true, encoding: .utf8) + + try await pod.addContainer("holder", rootfs: try cloneRootfs(bs.rootfs, testID: id, containerID: "holder")) { config in + config.process.arguments = ["sleep", "100"] + } + try await pod.addContainer("bystander", rootfs: try cloneRootfs(bs.rootfs, testID: id, containerID: "bystander")) { config in + config.process.arguments = ["sleep", "100"] + } + + do { + try await pod.create() + try await pod.startContainer("holder") + try await pod.startContainer("bystander") + + try await pod.copyIn( + "holder", + from: hostFile, + to: URL(filePath: "/tmp/copied-file.txt") + ) + + let buffer = BufferWriter() + let exec = try await pod.execInContainer("holder", processID: "verify-copy") { config in + config.arguments = ["cat", "/tmp/copied-file.txt"] + config.stdout = buffer + } + try await exec.start() + let status = try await exec.wait() + try await exec.delete() + + guard status.exitCode == 0 else { + throw IntegrationError.assert(msg: "cat in the container copied into failed with status \(status)") + } + guard String(data: buffer.data, encoding: .utf8) == testContent else { + throw IntegrationError.assert( + msg: "expected '\(testContent)', got '\(String(data: buffer.data, encoding: .utf8) ?? "nil")'") + } + + // The containers share a machine and not a filesystem, so the copy + // reached one of them alone. + let bystander = try await pod.execInContainer("bystander", processID: "verify-absent") { config in + config.arguments = ["test", "-e", "/tmp/copied-file.txt"] + } + try await bystander.start() + let bystanderStatus = try await bystander.wait() + try await bystander.delete() + + guard bystanderStatus.exitCode != 0 else { + throw IntegrationError.assert(msg: "the copy reached a container it was not addressed to") + } + + let returned = hostDirectory.appendingPathComponent("test-output.txt") + try await pod.copyOut( + "holder", + from: URL(filePath: "/tmp/copied-file.txt"), + to: returned + ) + + let returnedContent = try String(contentsOf: returned, encoding: .utf8) + guard returnedContent == testContent else { + throw IntegrationError.assert(msg: "expected '\(testContent)' back, got '\(returnedContent)'") + } + + try await pod.stop() + } catch { + try? await pod.stop() + throw error + } + } } diff --git a/Sources/Integration/Suite.swift b/Sources/Integration/Suite.swift index bd4ec656..769ccc12 100644 --- a/Sources/Integration/Suite.swift +++ b/Sources/Integration/Suite.swift @@ -559,6 +559,7 @@ struct IntegrationSuite: AsyncParsableCommand { Test("pod memory events OOM kill", testPodMemoryEventsOOMKill), Test("pod container resource limits", testPodContainerResourceLimits), Test("pod container filesystem isolation", testPodContainerFilesystemIsolation), + Test("pod copy round trip", testPodCopyRoundTrip), Test("pod container PID namespace isolation", testPodContainerPIDNamespaceIsolation), Test("pod container independent resource limits", testPodContainerIndependentResourceLimits), Test("pod shared PID namespace", testPodSharedPIDNamespace), From 8f7a007c5d83298f79b27a824e4882f835f8b1a0 Mon Sep 17 00:00:00 2001 From: Aaron Paterson Date: Fri, 7 Aug 2026 20:42:42 +0000 Subject: [PATCH 11/16] Give a pod the machine it asked for A container with a machine of its own is given the size it asked for plus the guest agent's, so a container that asks for a gibibyte has a gibibyte and the agent runs beside it. A pod hands its size to the machine as it stands, so the agent comes out of what the pod asked for and its containers are left with less: a pod given twelve gibibytes reports eleven and a half. The pod's size and the agent's are added the same way, and what a pod was given is what its containers have. --- Sources/Containerization/LinuxPod.swift | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/Sources/Containerization/LinuxPod.swift b/Sources/Containerization/LinuxPod.swift index 197f29bc..fab44f0f 100644 --- a/Sources/Containerization/LinuxPod.swift +++ b/Sources/Containerization/LinuxPod.swift @@ -57,6 +57,13 @@ public final class LinuxPod: Sendable { public var interfaces: [any Interface] = [] /// Whether nested virtualization should be turned on for the pod. public var virtualization: Bool = false + /// Additional CPU cores to allocate for the virtual machine on top of + /// the pod's configured `cpus`, so what the pod was given is what its + /// containers have rather than what the guest agent leaves of it. + public var cpuOverhead: Int = 1 + /// Additional memory in bytes to allocate for the virtual machine on + /// top of the pod's configured `memoryInBytes`. + public var memoryOverhead: UInt64 = 128.mib() /// Optional file path to store serial boot logs. public var bootLog: BootLog? /// Whether containers in the pod should share a PID namespace. @@ -669,9 +676,12 @@ extension LinuxPod { return false } + // The machine carries the guest agent as well as the containers, + // so it is given the pod's size and the agent's on top; what the + // pod was given is then what its containers have. var vmConfig = VMConfiguration( - cpus: self.config.cpus, - memoryInBytes: self.config.memoryInBytes, + cpus: self.config.cpus + self.config.cpuOverhead, + memoryInBytes: self.config.memoryInBytes + self.config.memoryOverhead, interfaces: self.config.interfaces, storage: machineStorage, bootLog: self.config.bootLog, @@ -681,9 +691,8 @@ extension LinuxPod { let creationConfig = StandardVMConfig(configuration: vmConfig) let vm = try await self.vmm.create(config: creationConfig) let relayManager = UnixSocketRelayManager(vm: vm) - try await vm.start() - do { + try await vm.start() let containers = state.containers let shareProcessNamespace = self.config.shareProcessNamespace let pauseProcessHolder = Mutex(nil) From ea2a9a221eaeacda38c6b9dc14375873d56e0b8e Mon Sep 17 00:00:00 2001 From: Aaron Paterson Date: Fri, 7 Aug 2026 21:10:11 +0000 Subject: [PATCH 12/16] Let a pod's containers be given a writable layer A container with a machine of its own can be given a block device to write into, with its image mounted read-only beneath, so the image is left as it is and every write lands in the layer. The same container placed in a pod has nowhere to be given one. The layer is attached with the container's other block devices, mounted in the guest as the upper layer of an overlay over the image, dropped from the mounts the runtime spec carries, and unmounted with the container. A pod's containers take one whether they were placed before the machine booted or hotplugged into it afterwards. --- .../Containerization/CHHotplugProvider.swift | 4 +- .../CHVirtualMachineInstance.swift | 4 +- .../Containerization/HotplugProvider.swift | 4 +- Sources/Containerization/LinuxContainer.swift | 43 +------ Sources/Containerization/LinuxPod.swift | 76 ++++++++++-- Sources/Containerization/OverlayRootfs.swift | 64 ++++++++++ .../VZVirtualMachineInstance.swift | 4 +- .../VirtualMachineInstance.swift | 6 +- Sources/Integration/PodTests.swift | 117 ++++++++++++++++++ Sources/Integration/Suite.swift | 2 + 10 files changed, 268 insertions(+), 56 deletions(-) create mode 100644 Sources/Containerization/OverlayRootfs.swift diff --git a/Sources/Containerization/CHHotplugProvider.swift b/Sources/Containerization/CHHotplugProvider.swift index 380cba76..e6a33d38 100644 --- a/Sources/Containerization/CHHotplugProvider.swift +++ b/Sources/Containerization/CHHotplugProvider.swift @@ -150,12 +150,12 @@ final class CHHotplugProvider: HotplugProvider { } } - func registerMounts(id: String, rootfs: AttachedFilesystem, additionalMounts: [Mount]) throws { + func registerMounts(id: String, rootfs: AttachedFilesystem, writableLayer: AttachedFilesystem?, additionalMounts: [Mount]) throws { var mounts: [AttachedFilesystem] = [] for mount in additionalMounts { mounts.append(try AttachedFilesystem(mount: mount, allocator: allocator)) } - let container = ContainerAttachments(rootfs: rootfs, mounts: mounts) + let container = ContainerAttachments(rootfs: rootfs, writableLayer: writableLayer, mounts: mounts) _storage.withLock { $0.containers[id] = container } diff --git a/Sources/Containerization/CHVirtualMachineInstance.swift b/Sources/Containerization/CHVirtualMachineInstance.swift index 1704ce92..11d7e0a3 100644 --- a/Sources/Containerization/CHVirtualMachineInstance.swift +++ b/Sources/Containerization/CHVirtualMachineInstance.swift @@ -490,8 +490,8 @@ extension CHVirtualMachineInstance: VirtualMachineInstance { try await hotplug.releaseVirtioFS(id: id) } - public func registerMounts(id: String, rootfs: AttachedFilesystem, additionalMounts: [Mount]) throws { - try hotplug.registerMounts(id: id, rootfs: rootfs, additionalMounts: additionalMounts) + public func registerMounts(id: String, rootfs: AttachedFilesystem, writableLayer: AttachedFilesystem?, additionalMounts: [Mount]) throws { + try hotplug.registerMounts(id: id, rootfs: rootfs, writableLayer: writableLayer, additionalMounts: additionalMounts) } } diff --git a/Sources/Containerization/HotplugProvider.swift b/Sources/Containerization/HotplugProvider.swift index 0835077f..94fad4b4 100644 --- a/Sources/Containerization/HotplugProvider.swift +++ b/Sources/Containerization/HotplugProvider.swift @@ -30,8 +30,10 @@ public protocol HotplugProvider: Sendable { /// - Parameters: /// - id: The container ID /// - rootfs: The rootfs attachment from hotplug + /// - writableLayer: The container's writable layer attachment when it + /// has one /// - additionalMounts: Additional mounts to register - func registerMounts(id: String, rootfs: AttachedFilesystem, additionalMounts: [Mount]) throws + func registerMounts(id: String, rootfs: AttachedFilesystem, writableLayer: AttachedFilesystem?, additionalMounts: [Mount]) throws /// Release a hotplug device. /// - Parameter id: The container ID who should be released diff --git a/Sources/Containerization/LinuxContainer.swift b/Sources/Containerization/LinuxContainer.swift index eca34a1a..ce57488f 100644 --- a/Sources/Containerization/LinuxContainer.swift +++ b/Sources/Containerization/LinuxContainer.swift @@ -575,47 +575,16 @@ extension LinuxContainer { rootfsPath: String, agent: VirtualMachineAgent ) async throws { - let rootfsAttachment = attached.rootfs - if let writableAttachment = attached.writableLayer { - // Set up overlayfs with image as lower layer and writable layer as upper. - let lowerPath = "/run/container/\(self.id)/lower" - let upperMountPath = "/run/container/\(self.id)/upper" - let upperPath = "/run/container/\(self.id)/upper/diff" - let workPath = "/run/container/\(self.id)/upper/work" - - // Mount the image (lower layer) as read-only. - var lowerMount = rootfsAttachment.to - lowerMount.destination = lowerPath - if !lowerMount.options.contains("ro") { - lowerMount.options.append("ro") - } - try await agent.mount(lowerMount) - - // Mount the writable layer. - var upperMount = writableAttachment.to - upperMount.destination = upperMountPath - try await agent.mount(upperMount) - - // Create the upper and work directories inside the writable layer. - try await agent.mkdir(path: upperPath, all: true, perms: 0o755) - try await agent.mkdir(path: workPath, all: true, perms: 0o755) - - // Mount the overlay. - let overlayMount = ContainerizationOCI.Mount( - type: "overlay", - source: "overlay", - destination: rootfsPath, - options: [ - "lowerdir=\(lowerPath)", - "upperdir=\(upperPath)", - "workdir=\(workPath)", - ] + try await agent.mountOverlayRootfs( + containerID: self.id, + rootfsAttachment: attached.rootfs, + writableAttachment: writableAttachment, + rootfsPath: rootfsPath ) - try await agent.mount(overlayMount) } else { // No writable layer. Mount rootfs directly. - var rootfs = rootfsAttachment.to + var rootfs = attached.rootfs.to rootfs.destination = rootfsPath try await agent.mount(rootfs) } diff --git a/Sources/Containerization/LinuxPod.swift b/Sources/Containerization/LinuxPod.swift index fab44f0f..76967821 100644 --- a/Sources/Containerization/LinuxPod.swift +++ b/Sources/Containerization/LinuxPod.swift @@ -202,6 +202,7 @@ public final class LinuxPod: Sendable { private struct PodContainer: Sendable { let id: String let rootfs: Mount + let writableLayer: Mount? let config: ContainerConfiguration var state: ContainerState var process: LinuxProcess? @@ -329,7 +330,7 @@ public final class LinuxPod: Sendable { ) } - private func generateRuntimeSpec(containerID: String, config: ContainerConfiguration, rootfs: Mount) -> Spec { + private func generateRuntimeSpec(containerID: String, config: ContainerConfiguration, rootfs: Mount, writableLayer: Mount? = nil) -> Spec { var spec = Self.createDefaultRuntimeSpec(containerID, podID: self.id) // Process configuration @@ -354,7 +355,7 @@ public final class LinuxPod: Sendable { // If the rootfs was requested as read-only, set it in the OCI spec. // We let the OCI runtime remount as ro, instead of doing it originally. - spec.root?.readonly = rootfs.options.contains("ro") + spec.root?.readonly = rootfs.options.contains("ro") && writableLayer == nil // Resource limits (if specified) if let cpus = config.cpus, cpus > 0 { @@ -413,9 +414,14 @@ extension LinuxPod { /// When called before `create()`, the container is registered for setup during VM creation. /// When called after `create()`, the container is hotplugged into the running VM. /// If the underlying VMM does not support hotplug, an error is thrown. + /// - Parameters: + /// - writableLayer: Optional writable layer mount. When provided, an overlayfs is used with + /// the container's rootfs as the lower layer and this as the upper layer, so all writes + /// go to this layer instead of the rootfs. public func addContainer( _ id: String, rootfs: Mount, + writableLayer: Mount? = nil, configuration: @Sendable @escaping (inout ContainerConfiguration) throws -> Void ) async throws { guard id.count <= Self.maxIDLength else { @@ -424,6 +430,14 @@ extension LinuxPod { message: "container id length \(id.count) exceeds maximum of \(Self.maxIDLength) characters" ) } + if let writableLayer { + guard writableLayer.isBlock else { + throw ContainerizationError( + .invalidArgument, + message: "writableLayer must be a block device" + ) + } + } try await self.state.withLock { state in guard state.containers[id] == nil else { throw ContainerizationError( @@ -451,6 +465,7 @@ extension LinuxPod { state.containers[id] = PodContainer( id: id, rootfs: rootfs, + writableLayer: writableLayer, config: config, state: .registered, process: nil, @@ -460,6 +475,9 @@ extension LinuxPod { case .created(let createdState): let vm = createdState.vm + // Strip "ro" as create() does: readonly is expressed through + // the OCI spec's root.readonly field and a remount in vmexec + // after setup completes, so the device attaches writable. var modifiedRootfs = rootfs modifiedRootfs.options.removeAll(where: { $0 == "ro" }) @@ -467,6 +485,13 @@ extension LinuxPod { var updatedFileMountContext = fileMountContext do { + // The writable layer is a block device like the rootfs, + // attached alongside it so the overlay has both layers. + var writableAttachment: AttachedFilesystem? + if let writableLayer { + writableAttachment = try await vm.hotplug(writableLayer, id: id) + } + let virtioFSMounts = fileMountContext.transformedMounts.filter { if case .virtiofs(_) = $0.runtimeOptions { return true } return false @@ -477,13 +502,23 @@ extension LinuxPod { let agent = try await vm.dialAgent() do { - var mount = attachment.to - mount.destination = Self.guestRootfsPath(id) - try await agent.mount(mount) + if let writableAttachment { + try await agent.mountOverlayRootfs( + containerID: id, + rootfsAttachment: attachment, + writableAttachment: writableAttachment, + rootfsPath: Self.guestRootfsPath(id) + ) + } else { + var mount = attachment.to + mount.destination = Self.guestRootfsPath(id) + try await agent.mount(mount) + } - // Filter out shared mounts — those are handled separately as - // pod volume bind mounts. Without it here, a container added to an - // already-created would add a duplicated mount into the shared VM. + // Shared mounts are handled separately as pod volume + // bind mounts; without the filter here, a container + // added to an already-created pod would add a + // duplicated mount into the shared VM. let nonSharedMounts = fileMountContext.transformedMounts.filter { if case .shared = $0.runtimeOptions { return false } return true @@ -491,6 +526,7 @@ extension LinuxPod { try vm.registerMounts( id: id, rootfs: attachment, + writableLayer: writableAttachment, additionalMounts: nonSharedMounts ) @@ -591,6 +627,7 @@ extension LinuxPod { state.containers[id] = PodContainer( id: id, rootfs: rootfs, + writableLayer: writableLayer, config: config, state: .created, process: nil, @@ -630,6 +667,7 @@ extension LinuxPod { } machineStorage.containers[id] = ContainerMounts( rootfs: modifiedRootfs, + writableLayer: container.writableLayer, mounts: containerMounts ) } @@ -809,6 +847,15 @@ extension LinuxPod { guard let attached = vm.storage.containers[container.id] else { throw ContainerizationError(.notFound, message: "rootfs mount not found for container \(container.id)") } + if let writableAttachment = attached.writableLayer { + try await agent.mountOverlayRootfs( + containerID: container.id, + rootfsAttachment: attached.rootfs, + writableAttachment: writableAttachment, + rootfsPath: Self.guestRootfsPath(container.id) + ) + continue + } var rootfs = attached.rootfs.to rootfs.destination = Self.guestRootfsPath(container.id) try await agent.mount(rootfs) @@ -936,7 +983,7 @@ extension LinuxPod { let agent = try await createdState.vm.dialAgent() do { - var spec = self.generateRuntimeSpec(containerID: containerID, config: container.config, rootfs: container.rootfs) + var spec = self.generateRuntimeSpec(containerID: containerID, config: container.config, rootfs: container.rootfs, writableLayer: container.writableLayer) // We don't need the rootfs, nor do OCI runtimes want it included. // Also filter out file mount holding directories - we mount those separately under /run. // Transform virtiofs mounts to bind mounts from /run/virtiofs/{tag} @@ -1101,12 +1148,21 @@ extension LinuxPod { try await process.kill(.kill) try await process.wait(timeoutInSeconds: 3) + let hasWritableLayer = container.writableLayer != nil try await createdState.vm.withAgent { agent in // Unmount the rootfs try await agent.umount( path: Self.guestRootfsPath(containerID), flags: 0 ) + + // If we have a writable layer, we also need to unmount the lower and upper layers. + if hasWritableLayer { + let upperPath = "/run/container/\(containerID)/upper" + let lowerPath = "/run/container/\(containerID)/lower" + try await agent.umount(path: upperPath, flags: 0) + try await agent.umount(path: lowerPath, flags: 0) + } } // Release the hotplug device and virtiofs shares so they can be reused by new containers @@ -1261,7 +1317,7 @@ extension LinuxPod { ) } - var spec = self.generateRuntimeSpec(containerID: containerID, config: container.config, rootfs: container.rootfs) + var spec = self.generateRuntimeSpec(containerID: containerID, config: container.config, rootfs: container.rootfs, writableLayer: container.writableLayer) // Inherit environment variables, working directory, user, capabilities, rlimits from container process. // Reset: process arguments, terminal, stdio as these are not supposed to be inherited. var config = container.config.process diff --git a/Sources/Containerization/OverlayRootfs.swift b/Sources/Containerization/OverlayRootfs.swift new file mode 100644 index 00000000..9af76763 --- /dev/null +++ b/Sources/Containerization/OverlayRootfs.swift @@ -0,0 +1,64 @@ +//===----------------------------------------------------------------------===// +// Copyright © 2026 Apple Inc. and the Containerization project authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//===----------------------------------------------------------------------===// + +import ContainerizationOCI + +extension VirtualMachineAgent { + /// Mount a container's rootfs as an overlay, with the image as the lower + /// layer and the container's writable layer as the upper, so that writes + /// land in the layer and the image stays as it is. + func mountOverlayRootfs( + containerID: String, + rootfsAttachment: AttachedFilesystem, + writableAttachment: AttachedFilesystem, + rootfsPath: String + ) async throws { + let lowerPath = "/run/container/\(containerID)/lower" + let upperMountPath = "/run/container/\(containerID)/upper" + let upperPath = "/run/container/\(containerID)/upper/diff" + let workPath = "/run/container/\(containerID)/upper/work" + + // Mount the image (lower layer) as read-only. + var lowerMount = rootfsAttachment.to + lowerMount.destination = lowerPath + if !lowerMount.options.contains("ro") { + lowerMount.options.append("ro") + } + try await self.mount(lowerMount) + + // Mount the writable layer. + var upperMount = writableAttachment.to + upperMount.destination = upperMountPath + try await self.mount(upperMount) + + // Create the upper and work directories inside the writable layer. + try await self.mkdir(path: upperPath, all: true, perms: 0o755) + try await self.mkdir(path: workPath, all: true, perms: 0o755) + + // Mount the overlay. + let overlayMount = ContainerizationOCI.Mount( + type: "overlay", + source: "overlay", + destination: rootfsPath, + options: [ + "lowerdir=\(lowerPath)", + "upperdir=\(upperPath)", + "workdir=\(workPath)", + ] + ) + try await self.mount(overlayMount) + } +} diff --git a/Sources/Containerization/VZVirtualMachineInstance.swift b/Sources/Containerization/VZVirtualMachineInstance.swift index 69828ec9..135879e0 100644 --- a/Sources/Containerization/VZVirtualMachineInstance.swift +++ b/Sources/Containerization/VZVirtualMachineInstance.swift @@ -325,9 +325,9 @@ extension VZVirtualMachineInstance: VirtualMachineInstance { return try await hotplugProvider.hotplug(block, id: id) } - public func registerMounts(id: String, rootfs: AttachedFilesystem, additionalMounts: [Mount]) throws { + public func registerMounts(id: String, rootfs: AttachedFilesystem, writableLayer: AttachedFilesystem?, additionalMounts: [Mount]) throws { guard let hotplugProvider else { return } - try hotplugProvider.registerMounts(id: id, rootfs: rootfs, additionalMounts: additionalMounts) + try hotplugProvider.registerMounts(id: id, rootfs: rootfs, writableLayer: writableLayer, additionalMounts: additionalMounts) } public func releaseHotplug(id: String) async throws { diff --git a/Sources/Containerization/VirtualMachineInstance.swift b/Sources/Containerization/VirtualMachineInstance.swift index b9340612..da7935d5 100644 --- a/Sources/Containerization/VirtualMachineInstance.swift +++ b/Sources/Containerization/VirtualMachineInstance.swift @@ -78,8 +78,10 @@ public protocol VirtualMachineInstance: Sendable { /// so they can be found when building the container's OCI spec. /// - Parameter id: The container ID /// - Parameter rootfs: The rootfs attachment from hotplug + /// - Parameter writableLayer: The container's writable layer attachment when it + /// has one /// - Parameter additionalMounts: Additional mounts (like /proc, /sys) to register - func registerMounts(id: String, rootfs: AttachedFilesystem, additionalMounts: [Mount]) throws + func registerMounts(id: String, rootfs: AttachedFilesystem, writableLayer: AttachedFilesystem?, additionalMounts: [Mount]) throws /// Release a hotplug device. /// This should be called when a hotplugged container is stopped or fails to start. @@ -107,7 +109,7 @@ extension VirtualMachineInstance { public func hotplug(_ block: Mount, id: String) async throws -> AttachedFilesystem { throw ContainerizationError(.unsupported, message: "hotplug not supported") } - public func registerMounts(id: String, rootfs: AttachedFilesystem, additionalMounts: [Mount]) throws { + public func registerMounts(id: String, rootfs: AttachedFilesystem, writableLayer: AttachedFilesystem?, additionalMounts: [Mount]) throws { // no-op default } public func releaseHotplug(id: String) async throws { diff --git a/Sources/Integration/PodTests.swift b/Sources/Integration/PodTests.swift index 5703bde1..2c0dffc2 100644 --- a/Sources/Integration/PodTests.swift +++ b/Sources/Integration/PodTests.swift @@ -2478,6 +2478,123 @@ extension IntegrationSuite { } #endif + /// A container in a pod given a writable layer writes into it, and the + /// image it was built from is left as it is for the pod's others. + /// Add a container with a writable layer to a pod whose machine is + /// already running: the overlay assembles from the two disks attached + /// while it runs, and writes land in the layer. + func testPodHotplugWritableLayer() async throws { + let id = "test-pod-hotplug-writable-layer" + let bs = try await bootstrap(id) + + let pod = try LinuxPod(id, vmm: bs.vmm) { config in + config.cpus = 4 + config.memoryInBytes = 1024.mib() + config.bootLog = bs.bootLog + } + + try await pod.addContainer("seed", rootfs: try cloneRootfs(bs.rootfs, testID: id, containerID: "seed")) { config in + config.process.arguments = ["/bin/sleep", "infinity"] + } + + try await pod.create() + + let writableLayerPath = Self.testDir.appending(component: "\(id)-writable.ext4") + try? FileManager.default.removeItem(at: writableLayerPath) + let filesystem = try EXT4.Formatter(FilePath(writableLayerPath.absolutePath()), minDiskSize: 512.mib()) + try filesystem.close() + let writableLayer = Mount.block( + format: "ext4", + source: writableLayerPath.absolutePath(), + destination: "/", + options: [] + ) + + let buffer = BufferWriter() + try await pod.addContainer( + "hot", + rootfs: try cloneRootfs(bs.rootfs, testID: id, containerID: "hot"), + writableLayer: writableLayer + ) { config in + config.process.arguments = ["/bin/sh", "-c", "echo 'written into a layer added while running' > /written && cat /written"] + config.process.stdout = buffer + } + + do { + try await pod.startContainer("hot") + let status = try await pod.waitContainer("hot") + + try await pod.stopContainer("hot") + try await pod.stop() + + guard status.exitCode == 0 else { + throw IntegrationError.assert(msg: "hot container status \(status) != 0") + } + let expected = "written into a layer added while running" + guard let output = String(data: buffer.data, encoding: .utf8), + output.trimmingCharacters(in: .whitespacesAndNewlines) == expected + else { + throw IntegrationError.assert( + msg: "expected '\(expected)', got '\(String(data: buffer.data, encoding: .utf8) ?? "nil")'") + } + } catch { + try? await pod.stop() + throw error + } + } + + func testPodWritableLayer() async throws { + let id = "test-pod-writable-layer" + + let bs = try await bootstrap(id) + let pod = try LinuxPod(id, vmm: bs.vmm) { config in + config.cpus = 4 + config.memoryInBytes = 1024.mib() + config.bootLog = bs.bootLog + } + + let writableLayerPath = Self.testDir.appending(component: "\(id)-writable.ext4") + try? FileManager.default.removeItem(at: writableLayerPath) + let filesystem = try EXT4.Formatter(FilePath(writableLayerPath.absolutePath()), minDiskSize: 512.mib()) + try filesystem.close() + let writableLayer = Mount.block( + format: "ext4", + source: writableLayerPath.absolutePath(), + destination: "/", + options: [] + ) + + let buffer = BufferWriter() + try await pod.addContainer( + "layered", + rootfs: try cloneRootfs(bs.rootfs, testID: id, containerID: "layered"), + writableLayer: writableLayer + ) { config in + config.process.arguments = ["/bin/sh", "-c", "echo 'writable layer test' > /tmp/testfile && cat /tmp/testfile"] + config.process.stdout = buffer + } + + do { + try await pod.create() + try await pod.startContainer("layered") + let status = try await pod.waitContainer("layered") + try await pod.stop() + + guard status.exitCode == 0 else { + throw IntegrationError.assert(msg: "process failed with status \(status)") + } + guard let output = String(data: buffer.data, encoding: .utf8) else { + throw IntegrationError.assert(msg: "failed to convert stdout to UTF8") + } + guard output.trimmingCharacters(in: .whitespacesAndNewlines) == "writable layer test" else { + throw IntegrationError.assert(msg: "unexpected output: \(output)") + } + } catch { + try? await pod.stop() + throw error + } + } + /// A file copied into one container in a pod arrives in that container and /// nowhere else, and comes back out with what it held. func testPodCopyRoundTrip() async throws { diff --git a/Sources/Integration/Suite.swift b/Sources/Integration/Suite.swift index 769ccc12..002e30a4 100644 --- a/Sources/Integration/Suite.swift +++ b/Sources/Integration/Suite.swift @@ -560,6 +560,7 @@ struct IntegrationSuite: AsyncParsableCommand { Test("pod container resource limits", testPodContainerResourceLimits), Test("pod container filesystem isolation", testPodContainerFilesystemIsolation), Test("pod copy round trip", testPodCopyRoundTrip), + Test("pod writable layer", testPodWritableLayer), Test("pod container PID namespace isolation", testPodContainerPIDNamespaceIsolation), Test("pod container independent resource limits", testPodContainerIndependentResourceLimits), Test("pod shared PID namespace", testPodSharedPIDNamespace), @@ -651,6 +652,7 @@ struct IntegrationSuite: AsyncParsableCommand { let linuxOnlyTests: [Test] = [ Test("pod hotplug block rootfs", testPodHotplugBlockRootfs), Test("pod hotplug virtiofs rootfs", testPodHotplugVirtiofsRootfs), + Test("pod hotplug writable layer", testPodHotplugWritableLayer), ] let tests: [Test] = crossPlatformTests + linuxOnlyTests #endif From 9adf57c175cc721fae870d0f3659523b0fe2e415 Mon Sep 17 00:00:00 2001 From: Aaron Paterson Date: Sat, 8 Aug 2026 02:01:02 +0000 Subject: [PATCH 13/16] Name the project the guest file transfer belongs to The file carried the sibling repository's copyright line, so the license header check found no header it recognized. --- Sources/Containerization/GuestFileTransfer.swift | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Sources/Containerization/GuestFileTransfer.swift b/Sources/Containerization/GuestFileTransfer.swift index d508c571..5eeb039d 100644 --- a/Sources/Containerization/GuestFileTransfer.swift +++ b/Sources/Containerization/GuestFileTransfer.swift @@ -1,5 +1,5 @@ //===----------------------------------------------------------------------===// -// Copyright © 2026 Apple Inc. and the container project authors. +// Copyright © 2026 Apple Inc. and the Containerization project authors. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. From bf57208a9871a0079fa6803f3f88c484f2e7b2a9 Mon Sep 17 00:00:00 2001 From: Aaron Paterson Date: Mon, 10 Aug 2026 05:40:19 +0000 Subject: [PATCH 14/16] Let a pod give a container's place back Stopping a container tears down what it was running and keeps its place: the name still answers for it, and placing another container under it is refused. Removal is the separate act the runtime specification names for giving the place up, taken once the container has stopped, so a name can run again in a machine that outlives what it last ran. https://github.com/kubernetes/cri-api/blob/master/pkg/apis/runtime/v1/api.proto --- Sources/Containerization/LinuxPod.swift | 29 +++++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/Sources/Containerization/LinuxPod.swift b/Sources/Containerization/LinuxPod.swift index 76967821..9d98d79f 100644 --- a/Sources/Containerization/LinuxPod.swift +++ b/Sources/Containerization/LinuxPod.swift @@ -1189,6 +1189,35 @@ extension LinuxPod { } } + /// Take a container out of the pod, so its name is free to place again. + /// + /// Stopping a container tears down what it was running and keeps its + /// place; the name still answers for it, and placing another container + /// under it is refused. Removal is the separate act the runtime + /// specification names for giving the place up, taken once the container + /// has stopped. A container that is running keeps its place and this + /// call refuses it. + /// https://github.com/kubernetes/cri-api/blob/master/pkg/apis/runtime/v1/api.proto + public func removeContainer(_ containerID: String) async throws { + try await self.state.withLock { state in + guard let container = state.containers[containerID] else { + throw ContainerizationError( + .notFound, + message: "container \(containerID) not found in pod" + ) + } + switch container.state { + case .registered, .stopped, .errored: + state.containers[containerID] = nil + default: + throw ContainerizationError( + .invalidState, + message: "container \(containerID) must stop before it is removed" + ) + } + } + } + /// Stop the pod's VM and all containers. public func stop() async throws { try await self.state.withLock { state in From a72170e0fdd7d1f7db2000fdf6a98ef7e0dc8ce1 Mon Sep 17 00:00:00 2001 From: Aaron Paterson Date: Sun, 2 Aug 2026 01:22:56 +0000 Subject: [PATCH 15/16] Let a caller set the runtime spec's hooks The runtime specification's hooks have a field on the spec type and no way to reach it: the configuration has nowhere to put them and the spec the container is built from never carries any. They travel intact once set, and the bundle's config.json contains them. Whether anything runs them depends on the runtime the container is launched under, which the field's documentation says. --- Sources/Containerization/LinuxContainer.swift | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/Sources/Containerization/LinuxContainer.swift b/Sources/Containerization/LinuxContainer.swift index ce57488f..0d0c7745 100644 --- a/Sources/Containerization/LinuxContainer.swift +++ b/Sources/Containerization/LinuxContainer.swift @@ -91,6 +91,16 @@ public final class LinuxContainer: Container, Sendable { /// Run the container with a minimal init process that handles signal /// forwarding and zombie reaping. public var useInit: Bool = false + /// Programs the runtime runs at points in the container's lifecycle, + /// as described by the runtime specification. + /// + /// The paths are resolved inside the guest, because that is where the + /// runtime that would run them lives. + /// + /// NOTE: these reach the bundle's config.json but are only acted on + /// when the container runs under an external OCI runtime. The default + /// launcher is not one, so they have no effect on that path. + public var hooks: ContainerizationOCI.Hooks? = nil /// Additional CPU cores to allocate for the virtual machine on top /// of the container's configured `cpus` value. public var cpuOverhead: Int = 1 @@ -405,6 +415,8 @@ public final class LinuxContainer: Container, Sendable { // Process toggles. spec.process = config.process.toOCI() + spec.hooks = config.hooks + // Wrap with init process if requested. if config.useInit { let originalArgs = spec.process?.args ?? [] From 12bf1f6840db7987e81dbd4565f806c9c66db38f Mon Sep 17 00:00:00 2001 From: Aaron Paterson Date: Sun, 2 Aug 2026 01:22:56 +0000 Subject: [PATCH 16/16] Let a pod's containers reach an OCI runtime and its hooks A container in a pod is spawned with the runtime path hardcoded to nil, so the runtime a standalone container reaches through `ociRuntimePath` is out of reach for the same container placed in a pod, and the spec it is built from carries no hooks for that runtime to run. Nothing about a pod prevents either: the runtime path is passed per process, and the pause process is the only one with a reason to stay on the default. The configuration carries both. The container's own runtime path is used when its process starts and when a process is executed in it, and its hooks travel on the spec, as they do for a standalone container. --- Sources/Containerization/LinuxPod.swift | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/Sources/Containerization/LinuxPod.swift b/Sources/Containerization/LinuxPod.swift index 9d98d79f..1dd65e07 100644 --- a/Sources/Containerization/LinuxPod.swift +++ b/Sources/Containerization/LinuxPod.swift @@ -137,6 +137,13 @@ public final class LinuxPod: Sendable { /// Run the container with a minimal init process that handles signal /// forwarding and zombie reaping. public var useInit: Bool = false + /// EXPERIMENTAL: Path in the root filesystem for the virtual + /// machine where the OCI runtime used to spawn the container lives. + public var ociRuntimePath: String? + /// Hooks the runtime specification carries for the container. An OCI + /// runtime is what runs them, so they take effect for a container + /// given an `ociRuntimePath`. + public var hooks: ContainerizationOCI.Hooks? = nil public init() {} } @@ -347,6 +354,7 @@ public final class LinuxPod: Sendable { if let hostname = config.hostname ?? self.config.hostname { spec.hostname = hostname } + spec.hooks = config.hooks // Linux toggles spec.linux?.sysctl = config.sysctl @@ -1085,7 +1093,7 @@ extension LinuxPod { containerID: containerID, spec: spec, io: stdio, - ociRuntimePath: nil, + ociRuntimePath: container.config.ociRuntimePath, agent: agent, vm: createdState.vm, logger: self.logger @@ -1370,7 +1378,7 @@ extension LinuxPod { containerID: containerID, spec: spec, io: stdio, - ociRuntimePath: nil, + ociRuntimePath: container.config.ociRuntimePath, agent: agent, vm: createdState.vm, logger: self.logger