From f10eaca613f6eef28add5d69f4fbf573ca6ed156 Mon Sep 17 00:00:00 2001 From: Aaron Paterson Date: Sat, 1 Aug 2026 18:48:34 +0000 Subject: [PATCH 01/21] Resolve test directory paths before reaping per-test files The reaper that runs at a concurrency of one deletes everything in the test directory except the unpacked rootfs it means to preserve. It held that rootfs path as FileManager reports it, under /var, and compared it against the entries of contentsOfDirectory, which reports them under /private/var, so the preserved path never matched any entry and the rootfs was deleted along with the per-test files. The unpack coordinator still held it as unpacked, so the next test opened a rootfs that was no longer there and failed with a missing file error. Resolve both sides with resolvingSymlinksInPathWithPrivate, which exists for this difference between the two views of the same directory. --- Sources/Integration/Suite.swift | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/Sources/Integration/Suite.swift b/Sources/Integration/Suite.swift index 9dae8435..e43cf372 100644 --- a/Sources/Integration/Suite.swift +++ b/Sources/Integration/Suite.swift @@ -259,12 +259,16 @@ struct IntegrationSuite: AsyncParsableCommand { // a ~2GB rootfs and a ~512MB initfs, so without reaping the dev // container fills its CoW layer in ~10 tests. if self.maxConcurrency == 1 { - let preserve = fsPath.absolutePath() + // contentsOfDirectory reports paths under /private, while testDir is + // built from FileManager's /var view of the same directory, so both + // sides are resolved before comparing. + let preserve = fsPath.resolvingSymlinksInPathWithPrivate().absolutePath() if let entries = try? FileManager.default.contentsOfDirectory( at: Self.testDir, includingPropertiesForKeys: nil ) { - for url in entries where url.absolutePath() != preserve { + for url in entries + where url.resolvingSymlinksInPathWithPrivate().absolutePath() != preserve { try? FileManager.default.removeItem(at: url) } } From 6499f7dfa7fff64c903642db690094d7e6c64ec8 Mon Sep 17 00:00:00 2001 From: Aaron Paterson Date: Sun, 2 Aug 2026 00:51:10 +0000 Subject: [PATCH 02/21] Say how to run the tests that macOS skips Tests guarded for Linux are compiled out on macOS, so `make test` reports success without having run them, and nothing says so. The target that does run them is not mentioned anywhere outside the makefile. --- README.md | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/README.md b/README.md index f658b08d..372a872a 100644 --- a/README.md +++ b/README.md @@ -130,6 +130,16 @@ After building, run basic and integration tests: make test integration ``` +Tests that only apply to Linux are compiled out on macOS, so `make test` passes +without running them. Run those with: + +```bash +make linux-test +``` + +which runs the same tests inside the Linux dev container, as the Linux build +workflow does. + A kernel is required to run integration tests. If you do not have a kernel locally, a default kernel can be fetched using the `make fetch-default-kernel` target. From bf4d7dcf8bbe64c745e9ef344a8c173d2a42a3af Mon Sep 17 00:00:00 2001 From: Aaron Paterson Date: Sun, 2 Aug 2026 01:20:29 +0000 Subject: [PATCH 03/21] Test against the kernel release the shipped tool installs The default kernel the tests fetch had drifted from the one users run. container installs kata 3.28.0 and its 6.18.15 kernel, while these tests fetched 3.17.0, so the suite exercised a guest with a different feature set to the one it is meant to represent. Nested runtimes are the visible case: 3.17.0 was built without nf_tables, so a docker daemon inside a container fails there and works on what ships. Kata moved from xz to zstd between those releases, so the archive is no longer named for its compression and tar is left to recognise it rather than being told, which also holds if the format changes again. --- Makefile | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Makefile b/Makefile index c79d6995..3d01b93a 100644 --- a/Makefile +++ b/Makefile @@ -64,7 +64,7 @@ LIBARCHIVE_UPSTREAM_REPO := https://github.com/libarchive/libarchive LIBARCHIVE_UPSTREAM_VERSION := v3.7.7 LIBARCHIVE_LOCAL_DIR := workdir/libarchive -KATA_BINARY_PACKAGE := https://github.com/kata-containers/kata-containers/releases/download/3.17.0/kata-static-3.17.0-arm64.tar.xz +KATA_BINARY_PACKAGE := https://github.com/kata-containers/kata-containers/releases/download/3.28.0/kata-static-3.28.0-arm64.tar.zst CLOUD_HYPERVISOR_URL := https://github.com/cloud-hypervisor/cloud-hypervisor/releases/download/v52.0/cloud-hypervisor-static-aarch64 # SHA256 of the v52.0 aarch64 static binary (verified locally from the # upstream release artifact). Bump alongside CLOUD_HYPERVISOR_URL. @@ -410,11 +410,11 @@ integration: .PHONY: fetch-default-kernel fetch-default-kernel: @mkdir -p .local/ bin/ -ifeq (,$(wildcard .local/kata.tar.gz)) - @curl -SsL -o .local/kata.tar.gz ${KATA_BINARY_PACKAGE} +ifeq (,$(wildcard .local/kata.tar)) + @curl -SsL -o .local/kata.tar ${KATA_BINARY_PACKAGE} endif ifeq (,$(wildcard .local/vmlinux-$(KERNEL_ARCH))) - @tar -zxf .local/kata.tar.gz -C .local/ --strip-components=1 + @tar -xf .local/kata.tar -C .local/ --strip-components=1 @cp -L .local/opt/kata/share/kata-containers/vmlinux.container .local/vmlinux-$(KERNEL_ARCH) endif ifeq (,$(wildcard bin/vmlinux-$(KERNEL_ARCH))) From 21f1f16eaaacc051bbc5be6566814cb606de5698 Mon Sep 17 00:00:00 2001 From: Aaron Paterson Date: Wed, 12 Aug 2026 05:25:08 +0000 Subject: [PATCH 04/21] Run each integration test in its own scratch directory A test's rootfs clones, init block, and writable layers land in a directory the runner creates for it and deletes when it finishes, so a test holds disk only while it runs at any concurrency. The run-shared directory holds the unpacked images every bootstrap clones from, and is removed when the run ends. --- Sources/Integration/PodTests.swift | 7 +++-- Sources/Integration/Suite.swift | 43 +++++++++++++----------------- 2 files changed, 21 insertions(+), 29 deletions(-) diff --git a/Sources/Integration/PodTests.swift b/Sources/Integration/PodTests.swift index ae1caec8..7dff20fa 100644 --- a/Sources/Integration/PodTests.swift +++ b/Sources/Integration/PodTests.swift @@ -2228,10 +2228,9 @@ extension IntegrationSuite { /// (directory-share) rootfs. Assumes a single-layer image (the alpine /// image used by the suite) so no OCI whiteout processing is required. /// - /// The extracted dir lives under `Self.testDir`; do NOT `defer`-remove it - /// here — virtiofsd shares it for the whole test. It is swept by - /// `bootstrap`'s `maxConcurrency == 1` reaper on the next test and by the - /// suite-end `removeItem(at: Self.testDir)`. + /// The extracted dir lives under the test's scratch directory; do NOT + /// `defer`-remove it here — virtiofsd shares it for the whole test. The + /// runner removes the scratch directory when the test finishes. private func unpackRootfsDirectory(_ image: Containerization.Image, testID: String) async throws -> Containerization.Mount { let dir = Self.testDir.appending(component: "\(testID)-rootfs-dir") try? FileManager.default.removeItem(at: dir) diff --git a/Sources/Integration/Suite.swift b/Sources/Integration/Suite.swift index e43cf372..84580dba 100644 --- a/Sources/Integration/Suite.swift +++ b/Sources/Integration/Suite.swift @@ -135,7 +135,19 @@ struct IntegrationSuite: AsyncParsableCommand { FileManager.default.uniqueTemporaryDirectory(create: true) }() + /// The scratch directory of the test running on this task. The runner + /// creates one per test and deletes it when the test finishes, so a + /// test's clones and layers hold disk only while it runs. + @TaskLocal static var currentTestDir: URL? + static var testDir: URL { + currentTestDir ?? _testDir + } + + /// The run-shared directory holding unpacked images, which every test's + /// bootstrap clones from. It lives for the whole run and is removed at + /// the end. + static var imageCacheDir: URL { _testDir } @@ -233,7 +245,7 @@ struct IntegrationSuite: AsyncParsableCommand { let platform = Platform(arch: "arm64", os: "linux", variant: "v8") // Unpack to shared location with coordination to prevent concurrent unpacks - let fsPath = Self.testDir.appending(component: image.digest) + let fsPath = Self.imageCacheDir.appending(component: image.digest) let fs = try await Self.unpackCoordinator.unpack(key: fsPath.absolutePath()) { do { let unpacker = EXT4Unpacker(capacityInBytes: 2.gib()) @@ -251,29 +263,6 @@ struct IntegrationSuite: AsyncParsableCommand { } } - // Reap any per-test artifacts left over from prior tests. With - // `--max-concurrency 1` (linux-integration default) this runs after - // the previous test has fully completed, so it's race-free; on - // macOS where tests can run in parallel we just keep all files — - // disk usage isn't a concern there. Each per-test bootstrap clones - // a ~2GB rootfs and a ~512MB initfs, so without reaping the dev - // container fills its CoW layer in ~10 tests. - if self.maxConcurrency == 1 { - // contentsOfDirectory reports paths under /private, while testDir is - // built from FileManager's /var view of the same directory, so both - // sides are resolved before comparing. - let preserve = fsPath.resolvingSymlinksInPathWithPrivate().absolutePath() - if let entries = try? FileManager.default.contentsOfDirectory( - at: Self.testDir, - includingPropertiesForKeys: nil - ) { - for url in entries - where url.resolvingSymlinksInPathWithPrivate().absolutePath() != preserve { - try? FileManager.default.removeItem(at: url) - } - } - } - // Clone to test-specific path let clPath = Self.testDir.appending(component: "\(testID).ext4").absolutePath() try? FileManager.default.removeItem(atPath: clPath) @@ -658,11 +647,14 @@ struct IntegrationSuite: AsyncParsableCommand { for _ in 0.. Date: Wed, 12 Aug 2026 15:00:44 +0000 Subject: [PATCH 05/21] Give the machine's storage the shape its containers have A container's storage is its rootfs, an optional writable layer, and its mounts; a machine's is its containers' plus the volumes they share. One generic shape describes both the Mount values a machine is configured with and the AttachedFilesystem values it reports once attached, so converting between the two is a map over the structure and the roles cannot drift between configuration and attachment. Device addresses are allocated walking the same sorted order the devices are created in, on both backends. Consumers read roles instead of list positions: the spec builders take a container's mounts without prefix arithmetic, pod volumes mount by name, and a cloud-hypervisor volume's virtiofsd is held by a machine-lifetime reference alongside its containers' reference counts. --- .../Containerization/CHHotplugProvider.swift | 79 ++++++----- .../CHVirtualMachineInstance.swift | 127 ++++++++++-------- .../CHVirtualMachineManager.swift | 2 +- .../Containerization/ContainerStorage.swift | 107 +++++++++++++++ .../Containerization/HotplugProvider.swift | 2 +- Sources/Containerization/LinuxContainer.swift | 51 +++---- Sources/Containerization/LinuxPod.swift | 98 +++++++------- .../Containerization/VMConfiguration.swift | 10 +- .../VZVirtualMachineInstance.swift | 100 +++++++------- .../VZVirtualMachineManager.swift | 2 +- .../VirtualMachineInstance.swift | 5 +- 11 files changed, 351 insertions(+), 232 deletions(-) create mode 100644 Sources/Containerization/ContainerStorage.swift diff --git a/Sources/Containerization/CHHotplugProvider.swift b/Sources/Containerization/CHHotplugProvider.swift index 870fb1ff..380cba76 100644 --- a/Sources/Containerization/CHHotplugProvider.swift +++ b/Sources/Containerization/CHHotplugProvider.swift @@ -27,8 +27,8 @@ import Synchronization /// /// Handles both block (`vm.add-disk`) and virtiofs (`vm.add-fs`, with one /// `virtiofsd` per unique source-hash tag) hotplug, plus the matching -/// `vm.remove-device` teardown. Owns the per-VM mount registry so -/// `CHVirtualMachineInstance.mounts` can forward to it. +/// `vm.remove-device` teardown. Owns the machine's storage so +/// `CHVirtualMachineInstance.storage` can forward to it. final class CHHotplugProvider: HotplugProvider { struct HotplugRecord: Sendable { let chDeviceId: String @@ -50,7 +50,7 @@ final class CHHotplugProvider: HotplugProvider { private let workDir: URL private let virtiofsdBinaryOverride: URL? private let allocator: any AddressAllocator - private let _mounts: Mutex<[String: [AttachedFilesystem]]> + private let _storage: Mutex private let _records: Mutex<[String: [HotplugRecord]]> private let _tags: Mutex<[String: VirtiofsdTagState]> /// Serializes per-tag virtiofsd spawn so a concurrent hotplug for the @@ -65,14 +65,14 @@ final class CHHotplugProvider: HotplugProvider { workDir: URL, virtiofsdBinary: URL?, allocator: any AddressAllocator, - initialMounts: [String: [AttachedFilesystem]], + initialStorage: MachineAttachments, logger: Logger? ) { self.client = client self.workDir = workDir self.virtiofsdBinaryOverride = virtiofsdBinary self.allocator = allocator - self._mounts = Mutex(initialMounts) + self._storage = Mutex(initialStorage) self._records = Mutex([:]) self._tags = Mutex([:]) self.spawnLock = AsyncLock() @@ -81,14 +81,14 @@ final class CHHotplugProvider: HotplugProvider { // MARK: - Read accessors - var mounts: [String: [AttachedFilesystem]] { - _mounts.withLock { $0 } + var storage: MachineAttachments { + _storage.withLock { $0 } } - func withMountRegistry( - _ body: (inout sending [String: [AttachedFilesystem]]) throws -> sending T + func withStorage( + _ body: (inout sending MachineAttachments) throws -> sending T ) rethrows -> T { - try _mounts.withLock(body) + try _storage.withLock(body) } // MARK: - HotplugProvider conformance @@ -151,12 +151,13 @@ final class CHHotplugProvider: HotplugProvider { } func registerMounts(id: String, rootfs: AttachedFilesystem, additionalMounts: [Mount]) throws { - var attached: [AttachedFilesystem] = [rootfs] + var mounts: [AttachedFilesystem] = [] for mount in additionalMounts { - attached.append(try AttachedFilesystem(mount: mount, allocator: allocator)) + mounts.append(try AttachedFilesystem(mount: mount, allocator: allocator)) } - _mounts.withLock { - $0[id, default: []].append(contentsOf: attached) + let container = ContainerAttachments(rootfs: rootfs, mounts: mounts) + _storage.withLock { + $0.containers[id] = container } } @@ -190,19 +191,10 @@ final class CHHotplugProvider: HotplugProvider { } } - // Drop block-derived AttachedFilesystem entries for `id`. Block entries - // are the ones whose source was rewritten to "/dev/vd" by - // `hotplug(_:)` (or by AttachedFilesystem(mount:allocator:) for an - // additionalMount of type virtio-blk). - _mounts.withLock { state in - guard var perID = state[id] else { return } - perID.removeAll { $0.source.hasPrefix("/dev/vd") } - if perID.isEmpty { - state.removeValue(forKey: id) - } else { - state[id] = perID - } - } + // The container's devices are gone, so the container leaves the + // registry with them; its shares are released separately and their + // processes reference-counted through `_tags`. + _ = _storage.withLock { $0.containers.removeValue(forKey: id) } } func hotplugVirtioFS(_ mounts: [Mount], id: String) async throws { @@ -226,7 +218,7 @@ final class CHHotplugProvider: HotplugProvider { let chDeviceId = try await ensureVirtiofsDevice(tag: tag, source: source, readonly: readonly) // Record once per tag for this container. The AttachedFilesystem // entries for these mounts are written by registerMounts (the sole - // _mounts writer), so we do NOT touch _mounts here. + // registry writer), so we do NOT touch the storage here. _records.withLock { $0[id, default: []].append(HotplugRecord(chDeviceId: chDeviceId, kind: .virtiofs(tag: tag))) } @@ -338,16 +330,17 @@ final class CHHotplugProvider: HotplugProvider { try? FileManager.default.removeItem(at: socket) } - // Drop virtiofs AttachedFilesystem entries for `id`. AttachedFilesystem - // sets `type = mount.type` which for a `.virtiofs` mount is "virtiofs". - _mounts.withLock { state in - guard var perID = state[id] else { return } - perID.removeAll { $0.type == "virtiofs" } - if perID.isEmpty { - state.removeValue(forKey: id) - } else { - state[id] = perID + // Drop the container's virtiofs entries. A container whose rootfs is + // itself a share leaves the registry whole; one that keeps block + // devices keeps its entry with the share entries dropped. + _storage.withLock { state in + guard var container = state.containers[id] else { return } + if container.rootfs.type == "virtiofs" { + state.containers.removeValue(forKey: id) + return } + container.mounts.removeAll { $0.type == "virtiofs" } + state.containers[id] = container } } @@ -358,15 +351,21 @@ final class CHHotplugProvider: HotplugProvider { /// is the user-supplied `FsConfig.id` (which `vm.remove-device` keys on). /// `ownerIds` are the container ids that count toward this tag's refcount; /// each gets a `HotplugRecord` so `releaseVirtioFS(id:)` walks them - /// uniformly. + /// uniformly. `machineHeld` adds one reference nothing releases, for a + /// share the machine itself owns (a volume). func recordBootTimeVirtiofs( tag: String, process: VirtiofsdProcess, chDeviceId: String, - ownerIds: [String] + ownerIds: [String], + machineHeld: Bool ) { _tags.withLock { - $0[tag] = VirtiofsdTagState(process: process, refcount: ownerIds.count, chDeviceId: chDeviceId) + $0[tag] = VirtiofsdTagState( + process: process, + refcount: ownerIds.count + (machineHeld ? 1 : 0), + chDeviceId: chDeviceId + ) } _records.withLock { records in for id in ownerIds { diff --git a/Sources/Containerization/CHVirtualMachineInstance.swift b/Sources/Containerization/CHVirtualMachineInstance.swift index 122f592a..e363d5ca 100644 --- a/Sources/Containerization/CHVirtualMachineInstance.swift +++ b/Sources/Containerization/CHVirtualMachineInstance.swift @@ -46,7 +46,7 @@ public final class CHVirtualMachineInstance: Sendable { public struct Configuration: Sendable { public var cpus: Int public var memoryInBytes: UInt64 - public var mountsByID: [String: [Mount]] + public var storage: MachineMounts public var interfaces: [any Interface] public var kernel: Kernel? public var initialFilesystem: Mount? @@ -56,7 +56,7 @@ public final class CHVirtualMachineInstance: Sendable { public init() { self.cpus = 4 self.memoryInBytes = 1024 * 1024 * 1024 - self.mountsByID = [:] + self.storage = MachineMounts() self.interfaces = [] } } @@ -65,8 +65,8 @@ public final class CHVirtualMachineInstance: Sendable { /// `start()`'s `VmConfig.disks` ordering matches the allocator letters. struct BootDisk: Sendable { let mount: Mount - let containerId: String? // nil for rootfs - let letter: Character + let chId: String + let readonly: Bool } // MARK: - State @@ -76,8 +76,8 @@ public final class CHVirtualMachineInstance: Sendable { _state.withLock { $0 } } - public var mounts: [String: [AttachedFilesystem]] { - hotplug.mounts + public var storage: MachineAttachments { + hotplug.storage } /// Cloud-hypervisor exposes one virtio-fs device per source-hash tag, so @@ -167,9 +167,9 @@ public final class CHVirtualMachineInstance: Sendable { ) self.workDir = workDir - // 2. Block allocator + boot inventory. Walks rootfs first, then - // mountsByID sorted by container id, allocating disk letters in - // that order. The same allocator is later handed to the hotplug + // 2. Block allocator + boot inventory. Walks rootfs first, then the + // machine's storage in its device order, allocating disk letters + // as it goes. The same allocator is later handed to the hotplug // provider so runtime add-disk picks up where boot wiring left off. let allocator = Character.blockDeviceTagAllocator() let inventory = try config.bootInventory(allocator: allocator) @@ -201,14 +201,14 @@ public final class CHVirtualMachineInstance: Sendable { logger: logger ?? Logger(label: "CloudHypervisor.Client") ) - // 5. Hotplug provider — owns the mount registry, seeded with the + // 5. Hotplug provider — owns the machine's storage, seeded with the // boot inventory so registerMounts can append to it. self.hotplug = CHHotplugProvider( client: self.client, workDir: workDir, virtiofsdBinary: virtiofsdBinary, allocator: allocator, - initialMounts: inventory.attachments, + initialStorage: inventory.attachments, logger: logger ) @@ -222,11 +222,11 @@ public final class CHVirtualMachineInstance: Sendable { self._stdioPool = Mutex([:]) } - /// Mutate the mount registry. Forwards to the hotplug provider, which + /// Mutate the storage registry. Forwards to the hotplug provider, which /// owns the registry. Kept on the instance for parity with the macOS - /// path's `withMountRegistry` API. - func withMountRegistry(_ body: (inout sending [String: [AttachedFilesystem]]) throws -> sending T) rethrows -> T { - try hotplug.withMountRegistry(body) + /// path's `withStorage` API. + func withStorage(_ body: (inout sending MachineAttachments) throws -> sending T) rethrows -> T { + try hotplug.withStorage(body) } } @@ -510,34 +510,46 @@ extension CHVirtualMachineInstance { throw ContainerizationError(.invalidArgument, message: "initialFilesystem is required for cloud-hypervisor backend") } - // Disks: rootfs forced read-only at the device level; container disks + // Disks: rootfs forced read-only at the device level; other disks // honor their `ro` option through chDiskConfig. var disks: [CloudHypervisor.DiskConfig] = [] for bd in bootDisks { - let chId = bd.containerId.map { "blk-\($0)-\(bd.letter)" } ?? "rootfs" - if var disk = bd.mount.chDiskConfig(id: chId) { - if bd.containerId == nil { + if var disk = bd.mount.chDiskConfig(id: bd.chId) { + if bd.readonly { disk.readonly = true } disks.append(disk) } } - // Virtiofs: group all .virtiofs mounts in mountsByID by source-hash - // tag, spawn one virtiofsd per tag, build matching FsConfigs. - var byTag: [String: (mounts: [Mount], owners: [String])] = [:] - for cid in config.mountsByID.keys.sorted() { - guard let mounts = config.mountsByID[cid] else { continue } - for mount in mounts { - guard case .virtiofs = mount.runtimeOptions else { continue } - let tag = try hashFilePath(path: mount.source) - var entry = byTag[tag] ?? (mounts: [], owners: []) - entry.mounts.append(mount) - if !entry.owners.contains(cid) { - entry.owners.append(cid) + // Virtiofs: group all virtiofs mounts in the machine's storage by + // source-hash tag, spawn one virtiofsd per tag, build matching + // FsConfigs. A volume's tag is the machine's own, held for its + // lifetime. + var byTag: [String: (mounts: [Mount], owners: [String], machineHeld: Bool)] = [:] + func groupVirtiofs(_ mount: Mount, owner: String?) throws { + guard case .virtiofs = mount.runtimeOptions else { return } + let tag = try hashFilePath(path: mount.source) + var entry = byTag[tag] ?? (mounts: [], owners: [], machineHeld: false) + entry.mounts.append(mount) + if let owner { + if !entry.owners.contains(owner) { + entry.owners.append(owner) } - byTag[tag] = entry + } else { + entry.machineHeld = true } + byTag[tag] = entry + } + for cid in config.storage.containers.keys.sorted() { + guard let container = config.storage.containers[cid] else { continue } + for mount in container.all { + try groupVirtiofs(mount, owner: cid) + } + } + for name in config.storage.volumes.keys.sorted() { + guard let mount = config.storage.volumes[name] else { continue } + try groupVirtiofs(mount, owner: nil) } var fsConfigs: [CloudHypervisor.FsConfig] = [] @@ -569,7 +581,8 @@ extension CHVirtualMachineInstance { tag: tag, process: process, chDeviceId: chDeviceId, - ownerIds: entry.owners + ownerIds: entry.owners, + machineHeld: entry.machineHeld ) fsConfigs.append( @@ -714,40 +727,46 @@ extension CHVirtualMachineInstance { // MARK: - Boot inventory extension CHVirtualMachineInstance.Configuration { - /// Walks boot-time mounts in deterministic order (rootfs first, then - /// `mountsByID` sorted by container id, then each container's mounts in - /// input order), allocating disk letters for virtio-blk mounts and seeding - /// the per-container `AttachedFilesystem` registry. + /// Walks boot-time storage in the machine's device order (rootfs first, + /// then containers sorted by id, each in role order, then volumes sorted + /// by name), allocating disk letters for virtio-blk mounts and seeding + /// the machine's `AttachedFilesystem` registry. /// /// The allocator is shared with the runtime hotplug provider, so block /// hotplug picks up at the next free letter after boot. func bootInventory( allocator: any AddressAllocator - ) throws -> (attachments: [String: [AttachedFilesystem]], bootDisks: [CHVirtualMachineInstance.BootDisk]) { + ) throws -> (attachments: MachineAttachments, bootDisks: [CHVirtualMachineInstance.BootDisk]) { var bootDisks: [CHVirtualMachineInstance.BootDisk] = [] - var attachments: [String: [AttachedFilesystem]] = [:] - // Rootfs is not part of mountsByID. If it's a block device, it claims - // the first letter (vda) so the kernel cmdline `root=/dev/vda` is right. + // The rootfs is the machine's own, not part of its containers' + // storage. If it's a block device, it claims the first letter (vda) + // so the kernel cmdline `root=/dev/vda` is right. if let rootfs = self.initialFilesystem, rootfs.isBlock { - let letter = try allocator.allocate() - bootDisks.append(.init(mount: rootfs, containerId: nil, letter: letter)) + _ = try allocator.allocate() + bootDisks.append(.init(mount: rootfs, chId: "rootfs", readonly: true)) } - for cid in self.mountsByID.keys.sorted() { - guard let mounts = self.mountsByID[cid] else { continue } - var perContainer: [AttachedFilesystem] = [] - for mount in mounts { - let attached = try AttachedFilesystem(mount: mount, allocator: allocator) - if mount.isBlock, let letter = attached.source.last { - bootDisks.append(.init(mount: mount, containerId: cid, letter: letter)) - } - perContainer.append(attached) + func attach(_ mount: Mount, chId: (Character) -> String) throws -> AttachedFilesystem { + let attached = try AttachedFilesystem(mount: mount, allocator: allocator) + if mount.isBlock, let letter = attached.source.last { + bootDisks.append(.init(mount: mount, chId: chId(letter), readonly: false)) } - attachments[cid] = perContainer + return attached + } + + var containers: [String: ContainerAttachments] = [:] + for cid in self.storage.containers.keys.sorted() { + guard let container = self.storage.containers[cid] else { continue } + containers[cid] = try container.map { try attach($0, chId: { "blk-\(cid)-\($0)" }) } + } + var volumes: [String: AttachedFilesystem] = [:] + for name in self.storage.volumes.keys.sorted() { + guard let mount = self.storage.volumes[name] else { continue } + volumes[name] = try attach(mount, chId: { "vol-\(name)-\($0)" }) } - return (attachments, bootDisks) + return (MachineAttachments(containers: containers, volumes: volumes), bootDisks) } } #endif diff --git a/Sources/Containerization/CHVirtualMachineManager.swift b/Sources/Containerization/CHVirtualMachineManager.swift index ba20b5d1..c444620d 100644 --- a/Sources/Containerization/CHVirtualMachineManager.swift +++ b/Sources/Containerization/CHVirtualMachineManager.swift @@ -99,7 +99,7 @@ public struct CHVirtualMachineManager: VirtualMachineManager { instanceConfig.cpus = vmConfig.cpus instanceConfig.memoryInBytes = vmConfig.memoryInBytes instanceConfig.interfaces = vmConfig.interfaces - instanceConfig.mountsByID = vmConfig.mountsByID + instanceConfig.storage = vmConfig.storage instanceConfig.bootLog = vmConfig.bootLog instanceConfig.extensions = vmConfig.extensions instanceConfig.kernel = kernel diff --git a/Sources/Containerization/ContainerStorage.swift b/Sources/Containerization/ContainerStorage.swift new file mode 100644 index 00000000..4d8de2ed --- /dev/null +++ b/Sources/Containerization/ContainerStorage.swift @@ -0,0 +1,107 @@ +//===----------------------------------------------------------------------===// +// Copyright © 2026 Apple Inc. and the Containerization project authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//===----------------------------------------------------------------------===// + +/// A container's storage, by role. +/// +/// The same shape describes storage at both stages of its life: as the +/// `Mount` values a container is configured with, and as the +/// `AttachedFilesystem` values the machine reports once they are attached. +/// Converting between the two is a `map` over the structure, so the roles +/// cannot drift apart between configuration and attachment. +public struct ContainerStorage: Sendable { + /// The container's root filesystem. + public var rootfs: Value + + /// The writable layer mounted over the rootfs as the upper side of an + /// overlay, when the container has one. + public var writableLayer: Value? + + /// The container's remaining mounts, in configuration order. + public var mounts: [Value] + + public init(rootfs: Value, writableLayer: Value? = nil, mounts: [Value] = []) { + self.rootfs = rootfs + self.writableLayer = writableLayer + self.mounts = mounts + } + + /// Every value in the structure: the rootfs, the writable layer when + /// present, then the mounts, in that order. + public var all: [Value] { + var values = [rootfs] + if let writableLayer { + values.append(writableLayer) + } + values.append(contentsOf: mounts) + return values + } + + /// The storage with `transform` applied to every value, each keeping + /// its role. + public func map(_ transform: (Value) throws -> U) rethrows -> ContainerStorage { + ContainerStorage( + rootfs: try transform(rootfs), + writableLayer: try writableLayer.map(transform), + mounts: try mounts.map(transform) + ) + } +} + +/// A container's storage as configured, before its machine exists. +public typealias ContainerMounts = ContainerStorage + +/// A container's storage as attached to its machine. +public typealias ContainerAttachments = ContainerStorage + +/// The storage a machine carries: each container's, and the resources its +/// containers share. +public struct MachineStorage: Sendable { + /// Each container's storage, by container ID. + public var containers: [String: ContainerStorage] + + /// Volumes shared by the machine's containers, by volume name. + public var volumes: [String: Value] + + public init(containers: [String: ContainerStorage] = [:], volumes: [String: Value] = [:]) { + self.containers = containers + self.volumes = volumes + } + + /// Every value the machine carries: containers sorted by ID, each in + /// role order, then volumes sorted by name. Walks that allocate device + /// addresses and walks that create the devices use this one order, so an + /// address always names the device it was handed out for. + public var ordered: [Value] { + containers.keys.sorted().flatMap { containers[$0]?.all ?? [] } + + volumes.keys.sorted().compactMap { volumes[$0] } + } + + /// The storage with `transform` applied to every value, each keeping its + /// place. The transform runs in no particular order, so an allocating + /// conversion walks the fields sorted itself. + public func map(_ transform: (Value) throws -> U) rethrows -> MachineStorage { + MachineStorage( + containers: try containers.mapValues { try $0.map(transform) }, + volumes: try volumes.mapValues(transform) + ) + } +} + +/// A machine's storage as configured, before it exists. +public typealias MachineMounts = MachineStorage + +/// A machine's storage as attached. +public typealias MachineAttachments = MachineStorage diff --git a/Sources/Containerization/HotplugProvider.swift b/Sources/Containerization/HotplugProvider.swift index f535ce3f..0835077f 100644 --- a/Sources/Containerization/HotplugProvider.swift +++ b/Sources/Containerization/HotplugProvider.swift @@ -26,7 +26,7 @@ public protocol HotplugProvider: Sendable { /// - Returns: The attached filesystem with the device path in the guest func hotplug(_ block: Mount, id: String) async throws -> AttachedFilesystem - /// Register mounts for a container in the VM's mount registry. + /// Register mounts for a container in the machine's storage. /// - Parameters: /// - id: The container ID /// - rootfs: The rootfs attachment from hotplug diff --git a/Sources/Containerization/LinuxContainer.swift b/Sources/Containerization/LinuxContainer.swift index 34964fcd..55bbb39a 100644 --- a/Sources/Containerization/LinuxContainer.swift +++ b/Sources/Containerization/LinuxContainer.swift @@ -537,24 +537,14 @@ extension LinuxContainer { } private func mountRootfs( - attachments: [AttachedFilesystem], + attached: ContainerAttachments, rootfsPath: String, agent: VirtualMachineAgent ) async throws { - guard let rootfsAttachment = attachments.first else { - throw ContainerizationError(.notFound, message: "rootfs mount not found") - } + let rootfsAttachment = attached.rootfs - if self.writableLayer != nil { + if let writableAttachment = attached.writableLayer { // Set up overlayfs with image as lower layer and writable layer as upper. - guard attachments.count >= 2 else { - throw ContainerizationError( - .notFound, - message: "writable layer mount not found" - ) - } - let writableAttachment = attachments[1] - let lowerPath = "/run/container/\(self.id)/lower" let upperMountPath = "/run/container/\(self.id)/upper" let upperPath = "/run/container/\(self.id)/upper/diff" @@ -623,17 +613,18 @@ extension LinuxContainer { // This is dumb, but alas. let fileMountContextHolder = Mutex(fileMountContext) - // Build the list of mounts to attach to the VM. - var containerMounts = [modifiedRootfs] + fileMountContext.transformedMounts - if let writableLayer = self.writableLayer { - containerMounts.insert(writableLayer, at: 1) - } + // Build the container's storage to attach to the VM. + let containerStorage = ContainerMounts( + rootfs: modifiedRootfs, + writableLayer: self.writableLayer, + mounts: fileMountContext.transformedMounts + ) let vmConfig = VMConfiguration( cpus: vmCpus, memoryInBytes: vmMemory, interfaces: self.interfaces, - mountsByID: [self.id: containerMounts], + storage: MachineMounts(containers: [self.id: containerStorage]), bootLog: self.config.bootLog, nestedVirtualization: self.config.virtualization ) @@ -643,7 +634,7 @@ extension LinuxContainer { do { try await vm.start() - let mountsForAgent = containerMounts + let storageForAgent = containerStorage try await vm.withAgent { agent in try await agent.standardSetup() @@ -656,7 +647,7 @@ extension LinuxContainer { // with zero shares), but the cloud-hypervisor backend // only spawns virtiofsd when shares exist; mounting an // unbacked tag fails with EINVAL. - let hasVirtiofsMount = mountsForAgent.contains { mount in + let hasVirtiofsMount = storageForAgent.all.contains { mount in if case .virtiofs = mount.runtimeOptions { return true } return false } @@ -671,7 +662,7 @@ extension LinuxContainer { // gets populated. if vm.virtiofsLayout == .perTag { try await agent.mkdir(path: "/run/virtiofs", all: true, perms: 0o755) - let virtiofsAttachments = (vm.mounts[self.id] ?? []).filter { $0.type == "virtiofs" } + let virtiofsAttachments = (vm.storage.containers[self.id]?.all ?? []).filter { $0.type == "virtiofs" } let uniqueTags = Set(virtiofsAttachments.map(\.source)) for tag in uniqueTags { let dest = "/run/virtiofs/\(tag)" @@ -695,18 +686,17 @@ extension LinuxContainer { } } - guard let attachments = vm.mounts[self.id] else { + guard let attached = vm.storage.containers[self.id] else { throw ContainerizationError(.notFound, message: "rootfs mount not found") } let rootfsPath = Self.guestRootfsPath(self.id) - try await self.mountRootfs(attachments: attachments, rootfsPath: rootfsPath, agent: agent) + try await self.mountRootfs(attached: attached, rootfsPath: rootfsPath, agent: agent) // Mount file mount holding directories under /run. if fileMountContext.hasFileMounts { - let containerMounts = vm.mounts[self.id] ?? [] var ctx = fileMountContextHolder.withLock { $0 } try await ctx.mountHoldingDirectories( - vmMounts: containerMounts, + vmMounts: attached.mounts, agent: agent ) fileMountContextHolder.withLock { $0 = ctx } @@ -764,15 +754,12 @@ extension LinuxContainer { let agent = try await createdState.vm.dialAgent() do { var spec = self.generateRuntimeSpec() - // We don't need the rootfs (or writable layer), nor do OCI runtimes want it included. - // Also filter out file mount holding directories. We'll mount those separately under /run. + // Filter out file mount holding directories. We'll mount those separately under /run. // Transform virtiofs mounts to bind mounts from /run/virtiofs/{tag} - let containerMounts = createdState.vm.mounts[self.id] ?? [] + let containerMounts = createdState.vm.storage.containers[self.id]?.mounts ?? [] let holdingTags = createdState.fileMountContext.holdingDirectoryTags - // Drop rootfs, and writable layer if present. - let mountsToSkip = self.writableLayer != nil ? 2 : 1 var mounts: [ContainerizationOCI.Mount] = - containerMounts.dropFirst(mountsToSkip) + containerMounts .filter { !holdingTags.contains($0.source) } .map { attached -> ContainerizationOCI.Mount in if attached.type == "virtiofs" { diff --git a/Sources/Containerization/LinuxPod.swift b/Sources/Containerization/LinuxPod.swift index 6275a4d4..fa4d1785 100644 --- a/Sources/Containerization/LinuxPod.swift +++ b/Sources/Containerization/LinuxPod.swift @@ -446,24 +446,28 @@ extension LinuxPod { // the container's bind mounts from /run/virtiofs/ fail // with ENOENT. // - // Derive the tags from the additional mounts directly rather - // than from vm.mounts[id], so this is independent of the - // rootfs (which may be virtiofs or virtio-blk) and of mount - // ordering. The rootfs is mounted at /run/container//rootfs - // and is never consumed from /run/virtiofs. + // Derive the new tags from the additional mounts being added; + // the machine's storage names what is already shared. let newVirtiofsTags = try virtioFSMounts.map { try hashFilePath(path: $0.source) } if !newVirtiofsTags.isEmpty { try await agent.mkdir(path: "/run/virtiofs", all: true, perms: 0o755) if vm.virtiofsLayout == .perTag { // Tags already mounted in the guest at boot or by a - // prior hotplug (i.e. present on another container). - let alreadyMounted = Set( - vm.mounts - .filter { $0.key != id } - .values.flatMap { $0 } - .filter { $0.type == "virtiofs" } - .map { $0.source } - ) + // prior hotplug (i.e. present on another container or + // a machine volume). + let alreadyMounted: Set = { + var mounted = Set( + vm.storage.containers + .filter { $0.key != id } + .values.flatMap { $0.all } + .filter { $0.type == "virtiofs" } + .map { $0.source }) + mounted.formUnion( + vm.storage.volumes.values + .filter { $0.type == "virtiofs" } + .map { $0.source }) + return mounted + }() var seen: Set = [] for tag in newVirtiofsTags where !alreadyMounted.contains(tag) && seen.insert(tag).inserted { @@ -493,7 +497,7 @@ extension LinuxPod { } if fileMountContext.hasFileMounts { - let containerMounts = vm.mounts[id] ?? [] + let containerMounts = vm.storage.containers[id]?.mounts ?? [] try await updatedFileMountContext.mountHoldingDirectories( vmMounts: containerMounts, agent: agent @@ -557,20 +561,23 @@ extension LinuxPod { try await self.state.withLock { state in try state.phase.validateForCreate() - // Build mountsByID for all containers. + // Build the machine's storage from its containers. // Strip "ro" from rootfs options - we handle readonly via the OCI spec's // root.readonly field and remount in vmexec after setup is complete. // Use transformedMounts from fileMountContext (file mounts become directory shares). - var mountsByID: [String: [Mount]] = [:] + var machineStorage = MachineMounts() for (id, container) in state.containers { var modifiedRootfs = container.rootfs modifiedRootfs.options.removeAll(where: { $0 == "ro" }) - // Filter out shared mounts — those are handled separately as pod volume bind mounts. + // Shared mounts are handled separately as pod volume bind mounts. let containerMounts = container.fileMountContext.transformedMounts.filter { if case .shared = $0.runtimeOptions { return false } return true } - mountsByID[id] = [modifiedRootfs] + containerMounts + machineStorage.containers[id] = ContainerMounts( + rootfs: modifiedRootfs, + mounts: containerMounts + ) } // Validate pod volume names are unique. @@ -597,9 +604,8 @@ extension LinuxPod { } } } - let podVolumeMounts = self.config.volumes.map { $0.toMount() } - if !podVolumeMounts.isEmpty { - mountsByID[self.id] = podVolumeMounts + for volume in self.config.volumes { + machineStorage.volumes[volume.name] = volume.toMount() } // Capture into an immutable `let` so the value is safely usable @@ -608,18 +614,16 @@ extension LinuxPod { // only attaches a virtiofs device when shares are configured, // so mounting an unbacked /run/virtiofs would fail with EINVAL // on the CH backend. - let hasVirtiofsMount = mountsByID.values.contains { mounts in - mounts.contains { mount in - if case .virtiofs = mount.runtimeOptions { return true } - return false - } + let hasVirtiofsMount = machineStorage.ordered.contains { mount in + if case .virtiofs = mount.runtimeOptions { return true } + return false } var vmConfig = VMConfiguration( cpus: self.config.cpus, memoryInBytes: self.config.memoryInBytes, interfaces: self.config.interfaces, - mountsByID: mountsByID, + storage: machineStorage, bootLog: self.config.bootLog, nestedVirtualization: self.config.virtualization ) @@ -649,19 +653,17 @@ extension LinuxPod { // /run/virtiofs/. See LinuxContainer for the // VZ vs. CH model split. var seenTags: Set = [] - for (_, attached) in vm.mounts { - for entry in attached where entry.type == "virtiofs" { - guard seenTags.insert(entry.source).inserted else { continue } - let dest = "/run/virtiofs/\(entry.source)" - try await agent.mkdir(path: dest, all: true, perms: 0o755) - try await agent.mount( - ContainerizationOCI.Mount( - type: "virtiofs", - source: entry.source, - destination: dest, - options: [] - )) - } + for entry in vm.storage.ordered where entry.type == "virtiofs" { + guard seenTags.insert(entry.source).inserted else { continue } + let dest = "/run/virtiofs/\(entry.source)" + try await agent.mkdir(path: dest, all: true, perms: 0o755) + try await agent.mount( + ContainerizationOCI.Mount( + type: "virtiofs", + source: entry.source, + destination: dest, + options: [] + )) } } else { try await agent.mount( @@ -728,10 +730,10 @@ extension LinuxPod { // Mount all container rootfs for (_, container) in containers { - guard let attachments = vm.mounts[container.id], let rootfsAttachment = attachments.first else { + guard let attached = vm.storage.containers[container.id] else { throw ContainerizationError(.notFound, message: "rootfs mount not found for container \(container.id)") } - var rootfs = rootfsAttachment.to + var rootfs = attached.rootfs.to rootfs.destination = Self.guestRootfsPath(container.id) try await agent.mount(rootfs) } @@ -740,7 +742,7 @@ extension LinuxPod { for (id, container) in containers { if container.fileMountContext.hasFileMounts { var ctx = container.fileMountContext - let containerMounts = vm.mounts[id] ?? [] + let containerMounts = vm.storage.containers[id]?.mounts ?? [] try await ctx.mountHoldingDirectories( vmMounts: containerMounts, agent: agent @@ -750,15 +752,13 @@ extension LinuxPod { } // Mount pod-level volumes. - let podVolumeAttachments = vm.mounts[self.id] ?? [] - for (index, volume) in self.config.volumes.enumerated() { - guard index < podVolumeAttachments.count else { + for volume in self.config.volumes { + guard let attachment = vm.storage.volumes[volume.name] else { throw ContainerizationError( .notFound, message: "attached filesystem not found for pod volume \"\(volume.name)\"" ) } - let attachment = podVolumeAttachments[index] let guestPath = Self.guestVolumePath(volume.name) try await agent.mount( ContainerizationOCI.Mount( @@ -864,10 +864,10 @@ extension LinuxPod { // We don't need the rootfs, nor do OCI runtimes want it included. // Also filter out file mount holding directories - we mount those separately under /run. // Transform virtiofs mounts to bind mounts from /run/virtiofs/{tag} - let containerMounts = createdState.vm.mounts[containerID] ?? [] + let containerMounts = createdState.vm.storage.containers[containerID]?.mounts ?? [] let holdingTags = container.fileMountContext.holdingDirectoryTags var mounts: [ContainerizationOCI.Mount] = - containerMounts.dropFirst() + containerMounts .filter { !holdingTags.contains($0.source) } .map { attached -> ContainerizationOCI.Mount in if attached.type == "virtiofs" { diff --git a/Sources/Containerization/VMConfiguration.swift b/Sources/Containerization/VMConfiguration.swift index 30faebc4..7a5e8e2c 100644 --- a/Sources/Containerization/VMConfiguration.swift +++ b/Sources/Containerization/VMConfiguration.swift @@ -72,9 +72,9 @@ public struct VMConfiguration: Sendable { public var memoryInBytes: UInt64 /// The network interfaces to attach. public var interfaces: [any Interface] - /// Mounts organized by metadata ID (e.g. container ID). - /// Each ID maps to an array of mounts for that workload. - public var mountsByID: [String: [Mount]] + /// The storage the machine carries: each container's mounts by role, + /// and the volumes its containers share. + public var storage: MachineMounts /// Optional destination for serial boot logs. public var bootLog: BootLog? /// Enable nested virtualization support. If the VirtualMachineManager @@ -89,14 +89,14 @@ public struct VMConfiguration: Sendable { cpus: Int = 4, memoryInBytes: UInt64 = 1024 * 1024 * 1024, interfaces: [any Interface] = [], - mountsByID: [String: [Mount]] = [:], + storage: MachineMounts = MachineMounts(), bootLog: BootLog? = nil, nestedVirtualization: Bool = false ) { self.cpus = cpus self.memoryInBytes = memoryInBytes self.interfaces = interfaces - self.mountsByID = mountsByID + self.storage = storage self.bootLog = bootLog self.nestedVirtualization = nestedVirtualization } diff --git a/Sources/Containerization/VZVirtualMachineInstance.swift b/Sources/Containerization/VZVirtualMachineInstance.swift index 160c5026..670ab512 100644 --- a/Sources/Containerization/VZVirtualMachineInstance.swift +++ b/Sources/Containerization/VZVirtualMachineInstance.swift @@ -28,10 +28,10 @@ import Synchronization public final class VZVirtualMachineInstance: Sendable { public typealias Agent = Vminitd - /// Attached mounts on the virtual machine, organized by metadata ID. - private let _mounts: Mutex<[String: [AttachedFilesystem]]> - public var mounts: [String: [AttachedFilesystem]] { - _mounts.withLock { $0 } + /// The machine's attached storage. + private let _storage: Mutex + public var storage: MachineAttachments { + _storage.withLock { $0 } } /// The underlying Virtualization framework virtual machine. @@ -40,9 +40,9 @@ public final class VZVirtualMachineInstance: Sendable { /// The dispatch queue used for VZ operations. public var vmQueue: DispatchQueue { queue } - /// Mutate the mount registry. - public func withMountRegistry(_ body: (inout sending [String: [AttachedFilesystem]]) throws -> sending T) rethrows -> T { - try _mounts.withLock(body) + /// Mutate the storage registry. + public func withStorage(_ body: (inout sending MachineAttachments) throws -> sending T) rethrows -> T { + try _storage.withLock(body) } /// Serialize VM operations with the instance lock. @@ -73,8 +73,9 @@ public final class VZVirtualMachineInstance: Sendable { public var rosetta: Bool /// Toggle nested virtualization support. public var nestedVirtualization: Bool - /// Mount attachments organized by metadata ID. - public var mountsByID: [String: [Mount]] + /// The machine's storage: each container's mounts by role, and the + /// volumes its containers share. + public var storage: MachineMounts /// Network interface attachments. public var interfaces: [any Interface] /// Kernel image. @@ -91,7 +92,7 @@ public final class VZVirtualMachineInstance: Sendable { self.memoryInBytes = 1024.mib() self.rosetta = false self.nestedVirtualization = false - self.mountsByID = [:] + self.storage = MachineMounts() self.interfaces = [] } } @@ -132,7 +133,7 @@ public final class VZVirtualMachineInstance: Sendable { let allocator = Character.blockDeviceTagAllocator() let (mountAttachments, _) = try config.mountAttachments(allocator: allocator) - self._mounts = Mutex(mountAttachments) + self._storage = Mutex(mountAttachments) self.vm = VZVirtualMachine( configuration: try config.toVZ(allocator: allocator), @@ -153,7 +154,7 @@ public protocol VZInstanceExtension: Sendable { _ config: inout VZVirtualMachineConfiguration, allocator: any AddressAllocator, storageDeviceCount: Int, - mountsByID: [String: [Mount]] + storage: MachineMounts ) throws /// Called after the VZVirtualMachine is created but before start. @@ -168,7 +169,7 @@ extension VZInstanceExtension { _ config: inout VZVirtualMachineConfiguration, allocator: any AddressAllocator, storageDeviceCount: Int, - mountsByID: [String: [Mount]] + storage: MachineMounts ) throws {} public func didCreate(_ instance: VZVirtualMachineInstance) throws {} @@ -477,35 +478,33 @@ extension VZVirtualMachineInstance.Configuration { // Track used virtiofs tags to avoid creating duplicate VZ devices. // The same source directory mounted to multiple destinations shares one device. + // The walk is the machine's device order, matching the addresses + // `mountAttachments` hands out walking the same way. var usedVirtioFSTags: Set = [] - for (_, mounts) in self.mountsByID { - for mount in mounts { - if case .virtiofs = mount.runtimeOptions { - let tag = try hashFilePath(path: mount.source) - if usedVirtioFSTags.contains(tag) { - continue - } - usedVirtioFSTags.insert(tag) + for mount in self.storage.ordered { + if case .virtiofs = mount.runtimeOptions { + let tag = try hashFilePath(path: mount.source) + if usedVirtioFSTags.contains(tag) { + continue } - try mount.configure(config: &config) + usedVirtioFSTags.insert(tag) } + try mount.configure(config: &config) } // Create the unified virtiofs device with VZMultipleDirectoryShare // This device hosts all virtiofs shares and supports runtime updates var directories: [String: VZSharedDirectory] = [:] - for (_, mounts) in self.mountsByID { - for mount in mounts { - guard case .virtiofs(_) = mount.runtimeOptions else { continue } - guard FileManager.default.fileExists(atPath: mount.source) else { - throw ContainerizationError(.notFound, message: "directory \(mount.source) does not exist") - } - let name = try hashFilePath(path: mount.source) - directories[name] = VZSharedDirectory( - url: URL(fileURLWithPath: mount.source), - readOnly: mount.options.contains("ro") - ) + for mount in self.storage.ordered { + guard case .virtiofs(_) = mount.runtimeOptions else { continue } + guard FileManager.default.fileExists(atPath: mount.source) else { + throw ContainerizationError(.notFound, message: "directory \(mount.source) does not exist") } + let name = try hashFilePath(path: mount.source) + directories[name] = VZSharedDirectory( + url: URL(fileURLWithPath: mount.source), + readOnly: mount.options.contains("ro") + ) } let multiShare = VZMultipleDirectoryShare(directories: directories) let virtiofsDevice = VZVirtioFileSystemDeviceConfiguration(tag: "virtiofs") @@ -527,7 +526,7 @@ extension VZVirtualMachineInstance.Configuration { config.platform = platform for ext in self.extensions.compactMap({ $0 as? any VZInstanceExtension }) { - try ext.configureVZ(&config, allocator: allocator, storageDeviceCount: storageDeviceCount, mountsByID: self.mountsByID) + try ext.configureVZ(&config, allocator: allocator, storageDeviceCount: storageDeviceCount, storage: self.storage) } try config.validate() @@ -535,7 +534,7 @@ extension VZVirtualMachineInstance.Configuration { } func mountAttachments(allocator: any AddressAllocator) throws -> ( - attachments: [String: [AttachedFilesystem]], storageDeviceCount: Int + attachments: MachineAttachments, storageDeviceCount: Int ) { var storageDeviceCount = 0 @@ -548,21 +547,28 @@ extension VZVirtualMachineInstance.Configuration { } } - var attachmentsByID: [String: [AttachedFilesystem]] = [:] - - for (id, mounts) in self.mountsByID { - var attachments: [AttachedFilesystem] = [] - for mount in mounts { - let attached = try AttachedFilesystem(mount: mount, allocator: allocator) - attachments.append(attached) - if mount.isBlock { - storageDeviceCount += 1 - } + // The machine's device order: addresses are handed out in the same + // walk `makeConfiguration` creates the devices in. + func attach(_ mount: Mount) throws -> AttachedFilesystem { + let attached = try AttachedFilesystem(mount: mount, allocator: allocator) + if mount.isBlock { + storageDeviceCount += 1 } - attachmentsByID[id] = attachments + return attached + } + + var containers: [String: ContainerAttachments] = [:] + for id in self.storage.containers.keys.sorted() { + guard let container = self.storage.containers[id] else { continue } + containers[id] = try container.map(attach) + } + var volumes: [String: AttachedFilesystem] = [:] + for name in self.storage.volumes.keys.sorted() { + guard let mount = self.storage.volumes[name] else { continue } + volumes[name] = try attach(mount) } - return (attachmentsByID, storageDeviceCount) + return (MachineAttachments(containers: containers, volumes: volumes), storageDeviceCount) } } diff --git a/Sources/Containerization/VZVirtualMachineManager.swift b/Sources/Containerization/VZVirtualMachineManager.swift index 4959bee4..d0888447 100644 --- a/Sources/Containerization/VZVirtualMachineManager.swift +++ b/Sources/Containerization/VZVirtualMachineManager.swift @@ -76,7 +76,7 @@ public struct VZVirtualMachineManager: VirtualMachineManager { instanceConfig.rosetta = self.rosetta instanceConfig.nestedVirtualization = useNestedVirtualization - instanceConfig.mountsByID = vmConfig.mountsByID + instanceConfig.storage = vmConfig.storage instanceConfig.extensions = vmConfig.extensions }) } diff --git a/Sources/Containerization/VirtualMachineInstance.swift b/Sources/Containerization/VirtualMachineInstance.swift index 302e97ae..b9340612 100644 --- a/Sources/Containerization/VirtualMachineInstance.swift +++ b/Sources/Containerization/VirtualMachineInstance.swift @@ -44,7 +44,8 @@ public protocol VirtualMachineInstance: Sendable { // The state of the virtual machine. var state: VirtualMachineInstanceState { get } - var mounts: [String: [AttachedFilesystem]] { get } + /// The machine's attached storage. + var storage: MachineAttachments { get } /// How this VMM exposes virtiofs devices to the guest. Defaults to /// `.unified` (the VZ-shaped behavior); CH overrides to `.perTag`. @@ -73,7 +74,7 @@ public protocol VirtualMachineInstance: Sendable { func hotplug(_ block: Mount, id: String) async throws -> AttachedFilesystem /// Register mounts for a container after hotplug. - /// This is used to add the rootfs and additional mounts to the VM's mount registry + /// This is used to add the rootfs and additional mounts to the machine's storage /// so they can be found when building the container's OCI spec. /// - Parameter id: The container ID /// - Parameter rootfs: The rootfs attachment from hotplug From b3d72088b9deaeda3c02e4dcf377c2831c1473a0 Mon Sep 17 00:00:00 2001 From: Aaron Paterson Date: Sat, 1 Aug 2026 19:06:18 +0000 Subject: [PATCH 06/21] Add guest swap support A container can be given a swap area, so a workload whose memory exceeds its limit is reclaimed rather than killed. The area is a raw file on the host attached as a block device, which is how a guest gets swap it can write to when its own root is read only, and vminitd formats and enables it while mounting the container's filesystems. The area is enabled with discard, and the device backing it is marked non rotational first. virtio block devices are rotational by default, and the kernel only tracks a swap area in clusters when its device is non rotational, so without that the discard flags are accepted and no discard is ever issued. Together they let the sparse file that backs the area release the blocks the guest stops using, rather than holding the area's high water mark for as long as the container runs. --- Sources/CShim/include/swap.h | 36 ++++ Sources/CShim/swap.c | 23 +++ .../Containerization/ContainerManager.swift | 57 +++++- .../Containerization/ContainerStorage.swift | 14 +- Sources/Containerization/LinuxContainer.swift | 40 +++- Sources/ContainerizationOS/Linux/Swap.swift | 176 +++++++++++++++++ Sources/Integration/ContainerTests.swift | 178 ++++++++++++++++++ Sources/Integration/Suite.swift | 26 +++ Sources/cctl/RunCommand.swift | 4 + Tests/ContainerizationTests/SwapTests.swift | 73 +++++++ vminitd/Sources/VminitdCore/Server+GRPC.swift | 15 ++ 11 files changed, 635 insertions(+), 7 deletions(-) create mode 100644 Sources/CShim/include/swap.h create mode 100644 Sources/CShim/swap.c create mode 100644 Sources/ContainerizationOS/Linux/Swap.swift create mode 100644 Tests/ContainerizationTests/SwapTests.swift diff --git a/Sources/CShim/include/swap.h b/Sources/CShim/include/swap.h new file mode 100644 index 00000000..f16000ee --- /dev/null +++ b/Sources/CShim/include/swap.h @@ -0,0 +1,36 @@ +/* + * Copyright © 2026 Apple Inc. and the Containerization project authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef __SWAP_H +#define __SWAP_H + +#if defined(__linux__) + +// swapon(2) lives in , which Swift's glibc modulemap does not +// carry, so it is reachable from Swift only through a wrapper. + +// Discard the whole area when it is enabled, and each page as it is freed. +// These come from the same header, and SWAP_FLAG_DISCARD_PAGES has no UAPI +// header of its own at all, so every consumer declares it; see util-linux +// sys-utils/swapon.c. +#define CZ_SWAP_DISCARD 0x10000 +#define CZ_SWAP_DISCARD_PAGES 0x40000 + +int CZ_swapon(const char *path, int flags); + +#endif + +#endif diff --git a/Sources/CShim/swap.c b/Sources/CShim/swap.c new file mode 100644 index 00000000..2a66461d --- /dev/null +++ b/Sources/CShim/swap.c @@ -0,0 +1,23 @@ +/* + * Copyright © 2026 Apple Inc. and the Containerization project authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifdef __linux__ +#include + +#include "swap.h" + +int CZ_swapon(const char *path, int flags) { return swapon(path, flags); } +#endif diff --git a/Sources/Containerization/ContainerManager.swift b/Sources/Containerization/ContainerManager.swift index 27e9fbe4..a1b082e1 100644 --- a/Sources/Containerization/ContainerManager.swift +++ b/Sources/Containerization/ContainerManager.swift @@ -195,6 +195,9 @@ public struct ContainerManager: Sendable { /// - writableLayerSizeInBytes: Optional size for a separate writable layer. When provided, /// the rootfs becomes read-only and an overlayfs is used with a separate writable layer of this size. /// - readOnly: Whether to mount the root filesystem as read-only. + /// - swapSizeInBytes: Optional size for a swap area. When provided, a raw block + /// device of this size is created and the guest enables it as swap, so a + /// workload exceeding its memory limit reclaims rather than being killed. /// - networking: Whether to create a network interface for this container. Defaults to `true`. /// When `false`, no network resources are allocated and `releaseNetwork`/`delete` remain safe to call. /// - progress: Optional handler for tracking rootfs unpacking progress. @@ -203,6 +206,7 @@ public struct ContainerManager: Sendable { reference: String, rootfsSizeInBytes: UInt64 = 8.gib(), writableLayerSizeInBytes: UInt64? = nil, + swapSizeInBytes: UInt64? = nil, readOnly: Bool = false, networking: Bool = true, progress: ProgressHandler? = nil, @@ -214,6 +218,7 @@ public struct ContainerManager: Sendable { image: image, rootfsSizeInBytes: rootfsSizeInBytes, writableLayerSizeInBytes: writableLayerSizeInBytes, + swapSizeInBytes: swapSizeInBytes, readOnly: readOnly, networking: networking, progress: progress, @@ -229,6 +234,9 @@ public struct ContainerManager: Sendable { /// - writableLayerSizeInBytes: Optional size for a separate writable layer. When provided, /// the rootfs becomes read-only and an overlayfs is used with a separate writable layer of this size. /// - readOnly: Whether to mount the root filesystem as read-only. + /// - swapSizeInBytes: Optional size for a swap area. When provided, a raw block + /// device of this size is created and the guest enables it as swap, so a + /// workload exceeding its memory limit reclaims rather than being killed. /// - networking: Whether to create a network interface for this container. Defaults to `true`. /// When `false`, no network resources are allocated and `releaseNetwork`/`delete` remain safe to call. /// - progress: Optional handler for tracking rootfs unpacking progress. @@ -237,6 +245,7 @@ public struct ContainerManager: Sendable { image: Image, rootfsSizeInBytes: UInt64 = 8.gib(), writableLayerSizeInBytes: UInt64? = nil, + swapSizeInBytes: UInt64? = nil, readOnly: Bool = false, networking: Bool = true, progress: ProgressHandler? = nil, @@ -263,14 +272,25 @@ public struct ContainerManager: Sendable { ) } + // Create the swap device if a size is specified. + var swapLayer: Mount? = nil + if let swapSize = swapSizeInBytes, swapSize > 0 { + swapLayer = try createSwapDevice( + at: path.appendingPathComponent("swap.raw"), + size: swapSize + ) + } + return try await create( id, image: image, rootfs: rootfs, writableLayer: writableLayer, - networking: networking, - configuration: configuration - ) + networking: networking + ) { config in + config.swapLayer = swapLayer + try configuration(&config) + } } /// Returns a new container from the provided image and root filesystem mount. @@ -357,6 +377,37 @@ public struct ContainerManager: Sendable { } } + /// Create a raw block file to back a container's swap area. + /// + /// It carries no filesystem: the agent writes the swap header to the device + /// and enables it. The file is sparse, so it costs the host only the pages + /// the guest has actually swapped out, and gives them back on discard. A + /// swap area held in a file has to be free of holes, since the kernel walks + /// its extents; the guest reaches this one as a block device, which the + /// kernel takes as a single extent without consulting the host's layout. + /// https://github.com/torvalds/linux/blob/master/mm/swapfile.c + private func createSwapDevice(at destination: URL, size: UInt64) throws -> Mount { + let path = destination.absolutePath() + guard !FileManager.default.fileExists(atPath: path) else { + throw ContainerizationError(.exists, message: "swap device already exists at \(path)") + } + guard FileManager.default.createFile(atPath: path, contents: nil) else { + throw ContainerizationError(.internalError, message: "failed to create swap device at \(path)") + } + let handle = try FileHandle(forWritingTo: destination) + defer { try? handle.close() } + try handle.truncate(atOffset: size) + // A swap area holds nothing that outlives the container, so the host + // has no reason to synchronize it to permanent storage. + return .block( + format: Swap.mountType, + source: path, + destination: "", + options: [], + runtimeOptions: ["vzDiskImageSynchronizationMode=none"] + ) + } + private func createEmptyFilesystem(at destination: URL, size: UInt64) throws -> Mount { let path = destination.absolutePath() guard !FileManager.default.fileExists(atPath: path) else { diff --git a/Sources/Containerization/ContainerStorage.swift b/Sources/Containerization/ContainerStorage.swift index 4d8de2ed..49fae6d4 100644 --- a/Sources/Containerization/ContainerStorage.swift +++ b/Sources/Containerization/ContainerStorage.swift @@ -29,22 +29,29 @@ public struct ContainerStorage: Sendable { /// overlay, when the container has one. public var writableLayer: Value? + /// The swap area enabled for the container, when it has one. + public var swap: Value? + /// The container's remaining mounts, in configuration order. public var mounts: [Value] - public init(rootfs: Value, writableLayer: Value? = nil, mounts: [Value] = []) { + public init(rootfs: Value, writableLayer: Value? = nil, swap: Value? = nil, mounts: [Value] = []) { self.rootfs = rootfs self.writableLayer = writableLayer + self.swap = swap self.mounts = mounts } - /// Every value in the structure: the rootfs, the writable layer when - /// present, then the mounts, in that order. + /// Every value in the structure: the rootfs, the writable layer and swap + /// when present, then the mounts, in that order. public var all: [Value] { var values = [rootfs] if let writableLayer { values.append(writableLayer) } + if let swap { + values.append(swap) + } values.append(contentsOf: mounts) return values } @@ -55,6 +62,7 @@ public struct ContainerStorage: Sendable { ContainerStorage( rootfs: try transform(rootfs), writableLayer: try writableLayer.map(transform), + swap: try swap.map(transform), mounts: try mounts.map(transform) ) } diff --git a/Sources/Containerization/LinuxContainer.swift b/Sources/Containerization/LinuxContainer.swift index 55bbb39a..4d5289a7 100644 --- a/Sources/Containerization/LinuxContainer.swift +++ b/Sources/Containerization/LinuxContainer.swift @@ -99,6 +99,16 @@ public final class LinuxContainer: Container, Sendable { /// The total is aligned to a 1 MiB boundary. public var memoryOverhead: UInt64 = 128.mib() + /// Optional swap area for the container, as a block device mount. + /// + /// Swap lets a workload whose resident set exceeds `memoryInBytes` + /// reclaim rather than meet the out of memory killer. The area is a + /// block device rather than a file in the guest, because no filesystem + /// the agent can write to exists when the sandbox starts: its root is + /// mounted read only. The `destination` field is ignored, as the area + /// is enabled rather than mounted. + public var swapLayer: Mount? = nil + public init() {} public init( @@ -536,6 +546,30 @@ extension LinuxContainer { config.interfaces } + /// Enable the container's swap area, if it has one. + /// + /// The device is attached with the container's other block devices, so the + /// agent is told the guest path the VMM allocated it. It is enabled rather + /// than mounted, which is why it travels as a mount of type `swap`. + private func enableSwap( + attached: ContainerAttachments, + agent: VirtualMachineAgent + ) async throws { + guard self.config.swapLayer != nil else { + return + } + guard let swap = attached.swap else { + throw ContainerizationError(.notFound, message: "swap mount not found") + } + try await agent.mount( + ContainerizationOCI.Mount( + type: Swap.mountType, + source: swap.source, + destination: "", + options: swap.options + )) + } + private func mountRootfs( attached: ContainerAttachments, rootfsPath: String, @@ -613,10 +647,13 @@ extension LinuxContainer { // This is dumb, but alas. let fileMountContextHolder = Mutex(fileMountContext) - // Build the container's storage to attach to the VM. + // Build the container's storage to attach to the VM. The swap + // device is attached with the container's other block devices so + // the guest is told the /dev path the VMM allocates it. let containerStorage = ContainerMounts( rootfs: modifiedRootfs, writableLayer: self.writableLayer, + swap: self.config.swapLayer, mounts: fileMountContext.transformedMounts ) @@ -691,6 +728,7 @@ extension LinuxContainer { } let rootfsPath = Self.guestRootfsPath(self.id) try await self.mountRootfs(attached: attached, rootfsPath: rootfsPath, agent: agent) + try await self.enableSwap(attached: attached, agent: agent) // Mount file mount holding directories under /run. if fileMountContext.hasFileMounts { diff --git a/Sources/ContainerizationOS/Linux/Swap.swift b/Sources/ContainerizationOS/Linux/Swap.swift new file mode 100644 index 00000000..82006738 --- /dev/null +++ b/Sources/ContainerizationOS/Linux/Swap.swift @@ -0,0 +1,176 @@ +//===----------------------------------------------------------------------===// +// Copyright © 2026 Apple Inc. and the Containerization project authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//===----------------------------------------------------------------------===// + +import CShim +import ContainerizationError + +#if canImport(FoundationEssentials) +import FoundationEssentials +#else +import Foundation +#endif + +#if canImport(Musl) +import Musl +#elseif canImport(Glibc) +import Glibc +#endif + +/// `Swap` is a utility type that contains static helpers for creating a swap +/// file and enabling it as swap space for the guest. +/// +/// The swap area header layout is the kernel's on-disk format, version 1: +/// a page of header whose last 10 bytes are the magic `SWAPSPACE2`, with the +/// version, the last usable page and the page count at fixed offsets from the +/// start of the second kilobyte. +/// https://docs.kernel.org/admin-guide/mm/concepts.html +/// https://github.com/torvalds/linux/blob/master/include/linux/swap.h +public struct Swap: Sendable { + /// Offset of the version field within the swap header, and of the last + /// usable page field that follows it. + private static let versionOffset = 1024 + private static let lastPageOffset = 1028 + /// The magic that marks the last bytes of the first page of a swap area. + private static let magic = "SWAPSPACE2" + /// The queue attributes the kernel exposes for each block device. + public static let blockPath = "/sys/block" + + /// Mount type that marks a block device as a container's swap area, which + /// the agent enables rather than mounts. It travels in the `type` field of + /// the mount the host sends, so both sides read it from here. + /// + /// Kata gives its guest swap the same shape, the host attaching a raw file + /// as a block device and the agent calling `swapon` on it, and carries the + /// request on an RPC of its own (`AddSwap`). Here it travels as a mount so + /// the device path the VMM allocates reaches the guest the way every other + /// attached device's does. + /// https://github.com/kata-containers/kata-containers/blob/main/src/agent/src/rpc.rs + public static let mountType = "swap" + + #if os(Linux) + /// Format the swap area at `path` and enable it. + /// + /// `path` may be a block device or a file that already has its final size, + /// which is what `size` describes; `create` makes such a file. + public static func enable(path: String, size: UInt64, pageSize: Int = 4096) throws { + try format(path: path, size: size, pageSize: pageSize) + try on(path: path) + } + + /// Write the swap area header the kernel expects to an existing block + /// device or fully allocated file. + /// + /// This is what `mkswap` writes, and kata has its host run `mkswap` before + /// attaching the device. That is not open to a host which is not Linux, so + /// the header is written here instead, from the guest that is about to + /// enable it. + /// https://github.com/kata-containers/kata-containers/blob/main/src/runtime-rs/crates/resource/src/cpu_mem/swap.rs + public static func format(path: String, size: UInt64, pageSize: Int = 4096) throws { + let pages = size / UInt64(pageSize) + guard pages > 1 else { + throw ContainerizationError( + .invalidArgument, + message: "swap size \(size) is smaller than the two pages a swap area needs" + ) + } + // The header carries the last page number in 32 bits. + guard let lastPage = UInt32(exactly: pages - 1) else { + throw ContainerizationError( + .invalidArgument, + message: "swap size \(size) exceeds the \(UInt64(UInt32.max) + 1) pages a swap header can carry" + ) + } + + let fd = open(path, O_WRONLY) + guard fd >= 0 else { + throw POSIXError.fromErrno() + } + defer { close(fd) } + + var header = [UInt8](repeating: 0, count: pageSize) + header.replaceSubrange(versionOffset..<(versionOffset + 4), with: littleEndianBytes(1)) + header.replaceSubrange( + lastPageOffset..<(lastPageOffset + 4), + with: littleEndianBytes(lastPage) + ) + // The magic occupies the last bytes of the header page. + header.replaceSubrange((pageSize - magic.count).. UInt64 { + let fd = open(path, O_RDONLY) + guard fd >= 0 else { + throw POSIXError.fromErrno() + } + defer { close(fd) } + let end = lseek(fd, 0, SEEK_END) + guard end > 0 else { + throw POSIXError.fromErrno() + } + return UInt64(end) + } + + /// Enable the swap area at `path`, asking the kernel to discard the blocks + /// it stops using so the file backing the area does not keep them. + /// + /// Ours: kata enables its swap with no flags, which suits an area on a + /// host disk that was sized once and stays that size. + /// https://github.com/kata-containers/kata-containers/blob/main/src/agent/src/rpc.rs + public static func on(path: String) throws { + try markSolidState(devicePath: path) + guard CZ_swapon(path, CZ_SWAP_DISCARD | CZ_SWAP_DISCARD_PAGES) == 0 else { + throw POSIXError.fromErrno() + } + } + + /// Mark the block device backing the swap area as non rotational. + /// + /// Ours: no other runtime does this, because no other runtime needs the + /// area to give its blocks back. Kata calls `swapon` with no flags at all. + /// + /// The kernel only tracks a swap area in clusters when its device is non + /// rotational, and freeing a cluster is the only thing that schedules a + /// discard. A virtio block device reports as rotational, so without this + /// the area is scanned rather than clustered and the discard flags above + /// never take effect, leaving the file that backs the area holding every + /// block the guest has ever swapped to. + /// https://github.com/torvalds/linux/blob/master/mm/swapfile.c + static func markSolidState(devicePath: String) throws { + let device = URL(fileURLWithPath: devicePath).lastPathComponent + try "0".write( + to: URL(fileURLWithPath: Self.blockPath) + .appendingPathComponent(device) + .appendingPathComponent("queue") + .appendingPathComponent("rotational"), + atomically: false, + encoding: .ascii + ) + } + + private static func littleEndianBytes(_ value: UInt32) -> [UInt8] { + withUnsafeBytes(of: value.littleEndian) { Array($0) } + } + + #endif // os(Linux) +} diff --git a/Sources/Integration/ContainerTests.swift b/Sources/Integration/ContainerTests.swift index dd2a9165..9980cf21 100644 --- a/Sources/Integration/ContainerTests.swift +++ b/Sources/Integration/ContainerTests.swift @@ -137,6 +137,184 @@ extension IntegrationSuite { } } + func testContainerSwap() async throws { + let id = "test-container-swap" + let bs = try await bootstrap(id) + + let swapPath = Self.binPath(name: "\(id)-swap.raw") + let swap = try Self.makeSwapDevice(at: swapPath, size: 64.mib()) + defer { try? FileManager.default.removeItem(at: swapPath) } + + let buffer = BufferWriter() + let container = try LinuxContainer(id, rootfs: bs.rootfs, vmm: bs.vmm) { config in + config.process.arguments = ["/bin/cat", "/proc/swaps"] + config.process.stdout = buffer + config.swapLayer = swap + config.bootLog = bs.bootLog + } + + do { + try await container.create() + try await container.start() + + let status = try await container.wait() + try await container.stop() + + guard status.exitCode == 0 else { + throw IntegrationError.assert(msg: "process status \(status) != 0") + } + + // The guest lists the area it enabled, so the device is present and + // the kernel accepted the header the agent wrote to it. + let swaps = String(data: buffer.data, encoding: .utf8) ?? "" + guard swaps.contains("partition") else { + throw IntegrationError.assert(msg: "swap not enabled in guest: '\(swaps)'") + } + } catch { + try? await container.stop() + throw error + } + } + + /// The point of swap: a workload whose pages exceed the memory limit + /// reclaims instead of meeting the out of memory killer. The tmpfs is + /// sized explicitly because its default is half of RAM, which cannot + /// exceed the limit and so would never drive a page out. + func testContainerSwapUnderPressure() async throws { + let id = "test-container-swap-pressure" + let bs = try await bootstrap(id) + + let swapPath = Self.binPath(name: "\(id)-swap.raw") + let swap = try Self.makeSwapDevice(at: swapPath, size: 512.mib()) + defer { try? FileManager.default.removeItem(at: swapPath) } + + let buffer = BufferWriter() + let container = try LinuxContainer(id, rootfs: bs.rootfs, vmm: bs.vmm) { config in + // A shell variable is anonymous memory, which is what the kernel + // reclaims to a swap area, and asking the container's cgroup to + // reclaim puts it there rather than leaving it to the pressure the + // allocation happens to produce. Reading the value back afterwards + // is what shows it made the round trip intact, so the length is + // checked after the area has been measured. + config.process.arguments = [ + "/bin/sh", "-c", + "fill=$(head -c 340000000 /dev/zero | tr '\\0' 'a'); " + + "echo 340M > /sys/fs/cgroup/memory.reclaim; " + + "awk '/SwapTotal|SwapFree/ { print $1, $2 }' /proc/meminfo; " + + "test ${#fill} -eq 340000000", + ] + config.process.stdout = buffer + config.memoryInBytes = 256.mib() + config.swapLayer = swap + config.bootLog = bs.bootLog + } + + do { + try await container.create() + try await container.start() + + let status = try await container.wait() + try await container.stop() + + // Survival is the first half of the claim: without swap this + // workload is killed rather than reclaimed. + guard status.exitCode == 0 else { + throw IntegrationError.assert(msg: "workload did not survive memory pressure: \(status)") + } + + // And the second half: pages actually reached the swap device. + let out = String(data: buffer.data, encoding: .utf8) ?? "" + let values = out.split(separator: "\n").reduce(into: [String: Int]()) { acc, line in + let parts = line.split(separator: " ") + if parts.count == 2 { acc[String(parts[0])] = Int(parts[1]) } + } + guard let total = values["SwapTotal:"], let free = values["SwapFree:"], total > 0 else { + throw IntegrationError.assert(msg: "guest reported no swap: '\(out)'") + } + guard UInt64(total - free) * 1024 > 64.mib() else { + throw IntegrationError.assert(msg: "little or nothing was swapped out: '\(out)'") + } + } catch { + try? await container.stop() + throw error + } + } + + func testContainerSwapReclaimsFreedBlocks() async throws { + let id = "test-container-swap-reclaim" + let bs = try await bootstrap(id) + + let swapPath = Self.binPath(name: "\(id)-swap.raw") + let swap = try Self.makeSwapDevice(at: swapPath, size: 512.mib()) + defer { try? FileManager.default.removeItem(at: swapPath) } + + let container = try LinuxContainer(id, rootfs: bs.rootfs, vmm: bs.vmm) { config in + // The allocation is held by a child that exits, because the swap + // slots are only freed once the pages themselves are, and a shell + // that drops a variable keeps the pages in its own allocator. + config.process.arguments = [ + "/bin/sh", "-c", + "sh -c 'fill=$(head -c 200000000 /dev/zero | tr \"\\0\" a); " + + "echo 200M > /sys/fs/cgroup/memory.reclaim; " + + "test ${#fill} -eq 200000000' && sleep 30", + ] + config.memoryInBytes = 256.mib() + config.swapLayer = swap + config.bootLog = bs.bootLog + } + + // The blocks the file holds, rather than the size it reports, because a + // sparse file only ever reports the whole area. URL resource values + // cache after their first read, which a sampler cannot use. + func allocatedBytes() -> UInt64 { + var info = stat() + guard stat(swapPath.absolutePath(), &info) == 0 else { + return 0 + } + return UInt64(info.st_blocks) * 512 + } + + do { + try await container.create() + try await container.start() + + // Watching the file while the workload runs is what separates an + // area that released its blocks from one that never held any. + let peak = Task { + var high: UInt64 = 0 + while !Task.isCancelled { + high = max(high, allocatedBytes()) + try? await Task.sleep(nanoseconds: 200_000_000) + } + return high + } + + let status = try await container.wait() + peak.cancel() + let highWater = await peak.value + try await container.stop() + + guard status.exitCode == 0 else { + throw IntegrationError.assert(msg: "workload did not complete: \(status)") + } + guard highWater > 100.mib() else { + throw IntegrationError.assert( + msg: "workload never filled the swap area, it held \(highWater) bytes") + } + + // Having been filled, the area releases what the guest stopped + // using, so the file backing it does not hold its high water mark. + let settled = allocatedBytes() + guard settled < highWater / 4 else { + throw IntegrationError.assert( + msg: "swap area kept \(settled) of \(highWater) bytes after the guest freed it") + } + } catch { + try? await container.stop() + throw error + } + } + func testProcessEchoHi() async throws { let id = "test-process-echo-hi" let bs = try await bootstrap(id) diff --git a/Sources/Integration/Suite.swift b/Sources/Integration/Suite.swift index 84580dba..6087aafe 100644 --- a/Sources/Integration/Suite.swift +++ b/Sources/Integration/Suite.swift @@ -207,6 +207,27 @@ struct IntegrationSuite: AsyncParsableCommand { static let eventLoop = MultiThreadedEventLoopGroup(numberOfThreads: System.coreCount) + /// The swap area a test hands to a container. A suite that builds its own + /// containers owns the file behind them, the way `ContainerManager` owns + /// the ones it makes for callers that do not. + static func makeSwapDevice(at path: URL, size: UInt64) throws -> Containerization.Mount { + guard FileManager.default.createFile(atPath: path.absolutePath(), contents: nil) else { + throw IntegrationError.assert(msg: "failed to create swap device at \(path.absolutePath())") + } + let handle = try FileHandle(forWritingTo: path) + defer { try? handle.close() } + try handle.truncate(atOffset: size) + // A swap area holds nothing that outlives the container, so the host + // has no reason to synchronize it to permanent storage. + return .block( + format: Swap.mountType, + source: path.absolutePath(), + destination: "", + options: [], + runtimeOptions: ["vzDiskImageSynchronizationMode=none"] + ) + } + func bootstrap(_ testID: String) async throws -> (rootfs: Containerization.Mount, vmm: VirtualMachineManager, image: Containerization.Image, bootLog: BootLog) { let reference = "ghcr.io/linuxcontainers/alpine:3.20" let store = Self.imageStore @@ -614,6 +635,11 @@ struct IntegrationSuite: AsyncParsableCommand { Test("pod filesystem operation", testPodFilesystemOperation), Test("pod shared disk image volume", testPodSharedDiskImageVolume), Test("pod shared tmpfs volume", testPodSharedTmpfsVolume), + + // Swap + Test("container swap", testContainerSwap), + Test("container swap under pressure", testContainerSwapUnderPressure), + Test("container swap reclaims freed blocks", testContainerSwapReclaimsFreedBlocks), ] + macOS26Tests() let tests: [Test] = crossPlatformTests + macOSOnlyTests #else diff --git a/Sources/cctl/RunCommand.swift b/Sources/cctl/RunCommand.swift index fceff172..21f13a3e 100644 --- a/Sources/cctl/RunCommand.swift +++ b/Sources/cctl/RunCommand.swift @@ -48,6 +48,9 @@ extension Application { @Flag(name: .customLong("rosetta"), help: "Enable rosetta x64 emulation") var rosetta = false + @Option(name: .customLong("swap"), help: "Amount of swap in megabytes to create in the guest") + var swap: UInt64 = 0 + @Option(name: .customLong("mount"), help: "Directory to share into the container (Example: /foo:/bar)") var mounts: [String] = [] @@ -106,6 +109,7 @@ extension Application { id, reference: imageReference, rootfsSizeInBytes: fsSizeInMB.mib(), + swapSizeInBytes: swap.mib(), readOnly: readOnly, networking: true ) { config in diff --git a/Tests/ContainerizationTests/SwapTests.swift b/Tests/ContainerizationTests/SwapTests.swift new file mode 100644 index 00000000..1b8aa8a9 --- /dev/null +++ b/Tests/ContainerizationTests/SwapTests.swift @@ -0,0 +1,73 @@ +//===----------------------------------------------------------------------===// +// Copyright © 2026 Apple Inc. and the Containerization project authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//===----------------------------------------------------------------------===// + +#if os(Linux) +import ContainerizationOS +import Foundation +import Testing + +struct SwapTests { + private func makeArea(pages: UInt64, pageSize: Int) throws -> String { + let path = FileManager.default.temporaryDirectory + .appendingPathComponent("swap-\(UUID().uuidString)").path + try #require( + FileManager.default.createFile( + atPath: path, + contents: Data(count: Int(pages) * pageSize) + ) + ) + return path + } + + /// The header the kernel reads back is the one `mkswap` writes: version 1 + /// and the last usable page at the documented offsets, and the magic in the + /// final bytes of the first page. + @Test func formatWritesTheSwapAreaHeader() throws { + let pageSize = 4096 + let pages: UInt64 = 32 + let path = try makeArea(pages: pages, pageSize: pageSize) + defer { try? FileManager.default.removeItem(atPath: path) } + + try Swap.format(path: path, size: pages * UInt64(pageSize), pageSize: pageSize) + + let header = try Data(contentsOf: URL(fileURLWithPath: path))[0.. Date: Sun, 2 Aug 2026 00:03:43 +0000 Subject: [PATCH 07/21] Give a pod a swap area its containers share A pod's containers share one virtual machine, so they can share one swap area as well, with the guest kernel deciding whose pages are reclaimed to it rather than each container carrying its own. The area is attached with the pod's other mounts and enabled once, after the agent comes up. A container may cap how much of the area it uses. That cap counts swap alone, while the runtime spec carries memory and swap as a single total, so the container's memory limit is added to it when the spec is built. A cap without a memory limit is rejected when the container is added, because the total cannot be worked out without one. Leaving the cap unset lets a container use the whole area, which is what containers sharing a pool generally want. --- .../CHVirtualMachineInstance.swift | 7 +- .../Containerization/ContainerStorage.swift | 18 +++-- Sources/Containerization/LinuxPod.swift | 66 ++++++++++++++++++- .../Containerization/VMConfiguration.swift | 2 +- .../VZVirtualMachineInstance.swift | 5 +- Sources/Integration/PodTests.swift | 60 +++++++++++++++++ Sources/Integration/Suite.swift | 1 + 7 files changed, 147 insertions(+), 12 deletions(-) diff --git a/Sources/Containerization/CHVirtualMachineInstance.swift b/Sources/Containerization/CHVirtualMachineInstance.swift index e363d5ca..1704ce92 100644 --- a/Sources/Containerization/CHVirtualMachineInstance.swift +++ b/Sources/Containerization/CHVirtualMachineInstance.swift @@ -729,8 +729,8 @@ extension CHVirtualMachineInstance { extension CHVirtualMachineInstance.Configuration { /// Walks boot-time storage in the machine's device order (rootfs first, /// then containers sorted by id, each in role order, then volumes sorted - /// by name), allocating disk letters for virtio-blk mounts and seeding - /// the machine's `AttachedFilesystem` registry. + /// by name, then swap), allocating disk letters for virtio-blk mounts + /// and seeding the machine's `AttachedFilesystem` registry. /// /// The allocator is shared with the runtime hotplug provider, so block /// hotplug picks up at the next free letter after boot. @@ -765,8 +765,9 @@ extension CHVirtualMachineInstance.Configuration { guard let mount = self.storage.volumes[name] else { continue } volumes[name] = try attach(mount, chId: { "vol-\(name)-\($0)" }) } + let swap = try self.storage.swap.map { try attach($0, chId: { "swap-\($0)" }) } - return (MachineAttachments(containers: containers, volumes: volumes), bootDisks) + return (MachineAttachments(containers: containers, volumes: volumes, swap: swap), bootDisks) } } #endif diff --git a/Sources/Containerization/ContainerStorage.swift b/Sources/Containerization/ContainerStorage.swift index 49fae6d4..c98b8558 100644 --- a/Sources/Containerization/ContainerStorage.swift +++ b/Sources/Containerization/ContainerStorage.swift @@ -83,18 +83,25 @@ public struct MachineStorage: Sendable { /// Volumes shared by the machine's containers, by volume name. public var volumes: [String: Value] - public init(containers: [String: ContainerStorage] = [:], volumes: [String: Value] = [:]) { + /// The swap area shared by every container in the machine, when it has + /// one. + public var swap: Value? + + public init(containers: [String: ContainerStorage] = [:], volumes: [String: Value] = [:], swap: Value? = nil) { self.containers = containers self.volumes = volumes + self.swap = swap } /// Every value the machine carries: containers sorted by ID, each in - /// role order, then volumes sorted by name. Walks that allocate device - /// addresses and walks that create the devices use this one order, so an - /// address always names the device it was handed out for. + /// role order, then volumes sorted by name, then swap. Walks that + /// allocate device addresses and walks that create the devices use this + /// one order, so an address always names the device it was handed out + /// for. public var ordered: [Value] { containers.keys.sorted().flatMap { containers[$0]?.all ?? [] } + volumes.keys.sorted().compactMap { volumes[$0] } + + (swap.map { [$0] } ?? []) } /// The storage with `transform` applied to every value, each keeping its @@ -103,7 +110,8 @@ public struct MachineStorage: Sendable { public func map(_ transform: (Value) throws -> U) rethrows -> MachineStorage { MachineStorage( containers: try containers.mapValues { try $0.map(transform) }, - volumes: try volumes.mapValues(transform) + volumes: try volumes.mapValues(transform), + swap: try swap.map(transform) ) } } diff --git a/Sources/Containerization/LinuxPod.swift b/Sources/Containerization/LinuxPod.swift index fa4d1785..140f01ab 100644 --- a/Sources/Containerization/LinuxPod.swift +++ b/Sources/Containerization/LinuxPod.swift @@ -21,6 +21,7 @@ import Foundation import Logging import Synchronization +import struct ContainerizationOS.Swap import struct ContainerizationOS.Terminal /// NOTE: Experimental API @@ -43,6 +44,15 @@ public final class LinuxPod: Sendable { public var cpus: Int = 4 /// The memory in bytes to give to the pod's VM. public var memoryInBytes: UInt64 = 1024.mib() + /// Optional swap area shared by every container in the pod, as a block + /// device mount. + /// + /// The area belongs to the pod rather than to any one container, so the + /// guest kernel decides which container's pages are reclaimed to it. + /// Containers are free to use all of it unless they carry a limit of + /// their own. The `destination` field is ignored, as the area is + /// enabled rather than mounted. + public var swapLayer: Mount? = nil /// The network interfaces for the pod. public var interfaces: [any Interface] = [] /// Whether nested virtualization should be turned on for the pod. @@ -77,6 +87,24 @@ public final class LinuxPod: Sendable { public var cpus: Int? /// Optional per-container memory limit in bytes (can exceed pod total for oversubscription). public var memoryInBytes: UInt64? + /// Optional cap on how much of the pod's swap area this container may + /// use, in bytes. Leaving it unset lets the container use the whole + /// area, which is what containers sharing a pool usually want. + /// + /// This counts swap alone. The runtime spec carries memory and swap + /// combined, so `memoryInBytes` is added to it when the spec is built, + /// and a swap limit without a memory limit is rejected because the + /// combined figure cannot be worked out without one. + /// + /// Kata and docker spell the same limit as the combined figure the spec + /// carries, subtracting the memory limit to size the area, so a + /// container asking there for 2 GiB against a 1 GiB memory limit is + /// asking for 1 GiB of swap and here for 2 GiB. That figure suits a + /// container whose swap is sized for it alone; this one is a share of + /// an area the pod owns, and how much of that share a container may + /// take is what the number says. + /// https://github.com/kata-containers/kata-containers/blob/main/docs/how-to/how-to-setup-swap-devices-in-guest-kernel.md + public var swapInBytes: UInt64? /// The hostname for the container. public var hostname: String? /// The system control options for the container. @@ -326,8 +354,15 @@ public final class LinuxPod: Sendable { ) } if let memoryInBytes = config.memoryInBytes, memoryInBytes > 0 { + // The runtime spec's `swap` is the memory and swap total, not the + // swap alone, so the container's memory limit is folded in here. + var swapTotal: Int64? = nil + if let swapInBytes = config.swapInBytes { + swapTotal = Int64(memoryInBytes + swapInBytes) + } spec.linux?.resources?.memory = LinuxMemory( - limit: Int64(memoryInBytes) + limit: Int64(memoryInBytes), + swap: swapTotal ) } @@ -390,6 +425,15 @@ extension LinuxPod { var config = ContainerConfiguration() try configuration(&config) + // The runtime spec carries memory and swap as one total, so a swap + // limit cannot be expressed without a memory limit to add it to. + if config.swapInBytes != nil, config.memoryInBytes == nil { + throw ContainerizationError( + .invalidArgument, + message: "container \(id) sets a swap limit without a memory limit" + ) + } + let fileMountContext = try FileMountContext.prepare(mounts: config.mounts) switch state.phase { @@ -607,6 +651,9 @@ extension LinuxPod { for volume in self.config.volumes { machineStorage.volumes[volume.name] = volume.toMount() } + // The swap area is attached with the machine's own storage so the + // guest is told the /dev path the VMM allocates it. + machineStorage.swap = self.config.swapLayer // Capture into an immutable `let` so the value is safely usable // from the concurrent `withAgent` closure below. The container @@ -638,10 +685,27 @@ extension LinuxPod { let shareProcessNamespace = self.config.shareProcessNamespace let pauseProcessHolder = Mutex(nil) let fileMountContextUpdates = Mutex<[String: FileMountContext]>([:]) + let hasSwapLayer = self.config.swapLayer != nil try await vm.withAgent { agent in try await agent.standardSetup() + // The swap area belongs to the pod rather than to any one + // container, so it is enabled once here and every container + // reclaims to it through the guest's own memory management. + if hasSwapLayer { + guard let swap = vm.storage.swap else { + throw ContainerizationError(.notFound, message: "swap mount not found") + } + try await agent.mount( + ContainerizationOCI.Mount( + type: Swap.mountType, + source: swap.source, + destination: "", + options: swap.options + )) + } + // Mount the unified virtiofs share at /run/virtiofs only // when at least one container has a virtiofs mount. VZ // tolerates the unbacked mount; CH does not. diff --git a/Sources/Containerization/VMConfiguration.swift b/Sources/Containerization/VMConfiguration.swift index 7a5e8e2c..f00e3e04 100644 --- a/Sources/Containerization/VMConfiguration.swift +++ b/Sources/Containerization/VMConfiguration.swift @@ -73,7 +73,7 @@ public struct VMConfiguration: Sendable { /// The network interfaces to attach. public var interfaces: [any Interface] /// The storage the machine carries: each container's mounts by role, - /// and the volumes its containers share. + /// and the volumes and swap its containers share. public var storage: MachineMounts /// Optional destination for serial boot logs. public var bootLog: BootLog? diff --git a/Sources/Containerization/VZVirtualMachineInstance.swift b/Sources/Containerization/VZVirtualMachineInstance.swift index 670ab512..69828ec9 100644 --- a/Sources/Containerization/VZVirtualMachineInstance.swift +++ b/Sources/Containerization/VZVirtualMachineInstance.swift @@ -74,7 +74,7 @@ public final class VZVirtualMachineInstance: Sendable { /// Toggle nested virtualization support. public var nestedVirtualization: Bool /// The machine's storage: each container's mounts by role, and the - /// volumes its containers share. + /// volumes and swap its containers share. public var storage: MachineMounts /// Network interface attachments. public var interfaces: [any Interface] @@ -567,8 +567,9 @@ extension VZVirtualMachineInstance.Configuration { guard let mount = self.storage.volumes[name] else { continue } volumes[name] = try attach(mount) } + let swap = try self.storage.swap.map(attach) - return (MachineAttachments(containers: containers, volumes: volumes), storageDeviceCount) + return (MachineAttachments(containers: containers, volumes: volumes, swap: swap), storageDeviceCount) } } diff --git a/Sources/Integration/PodTests.swift b/Sources/Integration/PodTests.swift index 7dff20fa..93b0b7c9 100644 --- a/Sources/Integration/PodTests.swift +++ b/Sources/Integration/PodTests.swift @@ -96,6 +96,66 @@ extension IntegrationSuite { } } + func testPodSharedSwap() async throws { + let id = "test-pod-shared-swap" + + let bs = try await bootstrap(id) + let swapPath = Self.binPath(name: "\(id)-swap.raw") + let swap = try Self.makeSwapDevice(at: swapPath, size: 512.mib()) + defer { try? FileManager.default.removeItem(at: swapPath) } + + let pod = try LinuxPod(id, vmm: bs.vmm) { config in + config.cpus = 2 + config.memoryInBytes = 512.mib() + config.bootLog = bs.bootLog + config.swapLayer = swap + } + + // Both containers report the same area, because the pod owns it and + // the guest kernel decides whose pages are reclaimed to it. + let names = ["swap1", "swap2"] + let buffers = [names[0]: BufferWriter(), names[1]: BufferWriter()] + for name in names { + let buffer = buffers[name]! + try await pod.addContainer( + name, + rootfs: try cloneRootfs(bs.rootfs, testID: id, containerID: name) + ) { config in + config.process.arguments = [ + "/bin/sh", "-c", + "awk '/SwapTotal/ { print $2 }' /proc/meminfo", + ] + config.process.stdout = buffer + } + } + + try await pod.create() + + var totals: [UInt64] = [] + for name in names { + try await pod.startContainer(name) + let status = try await pod.waitContainer(name) + guard status.exitCode == 0 else { + throw IntegrationError.assert(msg: "\(name) status \(status) != 0") + } + let out = String(data: buffers[name]!.data, encoding: .utf8) ?? "" + guard let total = UInt64(out.trimmingCharacters(in: .whitespacesAndNewlines)) else { + throw IntegrationError.assert(msg: "\(name) reported no swap total: '\(out)'") + } + totals.append(total) + } + + try await pod.stop() + + guard totals[0] > 0 else { + throw IntegrationError.assert(msg: "pod swap area was not enabled: \(totals)") + } + guard totals[0] == totals[1] else { + throw IntegrationError.assert( + msg: "containers saw different swap areas: \(totals)") + } + } + func testPodContainerOutput() async throws { let id = "test-pod-container-output" diff --git a/Sources/Integration/Suite.swift b/Sources/Integration/Suite.swift index 6087aafe..f74321fc 100644 --- a/Sources/Integration/Suite.swift +++ b/Sources/Integration/Suite.swift @@ -409,6 +409,7 @@ struct IntegrationSuite: AsyncParsableCommand { Test("container IPv6 only default route", testIPv6OnlyDefaultRoute), Test("container IPv6 only gateway outside subnet", testIPv6OnlyGatewayOutsideSubnet), Test("container IPv6 dual stack", testIPv6DualStack), + Test("pod shared swap", testPodSharedSwap), Test("pod IPv6 address", testPodIPv6AddressAdd), ] } From c0d0220cc589741b22efed6ba2fd84558c4889df Mon Sep 17 00:00:00 2001 From: Aaron Paterson Date: Sun, 2 Aug 2026 20:09:40 +0000 Subject: [PATCH 08/21] Hold a container to the swap limit it was given A pod works out the memory and swap total its containers are capped at, the way the runtime spec asks for it, and sends that along with the rest of the resources. The guest wrote the memory limit and dropped the rest, so a container drew on the whole of the pod's swap area whatever it had been given. Cgroup v2 wants the swap on its own rather than the total, so the memory limit comes back out of it before the limit is written, and the spec's unlimited and unset values are carried across as the controller spells them. Reading the limit back from the kernel is what the test does, since a limit the guest never applied leaves nothing else to see. https://github.com/opencontainers/cgroups/blob/main/utils.go --- Sources/Integration/PodTests.swift | 55 +++++++++++++++++++++ Sources/Integration/Suite.swift | 1 + vminitd/Sources/Cgroup/Cgroup2Manager.swift | 34 +++++++++++++ 3 files changed, 90 insertions(+) diff --git a/Sources/Integration/PodTests.swift b/Sources/Integration/PodTests.swift index 93b0b7c9..19854194 100644 --- a/Sources/Integration/PodTests.swift +++ b/Sources/Integration/PodTests.swift @@ -156,6 +156,61 @@ extension IntegrationSuite { } } + /// A container's cap names the swap alone while the runtime spec carries the + /// memory and swap total, so the guest has to take the memory back out of it + /// before the kernel will hold the container to it. Read the cap back from + /// the kernel, because a spec the guest ignores leaves the container drawing + /// on the whole pod area with nothing to show it. + func testPodContainerSwapLimit() async throws { + let id = "test-pod-container-swap-limit" + + let bs = try await bootstrap(id) + let swapPath = Self.binPath(name: "\(id)-swap.raw") + let swap = try Self.makeSwapDevice(at: swapPath, size: 512.mib()) + defer { try? FileManager.default.removeItem(at: swapPath) } + + let pod = try LinuxPod(id, vmm: bs.vmm) { config in + config.cpus = 2 + config.memoryInBytes = 512.mib() + config.bootLog = bs.bootLog + config.swapLayer = swap + } + + let capped: UInt64 = 64.mib() + let buffer = BufferWriter() + try await pod.addContainer( + "capped", + rootfs: try cloneRootfs(bs.rootfs, testID: id, containerID: "capped") + ) { config in + config.memoryInBytes = 128.mib() + config.swapInBytes = capped + config.process.arguments = [ + "/bin/sh", "-c", "cat /sys/fs/cgroup/memory.swap.max", + ] + config.process.stdout = buffer + } + + try await pod.create() + try await pod.startContainer("capped") + let status = try await pod.waitContainer("capped") + try await pod.stop() + guard status.exitCode == 0 else { + throw IntegrationError.assert(msg: "capped status \(status) != 0") + } + + let reported = + String(data: buffer.data, encoding: .utf8)? + .trimmingCharacters(in: .whitespacesAndNewlines) ?? "" + guard let limit = UInt64(reported) else { + throw IntegrationError.assert( + msg: "swap cap never reached the kernel, memory.swap.max is '\(reported)'") + } + guard limit == capped else { + throw IntegrationError.assert( + msg: "expected a \(capped) byte swap cap, kernel holds \(limit)") + } + } + func testPodContainerOutput() async throws { let id = "test-pod-container-output" diff --git a/Sources/Integration/Suite.swift b/Sources/Integration/Suite.swift index f74321fc..bd4ec656 100644 --- a/Sources/Integration/Suite.swift +++ b/Sources/Integration/Suite.swift @@ -410,6 +410,7 @@ struct IntegrationSuite: AsyncParsableCommand { Test("container IPv6 only gateway outside subnet", testIPv6OnlyGatewayOutsideSubnet), Test("container IPv6 dual stack", testIPv6DualStack), Test("pod shared swap", testPodSharedSwap), + Test("pod container swap limit", testPodContainerSwapLimit), Test("pod IPv6 address", testPodIPv6AddressAdd), ] } diff --git a/vminitd/Sources/Cgroup/Cgroup2Manager.swift b/vminitd/Sources/Cgroup/Cgroup2Manager.swift index 62fa78c7..9ab812bc 100644 --- a/vminitd/Sources/Cgroup/Cgroup2Manager.swift +++ b/vminitd/Sources/Cgroup/Cgroup2Manager.swift @@ -216,6 +216,37 @@ public struct Cgroup2Manager: Sendable { ) } + // The runtime spec's `swap` is the memory and swap total, the way cgroup + // v1 took it, while cgroup v2 wants the swap on its own. A zero is the + // spec's "unset", which leaves the limit at max. + // https://github.com/opencontainers/cgroups/blob/main/utils.go + if let memory = resources.memory, let swap = memory.swap, swap != 0 { + let value: String + if swap < 0 { + value = "max" + } else { + guard let limit = memory.limit, limit != 0 else { + throw Error.invalidResource( + message: "unable to set swap limit without memory limit") + } + if limit < 0 { + // Memory is unlimited, so the total that contains it is too. + value = "max" + } else { + guard swap >= limit else { + throw Error.invalidResource( + message: "memory and swap limit \(swap) is below the memory limit \(limit)") + } + value = String(swap - limit) + } + } + try Self.writeValue( + path: self.path, + value: value, + fileName: "memory.swap.max" + ) + } + if let cpu = resources.cpu, let quota = cpu.quota, let period = cpu.period { // cpu.max format is "quota period" let value = "\(quota) \(period)" @@ -751,6 +782,7 @@ extension Cgroup2Manager { case cgroup1 case errno(errno: Int32, message: String) case notExist(path: String) + case invalidResource(message: String) package var description: String { switch self { @@ -762,6 +794,8 @@ extension Cgroup2Manager { return "tried to load a cgroup v1 path" case .notCgroup: return "path is not a cgroup mountpoint" + case .invalidResource(let message): + return message } } } From 2cbb72321404631d517f957d57ebb98dd48abec1 Mon Sep 17 00:00:00 2001 From: Aaron Paterson Date: Sun, 2 Aug 2026 20:28:50 +0000 Subject: [PATCH 09/21] Take the swap total as given when memory is unlimited An unlimited memory limit leaves nothing to subtract from the memory and swap total, so the swap stands as it was given rather than becoming unlimited alongside it. A memory limit that is negative without being the unlimited sentinel is not a limit at all, and is refused rather than read as one. https://github.com/opencontainers/cgroups/blob/main/utils.go --- vminitd/Sources/Cgroup/Cgroup2Manager.swift | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/vminitd/Sources/Cgroup/Cgroup2Manager.swift b/vminitd/Sources/Cgroup/Cgroup2Manager.swift index 9ab812bc..a9dcd97d 100644 --- a/vminitd/Sources/Cgroup/Cgroup2Manager.swift +++ b/vminitd/Sources/Cgroup/Cgroup2Manager.swift @@ -229,9 +229,12 @@ public struct Cgroup2Manager: Sendable { throw Error.invalidResource( message: "unable to set swap limit without memory limit") } - if limit < 0 { - // Memory is unlimited, so the total that contains it is too. - value = "max" + if limit == -1 { + // Unlimited memory leaves nothing to take out of the total, + // so the swap stands as it was given. + value = String(swap) + } else if limit < 0 { + throw Error.invalidResource(message: "invalid memory value: \(limit)") } else { guard swap >= limit else { throw Error.invalidResource( From be4f9cca1aafdd58cde8d3bfaa75aa516ebbb9ba Mon Sep 17 00:00:00 2001 From: Aaron Paterson Date: Fri, 7 Aug 2026 17:31:13 +0000 Subject: [PATCH 10/21] Let a pod's containers copy files in and out Copying a file between the host and a container is reachable only for a container in a machine of its own; the same container placed in a pod has no way to be copied into or out of, so a tool that offers `cp` loses it the moment a container joins a pod. What the transfer needs is the machine, the path the container's filesystem sits at in the guest, and a vsock port, and only the path differs between the two cases. It moves to a type that takes those, which both a standalone container and a pod's containers hand it. The round trip is covered against a running pod, including that a file copied into one container reaches that container alone. --- .../Containerization/GuestFileTransfer.swift | 303 ++++++++++++++++++ Sources/Containerization/LinuxContainer.swift | 264 ++------------- Sources/Containerization/LinuxPod.swift | 79 +++++ Sources/Integration/PodTests.swift | 84 +++++ Sources/Integration/Suite.swift | 1 + 5 files changed, 492 insertions(+), 239 deletions(-) create mode 100644 Sources/Containerization/GuestFileTransfer.swift diff --git a/Sources/Containerization/GuestFileTransfer.swift b/Sources/Containerization/GuestFileTransfer.swift new file mode 100644 index 00000000..d508c571 --- /dev/null +++ b/Sources/Containerization/GuestFileTransfer.swift @@ -0,0 +1,303 @@ +//===----------------------------------------------------------------------===// +// Copyright © 2026 Apple Inc. and the container project authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//===----------------------------------------------------------------------===// + +import ContainerizationArchive +import ContainerizationError +import ContainerizationOS +import Foundation + +/// Moves files between the host and a container's filesystem inside a guest. +/// +/// The transfer runs over a dedicated vsock connection on a port the caller +/// allocates. A container in a machine of its own and a container among +/// several in a pod differ only in where their filesystem sits in the guest, +/// so that path is what this is given. +struct GuestFileTransfer: Sendable { + /// Default chunk size for file transfers (1MiB). + static let defaultChunkSize = 1024 * 1024 + + /// The machine holding the container's filesystem. + let vm: any VirtualMachineInstance + /// Where the container's filesystem sits in the guest. + let guestRoot: String + /// The vsock port the data travels over. + let port: UInt32 + /// Where the blocking read and write work runs. + let queue: DispatchQueue + + /// Copy a file or directory from the host into the container. + /// + /// For directories, the source is archived as tar+gzip and streamed + /// directly through vsock without intermediate temp files. + func copyIn( + from source: URL, + to destination: URL, + mode: UInt32, + createParents: Bool, + chunkSize: Int + ) async throws { + var isDirectory: ObjCBool = false + guard FileManager.default.fileExists(atPath: source.path, isDirectory: &isDirectory) else { + throw ContainerizationError(.notFound, message: "copyIn: source not found '\(source.path)'") + } + let isArchive = isDirectory.boolValue + + let guestPath: URL = try await vm.withAgent { agent in + guard let vminitd = agent as? Vminitd else { + throw ContainerizationError(.unsupported, message: "copyIn requires Vminitd agent") + } + + return try await self.resolveCopyInGuestPath( + from: source, + to: destination, + sourceIsDirectory: isArchive, + using: vminitd + ) + } + + let listener = try vm.listen(port) + + try await withThrowingTaskGroup(of: Void.self) { group in + group.addTask { + try await self.vm.withAgent { agent in + guard let vminitd = agent as? Vminitd else { + throw ContainerizationError(.unsupported, message: "copyIn requires Vminitd agent") + } + try await vminitd.copy( + direction: .copyIn, + guestPath: guestPath, + vsockPort: self.port, + mode: mode, + createParents: createParents, + isArchive: isArchive + ) + } + } + + group.addTask { + guard let conn = await listener.first(where: { _ in true }) else { + throw ContainerizationError(.internalError, message: "copyIn: vsock connection not established") + } + try listener.finish() + + try await withCheckedThrowingContinuation { (continuation: CheckedContinuation) in + self.queue.async { + do { + defer { conn.closeFile() } + + if isArchive { + let writer = try ArchiveWriter(configuration: .init(format: .pax, filter: .gzip)) + try writer.open(fileDescriptor: conn.fileDescriptor) + try writer.archiveDirectory(source) + try writer.finishEncoding() + } else { + let srcFd = open(source.path, O_RDONLY) + guard srcFd != -1 else { + throw ContainerizationError( + .internalError, + message: "copyIn: failed to open '\(source.path)': \(String(cString: strerror(errno)))" + ) + } + defer { close(srcFd) } + + var buf = [UInt8](repeating: 0, count: chunkSize) + while true { + let n = read(srcFd, &buf, buf.count) + if n == 0 { break } + guard n > 0 else { + throw ContainerizationError( + .internalError, + message: "copyIn: read error: \(String(cString: strerror(errno)))" + ) + } + var written = 0 + while written < n { + let w = buf.withUnsafeBytes { ptr in + write(conn.fileDescriptor, ptr.baseAddress! + written, n - written) + } + guard w > 0 else { + throw ContainerizationError( + .internalError, + message: "copyIn: vsock write error: \(String(cString: strerror(errno)))" + ) + } + written += w + } + } + } + continuation.resume() + } catch { + continuation.resume(throwing: error) + } + } + } + } + + try await group.waitForAll() + } + } + + /// Copy a file or directory from the container to the host. + /// + /// For directories, the guest archives the source as tar+gzip and streams + /// it directly through vsock. The host extracts the archive without + /// intermediate temp files. + func copyOut( + from source: URL, + to destination: URL, + createParents: Bool, + chunkSize: Int + ) async throws { + if createParents { + let parentDir = destination.deletingLastPathComponent() + try FileManager.default.createDirectory(at: parentDir, withIntermediateDirectories: true) + } + + let guestPath = URL(filePath: guestRoot).appending(path: source.path) + let listener = try vm.listen(port) + + let (metadataStream, metadataCont) = AsyncStream.makeStream(of: Vminitd.CopyMetadata.self) + + try await withThrowingTaskGroup(of: Void.self) { group in + group.addTask { + defer { metadataCont.finish() } + try await self.vm.withAgent { agent in + guard let vminitd = agent as? Vminitd else { + throw ContainerizationError(.unsupported, message: "copyOut requires Vminitd agent") + } + try await vminitd.copy( + direction: .copyOut, + guestPath: guestPath, + vsockPort: self.port, + onMetadata: { meta in + metadataCont.yield(meta) + metadataCont.finish() + } + ) + } + } + + group.addTask { + guard let metadata = await metadataStream.first(where: { _ in true }) else { + throw ContainerizationError(.internalError, message: "copyOut: no metadata received") + } + + guard let conn = await listener.first(where: { _ in true }) else { + throw ContainerizationError(.internalError, message: "copyOut: vsock connection not established") + } + try listener.finish() + + try await withCheckedThrowingContinuation { (continuation: CheckedContinuation) in + self.queue.async { + do { + defer { conn.closeFile() } + + if metadata.isArchive { + try FileManager.default.createDirectory(at: destination, withIntermediateDirectories: true) + let fh = FileHandle(fileDescriptor: dup(conn.fileDescriptor), closeOnDealloc: true) + let reader = try ArchiveReader(format: .pax, filter: .gzip, fileHandle: fh) + _ = try reader.extractContents(to: destination) + } else { + let destFd = open(destination.path, O_WRONLY | O_CREAT | O_TRUNC, 0o644) + guard destFd != -1 else { + throw ContainerizationError( + .internalError, + message: "copyOut: failed to open '\(destination.path)': \(String(cString: strerror(errno)))" + ) + } + defer { close(destFd) } + + var buf = [UInt8](repeating: 0, count: chunkSize) + while true { + let n = read(conn.fileDescriptor, &buf, buf.count) + if n == 0 { break } + guard n > 0 else { + throw ContainerizationError( + .internalError, + message: "copyOut: vsock read error: \(String(cString: strerror(errno)))" + ) + } + var written = 0 + while written < n { + let w = buf.withUnsafeBytes { ptr in + write(destFd, ptr.baseAddress! + written, n - written) + } + guard w > 0 else { + throw ContainerizationError( + .internalError, + message: "copyOut: write error: \(String(cString: strerror(errno)))" + ) + } + written += w + } + } + } + continuation.resume() + } catch { + continuation.resume(throwing: error) + } + } + } + } + + try await group.waitForAll() + } + } + + /// Where a copy lands in the guest, given what the destination already is. + /// + /// A destination that names an existing directory receives the source + /// under its own name, the way `cp` behaves. + private func resolveCopyInGuestPath( + from source: URL, + to destination: URL, + sourceIsDirectory: Bool, + using vminitd: Vminitd + ) async throws -> URL { + let guestDestination = URL(filePath: guestRoot).appending(path: destination.path) + + let stat: ContainerizationOS.Stat? + do { + stat = try await vminitd.stat(path: guestDestination) + } catch let error as ContainerizationError where error.code == .notFound { + stat = nil + } + // Any other error propagates so transport and permission failures are visible. + + guard let stat else { + if destination.hasDirectoryPath && !sourceIsDirectory { + throw ContainerizationError( + .invalidArgument, + message: "destination directory does not exist: \(destination.path)" + ) + } + return guestDestination + } + + let destinationIsDirectory = (stat.mode & UInt32(S_IFMT)) == UInt32(S_IFDIR) + guard destinationIsDirectory else { + if sourceIsDirectory { + throw ContainerizationError( + .invalidArgument, + message: "cannot copy directory over existing file: \(destination.path)" + ) + } + return guestDestination + } + + return guestDestination.appendingPathComponent(source.lastPathComponent) + } +} diff --git a/Sources/Containerization/LinuxContainer.swift b/Sources/Containerization/LinuxContainer.swift index 4d5289a7..eca34a1a 100644 --- a/Sources/Containerization/LinuxContainer.swift +++ b/Sources/Containerization/LinuxContainer.swift @@ -669,8 +669,8 @@ extension LinuxContainer { let vm = try await self.vmm.create(config: creationConfig) let relayManager = UnixSocketRelayManager(vm: vm, log: self.logger) + try await vm.start() do { - try await vm.start() let storageForAgent = containerStorage try await vm.withAgent { agent in try await agent.standardSetup() @@ -1194,7 +1194,7 @@ extension LinuxContainer { } /// Default chunk size for file transfers (1MiB). - public static let defaultCopyChunkSize = 1024 * 1024 + public static let defaultCopyChunkSize = GuestFileTransfer.defaultChunkSize /// Copy a file or directory from the host into the container. /// @@ -1210,148 +1210,14 @@ extension LinuxContainer { ) async throws { try await self.state.withLock { let state = try $0.startedState("copyIn") - - var isDirectory: ObjCBool = false - guard FileManager.default.fileExists(atPath: source.path, isDirectory: &isDirectory) else { - throw ContainerizationError(.notFound, message: "copyIn: source not found '\(source.path)'") - } - let isArchive = isDirectory.boolValue - - let guestPath: URL = try await state.vm.withAgent { agent in - guard let vminitd = agent as? Vminitd else { - throw ContainerizationError(.unsupported, message: "copyIn requires Vminitd agent") - } - - return try await self.resolveCopyInGuestPath( - from: source, - to: destination, - sourceIsDirectory: isArchive, - using: vminitd - ) - } - - let port = self.hostVsockPorts.wrappingAdd(1, ordering: .relaxed).oldValue - let listener = try state.vm.listen(port) - - try await withThrowingTaskGroup(of: Void.self) { group in - group.addTask { - try await state.vm.withAgent { agent in - guard let vminitd = agent as? Vminitd else { - throw ContainerizationError(.unsupported, message: "copyIn requires Vminitd agent") - } - try await vminitd.copy( - direction: .copyIn, - guestPath: guestPath, - vsockPort: port, - mode: mode, - createParents: createParents, - isArchive: isArchive - ) - } - } - - group.addTask { - guard let conn = await listener.first(where: { _ in true }) else { - throw ContainerizationError(.internalError, message: "copyIn: vsock connection not established") - } - try listener.finish() - - try await withCheckedThrowingContinuation { (continuation: CheckedContinuation) in - self.copyQueue.async { - do { - defer { conn.closeFile() } - - if isArchive { - let writer = try ArchiveWriter(configuration: .init(format: .pax, filter: .gzip)) - try writer.open(fileDescriptor: conn.fileDescriptor) - try writer.archiveDirectory(source) - try writer.finishEncoding() - } else { - let srcFd = open(source.path, O_RDONLY) - guard srcFd != -1 else { - throw ContainerizationError( - .internalError, - message: "copyIn: failed to open '\(source.path)': \(String(cString: strerror(errno)))" - ) - } - defer { close(srcFd) } - - var buf = [UInt8](repeating: 0, count: chunkSize) - while true { - let n = read(srcFd, &buf, buf.count) - if n == 0 { break } - guard n > 0 else { - throw ContainerizationError( - .internalError, - message: "copyIn: read error: \(String(cString: strerror(errno)))" - ) - } - var written = 0 - while written < n { - let w = buf.withUnsafeBytes { ptr in - write(conn.fileDescriptor, ptr.baseAddress! + written, n - written) - } - guard w > 0 else { - throw ContainerizationError( - .internalError, - message: "copyIn: vsock write error: \(String(cString: strerror(errno)))" - ) - } - written += w - } - } - } - continuation.resume() - } catch { - continuation.resume(throwing: error) - } - } - } - } - - try await group.waitForAll() - } - } - } - - private func resolveCopyInGuestPath( - from source: URL, - to destination: URL, - sourceIsDirectory: Bool, - using vminitd: Vminitd - ) async throws -> URL { - let guestDestination = URL(filePath: self.root).appending(path: destination.path) - - let stat: ContainerizationOS.Stat? - do { - stat = try await vminitd.stat(path: guestDestination) - } catch let error as ContainerizationError where error.code == .notFound { - stat = nil - } - // Any other error propagates so transport and permission failures are visible. - - guard let stat else { - if destination.hasDirectoryPath && !sourceIsDirectory { - throw ContainerizationError( - .invalidArgument, - message: "destination directory does not exist: \(destination.path)" - ) - } - return guestDestination - } - - let destinationIsDirectory = (stat.mode & UInt32(S_IFMT)) == UInt32(S_IFDIR) - guard destinationIsDirectory else { - if sourceIsDirectory { - throw ContainerizationError( - .invalidArgument, - message: "cannot copy directory over existing file: \(destination.path)" - ) - } - return guestDestination + try await self.transfer(vm: state.vm).copyIn( + from: source, + to: destination, + mode: mode, + createParents: createParents, + chunkSize: chunkSize + ) } - - return guestDestination.appendingPathComponent(source.lastPathComponent) } /// Copy a file or directory from the container to the host. @@ -1367,104 +1233,24 @@ extension LinuxContainer { ) async throws { try await self.state.withLock { let state = try $0.startedState("copyOut") - - if createParents { - let parentDir = destination.deletingLastPathComponent() - try FileManager.default.createDirectory(at: parentDir, withIntermediateDirectories: true) - } - - let guestPath = URL(filePath: self.root).appending(path: source.path) - let port = self.hostVsockPorts.wrappingAdd(1, ordering: .relaxed).oldValue - let listener = try state.vm.listen(port) - - let (metadataStream, metadataCont) = AsyncStream.makeStream(of: Vminitd.CopyMetadata.self) - - try await withThrowingTaskGroup(of: Void.self) { group in - group.addTask { - defer { metadataCont.finish() } - try await state.vm.withAgent { agent in - guard let vminitd = agent as? Vminitd else { - throw ContainerizationError(.unsupported, message: "copyOut requires Vminitd agent") - } - try await vminitd.copy( - direction: .copyOut, - guestPath: guestPath, - vsockPort: port, - onMetadata: { meta in - metadataCont.yield(meta) - metadataCont.finish() - } - ) - } - } - - group.addTask { - guard let metadata = await metadataStream.first(where: { _ in true }) else { - throw ContainerizationError(.internalError, message: "copyOut: no metadata received") - } - - guard let conn = await listener.first(where: { _ in true }) else { - throw ContainerizationError(.internalError, message: "copyOut: vsock connection not established") - } - try listener.finish() - - try await withCheckedThrowingContinuation { (continuation: CheckedContinuation) in - self.copyQueue.async { - do { - defer { conn.closeFile() } - - if metadata.isArchive { - try FileManager.default.createDirectory(at: destination, withIntermediateDirectories: true) - let fh = FileHandle(fileDescriptor: dup(conn.fileDescriptor), closeOnDealloc: true) - let reader = try ArchiveReader(format: .pax, filter: .gzip, fileHandle: fh) - _ = try reader.extractContents(to: destination) - } else { - let destFd = open(destination.path, O_WRONLY | O_CREAT | O_TRUNC, 0o644) - guard destFd != -1 else { - throw ContainerizationError( - .internalError, - message: "copyOut: failed to open '\(destination.path)': \(String(cString: strerror(errno)))" - ) - } - defer { close(destFd) } - - var buf = [UInt8](repeating: 0, count: chunkSize) - while true { - let n = read(conn.fileDescriptor, &buf, buf.count) - if n == 0 { break } - guard n > 0 else { - throw ContainerizationError( - .internalError, - message: "copyOut: vsock read error: \(String(cString: strerror(errno)))" - ) - } - var written = 0 - while written < n { - let w = buf.withUnsafeBytes { ptr in - write(destFd, ptr.baseAddress! + written, n - written) - } - guard w > 0 else { - throw ContainerizationError( - .internalError, - message: "copyOut: write error: \(String(cString: strerror(errno)))" - ) - } - written += w - } - } - } - continuation.resume() - } catch { - continuation.resume(throwing: error) - } - } - } - } - - try await group.waitForAll() - } + try await self.transfer(vm: state.vm).copyOut( + from: source, + to: destination, + createParents: createParents, + chunkSize: chunkSize + ) } } + + /// A transfer against this container's filesystem, on a port of its own. + private func transfer(vm: any VirtualMachineInstance) -> GuestFileTransfer { + GuestFileTransfer( + vm: vm, + guestRoot: self.root, + port: self.hostVsockPorts.wrappingAdd(1, ordering: .relaxed).oldValue, + queue: self.copyQueue + ) + } } extension VirtualMachineInstance { diff --git a/Sources/Containerization/LinuxPod.swift b/Sources/Containerization/LinuxPod.swift index 140f01ab..197f29bc 100644 --- a/Sources/Containerization/LinuxPod.swift +++ b/Sources/Containerization/LinuxPod.swift @@ -219,6 +219,9 @@ public final class LinuxPod: Sendable { // the host. private let guestVsockPorts: Atomic + // Where the blocking reads and writes of a file transfer run. + private let copyQueue = DispatchQueue(label: "com.apple.containerization.copy") + private struct State: Sendable { var phase: Phase var containers: [String: PodContainer] @@ -1355,6 +1358,82 @@ extension LinuxPod { } } + /// Default chunk size for file transfers (1MiB). + public static let defaultCopyChunkSize = GuestFileTransfer.defaultChunkSize + + /// Copy a file or directory from the host into a container in the pod. + /// + /// Data transfer happens over a dedicated vsock connection. For + /// directories, the source is archived as tar+gzip and streamed directly + /// through vsock without intermediate temp files. + public func copyIn( + _ containerID: String, + from source: URL, + to destination: URL, + mode: UInt32 = 0o644, + createParents: Bool = true, + chunkSize: Int = defaultCopyChunkSize + ) async throws { + try await self.state.withLock { state in + try await self.transfer(containerID, state: state, operation: "copyIn").copyIn( + from: source, + to: destination, + mode: mode, + createParents: createParents, + chunkSize: chunkSize + ) + } + } + + /// Copy a file or directory from a container in the pod to the host. + /// + /// Data transfer happens over a dedicated vsock connection. For + /// directories, the guest archives the source as tar+gzip and streams it + /// directly through vsock. The host extracts the archive without + /// intermediate temp files. + public func copyOut( + _ containerID: String, + from source: URL, + to destination: URL, + createParents: Bool = true, + chunkSize: Int = defaultCopyChunkSize + ) async throws { + try await self.state.withLock { state in + try await self.transfer(containerID, state: state, operation: "copyOut").copyOut( + from: source, + to: destination, + createParents: createParents, + chunkSize: chunkSize + ) + } + } + + /// A transfer against one container's filesystem, on a port of its own. + private func transfer(_ containerID: String, state: State, operation: String) throws -> GuestFileTransfer { + let createdState = try state.phase.createdState(operation) + + guard let container = state.containers[containerID] else { + throw ContainerizationError( + .notFound, + message: "container \(containerID) not found in pod" + ) + } + + guard container.state == .started else { + throw ContainerizationError( + .invalidState, + message: "container \(containerID) must be started to copy files" + ) + } + + return GuestFileTransfer( + vm: createdState.vm, + guestRoot: Self.guestRootfsPath(containerID), + port: self.hostVsockPorts.wrappingAdd(1, ordering: .relaxed).oldValue, + queue: self.copyQueue + ) + } + /// Close a container's standard input to signal no more input is arriving. public func closeContainerStdin(_ containerID: String) async throws { try await self.state.withLock { state in diff --git a/Sources/Integration/PodTests.swift b/Sources/Integration/PodTests.swift index 19854194..5703bde1 100644 --- a/Sources/Integration/PodTests.swift +++ b/Sources/Integration/PodTests.swift @@ -2477,4 +2477,88 @@ extension IntegrationSuite { } } #endif + + /// A file copied into one container in a pod arrives in that container and + /// nowhere else, and comes back out with what it held. + func testPodCopyRoundTrip() async throws { + let id = "test-pod-copy-round-trip" + + let bs = try await bootstrap(id) + let pod = try LinuxPod(id, vmm: bs.vmm) { config in + config.cpus = 4 + config.memoryInBytes = 1024.mib() + config.bootLog = bs.bootLog + } + + let testContent = "Hello from the host! This is a pod copy test." + let hostDirectory = FileManager.default.uniqueTemporaryDirectory(create: true) + let hostFile = hostDirectory.appendingPathComponent("test-input.txt") + try testContent.write(to: hostFile, atomically: true, encoding: .utf8) + + try await pod.addContainer("holder", rootfs: try cloneRootfs(bs.rootfs, testID: id, containerID: "holder")) { config in + config.process.arguments = ["sleep", "100"] + } + try await pod.addContainer("bystander", rootfs: try cloneRootfs(bs.rootfs, testID: id, containerID: "bystander")) { config in + config.process.arguments = ["sleep", "100"] + } + + do { + try await pod.create() + try await pod.startContainer("holder") + try await pod.startContainer("bystander") + + try await pod.copyIn( + "holder", + from: hostFile, + to: URL(filePath: "/tmp/copied-file.txt") + ) + + let buffer = BufferWriter() + let exec = try await pod.execInContainer("holder", processID: "verify-copy") { config in + config.arguments = ["cat", "/tmp/copied-file.txt"] + config.stdout = buffer + } + try await exec.start() + let status = try await exec.wait() + try await exec.delete() + + guard status.exitCode == 0 else { + throw IntegrationError.assert(msg: "cat in the container copied into failed with status \(status)") + } + guard String(data: buffer.data, encoding: .utf8) == testContent else { + throw IntegrationError.assert( + msg: "expected '\(testContent)', got '\(String(data: buffer.data, encoding: .utf8) ?? "nil")'") + } + + // The containers share a machine and not a filesystem, so the copy + // reached one of them alone. + let bystander = try await pod.execInContainer("bystander", processID: "verify-absent") { config in + config.arguments = ["test", "-e", "/tmp/copied-file.txt"] + } + try await bystander.start() + let bystanderStatus = try await bystander.wait() + try await bystander.delete() + + guard bystanderStatus.exitCode != 0 else { + throw IntegrationError.assert(msg: "the copy reached a container it was not addressed to") + } + + let returned = hostDirectory.appendingPathComponent("test-output.txt") + try await pod.copyOut( + "holder", + from: URL(filePath: "/tmp/copied-file.txt"), + to: returned + ) + + let returnedContent = try String(contentsOf: returned, encoding: .utf8) + guard returnedContent == testContent else { + throw IntegrationError.assert(msg: "expected '\(testContent)' back, got '\(returnedContent)'") + } + + try await pod.stop() + } catch { + try? await pod.stop() + throw error + } + } } diff --git a/Sources/Integration/Suite.swift b/Sources/Integration/Suite.swift index bd4ec656..769ccc12 100644 --- a/Sources/Integration/Suite.swift +++ b/Sources/Integration/Suite.swift @@ -559,6 +559,7 @@ struct IntegrationSuite: AsyncParsableCommand { Test("pod memory events OOM kill", testPodMemoryEventsOOMKill), Test("pod container resource limits", testPodContainerResourceLimits), Test("pod container filesystem isolation", testPodContainerFilesystemIsolation), + Test("pod copy round trip", testPodCopyRoundTrip), Test("pod container PID namespace isolation", testPodContainerPIDNamespaceIsolation), Test("pod container independent resource limits", testPodContainerIndependentResourceLimits), Test("pod shared PID namespace", testPodSharedPIDNamespace), From 8f7a007c5d83298f79b27a824e4882f835f8b1a0 Mon Sep 17 00:00:00 2001 From: Aaron Paterson Date: Fri, 7 Aug 2026 20:42:42 +0000 Subject: [PATCH 11/21] Give a pod the machine it asked for A container with a machine of its own is given the size it asked for plus the guest agent's, so a container that asks for a gibibyte has a gibibyte and the agent runs beside it. A pod hands its size to the machine as it stands, so the agent comes out of what the pod asked for and its containers are left with less: a pod given twelve gibibytes reports eleven and a half. The pod's size and the agent's are added the same way, and what a pod was given is what its containers have. --- Sources/Containerization/LinuxPod.swift | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/Sources/Containerization/LinuxPod.swift b/Sources/Containerization/LinuxPod.swift index 197f29bc..fab44f0f 100644 --- a/Sources/Containerization/LinuxPod.swift +++ b/Sources/Containerization/LinuxPod.swift @@ -57,6 +57,13 @@ public final class LinuxPod: Sendable { public var interfaces: [any Interface] = [] /// Whether nested virtualization should be turned on for the pod. public var virtualization: Bool = false + /// Additional CPU cores to allocate for the virtual machine on top of + /// the pod's configured `cpus`, so what the pod was given is what its + /// containers have rather than what the guest agent leaves of it. + public var cpuOverhead: Int = 1 + /// Additional memory in bytes to allocate for the virtual machine on + /// top of the pod's configured `memoryInBytes`. + public var memoryOverhead: UInt64 = 128.mib() /// Optional file path to store serial boot logs. public var bootLog: BootLog? /// Whether containers in the pod should share a PID namespace. @@ -669,9 +676,12 @@ extension LinuxPod { return false } + // The machine carries the guest agent as well as the containers, + // so it is given the pod's size and the agent's on top; what the + // pod was given is then what its containers have. var vmConfig = VMConfiguration( - cpus: self.config.cpus, - memoryInBytes: self.config.memoryInBytes, + cpus: self.config.cpus + self.config.cpuOverhead, + memoryInBytes: self.config.memoryInBytes + self.config.memoryOverhead, interfaces: self.config.interfaces, storage: machineStorage, bootLog: self.config.bootLog, @@ -681,9 +691,8 @@ extension LinuxPod { let creationConfig = StandardVMConfig(configuration: vmConfig) let vm = try await self.vmm.create(config: creationConfig) let relayManager = UnixSocketRelayManager(vm: vm) - try await vm.start() - do { + try await vm.start() let containers = state.containers let shareProcessNamespace = self.config.shareProcessNamespace let pauseProcessHolder = Mutex(nil) From ea2a9a221eaeacda38c6b9dc14375873d56e0b8e Mon Sep 17 00:00:00 2001 From: Aaron Paterson Date: Fri, 7 Aug 2026 21:10:11 +0000 Subject: [PATCH 12/21] Let a pod's containers be given a writable layer A container with a machine of its own can be given a block device to write into, with its image mounted read-only beneath, so the image is left as it is and every write lands in the layer. The same container placed in a pod has nowhere to be given one. The layer is attached with the container's other block devices, mounted in the guest as the upper layer of an overlay over the image, dropped from the mounts the runtime spec carries, and unmounted with the container. A pod's containers take one whether they were placed before the machine booted or hotplugged into it afterwards. --- .../Containerization/CHHotplugProvider.swift | 4 +- .../CHVirtualMachineInstance.swift | 4 +- .../Containerization/HotplugProvider.swift | 4 +- Sources/Containerization/LinuxContainer.swift | 43 +------ Sources/Containerization/LinuxPod.swift | 76 ++++++++++-- Sources/Containerization/OverlayRootfs.swift | 64 ++++++++++ .../VZVirtualMachineInstance.swift | 4 +- .../VirtualMachineInstance.swift | 6 +- Sources/Integration/PodTests.swift | 117 ++++++++++++++++++ Sources/Integration/Suite.swift | 2 + 10 files changed, 268 insertions(+), 56 deletions(-) create mode 100644 Sources/Containerization/OverlayRootfs.swift diff --git a/Sources/Containerization/CHHotplugProvider.swift b/Sources/Containerization/CHHotplugProvider.swift index 380cba76..e6a33d38 100644 --- a/Sources/Containerization/CHHotplugProvider.swift +++ b/Sources/Containerization/CHHotplugProvider.swift @@ -150,12 +150,12 @@ final class CHHotplugProvider: HotplugProvider { } } - func registerMounts(id: String, rootfs: AttachedFilesystem, additionalMounts: [Mount]) throws { + func registerMounts(id: String, rootfs: AttachedFilesystem, writableLayer: AttachedFilesystem?, additionalMounts: [Mount]) throws { var mounts: [AttachedFilesystem] = [] for mount in additionalMounts { mounts.append(try AttachedFilesystem(mount: mount, allocator: allocator)) } - let container = ContainerAttachments(rootfs: rootfs, mounts: mounts) + let container = ContainerAttachments(rootfs: rootfs, writableLayer: writableLayer, mounts: mounts) _storage.withLock { $0.containers[id] = container } diff --git a/Sources/Containerization/CHVirtualMachineInstance.swift b/Sources/Containerization/CHVirtualMachineInstance.swift index 1704ce92..11d7e0a3 100644 --- a/Sources/Containerization/CHVirtualMachineInstance.swift +++ b/Sources/Containerization/CHVirtualMachineInstance.swift @@ -490,8 +490,8 @@ extension CHVirtualMachineInstance: VirtualMachineInstance { try await hotplug.releaseVirtioFS(id: id) } - public func registerMounts(id: String, rootfs: AttachedFilesystem, additionalMounts: [Mount]) throws { - try hotplug.registerMounts(id: id, rootfs: rootfs, additionalMounts: additionalMounts) + public func registerMounts(id: String, rootfs: AttachedFilesystem, writableLayer: AttachedFilesystem?, additionalMounts: [Mount]) throws { + try hotplug.registerMounts(id: id, rootfs: rootfs, writableLayer: writableLayer, additionalMounts: additionalMounts) } } diff --git a/Sources/Containerization/HotplugProvider.swift b/Sources/Containerization/HotplugProvider.swift index 0835077f..94fad4b4 100644 --- a/Sources/Containerization/HotplugProvider.swift +++ b/Sources/Containerization/HotplugProvider.swift @@ -30,8 +30,10 @@ public protocol HotplugProvider: Sendable { /// - Parameters: /// - id: The container ID /// - rootfs: The rootfs attachment from hotplug + /// - writableLayer: The container's writable layer attachment when it + /// has one /// - additionalMounts: Additional mounts to register - func registerMounts(id: String, rootfs: AttachedFilesystem, additionalMounts: [Mount]) throws + func registerMounts(id: String, rootfs: AttachedFilesystem, writableLayer: AttachedFilesystem?, additionalMounts: [Mount]) throws /// Release a hotplug device. /// - Parameter id: The container ID who should be released diff --git a/Sources/Containerization/LinuxContainer.swift b/Sources/Containerization/LinuxContainer.swift index eca34a1a..ce57488f 100644 --- a/Sources/Containerization/LinuxContainer.swift +++ b/Sources/Containerization/LinuxContainer.swift @@ -575,47 +575,16 @@ extension LinuxContainer { rootfsPath: String, agent: VirtualMachineAgent ) async throws { - let rootfsAttachment = attached.rootfs - if let writableAttachment = attached.writableLayer { - // Set up overlayfs with image as lower layer and writable layer as upper. - let lowerPath = "/run/container/\(self.id)/lower" - let upperMountPath = "/run/container/\(self.id)/upper" - let upperPath = "/run/container/\(self.id)/upper/diff" - let workPath = "/run/container/\(self.id)/upper/work" - - // Mount the image (lower layer) as read-only. - var lowerMount = rootfsAttachment.to - lowerMount.destination = lowerPath - if !lowerMount.options.contains("ro") { - lowerMount.options.append("ro") - } - try await agent.mount(lowerMount) - - // Mount the writable layer. - var upperMount = writableAttachment.to - upperMount.destination = upperMountPath - try await agent.mount(upperMount) - - // Create the upper and work directories inside the writable layer. - try await agent.mkdir(path: upperPath, all: true, perms: 0o755) - try await agent.mkdir(path: workPath, all: true, perms: 0o755) - - // Mount the overlay. - let overlayMount = ContainerizationOCI.Mount( - type: "overlay", - source: "overlay", - destination: rootfsPath, - options: [ - "lowerdir=\(lowerPath)", - "upperdir=\(upperPath)", - "workdir=\(workPath)", - ] + try await agent.mountOverlayRootfs( + containerID: self.id, + rootfsAttachment: attached.rootfs, + writableAttachment: writableAttachment, + rootfsPath: rootfsPath ) - try await agent.mount(overlayMount) } else { // No writable layer. Mount rootfs directly. - var rootfs = rootfsAttachment.to + var rootfs = attached.rootfs.to rootfs.destination = rootfsPath try await agent.mount(rootfs) } diff --git a/Sources/Containerization/LinuxPod.swift b/Sources/Containerization/LinuxPod.swift index fab44f0f..76967821 100644 --- a/Sources/Containerization/LinuxPod.swift +++ b/Sources/Containerization/LinuxPod.swift @@ -202,6 +202,7 @@ public final class LinuxPod: Sendable { private struct PodContainer: Sendable { let id: String let rootfs: Mount + let writableLayer: Mount? let config: ContainerConfiguration var state: ContainerState var process: LinuxProcess? @@ -329,7 +330,7 @@ public final class LinuxPod: Sendable { ) } - private func generateRuntimeSpec(containerID: String, config: ContainerConfiguration, rootfs: Mount) -> Spec { + private func generateRuntimeSpec(containerID: String, config: ContainerConfiguration, rootfs: Mount, writableLayer: Mount? = nil) -> Spec { var spec = Self.createDefaultRuntimeSpec(containerID, podID: self.id) // Process configuration @@ -354,7 +355,7 @@ public final class LinuxPod: Sendable { // If the rootfs was requested as read-only, set it in the OCI spec. // We let the OCI runtime remount as ro, instead of doing it originally. - spec.root?.readonly = rootfs.options.contains("ro") + spec.root?.readonly = rootfs.options.contains("ro") && writableLayer == nil // Resource limits (if specified) if let cpus = config.cpus, cpus > 0 { @@ -413,9 +414,14 @@ extension LinuxPod { /// When called before `create()`, the container is registered for setup during VM creation. /// When called after `create()`, the container is hotplugged into the running VM. /// If the underlying VMM does not support hotplug, an error is thrown. + /// - Parameters: + /// - writableLayer: Optional writable layer mount. When provided, an overlayfs is used with + /// the container's rootfs as the lower layer and this as the upper layer, so all writes + /// go to this layer instead of the rootfs. public func addContainer( _ id: String, rootfs: Mount, + writableLayer: Mount? = nil, configuration: @Sendable @escaping (inout ContainerConfiguration) throws -> Void ) async throws { guard id.count <= Self.maxIDLength else { @@ -424,6 +430,14 @@ extension LinuxPod { message: "container id length \(id.count) exceeds maximum of \(Self.maxIDLength) characters" ) } + if let writableLayer { + guard writableLayer.isBlock else { + throw ContainerizationError( + .invalidArgument, + message: "writableLayer must be a block device" + ) + } + } try await self.state.withLock { state in guard state.containers[id] == nil else { throw ContainerizationError( @@ -451,6 +465,7 @@ extension LinuxPod { state.containers[id] = PodContainer( id: id, rootfs: rootfs, + writableLayer: writableLayer, config: config, state: .registered, process: nil, @@ -460,6 +475,9 @@ extension LinuxPod { case .created(let createdState): let vm = createdState.vm + // Strip "ro" as create() does: readonly is expressed through + // the OCI spec's root.readonly field and a remount in vmexec + // after setup completes, so the device attaches writable. var modifiedRootfs = rootfs modifiedRootfs.options.removeAll(where: { $0 == "ro" }) @@ -467,6 +485,13 @@ extension LinuxPod { var updatedFileMountContext = fileMountContext do { + // The writable layer is a block device like the rootfs, + // attached alongside it so the overlay has both layers. + var writableAttachment: AttachedFilesystem? + if let writableLayer { + writableAttachment = try await vm.hotplug(writableLayer, id: id) + } + let virtioFSMounts = fileMountContext.transformedMounts.filter { if case .virtiofs(_) = $0.runtimeOptions { return true } return false @@ -477,13 +502,23 @@ extension LinuxPod { let agent = try await vm.dialAgent() do { - var mount = attachment.to - mount.destination = Self.guestRootfsPath(id) - try await agent.mount(mount) + if let writableAttachment { + try await agent.mountOverlayRootfs( + containerID: id, + rootfsAttachment: attachment, + writableAttachment: writableAttachment, + rootfsPath: Self.guestRootfsPath(id) + ) + } else { + var mount = attachment.to + mount.destination = Self.guestRootfsPath(id) + try await agent.mount(mount) + } - // Filter out shared mounts — those are handled separately as - // pod volume bind mounts. Without it here, a container added to an - // already-created would add a duplicated mount into the shared VM. + // Shared mounts are handled separately as pod volume + // bind mounts; without the filter here, a container + // added to an already-created pod would add a + // duplicated mount into the shared VM. let nonSharedMounts = fileMountContext.transformedMounts.filter { if case .shared = $0.runtimeOptions { return false } return true @@ -491,6 +526,7 @@ extension LinuxPod { try vm.registerMounts( id: id, rootfs: attachment, + writableLayer: writableAttachment, additionalMounts: nonSharedMounts ) @@ -591,6 +627,7 @@ extension LinuxPod { state.containers[id] = PodContainer( id: id, rootfs: rootfs, + writableLayer: writableLayer, config: config, state: .created, process: nil, @@ -630,6 +667,7 @@ extension LinuxPod { } machineStorage.containers[id] = ContainerMounts( rootfs: modifiedRootfs, + writableLayer: container.writableLayer, mounts: containerMounts ) } @@ -809,6 +847,15 @@ extension LinuxPod { guard let attached = vm.storage.containers[container.id] else { throw ContainerizationError(.notFound, message: "rootfs mount not found for container \(container.id)") } + if let writableAttachment = attached.writableLayer { + try await agent.mountOverlayRootfs( + containerID: container.id, + rootfsAttachment: attached.rootfs, + writableAttachment: writableAttachment, + rootfsPath: Self.guestRootfsPath(container.id) + ) + continue + } var rootfs = attached.rootfs.to rootfs.destination = Self.guestRootfsPath(container.id) try await agent.mount(rootfs) @@ -936,7 +983,7 @@ extension LinuxPod { let agent = try await createdState.vm.dialAgent() do { - var spec = self.generateRuntimeSpec(containerID: containerID, config: container.config, rootfs: container.rootfs) + var spec = self.generateRuntimeSpec(containerID: containerID, config: container.config, rootfs: container.rootfs, writableLayer: container.writableLayer) // We don't need the rootfs, nor do OCI runtimes want it included. // Also filter out file mount holding directories - we mount those separately under /run. // Transform virtiofs mounts to bind mounts from /run/virtiofs/{tag} @@ -1101,12 +1148,21 @@ extension LinuxPod { try await process.kill(.kill) try await process.wait(timeoutInSeconds: 3) + let hasWritableLayer = container.writableLayer != nil try await createdState.vm.withAgent { agent in // Unmount the rootfs try await agent.umount( path: Self.guestRootfsPath(containerID), flags: 0 ) + + // If we have a writable layer, we also need to unmount the lower and upper layers. + if hasWritableLayer { + let upperPath = "/run/container/\(containerID)/upper" + let lowerPath = "/run/container/\(containerID)/lower" + try await agent.umount(path: upperPath, flags: 0) + try await agent.umount(path: lowerPath, flags: 0) + } } // Release the hotplug device and virtiofs shares so they can be reused by new containers @@ -1261,7 +1317,7 @@ extension LinuxPod { ) } - var spec = self.generateRuntimeSpec(containerID: containerID, config: container.config, rootfs: container.rootfs) + var spec = self.generateRuntimeSpec(containerID: containerID, config: container.config, rootfs: container.rootfs, writableLayer: container.writableLayer) // Inherit environment variables, working directory, user, capabilities, rlimits from container process. // Reset: process arguments, terminal, stdio as these are not supposed to be inherited. var config = container.config.process diff --git a/Sources/Containerization/OverlayRootfs.swift b/Sources/Containerization/OverlayRootfs.swift new file mode 100644 index 00000000..9af76763 --- /dev/null +++ b/Sources/Containerization/OverlayRootfs.swift @@ -0,0 +1,64 @@ +//===----------------------------------------------------------------------===// +// Copyright © 2026 Apple Inc. and the Containerization project authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//===----------------------------------------------------------------------===// + +import ContainerizationOCI + +extension VirtualMachineAgent { + /// Mount a container's rootfs as an overlay, with the image as the lower + /// layer and the container's writable layer as the upper, so that writes + /// land in the layer and the image stays as it is. + func mountOverlayRootfs( + containerID: String, + rootfsAttachment: AttachedFilesystem, + writableAttachment: AttachedFilesystem, + rootfsPath: String + ) async throws { + let lowerPath = "/run/container/\(containerID)/lower" + let upperMountPath = "/run/container/\(containerID)/upper" + let upperPath = "/run/container/\(containerID)/upper/diff" + let workPath = "/run/container/\(containerID)/upper/work" + + // Mount the image (lower layer) as read-only. + var lowerMount = rootfsAttachment.to + lowerMount.destination = lowerPath + if !lowerMount.options.contains("ro") { + lowerMount.options.append("ro") + } + try await self.mount(lowerMount) + + // Mount the writable layer. + var upperMount = writableAttachment.to + upperMount.destination = upperMountPath + try await self.mount(upperMount) + + // Create the upper and work directories inside the writable layer. + try await self.mkdir(path: upperPath, all: true, perms: 0o755) + try await self.mkdir(path: workPath, all: true, perms: 0o755) + + // Mount the overlay. + let overlayMount = ContainerizationOCI.Mount( + type: "overlay", + source: "overlay", + destination: rootfsPath, + options: [ + "lowerdir=\(lowerPath)", + "upperdir=\(upperPath)", + "workdir=\(workPath)", + ] + ) + try await self.mount(overlayMount) + } +} diff --git a/Sources/Containerization/VZVirtualMachineInstance.swift b/Sources/Containerization/VZVirtualMachineInstance.swift index 69828ec9..135879e0 100644 --- a/Sources/Containerization/VZVirtualMachineInstance.swift +++ b/Sources/Containerization/VZVirtualMachineInstance.swift @@ -325,9 +325,9 @@ extension VZVirtualMachineInstance: VirtualMachineInstance { return try await hotplugProvider.hotplug(block, id: id) } - public func registerMounts(id: String, rootfs: AttachedFilesystem, additionalMounts: [Mount]) throws { + public func registerMounts(id: String, rootfs: AttachedFilesystem, writableLayer: AttachedFilesystem?, additionalMounts: [Mount]) throws { guard let hotplugProvider else { return } - try hotplugProvider.registerMounts(id: id, rootfs: rootfs, additionalMounts: additionalMounts) + try hotplugProvider.registerMounts(id: id, rootfs: rootfs, writableLayer: writableLayer, additionalMounts: additionalMounts) } public func releaseHotplug(id: String) async throws { diff --git a/Sources/Containerization/VirtualMachineInstance.swift b/Sources/Containerization/VirtualMachineInstance.swift index b9340612..da7935d5 100644 --- a/Sources/Containerization/VirtualMachineInstance.swift +++ b/Sources/Containerization/VirtualMachineInstance.swift @@ -78,8 +78,10 @@ public protocol VirtualMachineInstance: Sendable { /// so they can be found when building the container's OCI spec. /// - Parameter id: The container ID /// - Parameter rootfs: The rootfs attachment from hotplug + /// - Parameter writableLayer: The container's writable layer attachment when it + /// has one /// - Parameter additionalMounts: Additional mounts (like /proc, /sys) to register - func registerMounts(id: String, rootfs: AttachedFilesystem, additionalMounts: [Mount]) throws + func registerMounts(id: String, rootfs: AttachedFilesystem, writableLayer: AttachedFilesystem?, additionalMounts: [Mount]) throws /// Release a hotplug device. /// This should be called when a hotplugged container is stopped or fails to start. @@ -107,7 +109,7 @@ extension VirtualMachineInstance { public func hotplug(_ block: Mount, id: String) async throws -> AttachedFilesystem { throw ContainerizationError(.unsupported, message: "hotplug not supported") } - public func registerMounts(id: String, rootfs: AttachedFilesystem, additionalMounts: [Mount]) throws { + public func registerMounts(id: String, rootfs: AttachedFilesystem, writableLayer: AttachedFilesystem?, additionalMounts: [Mount]) throws { // no-op default } public func releaseHotplug(id: String) async throws { diff --git a/Sources/Integration/PodTests.swift b/Sources/Integration/PodTests.swift index 5703bde1..2c0dffc2 100644 --- a/Sources/Integration/PodTests.swift +++ b/Sources/Integration/PodTests.swift @@ -2478,6 +2478,123 @@ extension IntegrationSuite { } #endif + /// A container in a pod given a writable layer writes into it, and the + /// image it was built from is left as it is for the pod's others. + /// Add a container with a writable layer to a pod whose machine is + /// already running: the overlay assembles from the two disks attached + /// while it runs, and writes land in the layer. + func testPodHotplugWritableLayer() async throws { + let id = "test-pod-hotplug-writable-layer" + let bs = try await bootstrap(id) + + let pod = try LinuxPod(id, vmm: bs.vmm) { config in + config.cpus = 4 + config.memoryInBytes = 1024.mib() + config.bootLog = bs.bootLog + } + + try await pod.addContainer("seed", rootfs: try cloneRootfs(bs.rootfs, testID: id, containerID: "seed")) { config in + config.process.arguments = ["/bin/sleep", "infinity"] + } + + try await pod.create() + + let writableLayerPath = Self.testDir.appending(component: "\(id)-writable.ext4") + try? FileManager.default.removeItem(at: writableLayerPath) + let filesystem = try EXT4.Formatter(FilePath(writableLayerPath.absolutePath()), minDiskSize: 512.mib()) + try filesystem.close() + let writableLayer = Mount.block( + format: "ext4", + source: writableLayerPath.absolutePath(), + destination: "/", + options: [] + ) + + let buffer = BufferWriter() + try await pod.addContainer( + "hot", + rootfs: try cloneRootfs(bs.rootfs, testID: id, containerID: "hot"), + writableLayer: writableLayer + ) { config in + config.process.arguments = ["/bin/sh", "-c", "echo 'written into a layer added while running' > /written && cat /written"] + config.process.stdout = buffer + } + + do { + try await pod.startContainer("hot") + let status = try await pod.waitContainer("hot") + + try await pod.stopContainer("hot") + try await pod.stop() + + guard status.exitCode == 0 else { + throw IntegrationError.assert(msg: "hot container status \(status) != 0") + } + let expected = "written into a layer added while running" + guard let output = String(data: buffer.data, encoding: .utf8), + output.trimmingCharacters(in: .whitespacesAndNewlines) == expected + else { + throw IntegrationError.assert( + msg: "expected '\(expected)', got '\(String(data: buffer.data, encoding: .utf8) ?? "nil")'") + } + } catch { + try? await pod.stop() + throw error + } + } + + func testPodWritableLayer() async throws { + let id = "test-pod-writable-layer" + + let bs = try await bootstrap(id) + let pod = try LinuxPod(id, vmm: bs.vmm) { config in + config.cpus = 4 + config.memoryInBytes = 1024.mib() + config.bootLog = bs.bootLog + } + + let writableLayerPath = Self.testDir.appending(component: "\(id)-writable.ext4") + try? FileManager.default.removeItem(at: writableLayerPath) + let filesystem = try EXT4.Formatter(FilePath(writableLayerPath.absolutePath()), minDiskSize: 512.mib()) + try filesystem.close() + let writableLayer = Mount.block( + format: "ext4", + source: writableLayerPath.absolutePath(), + destination: "/", + options: [] + ) + + let buffer = BufferWriter() + try await pod.addContainer( + "layered", + rootfs: try cloneRootfs(bs.rootfs, testID: id, containerID: "layered"), + writableLayer: writableLayer + ) { config in + config.process.arguments = ["/bin/sh", "-c", "echo 'writable layer test' > /tmp/testfile && cat /tmp/testfile"] + config.process.stdout = buffer + } + + do { + try await pod.create() + try await pod.startContainer("layered") + let status = try await pod.waitContainer("layered") + try await pod.stop() + + guard status.exitCode == 0 else { + throw IntegrationError.assert(msg: "process failed with status \(status)") + } + guard let output = String(data: buffer.data, encoding: .utf8) else { + throw IntegrationError.assert(msg: "failed to convert stdout to UTF8") + } + guard output.trimmingCharacters(in: .whitespacesAndNewlines) == "writable layer test" else { + throw IntegrationError.assert(msg: "unexpected output: \(output)") + } + } catch { + try? await pod.stop() + throw error + } + } + /// A file copied into one container in a pod arrives in that container and /// nowhere else, and comes back out with what it held. func testPodCopyRoundTrip() async throws { diff --git a/Sources/Integration/Suite.swift b/Sources/Integration/Suite.swift index 769ccc12..002e30a4 100644 --- a/Sources/Integration/Suite.swift +++ b/Sources/Integration/Suite.swift @@ -560,6 +560,7 @@ struct IntegrationSuite: AsyncParsableCommand { Test("pod container resource limits", testPodContainerResourceLimits), Test("pod container filesystem isolation", testPodContainerFilesystemIsolation), Test("pod copy round trip", testPodCopyRoundTrip), + Test("pod writable layer", testPodWritableLayer), Test("pod container PID namespace isolation", testPodContainerPIDNamespaceIsolation), Test("pod container independent resource limits", testPodContainerIndependentResourceLimits), Test("pod shared PID namespace", testPodSharedPIDNamespace), @@ -651,6 +652,7 @@ struct IntegrationSuite: AsyncParsableCommand { let linuxOnlyTests: [Test] = [ Test("pod hotplug block rootfs", testPodHotplugBlockRootfs), Test("pod hotplug virtiofs rootfs", testPodHotplugVirtiofsRootfs), + Test("pod hotplug writable layer", testPodHotplugWritableLayer), ] let tests: [Test] = crossPlatformTests + linuxOnlyTests #endif From 9adf57c175cc721fae870d0f3659523b0fe2e415 Mon Sep 17 00:00:00 2001 From: Aaron Paterson Date: Sat, 8 Aug 2026 02:01:02 +0000 Subject: [PATCH 13/21] Name the project the guest file transfer belongs to The file carried the sibling repository's copyright line, so the license header check found no header it recognized. --- Sources/Containerization/GuestFileTransfer.swift | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Sources/Containerization/GuestFileTransfer.swift b/Sources/Containerization/GuestFileTransfer.swift index d508c571..5eeb039d 100644 --- a/Sources/Containerization/GuestFileTransfer.swift +++ b/Sources/Containerization/GuestFileTransfer.swift @@ -1,5 +1,5 @@ //===----------------------------------------------------------------------===// -// Copyright © 2026 Apple Inc. and the container project authors. +// Copyright © 2026 Apple Inc. and the Containerization project authors. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. From bf57208a9871a0079fa6803f3f88c484f2e7b2a9 Mon Sep 17 00:00:00 2001 From: Aaron Paterson Date: Mon, 10 Aug 2026 05:40:19 +0000 Subject: [PATCH 14/21] Let a pod give a container's place back Stopping a container tears down what it was running and keeps its place: the name still answers for it, and placing another container under it is refused. Removal is the separate act the runtime specification names for giving the place up, taken once the container has stopped, so a name can run again in a machine that outlives what it last ran. https://github.com/kubernetes/cri-api/blob/master/pkg/apis/runtime/v1/api.proto --- Sources/Containerization/LinuxPod.swift | 29 +++++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/Sources/Containerization/LinuxPod.swift b/Sources/Containerization/LinuxPod.swift index 76967821..9d98d79f 100644 --- a/Sources/Containerization/LinuxPod.swift +++ b/Sources/Containerization/LinuxPod.swift @@ -1189,6 +1189,35 @@ extension LinuxPod { } } + /// Take a container out of the pod, so its name is free to place again. + /// + /// Stopping a container tears down what it was running and keeps its + /// place; the name still answers for it, and placing another container + /// under it is refused. Removal is the separate act the runtime + /// specification names for giving the place up, taken once the container + /// has stopped. A container that is running keeps its place and this + /// call refuses it. + /// https://github.com/kubernetes/cri-api/blob/master/pkg/apis/runtime/v1/api.proto + public func removeContainer(_ containerID: String) async throws { + try await self.state.withLock { state in + guard let container = state.containers[containerID] else { + throw ContainerizationError( + .notFound, + message: "container \(containerID) not found in pod" + ) + } + switch container.state { + case .registered, .stopped, .errored: + state.containers[containerID] = nil + default: + throw ContainerizationError( + .invalidState, + message: "container \(containerID) must stop before it is removed" + ) + } + } + } + /// Stop the pod's VM and all containers. public func stop() async throws { try await self.state.withLock { state in From df2bc3ec807037947b091fdb2bdfb7a3763dc9d3 Mon Sep 17 00:00:00 2001 From: Aaron Paterson Date: Sat, 1 Aug 2026 23:45:26 +0000 Subject: [PATCH 15/21] Let the host reclaim memory from a running container Pages a guest has freed stay with the machine, because the guest has no way to say it is done with them. They leave only when the host runs short and reclaims them as it would any cold memory, which pages out what the guest would have handed over for nothing. Attach a memory balloon to each virtual machine and expose the size the machine should hold as `setTargetMemorySize`. Lowering it hands memory back to the host; raising it, up to the size the machine was created with, returns memory to the guest. Backends without a balloon report the operation as unsupported, in the same shape as pause and hotplug. The guest keeps reporting the same total while the balloon holds its pages, and shows them as no longer free, because the balloon driver has to account for them in case it is asked to give them back. --- .../VZVirtualMachine+Helpers.swift | 19 +++++ .../VZVirtualMachineInstance.swift | 20 ++++++ .../VirtualMachineInstance.swift | 12 ++++ Sources/Integration/ContainerTests.swift | 69 +++++++++++++++++++ Sources/Integration/Suite.swift | 1 + 5 files changed, 121 insertions(+) diff --git a/Sources/Containerization/VZVirtualMachine+Helpers.swift b/Sources/Containerization/VZVirtualMachine+Helpers.swift index 2cbadb1c..6dc632a9 100644 --- a/Sources/Containerization/VZVirtualMachine+Helpers.swift +++ b/Sources/Containerization/VZVirtualMachine+Helpers.swift @@ -92,6 +92,25 @@ extension VZVirtualMachine { } } + /// Write the balloon's target size. Virtualization takes memory from the + /// guest and gives it to the host by the difference when the target is + /// lowered, and gives it back when the target is raised, so the guest's + /// operating system has to act before the change takes effect. + func setTargetMemorySize(_ bytes: UInt64, queue: DispatchQueue) throws { + try queue.sync { + guard + let balloon = self.memoryBalloonDevices.first + as? VZVirtioTraditionalMemoryBalloonDevice + else { + throw ContainerizationError( + .unsupported, + message: "no memory balloon device attached to the virtual machine" + ) + } + balloon.targetVirtualMachineMemorySize = bytes + } + } + func pause(queue: DispatchQueue) async throws { try await withCheckedThrowingContinuation { (cont: CheckedContinuation) in queue.sync { diff --git a/Sources/Containerization/VZVirtualMachineInstance.swift b/Sources/Containerization/VZVirtualMachineInstance.swift index 135879e0..86904ffb 100644 --- a/Sources/Containerization/VZVirtualMachineInstance.swift +++ b/Sources/Containerization/VZVirtualMachineInstance.swift @@ -253,6 +253,18 @@ extension VZVirtualMachineInstance: VirtualMachineInstance { } } + public func setTargetMemorySize(_ bytes: UInt64) async throws { + guard bytes <= self.config.memoryInBytes else { + throw ContainerizationError( + .invalidArgument, + message: "cannot hold \(bytes) bytes, the machine was created with \(self.config.memoryInBytes)" + ) + } + try await lock.withLock { _ in + try self.vm.setTargetMemorySize(bytes, queue: self.queue) + } + } + public func dialAgent() async throws -> Vminitd { try await lock.withLock { _ in do { @@ -419,6 +431,14 @@ extension VZVirtualMachineInstance.Configuration { config.memorySize = (self.memoryInBytes + mib - 1) & ~(mib - 1) config.entropyDevices = [VZVirtioEntropyDeviceConfiguration()] config.socketDevices = [VZVirtioSocketDeviceConfiguration()] + // A balloon lets the host take back memory the guest has stopped using. + // Nothing else can: the guest has no way to report a page it has freed, + // so without one those pages stay with the machine until the host runs + // short and reclaims them the way it reclaims any cold memory, paging + // out pages the guest would have given up for nothing. Lima attaches + // the same device beside the same entropy and socket devices. + // https://github.com/lima-vm/lima/blob/master/pkg/driver/vz/vm_darwin.go + config.memoryBalloonDevices = [VZVirtioTraditionalMemoryBalloonDeviceConfiguration()] if let bootLog = self.bootLog { config.serialPorts = try serialPort(destination: bootLog) diff --git a/Sources/Containerization/VirtualMachineInstance.swift b/Sources/Containerization/VirtualMachineInstance.swift index da7935d5..6df8fc97 100644 --- a/Sources/Containerization/VirtualMachineInstance.swift +++ b/Sources/Containerization/VirtualMachineInstance.swift @@ -96,6 +96,15 @@ public protocol VirtualMachineInstance: Sendable { /// Release virtiofs shares for a container. /// - Parameter id: The container ID whose virtiofs shares should be released func releaseVirtioFS(id: String) async throws + + /// Set how much memory the running virtual machine should hold. + /// + /// Lowering it hands memory back to the host, which is the only way to + /// recover pages the guest has touched and since freed. Raising it returns + /// memory to the guest. Throws if the VMM has no memory balloon. + /// - Parameter bytes: The size the virtual machine should hold, which must + /// not exceed the size it was created with. + func setTargetMemorySize(_ bytes: UInt64) async throws } extension VirtualMachineInstance { @@ -121,4 +130,7 @@ extension VirtualMachineInstance { public func releaseVirtioFS(id: String) async throws { // no-op default } + public func setTargetMemorySize(_ bytes: UInt64) async throws { + throw ContainerizationError(.unsupported, message: "memory balloon not supported") + } } diff --git a/Sources/Integration/ContainerTests.swift b/Sources/Integration/ContainerTests.swift index 9980cf21..0dfa4a83 100644 --- a/Sources/Integration/ContainerTests.swift +++ b/Sources/Integration/ContainerTests.swift @@ -315,6 +315,75 @@ extension IntegrationSuite { } } + func testContainerMemoryBalloon() async throws { + let id = "test-container-memory-balloon" + let bs = try await bootstrap(id) + + let memory: UInt64 = 2048.mib() + let target: UInt64 = 1024.mib() + let buffer = BufferWriter() + let container = try LinuxContainer(id, rootfs: bs.rootfs, vmm: bs.vmm) { config in + // Memory a guest has never touched is not backed on the host, so a + // balloon that takes only those pages moves nothing. The guest fills + // a tmpfs to put real pages behind its memory and frees them again, + // leaving the machine holding pages the guest no longer wants, which + // is the state the balloon exists to resolve. + config.mounts.append( + .any( + type: "tmpfs", source: "tmpfs", destination: "/fill", + options: ["rw", "size=1g"])) + config.process.arguments = [ + "/bin/sh", "-c", + "dd if=/dev/zero of=/fill/pages bs=1M count=768 2>/dev/null; " + + "rm /fill/pages; " + + "awk '/MemFree/ { print $2 }' /proc/meminfo; sleep 25; " + + "awk '/MemFree/ { print $2 }' /proc/meminfo", + ] + config.process.stdout = buffer + config.memoryInBytes = memory + config.bootLog = bs.bootLog + } + + do { + try await container.create() + try await container.start() + + try await Task.sleep(nanoseconds: 15_000_000_000) + try await container.withVirtualMachineInstance { vm in + try await vm.setTargetMemorySize(target) + } + + let status = try await container.wait() + try await container.stop() + guard status.exitCode == 0 else { + throw IntegrationError.assert(msg: "workload did not complete: \(status)") + } + + let out = String(data: buffer.data, encoding: .utf8) ?? "" + let readings = out.split(separator: "\n").compactMap { + UInt64($0.trimmingCharacters(in: .whitespaces)) + } + guard readings.count == 2 else { + throw IntegrationError.assert(msg: "expected two readings, got '\(out)'") + } + // The balloon holds what it takes, so the guest goes on reporting the + // same total while the memory it has free drops by close to the + // amount asked for. The driver keeps those pages accounted for in + // case it has to give them back, which is why the total does not + // move. + let takenByBalloon = Int64(readings[0]) - Int64(readings[1]) + let asked = Int64((memory - target) / 1024) + guard takenByBalloon > asked / 2 else { + throw IntegrationError.assert( + msg: "balloon took \(takenByBalloon) kB of the \(asked) kB asked for: " + + "free before=\(readings[0]) after=\(readings[1])") + } + } catch { + try? await container.stop() + throw error + } + } + func testProcessEchoHi() async throws { let id = "test-process-echo-hi" let bs = try await bootstrap(id) diff --git a/Sources/Integration/Suite.swift b/Sources/Integration/Suite.swift index 002e30a4..99f534fa 100644 --- a/Sources/Integration/Suite.swift +++ b/Sources/Integration/Suite.swift @@ -438,6 +438,7 @@ struct IntegrationSuite: AsyncParsableCommand { // Process basics Test("process true", testProcessTrue), Test("process false", testProcessFalse), + Test("container memory balloon", testContainerMemoryBalloon), Test("process echo hi", testProcessEchoHi), Test("process no executable", testProcessNoExecutable), Test("process user", testProcessUser), From 69565bf85301237273e59995ec13eebc336f3d55 Mon Sep 17 00:00:00 2001 From: Aaron Paterson Date: Sat, 1 Aug 2026 23:49:49 +0000 Subject: [PATCH 16/21] Reclaim guest freed memory on the cloud hypervisor backend Attach a balloon to the machines the cloud hypervisor backend creates, with free page reporting turned on, so the guest hands back the pages it frees and the host reclaims them without anything having to choose a balloon size. Kata drives the same device the same way, in src/runtime-rs/crates/hypervisor/ch-config/src/convert.rs. The type follows the Cloud Hypervisor OpenAPI spec, which carries size alongside deflate_on_oom and free_page_reporting. --- Sources/CloudHypervisor/Types/VmConfig.swift | 36 +++++++++++++++++++ .../CHVirtualMachineInstance.swift | 4 +++ 2 files changed, 40 insertions(+) diff --git a/Sources/CloudHypervisor/Types/VmConfig.swift b/Sources/CloudHypervisor/Types/VmConfig.swift index 429b7db9..f64562dc 100644 --- a/Sources/CloudHypervisor/Types/VmConfig.swift +++ b/Sources/CloudHypervisor/Types/VmConfig.swift @@ -28,6 +28,7 @@ extension CloudHypervisor { public var net: [NetConfig]? public var fs: [FsConfig]? public var vsock: VsockConfig? + public var balloon: BalloonConfig? public var console: ConsoleConfig public var serial: ConsoleConfig @@ -39,6 +40,7 @@ extension CloudHypervisor { net: [NetConfig]? = nil, fs: [FsConfig]? = nil, vsock: VsockConfig? = nil, + balloon: BalloonConfig? = nil, console: ConsoleConfig, serial: ConsoleConfig ) { @@ -49,6 +51,7 @@ extension CloudHypervisor { self.net = net self.fs = fs self.vsock = vsock + self.balloon = balloon self.console = console self.serial = serial } @@ -61,6 +64,7 @@ extension CloudHypervisor { case net case fs case vsock + case balloon case console case serial } @@ -121,6 +125,38 @@ extension CloudHypervisor { } } + // MARK: - BalloonConfig + + /// Memory balloon for handing guest memory back to the host. + /// + /// Maps to `BalloonConfig` in the Cloud Hypervisor OpenAPI spec. + public struct BalloonConfig: Sendable, Codable, Equatable { + /// Balloon size in bytes. Zero leaves every page with the guest until + /// something asks for it back. + public var size: UInt64 + /// Give pages back to the guest when it runs out of memory. + public var deflateOnOom: Bool? + /// Report the pages the guest frees, so the host can reclaim them + /// without anything having to decide a balloon size. + public var freePageReporting: Bool? + + public init( + size: UInt64 = 0, + deflateOnOom: Bool? = nil, + freePageReporting: Bool? = nil + ) { + self.size = size + self.deflateOnOom = deflateOnOom + self.freePageReporting = freePageReporting + } + + enum CodingKeys: String, CodingKey { + case size + case deflateOnOom = "deflate_on_oom" + case freePageReporting = "free_page_reporting" + } + } + // MARK: - PayloadConfig /// Kernel / initramfs / cmdline payload for a VM. diff --git a/Sources/Containerization/CHVirtualMachineInstance.swift b/Sources/Containerization/CHVirtualMachineInstance.swift index 11d7e0a3..99f8b7f6 100644 --- a/Sources/Containerization/CHVirtualMachineInstance.swift +++ b/Sources/Containerization/CHVirtualMachineInstance.swift @@ -626,6 +626,10 @@ extension CHVirtualMachineInstance { net: net.isEmpty ? nil : net, fs: fsConfigs.isEmpty ? nil : fsConfigs, vsock: vsock, + // Free page reporting lets the host reclaim what the guest frees + // without anything having to choose a balloon size, which is how + // Kata drives the same device (ch-config/src/convert.rs). + balloon: CloudHypervisor.BalloonConfig(freePageReporting: true), // Kernel cmdline is `console=hvc0`, so userspace (vminitd) writes // to hvc0 — capture that to the bootlog. We deliberately disable // the pl011 (`serial`) UART entirely with `.Off`. Any non-Off mode From 6d676f0f2d91edec7047c635c522afdaf3536f07 Mon Sep 17 00:00:00 2001 From: Aaron Paterson Date: Sun, 2 Aug 2026 00:08:06 +0000 Subject: [PATCH 17/21] Let the cloud hypervisor backend hold a target memory size Cloud Hypervisor sizes the balloon rather than the machine, so the balloon is asked to hold whatever the machine should not, and both backends answer the same request the same way. The resize request follows the Cloud Hypervisor OpenAPI spec, which carries desired_vcpus and desired_ram alongside desired_balloon. --- .../CloudHypervisor/Endpoints/Client+VM.swift | 7 +++++ Sources/CloudHypervisor/Types/VmConfig.swift | 27 +++++++++++++++++++ .../CHVirtualMachineInstance.swift | 21 +++++++++++++-- 3 files changed, 53 insertions(+), 2 deletions(-) diff --git a/Sources/CloudHypervisor/Endpoints/Client+VM.swift b/Sources/CloudHypervisor/Endpoints/Client+VM.swift index b1f79986..0373e2ad 100644 --- a/Sources/CloudHypervisor/Endpoints/Client+VM.swift +++ b/Sources/CloudHypervisor/Endpoints/Client+VM.swift @@ -31,6 +31,13 @@ extension CloudHypervisor.Client { try await put("/api/v1/vm.boot") } + /// Resize a running VM's cpus, memory or balloon. + /// + /// Maps to `PUT /api/v1/vm.resize` in the Cloud Hypervisor REST API. + public func vmResize(_ resize: CloudHypervisor.VmResize) async throws { + try await put("/api/v1/vm.resize", body: resize) + } + /// Shut down the VM. /// /// Maps to `PUT /api/v1/vm.shutdown` in the Cloud Hypervisor REST API. diff --git a/Sources/CloudHypervisor/Types/VmConfig.swift b/Sources/CloudHypervisor/Types/VmConfig.swift index f64562dc..0786678b 100644 --- a/Sources/CloudHypervisor/Types/VmConfig.swift +++ b/Sources/CloudHypervisor/Types/VmConfig.swift @@ -125,6 +125,33 @@ extension CloudHypervisor { } } + // MARK: - VmResize + + /// Resize request for a running VM. + /// + /// Maps to `VmResize` in the Cloud Hypervisor OpenAPI spec. + public struct VmResize: Sendable, Codable, Equatable { + /// Desired vCPU count. + public var desiredVcpus: Int? + /// Desired memory in bytes. + public var desiredRam: Int64? + /// Desired balloon size in bytes. Growing the balloon takes memory from + /// the guest and gives it to the host. + public var desiredBalloon: Int64? + + public init(desiredVcpus: Int? = nil, desiredRam: Int64? = nil, desiredBalloon: Int64? = nil) { + self.desiredVcpus = desiredVcpus + self.desiredRam = desiredRam + self.desiredBalloon = desiredBalloon + } + + enum CodingKeys: String, CodingKey { + case desiredVcpus = "desired_vcpus" + case desiredRam = "desired_ram" + case desiredBalloon = "desired_balloon" + } + } + // MARK: - BalloonConfig /// Memory balloon for handing guest memory back to the host. diff --git a/Sources/Containerization/CHVirtualMachineInstance.swift b/Sources/Containerization/CHVirtualMachineInstance.swift index 99f8b7f6..ed8ae1cf 100644 --- a/Sources/Containerization/CHVirtualMachineInstance.swift +++ b/Sources/Containerization/CHVirtualMachineInstance.swift @@ -317,6 +317,21 @@ extension CHVirtualMachineInstance: VirtualMachineInstance { } } + public func setTargetMemorySize(_ bytes: UInt64) async throws { + guard bytes <= self.config.memoryInBytes else { + throw ContainerizationError( + .invalidArgument, + message: "cannot hold \(bytes) bytes, the machine was created with \(self.config.memoryInBytes)" + ) + } + // Cloud Hypervisor sizes the balloon rather than the machine, so the + // balloon is asked to hold whatever the machine should not. + let balloon = Int64(self.config.memoryInBytes - bytes) + try await chCall { + try await self.client.vmResize(.init(desiredBalloon: balloon)) + } + } + public func stop() async throws { try await lock.withLock { _ in guard self.state == .running else { @@ -627,8 +642,10 @@ extension CHVirtualMachineInstance { fs: fsConfigs.isEmpty ? nil : fsConfigs, vsock: vsock, // Free page reporting lets the host reclaim what the guest frees - // without anything having to choose a balloon size, which is how - // Kata drives the same device (ch-config/src/convert.rs). + // without anything having to choose a balloon size, which is what + // Kata turns on to reclaim guest freed memory. + // https://github.com/kata-containers/kata-containers/blob/main/src/runtime-rs/crates/hypervisor/ch-config/src/convert.rs + // https://github.com/kata-containers/kata-containers/blob/main/docs/how-to/how-to-use-memory-agent.md balloon: CloudHypervisor.BalloonConfig(freePageReporting: true), // Kernel cmdline is `console=hvc0`, so userspace (vminitd) writes // to hvc0 — capture that to the bootlog. We deliberately disable From c8fe91c5352e43372a2e8b41dcfea85812e6cb09 Mon Sep 17 00:00:00 2001 From: Aaron Paterson Date: Sun, 2 Aug 2026 20:09:50 +0000 Subject: [PATCH 18/21] Compact a guest before taking memory back from it Virtualization asks for the guest to be compacted before the balloon is driven, so that the pages it gives up sit together well enough to be worth taking. Asking a machine to hold less memory now does that first, which puts the recommended order in the one place that drives the device rather than leaving each caller to remember it. The agent already carries the setting the guest needs, so it is the interface that grows rather than the protocol between the two halves. Memory a guest has never touched is not backed on the host, so a balloon that takes only those pages moves nothing at all. The test now fills a tmpfs and frees it, leaving the machine holding pages the guest no longer wants, which is the state the device exists to resolve. https://developer.apple.com/documentation/virtualization/vzvirtiotraditionalmemoryballoondevice --- Sources/Containerization/LinuxContainer.swift | 14 ++++++++++++++ Sources/Containerization/VirtualMachineAgent.swift | 1 + .../Containerization/VirtualMachineInstance.swift | 11 +++++++++++ Sources/Integration/ContainerTests.swift | 4 +--- 4 files changed, 27 insertions(+), 3 deletions(-) diff --git a/Sources/Containerization/LinuxContainer.swift b/Sources/Containerization/LinuxContainer.swift index ce57488f..03e1bcee 100644 --- a/Sources/Containerization/LinuxContainer.swift +++ b/Sources/Containerization/LinuxContainer.swift @@ -1080,6 +1080,20 @@ extension LinuxContainer { return try await fn(vm) } + /// Ask the machine to hold no more than `bytes`. The balloon takes the + /// difference from the guest, and hands it back when the target is raised + /// again, so nothing changes until the guest has acted on the request. + /// + /// The guest's free memory is gathered together first, which is what + /// Virtualization asks for before the device is driven. + public func setTargetMemorySize(_ bytes: UInt64) async throws { + let vm = try await self.state.withLock { state in + try state.vm("setTargetMemorySize") + } + try await vm.compactGuestMemory() + try await vm.setTargetMemorySize(bytes) + } + /// Close the containers standard input to signal no more input is /// arriving. public func closeStdin() async throws { diff --git a/Sources/Containerization/VirtualMachineAgent.swift b/Sources/Containerization/VirtualMachineAgent.swift index 05ea7950..e40702eb 100644 --- a/Sources/Containerization/VirtualMachineAgent.swift +++ b/Sources/Containerization/VirtualMachineAgent.swift @@ -53,6 +53,7 @@ public protocol VirtualMachineAgent: Sendable { func kill(pid: Int32, signal: Int32) async throws -> Int32 func sync() async throws func writeFile(path: String, data: Data, flags: WriteFileFlags, mode: UInt32) async throws + func sysctl(settings: [String: String]) async throws // Process lifecycle func createProcess( diff --git a/Sources/Containerization/VirtualMachineInstance.swift b/Sources/Containerization/VirtualMachineInstance.swift index 6df8fc97..2b1d96fe 100644 --- a/Sources/Containerization/VirtualMachineInstance.swift +++ b/Sources/Containerization/VirtualMachineInstance.swift @@ -133,4 +133,15 @@ extension VirtualMachineInstance { public func setTargetMemorySize(_ bytes: UInt64) async throws { throw ContainerizationError(.unsupported, message: "memory balloon not supported") } + + /// Gather the guest's free memory into contiguous runs. + /// + /// Virtualization asks for this before the balloon is driven, so that the + /// pages the guest gives up sit together well enough to be worth taking. + /// https://developer.apple.com/documentation/virtualization/vzvirtiotraditionalmemoryballoondevice + public func compactGuestMemory() async throws { + try await withAgent { agent in + try await agent.sysctl(settings: ["vm.compact_memory": "1"]) + } + } } diff --git a/Sources/Integration/ContainerTests.swift b/Sources/Integration/ContainerTests.swift index 0dfa4a83..863bb87a 100644 --- a/Sources/Integration/ContainerTests.swift +++ b/Sources/Integration/ContainerTests.swift @@ -349,9 +349,7 @@ extension IntegrationSuite { try await container.start() try await Task.sleep(nanoseconds: 15_000_000_000) - try await container.withVirtualMachineInstance { vm in - try await vm.setTargetMemorySize(target) - } + try await container.setTargetMemorySize(target) let status = try await container.wait() try await container.stop() From bb1a5b43ab80c303452f26e14bb6f98c1cfb81c5 Mon Sep 17 00:00:00 2001 From: Aaron Paterson Date: Wed, 12 Aug 2026 16:37:29 +0000 Subject: [PATCH 19/21] Let a pod gather its guest's memory before taking any back A pod's containers share one machine and one balloon, so bounding them together is the same operation a single container's machine takes, and the pod had no way to ask for it. The machine gathers the guest's free memory and drives the device, since dialing the agent and holding the balloon both already belong to it. --- Sources/Containerization/LinuxPod.swift | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/Sources/Containerization/LinuxPod.swift b/Sources/Containerization/LinuxPod.swift index 9d98d79f..73b64b43 100644 --- a/Sources/Containerization/LinuxPod.swift +++ b/Sources/Containerization/LinuxPod.swift @@ -1423,6 +1423,22 @@ extension LinuxPod { return try await fn(vm) } + /// Ask the machine to hold no more than `bytes`. The balloon takes the + /// difference from the guest, and hands it back when the target is raised + /// again, so nothing changes until the guest has acted on the request. + /// + /// The pod's containers share the machine's memory, so this bounds all of + /// them together rather than any one of them. The guest's free memory is + /// gathered together first, which is what Virtualization asks for before + /// the device is driven. + public func setTargetMemorySize(_ bytes: UInt64) async throws { + let vm = try await self.state.withLock { state in + try state.phase.createdState("setTargetMemorySize").vm + } + try await vm.compactGuestMemory() + try await vm.setTargetMemorySize(bytes) + } + // Perform filesystem operations in a container. public func filesystemOperation(_ containerID: String, operation: FilesystemOperation, path: String) async throws { try await self.state.withLock { state in From 33c07c75f6431ca3e2d5f787b1c9dca47271f70a Mon Sep 17 00:00:00 2001 From: Aaron Paterson Date: Sun, 2 Aug 2026 00:33:04 +0000 Subject: [PATCH 20/21] Work out how much memory to ask a machine to give back Attaching a balloon leaves the question of what to set it to. The guest cannot answer it: having freed memory, it has no way to say so, and the pages stay charged to the machine. So the reading is taken on the host, from the anonymous memory the guest reports holding, and the rest is asked for back. Reclaiming too far is the failure worth avoiding, because the guest then fetches back pages it was still using. The count of those refaults is already reported, so a rise in it since the last reading is treated as evidence the last target was too tight, and memory is given back rather than the reading being trusted. The decision is a plain function of one reading, so it is tested without a virtual machine. --- Sources/Containerization/MemoryReclaim.swift | 124 ++++++++++++++++++ .../MemoryReclaimTests.swift | 122 +++++++++++++++++ 2 files changed, 246 insertions(+) create mode 100644 Sources/Containerization/MemoryReclaim.swift create mode 100644 Tests/ContainerizationTests/MemoryReclaimTests.swift diff --git a/Sources/Containerization/MemoryReclaim.swift b/Sources/Containerization/MemoryReclaim.swift new file mode 100644 index 00000000..a09b13b7 --- /dev/null +++ b/Sources/Containerization/MemoryReclaim.swift @@ -0,0 +1,124 @@ +//===----------------------------------------------------------------------===// +// Copyright © 2026 Apple Inc. and the Containerization project authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//===----------------------------------------------------------------------===// + +import ContainerizationOS +import Foundation + +/// Works out how much memory a virtual machine should be asked to hold, from +/// what its guest is actually using. +/// +/// The decision is made on the host because Virtualization's balloon takes a +/// target size and nothing else: the guest cannot hand pages back of its own +/// accord, so something outside has to look at what it is using and ask for the +/// rest. Cloud Hypervisor needs none of this, its balloon reporting the pages +/// the guest frees so the host reclaims them with no size to choose, which is +/// why the backend turns that on instead. +/// +/// Kata answers the same question from inside the guest, its mem-agent using +/// MgLRU to reclaim cold pages per cgroup and free page reporting to give them +/// back. This is the host-side equivalent for a backend that offers only a +/// target. +/// https://github.com/kata-containers/kata-containers/blob/main/docs/how-to/how-to-use-memory-agent.md +/// +/// Reclaiming too far is the failure to avoid, because the guest then has to +/// fetch back pages it was still using. `workingsetRefaultAnon` counts exactly +/// that, so a rise in it since the last look is treated as evidence the last +/// target was too tight, and the policy gives the memory back and waits. +public struct MemoryReclaimPolicy: Sendable { + /// Room left above what the guest is using, so ordinary allocation does not + /// immediately have to wait for the balloon to deflate. + public var headroom: UInt64 + /// The least a machine will be asked to hold, whatever its guest reports. + public var floor: UInt64 + /// How much of the machine to give back at once after refaults are seen. + public var backoff: UInt64 + + public init( + headroom: UInt64 = 256.mib(), + floor: UInt64 = 256.mib(), + backoff: UInt64 = 256.mib() + ) { + self.headroom = headroom + self.floor = floor + self.backoff = backoff + } + + /// The size to ask for next. + /// + /// - Parameters: + /// - anon: Anonymous memory the guest holds. Page cache is left out + /// because the guest can drop it without help. + /// - refaulted: Whether anonymous pages have been fetched back since the + /// last look, meaning the previous target was too tight. + /// - current: What the machine is holding now. + /// - ceiling: The size the machine was created with, which it cannot + /// exceed. + public func nextTarget( + anon: UInt64, + refaulted: Bool, + current: UInt64, + ceiling: UInt64 + ) -> UInt64 { + if refaulted { + return min(ceiling, current + backoff) + } + let wanted = anon + headroom + return min(ceiling, max(floor, wanted)) + } +} + +/// Tracks a machine across looks so refaults can be compared against the +/// previous reading rather than treated as an absolute. +public actor MemoryReclaimer { + private let policy: MemoryReclaimPolicy + private let ceiling: UInt64 + private var lastRefaultAnon: UInt64? + private var current: UInt64 + + public init(policy: MemoryReclaimPolicy = .init(), ceiling: UInt64) { + self.policy = policy + self.ceiling = ceiling + self.current = ceiling + } + + /// Fold in one reading and return the size to ask for, or nil when it has + /// not changed enough to be worth asking. + public func step(memory: ContainerStatistics.MemoryStatistics) -> UInt64? { + let refaulted = + lastRefaultAnon.map { memory.workingsetRefaultAnon > $0 } ?? false + lastRefaultAnon = memory.workingsetRefaultAnon + + let target = policy.nextTarget( + anon: memory.anon, + refaulted: refaulted, + current: current, + ceiling: ceiling + ) + // Virtualization rounds the target to a megabyte, so anything smaller + // than that is not a change at all. + guard target != current, target.absoluteDistance(to: current) >= 1.mib() else { + return nil + } + current = target + return target + } +} + +extension UInt64 { + fileprivate func absoluteDistance(to other: UInt64) -> UInt64 { + self > other ? self - other : other - self + } +} diff --git a/Tests/ContainerizationTests/MemoryReclaimTests.swift b/Tests/ContainerizationTests/MemoryReclaimTests.swift new file mode 100644 index 00000000..842c150a --- /dev/null +++ b/Tests/ContainerizationTests/MemoryReclaimTests.swift @@ -0,0 +1,122 @@ +//===----------------------------------------------------------------------===// +// Copyright © 2026 Apple Inc. and the Containerization project authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//===----------------------------------------------------------------------===// + +import ContainerizationExtras +import Foundation +import Testing + +@testable import Containerization + +struct MemoryReclaimTests { + private let policy = MemoryReclaimPolicy( + headroom: 256.mib(), + floor: 256.mib(), + backoff: 256.mib() + ) + + @Test func targetFollowsWhatTheGuestHolds() { + let target = policy.nextTarget( + anon: 512.mib(), + refaulted: false, + current: 4096.mib(), + ceiling: 4096.mib() + ) + #expect(target == 768.mib()) + } + + @Test func targetNeverFallsBelowTheFloor() { + let target = policy.nextTarget( + anon: 0, + refaulted: false, + current: 4096.mib(), + ceiling: 4096.mib() + ) + #expect(target == 256.mib()) + } + + @Test func targetNeverExceedsTheSizeTheMachineWasCreatedWith() { + let target = policy.nextTarget( + anon: 8192.mib(), + refaulted: false, + current: 1024.mib(), + ceiling: 1024.mib() + ) + #expect(target == 1024.mib()) + } + + /// Refaults mean the guest had to fetch back pages it was still using, so + /// the previous target was too tight and memory goes back rather than the + /// reading being trusted. + @Test func refaultsGiveMemoryBackInsteadOfShrinkingFurther() { + let target = policy.nextTarget( + anon: 128.mib(), + refaulted: true, + current: 512.mib(), + ceiling: 4096.mib() + ) + #expect(target == 768.mib()) + } + + @Test func givingBackStopsAtTheCeiling() { + let target = policy.nextTarget( + anon: 128.mib(), + refaulted: true, + current: 4000.mib(), + ceiling: 4096.mib() + ) + #expect(target == 4096.mib()) + } + + @Test func firstReadingIsNotTreatedAsARefault() async { + let reclaimer = MemoryReclaimer(policy: policy, ceiling: 4096.mib()) + let target = await reclaimer.step(memory: Self.memory(anon: 512.mib(), refaults: 900)) + // Without a previous reading there is nothing to compare against, so + // the count is recorded and the guest's usage is followed. + #expect(target == 768.mib()) + } + + @Test func aRepeatedReadingAsksForNothing() async { + let reclaimer = MemoryReclaimer(policy: policy, ceiling: 4096.mib()) + _ = await reclaimer.step(memory: Self.memory(anon: 512.mib(), refaults: 0)) + let again = await reclaimer.step(memory: Self.memory(anon: 512.mib(), refaults: 0)) + #expect(again == nil) + } + + @Test func risingRefaultsReverseTheLastReclaim() async { + let reclaimer = MemoryReclaimer(policy: policy, ceiling: 4096.mib()) + let first = await reclaimer.step(memory: Self.memory(anon: 512.mib(), refaults: 10)) + #expect(first == 768.mib()) + let second = await reclaimer.step(memory: Self.memory(anon: 512.mib(), refaults: 99)) + #expect(second == 1024.mib()) + } + + private static func memory(anon: UInt64, refaults: UInt64) -> ContainerStatistics.MemoryStatistics { + .init( + usageBytes: anon, + limitBytes: 0, + swapUsageBytes: 0, + swapLimitBytes: 0, + cacheBytes: 0, + kernelStackBytes: 0, + slabBytes: 0, + pageFaults: 0, + majorPageFaults: 0, + inactiveFile: 0, + anon: anon, + workingsetRefaultAnon: refaults + ) + } +} From faddab1dc94f1c9eba23d74cee8aeceab173afa6 Mon Sep 17 00:00:00 2001 From: Aaron Paterson Date: Wed, 12 Aug 2026 06:55:15 +0000 Subject: [PATCH 21/21] Return a machine's freed memory to the host while it runs A loop looks at what a machine's containers hold on a cadence and asks the machine to hold that plus headroom, backing off when the guest refaults, so memory a workload frees goes back proactively instead of waiting for host pressure. The knob is proactiveMemoryReclaim on the pod and container configurations, off by default; the cloud-hypervisor backend reports freed pages continuously either way. The reclaimer reports what it has done, which is the deterministic ground the integration tests assert: the loop follows a fill-and-free with targets while a busy guest keeps what it uses. --- Sources/Containerization/LinuxContainer.swift | 84 +++++++++++- Sources/Containerization/LinuxPod.swift | 46 ++++++- Sources/Containerization/MemoryReclaim.swift | 68 +++++++++- Sources/Integration/PodTests.swift | 127 ++++++++++++++++++ Sources/Integration/Suite.swift | 5 + 5 files changed, 325 insertions(+), 5 deletions(-) diff --git a/Sources/Containerization/LinuxContainer.swift b/Sources/Containerization/LinuxContainer.swift index 03e1bcee..50218d3d 100644 --- a/Sources/Containerization/LinuxContainer.swift +++ b/Sources/Containerization/LinuxContainer.swift @@ -98,6 +98,15 @@ public final class LinuxContainer: Container, Sendable { /// on top of the container's configured `memoryInBytes` value. /// The total is aligned to a 1 MiB boundary. public var memoryOverhead: UInt64 = 128.mib() + /// Return memory the guest has freed to the host while the container + /// runs, instead of leaving it held until the host is under pressure. + /// A loop watches what the container holds and asks the machine to + /// hold that plus headroom, backing off when the guest refaults. The + /// cloud-hypervisor backend reports freed pages to the host + /// continuously whether or not this is set. + public var proactiveMemoryReclaim: Bool = false + /// How often the reclaim loop looks at the guest. + public var memoryReclaimInterval: Duration = .seconds(10) /// Optional swap area for the container, as a block device mount. /// @@ -183,6 +192,8 @@ public final class LinuxContainer: Container, Sendable { let vm: any VirtualMachineInstance let relayManager: UnixSocketRelayManager var fileMountContext: FileMountContext + let reclaimer: MemoryReclaimer? + let reclaimTask: Task? } struct StartedState: Sendable { @@ -191,6 +202,8 @@ public final class LinuxContainer: Container, Sendable { let relayManager: UnixSocketRelayManager var vendedProcesses: [String: LinuxProcess] let fileMountContext: FileMountContext + let reclaimer: MemoryReclaimer? + let reclaimTask: Task? init(_ state: CreatedState, process: LinuxProcess) { self.vm = state.vm @@ -198,6 +211,8 @@ public final class LinuxContainer: Container, Sendable { self.process = process self.vendedProcesses = [:] self.fileMountContext = state.fileMountContext + self.reclaimer = state.reclaimer + self.reclaimTask = state.reclaimTask } init(_ state: PausedState) { @@ -206,6 +221,8 @@ public final class LinuxContainer: Container, Sendable { self.process = state.process self.vendedProcesses = state.vendedProcesses self.fileMountContext = state.fileMountContext + self.reclaimer = state.reclaimer + self.reclaimTask = state.reclaimTask } } @@ -215,6 +232,8 @@ public final class LinuxContainer: Container, Sendable { let process: LinuxProcess var vendedProcesses: [String: LinuxProcess] let fileMountContext: FileMountContext + let reclaimer: MemoryReclaimer? + let reclaimTask: Task? init(_ state: StartedState) { self.vm = state.vm @@ -222,6 +241,8 @@ public final class LinuxContainer: Container, Sendable { self.process = state.process self.vendedProcesses = state.vendedProcesses self.fileMountContext = state.fileMountContext + self.reclaimer = state.reclaimer + self.reclaimTask = state.reclaimTask } } @@ -743,7 +764,40 @@ extension LinuxContainer { } } - state = .created(.init(vm: vm, relayManager: relayManager, fileMountContext: fileMountContextHolder.withLock { $0 })) + var reclaimer: MemoryReclaimer? = nil + var reclaimTask: Task? = nil + if self.config.proactiveMemoryReclaim { + let machineReclaimer = MemoryReclaimer(ceiling: vmMemory) + reclaimer = machineReclaimer + let interval = self.config.memoryReclaimInterval + let containerID = self.id + reclaimTask = Task { + await machineReclaimer.run(interval: interval) { + let stats = try await vm.withAgent { agent in + try await agent.containerStatistics(containerIDs: [containerID], categories: .memory) + } + var anon: UInt64 = 0 + var refaultAnon: UInt64 = 0 + for stat in stats { + guard let memory = stat.memory else { continue } + anon += memory.anon + refaultAnon += memory.workingsetRefaultAnon + } + return (anon, refaultAnon) + } apply: { target in + try await self.setTargetMemorySize(target) + } + } + } + state = .created( + .init( + vm: vm, + relayManager: relayManager, + fileMountContext: fileMountContextHolder.withLock { $0 }, + reclaimer: reclaimer, + reclaimTask: reclaimTask + ) + ) } catch { try? await relayManager.stopAll() try? await vm.stop() @@ -849,16 +903,20 @@ extension LinuxContainer { let vm: any VirtualMachineInstance let relayManager: UnixSocketRelayManager + let reclaimTask: Task? let startedState = try? state.startedState("stop") if let startedState { vm = startedState.vm relayManager = startedState.relayManager + reclaimTask = startedState.reclaimTask } else { let createdState = try state.createdState("stop") vm = createdState.vm relayManager = createdState.relayManager + reclaimTask = createdState.reclaimTask } + reclaimTask?.cancel() var firstError: Error? do { @@ -1094,6 +1152,30 @@ extension LinuxContainer { try await vm.setTargetMemorySize(bytes) } + /// How the machine's proactive reclaim has been going, or nil when the + /// container runs without it. + public func memoryReclaimReport() async throws -> MemoryReclaimer.Report? { + let reclaimer = try await self.state.withLock { state -> MemoryReclaimer? in + switch state { + case .created(let createdState): + return createdState.reclaimer + case .started(let startedState): + return startedState.reclaimer + case .paused(let pausedState): + return pausedState.reclaimer + case .errored(let err): + throw err + default: + throw ContainerizationError( + .invalidState, + message: "failed to memoryReclaimReport: container must be created" + ) + } + } + guard let reclaimer else { return nil } + return await reclaimer.currentReport() + } + /// Close the containers standard input to signal no more input is /// arriving. public func closeStdin() async throws { diff --git a/Sources/Containerization/LinuxPod.swift b/Sources/Containerization/LinuxPod.swift index 73b64b43..d2efffd9 100644 --- a/Sources/Containerization/LinuxPod.swift +++ b/Sources/Containerization/LinuxPod.swift @@ -53,6 +53,15 @@ public final class LinuxPod: Sendable { /// their own. The `destination` field is ignored, as the area is /// enabled rather than mounted. public var swapLayer: Mount? = nil + /// Return memory the guest has freed to the host while the pod runs, + /// instead of leaving it held until the host is under pressure. A + /// loop watches what the pod's containers hold and asks the machine + /// to hold that plus headroom, backing off when the guest refaults. + /// The cloud-hypervisor backend reports freed pages to the host + /// continuously whether or not this is set. + public var proactiveMemoryReclaim: Bool = false + /// How often the reclaim loop looks at the guest. + public var memoryReclaimInterval: Duration = .seconds(10) /// The network interfaces for the pod. public var interfaces: [any Interface] = [] /// Whether nested virtualization should be turned on for the pod. @@ -249,6 +258,8 @@ public final class LinuxPod: Sendable { struct CreatedState: Sendable { let vm: any VirtualMachineInstance let relayManager: UnixSocketRelayManager + let reclaimer: MemoryReclaimer? + let reclaimTask: Task? } func createdState(_ operation: String) throws -> CreatedState { @@ -952,7 +963,29 @@ extension LinuxPod { state.containers[id]?.state = .created } - state.phase = .created(.init(vm: vm, relayManager: relayManager)) + var reclaimer: MemoryReclaimer? = nil + var reclaimTask: Task? = nil + if self.config.proactiveMemoryReclaim { + let machineReclaimer = MemoryReclaimer(ceiling: self.config.memoryInBytes) + reclaimer = machineReclaimer + let interval = self.config.memoryReclaimInterval + reclaimTask = Task { + await machineReclaimer.run(interval: interval) { + let stats = try await self.statistics(categories: .memory) + var anon: UInt64 = 0 + var refaultAnon: UInt64 = 0 + for stat in stats { + guard let memory = stat.memory else { continue } + anon += memory.anon + refaultAnon += memory.workingsetRefaultAnon + } + return (anon, refaultAnon) + } apply: { target in + try await self.setTargetMemorySize(target) + } + } + } + state.phase = .created(.init(vm: vm, relayManager: relayManager, reclaimer: reclaimer, reclaimTask: reclaimTask)) } catch { try? await relayManager.stopAll() try? await vm.stop() @@ -1224,6 +1257,7 @@ extension LinuxPod { let createdState = try state.phase.createdState("stop") do { + createdState.reclaimTask?.cancel() try await createdState.relayManager.stopAll() // Stop all containers @@ -1401,6 +1435,16 @@ extension LinuxPod { return stats } + /// How the machine's proactive reclaim has been going, or nil when the + /// pod runs without it. + public func memoryReclaimReport() async throws -> MemoryReclaimer.Report? { + let createdState = try await self.state.withLock { state in + try state.phase.createdState("memoryReclaimReport") + } + guard let reclaimer = createdState.reclaimer else { return nil } + return await reclaimer.currentReport() + } + /// Dial a vsock port in the pod's VM. public func dialVsock(port: UInt32) async throws -> FileHandle { try await self.state.withLock { state in diff --git a/Sources/Containerization/MemoryReclaim.swift b/Sources/Containerization/MemoryReclaim.swift index a09b13b7..6130d3b1 100644 --- a/Sources/Containerization/MemoryReclaim.swift +++ b/Sources/Containerization/MemoryReclaim.swift @@ -83,10 +83,24 @@ public struct MemoryReclaimPolicy: Sendable { /// Tracks a machine across looks so refaults can be compared against the /// previous reading rather than treated as an absolute. public actor MemoryReclaimer { + /// What the reclaimer has done so far: how often it looked at the guest, + /// how often it asked the machine for a new size, how often a look or an + /// ask failed, and the size it last asked for. + public struct Report: Sendable { + public var looks: Int = 0 + public var applies: Int = 0 + public var failures: Int = 0 + public var target: UInt64? + public var lastAnon: UInt64 = 0 + public var maxAnon: UInt64 = 0 + public var lastRefaultAnon: UInt64 = 0 + } + private let policy: MemoryReclaimPolicy private let ceiling: UInt64 private var lastRefaultAnon: UInt64? private var current: UInt64 + private var report = Report() public init(policy: MemoryReclaimPolicy = .init(), ceiling: UInt64) { self.policy = policy @@ -94,15 +108,29 @@ public actor MemoryReclaimer { self.current = ceiling } + public func currentReport() -> Report { + report + } + /// Fold in one reading and return the size to ask for, or nil when it has /// not changed enough to be worth asking. public func step(memory: ContainerStatistics.MemoryStatistics) -> UInt64? { + step(anon: memory.anon, refaultAnon: memory.workingsetRefaultAnon) + } + + /// Fold in one reading and return the size to ask for, or nil when it has + /// not changed enough to be worth asking. + /// - Parameters: + /// - anon: Anonymous memory the guest holds. + /// - refaultAnon: The guest's running count of anonymous pages fetched + /// back after reclaim. + public func step(anon: UInt64, refaultAnon: UInt64) -> UInt64? { let refaulted = - lastRefaultAnon.map { memory.workingsetRefaultAnon > $0 } ?? false - lastRefaultAnon = memory.workingsetRefaultAnon + lastRefaultAnon.map { refaultAnon > $0 } ?? false + lastRefaultAnon = refaultAnon let target = policy.nextTarget( - anon: memory.anon, + anon: anon, refaulted: refaulted, current: current, ceiling: ceiling @@ -117,6 +145,40 @@ public actor MemoryReclaimer { } } +extension MemoryReclaimer { + /// Look at the guest on a cadence until cancelled: sample what it holds, + /// fold the reading in, and apply the size that comes out. + /// + /// A sample or apply that fails leaves the machine as it was; the next + /// look starts fresh. + public func run( + interval: Duration, + sample: @Sendable () async throws -> (anon: UInt64, refaultAnon: UInt64), + apply: @Sendable (UInt64) async throws -> Void + ) async { + while !Task.isCancelled { + do { + try await Task.sleep(for: interval) + let (anon, refaultAnon) = try await sample() + report.looks += 1 + report.lastAnon = anon + report.maxAnon = max(report.maxAnon, anon) + report.lastRefaultAnon = refaultAnon + if let target = step(anon: anon, refaultAnon: refaultAnon) { + report.target = target + try await apply(target) + report.applies += 1 + } + } catch is CancellationError { + return + } catch { + report.failures += 1 + continue + } + } + } +} + extension UInt64 { fileprivate func absoluteDistance(to other: UInt64) -> UInt64 { self > other ? self - other : other - self diff --git a/Sources/Integration/PodTests.swift b/Sources/Integration/PodTests.swift index 2c0dffc2..aa20bd5b 100644 --- a/Sources/Integration/PodTests.swift +++ b/Sources/Integration/PodTests.swift @@ -2049,6 +2049,133 @@ extension IntegrationSuite { return socketPath } + #if os(macOS) + /// Boot a pod with the reclaim loop on a fast cadence, run an + /// anonymous-memory fill that holds long enough for several looks and + /// frees by exiting, and return the reclaimer's report of what it saw + /// and did. + private func reclaimReport(id: String) async throws -> MemoryReclaimer.Report? { + let bs = try await bootstrap(id) + let pod = try LinuxPod(id, vmm: bs.vmm) { config in + config.cpus = 4 + config.memoryInBytes = 2048.mib() + config.bootLog = bs.bootLog + config.proactiveMemoryReclaim = true + config.memoryReclaimInterval = .seconds(1) + } + + try await pod.addContainer("filler", rootfs: try cloneRootfs(bs.rootfs, testID: id, containerID: "filler")) { config in + config.process.arguments = [ + "/bin/sh", "-c", + "awk 'BEGIN{ b=sprintf(\"%01000000d\",0); while(length(b)<700000000) b=b b; t=systime(); while(systime()= 500.mib() else { + throw IntegrationError.assert( + msg: "the loop never saw the fill: \(report)") + } + guard report.applies >= 2 else { + throw IntegrationError.assert( + msg: "the loop did not follow the fill and the free with targets: \(report)") + } + guard let target = report.target, target <= 512.mib() else { + throw IntegrationError.assert( + msg: "the loop did not ask for the freed memory back: \(report)") + } + } + + /// A guest that is using its memory keeps it under proactive reclaim: + /// the loop sees what the holder holds and never asks for a size below + /// it. + func testPodReclaimBusy() async throws { + let id = "test-pod-reclaim-busy" + let bs = try await bootstrap(id) + let pod = try LinuxPod(id, vmm: bs.vmm) { config in + config.cpus = 4 + config.memoryInBytes = 2048.mib() + config.bootLog = bs.bootLog + config.proactiveMemoryReclaim = true + config.memoryReclaimInterval = .seconds(1) + } + + try await pod.addContainer("holder", rootfs: try cloneRootfs(bs.rootfs, testID: id, containerID: "holder")) { config in + config.process.arguments = [ + "/usr/bin/awk", + "BEGIN{ b=sprintf(\"%01000000d\",0); while(length(b)<300000000) b=b b; while(1){} }", + ] + } + + do { + try await pod.create() + try await pod.startContainer("holder") + + // Let the holder build its anonymous buffer, then give the loop + // time to look at it holding. + try await Task.sleep(for: .seconds(20)) + let report = try await pod.memoryReclaimReport() + + try await pod.killContainer("holder", signal: .kill) + _ = try await pod.waitContainer("holder") + try await pod.stop() + + guard let report, report.failures == 0 else { + throw IntegrationError.assert( + msg: "reclaim looks or applies failed: \(String(describing: report))") + } + guard report.maxAnon >= 200.mib() else { + throw IntegrationError.assert( + msg: "the loop never saw the holder: \(report)") + } + guard (report.target ?? 2048.mib()) >= report.maxAnon else { + throw IntegrationError.assert( + msg: "the loop asked a busy guest to hold less than it uses: \(report)") + } + } catch { + try? await pod.stop() + throw error + } + } + #endif + func testPodSysctl() async throws { let id = "test-pod-sysctl" diff --git a/Sources/Integration/Suite.swift b/Sources/Integration/Suite.swift index 99f534fa..c942cca2 100644 --- a/Sources/Integration/Suite.swift +++ b/Sources/Integration/Suite.swift @@ -612,6 +612,11 @@ struct IntegrationSuite: AsyncParsableCommand { // Nested virtualization (VZ-only feature) Test("nested virt", testNestedVirtualizationEnabled), + // Proactive memory reclaim (VZ balloon loop), asserted on the + // reclaimer's own report of what it saw and did + Test("pod reclaim follows the guest", testPodReclaimFollowsTheGuest), + Test("pod reclaim busy", testPodReclaimBusy), + // Filesystem operations (TODO: promote to cross-platform once verified on CH) Test("container frozen ext4 clone", testFrozenExt4Clone), Test("container trim ext4 clone", testTrimExt4Clone),