From f10eaca613f6eef28add5d69f4fbf573ca6ed156 Mon Sep 17 00:00:00 2001 From: Aaron Paterson Date: Sat, 1 Aug 2026 18:48:34 +0000 Subject: [PATCH 01/10] Resolve test directory paths before reaping per-test files The reaper that runs at a concurrency of one deletes everything in the test directory except the unpacked rootfs it means to preserve. It held that rootfs path as FileManager reports it, under /var, and compared it against the entries of contentsOfDirectory, which reports them under /private/var, so the preserved path never matched any entry and the rootfs was deleted along with the per-test files. The unpack coordinator still held it as unpacked, so the next test opened a rootfs that was no longer there and failed with a missing file error. Resolve both sides with resolvingSymlinksInPathWithPrivate, which exists for this difference between the two views of the same directory. --- Sources/Integration/Suite.swift | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/Sources/Integration/Suite.swift b/Sources/Integration/Suite.swift index 9dae8435..e43cf372 100644 --- a/Sources/Integration/Suite.swift +++ b/Sources/Integration/Suite.swift @@ -259,12 +259,16 @@ struct IntegrationSuite: AsyncParsableCommand { // a ~2GB rootfs and a ~512MB initfs, so without reaping the dev // container fills its CoW layer in ~10 tests. if self.maxConcurrency == 1 { - let preserve = fsPath.absolutePath() + // contentsOfDirectory reports paths under /private, while testDir is + // built from FileManager's /var view of the same directory, so both + // sides are resolved before comparing. + let preserve = fsPath.resolvingSymlinksInPathWithPrivate().absolutePath() if let entries = try? FileManager.default.contentsOfDirectory( at: Self.testDir, includingPropertiesForKeys: nil ) { - for url in entries where url.absolutePath() != preserve { + for url in entries + where url.resolvingSymlinksInPathWithPrivate().absolutePath() != preserve { try? FileManager.default.removeItem(at: url) } } From 6499f7dfa7fff64c903642db690094d7e6c64ec8 Mon Sep 17 00:00:00 2001 From: Aaron Paterson Date: Sun, 2 Aug 2026 00:51:10 +0000 Subject: [PATCH 02/10] Say how to run the tests that macOS skips Tests guarded for Linux are compiled out on macOS, so `make test` reports success without having run them, and nothing says so. The target that does run them is not mentioned anywhere outside the makefile. --- README.md | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/README.md b/README.md index f658b08d..372a872a 100644 --- a/README.md +++ b/README.md @@ -130,6 +130,16 @@ After building, run basic and integration tests: make test integration ``` +Tests that only apply to Linux are compiled out on macOS, so `make test` passes +without running them. Run those with: + +```bash +make linux-test +``` + +which runs the same tests inside the Linux dev container, as the Linux build +workflow does. + A kernel is required to run integration tests. If you do not have a kernel locally, a default kernel can be fetched using the `make fetch-default-kernel` target. From bf4d7dcf8bbe64c745e9ef344a8c173d2a42a3af Mon Sep 17 00:00:00 2001 From: Aaron Paterson Date: Sun, 2 Aug 2026 01:20:29 +0000 Subject: [PATCH 03/10] Test against the kernel release the shipped tool installs The default kernel the tests fetch had drifted from the one users run. container installs kata 3.28.0 and its 6.18.15 kernel, while these tests fetched 3.17.0, so the suite exercised a guest with a different feature set to the one it is meant to represent. Nested runtimes are the visible case: 3.17.0 was built without nf_tables, so a docker daemon inside a container fails there and works on what ships. Kata moved from xz to zstd between those releases, so the archive is no longer named for its compression and tar is left to recognise it rather than being told, which also holds if the format changes again. --- Makefile | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Makefile b/Makefile index c79d6995..3d01b93a 100644 --- a/Makefile +++ b/Makefile @@ -64,7 +64,7 @@ LIBARCHIVE_UPSTREAM_REPO := https://github.com/libarchive/libarchive LIBARCHIVE_UPSTREAM_VERSION := v3.7.7 LIBARCHIVE_LOCAL_DIR := workdir/libarchive -KATA_BINARY_PACKAGE := https://github.com/kata-containers/kata-containers/releases/download/3.17.0/kata-static-3.17.0-arm64.tar.xz +KATA_BINARY_PACKAGE := https://github.com/kata-containers/kata-containers/releases/download/3.28.0/kata-static-3.28.0-arm64.tar.zst CLOUD_HYPERVISOR_URL := https://github.com/cloud-hypervisor/cloud-hypervisor/releases/download/v52.0/cloud-hypervisor-static-aarch64 # SHA256 of the v52.0 aarch64 static binary (verified locally from the # upstream release artifact). Bump alongside CLOUD_HYPERVISOR_URL. @@ -410,11 +410,11 @@ integration: .PHONY: fetch-default-kernel fetch-default-kernel: @mkdir -p .local/ bin/ -ifeq (,$(wildcard .local/kata.tar.gz)) - @curl -SsL -o .local/kata.tar.gz ${KATA_BINARY_PACKAGE} +ifeq (,$(wildcard .local/kata.tar)) + @curl -SsL -o .local/kata.tar ${KATA_BINARY_PACKAGE} endif ifeq (,$(wildcard .local/vmlinux-$(KERNEL_ARCH))) - @tar -zxf .local/kata.tar.gz -C .local/ --strip-components=1 + @tar -xf .local/kata.tar -C .local/ --strip-components=1 @cp -L .local/opt/kata/share/kata-containers/vmlinux.container .local/vmlinux-$(KERNEL_ARCH) endif ifeq (,$(wildcard bin/vmlinux-$(KERNEL_ARCH))) From 21f1f16eaaacc051bbc5be6566814cb606de5698 Mon Sep 17 00:00:00 2001 From: Aaron Paterson Date: Wed, 12 Aug 2026 05:25:08 +0000 Subject: [PATCH 04/10] Run each integration test in its own scratch directory A test's rootfs clones, init block, and writable layers land in a directory the runner creates for it and deletes when it finishes, so a test holds disk only while it runs at any concurrency. The run-shared directory holds the unpacked images every bootstrap clones from, and is removed when the run ends. --- Sources/Integration/PodTests.swift | 7 +++-- Sources/Integration/Suite.swift | 43 +++++++++++++----------------- 2 files changed, 21 insertions(+), 29 deletions(-) diff --git a/Sources/Integration/PodTests.swift b/Sources/Integration/PodTests.swift index ae1caec8..7dff20fa 100644 --- a/Sources/Integration/PodTests.swift +++ b/Sources/Integration/PodTests.swift @@ -2228,10 +2228,9 @@ extension IntegrationSuite { /// (directory-share) rootfs. Assumes a single-layer image (the alpine /// image used by the suite) so no OCI whiteout processing is required. /// - /// The extracted dir lives under `Self.testDir`; do NOT `defer`-remove it - /// here — virtiofsd shares it for the whole test. It is swept by - /// `bootstrap`'s `maxConcurrency == 1` reaper on the next test and by the - /// suite-end `removeItem(at: Self.testDir)`. + /// The extracted dir lives under the test's scratch directory; do NOT + /// `defer`-remove it here — virtiofsd shares it for the whole test. The + /// runner removes the scratch directory when the test finishes. private func unpackRootfsDirectory(_ image: Containerization.Image, testID: String) async throws -> Containerization.Mount { let dir = Self.testDir.appending(component: "\(testID)-rootfs-dir") try? FileManager.default.removeItem(at: dir) diff --git a/Sources/Integration/Suite.swift b/Sources/Integration/Suite.swift index e43cf372..84580dba 100644 --- a/Sources/Integration/Suite.swift +++ b/Sources/Integration/Suite.swift @@ -135,7 +135,19 @@ struct IntegrationSuite: AsyncParsableCommand { FileManager.default.uniqueTemporaryDirectory(create: true) }() + /// The scratch directory of the test running on this task. The runner + /// creates one per test and deletes it when the test finishes, so a + /// test's clones and layers hold disk only while it runs. + @TaskLocal static var currentTestDir: URL? + static var testDir: URL { + currentTestDir ?? _testDir + } + + /// The run-shared directory holding unpacked images, which every test's + /// bootstrap clones from. It lives for the whole run and is removed at + /// the end. + static var imageCacheDir: URL { _testDir } @@ -233,7 +245,7 @@ struct IntegrationSuite: AsyncParsableCommand { let platform = Platform(arch: "arm64", os: "linux", variant: "v8") // Unpack to shared location with coordination to prevent concurrent unpacks - let fsPath = Self.testDir.appending(component: image.digest) + let fsPath = Self.imageCacheDir.appending(component: image.digest) let fs = try await Self.unpackCoordinator.unpack(key: fsPath.absolutePath()) { do { let unpacker = EXT4Unpacker(capacityInBytes: 2.gib()) @@ -251,29 +263,6 @@ struct IntegrationSuite: AsyncParsableCommand { } } - // Reap any per-test artifacts left over from prior tests. With - // `--max-concurrency 1` (linux-integration default) this runs after - // the previous test has fully completed, so it's race-free; on - // macOS where tests can run in parallel we just keep all files — - // disk usage isn't a concern there. Each per-test bootstrap clones - // a ~2GB rootfs and a ~512MB initfs, so without reaping the dev - // container fills its CoW layer in ~10 tests. - if self.maxConcurrency == 1 { - // contentsOfDirectory reports paths under /private, while testDir is - // built from FileManager's /var view of the same directory, so both - // sides are resolved before comparing. - let preserve = fsPath.resolvingSymlinksInPathWithPrivate().absolutePath() - if let entries = try? FileManager.default.contentsOfDirectory( - at: Self.testDir, - includingPropertiesForKeys: nil - ) { - for url in entries - where url.resolvingSymlinksInPathWithPrivate().absolutePath() != preserve { - try? FileManager.default.removeItem(at: url) - } - } - } - // Clone to test-specific path let clPath = Self.testDir.appending(component: "\(testID).ext4").absolutePath() try? FileManager.default.removeItem(atPath: clPath) @@ -658,11 +647,14 @@ struct IntegrationSuite: AsyncParsableCommand { for _ in 0.. Date: Wed, 12 Aug 2026 15:00:44 +0000 Subject: [PATCH 05/10] Give the machine's storage the shape its containers have A container's storage is its rootfs, an optional writable layer, and its mounts; a machine's is its containers' plus the volumes they share. One generic shape describes both the Mount values a machine is configured with and the AttachedFilesystem values it reports once attached, so converting between the two is a map over the structure and the roles cannot drift between configuration and attachment. Device addresses are allocated walking the same sorted order the devices are created in, on both backends. Consumers read roles instead of list positions: the spec builders take a container's mounts without prefix arithmetic, pod volumes mount by name, and a cloud-hypervisor volume's virtiofsd is held by a machine-lifetime reference alongside its containers' reference counts. --- .../Containerization/CHHotplugProvider.swift | 79 ++++++----- .../CHVirtualMachineInstance.swift | 127 ++++++++++-------- .../CHVirtualMachineManager.swift | 2 +- .../Containerization/ContainerStorage.swift | 107 +++++++++++++++ .../Containerization/HotplugProvider.swift | 2 +- Sources/Containerization/LinuxContainer.swift | 51 +++---- Sources/Containerization/LinuxPod.swift | 98 +++++++------- .../Containerization/VMConfiguration.swift | 10 +- .../VZVirtualMachineInstance.swift | 100 +++++++------- .../VZVirtualMachineManager.swift | 2 +- .../VirtualMachineInstance.swift | 5 +- 11 files changed, 351 insertions(+), 232 deletions(-) create mode 100644 Sources/Containerization/ContainerStorage.swift diff --git a/Sources/Containerization/CHHotplugProvider.swift b/Sources/Containerization/CHHotplugProvider.swift index 870fb1ff..380cba76 100644 --- a/Sources/Containerization/CHHotplugProvider.swift +++ b/Sources/Containerization/CHHotplugProvider.swift @@ -27,8 +27,8 @@ import Synchronization /// /// Handles both block (`vm.add-disk`) and virtiofs (`vm.add-fs`, with one /// `virtiofsd` per unique source-hash tag) hotplug, plus the matching -/// `vm.remove-device` teardown. Owns the per-VM mount registry so -/// `CHVirtualMachineInstance.mounts` can forward to it. +/// `vm.remove-device` teardown. Owns the machine's storage so +/// `CHVirtualMachineInstance.storage` can forward to it. final class CHHotplugProvider: HotplugProvider { struct HotplugRecord: Sendable { let chDeviceId: String @@ -50,7 +50,7 @@ final class CHHotplugProvider: HotplugProvider { private let workDir: URL private let virtiofsdBinaryOverride: URL? private let allocator: any AddressAllocator - private let _mounts: Mutex<[String: [AttachedFilesystem]]> + private let _storage: Mutex private let _records: Mutex<[String: [HotplugRecord]]> private let _tags: Mutex<[String: VirtiofsdTagState]> /// Serializes per-tag virtiofsd spawn so a concurrent hotplug for the @@ -65,14 +65,14 @@ final class CHHotplugProvider: HotplugProvider { workDir: URL, virtiofsdBinary: URL?, allocator: any AddressAllocator, - initialMounts: [String: [AttachedFilesystem]], + initialStorage: MachineAttachments, logger: Logger? ) { self.client = client self.workDir = workDir self.virtiofsdBinaryOverride = virtiofsdBinary self.allocator = allocator - self._mounts = Mutex(initialMounts) + self._storage = Mutex(initialStorage) self._records = Mutex([:]) self._tags = Mutex([:]) self.spawnLock = AsyncLock() @@ -81,14 +81,14 @@ final class CHHotplugProvider: HotplugProvider { // MARK: - Read accessors - var mounts: [String: [AttachedFilesystem]] { - _mounts.withLock { $0 } + var storage: MachineAttachments { + _storage.withLock { $0 } } - func withMountRegistry( - _ body: (inout sending [String: [AttachedFilesystem]]) throws -> sending T + func withStorage( + _ body: (inout sending MachineAttachments) throws -> sending T ) rethrows -> T { - try _mounts.withLock(body) + try _storage.withLock(body) } // MARK: - HotplugProvider conformance @@ -151,12 +151,13 @@ final class CHHotplugProvider: HotplugProvider { } func registerMounts(id: String, rootfs: AttachedFilesystem, additionalMounts: [Mount]) throws { - var attached: [AttachedFilesystem] = [rootfs] + var mounts: [AttachedFilesystem] = [] for mount in additionalMounts { - attached.append(try AttachedFilesystem(mount: mount, allocator: allocator)) + mounts.append(try AttachedFilesystem(mount: mount, allocator: allocator)) } - _mounts.withLock { - $0[id, default: []].append(contentsOf: attached) + let container = ContainerAttachments(rootfs: rootfs, mounts: mounts) + _storage.withLock { + $0.containers[id] = container } } @@ -190,19 +191,10 @@ final class CHHotplugProvider: HotplugProvider { } } - // Drop block-derived AttachedFilesystem entries for `id`. Block entries - // are the ones whose source was rewritten to "/dev/vd" by - // `hotplug(_:)` (or by AttachedFilesystem(mount:allocator:) for an - // additionalMount of type virtio-blk). - _mounts.withLock { state in - guard var perID = state[id] else { return } - perID.removeAll { $0.source.hasPrefix("/dev/vd") } - if perID.isEmpty { - state.removeValue(forKey: id) - } else { - state[id] = perID - } - } + // The container's devices are gone, so the container leaves the + // registry with them; its shares are released separately and their + // processes reference-counted through `_tags`. + _ = _storage.withLock { $0.containers.removeValue(forKey: id) } } func hotplugVirtioFS(_ mounts: [Mount], id: String) async throws { @@ -226,7 +218,7 @@ final class CHHotplugProvider: HotplugProvider { let chDeviceId = try await ensureVirtiofsDevice(tag: tag, source: source, readonly: readonly) // Record once per tag for this container. The AttachedFilesystem // entries for these mounts are written by registerMounts (the sole - // _mounts writer), so we do NOT touch _mounts here. + // registry writer), so we do NOT touch the storage here. _records.withLock { $0[id, default: []].append(HotplugRecord(chDeviceId: chDeviceId, kind: .virtiofs(tag: tag))) } @@ -338,16 +330,17 @@ final class CHHotplugProvider: HotplugProvider { try? FileManager.default.removeItem(at: socket) } - // Drop virtiofs AttachedFilesystem entries for `id`. AttachedFilesystem - // sets `type = mount.type` which for a `.virtiofs` mount is "virtiofs". - _mounts.withLock { state in - guard var perID = state[id] else { return } - perID.removeAll { $0.type == "virtiofs" } - if perID.isEmpty { - state.removeValue(forKey: id) - } else { - state[id] = perID + // Drop the container's virtiofs entries. A container whose rootfs is + // itself a share leaves the registry whole; one that keeps block + // devices keeps its entry with the share entries dropped. + _storage.withLock { state in + guard var container = state.containers[id] else { return } + if container.rootfs.type == "virtiofs" { + state.containers.removeValue(forKey: id) + return } + container.mounts.removeAll { $0.type == "virtiofs" } + state.containers[id] = container } } @@ -358,15 +351,21 @@ final class CHHotplugProvider: HotplugProvider { /// is the user-supplied `FsConfig.id` (which `vm.remove-device` keys on). /// `ownerIds` are the container ids that count toward this tag's refcount; /// each gets a `HotplugRecord` so `releaseVirtioFS(id:)` walks them - /// uniformly. + /// uniformly. `machineHeld` adds one reference nothing releases, for a + /// share the machine itself owns (a volume). func recordBootTimeVirtiofs( tag: String, process: VirtiofsdProcess, chDeviceId: String, - ownerIds: [String] + ownerIds: [String], + machineHeld: Bool ) { _tags.withLock { - $0[tag] = VirtiofsdTagState(process: process, refcount: ownerIds.count, chDeviceId: chDeviceId) + $0[tag] = VirtiofsdTagState( + process: process, + refcount: ownerIds.count + (machineHeld ? 1 : 0), + chDeviceId: chDeviceId + ) } _records.withLock { records in for id in ownerIds { diff --git a/Sources/Containerization/CHVirtualMachineInstance.swift b/Sources/Containerization/CHVirtualMachineInstance.swift index 122f592a..e363d5ca 100644 --- a/Sources/Containerization/CHVirtualMachineInstance.swift +++ b/Sources/Containerization/CHVirtualMachineInstance.swift @@ -46,7 +46,7 @@ public final class CHVirtualMachineInstance: Sendable { public struct Configuration: Sendable { public var cpus: Int public var memoryInBytes: UInt64 - public var mountsByID: [String: [Mount]] + public var storage: MachineMounts public var interfaces: [any Interface] public var kernel: Kernel? public var initialFilesystem: Mount? @@ -56,7 +56,7 @@ public final class CHVirtualMachineInstance: Sendable { public init() { self.cpus = 4 self.memoryInBytes = 1024 * 1024 * 1024 - self.mountsByID = [:] + self.storage = MachineMounts() self.interfaces = [] } } @@ -65,8 +65,8 @@ public final class CHVirtualMachineInstance: Sendable { /// `start()`'s `VmConfig.disks` ordering matches the allocator letters. struct BootDisk: Sendable { let mount: Mount - let containerId: String? // nil for rootfs - let letter: Character + let chId: String + let readonly: Bool } // MARK: - State @@ -76,8 +76,8 @@ public final class CHVirtualMachineInstance: Sendable { _state.withLock { $0 } } - public var mounts: [String: [AttachedFilesystem]] { - hotplug.mounts + public var storage: MachineAttachments { + hotplug.storage } /// Cloud-hypervisor exposes one virtio-fs device per source-hash tag, so @@ -167,9 +167,9 @@ public final class CHVirtualMachineInstance: Sendable { ) self.workDir = workDir - // 2. Block allocator + boot inventory. Walks rootfs first, then - // mountsByID sorted by container id, allocating disk letters in - // that order. The same allocator is later handed to the hotplug + // 2. Block allocator + boot inventory. Walks rootfs first, then the + // machine's storage in its device order, allocating disk letters + // as it goes. The same allocator is later handed to the hotplug // provider so runtime add-disk picks up where boot wiring left off. let allocator = Character.blockDeviceTagAllocator() let inventory = try config.bootInventory(allocator: allocator) @@ -201,14 +201,14 @@ public final class CHVirtualMachineInstance: Sendable { logger: logger ?? Logger(label: "CloudHypervisor.Client") ) - // 5. Hotplug provider — owns the mount registry, seeded with the + // 5. Hotplug provider — owns the machine's storage, seeded with the // boot inventory so registerMounts can append to it. self.hotplug = CHHotplugProvider( client: self.client, workDir: workDir, virtiofsdBinary: virtiofsdBinary, allocator: allocator, - initialMounts: inventory.attachments, + initialStorage: inventory.attachments, logger: logger ) @@ -222,11 +222,11 @@ public final class CHVirtualMachineInstance: Sendable { self._stdioPool = Mutex([:]) } - /// Mutate the mount registry. Forwards to the hotplug provider, which + /// Mutate the storage registry. Forwards to the hotplug provider, which /// owns the registry. Kept on the instance for parity with the macOS - /// path's `withMountRegistry` API. - func withMountRegistry(_ body: (inout sending [String: [AttachedFilesystem]]) throws -> sending T) rethrows -> T { - try hotplug.withMountRegistry(body) + /// path's `withStorage` API. + func withStorage(_ body: (inout sending MachineAttachments) throws -> sending T) rethrows -> T { + try hotplug.withStorage(body) } } @@ -510,34 +510,46 @@ extension CHVirtualMachineInstance { throw ContainerizationError(.invalidArgument, message: "initialFilesystem is required for cloud-hypervisor backend") } - // Disks: rootfs forced read-only at the device level; container disks + // Disks: rootfs forced read-only at the device level; other disks // honor their `ro` option through chDiskConfig. var disks: [CloudHypervisor.DiskConfig] = [] for bd in bootDisks { - let chId = bd.containerId.map { "blk-\($0)-\(bd.letter)" } ?? "rootfs" - if var disk = bd.mount.chDiskConfig(id: chId) { - if bd.containerId == nil { + if var disk = bd.mount.chDiskConfig(id: bd.chId) { + if bd.readonly { disk.readonly = true } disks.append(disk) } } - // Virtiofs: group all .virtiofs mounts in mountsByID by source-hash - // tag, spawn one virtiofsd per tag, build matching FsConfigs. - var byTag: [String: (mounts: [Mount], owners: [String])] = [:] - for cid in config.mountsByID.keys.sorted() { - guard let mounts = config.mountsByID[cid] else { continue } - for mount in mounts { - guard case .virtiofs = mount.runtimeOptions else { continue } - let tag = try hashFilePath(path: mount.source) - var entry = byTag[tag] ?? (mounts: [], owners: []) - entry.mounts.append(mount) - if !entry.owners.contains(cid) { - entry.owners.append(cid) + // Virtiofs: group all virtiofs mounts in the machine's storage by + // source-hash tag, spawn one virtiofsd per tag, build matching + // FsConfigs. A volume's tag is the machine's own, held for its + // lifetime. + var byTag: [String: (mounts: [Mount], owners: [String], machineHeld: Bool)] = [:] + func groupVirtiofs(_ mount: Mount, owner: String?) throws { + guard case .virtiofs = mount.runtimeOptions else { return } + let tag = try hashFilePath(path: mount.source) + var entry = byTag[tag] ?? (mounts: [], owners: [], machineHeld: false) + entry.mounts.append(mount) + if let owner { + if !entry.owners.contains(owner) { + entry.owners.append(owner) } - byTag[tag] = entry + } else { + entry.machineHeld = true } + byTag[tag] = entry + } + for cid in config.storage.containers.keys.sorted() { + guard let container = config.storage.containers[cid] else { continue } + for mount in container.all { + try groupVirtiofs(mount, owner: cid) + } + } + for name in config.storage.volumes.keys.sorted() { + guard let mount = config.storage.volumes[name] else { continue } + try groupVirtiofs(mount, owner: nil) } var fsConfigs: [CloudHypervisor.FsConfig] = [] @@ -569,7 +581,8 @@ extension CHVirtualMachineInstance { tag: tag, process: process, chDeviceId: chDeviceId, - ownerIds: entry.owners + ownerIds: entry.owners, + machineHeld: entry.machineHeld ) fsConfigs.append( @@ -714,40 +727,46 @@ extension CHVirtualMachineInstance { // MARK: - Boot inventory extension CHVirtualMachineInstance.Configuration { - /// Walks boot-time mounts in deterministic order (rootfs first, then - /// `mountsByID` sorted by container id, then each container's mounts in - /// input order), allocating disk letters for virtio-blk mounts and seeding - /// the per-container `AttachedFilesystem` registry. + /// Walks boot-time storage in the machine's device order (rootfs first, + /// then containers sorted by id, each in role order, then volumes sorted + /// by name), allocating disk letters for virtio-blk mounts and seeding + /// the machine's `AttachedFilesystem` registry. /// /// The allocator is shared with the runtime hotplug provider, so block /// hotplug picks up at the next free letter after boot. func bootInventory( allocator: any AddressAllocator - ) throws -> (attachments: [String: [AttachedFilesystem]], bootDisks: [CHVirtualMachineInstance.BootDisk]) { + ) throws -> (attachments: MachineAttachments, bootDisks: [CHVirtualMachineInstance.BootDisk]) { var bootDisks: [CHVirtualMachineInstance.BootDisk] = [] - var attachments: [String: [AttachedFilesystem]] = [:] - // Rootfs is not part of mountsByID. If it's a block device, it claims - // the first letter (vda) so the kernel cmdline `root=/dev/vda` is right. + // The rootfs is the machine's own, not part of its containers' + // storage. If it's a block device, it claims the first letter (vda) + // so the kernel cmdline `root=/dev/vda` is right. if let rootfs = self.initialFilesystem, rootfs.isBlock { - let letter = try allocator.allocate() - bootDisks.append(.init(mount: rootfs, containerId: nil, letter: letter)) + _ = try allocator.allocate() + bootDisks.append(.init(mount: rootfs, chId: "rootfs", readonly: true)) } - for cid in self.mountsByID.keys.sorted() { - guard let mounts = self.mountsByID[cid] else { continue } - var perContainer: [AttachedFilesystem] = [] - for mount in mounts { - let attached = try AttachedFilesystem(mount: mount, allocator: allocator) - if mount.isBlock, let letter = attached.source.last { - bootDisks.append(.init(mount: mount, containerId: cid, letter: letter)) - } - perContainer.append(attached) + func attach(_ mount: Mount, chId: (Character) -> String) throws -> AttachedFilesystem { + let attached = try AttachedFilesystem(mount: mount, allocator: allocator) + if mount.isBlock, let letter = attached.source.last { + bootDisks.append(.init(mount: mount, chId: chId(letter), readonly: false)) } - attachments[cid] = perContainer + return attached + } + + var containers: [String: ContainerAttachments] = [:] + for cid in self.storage.containers.keys.sorted() { + guard let container = self.storage.containers[cid] else { continue } + containers[cid] = try container.map { try attach($0, chId: { "blk-\(cid)-\($0)" }) } + } + var volumes: [String: AttachedFilesystem] = [:] + for name in self.storage.volumes.keys.sorted() { + guard let mount = self.storage.volumes[name] else { continue } + volumes[name] = try attach(mount, chId: { "vol-\(name)-\($0)" }) } - return (attachments, bootDisks) + return (MachineAttachments(containers: containers, volumes: volumes), bootDisks) } } #endif diff --git a/Sources/Containerization/CHVirtualMachineManager.swift b/Sources/Containerization/CHVirtualMachineManager.swift index ba20b5d1..c444620d 100644 --- a/Sources/Containerization/CHVirtualMachineManager.swift +++ b/Sources/Containerization/CHVirtualMachineManager.swift @@ -99,7 +99,7 @@ public struct CHVirtualMachineManager: VirtualMachineManager { instanceConfig.cpus = vmConfig.cpus instanceConfig.memoryInBytes = vmConfig.memoryInBytes instanceConfig.interfaces = vmConfig.interfaces - instanceConfig.mountsByID = vmConfig.mountsByID + instanceConfig.storage = vmConfig.storage instanceConfig.bootLog = vmConfig.bootLog instanceConfig.extensions = vmConfig.extensions instanceConfig.kernel = kernel diff --git a/Sources/Containerization/ContainerStorage.swift b/Sources/Containerization/ContainerStorage.swift new file mode 100644 index 00000000..4d8de2ed --- /dev/null +++ b/Sources/Containerization/ContainerStorage.swift @@ -0,0 +1,107 @@ +//===----------------------------------------------------------------------===// +// Copyright © 2026 Apple Inc. and the Containerization project authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//===----------------------------------------------------------------------===// + +/// A container's storage, by role. +/// +/// The same shape describes storage at both stages of its life: as the +/// `Mount` values a container is configured with, and as the +/// `AttachedFilesystem` values the machine reports once they are attached. +/// Converting between the two is a `map` over the structure, so the roles +/// cannot drift apart between configuration and attachment. +public struct ContainerStorage: Sendable { + /// The container's root filesystem. + public var rootfs: Value + + /// The writable layer mounted over the rootfs as the upper side of an + /// overlay, when the container has one. + public var writableLayer: Value? + + /// The container's remaining mounts, in configuration order. + public var mounts: [Value] + + public init(rootfs: Value, writableLayer: Value? = nil, mounts: [Value] = []) { + self.rootfs = rootfs + self.writableLayer = writableLayer + self.mounts = mounts + } + + /// Every value in the structure: the rootfs, the writable layer when + /// present, then the mounts, in that order. + public var all: [Value] { + var values = [rootfs] + if let writableLayer { + values.append(writableLayer) + } + values.append(contentsOf: mounts) + return values + } + + /// The storage with `transform` applied to every value, each keeping + /// its role. + public func map(_ transform: (Value) throws -> U) rethrows -> ContainerStorage { + ContainerStorage( + rootfs: try transform(rootfs), + writableLayer: try writableLayer.map(transform), + mounts: try mounts.map(transform) + ) + } +} + +/// A container's storage as configured, before its machine exists. +public typealias ContainerMounts = ContainerStorage + +/// A container's storage as attached to its machine. +public typealias ContainerAttachments = ContainerStorage + +/// The storage a machine carries: each container's, and the resources its +/// containers share. +public struct MachineStorage: Sendable { + /// Each container's storage, by container ID. + public var containers: [String: ContainerStorage] + + /// Volumes shared by the machine's containers, by volume name. + public var volumes: [String: Value] + + public init(containers: [String: ContainerStorage] = [:], volumes: [String: Value] = [:]) { + self.containers = containers + self.volumes = volumes + } + + /// Every value the machine carries: containers sorted by ID, each in + /// role order, then volumes sorted by name. Walks that allocate device + /// addresses and walks that create the devices use this one order, so an + /// address always names the device it was handed out for. + public var ordered: [Value] { + containers.keys.sorted().flatMap { containers[$0]?.all ?? [] } + + volumes.keys.sorted().compactMap { volumes[$0] } + } + + /// The storage with `transform` applied to every value, each keeping its + /// place. The transform runs in no particular order, so an allocating + /// conversion walks the fields sorted itself. + public func map(_ transform: (Value) throws -> U) rethrows -> MachineStorage { + MachineStorage( + containers: try containers.mapValues { try $0.map(transform) }, + volumes: try volumes.mapValues(transform) + ) + } +} + +/// A machine's storage as configured, before it exists. +public typealias MachineMounts = MachineStorage + +/// A machine's storage as attached. +public typealias MachineAttachments = MachineStorage diff --git a/Sources/Containerization/HotplugProvider.swift b/Sources/Containerization/HotplugProvider.swift index f535ce3f..0835077f 100644 --- a/Sources/Containerization/HotplugProvider.swift +++ b/Sources/Containerization/HotplugProvider.swift @@ -26,7 +26,7 @@ public protocol HotplugProvider: Sendable { /// - Returns: The attached filesystem with the device path in the guest func hotplug(_ block: Mount, id: String) async throws -> AttachedFilesystem - /// Register mounts for a container in the VM's mount registry. + /// Register mounts for a container in the machine's storage. /// - Parameters: /// - id: The container ID /// - rootfs: The rootfs attachment from hotplug diff --git a/Sources/Containerization/LinuxContainer.swift b/Sources/Containerization/LinuxContainer.swift index 34964fcd..55bbb39a 100644 --- a/Sources/Containerization/LinuxContainer.swift +++ b/Sources/Containerization/LinuxContainer.swift @@ -537,24 +537,14 @@ extension LinuxContainer { } private func mountRootfs( - attachments: [AttachedFilesystem], + attached: ContainerAttachments, rootfsPath: String, agent: VirtualMachineAgent ) async throws { - guard let rootfsAttachment = attachments.first else { - throw ContainerizationError(.notFound, message: "rootfs mount not found") - } + let rootfsAttachment = attached.rootfs - if self.writableLayer != nil { + if let writableAttachment = attached.writableLayer { // Set up overlayfs with image as lower layer and writable layer as upper. - guard attachments.count >= 2 else { - throw ContainerizationError( - .notFound, - message: "writable layer mount not found" - ) - } - let writableAttachment = attachments[1] - let lowerPath = "/run/container/\(self.id)/lower" let upperMountPath = "/run/container/\(self.id)/upper" let upperPath = "/run/container/\(self.id)/upper/diff" @@ -623,17 +613,18 @@ extension LinuxContainer { // This is dumb, but alas. let fileMountContextHolder = Mutex(fileMountContext) - // Build the list of mounts to attach to the VM. - var containerMounts = [modifiedRootfs] + fileMountContext.transformedMounts - if let writableLayer = self.writableLayer { - containerMounts.insert(writableLayer, at: 1) - } + // Build the container's storage to attach to the VM. + let containerStorage = ContainerMounts( + rootfs: modifiedRootfs, + writableLayer: self.writableLayer, + mounts: fileMountContext.transformedMounts + ) let vmConfig = VMConfiguration( cpus: vmCpus, memoryInBytes: vmMemory, interfaces: self.interfaces, - mountsByID: [self.id: containerMounts], + storage: MachineMounts(containers: [self.id: containerStorage]), bootLog: self.config.bootLog, nestedVirtualization: self.config.virtualization ) @@ -643,7 +634,7 @@ extension LinuxContainer { do { try await vm.start() - let mountsForAgent = containerMounts + let storageForAgent = containerStorage try await vm.withAgent { agent in try await agent.standardSetup() @@ -656,7 +647,7 @@ extension LinuxContainer { // with zero shares), but the cloud-hypervisor backend // only spawns virtiofsd when shares exist; mounting an // unbacked tag fails with EINVAL. - let hasVirtiofsMount = mountsForAgent.contains { mount in + let hasVirtiofsMount = storageForAgent.all.contains { mount in if case .virtiofs = mount.runtimeOptions { return true } return false } @@ -671,7 +662,7 @@ extension LinuxContainer { // gets populated. if vm.virtiofsLayout == .perTag { try await agent.mkdir(path: "/run/virtiofs", all: true, perms: 0o755) - let virtiofsAttachments = (vm.mounts[self.id] ?? []).filter { $0.type == "virtiofs" } + let virtiofsAttachments = (vm.storage.containers[self.id]?.all ?? []).filter { $0.type == "virtiofs" } let uniqueTags = Set(virtiofsAttachments.map(\.source)) for tag in uniqueTags { let dest = "/run/virtiofs/\(tag)" @@ -695,18 +686,17 @@ extension LinuxContainer { } } - guard let attachments = vm.mounts[self.id] else { + guard let attached = vm.storage.containers[self.id] else { throw ContainerizationError(.notFound, message: "rootfs mount not found") } let rootfsPath = Self.guestRootfsPath(self.id) - try await self.mountRootfs(attachments: attachments, rootfsPath: rootfsPath, agent: agent) + try await self.mountRootfs(attached: attached, rootfsPath: rootfsPath, agent: agent) // Mount file mount holding directories under /run. if fileMountContext.hasFileMounts { - let containerMounts = vm.mounts[self.id] ?? [] var ctx = fileMountContextHolder.withLock { $0 } try await ctx.mountHoldingDirectories( - vmMounts: containerMounts, + vmMounts: attached.mounts, agent: agent ) fileMountContextHolder.withLock { $0 = ctx } @@ -764,15 +754,12 @@ extension LinuxContainer { let agent = try await createdState.vm.dialAgent() do { var spec = self.generateRuntimeSpec() - // We don't need the rootfs (or writable layer), nor do OCI runtimes want it included. - // Also filter out file mount holding directories. We'll mount those separately under /run. + // Filter out file mount holding directories. We'll mount those separately under /run. // Transform virtiofs mounts to bind mounts from /run/virtiofs/{tag} - let containerMounts = createdState.vm.mounts[self.id] ?? [] + let containerMounts = createdState.vm.storage.containers[self.id]?.mounts ?? [] let holdingTags = createdState.fileMountContext.holdingDirectoryTags - // Drop rootfs, and writable layer if present. - let mountsToSkip = self.writableLayer != nil ? 2 : 1 var mounts: [ContainerizationOCI.Mount] = - containerMounts.dropFirst(mountsToSkip) + containerMounts .filter { !holdingTags.contains($0.source) } .map { attached -> ContainerizationOCI.Mount in if attached.type == "virtiofs" { diff --git a/Sources/Containerization/LinuxPod.swift b/Sources/Containerization/LinuxPod.swift index 6275a4d4..fa4d1785 100644 --- a/Sources/Containerization/LinuxPod.swift +++ b/Sources/Containerization/LinuxPod.swift @@ -446,24 +446,28 @@ extension LinuxPod { // the container's bind mounts from /run/virtiofs/ fail // with ENOENT. // - // Derive the tags from the additional mounts directly rather - // than from vm.mounts[id], so this is independent of the - // rootfs (which may be virtiofs or virtio-blk) and of mount - // ordering. The rootfs is mounted at /run/container//rootfs - // and is never consumed from /run/virtiofs. + // Derive the new tags from the additional mounts being added; + // the machine's storage names what is already shared. let newVirtiofsTags = try virtioFSMounts.map { try hashFilePath(path: $0.source) } if !newVirtiofsTags.isEmpty { try await agent.mkdir(path: "/run/virtiofs", all: true, perms: 0o755) if vm.virtiofsLayout == .perTag { // Tags already mounted in the guest at boot or by a - // prior hotplug (i.e. present on another container). - let alreadyMounted = Set( - vm.mounts - .filter { $0.key != id } - .values.flatMap { $0 } - .filter { $0.type == "virtiofs" } - .map { $0.source } - ) + // prior hotplug (i.e. present on another container or + // a machine volume). + let alreadyMounted: Set = { + var mounted = Set( + vm.storage.containers + .filter { $0.key != id } + .values.flatMap { $0.all } + .filter { $0.type == "virtiofs" } + .map { $0.source }) + mounted.formUnion( + vm.storage.volumes.values + .filter { $0.type == "virtiofs" } + .map { $0.source }) + return mounted + }() var seen: Set = [] for tag in newVirtiofsTags where !alreadyMounted.contains(tag) && seen.insert(tag).inserted { @@ -493,7 +497,7 @@ extension LinuxPod { } if fileMountContext.hasFileMounts { - let containerMounts = vm.mounts[id] ?? [] + let containerMounts = vm.storage.containers[id]?.mounts ?? [] try await updatedFileMountContext.mountHoldingDirectories( vmMounts: containerMounts, agent: agent @@ -557,20 +561,23 @@ extension LinuxPod { try await self.state.withLock { state in try state.phase.validateForCreate() - // Build mountsByID for all containers. + // Build the machine's storage from its containers. // Strip "ro" from rootfs options - we handle readonly via the OCI spec's // root.readonly field and remount in vmexec after setup is complete. // Use transformedMounts from fileMountContext (file mounts become directory shares). - var mountsByID: [String: [Mount]] = [:] + var machineStorage = MachineMounts() for (id, container) in state.containers { var modifiedRootfs = container.rootfs modifiedRootfs.options.removeAll(where: { $0 == "ro" }) - // Filter out shared mounts — those are handled separately as pod volume bind mounts. + // Shared mounts are handled separately as pod volume bind mounts. let containerMounts = container.fileMountContext.transformedMounts.filter { if case .shared = $0.runtimeOptions { return false } return true } - mountsByID[id] = [modifiedRootfs] + containerMounts + machineStorage.containers[id] = ContainerMounts( + rootfs: modifiedRootfs, + mounts: containerMounts + ) } // Validate pod volume names are unique. @@ -597,9 +604,8 @@ extension LinuxPod { } } } - let podVolumeMounts = self.config.volumes.map { $0.toMount() } - if !podVolumeMounts.isEmpty { - mountsByID[self.id] = podVolumeMounts + for volume in self.config.volumes { + machineStorage.volumes[volume.name] = volume.toMount() } // Capture into an immutable `let` so the value is safely usable @@ -608,18 +614,16 @@ extension LinuxPod { // only attaches a virtiofs device when shares are configured, // so mounting an unbacked /run/virtiofs would fail with EINVAL // on the CH backend. - let hasVirtiofsMount = mountsByID.values.contains { mounts in - mounts.contains { mount in - if case .virtiofs = mount.runtimeOptions { return true } - return false - } + let hasVirtiofsMount = machineStorage.ordered.contains { mount in + if case .virtiofs = mount.runtimeOptions { return true } + return false } var vmConfig = VMConfiguration( cpus: self.config.cpus, memoryInBytes: self.config.memoryInBytes, interfaces: self.config.interfaces, - mountsByID: mountsByID, + storage: machineStorage, bootLog: self.config.bootLog, nestedVirtualization: self.config.virtualization ) @@ -649,19 +653,17 @@ extension LinuxPod { // /run/virtiofs/. See LinuxContainer for the // VZ vs. CH model split. var seenTags: Set = [] - for (_, attached) in vm.mounts { - for entry in attached where entry.type == "virtiofs" { - guard seenTags.insert(entry.source).inserted else { continue } - let dest = "/run/virtiofs/\(entry.source)" - try await agent.mkdir(path: dest, all: true, perms: 0o755) - try await agent.mount( - ContainerizationOCI.Mount( - type: "virtiofs", - source: entry.source, - destination: dest, - options: [] - )) - } + for entry in vm.storage.ordered where entry.type == "virtiofs" { + guard seenTags.insert(entry.source).inserted else { continue } + let dest = "/run/virtiofs/\(entry.source)" + try await agent.mkdir(path: dest, all: true, perms: 0o755) + try await agent.mount( + ContainerizationOCI.Mount( + type: "virtiofs", + source: entry.source, + destination: dest, + options: [] + )) } } else { try await agent.mount( @@ -728,10 +730,10 @@ extension LinuxPod { // Mount all container rootfs for (_, container) in containers { - guard let attachments = vm.mounts[container.id], let rootfsAttachment = attachments.first else { + guard let attached = vm.storage.containers[container.id] else { throw ContainerizationError(.notFound, message: "rootfs mount not found for container \(container.id)") } - var rootfs = rootfsAttachment.to + var rootfs = attached.rootfs.to rootfs.destination = Self.guestRootfsPath(container.id) try await agent.mount(rootfs) } @@ -740,7 +742,7 @@ extension LinuxPod { for (id, container) in containers { if container.fileMountContext.hasFileMounts { var ctx = container.fileMountContext - let containerMounts = vm.mounts[id] ?? [] + let containerMounts = vm.storage.containers[id]?.mounts ?? [] try await ctx.mountHoldingDirectories( vmMounts: containerMounts, agent: agent @@ -750,15 +752,13 @@ extension LinuxPod { } // Mount pod-level volumes. - let podVolumeAttachments = vm.mounts[self.id] ?? [] - for (index, volume) in self.config.volumes.enumerated() { - guard index < podVolumeAttachments.count else { + for volume in self.config.volumes { + guard let attachment = vm.storage.volumes[volume.name] else { throw ContainerizationError( .notFound, message: "attached filesystem not found for pod volume \"\(volume.name)\"" ) } - let attachment = podVolumeAttachments[index] let guestPath = Self.guestVolumePath(volume.name) try await agent.mount( ContainerizationOCI.Mount( @@ -864,10 +864,10 @@ extension LinuxPod { // We don't need the rootfs, nor do OCI runtimes want it included. // Also filter out file mount holding directories - we mount those separately under /run. // Transform virtiofs mounts to bind mounts from /run/virtiofs/{tag} - let containerMounts = createdState.vm.mounts[containerID] ?? [] + let containerMounts = createdState.vm.storage.containers[containerID]?.mounts ?? [] let holdingTags = container.fileMountContext.holdingDirectoryTags var mounts: [ContainerizationOCI.Mount] = - containerMounts.dropFirst() + containerMounts .filter { !holdingTags.contains($0.source) } .map { attached -> ContainerizationOCI.Mount in if attached.type == "virtiofs" { diff --git a/Sources/Containerization/VMConfiguration.swift b/Sources/Containerization/VMConfiguration.swift index 30faebc4..7a5e8e2c 100644 --- a/Sources/Containerization/VMConfiguration.swift +++ b/Sources/Containerization/VMConfiguration.swift @@ -72,9 +72,9 @@ public struct VMConfiguration: Sendable { public var memoryInBytes: UInt64 /// The network interfaces to attach. public var interfaces: [any Interface] - /// Mounts organized by metadata ID (e.g. container ID). - /// Each ID maps to an array of mounts for that workload. - public var mountsByID: [String: [Mount]] + /// The storage the machine carries: each container's mounts by role, + /// and the volumes its containers share. + public var storage: MachineMounts /// Optional destination for serial boot logs. public var bootLog: BootLog? /// Enable nested virtualization support. If the VirtualMachineManager @@ -89,14 +89,14 @@ public struct VMConfiguration: Sendable { cpus: Int = 4, memoryInBytes: UInt64 = 1024 * 1024 * 1024, interfaces: [any Interface] = [], - mountsByID: [String: [Mount]] = [:], + storage: MachineMounts = MachineMounts(), bootLog: BootLog? = nil, nestedVirtualization: Bool = false ) { self.cpus = cpus self.memoryInBytes = memoryInBytes self.interfaces = interfaces - self.mountsByID = mountsByID + self.storage = storage self.bootLog = bootLog self.nestedVirtualization = nestedVirtualization } diff --git a/Sources/Containerization/VZVirtualMachineInstance.swift b/Sources/Containerization/VZVirtualMachineInstance.swift index 160c5026..670ab512 100644 --- a/Sources/Containerization/VZVirtualMachineInstance.swift +++ b/Sources/Containerization/VZVirtualMachineInstance.swift @@ -28,10 +28,10 @@ import Synchronization public final class VZVirtualMachineInstance: Sendable { public typealias Agent = Vminitd - /// Attached mounts on the virtual machine, organized by metadata ID. - private let _mounts: Mutex<[String: [AttachedFilesystem]]> - public var mounts: [String: [AttachedFilesystem]] { - _mounts.withLock { $0 } + /// The machine's attached storage. + private let _storage: Mutex + public var storage: MachineAttachments { + _storage.withLock { $0 } } /// The underlying Virtualization framework virtual machine. @@ -40,9 +40,9 @@ public final class VZVirtualMachineInstance: Sendable { /// The dispatch queue used for VZ operations. public var vmQueue: DispatchQueue { queue } - /// Mutate the mount registry. - public func withMountRegistry(_ body: (inout sending [String: [AttachedFilesystem]]) throws -> sending T) rethrows -> T { - try _mounts.withLock(body) + /// Mutate the storage registry. + public func withStorage(_ body: (inout sending MachineAttachments) throws -> sending T) rethrows -> T { + try _storage.withLock(body) } /// Serialize VM operations with the instance lock. @@ -73,8 +73,9 @@ public final class VZVirtualMachineInstance: Sendable { public var rosetta: Bool /// Toggle nested virtualization support. public var nestedVirtualization: Bool - /// Mount attachments organized by metadata ID. - public var mountsByID: [String: [Mount]] + /// The machine's storage: each container's mounts by role, and the + /// volumes its containers share. + public var storage: MachineMounts /// Network interface attachments. public var interfaces: [any Interface] /// Kernel image. @@ -91,7 +92,7 @@ public final class VZVirtualMachineInstance: Sendable { self.memoryInBytes = 1024.mib() self.rosetta = false self.nestedVirtualization = false - self.mountsByID = [:] + self.storage = MachineMounts() self.interfaces = [] } } @@ -132,7 +133,7 @@ public final class VZVirtualMachineInstance: Sendable { let allocator = Character.blockDeviceTagAllocator() let (mountAttachments, _) = try config.mountAttachments(allocator: allocator) - self._mounts = Mutex(mountAttachments) + self._storage = Mutex(mountAttachments) self.vm = VZVirtualMachine( configuration: try config.toVZ(allocator: allocator), @@ -153,7 +154,7 @@ public protocol VZInstanceExtension: Sendable { _ config: inout VZVirtualMachineConfiguration, allocator: any AddressAllocator, storageDeviceCount: Int, - mountsByID: [String: [Mount]] + storage: MachineMounts ) throws /// Called after the VZVirtualMachine is created but before start. @@ -168,7 +169,7 @@ extension VZInstanceExtension { _ config: inout VZVirtualMachineConfiguration, allocator: any AddressAllocator, storageDeviceCount: Int, - mountsByID: [String: [Mount]] + storage: MachineMounts ) throws {} public func didCreate(_ instance: VZVirtualMachineInstance) throws {} @@ -477,35 +478,33 @@ extension VZVirtualMachineInstance.Configuration { // Track used virtiofs tags to avoid creating duplicate VZ devices. // The same source directory mounted to multiple destinations shares one device. + // The walk is the machine's device order, matching the addresses + // `mountAttachments` hands out walking the same way. var usedVirtioFSTags: Set = [] - for (_, mounts) in self.mountsByID { - for mount in mounts { - if case .virtiofs = mount.runtimeOptions { - let tag = try hashFilePath(path: mount.source) - if usedVirtioFSTags.contains(tag) { - continue - } - usedVirtioFSTags.insert(tag) + for mount in self.storage.ordered { + if case .virtiofs = mount.runtimeOptions { + let tag = try hashFilePath(path: mount.source) + if usedVirtioFSTags.contains(tag) { + continue } - try mount.configure(config: &config) + usedVirtioFSTags.insert(tag) } + try mount.configure(config: &config) } // Create the unified virtiofs device with VZMultipleDirectoryShare // This device hosts all virtiofs shares and supports runtime updates var directories: [String: VZSharedDirectory] = [:] - for (_, mounts) in self.mountsByID { - for mount in mounts { - guard case .virtiofs(_) = mount.runtimeOptions else { continue } - guard FileManager.default.fileExists(atPath: mount.source) else { - throw ContainerizationError(.notFound, message: "directory \(mount.source) does not exist") - } - let name = try hashFilePath(path: mount.source) - directories[name] = VZSharedDirectory( - url: URL(fileURLWithPath: mount.source), - readOnly: mount.options.contains("ro") - ) + for mount in self.storage.ordered { + guard case .virtiofs(_) = mount.runtimeOptions else { continue } + guard FileManager.default.fileExists(atPath: mount.source) else { + throw ContainerizationError(.notFound, message: "directory \(mount.source) does not exist") } + let name = try hashFilePath(path: mount.source) + directories[name] = VZSharedDirectory( + url: URL(fileURLWithPath: mount.source), + readOnly: mount.options.contains("ro") + ) } let multiShare = VZMultipleDirectoryShare(directories: directories) let virtiofsDevice = VZVirtioFileSystemDeviceConfiguration(tag: "virtiofs") @@ -527,7 +526,7 @@ extension VZVirtualMachineInstance.Configuration { config.platform = platform for ext in self.extensions.compactMap({ $0 as? any VZInstanceExtension }) { - try ext.configureVZ(&config, allocator: allocator, storageDeviceCount: storageDeviceCount, mountsByID: self.mountsByID) + try ext.configureVZ(&config, allocator: allocator, storageDeviceCount: storageDeviceCount, storage: self.storage) } try config.validate() @@ -535,7 +534,7 @@ extension VZVirtualMachineInstance.Configuration { } func mountAttachments(allocator: any AddressAllocator) throws -> ( - attachments: [String: [AttachedFilesystem]], storageDeviceCount: Int + attachments: MachineAttachments, storageDeviceCount: Int ) { var storageDeviceCount = 0 @@ -548,21 +547,28 @@ extension VZVirtualMachineInstance.Configuration { } } - var attachmentsByID: [String: [AttachedFilesystem]] = [:] - - for (id, mounts) in self.mountsByID { - var attachments: [AttachedFilesystem] = [] - for mount in mounts { - let attached = try AttachedFilesystem(mount: mount, allocator: allocator) - attachments.append(attached) - if mount.isBlock { - storageDeviceCount += 1 - } + // The machine's device order: addresses are handed out in the same + // walk `makeConfiguration` creates the devices in. + func attach(_ mount: Mount) throws -> AttachedFilesystem { + let attached = try AttachedFilesystem(mount: mount, allocator: allocator) + if mount.isBlock { + storageDeviceCount += 1 } - attachmentsByID[id] = attachments + return attached + } + + var containers: [String: ContainerAttachments] = [:] + for id in self.storage.containers.keys.sorted() { + guard let container = self.storage.containers[id] else { continue } + containers[id] = try container.map(attach) + } + var volumes: [String: AttachedFilesystem] = [:] + for name in self.storage.volumes.keys.sorted() { + guard let mount = self.storage.volumes[name] else { continue } + volumes[name] = try attach(mount) } - return (attachmentsByID, storageDeviceCount) + return (MachineAttachments(containers: containers, volumes: volumes), storageDeviceCount) } } diff --git a/Sources/Containerization/VZVirtualMachineManager.swift b/Sources/Containerization/VZVirtualMachineManager.swift index 4959bee4..d0888447 100644 --- a/Sources/Containerization/VZVirtualMachineManager.swift +++ b/Sources/Containerization/VZVirtualMachineManager.swift @@ -76,7 +76,7 @@ public struct VZVirtualMachineManager: VirtualMachineManager { instanceConfig.rosetta = self.rosetta instanceConfig.nestedVirtualization = useNestedVirtualization - instanceConfig.mountsByID = vmConfig.mountsByID + instanceConfig.storage = vmConfig.storage instanceConfig.extensions = vmConfig.extensions }) } diff --git a/Sources/Containerization/VirtualMachineInstance.swift b/Sources/Containerization/VirtualMachineInstance.swift index 302e97ae..b9340612 100644 --- a/Sources/Containerization/VirtualMachineInstance.swift +++ b/Sources/Containerization/VirtualMachineInstance.swift @@ -44,7 +44,8 @@ public protocol VirtualMachineInstance: Sendable { // The state of the virtual machine. var state: VirtualMachineInstanceState { get } - var mounts: [String: [AttachedFilesystem]] { get } + /// The machine's attached storage. + var storage: MachineAttachments { get } /// How this VMM exposes virtiofs devices to the guest. Defaults to /// `.unified` (the VZ-shaped behavior); CH overrides to `.perTag`. @@ -73,7 +74,7 @@ public protocol VirtualMachineInstance: Sendable { func hotplug(_ block: Mount, id: String) async throws -> AttachedFilesystem /// Register mounts for a container after hotplug. - /// This is used to add the rootfs and additional mounts to the VM's mount registry + /// This is used to add the rootfs and additional mounts to the machine's storage /// so they can be found when building the container's OCI spec. /// - Parameter id: The container ID /// - Parameter rootfs: The rootfs attachment from hotplug From 2f6797a5cf27a9f481805b898a8797f54154526e Mon Sep 17 00:00:00 2001 From: Aaron Paterson Date: Fri, 7 Aug 2026 17:31:13 +0000 Subject: [PATCH 06/10] Let a pod's containers copy files in and out Copying a file between the host and a container is reachable only for a container in a machine of its own; the same container placed in a pod has no way to be copied into or out of, so a tool that offers `cp` loses it the moment a container joins a pod. What the transfer needs is the machine, the path the container's filesystem sits at in the guest, and a vsock port, and only the path differs between the two cases. It moves to a type that takes those, which both a standalone container and a pod's containers hand it. The round trip is covered against a running pod, including that a file copied into one container reaches that container alone. --- .../Containerization/GuestFileTransfer.swift | 303 ++++++++++++++++++ Sources/Containerization/LinuxContainer.swift | 264 ++------------- Sources/Containerization/LinuxPod.swift | 79 +++++ Sources/Integration/PodTests.swift | 84 +++++ Sources/Integration/Suite.swift | 1 + 5 files changed, 492 insertions(+), 239 deletions(-) create mode 100644 Sources/Containerization/GuestFileTransfer.swift diff --git a/Sources/Containerization/GuestFileTransfer.swift b/Sources/Containerization/GuestFileTransfer.swift new file mode 100644 index 00000000..d508c571 --- /dev/null +++ b/Sources/Containerization/GuestFileTransfer.swift @@ -0,0 +1,303 @@ +//===----------------------------------------------------------------------===// +// Copyright © 2026 Apple Inc. and the container project authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//===----------------------------------------------------------------------===// + +import ContainerizationArchive +import ContainerizationError +import ContainerizationOS +import Foundation + +/// Moves files between the host and a container's filesystem inside a guest. +/// +/// The transfer runs over a dedicated vsock connection on a port the caller +/// allocates. A container in a machine of its own and a container among +/// several in a pod differ only in where their filesystem sits in the guest, +/// so that path is what this is given. +struct GuestFileTransfer: Sendable { + /// Default chunk size for file transfers (1MiB). + static let defaultChunkSize = 1024 * 1024 + + /// The machine holding the container's filesystem. + let vm: any VirtualMachineInstance + /// Where the container's filesystem sits in the guest. + let guestRoot: String + /// The vsock port the data travels over. + let port: UInt32 + /// Where the blocking read and write work runs. + let queue: DispatchQueue + + /// Copy a file or directory from the host into the container. + /// + /// For directories, the source is archived as tar+gzip and streamed + /// directly through vsock without intermediate temp files. + func copyIn( + from source: URL, + to destination: URL, + mode: UInt32, + createParents: Bool, + chunkSize: Int + ) async throws { + var isDirectory: ObjCBool = false + guard FileManager.default.fileExists(atPath: source.path, isDirectory: &isDirectory) else { + throw ContainerizationError(.notFound, message: "copyIn: source not found '\(source.path)'") + } + let isArchive = isDirectory.boolValue + + let guestPath: URL = try await vm.withAgent { agent in + guard let vminitd = agent as? Vminitd else { + throw ContainerizationError(.unsupported, message: "copyIn requires Vminitd agent") + } + + return try await self.resolveCopyInGuestPath( + from: source, + to: destination, + sourceIsDirectory: isArchive, + using: vminitd + ) + } + + let listener = try vm.listen(port) + + try await withThrowingTaskGroup(of: Void.self) { group in + group.addTask { + try await self.vm.withAgent { agent in + guard let vminitd = agent as? Vminitd else { + throw ContainerizationError(.unsupported, message: "copyIn requires Vminitd agent") + } + try await vminitd.copy( + direction: .copyIn, + guestPath: guestPath, + vsockPort: self.port, + mode: mode, + createParents: createParents, + isArchive: isArchive + ) + } + } + + group.addTask { + guard let conn = await listener.first(where: { _ in true }) else { + throw ContainerizationError(.internalError, message: "copyIn: vsock connection not established") + } + try listener.finish() + + try await withCheckedThrowingContinuation { (continuation: CheckedContinuation) in + self.queue.async { + do { + defer { conn.closeFile() } + + if isArchive { + let writer = try ArchiveWriter(configuration: .init(format: .pax, filter: .gzip)) + try writer.open(fileDescriptor: conn.fileDescriptor) + try writer.archiveDirectory(source) + try writer.finishEncoding() + } else { + let srcFd = open(source.path, O_RDONLY) + guard srcFd != -1 else { + throw ContainerizationError( + .internalError, + message: "copyIn: failed to open '\(source.path)': \(String(cString: strerror(errno)))" + ) + } + defer { close(srcFd) } + + var buf = [UInt8](repeating: 0, count: chunkSize) + while true { + let n = read(srcFd, &buf, buf.count) + if n == 0 { break } + guard n > 0 else { + throw ContainerizationError( + .internalError, + message: "copyIn: read error: \(String(cString: strerror(errno)))" + ) + } + var written = 0 + while written < n { + let w = buf.withUnsafeBytes { ptr in + write(conn.fileDescriptor, ptr.baseAddress! + written, n - written) + } + guard w > 0 else { + throw ContainerizationError( + .internalError, + message: "copyIn: vsock write error: \(String(cString: strerror(errno)))" + ) + } + written += w + } + } + } + continuation.resume() + } catch { + continuation.resume(throwing: error) + } + } + } + } + + try await group.waitForAll() + } + } + + /// Copy a file or directory from the container to the host. + /// + /// For directories, the guest archives the source as tar+gzip and streams + /// it directly through vsock. The host extracts the archive without + /// intermediate temp files. + func copyOut( + from source: URL, + to destination: URL, + createParents: Bool, + chunkSize: Int + ) async throws { + if createParents { + let parentDir = destination.deletingLastPathComponent() + try FileManager.default.createDirectory(at: parentDir, withIntermediateDirectories: true) + } + + let guestPath = URL(filePath: guestRoot).appending(path: source.path) + let listener = try vm.listen(port) + + let (metadataStream, metadataCont) = AsyncStream.makeStream(of: Vminitd.CopyMetadata.self) + + try await withThrowingTaskGroup(of: Void.self) { group in + group.addTask { + defer { metadataCont.finish() } + try await self.vm.withAgent { agent in + guard let vminitd = agent as? Vminitd else { + throw ContainerizationError(.unsupported, message: "copyOut requires Vminitd agent") + } + try await vminitd.copy( + direction: .copyOut, + guestPath: guestPath, + vsockPort: self.port, + onMetadata: { meta in + metadataCont.yield(meta) + metadataCont.finish() + } + ) + } + } + + group.addTask { + guard let metadata = await metadataStream.first(where: { _ in true }) else { + throw ContainerizationError(.internalError, message: "copyOut: no metadata received") + } + + guard let conn = await listener.first(where: { _ in true }) else { + throw ContainerizationError(.internalError, message: "copyOut: vsock connection not established") + } + try listener.finish() + + try await withCheckedThrowingContinuation { (continuation: CheckedContinuation) in + self.queue.async { + do { + defer { conn.closeFile() } + + if metadata.isArchive { + try FileManager.default.createDirectory(at: destination, withIntermediateDirectories: true) + let fh = FileHandle(fileDescriptor: dup(conn.fileDescriptor), closeOnDealloc: true) + let reader = try ArchiveReader(format: .pax, filter: .gzip, fileHandle: fh) + _ = try reader.extractContents(to: destination) + } else { + let destFd = open(destination.path, O_WRONLY | O_CREAT | O_TRUNC, 0o644) + guard destFd != -1 else { + throw ContainerizationError( + .internalError, + message: "copyOut: failed to open '\(destination.path)': \(String(cString: strerror(errno)))" + ) + } + defer { close(destFd) } + + var buf = [UInt8](repeating: 0, count: chunkSize) + while true { + let n = read(conn.fileDescriptor, &buf, buf.count) + if n == 0 { break } + guard n > 0 else { + throw ContainerizationError( + .internalError, + message: "copyOut: vsock read error: \(String(cString: strerror(errno)))" + ) + } + var written = 0 + while written < n { + let w = buf.withUnsafeBytes { ptr in + write(destFd, ptr.baseAddress! + written, n - written) + } + guard w > 0 else { + throw ContainerizationError( + .internalError, + message: "copyOut: write error: \(String(cString: strerror(errno)))" + ) + } + written += w + } + } + } + continuation.resume() + } catch { + continuation.resume(throwing: error) + } + } + } + } + + try await group.waitForAll() + } + } + + /// Where a copy lands in the guest, given what the destination already is. + /// + /// A destination that names an existing directory receives the source + /// under its own name, the way `cp` behaves. + private func resolveCopyInGuestPath( + from source: URL, + to destination: URL, + sourceIsDirectory: Bool, + using vminitd: Vminitd + ) async throws -> URL { + let guestDestination = URL(filePath: guestRoot).appending(path: destination.path) + + let stat: ContainerizationOS.Stat? + do { + stat = try await vminitd.stat(path: guestDestination) + } catch let error as ContainerizationError where error.code == .notFound { + stat = nil + } + // Any other error propagates so transport and permission failures are visible. + + guard let stat else { + if destination.hasDirectoryPath && !sourceIsDirectory { + throw ContainerizationError( + .invalidArgument, + message: "destination directory does not exist: \(destination.path)" + ) + } + return guestDestination + } + + let destinationIsDirectory = (stat.mode & UInt32(S_IFMT)) == UInt32(S_IFDIR) + guard destinationIsDirectory else { + if sourceIsDirectory { + throw ContainerizationError( + .invalidArgument, + message: "cannot copy directory over existing file: \(destination.path)" + ) + } + return guestDestination + } + + return guestDestination.appendingPathComponent(source.lastPathComponent) + } +} diff --git a/Sources/Containerization/LinuxContainer.swift b/Sources/Containerization/LinuxContainer.swift index 55bbb39a..8aaedb10 100644 --- a/Sources/Containerization/LinuxContainer.swift +++ b/Sources/Containerization/LinuxContainer.swift @@ -632,8 +632,8 @@ extension LinuxContainer { let vm = try await self.vmm.create(config: creationConfig) let relayManager = UnixSocketRelayManager(vm: vm, log: self.logger) + try await vm.start() do { - try await vm.start() let storageForAgent = containerStorage try await vm.withAgent { agent in try await agent.standardSetup() @@ -1156,7 +1156,7 @@ extension LinuxContainer { } /// Default chunk size for file transfers (1MiB). - public static let defaultCopyChunkSize = 1024 * 1024 + public static let defaultCopyChunkSize = GuestFileTransfer.defaultChunkSize /// Copy a file or directory from the host into the container. /// @@ -1172,148 +1172,14 @@ extension LinuxContainer { ) async throws { try await self.state.withLock { let state = try $0.startedState("copyIn") - - var isDirectory: ObjCBool = false - guard FileManager.default.fileExists(atPath: source.path, isDirectory: &isDirectory) else { - throw ContainerizationError(.notFound, message: "copyIn: source not found '\(source.path)'") - } - let isArchive = isDirectory.boolValue - - let guestPath: URL = try await state.vm.withAgent { agent in - guard let vminitd = agent as? Vminitd else { - throw ContainerizationError(.unsupported, message: "copyIn requires Vminitd agent") - } - - return try await self.resolveCopyInGuestPath( - from: source, - to: destination, - sourceIsDirectory: isArchive, - using: vminitd - ) - } - - let port = self.hostVsockPorts.wrappingAdd(1, ordering: .relaxed).oldValue - let listener = try state.vm.listen(port) - - try await withThrowingTaskGroup(of: Void.self) { group in - group.addTask { - try await state.vm.withAgent { agent in - guard let vminitd = agent as? Vminitd else { - throw ContainerizationError(.unsupported, message: "copyIn requires Vminitd agent") - } - try await vminitd.copy( - direction: .copyIn, - guestPath: guestPath, - vsockPort: port, - mode: mode, - createParents: createParents, - isArchive: isArchive - ) - } - } - - group.addTask { - guard let conn = await listener.first(where: { _ in true }) else { - throw ContainerizationError(.internalError, message: "copyIn: vsock connection not established") - } - try listener.finish() - - try await withCheckedThrowingContinuation { (continuation: CheckedContinuation) in - self.copyQueue.async { - do { - defer { conn.closeFile() } - - if isArchive { - let writer = try ArchiveWriter(configuration: .init(format: .pax, filter: .gzip)) - try writer.open(fileDescriptor: conn.fileDescriptor) - try writer.archiveDirectory(source) - try writer.finishEncoding() - } else { - let srcFd = open(source.path, O_RDONLY) - guard srcFd != -1 else { - throw ContainerizationError( - .internalError, - message: "copyIn: failed to open '\(source.path)': \(String(cString: strerror(errno)))" - ) - } - defer { close(srcFd) } - - var buf = [UInt8](repeating: 0, count: chunkSize) - while true { - let n = read(srcFd, &buf, buf.count) - if n == 0 { break } - guard n > 0 else { - throw ContainerizationError( - .internalError, - message: "copyIn: read error: \(String(cString: strerror(errno)))" - ) - } - var written = 0 - while written < n { - let w = buf.withUnsafeBytes { ptr in - write(conn.fileDescriptor, ptr.baseAddress! + written, n - written) - } - guard w > 0 else { - throw ContainerizationError( - .internalError, - message: "copyIn: vsock write error: \(String(cString: strerror(errno)))" - ) - } - written += w - } - } - } - continuation.resume() - } catch { - continuation.resume(throwing: error) - } - } - } - } - - try await group.waitForAll() - } - } - } - - private func resolveCopyInGuestPath( - from source: URL, - to destination: URL, - sourceIsDirectory: Bool, - using vminitd: Vminitd - ) async throws -> URL { - let guestDestination = URL(filePath: self.root).appending(path: destination.path) - - let stat: ContainerizationOS.Stat? - do { - stat = try await vminitd.stat(path: guestDestination) - } catch let error as ContainerizationError where error.code == .notFound { - stat = nil - } - // Any other error propagates so transport and permission failures are visible. - - guard let stat else { - if destination.hasDirectoryPath && !sourceIsDirectory { - throw ContainerizationError( - .invalidArgument, - message: "destination directory does not exist: \(destination.path)" - ) - } - return guestDestination - } - - let destinationIsDirectory = (stat.mode & UInt32(S_IFMT)) == UInt32(S_IFDIR) - guard destinationIsDirectory else { - if sourceIsDirectory { - throw ContainerizationError( - .invalidArgument, - message: "cannot copy directory over existing file: \(destination.path)" - ) - } - return guestDestination + try await self.transfer(vm: state.vm).copyIn( + from: source, + to: destination, + mode: mode, + createParents: createParents, + chunkSize: chunkSize + ) } - - return guestDestination.appendingPathComponent(source.lastPathComponent) } /// Copy a file or directory from the container to the host. @@ -1329,104 +1195,24 @@ extension LinuxContainer { ) async throws { try await self.state.withLock { let state = try $0.startedState("copyOut") - - if createParents { - let parentDir = destination.deletingLastPathComponent() - try FileManager.default.createDirectory(at: parentDir, withIntermediateDirectories: true) - } - - let guestPath = URL(filePath: self.root).appending(path: source.path) - let port = self.hostVsockPorts.wrappingAdd(1, ordering: .relaxed).oldValue - let listener = try state.vm.listen(port) - - let (metadataStream, metadataCont) = AsyncStream.makeStream(of: Vminitd.CopyMetadata.self) - - try await withThrowingTaskGroup(of: Void.self) { group in - group.addTask { - defer { metadataCont.finish() } - try await state.vm.withAgent { agent in - guard let vminitd = agent as? Vminitd else { - throw ContainerizationError(.unsupported, message: "copyOut requires Vminitd agent") - } - try await vminitd.copy( - direction: .copyOut, - guestPath: guestPath, - vsockPort: port, - onMetadata: { meta in - metadataCont.yield(meta) - metadataCont.finish() - } - ) - } - } - - group.addTask { - guard let metadata = await metadataStream.first(where: { _ in true }) else { - throw ContainerizationError(.internalError, message: "copyOut: no metadata received") - } - - guard let conn = await listener.first(where: { _ in true }) else { - throw ContainerizationError(.internalError, message: "copyOut: vsock connection not established") - } - try listener.finish() - - try await withCheckedThrowingContinuation { (continuation: CheckedContinuation) in - self.copyQueue.async { - do { - defer { conn.closeFile() } - - if metadata.isArchive { - try FileManager.default.createDirectory(at: destination, withIntermediateDirectories: true) - let fh = FileHandle(fileDescriptor: dup(conn.fileDescriptor), closeOnDealloc: true) - let reader = try ArchiveReader(format: .pax, filter: .gzip, fileHandle: fh) - _ = try reader.extractContents(to: destination) - } else { - let destFd = open(destination.path, O_WRONLY | O_CREAT | O_TRUNC, 0o644) - guard destFd != -1 else { - throw ContainerizationError( - .internalError, - message: "copyOut: failed to open '\(destination.path)': \(String(cString: strerror(errno)))" - ) - } - defer { close(destFd) } - - var buf = [UInt8](repeating: 0, count: chunkSize) - while true { - let n = read(conn.fileDescriptor, &buf, buf.count) - if n == 0 { break } - guard n > 0 else { - throw ContainerizationError( - .internalError, - message: "copyOut: vsock read error: \(String(cString: strerror(errno)))" - ) - } - var written = 0 - while written < n { - let w = buf.withUnsafeBytes { ptr in - write(destFd, ptr.baseAddress! + written, n - written) - } - guard w > 0 else { - throw ContainerizationError( - .internalError, - message: "copyOut: write error: \(String(cString: strerror(errno)))" - ) - } - written += w - } - } - } - continuation.resume() - } catch { - continuation.resume(throwing: error) - } - } - } - } - - try await group.waitForAll() - } + try await self.transfer(vm: state.vm).copyOut( + from: source, + to: destination, + createParents: createParents, + chunkSize: chunkSize + ) } } + + /// A transfer against this container's filesystem, on a port of its own. + private func transfer(vm: any VirtualMachineInstance) -> GuestFileTransfer { + GuestFileTransfer( + vm: vm, + guestRoot: self.root, + port: self.hostVsockPorts.wrappingAdd(1, ordering: .relaxed).oldValue, + queue: self.copyQueue + ) + } } extension VirtualMachineInstance { diff --git a/Sources/Containerization/LinuxPod.swift b/Sources/Containerization/LinuxPod.swift index fa4d1785..457a7ebe 100644 --- a/Sources/Containerization/LinuxPod.swift +++ b/Sources/Containerization/LinuxPod.swift @@ -191,6 +191,9 @@ public final class LinuxPod: Sendable { // the host. private let guestVsockPorts: Atomic + // Where the blocking reads and writes of a file transfer run. + private let copyQueue = DispatchQueue(label: "com.apple.containerization.copy") + private struct State: Sendable { var phase: Phase var containers: [String: PodContainer] @@ -1291,6 +1294,82 @@ extension LinuxPod { } } + /// Default chunk size for file transfers (1MiB). + public static let defaultCopyChunkSize = GuestFileTransfer.defaultChunkSize + + /// Copy a file or directory from the host into a container in the pod. + /// + /// Data transfer happens over a dedicated vsock connection. For + /// directories, the source is archived as tar+gzip and streamed directly + /// through vsock without intermediate temp files. + public func copyIn( + _ containerID: String, + from source: URL, + to destination: URL, + mode: UInt32 = 0o644, + createParents: Bool = true, + chunkSize: Int = defaultCopyChunkSize + ) async throws { + try await self.state.withLock { state in + try await self.transfer(containerID, state: state, operation: "copyIn").copyIn( + from: source, + to: destination, + mode: mode, + createParents: createParents, + chunkSize: chunkSize + ) + } + } + + /// Copy a file or directory from a container in the pod to the host. + /// + /// Data transfer happens over a dedicated vsock connection. For + /// directories, the guest archives the source as tar+gzip and streams it + /// directly through vsock. The host extracts the archive without + /// intermediate temp files. + public func copyOut( + _ containerID: String, + from source: URL, + to destination: URL, + createParents: Bool = true, + chunkSize: Int = defaultCopyChunkSize + ) async throws { + try await self.state.withLock { state in + try await self.transfer(containerID, state: state, operation: "copyOut").copyOut( + from: source, + to: destination, + createParents: createParents, + chunkSize: chunkSize + ) + } + } + + /// A transfer against one container's filesystem, on a port of its own. + private func transfer(_ containerID: String, state: State, operation: String) throws -> GuestFileTransfer { + let createdState = try state.phase.createdState(operation) + + guard let container = state.containers[containerID] else { + throw ContainerizationError( + .notFound, + message: "container \(containerID) not found in pod" + ) + } + + guard container.state == .started else { + throw ContainerizationError( + .invalidState, + message: "container \(containerID) must be started to copy files" + ) + } + + return GuestFileTransfer( + vm: createdState.vm, + guestRoot: Self.guestRootfsPath(containerID), + port: self.hostVsockPorts.wrappingAdd(1, ordering: .relaxed).oldValue, + queue: self.copyQueue + ) + } + /// Close a container's standard input to signal no more input is arriving. public func closeContainerStdin(_ containerID: String) async throws { try await self.state.withLock { state in diff --git a/Sources/Integration/PodTests.swift b/Sources/Integration/PodTests.swift index 7dff20fa..878cd323 100644 --- a/Sources/Integration/PodTests.swift +++ b/Sources/Integration/PodTests.swift @@ -2362,4 +2362,88 @@ extension IntegrationSuite { } } #endif + + /// A file copied into one container in a pod arrives in that container and + /// nowhere else, and comes back out with what it held. + func testPodCopyRoundTrip() async throws { + let id = "test-pod-copy-round-trip" + + let bs = try await bootstrap(id) + let pod = try LinuxPod(id, vmm: bs.vmm) { config in + config.cpus = 4 + config.memoryInBytes = 1024.mib() + config.bootLog = bs.bootLog + } + + let testContent = "Hello from the host! This is a pod copy test." + let hostDirectory = FileManager.default.uniqueTemporaryDirectory(create: true) + let hostFile = hostDirectory.appendingPathComponent("test-input.txt") + try testContent.write(to: hostFile, atomically: true, encoding: .utf8) + + try await pod.addContainer("holder", rootfs: try cloneRootfs(bs.rootfs, testID: id, containerID: "holder")) { config in + config.process.arguments = ["sleep", "100"] + } + try await pod.addContainer("bystander", rootfs: try cloneRootfs(bs.rootfs, testID: id, containerID: "bystander")) { config in + config.process.arguments = ["sleep", "100"] + } + + do { + try await pod.create() + try await pod.startContainer("holder") + try await pod.startContainer("bystander") + + try await pod.copyIn( + "holder", + from: hostFile, + to: URL(filePath: "/tmp/copied-file.txt") + ) + + let buffer = BufferWriter() + let exec = try await pod.execInContainer("holder", processID: "verify-copy") { config in + config.arguments = ["cat", "/tmp/copied-file.txt"] + config.stdout = buffer + } + try await exec.start() + let status = try await exec.wait() + try await exec.delete() + + guard status.exitCode == 0 else { + throw IntegrationError.assert(msg: "cat in the container copied into failed with status \(status)") + } + guard String(data: buffer.data, encoding: .utf8) == testContent else { + throw IntegrationError.assert( + msg: "expected '\(testContent)', got '\(String(data: buffer.data, encoding: .utf8) ?? "nil")'") + } + + // The containers share a machine and not a filesystem, so the copy + // reached one of them alone. + let bystander = try await pod.execInContainer("bystander", processID: "verify-absent") { config in + config.arguments = ["test", "-e", "/tmp/copied-file.txt"] + } + try await bystander.start() + let bystanderStatus = try await bystander.wait() + try await bystander.delete() + + guard bystanderStatus.exitCode != 0 else { + throw IntegrationError.assert(msg: "the copy reached a container it was not addressed to") + } + + let returned = hostDirectory.appendingPathComponent("test-output.txt") + try await pod.copyOut( + "holder", + from: URL(filePath: "/tmp/copied-file.txt"), + to: returned + ) + + let returnedContent = try String(contentsOf: returned, encoding: .utf8) + guard returnedContent == testContent else { + throw IntegrationError.assert(msg: "expected '\(testContent)' back, got '\(returnedContent)'") + } + + try await pod.stop() + } catch { + try? await pod.stop() + throw error + } + } } diff --git a/Sources/Integration/Suite.swift b/Sources/Integration/Suite.swift index 84580dba..571c0307 100644 --- a/Sources/Integration/Suite.swift +++ b/Sources/Integration/Suite.swift @@ -536,6 +536,7 @@ struct IntegrationSuite: AsyncParsableCommand { Test("pod memory events OOM kill", testPodMemoryEventsOOMKill), Test("pod container resource limits", testPodContainerResourceLimits), Test("pod container filesystem isolation", testPodContainerFilesystemIsolation), + Test("pod copy round trip", testPodCopyRoundTrip), Test("pod container PID namespace isolation", testPodContainerPIDNamespaceIsolation), Test("pod container independent resource limits", testPodContainerIndependentResourceLimits), Test("pod shared PID namespace", testPodSharedPIDNamespace), From 3a87186ad7abbbfe8bca627d0a1bed84cb6bcbb7 Mon Sep 17 00:00:00 2001 From: Aaron Paterson Date: Fri, 7 Aug 2026 20:42:42 +0000 Subject: [PATCH 07/10] Give a pod the machine it asked for A container with a machine of its own is given the size it asked for plus the guest agent's, so a container that asks for a gibibyte has a gibibyte and the agent runs beside it. A pod hands its size to the machine as it stands, so the agent comes out of what the pod asked for and its containers are left with less: a pod given twelve gibibytes reports eleven and a half. The pod's size and the agent's are added the same way, and what a pod was given is what its containers have. --- Sources/Containerization/LinuxPod.swift | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/Sources/Containerization/LinuxPod.swift b/Sources/Containerization/LinuxPod.swift index 457a7ebe..039f4aa7 100644 --- a/Sources/Containerization/LinuxPod.swift +++ b/Sources/Containerization/LinuxPod.swift @@ -47,6 +47,13 @@ public final class LinuxPod: Sendable { public var interfaces: [any Interface] = [] /// Whether nested virtualization should be turned on for the pod. public var virtualization: Bool = false + /// Additional CPU cores to allocate for the virtual machine on top of + /// the pod's configured `cpus`, so what the pod was given is what its + /// containers have rather than what the guest agent leaves of it. + public var cpuOverhead: Int = 1 + /// Additional memory in bytes to allocate for the virtual machine on + /// top of the pod's configured `memoryInBytes`. + public var memoryOverhead: UInt64 = 128.mib() /// Optional file path to store serial boot logs. public var bootLog: BootLog? /// Whether containers in the pod should share a PID namespace. @@ -622,9 +629,12 @@ extension LinuxPod { return false } + // The machine carries the guest agent as well as the containers, + // so it is given the pod's size and the agent's on top; what the + // pod was given is then what its containers have. var vmConfig = VMConfiguration( - cpus: self.config.cpus, - memoryInBytes: self.config.memoryInBytes, + cpus: self.config.cpus + self.config.cpuOverhead, + memoryInBytes: self.config.memoryInBytes + self.config.memoryOverhead, interfaces: self.config.interfaces, storage: machineStorage, bootLog: self.config.bootLog, @@ -634,9 +644,8 @@ extension LinuxPod { let creationConfig = StandardVMConfig(configuration: vmConfig) let vm = try await self.vmm.create(config: creationConfig) let relayManager = UnixSocketRelayManager(vm: vm) - try await vm.start() - do { + try await vm.start() let containers = state.containers let shareProcessNamespace = self.config.shareProcessNamespace let pauseProcessHolder = Mutex(nil) From ff841afe577d7502229fe40b034dadda82243a75 Mon Sep 17 00:00:00 2001 From: Aaron Paterson Date: Fri, 7 Aug 2026 21:10:11 +0000 Subject: [PATCH 08/10] Let a pod's containers be given a writable layer A container with a machine of its own can be given a block device to write into, with its image mounted read-only beneath, so the image is left as it is and every write lands in the layer. The same container placed in a pod has nowhere to be given one. The layer is attached with the container's other block devices, mounted in the guest as the upper layer of an overlay over the image, dropped from the mounts the runtime spec carries, and unmounted with the container. A pod's containers take one whether they were placed before the machine booted or hotplugged into it afterwards. --- .../Containerization/CHHotplugProvider.swift | 4 +- .../CHVirtualMachineInstance.swift | 4 +- .../Containerization/HotplugProvider.swift | 4 +- Sources/Containerization/LinuxContainer.swift | 43 +------ Sources/Containerization/LinuxPod.swift | 76 ++++++++++-- Sources/Containerization/OverlayRootfs.swift | 64 ++++++++++ .../VZVirtualMachineInstance.swift | 4 +- .../VirtualMachineInstance.swift | 6 +- Sources/Integration/PodTests.swift | 117 ++++++++++++++++++ Sources/Integration/Suite.swift | 2 + 10 files changed, 268 insertions(+), 56 deletions(-) create mode 100644 Sources/Containerization/OverlayRootfs.swift diff --git a/Sources/Containerization/CHHotplugProvider.swift b/Sources/Containerization/CHHotplugProvider.swift index 380cba76..e6a33d38 100644 --- a/Sources/Containerization/CHHotplugProvider.swift +++ b/Sources/Containerization/CHHotplugProvider.swift @@ -150,12 +150,12 @@ final class CHHotplugProvider: HotplugProvider { } } - func registerMounts(id: String, rootfs: AttachedFilesystem, additionalMounts: [Mount]) throws { + func registerMounts(id: String, rootfs: AttachedFilesystem, writableLayer: AttachedFilesystem?, additionalMounts: [Mount]) throws { var mounts: [AttachedFilesystem] = [] for mount in additionalMounts { mounts.append(try AttachedFilesystem(mount: mount, allocator: allocator)) } - let container = ContainerAttachments(rootfs: rootfs, mounts: mounts) + let container = ContainerAttachments(rootfs: rootfs, writableLayer: writableLayer, mounts: mounts) _storage.withLock { $0.containers[id] = container } diff --git a/Sources/Containerization/CHVirtualMachineInstance.swift b/Sources/Containerization/CHVirtualMachineInstance.swift index e363d5ca..5c427116 100644 --- a/Sources/Containerization/CHVirtualMachineInstance.swift +++ b/Sources/Containerization/CHVirtualMachineInstance.swift @@ -490,8 +490,8 @@ extension CHVirtualMachineInstance: VirtualMachineInstance { try await hotplug.releaseVirtioFS(id: id) } - public func registerMounts(id: String, rootfs: AttachedFilesystem, additionalMounts: [Mount]) throws { - try hotplug.registerMounts(id: id, rootfs: rootfs, additionalMounts: additionalMounts) + public func registerMounts(id: String, rootfs: AttachedFilesystem, writableLayer: AttachedFilesystem?, additionalMounts: [Mount]) throws { + try hotplug.registerMounts(id: id, rootfs: rootfs, writableLayer: writableLayer, additionalMounts: additionalMounts) } } diff --git a/Sources/Containerization/HotplugProvider.swift b/Sources/Containerization/HotplugProvider.swift index 0835077f..94fad4b4 100644 --- a/Sources/Containerization/HotplugProvider.swift +++ b/Sources/Containerization/HotplugProvider.swift @@ -30,8 +30,10 @@ public protocol HotplugProvider: Sendable { /// - Parameters: /// - id: The container ID /// - rootfs: The rootfs attachment from hotplug + /// - writableLayer: The container's writable layer attachment when it + /// has one /// - additionalMounts: Additional mounts to register - func registerMounts(id: String, rootfs: AttachedFilesystem, additionalMounts: [Mount]) throws + func registerMounts(id: String, rootfs: AttachedFilesystem, writableLayer: AttachedFilesystem?, additionalMounts: [Mount]) throws /// Release a hotplug device. /// - Parameter id: The container ID who should be released diff --git a/Sources/Containerization/LinuxContainer.swift b/Sources/Containerization/LinuxContainer.swift index 8aaedb10..36b07de7 100644 --- a/Sources/Containerization/LinuxContainer.swift +++ b/Sources/Containerization/LinuxContainer.swift @@ -541,47 +541,16 @@ extension LinuxContainer { rootfsPath: String, agent: VirtualMachineAgent ) async throws { - let rootfsAttachment = attached.rootfs - if let writableAttachment = attached.writableLayer { - // Set up overlayfs with image as lower layer and writable layer as upper. - let lowerPath = "/run/container/\(self.id)/lower" - let upperMountPath = "/run/container/\(self.id)/upper" - let upperPath = "/run/container/\(self.id)/upper/diff" - let workPath = "/run/container/\(self.id)/upper/work" - - // Mount the image (lower layer) as read-only. - var lowerMount = rootfsAttachment.to - lowerMount.destination = lowerPath - if !lowerMount.options.contains("ro") { - lowerMount.options.append("ro") - } - try await agent.mount(lowerMount) - - // Mount the writable layer. - var upperMount = writableAttachment.to - upperMount.destination = upperMountPath - try await agent.mount(upperMount) - - // Create the upper and work directories inside the writable layer. - try await agent.mkdir(path: upperPath, all: true, perms: 0o755) - try await agent.mkdir(path: workPath, all: true, perms: 0o755) - - // Mount the overlay. - let overlayMount = ContainerizationOCI.Mount( - type: "overlay", - source: "overlay", - destination: rootfsPath, - options: [ - "lowerdir=\(lowerPath)", - "upperdir=\(upperPath)", - "workdir=\(workPath)", - ] + try await agent.mountOverlayRootfs( + containerID: self.id, + rootfsAttachment: attached.rootfs, + writableAttachment: writableAttachment, + rootfsPath: rootfsPath ) - try await agent.mount(overlayMount) } else { // No writable layer. Mount rootfs directly. - var rootfs = rootfsAttachment.to + var rootfs = attached.rootfs.to rootfs.destination = rootfsPath try await agent.mount(rootfs) } diff --git a/Sources/Containerization/LinuxPod.swift b/Sources/Containerization/LinuxPod.swift index 039f4aa7..e3816e13 100644 --- a/Sources/Containerization/LinuxPod.swift +++ b/Sources/Containerization/LinuxPod.swift @@ -174,6 +174,7 @@ public final class LinuxPod: Sendable { private struct PodContainer: Sendable { let id: String let rootfs: Mount + let writableLayer: Mount? let config: ContainerConfiguration var state: ContainerState var process: LinuxProcess? @@ -301,7 +302,7 @@ public final class LinuxPod: Sendable { ) } - private func generateRuntimeSpec(containerID: String, config: ContainerConfiguration, rootfs: Mount) -> Spec { + private func generateRuntimeSpec(containerID: String, config: ContainerConfiguration, rootfs: Mount, writableLayer: Mount? = nil) -> Spec { var spec = Self.createDefaultRuntimeSpec(containerID, podID: self.id) // Process configuration @@ -326,7 +327,7 @@ public final class LinuxPod: Sendable { // If the rootfs was requested as read-only, set it in the OCI spec. // We let the OCI runtime remount as ro, instead of doing it originally. - spec.root?.readonly = rootfs.options.contains("ro") + spec.root?.readonly = rootfs.options.contains("ro") && writableLayer == nil // Resource limits (if specified) if let cpus = config.cpus, cpus > 0 { @@ -378,9 +379,14 @@ extension LinuxPod { /// When called before `create()`, the container is registered for setup during VM creation. /// When called after `create()`, the container is hotplugged into the running VM. /// If the underlying VMM does not support hotplug, an error is thrown. + /// - Parameters: + /// - writableLayer: Optional writable layer mount. When provided, an overlayfs is used with + /// the container's rootfs as the lower layer and this as the upper layer, so all writes + /// go to this layer instead of the rootfs. public func addContainer( _ id: String, rootfs: Mount, + writableLayer: Mount? = nil, configuration: @Sendable @escaping (inout ContainerConfiguration) throws -> Void ) async throws { guard id.count <= Self.maxIDLength else { @@ -389,6 +395,14 @@ extension LinuxPod { message: "container id length \(id.count) exceeds maximum of \(Self.maxIDLength) characters" ) } + if let writableLayer { + guard writableLayer.isBlock else { + throw ContainerizationError( + .invalidArgument, + message: "writableLayer must be a block device" + ) + } + } try await self.state.withLock { state in guard state.containers[id] == nil else { throw ContainerizationError( @@ -407,6 +421,7 @@ extension LinuxPod { state.containers[id] = PodContainer( id: id, rootfs: rootfs, + writableLayer: writableLayer, config: config, state: .registered, process: nil, @@ -416,6 +431,9 @@ extension LinuxPod { case .created(let createdState): let vm = createdState.vm + // Strip "ro" as create() does: readonly is expressed through + // the OCI spec's root.readonly field and a remount in vmexec + // after setup completes, so the device attaches writable. var modifiedRootfs = rootfs modifiedRootfs.options.removeAll(where: { $0 == "ro" }) @@ -423,6 +441,13 @@ extension LinuxPod { var updatedFileMountContext = fileMountContext do { + // The writable layer is a block device like the rootfs, + // attached alongside it so the overlay has both layers. + var writableAttachment: AttachedFilesystem? + if let writableLayer { + writableAttachment = try await vm.hotplug(writableLayer, id: id) + } + let virtioFSMounts = fileMountContext.transformedMounts.filter { if case .virtiofs(_) = $0.runtimeOptions { return true } return false @@ -433,13 +458,23 @@ extension LinuxPod { let agent = try await vm.dialAgent() do { - var mount = attachment.to - mount.destination = Self.guestRootfsPath(id) - try await agent.mount(mount) + if let writableAttachment { + try await agent.mountOverlayRootfs( + containerID: id, + rootfsAttachment: attachment, + writableAttachment: writableAttachment, + rootfsPath: Self.guestRootfsPath(id) + ) + } else { + var mount = attachment.to + mount.destination = Self.guestRootfsPath(id) + try await agent.mount(mount) + } - // Filter out shared mounts — those are handled separately as - // pod volume bind mounts. Without it here, a container added to an - // already-created would add a duplicated mount into the shared VM. + // Shared mounts are handled separately as pod volume + // bind mounts; without the filter here, a container + // added to an already-created pod would add a + // duplicated mount into the shared VM. let nonSharedMounts = fileMountContext.transformedMounts.filter { if case .shared = $0.runtimeOptions { return false } return true @@ -447,6 +482,7 @@ extension LinuxPod { try vm.registerMounts( id: id, rootfs: attachment, + writableLayer: writableAttachment, additionalMounts: nonSharedMounts ) @@ -547,6 +583,7 @@ extension LinuxPod { state.containers[id] = PodContainer( id: id, rootfs: rootfs, + writableLayer: writableLayer, config: config, state: .created, process: nil, @@ -586,6 +623,7 @@ extension LinuxPod { } machineStorage.containers[id] = ContainerMounts( rootfs: modifiedRootfs, + writableLayer: container.writableLayer, mounts: containerMounts ) } @@ -745,6 +783,15 @@ extension LinuxPod { guard let attached = vm.storage.containers[container.id] else { throw ContainerizationError(.notFound, message: "rootfs mount not found for container \(container.id)") } + if let writableAttachment = attached.writableLayer { + try await agent.mountOverlayRootfs( + containerID: container.id, + rootfsAttachment: attached.rootfs, + writableAttachment: writableAttachment, + rootfsPath: Self.guestRootfsPath(container.id) + ) + continue + } var rootfs = attached.rootfs.to rootfs.destination = Self.guestRootfsPath(container.id) try await agent.mount(rootfs) @@ -872,7 +919,7 @@ extension LinuxPod { let agent = try await createdState.vm.dialAgent() do { - var spec = self.generateRuntimeSpec(containerID: containerID, config: container.config, rootfs: container.rootfs) + var spec = self.generateRuntimeSpec(containerID: containerID, config: container.config, rootfs: container.rootfs, writableLayer: container.writableLayer) // We don't need the rootfs, nor do OCI runtimes want it included. // Also filter out file mount holding directories - we mount those separately under /run. // Transform virtiofs mounts to bind mounts from /run/virtiofs/{tag} @@ -1037,12 +1084,21 @@ extension LinuxPod { try await process.kill(.kill) try await process.wait(timeoutInSeconds: 3) + let hasWritableLayer = container.writableLayer != nil try await createdState.vm.withAgent { agent in // Unmount the rootfs try await agent.umount( path: Self.guestRootfsPath(containerID), flags: 0 ) + + // If we have a writable layer, we also need to unmount the lower and upper layers. + if hasWritableLayer { + let upperPath = "/run/container/\(containerID)/upper" + let lowerPath = "/run/container/\(containerID)/lower" + try await agent.umount(path: upperPath, flags: 0) + try await agent.umount(path: lowerPath, flags: 0) + } } // Release the hotplug device and virtiofs shares so they can be reused by new containers @@ -1197,7 +1253,7 @@ extension LinuxPod { ) } - var spec = self.generateRuntimeSpec(containerID: containerID, config: container.config, rootfs: container.rootfs) + var spec = self.generateRuntimeSpec(containerID: containerID, config: container.config, rootfs: container.rootfs, writableLayer: container.writableLayer) // Inherit environment variables, working directory, user, capabilities, rlimits from container process. // Reset: process arguments, terminal, stdio as these are not supposed to be inherited. var config = container.config.process diff --git a/Sources/Containerization/OverlayRootfs.swift b/Sources/Containerization/OverlayRootfs.swift new file mode 100644 index 00000000..9af76763 --- /dev/null +++ b/Sources/Containerization/OverlayRootfs.swift @@ -0,0 +1,64 @@ +//===----------------------------------------------------------------------===// +// Copyright © 2026 Apple Inc. and the Containerization project authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//===----------------------------------------------------------------------===// + +import ContainerizationOCI + +extension VirtualMachineAgent { + /// Mount a container's rootfs as an overlay, with the image as the lower + /// layer and the container's writable layer as the upper, so that writes + /// land in the layer and the image stays as it is. + func mountOverlayRootfs( + containerID: String, + rootfsAttachment: AttachedFilesystem, + writableAttachment: AttachedFilesystem, + rootfsPath: String + ) async throws { + let lowerPath = "/run/container/\(containerID)/lower" + let upperMountPath = "/run/container/\(containerID)/upper" + let upperPath = "/run/container/\(containerID)/upper/diff" + let workPath = "/run/container/\(containerID)/upper/work" + + // Mount the image (lower layer) as read-only. + var lowerMount = rootfsAttachment.to + lowerMount.destination = lowerPath + if !lowerMount.options.contains("ro") { + lowerMount.options.append("ro") + } + try await self.mount(lowerMount) + + // Mount the writable layer. + var upperMount = writableAttachment.to + upperMount.destination = upperMountPath + try await self.mount(upperMount) + + // Create the upper and work directories inside the writable layer. + try await self.mkdir(path: upperPath, all: true, perms: 0o755) + try await self.mkdir(path: workPath, all: true, perms: 0o755) + + // Mount the overlay. + let overlayMount = ContainerizationOCI.Mount( + type: "overlay", + source: "overlay", + destination: rootfsPath, + options: [ + "lowerdir=\(lowerPath)", + "upperdir=\(upperPath)", + "workdir=\(workPath)", + ] + ) + try await self.mount(overlayMount) + } +} diff --git a/Sources/Containerization/VZVirtualMachineInstance.swift b/Sources/Containerization/VZVirtualMachineInstance.swift index 670ab512..9ed965cd 100644 --- a/Sources/Containerization/VZVirtualMachineInstance.swift +++ b/Sources/Containerization/VZVirtualMachineInstance.swift @@ -325,9 +325,9 @@ extension VZVirtualMachineInstance: VirtualMachineInstance { return try await hotplugProvider.hotplug(block, id: id) } - public func registerMounts(id: String, rootfs: AttachedFilesystem, additionalMounts: [Mount]) throws { + public func registerMounts(id: String, rootfs: AttachedFilesystem, writableLayer: AttachedFilesystem?, additionalMounts: [Mount]) throws { guard let hotplugProvider else { return } - try hotplugProvider.registerMounts(id: id, rootfs: rootfs, additionalMounts: additionalMounts) + try hotplugProvider.registerMounts(id: id, rootfs: rootfs, writableLayer: writableLayer, additionalMounts: additionalMounts) } public func releaseHotplug(id: String) async throws { diff --git a/Sources/Containerization/VirtualMachineInstance.swift b/Sources/Containerization/VirtualMachineInstance.swift index b9340612..da7935d5 100644 --- a/Sources/Containerization/VirtualMachineInstance.swift +++ b/Sources/Containerization/VirtualMachineInstance.swift @@ -78,8 +78,10 @@ public protocol VirtualMachineInstance: Sendable { /// so they can be found when building the container's OCI spec. /// - Parameter id: The container ID /// - Parameter rootfs: The rootfs attachment from hotplug + /// - Parameter writableLayer: The container's writable layer attachment when it + /// has one /// - Parameter additionalMounts: Additional mounts (like /proc, /sys) to register - func registerMounts(id: String, rootfs: AttachedFilesystem, additionalMounts: [Mount]) throws + func registerMounts(id: String, rootfs: AttachedFilesystem, writableLayer: AttachedFilesystem?, additionalMounts: [Mount]) throws /// Release a hotplug device. /// This should be called when a hotplugged container is stopped or fails to start. @@ -107,7 +109,7 @@ extension VirtualMachineInstance { public func hotplug(_ block: Mount, id: String) async throws -> AttachedFilesystem { throw ContainerizationError(.unsupported, message: "hotplug not supported") } - public func registerMounts(id: String, rootfs: AttachedFilesystem, additionalMounts: [Mount]) throws { + public func registerMounts(id: String, rootfs: AttachedFilesystem, writableLayer: AttachedFilesystem?, additionalMounts: [Mount]) throws { // no-op default } public func releaseHotplug(id: String) async throws { diff --git a/Sources/Integration/PodTests.swift b/Sources/Integration/PodTests.swift index 878cd323..7aeb011a 100644 --- a/Sources/Integration/PodTests.swift +++ b/Sources/Integration/PodTests.swift @@ -2363,6 +2363,123 @@ extension IntegrationSuite { } #endif + /// A container in a pod given a writable layer writes into it, and the + /// image it was built from is left as it is for the pod's others. + /// Add a container with a writable layer to a pod whose machine is + /// already running: the overlay assembles from the two disks attached + /// while it runs, and writes land in the layer. + func testPodHotplugWritableLayer() async throws { + let id = "test-pod-hotplug-writable-layer" + let bs = try await bootstrap(id) + + let pod = try LinuxPod(id, vmm: bs.vmm) { config in + config.cpus = 4 + config.memoryInBytes = 1024.mib() + config.bootLog = bs.bootLog + } + + try await pod.addContainer("seed", rootfs: try cloneRootfs(bs.rootfs, testID: id, containerID: "seed")) { config in + config.process.arguments = ["/bin/sleep", "infinity"] + } + + try await pod.create() + + let writableLayerPath = Self.testDir.appending(component: "\(id)-writable.ext4") + try? FileManager.default.removeItem(at: writableLayerPath) + let filesystem = try EXT4.Formatter(FilePath(writableLayerPath.absolutePath()), minDiskSize: 512.mib()) + try filesystem.close() + let writableLayer = Mount.block( + format: "ext4", + source: writableLayerPath.absolutePath(), + destination: "/", + options: [] + ) + + let buffer = BufferWriter() + try await pod.addContainer( + "hot", + rootfs: try cloneRootfs(bs.rootfs, testID: id, containerID: "hot"), + writableLayer: writableLayer + ) { config in + config.process.arguments = ["/bin/sh", "-c", "echo 'written into a layer added while running' > /written && cat /written"] + config.process.stdout = buffer + } + + do { + try await pod.startContainer("hot") + let status = try await pod.waitContainer("hot") + + try await pod.stopContainer("hot") + try await pod.stop() + + guard status.exitCode == 0 else { + throw IntegrationError.assert(msg: "hot container status \(status) != 0") + } + let expected = "written into a layer added while running" + guard let output = String(data: buffer.data, encoding: .utf8), + output.trimmingCharacters(in: .whitespacesAndNewlines) == expected + else { + throw IntegrationError.assert( + msg: "expected '\(expected)', got '\(String(data: buffer.data, encoding: .utf8) ?? "nil")'") + } + } catch { + try? await pod.stop() + throw error + } + } + + func testPodWritableLayer() async throws { + let id = "test-pod-writable-layer" + + let bs = try await bootstrap(id) + let pod = try LinuxPod(id, vmm: bs.vmm) { config in + config.cpus = 4 + config.memoryInBytes = 1024.mib() + config.bootLog = bs.bootLog + } + + let writableLayerPath = Self.testDir.appending(component: "\(id)-writable.ext4") + try? FileManager.default.removeItem(at: writableLayerPath) + let filesystem = try EXT4.Formatter(FilePath(writableLayerPath.absolutePath()), minDiskSize: 512.mib()) + try filesystem.close() + let writableLayer = Mount.block( + format: "ext4", + source: writableLayerPath.absolutePath(), + destination: "/", + options: [] + ) + + let buffer = BufferWriter() + try await pod.addContainer( + "layered", + rootfs: try cloneRootfs(bs.rootfs, testID: id, containerID: "layered"), + writableLayer: writableLayer + ) { config in + config.process.arguments = ["/bin/sh", "-c", "echo 'writable layer test' > /tmp/testfile && cat /tmp/testfile"] + config.process.stdout = buffer + } + + do { + try await pod.create() + try await pod.startContainer("layered") + let status = try await pod.waitContainer("layered") + try await pod.stop() + + guard status.exitCode == 0 else { + throw IntegrationError.assert(msg: "process failed with status \(status)") + } + guard let output = String(data: buffer.data, encoding: .utf8) else { + throw IntegrationError.assert(msg: "failed to convert stdout to UTF8") + } + guard output.trimmingCharacters(in: .whitespacesAndNewlines) == "writable layer test" else { + throw IntegrationError.assert(msg: "unexpected output: \(output)") + } + } catch { + try? await pod.stop() + throw error + } + } + /// A file copied into one container in a pod arrives in that container and /// nowhere else, and comes back out with what it held. func testPodCopyRoundTrip() async throws { diff --git a/Sources/Integration/Suite.swift b/Sources/Integration/Suite.swift index 571c0307..2af578b6 100644 --- a/Sources/Integration/Suite.swift +++ b/Sources/Integration/Suite.swift @@ -537,6 +537,7 @@ struct IntegrationSuite: AsyncParsableCommand { Test("pod container resource limits", testPodContainerResourceLimits), Test("pod container filesystem isolation", testPodContainerFilesystemIsolation), Test("pod copy round trip", testPodCopyRoundTrip), + Test("pod writable layer", testPodWritableLayer), Test("pod container PID namespace isolation", testPodContainerPIDNamespaceIsolation), Test("pod container independent resource limits", testPodContainerIndependentResourceLimits), Test("pod shared PID namespace", testPodSharedPIDNamespace), @@ -623,6 +624,7 @@ struct IntegrationSuite: AsyncParsableCommand { let linuxOnlyTests: [Test] = [ Test("pod hotplug block rootfs", testPodHotplugBlockRootfs), Test("pod hotplug virtiofs rootfs", testPodHotplugVirtiofsRootfs), + Test("pod hotplug writable layer", testPodHotplugWritableLayer), ] let tests: [Test] = crossPlatformTests + linuxOnlyTests #endif From baf30df6678da72c73520ccabc5dba1352fb9f5a Mon Sep 17 00:00:00 2001 From: Aaron Paterson Date: Sat, 8 Aug 2026 02:01:02 +0000 Subject: [PATCH 09/10] Name the project the guest file transfer belongs to The file carried the sibling repository's copyright line, so the license header check found no header it recognized. --- Sources/Containerization/GuestFileTransfer.swift | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Sources/Containerization/GuestFileTransfer.swift b/Sources/Containerization/GuestFileTransfer.swift index d508c571..5eeb039d 100644 --- a/Sources/Containerization/GuestFileTransfer.swift +++ b/Sources/Containerization/GuestFileTransfer.swift @@ -1,5 +1,5 @@ //===----------------------------------------------------------------------===// -// Copyright © 2026 Apple Inc. and the container project authors. +// Copyright © 2026 Apple Inc. and the Containerization project authors. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. From 1bfb102cd25c2b6d8f8c370023853df08d25d06b Mon Sep 17 00:00:00 2001 From: Aaron Paterson Date: Mon, 10 Aug 2026 05:40:19 +0000 Subject: [PATCH 10/10] Let a pod give a container's place back Stopping a container tears down what it was running and keeps its place: the name still answers for it, and placing another container under it is refused. Removal is the separate act the runtime specification names for giving the place up, taken once the container has stopped, so a name can run again in a machine that outlives what it last ran. https://github.com/kubernetes/cri-api/blob/master/pkg/apis/runtime/v1/api.proto --- Sources/Containerization/LinuxPod.swift | 29 +++++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/Sources/Containerization/LinuxPod.swift b/Sources/Containerization/LinuxPod.swift index e3816e13..16a485c3 100644 --- a/Sources/Containerization/LinuxPod.swift +++ b/Sources/Containerization/LinuxPod.swift @@ -1125,6 +1125,35 @@ extension LinuxPod { } } + /// Take a container out of the pod, so its name is free to place again. + /// + /// Stopping a container tears down what it was running and keeps its + /// place; the name still answers for it, and placing another container + /// under it is refused. Removal is the separate act the runtime + /// specification names for giving the place up, taken once the container + /// has stopped. A container that is running keeps its place and this + /// call refuses it. + /// https://github.com/kubernetes/cri-api/blob/master/pkg/apis/runtime/v1/api.proto + public func removeContainer(_ containerID: String) async throws { + try await self.state.withLock { state in + guard let container = state.containers[containerID] else { + throw ContainerizationError( + .notFound, + message: "container \(containerID) not found in pod" + ) + } + switch container.state { + case .registered, .stopped, .errored: + state.containers[containerID] = nil + default: + throw ContainerizationError( + .invalidState, + message: "container \(containerID) must stop before it is removed" + ) + } + } + } + /// Stop the pod's VM and all containers. public func stop() async throws { try await self.state.withLock { state in