diff --git a/Makefile b/Makefile index c79d6995..c5645fb6 100644 --- a/Makefile +++ b/Makefile @@ -64,7 +64,7 @@ LIBARCHIVE_UPSTREAM_REPO := https://github.com/libarchive/libarchive LIBARCHIVE_UPSTREAM_VERSION := v3.7.7 LIBARCHIVE_LOCAL_DIR := workdir/libarchive -KATA_BINARY_PACKAGE := https://github.com/kata-containers/kata-containers/releases/download/3.17.0/kata-static-3.17.0-arm64.tar.xz +KATA_BINARY_PACKAGE := https://github.com/kata-containers/kata-containers/releases/download/3.28.0/kata-static-3.28.0-arm64.tar.zst CLOUD_HYPERVISOR_URL := https://github.com/cloud-hypervisor/cloud-hypervisor/releases/download/v52.0/cloud-hypervisor-static-aarch64 # SHA256 of the v52.0 aarch64 static binary (verified locally from the # upstream release artifact). Bump alongside CLOUD_HYPERVISOR_URL. @@ -410,11 +410,11 @@ integration: .PHONY: fetch-default-kernel fetch-default-kernel: @mkdir -p .local/ bin/ -ifeq (,$(wildcard .local/kata.tar.gz)) - @curl -SsL -o .local/kata.tar.gz ${KATA_BINARY_PACKAGE} +ifeq (,$(wildcard .local/kata.tar)) + @curl -SsL -o .local/kata.tar ${KATA_BINARY_PACKAGE} endif ifeq (,$(wildcard .local/vmlinux-$(KERNEL_ARCH))) - @tar -zxf .local/kata.tar.gz -C .local/ --strip-components=1 + @tar -xf .local/kata.tar -C .local/ --strip-components=1 @cp -L .local/opt/kata/share/kata-containers/vmlinux.container .local/vmlinux-$(KERNEL_ARCH) endif ifeq (,$(wildcard bin/vmlinux-$(KERNEL_ARCH))) @@ -486,10 +486,22 @@ docs: @rm -rf _site @scripts/make-docs.sh _site containerization +# The Linux dev container keeps its content beside the repository rather than +# under a home directory of its own, which does not survive a `container run`. +# It is the same store either way, so cleaning the content means both of them. .PHONY: cleancontent cleancontent: @echo Cleaning the content... @rm -rf ~/Library/Application\ Support/com.apple.containerization + @rm -rf $(ROOT_DIR)/.local/integration-cache + +# The integration suite takes its files away as it goes, so this is for what a +# run that was interrupted, or one asked to keep them, has left behind. The +# directory is the one IntegrationSuite.testRootName names. +.PHONY: cleantests +cleantests: + @echo Cleaning the integration test files... + @rm -rf "$${TMPDIR:-/tmp}/containerization-integration" .PHONY: examples examples: diff --git a/README.md b/README.md index f658b08d..372a872a 100644 --- a/README.md +++ b/README.md @@ -130,6 +130,16 @@ After building, run basic and integration tests: make test integration ``` +Tests that only apply to Linux are compiled out on macOS, so `make test` passes +without running them. Run those with: + +```bash +make linux-test +``` + +which runs the same tests inside the Linux dev container, as the Linux build +workflow does. + A kernel is required to run integration tests. If you do not have a kernel locally, a default kernel can be fetched using the `make fetch-default-kernel` target. diff --git a/Sources/Containerization/CHHotplugProvider.swift b/Sources/Containerization/CHHotplugProvider.swift index 870fb1ff..380cba76 100644 --- a/Sources/Containerization/CHHotplugProvider.swift +++ b/Sources/Containerization/CHHotplugProvider.swift @@ -27,8 +27,8 @@ import Synchronization /// /// Handles both block (`vm.add-disk`) and virtiofs (`vm.add-fs`, with one /// `virtiofsd` per unique source-hash tag) hotplug, plus the matching -/// `vm.remove-device` teardown. Owns the per-VM mount registry so -/// `CHVirtualMachineInstance.mounts` can forward to it. +/// `vm.remove-device` teardown. Owns the machine's storage so +/// `CHVirtualMachineInstance.storage` can forward to it. final class CHHotplugProvider: HotplugProvider { struct HotplugRecord: Sendable { let chDeviceId: String @@ -50,7 +50,7 @@ final class CHHotplugProvider: HotplugProvider { private let workDir: URL private let virtiofsdBinaryOverride: URL? private let allocator: any AddressAllocator - private let _mounts: Mutex<[String: [AttachedFilesystem]]> + private let _storage: Mutex private let _records: Mutex<[String: [HotplugRecord]]> private let _tags: Mutex<[String: VirtiofsdTagState]> /// Serializes per-tag virtiofsd spawn so a concurrent hotplug for the @@ -65,14 +65,14 @@ final class CHHotplugProvider: HotplugProvider { workDir: URL, virtiofsdBinary: URL?, allocator: any AddressAllocator, - initialMounts: [String: [AttachedFilesystem]], + initialStorage: MachineAttachments, logger: Logger? ) { self.client = client self.workDir = workDir self.virtiofsdBinaryOverride = virtiofsdBinary self.allocator = allocator - self._mounts = Mutex(initialMounts) + self._storage = Mutex(initialStorage) self._records = Mutex([:]) self._tags = Mutex([:]) self.spawnLock = AsyncLock() @@ -81,14 +81,14 @@ final class CHHotplugProvider: HotplugProvider { // MARK: - Read accessors - var mounts: [String: [AttachedFilesystem]] { - _mounts.withLock { $0 } + var storage: MachineAttachments { + _storage.withLock { $0 } } - func withMountRegistry( - _ body: (inout sending [String: [AttachedFilesystem]]) throws -> sending T + func withStorage( + _ body: (inout sending MachineAttachments) throws -> sending T ) rethrows -> T { - try _mounts.withLock(body) + try _storage.withLock(body) } // MARK: - HotplugProvider conformance @@ -151,12 +151,13 @@ final class CHHotplugProvider: HotplugProvider { } func registerMounts(id: String, rootfs: AttachedFilesystem, additionalMounts: [Mount]) throws { - var attached: [AttachedFilesystem] = [rootfs] + var mounts: [AttachedFilesystem] = [] for mount in additionalMounts { - attached.append(try AttachedFilesystem(mount: mount, allocator: allocator)) + mounts.append(try AttachedFilesystem(mount: mount, allocator: allocator)) } - _mounts.withLock { - $0[id, default: []].append(contentsOf: attached) + let container = ContainerAttachments(rootfs: rootfs, mounts: mounts) + _storage.withLock { + $0.containers[id] = container } } @@ -190,19 +191,10 @@ final class CHHotplugProvider: HotplugProvider { } } - // Drop block-derived AttachedFilesystem entries for `id`. Block entries - // are the ones whose source was rewritten to "/dev/vd" by - // `hotplug(_:)` (or by AttachedFilesystem(mount:allocator:) for an - // additionalMount of type virtio-blk). - _mounts.withLock { state in - guard var perID = state[id] else { return } - perID.removeAll { $0.source.hasPrefix("/dev/vd") } - if perID.isEmpty { - state.removeValue(forKey: id) - } else { - state[id] = perID - } - } + // The container's devices are gone, so the container leaves the + // registry with them; its shares are released separately and their + // processes reference-counted through `_tags`. + _ = _storage.withLock { $0.containers.removeValue(forKey: id) } } func hotplugVirtioFS(_ mounts: [Mount], id: String) async throws { @@ -226,7 +218,7 @@ final class CHHotplugProvider: HotplugProvider { let chDeviceId = try await ensureVirtiofsDevice(tag: tag, source: source, readonly: readonly) // Record once per tag for this container. The AttachedFilesystem // entries for these mounts are written by registerMounts (the sole - // _mounts writer), so we do NOT touch _mounts here. + // registry writer), so we do NOT touch the storage here. _records.withLock { $0[id, default: []].append(HotplugRecord(chDeviceId: chDeviceId, kind: .virtiofs(tag: tag))) } @@ -338,16 +330,17 @@ final class CHHotplugProvider: HotplugProvider { try? FileManager.default.removeItem(at: socket) } - // Drop virtiofs AttachedFilesystem entries for `id`. AttachedFilesystem - // sets `type = mount.type` which for a `.virtiofs` mount is "virtiofs". - _mounts.withLock { state in - guard var perID = state[id] else { return } - perID.removeAll { $0.type == "virtiofs" } - if perID.isEmpty { - state.removeValue(forKey: id) - } else { - state[id] = perID + // Drop the container's virtiofs entries. A container whose rootfs is + // itself a share leaves the registry whole; one that keeps block + // devices keeps its entry with the share entries dropped. + _storage.withLock { state in + guard var container = state.containers[id] else { return } + if container.rootfs.type == "virtiofs" { + state.containers.removeValue(forKey: id) + return } + container.mounts.removeAll { $0.type == "virtiofs" } + state.containers[id] = container } } @@ -358,15 +351,21 @@ final class CHHotplugProvider: HotplugProvider { /// is the user-supplied `FsConfig.id` (which `vm.remove-device` keys on). /// `ownerIds` are the container ids that count toward this tag's refcount; /// each gets a `HotplugRecord` so `releaseVirtioFS(id:)` walks them - /// uniformly. + /// uniformly. `machineHeld` adds one reference nothing releases, for a + /// share the machine itself owns (a volume). func recordBootTimeVirtiofs( tag: String, process: VirtiofsdProcess, chDeviceId: String, - ownerIds: [String] + ownerIds: [String], + machineHeld: Bool ) { _tags.withLock { - $0[tag] = VirtiofsdTagState(process: process, refcount: ownerIds.count, chDeviceId: chDeviceId) + $0[tag] = VirtiofsdTagState( + process: process, + refcount: ownerIds.count + (machineHeld ? 1 : 0), + chDeviceId: chDeviceId + ) } _records.withLock { records in for id in ownerIds { diff --git a/Sources/Containerization/CHVirtualMachineInstance.swift b/Sources/Containerization/CHVirtualMachineInstance.swift index 122f592a..e363d5ca 100644 --- a/Sources/Containerization/CHVirtualMachineInstance.swift +++ b/Sources/Containerization/CHVirtualMachineInstance.swift @@ -46,7 +46,7 @@ public final class CHVirtualMachineInstance: Sendable { public struct Configuration: Sendable { public var cpus: Int public var memoryInBytes: UInt64 - public var mountsByID: [String: [Mount]] + public var storage: MachineMounts public var interfaces: [any Interface] public var kernel: Kernel? public var initialFilesystem: Mount? @@ -56,7 +56,7 @@ public final class CHVirtualMachineInstance: Sendable { public init() { self.cpus = 4 self.memoryInBytes = 1024 * 1024 * 1024 - self.mountsByID = [:] + self.storage = MachineMounts() self.interfaces = [] } } @@ -65,8 +65,8 @@ public final class CHVirtualMachineInstance: Sendable { /// `start()`'s `VmConfig.disks` ordering matches the allocator letters. struct BootDisk: Sendable { let mount: Mount - let containerId: String? // nil for rootfs - let letter: Character + let chId: String + let readonly: Bool } // MARK: - State @@ -76,8 +76,8 @@ public final class CHVirtualMachineInstance: Sendable { _state.withLock { $0 } } - public var mounts: [String: [AttachedFilesystem]] { - hotplug.mounts + public var storage: MachineAttachments { + hotplug.storage } /// Cloud-hypervisor exposes one virtio-fs device per source-hash tag, so @@ -167,9 +167,9 @@ public final class CHVirtualMachineInstance: Sendable { ) self.workDir = workDir - // 2. Block allocator + boot inventory. Walks rootfs first, then - // mountsByID sorted by container id, allocating disk letters in - // that order. The same allocator is later handed to the hotplug + // 2. Block allocator + boot inventory. Walks rootfs first, then the + // machine's storage in its device order, allocating disk letters + // as it goes. The same allocator is later handed to the hotplug // provider so runtime add-disk picks up where boot wiring left off. let allocator = Character.blockDeviceTagAllocator() let inventory = try config.bootInventory(allocator: allocator) @@ -201,14 +201,14 @@ public final class CHVirtualMachineInstance: Sendable { logger: logger ?? Logger(label: "CloudHypervisor.Client") ) - // 5. Hotplug provider — owns the mount registry, seeded with the + // 5. Hotplug provider — owns the machine's storage, seeded with the // boot inventory so registerMounts can append to it. self.hotplug = CHHotplugProvider( client: self.client, workDir: workDir, virtiofsdBinary: virtiofsdBinary, allocator: allocator, - initialMounts: inventory.attachments, + initialStorage: inventory.attachments, logger: logger ) @@ -222,11 +222,11 @@ public final class CHVirtualMachineInstance: Sendable { self._stdioPool = Mutex([:]) } - /// Mutate the mount registry. Forwards to the hotplug provider, which + /// Mutate the storage registry. Forwards to the hotplug provider, which /// owns the registry. Kept on the instance for parity with the macOS - /// path's `withMountRegistry` API. - func withMountRegistry(_ body: (inout sending [String: [AttachedFilesystem]]) throws -> sending T) rethrows -> T { - try hotplug.withMountRegistry(body) + /// path's `withStorage` API. + func withStorage(_ body: (inout sending MachineAttachments) throws -> sending T) rethrows -> T { + try hotplug.withStorage(body) } } @@ -510,34 +510,46 @@ extension CHVirtualMachineInstance { throw ContainerizationError(.invalidArgument, message: "initialFilesystem is required for cloud-hypervisor backend") } - // Disks: rootfs forced read-only at the device level; container disks + // Disks: rootfs forced read-only at the device level; other disks // honor their `ro` option through chDiskConfig. var disks: [CloudHypervisor.DiskConfig] = [] for bd in bootDisks { - let chId = bd.containerId.map { "blk-\($0)-\(bd.letter)" } ?? "rootfs" - if var disk = bd.mount.chDiskConfig(id: chId) { - if bd.containerId == nil { + if var disk = bd.mount.chDiskConfig(id: bd.chId) { + if bd.readonly { disk.readonly = true } disks.append(disk) } } - // Virtiofs: group all .virtiofs mounts in mountsByID by source-hash - // tag, spawn one virtiofsd per tag, build matching FsConfigs. - var byTag: [String: (mounts: [Mount], owners: [String])] = [:] - for cid in config.mountsByID.keys.sorted() { - guard let mounts = config.mountsByID[cid] else { continue } - for mount in mounts { - guard case .virtiofs = mount.runtimeOptions else { continue } - let tag = try hashFilePath(path: mount.source) - var entry = byTag[tag] ?? (mounts: [], owners: []) - entry.mounts.append(mount) - if !entry.owners.contains(cid) { - entry.owners.append(cid) + // Virtiofs: group all virtiofs mounts in the machine's storage by + // source-hash tag, spawn one virtiofsd per tag, build matching + // FsConfigs. A volume's tag is the machine's own, held for its + // lifetime. + var byTag: [String: (mounts: [Mount], owners: [String], machineHeld: Bool)] = [:] + func groupVirtiofs(_ mount: Mount, owner: String?) throws { + guard case .virtiofs = mount.runtimeOptions else { return } + let tag = try hashFilePath(path: mount.source) + var entry = byTag[tag] ?? (mounts: [], owners: [], machineHeld: false) + entry.mounts.append(mount) + if let owner { + if !entry.owners.contains(owner) { + entry.owners.append(owner) } - byTag[tag] = entry + } else { + entry.machineHeld = true } + byTag[tag] = entry + } + for cid in config.storage.containers.keys.sorted() { + guard let container = config.storage.containers[cid] else { continue } + for mount in container.all { + try groupVirtiofs(mount, owner: cid) + } + } + for name in config.storage.volumes.keys.sorted() { + guard let mount = config.storage.volumes[name] else { continue } + try groupVirtiofs(mount, owner: nil) } var fsConfigs: [CloudHypervisor.FsConfig] = [] @@ -569,7 +581,8 @@ extension CHVirtualMachineInstance { tag: tag, process: process, chDeviceId: chDeviceId, - ownerIds: entry.owners + ownerIds: entry.owners, + machineHeld: entry.machineHeld ) fsConfigs.append( @@ -714,40 +727,46 @@ extension CHVirtualMachineInstance { // MARK: - Boot inventory extension CHVirtualMachineInstance.Configuration { - /// Walks boot-time mounts in deterministic order (rootfs first, then - /// `mountsByID` sorted by container id, then each container's mounts in - /// input order), allocating disk letters for virtio-blk mounts and seeding - /// the per-container `AttachedFilesystem` registry. + /// Walks boot-time storage in the machine's device order (rootfs first, + /// then containers sorted by id, each in role order, then volumes sorted + /// by name), allocating disk letters for virtio-blk mounts and seeding + /// the machine's `AttachedFilesystem` registry. /// /// The allocator is shared with the runtime hotplug provider, so block /// hotplug picks up at the next free letter after boot. func bootInventory( allocator: any AddressAllocator - ) throws -> (attachments: [String: [AttachedFilesystem]], bootDisks: [CHVirtualMachineInstance.BootDisk]) { + ) throws -> (attachments: MachineAttachments, bootDisks: [CHVirtualMachineInstance.BootDisk]) { var bootDisks: [CHVirtualMachineInstance.BootDisk] = [] - var attachments: [String: [AttachedFilesystem]] = [:] - // Rootfs is not part of mountsByID. If it's a block device, it claims - // the first letter (vda) so the kernel cmdline `root=/dev/vda` is right. + // The rootfs is the machine's own, not part of its containers' + // storage. If it's a block device, it claims the first letter (vda) + // so the kernel cmdline `root=/dev/vda` is right. if let rootfs = self.initialFilesystem, rootfs.isBlock { - let letter = try allocator.allocate() - bootDisks.append(.init(mount: rootfs, containerId: nil, letter: letter)) + _ = try allocator.allocate() + bootDisks.append(.init(mount: rootfs, chId: "rootfs", readonly: true)) } - for cid in self.mountsByID.keys.sorted() { - guard let mounts = self.mountsByID[cid] else { continue } - var perContainer: [AttachedFilesystem] = [] - for mount in mounts { - let attached = try AttachedFilesystem(mount: mount, allocator: allocator) - if mount.isBlock, let letter = attached.source.last { - bootDisks.append(.init(mount: mount, containerId: cid, letter: letter)) - } - perContainer.append(attached) + func attach(_ mount: Mount, chId: (Character) -> String) throws -> AttachedFilesystem { + let attached = try AttachedFilesystem(mount: mount, allocator: allocator) + if mount.isBlock, let letter = attached.source.last { + bootDisks.append(.init(mount: mount, chId: chId(letter), readonly: false)) } - attachments[cid] = perContainer + return attached + } + + var containers: [String: ContainerAttachments] = [:] + for cid in self.storage.containers.keys.sorted() { + guard let container = self.storage.containers[cid] else { continue } + containers[cid] = try container.map { try attach($0, chId: { "blk-\(cid)-\($0)" }) } + } + var volumes: [String: AttachedFilesystem] = [:] + for name in self.storage.volumes.keys.sorted() { + guard let mount = self.storage.volumes[name] else { continue } + volumes[name] = try attach(mount, chId: { "vol-\(name)-\($0)" }) } - return (attachments, bootDisks) + return (MachineAttachments(containers: containers, volumes: volumes), bootDisks) } } #endif diff --git a/Sources/Containerization/CHVirtualMachineManager.swift b/Sources/Containerization/CHVirtualMachineManager.swift index ba20b5d1..c444620d 100644 --- a/Sources/Containerization/CHVirtualMachineManager.swift +++ b/Sources/Containerization/CHVirtualMachineManager.swift @@ -99,7 +99,7 @@ public struct CHVirtualMachineManager: VirtualMachineManager { instanceConfig.cpus = vmConfig.cpus instanceConfig.memoryInBytes = vmConfig.memoryInBytes instanceConfig.interfaces = vmConfig.interfaces - instanceConfig.mountsByID = vmConfig.mountsByID + instanceConfig.storage = vmConfig.storage instanceConfig.bootLog = vmConfig.bootLog instanceConfig.extensions = vmConfig.extensions instanceConfig.kernel = kernel diff --git a/Sources/Containerization/ContainerStorage.swift b/Sources/Containerization/ContainerStorage.swift new file mode 100644 index 00000000..4d8de2ed --- /dev/null +++ b/Sources/Containerization/ContainerStorage.swift @@ -0,0 +1,107 @@ +//===----------------------------------------------------------------------===// +// Copyright © 2026 Apple Inc. and the Containerization project authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//===----------------------------------------------------------------------===// + +/// A container's storage, by role. +/// +/// The same shape describes storage at both stages of its life: as the +/// `Mount` values a container is configured with, and as the +/// `AttachedFilesystem` values the machine reports once they are attached. +/// Converting between the two is a `map` over the structure, so the roles +/// cannot drift apart between configuration and attachment. +public struct ContainerStorage: Sendable { + /// The container's root filesystem. + public var rootfs: Value + + /// The writable layer mounted over the rootfs as the upper side of an + /// overlay, when the container has one. + public var writableLayer: Value? + + /// The container's remaining mounts, in configuration order. + public var mounts: [Value] + + public init(rootfs: Value, writableLayer: Value? = nil, mounts: [Value] = []) { + self.rootfs = rootfs + self.writableLayer = writableLayer + self.mounts = mounts + } + + /// Every value in the structure: the rootfs, the writable layer when + /// present, then the mounts, in that order. + public var all: [Value] { + var values = [rootfs] + if let writableLayer { + values.append(writableLayer) + } + values.append(contentsOf: mounts) + return values + } + + /// The storage with `transform` applied to every value, each keeping + /// its role. + public func map(_ transform: (Value) throws -> U) rethrows -> ContainerStorage { + ContainerStorage( + rootfs: try transform(rootfs), + writableLayer: try writableLayer.map(transform), + mounts: try mounts.map(transform) + ) + } +} + +/// A container's storage as configured, before its machine exists. +public typealias ContainerMounts = ContainerStorage + +/// A container's storage as attached to its machine. +public typealias ContainerAttachments = ContainerStorage + +/// The storage a machine carries: each container's, and the resources its +/// containers share. +public struct MachineStorage: Sendable { + /// Each container's storage, by container ID. + public var containers: [String: ContainerStorage] + + /// Volumes shared by the machine's containers, by volume name. + public var volumes: [String: Value] + + public init(containers: [String: ContainerStorage] = [:], volumes: [String: Value] = [:]) { + self.containers = containers + self.volumes = volumes + } + + /// Every value the machine carries: containers sorted by ID, each in + /// role order, then volumes sorted by name. Walks that allocate device + /// addresses and walks that create the devices use this one order, so an + /// address always names the device it was handed out for. + public var ordered: [Value] { + containers.keys.sorted().flatMap { containers[$0]?.all ?? [] } + + volumes.keys.sorted().compactMap { volumes[$0] } + } + + /// The storage with `transform` applied to every value, each keeping its + /// place. The transform runs in no particular order, so an allocating + /// conversion walks the fields sorted itself. + public func map(_ transform: (Value) throws -> U) rethrows -> MachineStorage { + MachineStorage( + containers: try containers.mapValues { try $0.map(transform) }, + volumes: try volumes.mapValues(transform) + ) + } +} + +/// A machine's storage as configured, before it exists. +public typealias MachineMounts = MachineStorage + +/// A machine's storage as attached. +public typealias MachineAttachments = MachineStorage diff --git a/Sources/Containerization/HotplugProvider.swift b/Sources/Containerization/HotplugProvider.swift index f535ce3f..0835077f 100644 --- a/Sources/Containerization/HotplugProvider.swift +++ b/Sources/Containerization/HotplugProvider.swift @@ -26,7 +26,7 @@ public protocol HotplugProvider: Sendable { /// - Returns: The attached filesystem with the device path in the guest func hotplug(_ block: Mount, id: String) async throws -> AttachedFilesystem - /// Register mounts for a container in the VM's mount registry. + /// Register mounts for a container in the machine's storage. /// - Parameters: /// - id: The container ID /// - rootfs: The rootfs attachment from hotplug diff --git a/Sources/Containerization/LinuxContainer.swift b/Sources/Containerization/LinuxContainer.swift index 34964fcd..55bbb39a 100644 --- a/Sources/Containerization/LinuxContainer.swift +++ b/Sources/Containerization/LinuxContainer.swift @@ -537,24 +537,14 @@ extension LinuxContainer { } private func mountRootfs( - attachments: [AttachedFilesystem], + attached: ContainerAttachments, rootfsPath: String, agent: VirtualMachineAgent ) async throws { - guard let rootfsAttachment = attachments.first else { - throw ContainerizationError(.notFound, message: "rootfs mount not found") - } + let rootfsAttachment = attached.rootfs - if self.writableLayer != nil { + if let writableAttachment = attached.writableLayer { // Set up overlayfs with image as lower layer and writable layer as upper. - guard attachments.count >= 2 else { - throw ContainerizationError( - .notFound, - message: "writable layer mount not found" - ) - } - let writableAttachment = attachments[1] - let lowerPath = "/run/container/\(self.id)/lower" let upperMountPath = "/run/container/\(self.id)/upper" let upperPath = "/run/container/\(self.id)/upper/diff" @@ -623,17 +613,18 @@ extension LinuxContainer { // This is dumb, but alas. let fileMountContextHolder = Mutex(fileMountContext) - // Build the list of mounts to attach to the VM. - var containerMounts = [modifiedRootfs] + fileMountContext.transformedMounts - if let writableLayer = self.writableLayer { - containerMounts.insert(writableLayer, at: 1) - } + // Build the container's storage to attach to the VM. + let containerStorage = ContainerMounts( + rootfs: modifiedRootfs, + writableLayer: self.writableLayer, + mounts: fileMountContext.transformedMounts + ) let vmConfig = VMConfiguration( cpus: vmCpus, memoryInBytes: vmMemory, interfaces: self.interfaces, - mountsByID: [self.id: containerMounts], + storage: MachineMounts(containers: [self.id: containerStorage]), bootLog: self.config.bootLog, nestedVirtualization: self.config.virtualization ) @@ -643,7 +634,7 @@ extension LinuxContainer { do { try await vm.start() - let mountsForAgent = containerMounts + let storageForAgent = containerStorage try await vm.withAgent { agent in try await agent.standardSetup() @@ -656,7 +647,7 @@ extension LinuxContainer { // with zero shares), but the cloud-hypervisor backend // only spawns virtiofsd when shares exist; mounting an // unbacked tag fails with EINVAL. - let hasVirtiofsMount = mountsForAgent.contains { mount in + let hasVirtiofsMount = storageForAgent.all.contains { mount in if case .virtiofs = mount.runtimeOptions { return true } return false } @@ -671,7 +662,7 @@ extension LinuxContainer { // gets populated. if vm.virtiofsLayout == .perTag { try await agent.mkdir(path: "/run/virtiofs", all: true, perms: 0o755) - let virtiofsAttachments = (vm.mounts[self.id] ?? []).filter { $0.type == "virtiofs" } + let virtiofsAttachments = (vm.storage.containers[self.id]?.all ?? []).filter { $0.type == "virtiofs" } let uniqueTags = Set(virtiofsAttachments.map(\.source)) for tag in uniqueTags { let dest = "/run/virtiofs/\(tag)" @@ -695,18 +686,17 @@ extension LinuxContainer { } } - guard let attachments = vm.mounts[self.id] else { + guard let attached = vm.storage.containers[self.id] else { throw ContainerizationError(.notFound, message: "rootfs mount not found") } let rootfsPath = Self.guestRootfsPath(self.id) - try await self.mountRootfs(attachments: attachments, rootfsPath: rootfsPath, agent: agent) + try await self.mountRootfs(attached: attached, rootfsPath: rootfsPath, agent: agent) // Mount file mount holding directories under /run. if fileMountContext.hasFileMounts { - let containerMounts = vm.mounts[self.id] ?? [] var ctx = fileMountContextHolder.withLock { $0 } try await ctx.mountHoldingDirectories( - vmMounts: containerMounts, + vmMounts: attached.mounts, agent: agent ) fileMountContextHolder.withLock { $0 = ctx } @@ -764,15 +754,12 @@ extension LinuxContainer { let agent = try await createdState.vm.dialAgent() do { var spec = self.generateRuntimeSpec() - // We don't need the rootfs (or writable layer), nor do OCI runtimes want it included. - // Also filter out file mount holding directories. We'll mount those separately under /run. + // Filter out file mount holding directories. We'll mount those separately under /run. // Transform virtiofs mounts to bind mounts from /run/virtiofs/{tag} - let containerMounts = createdState.vm.mounts[self.id] ?? [] + let containerMounts = createdState.vm.storage.containers[self.id]?.mounts ?? [] let holdingTags = createdState.fileMountContext.holdingDirectoryTags - // Drop rootfs, and writable layer if present. - let mountsToSkip = self.writableLayer != nil ? 2 : 1 var mounts: [ContainerizationOCI.Mount] = - containerMounts.dropFirst(mountsToSkip) + containerMounts .filter { !holdingTags.contains($0.source) } .map { attached -> ContainerizationOCI.Mount in if attached.type == "virtiofs" { diff --git a/Sources/Containerization/LinuxPod.swift b/Sources/Containerization/LinuxPod.swift index 6275a4d4..fa4d1785 100644 --- a/Sources/Containerization/LinuxPod.swift +++ b/Sources/Containerization/LinuxPod.swift @@ -446,24 +446,28 @@ extension LinuxPod { // the container's bind mounts from /run/virtiofs/ fail // with ENOENT. // - // Derive the tags from the additional mounts directly rather - // than from vm.mounts[id], so this is independent of the - // rootfs (which may be virtiofs or virtio-blk) and of mount - // ordering. The rootfs is mounted at /run/container//rootfs - // and is never consumed from /run/virtiofs. + // Derive the new tags from the additional mounts being added; + // the machine's storage names what is already shared. let newVirtiofsTags = try virtioFSMounts.map { try hashFilePath(path: $0.source) } if !newVirtiofsTags.isEmpty { try await agent.mkdir(path: "/run/virtiofs", all: true, perms: 0o755) if vm.virtiofsLayout == .perTag { // Tags already mounted in the guest at boot or by a - // prior hotplug (i.e. present on another container). - let alreadyMounted = Set( - vm.mounts - .filter { $0.key != id } - .values.flatMap { $0 } - .filter { $0.type == "virtiofs" } - .map { $0.source } - ) + // prior hotplug (i.e. present on another container or + // a machine volume). + let alreadyMounted: Set = { + var mounted = Set( + vm.storage.containers + .filter { $0.key != id } + .values.flatMap { $0.all } + .filter { $0.type == "virtiofs" } + .map { $0.source }) + mounted.formUnion( + vm.storage.volumes.values + .filter { $0.type == "virtiofs" } + .map { $0.source }) + return mounted + }() var seen: Set = [] for tag in newVirtiofsTags where !alreadyMounted.contains(tag) && seen.insert(tag).inserted { @@ -493,7 +497,7 @@ extension LinuxPod { } if fileMountContext.hasFileMounts { - let containerMounts = vm.mounts[id] ?? [] + let containerMounts = vm.storage.containers[id]?.mounts ?? [] try await updatedFileMountContext.mountHoldingDirectories( vmMounts: containerMounts, agent: agent @@ -557,20 +561,23 @@ extension LinuxPod { try await self.state.withLock { state in try state.phase.validateForCreate() - // Build mountsByID for all containers. + // Build the machine's storage from its containers. // Strip "ro" from rootfs options - we handle readonly via the OCI spec's // root.readonly field and remount in vmexec after setup is complete. // Use transformedMounts from fileMountContext (file mounts become directory shares). - var mountsByID: [String: [Mount]] = [:] + var machineStorage = MachineMounts() for (id, container) in state.containers { var modifiedRootfs = container.rootfs modifiedRootfs.options.removeAll(where: { $0 == "ro" }) - // Filter out shared mounts — those are handled separately as pod volume bind mounts. + // Shared mounts are handled separately as pod volume bind mounts. let containerMounts = container.fileMountContext.transformedMounts.filter { if case .shared = $0.runtimeOptions { return false } return true } - mountsByID[id] = [modifiedRootfs] + containerMounts + machineStorage.containers[id] = ContainerMounts( + rootfs: modifiedRootfs, + mounts: containerMounts + ) } // Validate pod volume names are unique. @@ -597,9 +604,8 @@ extension LinuxPod { } } } - let podVolumeMounts = self.config.volumes.map { $0.toMount() } - if !podVolumeMounts.isEmpty { - mountsByID[self.id] = podVolumeMounts + for volume in self.config.volumes { + machineStorage.volumes[volume.name] = volume.toMount() } // Capture into an immutable `let` so the value is safely usable @@ -608,18 +614,16 @@ extension LinuxPod { // only attaches a virtiofs device when shares are configured, // so mounting an unbacked /run/virtiofs would fail with EINVAL // on the CH backend. - let hasVirtiofsMount = mountsByID.values.contains { mounts in - mounts.contains { mount in - if case .virtiofs = mount.runtimeOptions { return true } - return false - } + let hasVirtiofsMount = machineStorage.ordered.contains { mount in + if case .virtiofs = mount.runtimeOptions { return true } + return false } var vmConfig = VMConfiguration( cpus: self.config.cpus, memoryInBytes: self.config.memoryInBytes, interfaces: self.config.interfaces, - mountsByID: mountsByID, + storage: machineStorage, bootLog: self.config.bootLog, nestedVirtualization: self.config.virtualization ) @@ -649,19 +653,17 @@ extension LinuxPod { // /run/virtiofs/. See LinuxContainer for the // VZ vs. CH model split. var seenTags: Set = [] - for (_, attached) in vm.mounts { - for entry in attached where entry.type == "virtiofs" { - guard seenTags.insert(entry.source).inserted else { continue } - let dest = "/run/virtiofs/\(entry.source)" - try await agent.mkdir(path: dest, all: true, perms: 0o755) - try await agent.mount( - ContainerizationOCI.Mount( - type: "virtiofs", - source: entry.source, - destination: dest, - options: [] - )) - } + for entry in vm.storage.ordered where entry.type == "virtiofs" { + guard seenTags.insert(entry.source).inserted else { continue } + let dest = "/run/virtiofs/\(entry.source)" + try await agent.mkdir(path: dest, all: true, perms: 0o755) + try await agent.mount( + ContainerizationOCI.Mount( + type: "virtiofs", + source: entry.source, + destination: dest, + options: [] + )) } } else { try await agent.mount( @@ -728,10 +730,10 @@ extension LinuxPod { // Mount all container rootfs for (_, container) in containers { - guard let attachments = vm.mounts[container.id], let rootfsAttachment = attachments.first else { + guard let attached = vm.storage.containers[container.id] else { throw ContainerizationError(.notFound, message: "rootfs mount not found for container \(container.id)") } - var rootfs = rootfsAttachment.to + var rootfs = attached.rootfs.to rootfs.destination = Self.guestRootfsPath(container.id) try await agent.mount(rootfs) } @@ -740,7 +742,7 @@ extension LinuxPod { for (id, container) in containers { if container.fileMountContext.hasFileMounts { var ctx = container.fileMountContext - let containerMounts = vm.mounts[id] ?? [] + let containerMounts = vm.storage.containers[id]?.mounts ?? [] try await ctx.mountHoldingDirectories( vmMounts: containerMounts, agent: agent @@ -750,15 +752,13 @@ extension LinuxPod { } // Mount pod-level volumes. - let podVolumeAttachments = vm.mounts[self.id] ?? [] - for (index, volume) in self.config.volumes.enumerated() { - guard index < podVolumeAttachments.count else { + for volume in self.config.volumes { + guard let attachment = vm.storage.volumes[volume.name] else { throw ContainerizationError( .notFound, message: "attached filesystem not found for pod volume \"\(volume.name)\"" ) } - let attachment = podVolumeAttachments[index] let guestPath = Self.guestVolumePath(volume.name) try await agent.mount( ContainerizationOCI.Mount( @@ -864,10 +864,10 @@ extension LinuxPod { // We don't need the rootfs, nor do OCI runtimes want it included. // Also filter out file mount holding directories - we mount those separately under /run. // Transform virtiofs mounts to bind mounts from /run/virtiofs/{tag} - let containerMounts = createdState.vm.mounts[containerID] ?? [] + let containerMounts = createdState.vm.storage.containers[containerID]?.mounts ?? [] let holdingTags = container.fileMountContext.holdingDirectoryTags var mounts: [ContainerizationOCI.Mount] = - containerMounts.dropFirst() + containerMounts .filter { !holdingTags.contains($0.source) } .map { attached -> ContainerizationOCI.Mount in if attached.type == "virtiofs" { diff --git a/Sources/Containerization/VMConfiguration.swift b/Sources/Containerization/VMConfiguration.swift index 30faebc4..7a5e8e2c 100644 --- a/Sources/Containerization/VMConfiguration.swift +++ b/Sources/Containerization/VMConfiguration.swift @@ -72,9 +72,9 @@ public struct VMConfiguration: Sendable { public var memoryInBytes: UInt64 /// The network interfaces to attach. public var interfaces: [any Interface] - /// Mounts organized by metadata ID (e.g. container ID). - /// Each ID maps to an array of mounts for that workload. - public var mountsByID: [String: [Mount]] + /// The storage the machine carries: each container's mounts by role, + /// and the volumes its containers share. + public var storage: MachineMounts /// Optional destination for serial boot logs. public var bootLog: BootLog? /// Enable nested virtualization support. If the VirtualMachineManager @@ -89,14 +89,14 @@ public struct VMConfiguration: Sendable { cpus: Int = 4, memoryInBytes: UInt64 = 1024 * 1024 * 1024, interfaces: [any Interface] = [], - mountsByID: [String: [Mount]] = [:], + storage: MachineMounts = MachineMounts(), bootLog: BootLog? = nil, nestedVirtualization: Bool = false ) { self.cpus = cpus self.memoryInBytes = memoryInBytes self.interfaces = interfaces - self.mountsByID = mountsByID + self.storage = storage self.bootLog = bootLog self.nestedVirtualization = nestedVirtualization } diff --git a/Sources/Containerization/VZVirtualMachineInstance.swift b/Sources/Containerization/VZVirtualMachineInstance.swift index 160c5026..670ab512 100644 --- a/Sources/Containerization/VZVirtualMachineInstance.swift +++ b/Sources/Containerization/VZVirtualMachineInstance.swift @@ -28,10 +28,10 @@ import Synchronization public final class VZVirtualMachineInstance: Sendable { public typealias Agent = Vminitd - /// Attached mounts on the virtual machine, organized by metadata ID. - private let _mounts: Mutex<[String: [AttachedFilesystem]]> - public var mounts: [String: [AttachedFilesystem]] { - _mounts.withLock { $0 } + /// The machine's attached storage. + private let _storage: Mutex + public var storage: MachineAttachments { + _storage.withLock { $0 } } /// The underlying Virtualization framework virtual machine. @@ -40,9 +40,9 @@ public final class VZVirtualMachineInstance: Sendable { /// The dispatch queue used for VZ operations. public var vmQueue: DispatchQueue { queue } - /// Mutate the mount registry. - public func withMountRegistry(_ body: (inout sending [String: [AttachedFilesystem]]) throws -> sending T) rethrows -> T { - try _mounts.withLock(body) + /// Mutate the storage registry. + public func withStorage(_ body: (inout sending MachineAttachments) throws -> sending T) rethrows -> T { + try _storage.withLock(body) } /// Serialize VM operations with the instance lock. @@ -73,8 +73,9 @@ public final class VZVirtualMachineInstance: Sendable { public var rosetta: Bool /// Toggle nested virtualization support. public var nestedVirtualization: Bool - /// Mount attachments organized by metadata ID. - public var mountsByID: [String: [Mount]] + /// The machine's storage: each container's mounts by role, and the + /// volumes its containers share. + public var storage: MachineMounts /// Network interface attachments. public var interfaces: [any Interface] /// Kernel image. @@ -91,7 +92,7 @@ public final class VZVirtualMachineInstance: Sendable { self.memoryInBytes = 1024.mib() self.rosetta = false self.nestedVirtualization = false - self.mountsByID = [:] + self.storage = MachineMounts() self.interfaces = [] } } @@ -132,7 +133,7 @@ public final class VZVirtualMachineInstance: Sendable { let allocator = Character.blockDeviceTagAllocator() let (mountAttachments, _) = try config.mountAttachments(allocator: allocator) - self._mounts = Mutex(mountAttachments) + self._storage = Mutex(mountAttachments) self.vm = VZVirtualMachine( configuration: try config.toVZ(allocator: allocator), @@ -153,7 +154,7 @@ public protocol VZInstanceExtension: Sendable { _ config: inout VZVirtualMachineConfiguration, allocator: any AddressAllocator, storageDeviceCount: Int, - mountsByID: [String: [Mount]] + storage: MachineMounts ) throws /// Called after the VZVirtualMachine is created but before start. @@ -168,7 +169,7 @@ extension VZInstanceExtension { _ config: inout VZVirtualMachineConfiguration, allocator: any AddressAllocator, storageDeviceCount: Int, - mountsByID: [String: [Mount]] + storage: MachineMounts ) throws {} public func didCreate(_ instance: VZVirtualMachineInstance) throws {} @@ -477,35 +478,33 @@ extension VZVirtualMachineInstance.Configuration { // Track used virtiofs tags to avoid creating duplicate VZ devices. // The same source directory mounted to multiple destinations shares one device. + // The walk is the machine's device order, matching the addresses + // `mountAttachments` hands out walking the same way. var usedVirtioFSTags: Set = [] - for (_, mounts) in self.mountsByID { - for mount in mounts { - if case .virtiofs = mount.runtimeOptions { - let tag = try hashFilePath(path: mount.source) - if usedVirtioFSTags.contains(tag) { - continue - } - usedVirtioFSTags.insert(tag) + for mount in self.storage.ordered { + if case .virtiofs = mount.runtimeOptions { + let tag = try hashFilePath(path: mount.source) + if usedVirtioFSTags.contains(tag) { + continue } - try mount.configure(config: &config) + usedVirtioFSTags.insert(tag) } + try mount.configure(config: &config) } // Create the unified virtiofs device with VZMultipleDirectoryShare // This device hosts all virtiofs shares and supports runtime updates var directories: [String: VZSharedDirectory] = [:] - for (_, mounts) in self.mountsByID { - for mount in mounts { - guard case .virtiofs(_) = mount.runtimeOptions else { continue } - guard FileManager.default.fileExists(atPath: mount.source) else { - throw ContainerizationError(.notFound, message: "directory \(mount.source) does not exist") - } - let name = try hashFilePath(path: mount.source) - directories[name] = VZSharedDirectory( - url: URL(fileURLWithPath: mount.source), - readOnly: mount.options.contains("ro") - ) + for mount in self.storage.ordered { + guard case .virtiofs(_) = mount.runtimeOptions else { continue } + guard FileManager.default.fileExists(atPath: mount.source) else { + throw ContainerizationError(.notFound, message: "directory \(mount.source) does not exist") } + let name = try hashFilePath(path: mount.source) + directories[name] = VZSharedDirectory( + url: URL(fileURLWithPath: mount.source), + readOnly: mount.options.contains("ro") + ) } let multiShare = VZMultipleDirectoryShare(directories: directories) let virtiofsDevice = VZVirtioFileSystemDeviceConfiguration(tag: "virtiofs") @@ -527,7 +526,7 @@ extension VZVirtualMachineInstance.Configuration { config.platform = platform for ext in self.extensions.compactMap({ $0 as? any VZInstanceExtension }) { - try ext.configureVZ(&config, allocator: allocator, storageDeviceCount: storageDeviceCount, mountsByID: self.mountsByID) + try ext.configureVZ(&config, allocator: allocator, storageDeviceCount: storageDeviceCount, storage: self.storage) } try config.validate() @@ -535,7 +534,7 @@ extension VZVirtualMachineInstance.Configuration { } func mountAttachments(allocator: any AddressAllocator) throws -> ( - attachments: [String: [AttachedFilesystem]], storageDeviceCount: Int + attachments: MachineAttachments, storageDeviceCount: Int ) { var storageDeviceCount = 0 @@ -548,21 +547,28 @@ extension VZVirtualMachineInstance.Configuration { } } - var attachmentsByID: [String: [AttachedFilesystem]] = [:] - - for (id, mounts) in self.mountsByID { - var attachments: [AttachedFilesystem] = [] - for mount in mounts { - let attached = try AttachedFilesystem(mount: mount, allocator: allocator) - attachments.append(attached) - if mount.isBlock { - storageDeviceCount += 1 - } + // The machine's device order: addresses are handed out in the same + // walk `makeConfiguration` creates the devices in. + func attach(_ mount: Mount) throws -> AttachedFilesystem { + let attached = try AttachedFilesystem(mount: mount, allocator: allocator) + if mount.isBlock { + storageDeviceCount += 1 } - attachmentsByID[id] = attachments + return attached + } + + var containers: [String: ContainerAttachments] = [:] + for id in self.storage.containers.keys.sorted() { + guard let container = self.storage.containers[id] else { continue } + containers[id] = try container.map(attach) + } + var volumes: [String: AttachedFilesystem] = [:] + for name in self.storage.volumes.keys.sorted() { + guard let mount = self.storage.volumes[name] else { continue } + volumes[name] = try attach(mount) } - return (attachmentsByID, storageDeviceCount) + return (MachineAttachments(containers: containers, volumes: volumes), storageDeviceCount) } } diff --git a/Sources/Containerization/VZVirtualMachineManager.swift b/Sources/Containerization/VZVirtualMachineManager.swift index 4959bee4..d0888447 100644 --- a/Sources/Containerization/VZVirtualMachineManager.swift +++ b/Sources/Containerization/VZVirtualMachineManager.swift @@ -76,7 +76,7 @@ public struct VZVirtualMachineManager: VirtualMachineManager { instanceConfig.rosetta = self.rosetta instanceConfig.nestedVirtualization = useNestedVirtualization - instanceConfig.mountsByID = vmConfig.mountsByID + instanceConfig.storage = vmConfig.storage instanceConfig.extensions = vmConfig.extensions }) } diff --git a/Sources/Containerization/VirtualMachineInstance.swift b/Sources/Containerization/VirtualMachineInstance.swift index 302e97ae..b9340612 100644 --- a/Sources/Containerization/VirtualMachineInstance.swift +++ b/Sources/Containerization/VirtualMachineInstance.swift @@ -44,7 +44,8 @@ public protocol VirtualMachineInstance: Sendable { // The state of the virtual machine. var state: VirtualMachineInstanceState { get } - var mounts: [String: [AttachedFilesystem]] { get } + /// The machine's attached storage. + var storage: MachineAttachments { get } /// How this VMM exposes virtiofs devices to the guest. Defaults to /// `.unified` (the VZ-shaped behavior); CH overrides to `.perTag`. @@ -73,7 +74,7 @@ public protocol VirtualMachineInstance: Sendable { func hotplug(_ block: Mount, id: String) async throws -> AttachedFilesystem /// Register mounts for a container after hotplug. - /// This is used to add the rootfs and additional mounts to the VM's mount registry + /// This is used to add the rootfs and additional mounts to the machine's storage /// so they can be found when building the container's OCI spec. /// - Parameter id: The container ID /// - Parameter rootfs: The rootfs attachment from hotplug diff --git a/Sources/Integration/PodTests.swift b/Sources/Integration/PodTests.swift index ae1caec8..7dff20fa 100644 --- a/Sources/Integration/PodTests.swift +++ b/Sources/Integration/PodTests.swift @@ -2228,10 +2228,9 @@ extension IntegrationSuite { /// (directory-share) rootfs. Assumes a single-layer image (the alpine /// image used by the suite) so no OCI whiteout processing is required. /// - /// The extracted dir lives under `Self.testDir`; do NOT `defer`-remove it - /// here — virtiofsd shares it for the whole test. It is swept by - /// `bootstrap`'s `maxConcurrency == 1` reaper on the next test and by the - /// suite-end `removeItem(at: Self.testDir)`. + /// The extracted dir lives under the test's scratch directory; do NOT + /// `defer`-remove it here — virtiofsd shares it for the whole test. The + /// runner removes the scratch directory when the test finishes. private func unpackRootfsDirectory(_ image: Containerization.Image, testID: String) async throws -> Containerization.Mount { let dir = Self.testDir.appending(component: "\(testID)-rootfs-dir") try? FileManager.default.removeItem(at: dir) diff --git a/Sources/Integration/Suite.swift b/Sources/Integration/Suite.swift index 9dae8435..85e623c9 100644 --- a/Sources/Integration/Suite.swift +++ b/Sources/Integration/Suite.swift @@ -131,11 +131,35 @@ struct IntegrationSuite: AsyncParsableCommand { ) }() + /// Where a run keeps everything it makes. + /// + /// Runs gather under one directory rather than each taking a name of its + /// own among the system's temporary files, so that what a run leaves behind + /// can be swept without knowing which run left it. `make cleantests` sweeps + /// this same directory, so the name is shared between the two. + static let testRootName = "containerization-integration" + static let _testDir: URL = { - FileManager.default.uniqueTemporaryDirectory(create: true) + let root = FileManager.default.temporaryDirectory + .appendingPathComponent(testRootName) + let run = root.appendingPathComponent(UUID().uuidString) + try? FileManager.default.createDirectory(at: run, withIntermediateDirectories: true) + return run }() + /// The scratch directory of the test running on this task. The runner + /// creates one per test and deletes it when the test finishes, so a + /// test's clones and layers hold disk only while it runs. + @TaskLocal static var currentTestDir: URL? + static var testDir: URL { + currentTestDir ?? _testDir + } + + /// The run-shared directory holding unpacked images, which every test's + /// bootstrap clones from. It lives for the whole run and is removed at + /// the end. + static var imageCacheDir: URL { _testDir } @@ -233,7 +257,7 @@ struct IntegrationSuite: AsyncParsableCommand { let platform = Platform(arch: "arm64", os: "linux", variant: "v8") // Unpack to shared location with coordination to prevent concurrent unpacks - let fsPath = Self.testDir.appending(component: image.digest) + let fsPath = Self.imageCacheDir.appending(component: image.digest) let fs = try await Self.unpackCoordinator.unpack(key: fsPath.absolutePath()) { do { let unpacker = EXT4Unpacker(capacityInBytes: 2.gib()) @@ -251,25 +275,6 @@ struct IntegrationSuite: AsyncParsableCommand { } } - // Reap any per-test artifacts left over from prior tests. With - // `--max-concurrency 1` (linux-integration default) this runs after - // the previous test has fully completed, so it's race-free; on - // macOS where tests can run in parallel we just keep all files — - // disk usage isn't a concern there. Each per-test bootstrap clones - // a ~2GB rootfs and a ~512MB initfs, so without reaping the dev - // container fills its CoW layer in ~10 tests. - if self.maxConcurrency == 1 { - let preserve = fsPath.absolutePath() - if let entries = try? FileManager.default.contentsOfDirectory( - at: Self.testDir, - includingPropertiesForKeys: nil - ) { - for url in entries where url.absolutePath() != preserve { - try? FileManager.default.removeItem(at: url) - } - } - } - // Clone to test-specific path let clPath = Self.testDir.appending(component: "\(testID).ext4").absolutePath() try? FileManager.default.removeItem(atPath: clPath) @@ -415,6 +420,13 @@ struct IntegrationSuite: AsyncParsableCommand { // Hopefully this improves over time. func run() async throws { try Self.adjustLimits() + + // Tests reach for a temporary directory of their own freely and none of + // them take it away again. Pointing the temporary directory at the + // suite's own puts whatever they make where the removal at the end of + // the run already reaches, and takes any a test adds later with it. + setenv("TMPDIR", Self.testDir.absolutePath(), 1) + let suiteStarted = Date().timeIntervalSinceReferenceDate log.info("starting integration suite\n") @@ -654,11 +666,14 @@ struct IntegrationSuite: AsyncParsableCommand { for _ in 0..