Run docker inside a container, rootless and rootful

Makefile

@@ -64,7 +64,7 @@ LIBARCHIVE_UPSTREAM_REPO := https://github.com/libarchive/libarchive
 LIBARCHIVE_UPSTREAM_VERSION := v3.7.7
 LIBARCHIVE_LOCAL_DIR := workdir/libarchive
 
-KATA_BINARY_PACKAGE := https://github.com/kata-containers/kata-containers/releases/download/3.17.0/kata-static-3.17.0-arm64.tar.xz
+KATA_BINARY_PACKAGE := https://github.com/kata-containers/kata-containers/releases/download/3.28.0/kata-static-3.28.0-arm64.tar.zst
 CLOUD_HYPERVISOR_URL := https://github.com/cloud-hypervisor/cloud-hypervisor/releases/download/v52.0/cloud-hypervisor-static-aarch64
 # SHA256 of the v52.0 aarch64 static binary (verified locally from the
 # upstream release artifact). Bump alongside CLOUD_HYPERVISOR_URL.
@@ -410,11 +410,11 @@ integration:
 .PHONY: fetch-default-kernel
 fetch-default-kernel:
 	@mkdir -p .local/ bin/
-ifeq (,$(wildcard .local/kata.tar.gz))
-	@curl -SsL -o .local/kata.tar.gz ${KATA_BINARY_PACKAGE}
+ifeq (,$(wildcard .local/kata.tar))
+	@curl -SsL -o .local/kata.tar ${KATA_BINARY_PACKAGE}
 endif
 ifeq (,$(wildcard .local/vmlinux-$(KERNEL_ARCH)))
-	@tar -zxf .local/kata.tar.gz -C .local/ --strip-components=1
+	@tar -xf .local/kata.tar -C .local/ --strip-components=1
 	@cp -L .local/opt/kata/share/kata-containers/vmlinux.container .local/vmlinux-$(KERNEL_ARCH)
 endif
 ifeq (,$(wildcard bin/vmlinux-$(KERNEL_ARCH)))

README.md

@@ -130,6 +130,16 @@ After building, run basic and integration tests:
 make test integration
 ```
 
+Tests that only apply to Linux are compiled out on macOS, so `make test` passes
+without running them. Run those with:
+
+```bash
+make linux-test
+```
+
+which runs the same tests inside the Linux dev container, as the Linux build
+workflow does.
+
 A kernel is required to run integration tests.
 If you do not have a kernel locally, a default kernel can be fetched using the `make fetch-default-kernel` target.
 

Sources/CShim/include/swap.h

@@ -0,0 +1,36 @@
+/*
+ * Copyright © 2026 Apple Inc. and the Containerization project authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *   https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef __SWAP_H
+#define __SWAP_H
+
+#if defined(__linux__)
+
+// swapon(2) lives in <sys/swap.h>, which Swift's glibc modulemap does not
+// carry, so it is reachable from Swift only through a wrapper.
+
+// Discard the whole area when it is enabled, and each page as it is freed.
+// These come from the same header, and SWAP_FLAG_DISCARD_PAGES has no UAPI
+// header of its own at all, so every consumer declares it; see util-linux
+// sys-utils/swapon.c.
+#define CZ_SWAP_DISCARD 0x10000
+#define CZ_SWAP_DISCARD_PAGES 0x40000
+
+int CZ_swapon(const char *path, int flags);
+
+#endif
+
+#endif

Sources/CShim/swap.c

@@ -0,0 +1,23 @@
+/*
+ * Copyright © 2026 Apple Inc. and the Containerization project authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *   https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifdef __linux__
+#include <sys/swap.h>
+
+#include "swap.h"
+
+int CZ_swapon(const char *path, int flags) { return swapon(path, flags); }
+#endif

Sources/Containerization/CHHotplugProvider.swift

@@ -27,8 +27,8 @@ import Synchronization
 ///
 /// Handles both block (`vm.add-disk`) and virtiofs (`vm.add-fs`, with one
 /// `virtiofsd` per unique source-hash tag) hotplug, plus the matching
-/// `vm.remove-device` teardown. Owns the per-VM mount registry so
-/// `CHVirtualMachineInstance.mounts` can forward to it.
+/// `vm.remove-device` teardown. Owns the machine's storage so
+/// `CHVirtualMachineInstance.storage` can forward to it.
 final class CHHotplugProvider: HotplugProvider {
     struct HotplugRecord: Sendable {
         let chDeviceId: String
@@ -50,7 +50,7 @@ final class CHHotplugProvider: HotplugProvider {
     private let workDir: URL
     private let virtiofsdBinaryOverride: URL?
     private let allocator: any AddressAllocator<Character>
-    private let _mounts: Mutex<[String: [AttachedFilesystem]]>
+    private let _storage: Mutex<MachineAttachments>
     private let _records: Mutex<[String: [HotplugRecord]]>
     private let _tags: Mutex<[String: VirtiofsdTagState]>
     /// Serializes per-tag virtiofsd spawn so a concurrent hotplug for the
@@ -65,14 +65,14 @@ final class CHHotplugProvider: HotplugProvider {
         workDir: URL,
         virtiofsdBinary: URL?,
         allocator: any AddressAllocator<Character>,
-        initialMounts: [String: [AttachedFilesystem]],
+        initialStorage: MachineAttachments,
         logger: Logger?
     ) {
         self.client = client
         self.workDir = workDir
         self.virtiofsdBinaryOverride = virtiofsdBinary
         self.allocator = allocator
-        self._mounts = Mutex(initialMounts)
+        self._storage = Mutex(initialStorage)
         self._records = Mutex([:])
         self._tags = Mutex([:])
         self.spawnLock = AsyncLock()
@@ -81,14 +81,14 @@ final class CHHotplugProvider: HotplugProvider {
 
     // MARK: - Read accessors
 
-    var mounts: [String: [AttachedFilesystem]] {
-        _mounts.withLock { $0 }
+    var storage: MachineAttachments {
+        _storage.withLock { $0 }
     }
 
-    func withMountRegistry<T: Sendable>(
-        _ body: (inout sending [String: [AttachedFilesystem]]) throws -> sending T
+    func withStorage<T: Sendable>(
+        _ body: (inout sending MachineAttachments) throws -> sending T
     ) rethrows -> T {
-        try _mounts.withLock(body)
+        try _storage.withLock(body)
     }
 
     // MARK: - HotplugProvider conformance
@@ -150,13 +150,14 @@ final class CHHotplugProvider: HotplugProvider {
         }
     }
 
-    func registerMounts(id: String, rootfs: AttachedFilesystem, additionalMounts: [Mount]) throws {
-        var attached: [AttachedFilesystem] = [rootfs]
+    func registerMounts(id: String, rootfs: AttachedFilesystem, writableLayer: AttachedFilesystem?, additionalMounts: [Mount]) throws {
+        var mounts: [AttachedFilesystem] = []
         for mount in additionalMounts {
-            attached.append(try AttachedFilesystem(mount: mount, allocator: allocator))
+            mounts.append(try AttachedFilesystem(mount: mount, allocator: allocator))
         }
-        _mounts.withLock {
-            $0[id, default: []].append(contentsOf: attached)
+        let container = ContainerAttachments(rootfs: rootfs, writableLayer: writableLayer, mounts: mounts)
+        _storage.withLock {
+            $0.containers[id] = container
         }
     }
 
@@ -190,19 +191,10 @@ final class CHHotplugProvider: HotplugProvider {
             }
         }
 
-        // Drop block-derived AttachedFilesystem entries for `id`. Block entries
-        // are the ones whose source was rewritten to "/dev/vd<letter>" by
-        // `hotplug(_:)` (or by AttachedFilesystem(mount:allocator:) for an
-        // additionalMount of type virtio-blk).
-        _mounts.withLock { state in
-            guard var perID = state[id] else { return }
-            perID.removeAll { $0.source.hasPrefix("/dev/vd") }
-            if perID.isEmpty {
-                state.removeValue(forKey: id)
-            } else {
-                state[id] = perID
-            }
-        }
+        // The container's devices are gone, so the container leaves the
+        // registry with them; its shares are released separately and their
+        // processes reference-counted through `_tags`.
+        _ = _storage.withLock { $0.containers.removeValue(forKey: id) }
     }
 
     func hotplugVirtioFS(_ mounts: [Mount], id: String) async throws {
@@ -226,7 +218,7 @@ final class CHHotplugProvider: HotplugProvider {
             let chDeviceId = try await ensureVirtiofsDevice(tag: tag, source: source, readonly: readonly)
             // Record once per tag for this container. The AttachedFilesystem
             // entries for these mounts are written by registerMounts (the sole
-            // _mounts writer), so we do NOT touch _mounts here.
+            // registry writer), so we do NOT touch the storage here.
             _records.withLock {
                 $0[id, default: []].append(HotplugRecord(chDeviceId: chDeviceId, kind: .virtiofs(tag: tag)))
             }
@@ -338,16 +330,17 @@ final class CHHotplugProvider: HotplugProvider {
             try? FileManager.default.removeItem(at: socket)
         }
 
-        // Drop virtiofs AttachedFilesystem entries for `id`. AttachedFilesystem
-        // sets `type = mount.type` which for a `.virtiofs` mount is "virtiofs".
-        _mounts.withLock { state in
-            guard var perID = state[id] else { return }
-            perID.removeAll { $0.type == "virtiofs" }
-            if perID.isEmpty {
-                state.removeValue(forKey: id)
-            } else {
-                state[id] = perID
+        // Drop the container's virtiofs entries. A container whose rootfs is
+        // itself a share leaves the registry whole; one that keeps block
+        // devices keeps its entry with the share entries dropped.
+        _storage.withLock { state in
+            guard var container = state.containers[id] else { return }
+            if container.rootfs.type == "virtiofs" {
+                state.containers.removeValue(forKey: id)
+                return
             }
+            container.mounts.removeAll { $0.type == "virtiofs" }
+            state.containers[id] = container
         }
     }
 
@@ -358,15 +351,21 @@ final class CHHotplugProvider: HotplugProvider {
     /// is the user-supplied `FsConfig.id` (which `vm.remove-device` keys on).
     /// `ownerIds` are the container ids that count toward this tag's refcount;
     /// each gets a `HotplugRecord` so `releaseVirtioFS(id:)` walks them
-    /// uniformly.
+    /// uniformly. `machineHeld` adds one reference nothing releases, for a
+    /// share the machine itself owns (a volume).
     func recordBootTimeVirtiofs(
         tag: String,
         process: VirtiofsdProcess,
         chDeviceId: String,
-        ownerIds: [String]
+        ownerIds: [String],
+        machineHeld: Bool
     ) {
         _tags.withLock {
-            $0[tag] = VirtiofsdTagState(process: process, refcount: ownerIds.count, chDeviceId: chDeviceId)
+            $0[tag] = VirtiofsdTagState(
+                process: process,
+                refcount: ownerIds.count + (machineHeld ? 1 : 0),
+                chDeviceId: chDeviceId
+            )
         }
         _records.withLock { records in
             for id in ownerIds {