From f10eaca613f6eef28add5d69f4fbf573ca6ed156 Mon Sep 17 00:00:00 2001 From: Aaron Paterson Date: Sat, 1 Aug 2026 18:48:34 +0000 Subject: [PATCH 1/4] Resolve test directory paths before reaping per-test files The reaper that runs at a concurrency of one deletes everything in the test directory except the unpacked rootfs it means to preserve. It held that rootfs path as FileManager reports it, under /var, and compared it against the entries of contentsOfDirectory, which reports them under /private/var, so the preserved path never matched any entry and the rootfs was deleted along with the per-test files. The unpack coordinator still held it as unpacked, so the next test opened a rootfs that was no longer there and failed with a missing file error. Resolve both sides with resolvingSymlinksInPathWithPrivate, which exists for this difference between the two views of the same directory. --- Sources/Integration/Suite.swift | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/Sources/Integration/Suite.swift b/Sources/Integration/Suite.swift index 9dae8435..e43cf372 100644 --- a/Sources/Integration/Suite.swift +++ b/Sources/Integration/Suite.swift @@ -259,12 +259,16 @@ struct IntegrationSuite: AsyncParsableCommand { // a ~2GB rootfs and a ~512MB initfs, so without reaping the dev // container fills its CoW layer in ~10 tests. if self.maxConcurrency == 1 { - let preserve = fsPath.absolutePath() + // contentsOfDirectory reports paths under /private, while testDir is + // built from FileManager's /var view of the same directory, so both + // sides are resolved before comparing. + let preserve = fsPath.resolvingSymlinksInPathWithPrivate().absolutePath() if let entries = try? FileManager.default.contentsOfDirectory( at: Self.testDir, includingPropertiesForKeys: nil ) { - for url in entries where url.absolutePath() != preserve { + for url in entries + where url.resolvingSymlinksInPathWithPrivate().absolutePath() != preserve { try? FileManager.default.removeItem(at: url) } } From 6499f7dfa7fff64c903642db690094d7e6c64ec8 Mon Sep 17 00:00:00 2001 From: Aaron Paterson Date: Sun, 2 Aug 2026 00:51:10 +0000 Subject: [PATCH 2/4] Say how to run the tests that macOS skips Tests guarded for Linux are compiled out on macOS, so `make test` reports success without having run them, and nothing says so. The target that does run them is not mentioned anywhere outside the makefile. --- README.md | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/README.md b/README.md index f658b08d..372a872a 100644 --- a/README.md +++ b/README.md @@ -130,6 +130,16 @@ After building, run basic and integration tests: make test integration ``` +Tests that only apply to Linux are compiled out on macOS, so `make test` passes +without running them. Run those with: + +```bash +make linux-test +``` + +which runs the same tests inside the Linux dev container, as the Linux build +workflow does. + A kernel is required to run integration tests. If you do not have a kernel locally, a default kernel can be fetched using the `make fetch-default-kernel` target. From bf4d7dcf8bbe64c745e9ef344a8c173d2a42a3af Mon Sep 17 00:00:00 2001 From: Aaron Paterson Date: Sun, 2 Aug 2026 01:20:29 +0000 Subject: [PATCH 3/4] Test against the kernel release the shipped tool installs The default kernel the tests fetch had drifted from the one users run. container installs kata 3.28.0 and its 6.18.15 kernel, while these tests fetched 3.17.0, so the suite exercised a guest with a different feature set to the one it is meant to represent. Nested runtimes are the visible case: 3.17.0 was built without nf_tables, so a docker daemon inside a container fails there and works on what ships. Kata moved from xz to zstd between those releases, so the archive is no longer named for its compression and tar is left to recognise it rather than being told, which also holds if the format changes again. --- Makefile | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Makefile b/Makefile index c79d6995..3d01b93a 100644 --- a/Makefile +++ b/Makefile @@ -64,7 +64,7 @@ LIBARCHIVE_UPSTREAM_REPO := https://github.com/libarchive/libarchive LIBARCHIVE_UPSTREAM_VERSION := v3.7.7 LIBARCHIVE_LOCAL_DIR := workdir/libarchive -KATA_BINARY_PACKAGE := https://github.com/kata-containers/kata-containers/releases/download/3.17.0/kata-static-3.17.0-arm64.tar.xz +KATA_BINARY_PACKAGE := https://github.com/kata-containers/kata-containers/releases/download/3.28.0/kata-static-3.28.0-arm64.tar.zst CLOUD_HYPERVISOR_URL := https://github.com/cloud-hypervisor/cloud-hypervisor/releases/download/v52.0/cloud-hypervisor-static-aarch64 # SHA256 of the v52.0 aarch64 static binary (verified locally from the # upstream release artifact). Bump alongside CLOUD_HYPERVISOR_URL. @@ -410,11 +410,11 @@ integration: .PHONY: fetch-default-kernel fetch-default-kernel: @mkdir -p .local/ bin/ -ifeq (,$(wildcard .local/kata.tar.gz)) - @curl -SsL -o .local/kata.tar.gz ${KATA_BINARY_PACKAGE} +ifeq (,$(wildcard .local/kata.tar)) + @curl -SsL -o .local/kata.tar ${KATA_BINARY_PACKAGE} endif ifeq (,$(wildcard .local/vmlinux-$(KERNEL_ARCH))) - @tar -zxf .local/kata.tar.gz -C .local/ --strip-components=1 + @tar -xf .local/kata.tar -C .local/ --strip-components=1 @cp -L .local/opt/kata/share/kata-containers/vmlinux.container .local/vmlinux-$(KERNEL_ARCH) endif ifeq (,$(wildcard bin/vmlinux-$(KERNEL_ARCH))) From 2086043dc79eab6ba3f237b4ff2283eae1a91421 Mon Sep 17 00:00:00 2001 From: Aaron Paterson Date: Sun, 2 Aug 2026 06:00:51 +0000 Subject: [PATCH 4/4] Let a service manager run as a container's init An image built around a service manager expects to own the runtime state it keeps, and finds nowhere to write it: a container gets its image's filesystem and the mounts a single process needs, so the manager cannot start and takes the container down with it. Give it somewhere to write, which is what nerdctl and podman give such a container, and take the same paths to mean one is being asked for. The container says which of the three it wants: never, whenever the process it starts is a service manager, or whatever it starts. Never is the default, as it is for the runtimes that create cgroups themselves. A mount the caller supplied for one of those places is left as they asked for it, and a pod's containers ask the same way. --- Sources/Containerization/LinuxContainer.swift | 81 ++++++++++++++++++- Sources/Containerization/LinuxPod.swift | 8 +- Sources/Integration/ContainerTests.swift | 42 ++++++++++ Sources/Integration/Suite.swift | 10 ++- 4 files changed, 136 insertions(+), 5 deletions(-) diff --git a/Sources/Containerization/LinuxContainer.swift b/Sources/Containerization/LinuxContainer.swift index 34964fcd..61512dc6 100644 --- a/Sources/Containerization/LinuxContainer.swift +++ b/Sources/Containerization/LinuxContainer.swift @@ -26,6 +26,17 @@ import SystemPackage import struct ContainerizationOS.Terminal +/// When a container is prepared for a service manager to run as its init. +public enum SystemdMode: Sendable { + /// Never, which is what a container that runs a single process wants. + case disabled + /// When the process the container starts is one, by the paths that + /// nerdctl and podman both take to mean a service manager. + case detected + /// Whatever the container starts. + case always +} + /// `LinuxContainer` is an easy to use type for launching and managing the /// full lifecycle of a Linux container ran inside of a virtual machine. public final class LinuxContainer: Container, Sendable { @@ -91,6 +102,9 @@ public final class LinuxContainer: Container, Sendable { /// Run the container with a minimal init process that handles signal /// forwarding and zombie reaping. public var useInit: Bool = false + /// Prepare the container for a service manager to run as its init, + /// which needs somewhere writable to keep runtime state. + public var systemd: SystemdMode = .disabled /// Additional CPU cores to allocate for the virtual machine on top /// of the container's configured `cpus` value. public var cpuOverhead: Int = 1 @@ -454,6 +468,68 @@ public final class LinuxContainer: Container, Sendable { ] } + /// Whether a service manager is to run as a container's init, which is + /// either asked for outright or taken from the process the container + /// starts, as nerdctl and podman both decide it. + static func runsSystemd(_ systemd: SystemdMode, arguments: [String]) -> Bool { + switch systemd { + case .disabled: + return false + case .always: + return true + case .detected: + guard let command = arguments.first else { + return false + } + return Self.systemdPaths().contains(command) + } + } + + /// A container's mounts, plus somewhere writable for a service manager to + /// keep its runtime state when one is to run as the container's init. A + /// mount the caller supplied for the same place is left as they asked. + static func systemdAwareMounts( + _ mounts: [Mount], systemd: SystemdMode, arguments: [String] + ) -> [Mount] { + guard Self.runsSystemd(systemd, arguments: arguments) else { + return mounts + } + let asked = Set(mounts.map { $0.destination }) + return mounts + Self.systemdMounts().filter { !asked.contains($0.destination) } + } + + /// The paths a container's process is taken to be a service manager by, + /// which is what nerdctl and podman look for when told to detect one. + /// https://github.com/containerd/nerdctl/blob/main/pkg/cmd/container/create.go + public static func systemdPaths() -> [String] { + [ + "/sbin/init", + "/usr/sbin/init", + "/usr/local/sbin/init", + ] + } + + /// Somewhere writable for a service manager to keep the runtime state it + /// expects to own, which is what podman gives one. + /// https://github.com/containers/podman/blob/main/libpod/container_internal_linux.go + /// + /// The journal directory is the one journald reads. Persistent logs go to + /// `/var/log/journal` and volatile ones to `/run/log/journal`, so those are + /// the two paths worth making writable and `/var/log/journal` is the one a + /// tmpfs belongs on. + /// https://www.freedesktop.org/software/systemd/man/latest/journald.conf.html + /// + /// Two things that belong with these are seen to elsewhere. `/sys/fs/cgroup` + /// is mounted writable by ``defaultMounts()``, which is what podman gives a + /// service manager alongside its own list, and the `SIGRTMIN+3` one takes as + /// the request to shut down is carried on the image configuration here, with + /// nothing on the runtime side to set it. + public static func systemdMounts() -> [Mount] { + ["/run", "/run/lock", "/tmp", "/var/log/journal"].map { + .any(type: "tmpfs", source: "tmpfs", destination: $0) + } + } + /// The default set of paths to mask inside a container, matching the OCI /// runtime spec defaults that runc and other production runtimes apply. /// Each path is hidden from the workload (replaced by `/dev/null` for files @@ -619,7 +695,10 @@ extension LinuxContainer { let vmCpus = self.cpus + self.config.cpuOverhead // Prepare file mounts. This transforms single-file mounts into directory shares. - let fileMountContext = try FileMountContext.prepare(mounts: self.config.mounts) + let fileMountContext = try FileMountContext.prepare( + mounts: Self.systemdAwareMounts( + self.config.mounts, systemd: self.config.systemd, + arguments: self.config.process.arguments)) // This is dumb, but alas. let fileMountContextHolder = Mutex(fileMountContext) diff --git a/Sources/Containerization/LinuxPod.swift b/Sources/Containerization/LinuxPod.swift index 6275a4d4..5e9694b7 100644 --- a/Sources/Containerization/LinuxPod.swift +++ b/Sources/Containerization/LinuxPod.swift @@ -83,6 +83,9 @@ public final class LinuxPod: Sendable { public var sysctl: [String: String] = [:] /// The mounts for the container. public var mounts: [Mount] = LinuxContainer.defaultMounts() + /// Prepare this container for a service manager to run as its init, + /// which needs somewhere writable to keep runtime state. + public var systemd: SystemdMode = .disabled /// Paths inside the container that vmexec hides from the workload. /// Defaults to the OCI standard set (``LinuxContainer/defaultMaskedPaths()``), /// matching the restricted capability baseline. Set to `[]` to opt out, @@ -390,7 +393,10 @@ extension LinuxPod { var config = ContainerConfiguration() try configuration(&config) - let fileMountContext = try FileMountContext.prepare(mounts: config.mounts) + let fileMountContext = try FileMountContext.prepare( + mounts: LinuxContainer.systemdAwareMounts( + config.mounts, systemd: config.systemd, + arguments: config.process.arguments)) switch state.phase { case .initialized: diff --git a/Sources/Integration/ContainerTests.swift b/Sources/Integration/ContainerTests.swift index dd2a9165..219abed9 100644 --- a/Sources/Integration/ContainerTests.swift +++ b/Sources/Integration/ContainerTests.swift @@ -137,6 +137,48 @@ extension IntegrationSuite { } } + /// A service manager running as the container's init, which is what an + /// image built around one expects, and which needs somewhere writable to + /// keep the runtime state it owns. + func testContainerSystemd() async throws { + let id = "test-container-systemd" + let bs = try await bootstrap( + id, reference: "docker.io/library/almalinux:9", capacityInBytes: 8.gib()) + + let container = try LinuxContainer(id, rootfs: bs.rootfs, vmm: bs.vmm) { config in + // Asked for by the process the container starts, rather than outright. + config.systemd = .detected + config.process.arguments = ["/usr/sbin/init"] + config.bootLog = bs.bootLog + } + try await container.create() + try await container.start() + try await Task.sleep(for: .seconds(30)) + + // It logs to the console rather than to us, so ask it from inside. + let buffer = BufferWriter() + let query = try await container.exec("state") { config in + config.arguments = [ + "/bin/sh", "-c", + "echo PID1=$(cat /proc/1/comm); echo STATE=$(systemctl is-system-running)", + ] + config.stdout = buffer + config.stderr = buffer + } + try await query.start() + _ = try await query.wait() + try await query.delete() + try? await container.stop() + + let out = String(data: buffer.data, encoding: .utf8) ?? "" + guard out.contains("PID1=systemd") else { + throw IntegrationError.assert(msg: "systemd is not the container's init: '\(out)'") + } + guard out.contains("STATE=running") || out.contains("STATE=degraded") else { + throw IntegrationError.assert(msg: "systemd did not come up: '\(out)'") + } + } + func testProcessEchoHi() async throws { let id = "test-process-echo-hi" let bs = try await bootstrap(id) diff --git a/Sources/Integration/Suite.swift b/Sources/Integration/Suite.swift index e43cf372..600b05d3 100644 --- a/Sources/Integration/Suite.swift +++ b/Sources/Integration/Suite.swift @@ -195,8 +195,11 @@ struct IntegrationSuite: AsyncParsableCommand { static let eventLoop = MultiThreadedEventLoopGroup(numberOfThreads: System.coreCount) - func bootstrap(_ testID: String) async throws -> (rootfs: Containerization.Mount, vmm: VirtualMachineManager, image: Containerization.Image, bootLog: BootLog) { - let reference = "ghcr.io/linuxcontainers/alpine:3.20" + func bootstrap( + _ testID: String, + reference: String = "ghcr.io/linuxcontainers/alpine:3.20", + capacityInBytes: UInt64 = 2.gib() + ) async throws -> (rootfs: Containerization.Mount, vmm: VirtualMachineManager, image: Containerization.Image, bootLog: BootLog) { let store = Self.imageStore let initImage = try await store.getInitImage(reference: Self.initImage) @@ -236,7 +239,7 @@ struct IntegrationSuite: AsyncParsableCommand { let fsPath = Self.testDir.appending(component: image.digest) let fs = try await Self.unpackCoordinator.unpack(key: fsPath.absolutePath()) { do { - let unpacker = EXT4Unpacker(capacityInBytes: 2.gib()) + let unpacker = EXT4Unpacker(capacityInBytes: capacityInBytes) return try await unpacker.unpack(image, for: platform, at: fsPath) } catch let err as ContainerizationError { if err.code == .exists { @@ -426,6 +429,7 @@ struct IntegrationSuite: AsyncParsableCommand { // Process basics Test("process true", testProcessTrue), Test("process false", testProcessFalse), + Test("container systemd", testContainerSystemd), Test("process echo hi", testProcessEchoHi), Test("process no executable", testProcessNoExecutable), Test("process user", testProcessUser),