Let a service manager run as a container's init

Makefile

@@ -64,7 +64,7 @@ LIBARCHIVE_UPSTREAM_REPO := https://github.com/libarchive/libarchive
 LIBARCHIVE_UPSTREAM_VERSION := v3.7.7
 LIBARCHIVE_LOCAL_DIR := workdir/libarchive
 
-KATA_BINARY_PACKAGE := https://github.com/kata-containers/kata-containers/releases/download/3.17.0/kata-static-3.17.0-arm64.tar.xz
+KATA_BINARY_PACKAGE := https://github.com/kata-containers/kata-containers/releases/download/3.28.0/kata-static-3.28.0-arm64.tar.zst
 CLOUD_HYPERVISOR_URL := https://github.com/cloud-hypervisor/cloud-hypervisor/releases/download/v52.0/cloud-hypervisor-static-aarch64
 # SHA256 of the v52.0 aarch64 static binary (verified locally from the
 # upstream release artifact). Bump alongside CLOUD_HYPERVISOR_URL.
@@ -410,11 +410,11 @@ integration:
 .PHONY: fetch-default-kernel
 fetch-default-kernel:
 	@mkdir -p .local/ bin/
-ifeq (,$(wildcard .local/kata.tar.gz))
-	@curl -SsL -o .local/kata.tar.gz ${KATA_BINARY_PACKAGE}
+ifeq (,$(wildcard .local/kata.tar))
+	@curl -SsL -o .local/kata.tar ${KATA_BINARY_PACKAGE}
 endif
 ifeq (,$(wildcard .local/vmlinux-$(KERNEL_ARCH)))
-	@tar -zxf .local/kata.tar.gz -C .local/ --strip-components=1
+	@tar -xf .local/kata.tar -C .local/ --strip-components=1
 	@cp -L .local/opt/kata/share/kata-containers/vmlinux.container .local/vmlinux-$(KERNEL_ARCH)
 endif
 ifeq (,$(wildcard bin/vmlinux-$(KERNEL_ARCH)))

README.md

@@ -130,6 +130,16 @@ After building, run basic and integration tests:
 make test integration
 ```
 
+Tests that only apply to Linux are compiled out on macOS, so `make test` passes
+without running them. Run those with:
+
+```bash
+make linux-test
+```
+
+which runs the same tests inside the Linux dev container, as the Linux build
+workflow does.
+
 A kernel is required to run integration tests.
 If you do not have a kernel locally, a default kernel can be fetched using the `make fetch-default-kernel` target.
 

Sources/Containerization/LinuxContainer.swift

@@ -26,6 +26,17 @@ import SystemPackage
 
 import struct ContainerizationOS.Terminal
 
+/// When a container is prepared for a service manager to run as its init.
+public enum SystemdMode: Sendable {
+    /// Never, which is what a container that runs a single process wants.
+    case disabled
+    /// When the process the container starts is one, by the paths that
+    /// nerdctl and podman both take to mean a service manager.
+    case detected
+    /// Whatever the container starts.
+    case always
+}
+
 /// `LinuxContainer` is an easy to use type for launching and managing the
 /// full lifecycle of a Linux container ran inside of a virtual machine.
 public final class LinuxContainer: Container, Sendable {
@@ -91,6 +102,9 @@ public final class LinuxContainer: Container, Sendable {
         /// Run the container with a minimal init process that handles signal
         /// forwarding and zombie reaping.
         public var useInit: Bool = false
+        /// Prepare the container for a service manager to run as its init,
+        /// which needs somewhere writable to keep runtime state.
+        public var systemd: SystemdMode = .disabled
         /// Additional CPU cores to allocate for the virtual machine on top
         /// of the container's configured `cpus` value.
         public var cpuOverhead: Int = 1
@@ -454,6 +468,68 @@ public final class LinuxContainer: Container, Sendable {
         ]
     }
 
+    /// Whether a service manager is to run as a container's init, which is
+    /// either asked for outright or taken from the process the container
+    /// starts, as nerdctl and podman both decide it.
+    static func runsSystemd(_ systemd: SystemdMode, arguments: [String]) -> Bool {
+        switch systemd {
+        case .disabled:
+            return false
+        case .always:
+            return true
+        case .detected:
+            guard let command = arguments.first else {
+                return false
+            }
+            return Self.systemdPaths().contains(command)
+        }
+    }
+
+    /// A container's mounts, plus somewhere writable for a service manager to
+    /// keep its runtime state when one is to run as the container's init. A
+    /// mount the caller supplied for the same place is left as they asked.
+    static func systemdAwareMounts(
+        _ mounts: [Mount], systemd: SystemdMode, arguments: [String]
+    ) -> [Mount] {
+        guard Self.runsSystemd(systemd, arguments: arguments) else {
+            return mounts
+        }
+        let asked = Set(mounts.map { $0.destination })
+        return mounts + Self.systemdMounts().filter { !asked.contains($0.destination) }
+    }
+
+    /// The paths a container's process is taken to be a service manager by,
+    /// which is what nerdctl and podman look for when told to detect one.
+    /// https://github.com/containerd/nerdctl/blob/main/pkg/cmd/container/create.go
+    public static func systemdPaths() -> [String] {
+        [
+            "/sbin/init",
+            "/usr/sbin/init",
+            "/usr/local/sbin/init",
+        ]
+    }
+
+    /// Somewhere writable for a service manager to keep the runtime state it
+    /// expects to own, which is what podman gives one.
+    /// https://github.com/containers/podman/blob/main/libpod/container_internal_linux.go
+    ///
+    /// The journal directory is the one journald reads. Persistent logs go to
+    /// `/var/log/journal` and volatile ones to `/run/log/journal`, so those are
+    /// the two paths worth making writable and `/var/log/journal` is the one a
+    /// tmpfs belongs on.
+    /// https://www.freedesktop.org/software/systemd/man/latest/journald.conf.html
+    ///
+    /// Two things that belong with these are seen to elsewhere. `/sys/fs/cgroup`
+    /// is mounted writable by ``defaultMounts()``, which is what podman gives a
+    /// service manager alongside its own list, and the `SIGRTMIN+3` one takes as
+    /// the request to shut down is carried on the image configuration here, with
+    /// nothing on the runtime side to set it.
+    public static func systemdMounts() -> [Mount] {
+        ["/run", "/run/lock", "/tmp", "/var/log/journal"].map {
+            .any(type: "tmpfs", source: "tmpfs", destination: $0)
+        }
+    }
+
     /// The default set of paths to mask inside a container, matching the OCI
     /// runtime spec defaults that runc and other production runtimes apply.
     /// Each path is hidden from the workload (replaced by `/dev/null` for files
@@ -619,7 +695,10 @@ extension LinuxContainer {
             let vmCpus = self.cpus + self.config.cpuOverhead
 
             // Prepare file mounts. This transforms single-file mounts into directory shares.
-            let fileMountContext = try FileMountContext.prepare(mounts: self.config.mounts)
+            let fileMountContext = try FileMountContext.prepare(
+                mounts: Self.systemdAwareMounts(
+                    self.config.mounts, systemd: self.config.systemd,
+                    arguments: self.config.process.arguments))
             // This is dumb, but alas.
             let fileMountContextHolder = Mutex<FileMountContext>(fileMountContext)
 

Sources/Containerization/LinuxPod.swift

@@ -83,6 +83,9 @@ public final class LinuxPod: Sendable {
         public var sysctl: [String: String] = [:]
         /// The mounts for the container.
         public var mounts: [Mount] = LinuxContainer.defaultMounts()
+        /// Prepare this container for a service manager to run as its init,
+        /// which needs somewhere writable to keep runtime state.
+        public var systemd: SystemdMode = .disabled
         /// Paths inside the container that vmexec hides from the workload.
         /// Defaults to the OCI standard set (``LinuxContainer/defaultMaskedPaths()``),
         /// matching the restricted capability baseline. Set to `[]` to opt out,
@@ -390,7 +393,10 @@ extension LinuxPod {
             var config = ContainerConfiguration()
             try configuration(&config)
 
-            let fileMountContext = try FileMountContext.prepare(mounts: config.mounts)
+            let fileMountContext = try FileMountContext.prepare(
+                mounts: LinuxContainer.systemdAwareMounts(
+                    config.mounts, systemd: config.systemd,
+                    arguments: config.process.arguments))
 
             switch state.phase {
             case .initialized:

Sources/Integration/ContainerTests.swift

@@ -137,6 +137,48 @@ extension IntegrationSuite {
         }
     }
 
+    /// A service manager running as the container's init, which is what an
+    /// image built around one expects, and which needs somewhere writable to
+    /// keep the runtime state it owns.
+    func testContainerSystemd() async throws {
+        let id = "test-container-systemd"
+        let bs = try await bootstrap(
+            id, reference: "docker.io/library/almalinux:9", capacityInBytes: 8.gib())
+
+        let container = try LinuxContainer(id, rootfs: bs.rootfs, vmm: bs.vmm) { config in
+            // Asked for by the process the container starts, rather than outright.
+            config.systemd = .detected
+            config.process.arguments = ["/usr/sbin/init"]
+            config.bootLog = bs.bootLog
+        }
+        try await container.create()
+        try await container.start()
+        try await Task.sleep(for: .seconds(30))
+
+        // It logs to the console rather than to us, so ask it from inside.
+        let buffer = BufferWriter()
+        let query = try await container.exec("state") { config in
+            config.arguments = [
+                "/bin/sh", "-c",
+                "echo PID1=$(cat /proc/1/comm); echo STATE=$(systemctl is-system-running)",
+            ]
+            config.stdout = buffer
+            config.stderr = buffer
+        }
+        try await query.start()
+        _ = try await query.wait()
+        try await query.delete()
+        try? await container.stop()
+
+        let out = String(data: buffer.data, encoding: .utf8) ?? ""
+        guard out.contains("PID1=systemd") else {
+            throw IntegrationError.assert(msg: "systemd is not the container's init: '\(out)'")
+        }
+        guard out.contains("STATE=running") || out.contains("STATE=degraded") else {
+            throw IntegrationError.assert(msg: "systemd did not come up: '\(out)'")
+        }
+    }
+
     func testProcessEchoHi() async throws {
         let id = "test-process-echo-hi"
         let bs = try await bootstrap(id)

Sources/Integration/Suite.swift

@@ -195,8 +195,11 @@ struct IntegrationSuite: AsyncParsableCommand {
 
     static let eventLoop = MultiThreadedEventLoopGroup(numberOfThreads: System.coreCount)
 
-    func bootstrap(_ testID: String) async throws -> (rootfs: Containerization.Mount, vmm: VirtualMachineManager, image: Containerization.Image, bootLog: BootLog) {
-        let reference = "ghcr.io/linuxcontainers/alpine:3.20"
+    func bootstrap(
+        _ testID: String,
+        reference: String = "ghcr.io/linuxcontainers/alpine:3.20",
+        capacityInBytes: UInt64 = 2.gib()
+    ) async throws -> (rootfs: Containerization.Mount, vmm: VirtualMachineManager, image: Containerization.Image, bootLog: BootLog) {
         let store = Self.imageStore
 
         let initImage = try await store.getInitImage(reference: Self.initImage)
@@ -236,7 +239,7 @@ struct IntegrationSuite: AsyncParsableCommand {
         let fsPath = Self.testDir.appending(component: image.digest)
         let fs = try await Self.unpackCoordinator.unpack(key: fsPath.absolutePath()) {
             do {
-                let unpacker = EXT4Unpacker(capacityInBytes: 2.gib())
+                let unpacker = EXT4Unpacker(capacityInBytes: capacityInBytes)
                 return try await unpacker.unpack(image, for: platform, at: fsPath)
             } catch let err as ContainerizationError {
                 if err.code == .exists {
@@ -259,12 +262,16 @@ struct IntegrationSuite: AsyncParsableCommand {
         // a ~2GB rootfs and a ~512MB initfs, so without reaping the dev
         // container fills its CoW layer in ~10 tests.
         if self.maxConcurrency == 1 {
-            let preserve = fsPath.absolutePath()
+            // contentsOfDirectory reports paths under /private, while testDir is
+            // built from FileManager's /var view of the same directory, so both
+            // sides are resolved before comparing.
+            let preserve = fsPath.resolvingSymlinksInPathWithPrivate().absolutePath()
             if let entries = try? FileManager.default.contentsOfDirectory(
                 at: Self.testDir,
                 includingPropertiesForKeys: nil
             ) {
-                for url in entries where url.absolutePath() != preserve {
+                for url in entries
+                where url.resolvingSymlinksInPathWithPrivate().absolutePath() != preserve {
                     try? FileManager.default.removeItem(at: url)
                 }
             }
@@ -422,6 +429,7 @@ struct IntegrationSuite: AsyncParsableCommand {
             // Process basics
             Test("process true", testProcessTrue),
             Test("process false", testProcessFalse),
+            Test("container systemd", testContainerSystemd),
             Test("process echo hi", testProcessEchoHi),
             Test("process no executable", testProcessNoExecutable),
             Test("process user", testProcessUser),