diff --git a/Makefile b/Makefile index c79d6995..3d01b93a 100644 --- a/Makefile +++ b/Makefile @@ -64,7 +64,7 @@ LIBARCHIVE_UPSTREAM_REPO := https://github.com/libarchive/libarchive LIBARCHIVE_UPSTREAM_VERSION := v3.7.7 LIBARCHIVE_LOCAL_DIR := workdir/libarchive -KATA_BINARY_PACKAGE := https://github.com/kata-containers/kata-containers/releases/download/3.17.0/kata-static-3.17.0-arm64.tar.xz +KATA_BINARY_PACKAGE := https://github.com/kata-containers/kata-containers/releases/download/3.28.0/kata-static-3.28.0-arm64.tar.zst CLOUD_HYPERVISOR_URL := https://github.com/cloud-hypervisor/cloud-hypervisor/releases/download/v52.0/cloud-hypervisor-static-aarch64 # SHA256 of the v52.0 aarch64 static binary (verified locally from the # upstream release artifact). Bump alongside CLOUD_HYPERVISOR_URL. @@ -410,11 +410,11 @@ integration: .PHONY: fetch-default-kernel fetch-default-kernel: @mkdir -p .local/ bin/ -ifeq (,$(wildcard .local/kata.tar.gz)) - @curl -SsL -o .local/kata.tar.gz ${KATA_BINARY_PACKAGE} +ifeq (,$(wildcard .local/kata.tar)) + @curl -SsL -o .local/kata.tar ${KATA_BINARY_PACKAGE} endif ifeq (,$(wildcard .local/vmlinux-$(KERNEL_ARCH))) - @tar -zxf .local/kata.tar.gz -C .local/ --strip-components=1 + @tar -xf .local/kata.tar -C .local/ --strip-components=1 @cp -L .local/opt/kata/share/kata-containers/vmlinux.container .local/vmlinux-$(KERNEL_ARCH) endif ifeq (,$(wildcard bin/vmlinux-$(KERNEL_ARCH))) diff --git a/README.md b/README.md index f658b08d..372a872a 100644 --- a/README.md +++ b/README.md @@ -130,6 +130,16 @@ After building, run basic and integration tests: make test integration ``` +Tests that only apply to Linux are compiled out on macOS, so `make test` passes +without running them. Run those with: + +```bash +make linux-test +``` + +which runs the same tests inside the Linux dev container, as the Linux build +workflow does. + A kernel is required to run integration tests. If you do not have a kernel locally, a default kernel can be fetched using the `make fetch-default-kernel` target. diff --git a/Sources/Containerization/LinuxContainer.swift b/Sources/Containerization/LinuxContainer.swift index 34964fcd..61512dc6 100644 --- a/Sources/Containerization/LinuxContainer.swift +++ b/Sources/Containerization/LinuxContainer.swift @@ -26,6 +26,17 @@ import SystemPackage import struct ContainerizationOS.Terminal +/// When a container is prepared for a service manager to run as its init. +public enum SystemdMode: Sendable { + /// Never, which is what a container that runs a single process wants. + case disabled + /// When the process the container starts is one, by the paths that + /// nerdctl and podman both take to mean a service manager. + case detected + /// Whatever the container starts. + case always +} + /// `LinuxContainer` is an easy to use type for launching and managing the /// full lifecycle of a Linux container ran inside of a virtual machine. public final class LinuxContainer: Container, Sendable { @@ -91,6 +102,9 @@ public final class LinuxContainer: Container, Sendable { /// Run the container with a minimal init process that handles signal /// forwarding and zombie reaping. public var useInit: Bool = false + /// Prepare the container for a service manager to run as its init, + /// which needs somewhere writable to keep runtime state. + public var systemd: SystemdMode = .disabled /// Additional CPU cores to allocate for the virtual machine on top /// of the container's configured `cpus` value. public var cpuOverhead: Int = 1 @@ -454,6 +468,68 @@ public final class LinuxContainer: Container, Sendable { ] } + /// Whether a service manager is to run as a container's init, which is + /// either asked for outright or taken from the process the container + /// starts, as nerdctl and podman both decide it. + static func runsSystemd(_ systemd: SystemdMode, arguments: [String]) -> Bool { + switch systemd { + case .disabled: + return false + case .always: + return true + case .detected: + guard let command = arguments.first else { + return false + } + return Self.systemdPaths().contains(command) + } + } + + /// A container's mounts, plus somewhere writable for a service manager to + /// keep its runtime state when one is to run as the container's init. A + /// mount the caller supplied for the same place is left as they asked. + static func systemdAwareMounts( + _ mounts: [Mount], systemd: SystemdMode, arguments: [String] + ) -> [Mount] { + guard Self.runsSystemd(systemd, arguments: arguments) else { + return mounts + } + let asked = Set(mounts.map { $0.destination }) + return mounts + Self.systemdMounts().filter { !asked.contains($0.destination) } + } + + /// The paths a container's process is taken to be a service manager by, + /// which is what nerdctl and podman look for when told to detect one. + /// https://github.com/containerd/nerdctl/blob/main/pkg/cmd/container/create.go + public static func systemdPaths() -> [String] { + [ + "/sbin/init", + "/usr/sbin/init", + "/usr/local/sbin/init", + ] + } + + /// Somewhere writable for a service manager to keep the runtime state it + /// expects to own, which is what podman gives one. + /// https://github.com/containers/podman/blob/main/libpod/container_internal_linux.go + /// + /// The journal directory is the one journald reads. Persistent logs go to + /// `/var/log/journal` and volatile ones to `/run/log/journal`, so those are + /// the two paths worth making writable and `/var/log/journal` is the one a + /// tmpfs belongs on. + /// https://www.freedesktop.org/software/systemd/man/latest/journald.conf.html + /// + /// Two things that belong with these are seen to elsewhere. `/sys/fs/cgroup` + /// is mounted writable by ``defaultMounts()``, which is what podman gives a + /// service manager alongside its own list, and the `SIGRTMIN+3` one takes as + /// the request to shut down is carried on the image configuration here, with + /// nothing on the runtime side to set it. + public static func systemdMounts() -> [Mount] { + ["/run", "/run/lock", "/tmp", "/var/log/journal"].map { + .any(type: "tmpfs", source: "tmpfs", destination: $0) + } + } + /// The default set of paths to mask inside a container, matching the OCI /// runtime spec defaults that runc and other production runtimes apply. /// Each path is hidden from the workload (replaced by `/dev/null` for files @@ -619,7 +695,10 @@ extension LinuxContainer { let vmCpus = self.cpus + self.config.cpuOverhead // Prepare file mounts. This transforms single-file mounts into directory shares. - let fileMountContext = try FileMountContext.prepare(mounts: self.config.mounts) + let fileMountContext = try FileMountContext.prepare( + mounts: Self.systemdAwareMounts( + self.config.mounts, systemd: self.config.systemd, + arguments: self.config.process.arguments)) // This is dumb, but alas. let fileMountContextHolder = Mutex(fileMountContext) diff --git a/Sources/Containerization/LinuxPod.swift b/Sources/Containerization/LinuxPod.swift index 6275a4d4..5e9694b7 100644 --- a/Sources/Containerization/LinuxPod.swift +++ b/Sources/Containerization/LinuxPod.swift @@ -83,6 +83,9 @@ public final class LinuxPod: Sendable { public var sysctl: [String: String] = [:] /// The mounts for the container. public var mounts: [Mount] = LinuxContainer.defaultMounts() + /// Prepare this container for a service manager to run as its init, + /// which needs somewhere writable to keep runtime state. + public var systemd: SystemdMode = .disabled /// Paths inside the container that vmexec hides from the workload. /// Defaults to the OCI standard set (``LinuxContainer/defaultMaskedPaths()``), /// matching the restricted capability baseline. Set to `[]` to opt out, @@ -390,7 +393,10 @@ extension LinuxPod { var config = ContainerConfiguration() try configuration(&config) - let fileMountContext = try FileMountContext.prepare(mounts: config.mounts) + let fileMountContext = try FileMountContext.prepare( + mounts: LinuxContainer.systemdAwareMounts( + config.mounts, systemd: config.systemd, + arguments: config.process.arguments)) switch state.phase { case .initialized: diff --git a/Sources/Integration/ContainerTests.swift b/Sources/Integration/ContainerTests.swift index dd2a9165..219abed9 100644 --- a/Sources/Integration/ContainerTests.swift +++ b/Sources/Integration/ContainerTests.swift @@ -137,6 +137,48 @@ extension IntegrationSuite { } } + /// A service manager running as the container's init, which is what an + /// image built around one expects, and which needs somewhere writable to + /// keep the runtime state it owns. + func testContainerSystemd() async throws { + let id = "test-container-systemd" + let bs = try await bootstrap( + id, reference: "docker.io/library/almalinux:9", capacityInBytes: 8.gib()) + + let container = try LinuxContainer(id, rootfs: bs.rootfs, vmm: bs.vmm) { config in + // Asked for by the process the container starts, rather than outright. + config.systemd = .detected + config.process.arguments = ["/usr/sbin/init"] + config.bootLog = bs.bootLog + } + try await container.create() + try await container.start() + try await Task.sleep(for: .seconds(30)) + + // It logs to the console rather than to us, so ask it from inside. + let buffer = BufferWriter() + let query = try await container.exec("state") { config in + config.arguments = [ + "/bin/sh", "-c", + "echo PID1=$(cat /proc/1/comm); echo STATE=$(systemctl is-system-running)", + ] + config.stdout = buffer + config.stderr = buffer + } + try await query.start() + _ = try await query.wait() + try await query.delete() + try? await container.stop() + + let out = String(data: buffer.data, encoding: .utf8) ?? "" + guard out.contains("PID1=systemd") else { + throw IntegrationError.assert(msg: "systemd is not the container's init: '\(out)'") + } + guard out.contains("STATE=running") || out.contains("STATE=degraded") else { + throw IntegrationError.assert(msg: "systemd did not come up: '\(out)'") + } + } + func testProcessEchoHi() async throws { let id = "test-process-echo-hi" let bs = try await bootstrap(id) diff --git a/Sources/Integration/Suite.swift b/Sources/Integration/Suite.swift index 9dae8435..600b05d3 100644 --- a/Sources/Integration/Suite.swift +++ b/Sources/Integration/Suite.swift @@ -195,8 +195,11 @@ struct IntegrationSuite: AsyncParsableCommand { static let eventLoop = MultiThreadedEventLoopGroup(numberOfThreads: System.coreCount) - func bootstrap(_ testID: String) async throws -> (rootfs: Containerization.Mount, vmm: VirtualMachineManager, image: Containerization.Image, bootLog: BootLog) { - let reference = "ghcr.io/linuxcontainers/alpine:3.20" + func bootstrap( + _ testID: String, + reference: String = "ghcr.io/linuxcontainers/alpine:3.20", + capacityInBytes: UInt64 = 2.gib() + ) async throws -> (rootfs: Containerization.Mount, vmm: VirtualMachineManager, image: Containerization.Image, bootLog: BootLog) { let store = Self.imageStore let initImage = try await store.getInitImage(reference: Self.initImage) @@ -236,7 +239,7 @@ struct IntegrationSuite: AsyncParsableCommand { let fsPath = Self.testDir.appending(component: image.digest) let fs = try await Self.unpackCoordinator.unpack(key: fsPath.absolutePath()) { do { - let unpacker = EXT4Unpacker(capacityInBytes: 2.gib()) + let unpacker = EXT4Unpacker(capacityInBytes: capacityInBytes) return try await unpacker.unpack(image, for: platform, at: fsPath) } catch let err as ContainerizationError { if err.code == .exists { @@ -259,12 +262,16 @@ struct IntegrationSuite: AsyncParsableCommand { // a ~2GB rootfs and a ~512MB initfs, so without reaping the dev // container fills its CoW layer in ~10 tests. if self.maxConcurrency == 1 { - let preserve = fsPath.absolutePath() + // contentsOfDirectory reports paths under /private, while testDir is + // built from FileManager's /var view of the same directory, so both + // sides are resolved before comparing. + let preserve = fsPath.resolvingSymlinksInPathWithPrivate().absolutePath() if let entries = try? FileManager.default.contentsOfDirectory( at: Self.testDir, includingPropertiesForKeys: nil ) { - for url in entries where url.absolutePath() != preserve { + for url in entries + where url.resolvingSymlinksInPathWithPrivate().absolutePath() != preserve { try? FileManager.default.removeItem(at: url) } } @@ -422,6 +429,7 @@ struct IntegrationSuite: AsyncParsableCommand { // Process basics Test("process true", testProcessTrue), Test("process false", testProcessFalse), + Test("container systemd", testContainerSystemd), Test("process echo hi", testProcessEchoHi), Test("process no executable", testProcessNoExecutable), Test("process user", testProcessUser),