From f10eaca613f6eef28add5d69f4fbf573ca6ed156 Mon Sep 17 00:00:00 2001 From: Aaron Paterson Date: Sat, 1 Aug 2026 18:48:34 +0000 Subject: [PATCH 1/4] Resolve test directory paths before reaping per-test files The reaper that runs at a concurrency of one deletes everything in the test directory except the unpacked rootfs it means to preserve. It held that rootfs path as FileManager reports it, under /var, and compared it against the entries of contentsOfDirectory, which reports them under /private/var, so the preserved path never matched any entry and the rootfs was deleted along with the per-test files. The unpack coordinator still held it as unpacked, so the next test opened a rootfs that was no longer there and failed with a missing file error. Resolve both sides with resolvingSymlinksInPathWithPrivate, which exists for this difference between the two views of the same directory. --- Sources/Integration/Suite.swift | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/Sources/Integration/Suite.swift b/Sources/Integration/Suite.swift index 9dae8435..e43cf372 100644 --- a/Sources/Integration/Suite.swift +++ b/Sources/Integration/Suite.swift @@ -259,12 +259,16 @@ struct IntegrationSuite: AsyncParsableCommand { // a ~2GB rootfs and a ~512MB initfs, so without reaping the dev // container fills its CoW layer in ~10 tests. if self.maxConcurrency == 1 { - let preserve = fsPath.absolutePath() + // contentsOfDirectory reports paths under /private, while testDir is + // built from FileManager's /var view of the same directory, so both + // sides are resolved before comparing. + let preserve = fsPath.resolvingSymlinksInPathWithPrivate().absolutePath() if let entries = try? FileManager.default.contentsOfDirectory( at: Self.testDir, includingPropertiesForKeys: nil ) { - for url in entries where url.absolutePath() != preserve { + for url in entries + where url.resolvingSymlinksInPathWithPrivate().absolutePath() != preserve { try? FileManager.default.removeItem(at: url) } } From 6499f7dfa7fff64c903642db690094d7e6c64ec8 Mon Sep 17 00:00:00 2001 From: Aaron Paterson Date: Sun, 2 Aug 2026 00:51:10 +0000 Subject: [PATCH 2/4] Say how to run the tests that macOS skips Tests guarded for Linux are compiled out on macOS, so `make test` reports success without having run them, and nothing says so. The target that does run them is not mentioned anywhere outside the makefile. --- README.md | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/README.md b/README.md index f658b08d..372a872a 100644 --- a/README.md +++ b/README.md @@ -130,6 +130,16 @@ After building, run basic and integration tests: make test integration ``` +Tests that only apply to Linux are compiled out on macOS, so `make test` passes +without running them. Run those with: + +```bash +make linux-test +``` + +which runs the same tests inside the Linux dev container, as the Linux build +workflow does. + A kernel is required to run integration tests. If you do not have a kernel locally, a default kernel can be fetched using the `make fetch-default-kernel` target. From bf4d7dcf8bbe64c745e9ef344a8c173d2a42a3af Mon Sep 17 00:00:00 2001 From: Aaron Paterson Date: Sun, 2 Aug 2026 01:20:29 +0000 Subject: [PATCH 3/4] Test against the kernel release the shipped tool installs The default kernel the tests fetch had drifted from the one users run. container installs kata 3.28.0 and its 6.18.15 kernel, while these tests fetched 3.17.0, so the suite exercised a guest with a different feature set to the one it is meant to represent. Nested runtimes are the visible case: 3.17.0 was built without nf_tables, so a docker daemon inside a container fails there and works on what ships. Kata moved from xz to zstd between those releases, so the archive is no longer named for its compression and tar is left to recognise it rather than being told, which also holds if the format changes again. --- Makefile | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Makefile b/Makefile index c79d6995..3d01b93a 100644 --- a/Makefile +++ b/Makefile @@ -64,7 +64,7 @@ LIBARCHIVE_UPSTREAM_REPO := https://github.com/libarchive/libarchive LIBARCHIVE_UPSTREAM_VERSION := v3.7.7 LIBARCHIVE_LOCAL_DIR := workdir/libarchive -KATA_BINARY_PACKAGE := https://github.com/kata-containers/kata-containers/releases/download/3.17.0/kata-static-3.17.0-arm64.tar.xz +KATA_BINARY_PACKAGE := https://github.com/kata-containers/kata-containers/releases/download/3.28.0/kata-static-3.28.0-arm64.tar.zst CLOUD_HYPERVISOR_URL := https://github.com/cloud-hypervisor/cloud-hypervisor/releases/download/v52.0/cloud-hypervisor-static-aarch64 # SHA256 of the v52.0 aarch64 static binary (verified locally from the # upstream release artifact). Bump alongside CLOUD_HYPERVISOR_URL. @@ -410,11 +410,11 @@ integration: .PHONY: fetch-default-kernel fetch-default-kernel: @mkdir -p .local/ bin/ -ifeq (,$(wildcard .local/kata.tar.gz)) - @curl -SsL -o .local/kata.tar.gz ${KATA_BINARY_PACKAGE} +ifeq (,$(wildcard .local/kata.tar)) + @curl -SsL -o .local/kata.tar ${KATA_BINARY_PACKAGE} endif ifeq (,$(wildcard .local/vmlinux-$(KERNEL_ARCH))) - @tar -zxf .local/kata.tar.gz -C .local/ --strip-components=1 + @tar -xf .local/kata.tar -C .local/ --strip-components=1 @cp -L .local/opt/kata/share/kata-containers/vmlinux.container .local/vmlinux-$(KERNEL_ARCH) endif ifeq (,$(wildcard bin/vmlinux-$(KERNEL_ARCH))) From 5c0d8dc22b50aede590f65e794ece030af90500e Mon Sep 17 00:00:00 2001 From: Aaron Paterson Date: Sun, 2 Aug 2026 05:08:54 +0000 Subject: [PATCH 4/4] Hand a container's cgroup to the user it runs as A container that runs as somebody other than root cannot use the cgroup it was placed in: the kernel keeps it for whoever owns the directory, so anything nested inside that wants to hold its own work to a limit finds the tree closed. On a machine the service manager solves this by delegating a subtree to a user session. Nothing here does, because the runtime is what creates the cgroup, and so is already the one privileged to give it away. Grant the user the directory, cgroup.procs, cgroup.threads and cgroup.subtree_control, which is what the kernel documents delegation to mean. The resource files stay where they are: they distribute the parent's resources rather than this cgroup's own, so the limits a container was given still bind everything it puts underneath. A container asks for this through its configuration, and a pod's containers ask the same way. --- Sources/Containerization/LinuxContainer.swift | 19 +++++++++ Sources/Containerization/LinuxPod.swift | 10 +++++ .../ContainerizationOCI/AnnotationKeys.swift | 1 + Sources/Integration/ContainerTests.swift | 36 +++++++++++++++++ Sources/Integration/PodTests.swift | 39 +++++++++++++++++++ Sources/Integration/Suite.swift | 2 + vminitd/Sources/Cgroup/Cgroup2Manager.swift | 22 +++++++++++ .../VminitdCore/ManagedContainer.swift | 9 +++++ 8 files changed, 138 insertions(+) diff --git a/Sources/Containerization/LinuxContainer.swift b/Sources/Containerization/LinuxContainer.swift index 34964fcd..cc5dce93 100644 --- a/Sources/Containerization/LinuxContainer.swift +++ b/Sources/Containerization/LinuxContainer.swift @@ -91,6 +91,19 @@ public final class LinuxContainer: Container, Sendable { /// Run the container with a minimal init process that handles signal /// forwarding and zombie reaping. public var useInit: Bool = false + /// Hand the container's cgroup to the user the container runs as, so + /// that what runs inside can place its own work under limits of its + /// own. This is what a service manager does for a user session, and + /// what a container runtime nested in this one looks for. + /// + /// The user is given the cgroup it is placed in and nothing above it, + /// so the limits this container was given still bind everything within. + /// + /// A nested runtime asks a service manager for this where one exists: + /// rootless runc talks to systemd to have a cgroup delegated to it, and + /// without that it is left with no cgroup at all. + /// https://github.com/opencontainers/runc/blob/main/docs/cgroup-v2.md + public var cgroupDelegation: Bool = false /// Additional CPU cores to allocate for the virtual machine on top /// of the container's configured `cpus` value. public var cpuOverhead: Int = 1 @@ -407,6 +420,12 @@ public final class LinuxContainer: Container, Sendable { } // Linux toggles. + if config.cgroupDelegation { + var annotations = spec.annotations ?? [:] + annotations[AnnotationKeys.containerizationCgroupDelegation] = "true" + spec.annotations = annotations + } + spec.linux?.sysctl = config.sysctl spec.linux?.maskedPaths = config.maskedPaths spec.linux?.readonlyPaths = config.readonlyPaths diff --git a/Sources/Containerization/LinuxPod.swift b/Sources/Containerization/LinuxPod.swift index 6275a4d4..2b2360b9 100644 --- a/Sources/Containerization/LinuxPod.swift +++ b/Sources/Containerization/LinuxPod.swift @@ -77,6 +77,11 @@ public final class LinuxPod: Sendable { public var cpus: Int? /// Optional per-container memory limit in bytes (can exceed pod total for oversubscription). public var memoryInBytes: UInt64? + /// Hand this container's cgroup to the user it runs as, so that what + /// runs inside can place its own work under limits of its own. The + /// user is given the cgroup it is placed in and nothing above it, so + /// the limits the pod and this container were given still bind. + public var cgroupDelegation: Bool = false /// The hostname for the container. public var hostname: String? /// The system control options for the container. @@ -310,6 +315,11 @@ public final class LinuxPod: Sendable { } // Linux toggles + if config.cgroupDelegation { + var annotations = spec.annotations ?? [:] + annotations[AnnotationKeys.containerizationCgroupDelegation] = "true" + spec.annotations = annotations + } spec.linux?.sysctl = config.sysctl spec.linux?.maskedPaths = config.maskedPaths spec.linux?.readonlyPaths = config.readonlyPaths diff --git a/Sources/ContainerizationOCI/AnnotationKeys.swift b/Sources/ContainerizationOCI/AnnotationKeys.swift index 73f8889b..d46227f1 100644 --- a/Sources/ContainerizationOCI/AnnotationKeys.swift +++ b/Sources/ContainerizationOCI/AnnotationKeys.swift @@ -19,6 +19,7 @@ public struct AnnotationKeys: Codable, Sendable { public static let containerizationIndexIndirect = "com.apple.containerization.index.indirect" public static let containerizationImageName = "com.apple.containerization.image.name" + public static let containerizationCgroupDelegation = "com.apple.containerization.cgroup.delegation" public static let containerdImageName = "io.containerd.image.name" public static let openContainersImageName = "org.opencontainers.image.ref.name" } diff --git a/Sources/Integration/ContainerTests.swift b/Sources/Integration/ContainerTests.swift index dd2a9165..334acec9 100644 --- a/Sources/Integration/ContainerTests.swift +++ b/Sources/Integration/ContainerTests.swift @@ -137,6 +137,42 @@ extension IntegrationSuite { } } + /// What delegation is for: the user the container runs as can put its own + /// work under a limit, which the kernel otherwise reserves for whoever owns + /// the cgroup. + func testContainerCgroupDelegation() async throws { + let id = "test-container-cgroup-delegation" + let bs = try await bootstrap(id) + + let buffer = BufferWriter() + let container = try LinuxContainer(id, rootfs: bs.rootfs, vmm: bs.vmm) { config in + config.cgroupDelegation = true + config.process.user = ContainerizationOCI.User(uid: 1000, gid: 1000) + config.process.arguments = [ + "/bin/sh", "-c", + // A cgroup holding processes cannot enable controllers for its + // children, so step into a leaf before handing memory down. + "mkdir -p /sys/fs/cgroup/init /sys/fs/cgroup/work && " + + "echo $$ > /sys/fs/cgroup/init/cgroup.procs && " + + "echo '+memory' > /sys/fs/cgroup/cgroup.subtree_control && " + + "echo 64000000 > /sys/fs/cgroup/work/memory.max && " + + "echo LIMIT=$(cat /sys/fs/cgroup/work/memory.max)", + ] + config.process.stdout = buffer + config.bootLog = bs.bootLog + } + try await container.create() + try await container.start() + _ = try await container.wait() + try await container.stop() + + let out = String(data: buffer.data, encoding: .utf8) ?? "" + guard out.contains("LIMIT=64000000") else { + throw IntegrationError.assert( + msg: "container could not limit its own work: '\(out)'") + } + } + func testProcessEchoHi() async throws { let id = "test-process-echo-hi" let bs = try await bootstrap(id) diff --git a/Sources/Integration/PodTests.swift b/Sources/Integration/PodTests.swift index ae1caec8..eeeb7ceb 100644 --- a/Sources/Integration/PodTests.swift +++ b/Sources/Integration/PodTests.swift @@ -59,6 +59,45 @@ extension IntegrationSuite { } } + /// Delegation reaches a pod's containers too, so one container's user can + /// limit its own work without touching what the pod gave any other. + func testPodCgroupDelegation() async throws { + let id = "test-pod-cgroup-delegation" + + let bs = try await bootstrap(id) + let pod = try LinuxPod(id, vmm: bs.vmm) { config in + config.cpus = 4 + config.memoryInBytes = 1024.mib() + config.bootLog = bs.bootLog + } + + let buffer = BufferWriter() + try await pod.addContainer("delegated", rootfs: bs.rootfs) { config in + config.cgroupDelegation = true + config.process.user = ContainerizationOCI.User(uid: 1000, gid: 1000) + config.process.arguments = [ + "/bin/sh", "-c", + "mkdir -p /sys/fs/cgroup/init /sys/fs/cgroup/work && " + + "echo $$ > /sys/fs/cgroup/init/cgroup.procs && " + + "echo '+memory' > /sys/fs/cgroup/cgroup.subtree_control && " + + "echo 64000000 > /sys/fs/cgroup/work/memory.max && " + + "echo LIMIT=$(cat /sys/fs/cgroup/work/memory.max)", + ] + config.process.stdout = buffer + } + + try await pod.create() + try await pod.startContainer("delegated") + _ = try await pod.waitContainer("delegated") + try await pod.stop() + + let out = String(data: buffer.data, encoding: .utf8) ?? "" + guard out.contains("LIMIT=64000000") else { + throw IntegrationError.assert( + msg: "pod container could not limit its own work: '\(out)'") + } + } + func testPodMultipleContainers() async throws { let id = "test-pod-multiple-containers" diff --git a/Sources/Integration/Suite.swift b/Sources/Integration/Suite.swift index e43cf372..b6dc829b 100644 --- a/Sources/Integration/Suite.swift +++ b/Sources/Integration/Suite.swift @@ -426,6 +426,7 @@ struct IntegrationSuite: AsyncParsableCommand { // Process basics Test("process true", testProcessTrue), Test("process false", testProcessFalse), + Test("container cgroup delegation", testContainerCgroupDelegation), Test("process echo hi", testProcessEchoHi), Test("process no executable", testProcessNoExecutable), Test("process user", testProcessUser), @@ -534,6 +535,7 @@ struct IntegrationSuite: AsyncParsableCommand { // Pods Test("pod single container", testPodSingleContainer), + Test("pod cgroup delegation", testPodCgroupDelegation), Test("pod multiple containers", testPodMultipleContainers), Test("pod container output", testPodContainerOutput), Test("pod concurrent containers", testPodConcurrentContainers), diff --git a/vminitd/Sources/Cgroup/Cgroup2Manager.swift b/vminitd/Sources/Cgroup/Cgroup2Manager.swift index 62fa78c7..2ac25301 100644 --- a/vminitd/Sources/Cgroup/Cgroup2Manager.swift +++ b/vminitd/Sources/Cgroup/Cgroup2Manager.swift @@ -49,6 +49,7 @@ public struct Cgroup2Manager: Sendable { private static let killFile = "cgroup.kill" private static let procsFile = "cgroup.procs" private static let subtreeControlFile = "cgroup.subtree_control" + private static let threadsFile = "cgroup.threads" private static let cg2Magic = 0x6367_7270 @@ -122,6 +123,27 @@ public struct Cgroup2Manager: Sendable { ) } + /// Hand control of this cgroup to a less privileged user, which the kernel + /// defines as write access to the directory and to its `cgroup.procs`, + /// `cgroup.threads` and `cgroup.subtree_control` files. The resource files + /// are deliberately left alone: they distribute the resources of the parent + /// rather than of this cgroup, so the user they are delegated to must not + /// be able to write them. + /// https://github.com/torvalds/linux/blob/master/Documentation/admin-guide/cgroup-v2.rst + package func delegate(uid: UInt32, gid: UInt32) throws { + let owned = + [self.path] + + [Self.procsFile, Self.threadsFile, Self.subtreeControlFile].map { + self.path.appending(path: $0) + } + + for file in owned { + guard chown(file.path, uid, gid) == 0 else { + throw Error.errno(errno: errno, message: "failed to chown \(file.path)") + } + } + } + private static func writeValue(path: URL, value: String, fileName: String) throws { let file = path.appending(path: fileName) let fd = open(file.path, O_WRONLY, 0) diff --git a/vminitd/Sources/VminitdCore/ManagedContainer.swift b/vminitd/Sources/VminitdCore/ManagedContainer.swift index 545046a8..bd07c1cd 100644 --- a/vminitd/Sources/VminitdCore/ManagedContainer.swift +++ b/vminitd/Sources/VminitdCore/ManagedContainer.swift @@ -66,6 +66,15 @@ public actor ManagedContainer { do { try cgManager.toggleAllAvailableControllers(enable: true) + // A container that asked for its cgroup gets it, so long as it runs + // as somebody who would otherwise be unable to use it. + if spec.annotations?[AnnotationKeys.containerizationCgroupDelegation] == "true", + let user = spec.process?.user, user.uid != 0 + { + try cgManager.delegate(uid: user.uid, gid: user.gid) + log.info("delegated cgroup to \(user.uid):\(user.gid)") + } + let initProcess: any ContainerProcess if let runtimePath = ociRuntimePath {