Hand a container's cgroup to the user it runs as

Makefile

@@ -64,7 +64,7 @@ LIBARCHIVE_UPSTREAM_REPO := https://github.com/libarchive/libarchive
 LIBARCHIVE_UPSTREAM_VERSION := v3.7.7
 LIBARCHIVE_LOCAL_DIR := workdir/libarchive
 
-KATA_BINARY_PACKAGE := https://github.com/kata-containers/kata-containers/releases/download/3.17.0/kata-static-3.17.0-arm64.tar.xz
+KATA_BINARY_PACKAGE := https://github.com/kata-containers/kata-containers/releases/download/3.28.0/kata-static-3.28.0-arm64.tar.zst
 CLOUD_HYPERVISOR_URL := https://github.com/cloud-hypervisor/cloud-hypervisor/releases/download/v52.0/cloud-hypervisor-static-aarch64
 # SHA256 of the v52.0 aarch64 static binary (verified locally from the
 # upstream release artifact). Bump alongside CLOUD_HYPERVISOR_URL.
@@ -410,11 +410,11 @@ integration:
 .PHONY: fetch-default-kernel
 fetch-default-kernel:
 	@mkdir -p .local/ bin/
-ifeq (,$(wildcard .local/kata.tar.gz))
-	@curl -SsL -o .local/kata.tar.gz ${KATA_BINARY_PACKAGE}
+ifeq (,$(wildcard .local/kata.tar))
+	@curl -SsL -o .local/kata.tar ${KATA_BINARY_PACKAGE}
 endif
 ifeq (,$(wildcard .local/vmlinux-$(KERNEL_ARCH)))
-	@tar -zxf .local/kata.tar.gz -C .local/ --strip-components=1
+	@tar -xf .local/kata.tar -C .local/ --strip-components=1
 	@cp -L .local/opt/kata/share/kata-containers/vmlinux.container .local/vmlinux-$(KERNEL_ARCH)
 endif
 ifeq (,$(wildcard bin/vmlinux-$(KERNEL_ARCH)))

README.md

@@ -130,6 +130,16 @@ After building, run basic and integration tests:
 make test integration
 ```
 
+Tests that only apply to Linux are compiled out on macOS, so `make test` passes
+without running them. Run those with:
+
+```bash
+make linux-test
+```
+
+which runs the same tests inside the Linux dev container, as the Linux build
+workflow does.
+
 A kernel is required to run integration tests.
 If you do not have a kernel locally, a default kernel can be fetched using the `make fetch-default-kernel` target.
 

Sources/Containerization/LinuxContainer.swift

@@ -91,6 +91,19 @@ public final class LinuxContainer: Container, Sendable {
         /// Run the container with a minimal init process that handles signal
         /// forwarding and zombie reaping.
         public var useInit: Bool = false
+        /// Hand the container's cgroup to the user the container runs as, so
+        /// that what runs inside can place its own work under limits of its
+        /// own. This is what a service manager does for a user session, and
+        /// what a container runtime nested in this one looks for.
+        ///
+        /// The user is given the cgroup it is placed in and nothing above it,
+        /// so the limits this container was given still bind everything within.
+        ///
+        /// A nested runtime asks a service manager for this where one exists:
+        /// rootless runc talks to systemd to have a cgroup delegated to it, and
+        /// without that it is left with no cgroup at all.
+        /// https://github.com/opencontainers/runc/blob/main/docs/cgroup-v2.md
+        public var cgroupDelegation: Bool = false
         /// Additional CPU cores to allocate for the virtual machine on top
         /// of the container's configured `cpus` value.
         public var cpuOverhead: Int = 1
@@ -407,6 +420,12 @@ public final class LinuxContainer: Container, Sendable {
         }
 
         // Linux toggles.
+        if config.cgroupDelegation {
+            var annotations = spec.annotations ?? [:]
+            annotations[AnnotationKeys.containerizationCgroupDelegation] = "true"
+            spec.annotations = annotations
+        }
+
         spec.linux?.sysctl = config.sysctl
         spec.linux?.maskedPaths = config.maskedPaths
         spec.linux?.readonlyPaths = config.readonlyPaths

Sources/Containerization/LinuxPod.swift

@@ -77,6 +77,11 @@ public final class LinuxPod: Sendable {
         public var cpus: Int?
         /// Optional per-container memory limit in bytes (can exceed pod total for oversubscription).
         public var memoryInBytes: UInt64?
+        /// Hand this container's cgroup to the user it runs as, so that what
+        /// runs inside can place its own work under limits of its own. The
+        /// user is given the cgroup it is placed in and nothing above it, so
+        /// the limits the pod and this container were given still bind.
+        public var cgroupDelegation: Bool = false
         /// The hostname for the container.
         public var hostname: String?
         /// The system control options for the container.
@@ -310,6 +315,11 @@ public final class LinuxPod: Sendable {
         }
 
         // Linux toggles
+        if config.cgroupDelegation {
+            var annotations = spec.annotations ?? [:]
+            annotations[AnnotationKeys.containerizationCgroupDelegation] = "true"
+            spec.annotations = annotations
+        }
         spec.linux?.sysctl = config.sysctl
         spec.linux?.maskedPaths = config.maskedPaths
         spec.linux?.readonlyPaths = config.readonlyPaths

Sources/ContainerizationOCI/AnnotationKeys.swift

@@ -19,6 +19,7 @@
 public struct AnnotationKeys: Codable, Sendable {
     public static let containerizationIndexIndirect = "com.apple.containerization.index.indirect"
     public static let containerizationImageName = "com.apple.containerization.image.name"
+    public static let containerizationCgroupDelegation = "com.apple.containerization.cgroup.delegation"
     public static let containerdImageName = "io.containerd.image.name"
     public static let openContainersImageName = "org.opencontainers.image.ref.name"
 }

Sources/Integration/ContainerTests.swift

@@ -137,6 +137,42 @@ extension IntegrationSuite {
         }
     }
 
+    /// What delegation is for: the user the container runs as can put its own
+    /// work under a limit, which the kernel otherwise reserves for whoever owns
+    /// the cgroup.
+    func testContainerCgroupDelegation() async throws {
+        let id = "test-container-cgroup-delegation"
+        let bs = try await bootstrap(id)
+
+        let buffer = BufferWriter()
+        let container = try LinuxContainer(id, rootfs: bs.rootfs, vmm: bs.vmm) { config in
+            config.cgroupDelegation = true
+            config.process.user = ContainerizationOCI.User(uid: 1000, gid: 1000)
+            config.process.arguments = [
+                "/bin/sh", "-c",
+                // A cgroup holding processes cannot enable controllers for its
+                // children, so step into a leaf before handing memory down.
+                "mkdir -p /sys/fs/cgroup/init /sys/fs/cgroup/work && "
+                    + "echo $$ > /sys/fs/cgroup/init/cgroup.procs && "
+                    + "echo '+memory' > /sys/fs/cgroup/cgroup.subtree_control && "
+                    + "echo 64000000 > /sys/fs/cgroup/work/memory.max && "
+                    + "echo LIMIT=$(cat /sys/fs/cgroup/work/memory.max)",
+            ]
+            config.process.stdout = buffer
+            config.bootLog = bs.bootLog
+        }
+        try await container.create()
+        try await container.start()
+        _ = try await container.wait()
+        try await container.stop()
+
+        let out = String(data: buffer.data, encoding: .utf8) ?? ""
+        guard out.contains("LIMIT=64000000") else {
+            throw IntegrationError.assert(
+                msg: "container could not limit its own work: '\(out)'")
+        }
+    }
+
     func testProcessEchoHi() async throws {
         let id = "test-process-echo-hi"
         let bs = try await bootstrap(id)

Sources/Integration/PodTests.swift

@@ -59,6 +59,45 @@ extension IntegrationSuite {
         }
     }
 
+    /// Delegation reaches a pod's containers too, so one container's user can
+    /// limit its own work without touching what the pod gave any other.
+    func testPodCgroupDelegation() async throws {
+        let id = "test-pod-cgroup-delegation"
+
+        let bs = try await bootstrap(id)
+        let pod = try LinuxPod(id, vmm: bs.vmm) { config in
+            config.cpus = 4
+            config.memoryInBytes = 1024.mib()
+            config.bootLog = bs.bootLog
+        }
+
+        let buffer = BufferWriter()
+        try await pod.addContainer("delegated", rootfs: bs.rootfs) { config in
+            config.cgroupDelegation = true
+            config.process.user = ContainerizationOCI.User(uid: 1000, gid: 1000)
+            config.process.arguments = [
+                "/bin/sh", "-c",
+                "mkdir -p /sys/fs/cgroup/init /sys/fs/cgroup/work && "
+                    + "echo $$ > /sys/fs/cgroup/init/cgroup.procs && "
+                    + "echo '+memory' > /sys/fs/cgroup/cgroup.subtree_control && "
+                    + "echo 64000000 > /sys/fs/cgroup/work/memory.max && "
+                    + "echo LIMIT=$(cat /sys/fs/cgroup/work/memory.max)",
+            ]
+            config.process.stdout = buffer
+        }
+
+        try await pod.create()
+        try await pod.startContainer("delegated")
+        _ = try await pod.waitContainer("delegated")
+        try await pod.stop()
+
+        let out = String(data: buffer.data, encoding: .utf8) ?? ""
+        guard out.contains("LIMIT=64000000") else {
+            throw IntegrationError.assert(
+                msg: "pod container could not limit its own work: '\(out)'")
+        }
+    }
+
     func testPodMultipleContainers() async throws {
         let id = "test-pod-multiple-containers"
 

Sources/Integration/Suite.swift

@@ -259,12 +259,16 @@ struct IntegrationSuite: AsyncParsableCommand {
         // a ~2GB rootfs and a ~512MB initfs, so without reaping the dev
         // container fills its CoW layer in ~10 tests.
         if self.maxConcurrency == 1 {
-            let preserve = fsPath.absolutePath()
+            // contentsOfDirectory reports paths under /private, while testDir is
+            // built from FileManager's /var view of the same directory, so both
+            // sides are resolved before comparing.
+            let preserve = fsPath.resolvingSymlinksInPathWithPrivate().absolutePath()
             if let entries = try? FileManager.default.contentsOfDirectory(
                 at: Self.testDir,
                 includingPropertiesForKeys: nil
             ) {
-                for url in entries where url.absolutePath() != preserve {
+                for url in entries
+                where url.resolvingSymlinksInPathWithPrivate().absolutePath() != preserve {
                     try? FileManager.default.removeItem(at: url)
                 }
             }
@@ -422,6 +426,7 @@ struct IntegrationSuite: AsyncParsableCommand {
             // Process basics
             Test("process true", testProcessTrue),
             Test("process false", testProcessFalse),
+            Test("container cgroup delegation", testContainerCgroupDelegation),
             Test("process echo hi", testProcessEchoHi),
             Test("process no executable", testProcessNoExecutable),
             Test("process user", testProcessUser),
@@ -530,6 +535,7 @@ struct IntegrationSuite: AsyncParsableCommand {
 
             // Pods
             Test("pod single container", testPodSingleContainer),
+            Test("pod cgroup delegation", testPodCgroupDelegation),
             Test("pod multiple containers", testPodMultipleContainers),
             Test("pod container output", testPodContainerOutput),
             Test("pod concurrent containers", testPodConcurrentContainers),

vminitd/Sources/Cgroup/Cgroup2Manager.swift

@@ -49,6 +49,7 @@ public struct Cgroup2Manager: Sendable {
     private static let killFile = "cgroup.kill"
     private static let procsFile = "cgroup.procs"
     private static let subtreeControlFile = "cgroup.subtree_control"
+    private static let threadsFile = "cgroup.threads"
 
     private static let cg2Magic = 0x6367_7270
 
@@ -122,6 +123,27 @@ public struct Cgroup2Manager: Sendable {
         )
     }
 
+    /// Hand control of this cgroup to a less privileged user, which the kernel
+    /// defines as write access to the directory and to its `cgroup.procs`,
+    /// `cgroup.threads` and `cgroup.subtree_control` files. The resource files
+    /// are deliberately left alone: they distribute the resources of the parent
+    /// rather than of this cgroup, so the user they are delegated to must not
+    /// be able to write them.
+    /// https://github.com/torvalds/linux/blob/master/Documentation/admin-guide/cgroup-v2.rst
+    package func delegate(uid: UInt32, gid: UInt32) throws {
+        let owned =
+            [self.path]
+            + [Self.procsFile, Self.threadsFile, Self.subtreeControlFile].map {
+                self.path.appending(path: $0)
+            }
+
+        for file in owned {
+            guard chown(file.path, uid, gid) == 0 else {
+                throw Error.errno(errno: errno, message: "failed to chown \(file.path)")
+            }
+        }
+    }
+
     private static func writeValue(path: URL, value: String, fileName: String) throws {
         let file = path.appending(path: fileName)
         let fd = open(file.path, O_WRONLY, 0)

vminitd/Sources/VminitdCore/ManagedContainer.swift

@@ -66,6 +66,15 @@ public actor ManagedContainer {
         do {
             try cgManager.toggleAllAvailableControllers(enable: true)
 
+            // A container that asked for its cgroup gets it, so long as it runs
+            // as somebody who would otherwise be unable to use it.
+            if spec.annotations?[AnnotationKeys.containerizationCgroupDelegation] == "true",
+                let user = spec.process?.user, user.uid != 0
+            {
+                try cgManager.delegate(uid: user.uid, gid: user.gid)
+                log.info("delegated cgroup to \(user.uid):\(user.gid)")
+            }
+
             let initProcess: any ContainerProcess
 
             if let runtimePath = ociRuntimePath {