diff --git a/Makefile b/Makefile index c79d6995d..3d01b93a1 100644 --- a/Makefile +++ b/Makefile @@ -64,7 +64,7 @@ LIBARCHIVE_UPSTREAM_REPO := https://github.com/libarchive/libarchive LIBARCHIVE_UPSTREAM_VERSION := v3.7.7 LIBARCHIVE_LOCAL_DIR := workdir/libarchive -KATA_BINARY_PACKAGE := https://github.com/kata-containers/kata-containers/releases/download/3.17.0/kata-static-3.17.0-arm64.tar.xz +KATA_BINARY_PACKAGE := https://github.com/kata-containers/kata-containers/releases/download/3.28.0/kata-static-3.28.0-arm64.tar.zst CLOUD_HYPERVISOR_URL := https://github.com/cloud-hypervisor/cloud-hypervisor/releases/download/v52.0/cloud-hypervisor-static-aarch64 # SHA256 of the v52.0 aarch64 static binary (verified locally from the # upstream release artifact). Bump alongside CLOUD_HYPERVISOR_URL. @@ -410,11 +410,11 @@ integration: .PHONY: fetch-default-kernel fetch-default-kernel: @mkdir -p .local/ bin/ -ifeq (,$(wildcard .local/kata.tar.gz)) - @curl -SsL -o .local/kata.tar.gz ${KATA_BINARY_PACKAGE} +ifeq (,$(wildcard .local/kata.tar)) + @curl -SsL -o .local/kata.tar ${KATA_BINARY_PACKAGE} endif ifeq (,$(wildcard .local/vmlinux-$(KERNEL_ARCH))) - @tar -zxf .local/kata.tar.gz -C .local/ --strip-components=1 + @tar -xf .local/kata.tar -C .local/ --strip-components=1 @cp -L .local/opt/kata/share/kata-containers/vmlinux.container .local/vmlinux-$(KERNEL_ARCH) endif ifeq (,$(wildcard bin/vmlinux-$(KERNEL_ARCH))) diff --git a/README.md b/README.md index f658b08dd..372a872a1 100644 --- a/README.md +++ b/README.md @@ -130,6 +130,16 @@ After building, run basic and integration tests: make test integration ``` +Tests that only apply to Linux are compiled out on macOS, so `make test` passes +without running them. Run those with: + +```bash +make linux-test +``` + +which runs the same tests inside the Linux dev container, as the Linux build +workflow does. + A kernel is required to run integration tests. If you do not have a kernel locally, a default kernel can be fetched using the `make fetch-default-kernel` target. diff --git a/Sources/Containerization/LinuxContainer.swift b/Sources/Containerization/LinuxContainer.swift index 34964fcdc..cc5dce93c 100644 --- a/Sources/Containerization/LinuxContainer.swift +++ b/Sources/Containerization/LinuxContainer.swift @@ -91,6 +91,19 @@ public final class LinuxContainer: Container, Sendable { /// Run the container with a minimal init process that handles signal /// forwarding and zombie reaping. public var useInit: Bool = false + /// Hand the container's cgroup to the user the container runs as, so + /// that what runs inside can place its own work under limits of its + /// own. This is what a service manager does for a user session, and + /// what a container runtime nested in this one looks for. + /// + /// The user is given the cgroup it is placed in and nothing above it, + /// so the limits this container was given still bind everything within. + /// + /// A nested runtime asks a service manager for this where one exists: + /// rootless runc talks to systemd to have a cgroup delegated to it, and + /// without that it is left with no cgroup at all. + /// https://github.com/opencontainers/runc/blob/main/docs/cgroup-v2.md + public var cgroupDelegation: Bool = false /// Additional CPU cores to allocate for the virtual machine on top /// of the container's configured `cpus` value. public var cpuOverhead: Int = 1 @@ -407,6 +420,12 @@ public final class LinuxContainer: Container, Sendable { } // Linux toggles. + if config.cgroupDelegation { + var annotations = spec.annotations ?? [:] + annotations[AnnotationKeys.containerizationCgroupDelegation] = "true" + spec.annotations = annotations + } + spec.linux?.sysctl = config.sysctl spec.linux?.maskedPaths = config.maskedPaths spec.linux?.readonlyPaths = config.readonlyPaths diff --git a/Sources/Containerization/LinuxPod.swift b/Sources/Containerization/LinuxPod.swift index 6275a4d49..2b2360b95 100644 --- a/Sources/Containerization/LinuxPod.swift +++ b/Sources/Containerization/LinuxPod.swift @@ -77,6 +77,11 @@ public final class LinuxPod: Sendable { public var cpus: Int? /// Optional per-container memory limit in bytes (can exceed pod total for oversubscription). public var memoryInBytes: UInt64? + /// Hand this container's cgroup to the user it runs as, so that what + /// runs inside can place its own work under limits of its own. The + /// user is given the cgroup it is placed in and nothing above it, so + /// the limits the pod and this container were given still bind. + public var cgroupDelegation: Bool = false /// The hostname for the container. public var hostname: String? /// The system control options for the container. @@ -310,6 +315,11 @@ public final class LinuxPod: Sendable { } // Linux toggles + if config.cgroupDelegation { + var annotations = spec.annotations ?? [:] + annotations[AnnotationKeys.containerizationCgroupDelegation] = "true" + spec.annotations = annotations + } spec.linux?.sysctl = config.sysctl spec.linux?.maskedPaths = config.maskedPaths spec.linux?.readonlyPaths = config.readonlyPaths diff --git a/Sources/ContainerizationOCI/AnnotationKeys.swift b/Sources/ContainerizationOCI/AnnotationKeys.swift index 73f8889b7..d46227f1a 100644 --- a/Sources/ContainerizationOCI/AnnotationKeys.swift +++ b/Sources/ContainerizationOCI/AnnotationKeys.swift @@ -19,6 +19,7 @@ public struct AnnotationKeys: Codable, Sendable { public static let containerizationIndexIndirect = "com.apple.containerization.index.indirect" public static let containerizationImageName = "com.apple.containerization.image.name" + public static let containerizationCgroupDelegation = "com.apple.containerization.cgroup.delegation" public static let containerdImageName = "io.containerd.image.name" public static let openContainersImageName = "org.opencontainers.image.ref.name" } diff --git a/Sources/Integration/ContainerTests.swift b/Sources/Integration/ContainerTests.swift index dd2a91656..334acec9d 100644 --- a/Sources/Integration/ContainerTests.swift +++ b/Sources/Integration/ContainerTests.swift @@ -137,6 +137,42 @@ extension IntegrationSuite { } } + /// What delegation is for: the user the container runs as can put its own + /// work under a limit, which the kernel otherwise reserves for whoever owns + /// the cgroup. + func testContainerCgroupDelegation() async throws { + let id = "test-container-cgroup-delegation" + let bs = try await bootstrap(id) + + let buffer = BufferWriter() + let container = try LinuxContainer(id, rootfs: bs.rootfs, vmm: bs.vmm) { config in + config.cgroupDelegation = true + config.process.user = ContainerizationOCI.User(uid: 1000, gid: 1000) + config.process.arguments = [ + "/bin/sh", "-c", + // A cgroup holding processes cannot enable controllers for its + // children, so step into a leaf before handing memory down. + "mkdir -p /sys/fs/cgroup/init /sys/fs/cgroup/work && " + + "echo $$ > /sys/fs/cgroup/init/cgroup.procs && " + + "echo '+memory' > /sys/fs/cgroup/cgroup.subtree_control && " + + "echo 64000000 > /sys/fs/cgroup/work/memory.max && " + + "echo LIMIT=$(cat /sys/fs/cgroup/work/memory.max)", + ] + config.process.stdout = buffer + config.bootLog = bs.bootLog + } + try await container.create() + try await container.start() + _ = try await container.wait() + try await container.stop() + + let out = String(data: buffer.data, encoding: .utf8) ?? "" + guard out.contains("LIMIT=64000000") else { + throw IntegrationError.assert( + msg: "container could not limit its own work: '\(out)'") + } + } + func testProcessEchoHi() async throws { let id = "test-process-echo-hi" let bs = try await bootstrap(id) diff --git a/Sources/Integration/PodTests.swift b/Sources/Integration/PodTests.swift index ae1caec86..eeeb7cebe 100644 --- a/Sources/Integration/PodTests.swift +++ b/Sources/Integration/PodTests.swift @@ -59,6 +59,45 @@ extension IntegrationSuite { } } + /// Delegation reaches a pod's containers too, so one container's user can + /// limit its own work without touching what the pod gave any other. + func testPodCgroupDelegation() async throws { + let id = "test-pod-cgroup-delegation" + + let bs = try await bootstrap(id) + let pod = try LinuxPod(id, vmm: bs.vmm) { config in + config.cpus = 4 + config.memoryInBytes = 1024.mib() + config.bootLog = bs.bootLog + } + + let buffer = BufferWriter() + try await pod.addContainer("delegated", rootfs: bs.rootfs) { config in + config.cgroupDelegation = true + config.process.user = ContainerizationOCI.User(uid: 1000, gid: 1000) + config.process.arguments = [ + "/bin/sh", "-c", + "mkdir -p /sys/fs/cgroup/init /sys/fs/cgroup/work && " + + "echo $$ > /sys/fs/cgroup/init/cgroup.procs && " + + "echo '+memory' > /sys/fs/cgroup/cgroup.subtree_control && " + + "echo 64000000 > /sys/fs/cgroup/work/memory.max && " + + "echo LIMIT=$(cat /sys/fs/cgroup/work/memory.max)", + ] + config.process.stdout = buffer + } + + try await pod.create() + try await pod.startContainer("delegated") + _ = try await pod.waitContainer("delegated") + try await pod.stop() + + let out = String(data: buffer.data, encoding: .utf8) ?? "" + guard out.contains("LIMIT=64000000") else { + throw IntegrationError.assert( + msg: "pod container could not limit its own work: '\(out)'") + } + } + func testPodMultipleContainers() async throws { let id = "test-pod-multiple-containers" diff --git a/Sources/Integration/Suite.swift b/Sources/Integration/Suite.swift index 9dae84351..b6dc829b3 100644 --- a/Sources/Integration/Suite.swift +++ b/Sources/Integration/Suite.swift @@ -259,12 +259,16 @@ struct IntegrationSuite: AsyncParsableCommand { // a ~2GB rootfs and a ~512MB initfs, so without reaping the dev // container fills its CoW layer in ~10 tests. if self.maxConcurrency == 1 { - let preserve = fsPath.absolutePath() + // contentsOfDirectory reports paths under /private, while testDir is + // built from FileManager's /var view of the same directory, so both + // sides are resolved before comparing. + let preserve = fsPath.resolvingSymlinksInPathWithPrivate().absolutePath() if let entries = try? FileManager.default.contentsOfDirectory( at: Self.testDir, includingPropertiesForKeys: nil ) { - for url in entries where url.absolutePath() != preserve { + for url in entries + where url.resolvingSymlinksInPathWithPrivate().absolutePath() != preserve { try? FileManager.default.removeItem(at: url) } } @@ -422,6 +426,7 @@ struct IntegrationSuite: AsyncParsableCommand { // Process basics Test("process true", testProcessTrue), Test("process false", testProcessFalse), + Test("container cgroup delegation", testContainerCgroupDelegation), Test("process echo hi", testProcessEchoHi), Test("process no executable", testProcessNoExecutable), Test("process user", testProcessUser), @@ -530,6 +535,7 @@ struct IntegrationSuite: AsyncParsableCommand { // Pods Test("pod single container", testPodSingleContainer), + Test("pod cgroup delegation", testPodCgroupDelegation), Test("pod multiple containers", testPodMultipleContainers), Test("pod container output", testPodContainerOutput), Test("pod concurrent containers", testPodConcurrentContainers), diff --git a/vminitd/Sources/Cgroup/Cgroup2Manager.swift b/vminitd/Sources/Cgroup/Cgroup2Manager.swift index 62fa78c7a..2ac253015 100644 --- a/vminitd/Sources/Cgroup/Cgroup2Manager.swift +++ b/vminitd/Sources/Cgroup/Cgroup2Manager.swift @@ -49,6 +49,7 @@ public struct Cgroup2Manager: Sendable { private static let killFile = "cgroup.kill" private static let procsFile = "cgroup.procs" private static let subtreeControlFile = "cgroup.subtree_control" + private static let threadsFile = "cgroup.threads" private static let cg2Magic = 0x6367_7270 @@ -122,6 +123,27 @@ public struct Cgroup2Manager: Sendable { ) } + /// Hand control of this cgroup to a less privileged user, which the kernel + /// defines as write access to the directory and to its `cgroup.procs`, + /// `cgroup.threads` and `cgroup.subtree_control` files. The resource files + /// are deliberately left alone: they distribute the resources of the parent + /// rather than of this cgroup, so the user they are delegated to must not + /// be able to write them. + /// https://github.com/torvalds/linux/blob/master/Documentation/admin-guide/cgroup-v2.rst + package func delegate(uid: UInt32, gid: UInt32) throws { + let owned = + [self.path] + + [Self.procsFile, Self.threadsFile, Self.subtreeControlFile].map { + self.path.appending(path: $0) + } + + for file in owned { + guard chown(file.path, uid, gid) == 0 else { + throw Error.errno(errno: errno, message: "failed to chown \(file.path)") + } + } + } + private static func writeValue(path: URL, value: String, fileName: String) throws { let file = path.appending(path: fileName) let fd = open(file.path, O_WRONLY, 0) diff --git a/vminitd/Sources/VminitdCore/ManagedContainer.swift b/vminitd/Sources/VminitdCore/ManagedContainer.swift index 545046a8f..bd07c1cd2 100644 --- a/vminitd/Sources/VminitdCore/ManagedContainer.swift +++ b/vminitd/Sources/VminitdCore/ManagedContainer.swift @@ -66,6 +66,15 @@ public actor ManagedContainer { do { try cgManager.toggleAllAvailableControllers(enable: true) + // A container that asked for its cgroup gets it, so long as it runs + // as somebody who would otherwise be unable to use it. + if spec.annotations?[AnnotationKeys.containerizationCgroupDelegation] == "true", + let user = spec.process?.user, user.uid != 0 + { + try cgManager.delegate(uid: user.uid, gid: user.gid) + log.info("delegated cgroup to \(user.uid):\(user.gid)") + } + let initProcess: any ContainerProcess if let runtimePath = ociRuntimePath {