From 979794f71dd6c0b75649b2bb7f10ee0916386b1a Mon Sep 17 00:00:00 2001 From: Aaron Paterson Date: Sun, 2 Aug 2026 18:47:54 +0000 Subject: [PATCH 01/29] Give a container the swap it asks for A container whose workload exceeds its memory limit meets the out of memory killer, with nothing between the two. The runtime it sits on can hand the guest a swap area to reclaim to instead, but nothing here asks for one. Take a size on the command line, keep it on the container's resources beside the memory limit it plays against, and make the area from it at bootstrap: a raw block file in the container's own bundle, alongside the root filesystem, which the guest agent writes a swap header to and enables. The size counts swap alone rather than the memory and swap total the runtime spec carries, so it reads against `--memory` rather than including it. A container that asks for none gets none, which is what one expected to stay within its memory wants. The area holds nothing that outlives the container, so it is made afresh at each bootstrap and the host is told not to synchronize it. --- .../ContainerResource/Container/Bundle.swift | 6 ++++ .../Container/ContainerConfiguration.swift | 6 ++++ .../ContainerAPIService/Client/Flags.swift | 13 +++++++- .../ContainerAPIService/Client/Parser.swift | 7 +++++ .../ContainerAPIService/Client/Utility.swift | 1 + .../RuntimeLinux/Server/RuntimeService.swift | 30 +++++++++++++++++++ .../ContainerConfigurationTests.swift | 23 ++++++++++++++ 7 files changed, 85 insertions(+), 1 deletion(-) diff --git a/Sources/ContainerResource/Container/Bundle.swift b/Sources/ContainerResource/Container/Bundle.swift index 217531b8c..2fd005ca0 100644 --- a/Sources/ContainerResource/Container/Bundle.swift +++ b/Sources/ContainerResource/Container/Bundle.swift @@ -23,6 +23,7 @@ public struct Bundle: Sendable { private static let kernelFilename = "kernel.json" private static let kernelBinaryFilename = "kernel.bin" private static let containerRootFsBlockFilename = "rootfs.ext4" + private static let containerSwapBlockFilename = "swap.raw" private static let containerRootFsFilename = "rootfs.json" static let containerConfigFilename = "config.json" @@ -42,6 +43,11 @@ public struct Bundle: Sendable { self.path.appendingPathComponent(Self.containerRootFsBlockFilename) } + /// The raw block file backing the container's swap area, when it has one. + public var containerSwapBlock: URL { + self.path.appendingPathComponent(Self.containerSwapBlockFilename) + } + private var containerRootfsConfig: URL { self.path.appendingPathComponent(Self.containerRootFsFilename) } diff --git a/Sources/ContainerResource/Container/ContainerConfiguration.swift b/Sources/ContainerResource/Container/ContainerConfiguration.swift index 87e0f9049..b8a868b17 100644 --- a/Sources/ContainerResource/Container/ContainerConfiguration.swift +++ b/Sources/ContainerResource/Container/ContainerConfiguration.swift @@ -166,6 +166,11 @@ public struct ContainerConfiguration: Sendable, Codable { public var cpus: Int = 4 /// Memory in bytes allocated. public var memoryInBytes: UInt64 = 1024.mib() + /// Swap in bytes allocated. When set, a raw block device of this size + /// backs the container's swap area, which the guest enables so that a + /// workload exceeding `memoryInBytes` reclaims to it. Counts swap + /// alone, not the memory and swap total the runtime spec carries. + public var swapInBytes: UInt64? /// Storage quota/size in bytes. public var storage: UInt64? /// Additional CPU cores allocated for VM overhead (guest agent, etc). @@ -177,6 +182,7 @@ public struct ContainerConfiguration: Sendable, Codable { let c = try decoder.container(keyedBy: CodingKeys.self) self.cpus = try c.decodeIfPresent(Int.self, forKey: .cpus) ?? 4 self.memoryInBytes = try c.decodeIfPresent(UInt64.self, forKey: .memoryInBytes) ?? 1024.mib() + self.swapInBytes = try c.decodeIfPresent(UInt64.self, forKey: .swapInBytes) self.storage = try c.decodeIfPresent(UInt64.self, forKey: .storage) self.cpuOverhead = try c.decodeIfPresent(Int.self, forKey: .cpuOverhead) ?? 1 } diff --git a/Sources/Services/ContainerAPIService/Client/Flags.swift b/Sources/Services/ContainerAPIService/Client/Flags.swift index 39962d436..19947f555 100644 --- a/Sources/Services/ContainerAPIService/Client/Flags.swift +++ b/Sources/Services/ContainerAPIService/Client/Flags.swift @@ -101,9 +101,10 @@ public struct Flags { public struct Resource: ParsableArguments { public init() {} - public init(cpus: Int64?, memory: String?) { + public init(cpus: Int64?, memory: String?, swap: String? = nil) { self.cpus = cpus self.memory = memory + self.swap = swap } @Option(name: .shortAndLong, help: "Number of CPUs to allocate to the container") @@ -114,6 +115,16 @@ public struct Flags { help: "Amount of memory (1MiByte granularity), with optional K, M, G, T, or P suffix" ) public var memory: String? + + @Option( + name: .customLong("swap"), + help: """ + Amount of swap to give the container (1MiByte granularity), with optional K, M, G, \ + T, or P suffix. A workload whose memory exceeds its limit reclaims to it rather \ + than meeting the out of memory killer. Counts swap alone, not memory plus swap. + """ + ) + public var swap: String? } public struct DNS: ParsableArguments { diff --git a/Sources/Services/ContainerAPIService/Client/Parser.swift b/Sources/Services/ContainerAPIService/Client/Parser.swift index a52c1499e..d0c900f9c 100644 --- a/Sources/Services/ContainerAPIService/Client/Parser.swift +++ b/Sources/Services/ContainerAPIService/Client/Parser.swift @@ -105,6 +105,7 @@ public struct Parser { public static func resources( cpus: Int64?, memory: String?, + swap: String? = nil, defaultCPUs: Int, defaultMemory: MemorySize, ) throws -> ContainerConfiguration.Resources { @@ -120,6 +121,12 @@ public struct Parser { resource.memoryInBytes = try Parser.memoryStringAsMiB(memory).mib() } + // Left unset the container gets no swap area at all, which is what a + // container that is expected to stay within its memory wants. + if let swap { + resource.swapInBytes = try Parser.memoryStringAsMiB(swap).mib() + } + return resource } diff --git a/Sources/Services/ContainerAPIService/Client/Utility.swift b/Sources/Services/ContainerAPIService/Client/Utility.swift index f6329c35a..ee7d0ff83 100644 --- a/Sources/Services/ContainerAPIService/Client/Utility.swift +++ b/Sources/Services/ContainerAPIService/Client/Utility.swift @@ -156,6 +156,7 @@ public struct Utility { config.resources = try Parser.resources( cpus: resource.cpus, memory: resource.memory, + swap: resource.swap, defaultCPUs: containerSystemConfig.container.cpus, defaultMemory: containerSystemConfig.container.memory ) diff --git a/Sources/Services/RuntimeLinux/Server/RuntimeService.swift b/Sources/Services/RuntimeLinux/Server/RuntimeService.swift index 948a65603..c1d284941 100644 --- a/Sources/Services/RuntimeLinux/Server/RuntimeService.swift +++ b/Sources/Services/RuntimeLinux/Server/RuntimeService.swift @@ -263,8 +263,13 @@ public actor RuntimeService { let id = config.id let rootfs = try bundle.containerRootfs.asMount + // A container is only given a swap area when it asked for one. + let swapLayer = try config.resources.swapInBytes.map { + try bundle.createSwapDevice(size: $0).asMount + } let container = try LinuxContainer(id, rootfs: rootfs, vmm: vmm, logger: self.log) { czConfig in try Self.configureContainer(czConfig: &czConfig, config: config, dynamicEnv: dynamicEnv, log: self.log) + czConfig.swapLayer = swapLayer czConfig.interfaces = interfaces czConfig.process.stdout = stdout czConfig.process.stderr = stderr @@ -1414,6 +1419,31 @@ extension XPCMessage { } extension ContainerResource.Bundle { + /// Create the raw block file backing the container's swap area. + /// + /// It carries no filesystem: the guest agent writes the swap header to the + /// device and enables it. Fully allocated rather than sparse, because the + /// kernel maps a swap area's blocks directly and refuses one with holes. + /// The area holds nothing that outlives the container, so it is made afresh + /// with every bootstrap and the host is told not to synchronize it. + func createSwapDevice(size: UInt64) throws -> Filesystem { + let path = self.containerSwapBlock + guard FileManager.default.createFile(atPath: path.path, contents: nil) else { + throw ContainerizationError( + .internalError, message: "failed to create swap device at \(path.path)") + } + let handle = try FileHandle(forWritingTo: path) + defer { try? handle.close() } + try handle.truncate(atOffset: size) + return .block( + format: Swap.mountType, + source: path.path, + destination: "", + options: [], + sync: .nosync + ) + } + func createLogFile() throws { // Create the log file we'll write stdio to. // O_TRUNC resolves a log delay issue on restarted containers by force-updating internal state diff --git a/Tests/ContainerResourceTests/ContainerConfigurationTests.swift b/Tests/ContainerResourceTests/ContainerConfigurationTests.swift index b1aafa0b0..eb28564f9 100644 --- a/Tests/ContainerResourceTests/ContainerConfigurationTests.swift +++ b/Tests/ContainerResourceTests/ContainerConfigurationTests.swift @@ -71,6 +71,29 @@ struct ContainerConfigurationResourcesTests { let decoded = try JSONDecoder().decode(ContainerConfiguration.self, from: stripped) #expect(decoded.resources.cpuOverhead == 1) } + + /// The size asked for has to reach the runtime, which reads the container's + /// configuration back rather than being handed the flags. + @Test func roundTripsSwap() throws { + var config = makeTestConfiguration() + config.resources.swapInBytes = 512.mib() + let data = try JSONEncoder().encode(config) + let decoded = try JSONDecoder().decode(ContainerConfiguration.self, from: data) + #expect(decoded.resources.swapInBytes == 512.mib()) + } + + /// A container that asked for no swap gets none, rather than a default size. + @Test func decodesMissingSwapAsNone() throws { + let config = makeTestConfiguration() + let data = try JSONEncoder().encode(config) + var obj = try #require(try JSONSerialization.jsonObject(with: data) as? [String: Any]) + var resources = try #require(obj["resources"] as? [String: Any]) + resources.removeValue(forKey: "swapInBytes") + obj["resources"] = resources + let stripped = try JSONSerialization.data(withJSONObject: obj) + let decoded = try JSONDecoder().decode(ContainerConfiguration.self, from: stripped) + #expect(decoded.resources.swapInBytes == nil) + } } struct ContainerConfigurationCreationDateTests { From bab4965db6a7b713a677665a86bed6669fdb7014 Mon Sep 17 00:00:00 2001 From: Aaron Paterson Date: Thu, 6 Aug 2026 18:45:04 +0000 Subject: [PATCH 02/29] Back the swap device with a sparse file The guest reaches the swap area as a block device, which the kernel takes as a single extent without consulting the host's layout, so the hole-free requirement that binds a swap file inside the guest does not bind the host file backing the device. A sparse backing costs the host only the pages the guest has actually swapped out and gives them back on discard. https://github.com/torvalds/linux/blob/master/mm/swapfile.c --- Sources/Services/RuntimeLinux/Server/RuntimeService.swift | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/Sources/Services/RuntimeLinux/Server/RuntimeService.swift b/Sources/Services/RuntimeLinux/Server/RuntimeService.swift index c1d284941..57f756b89 100644 --- a/Sources/Services/RuntimeLinux/Server/RuntimeService.swift +++ b/Sources/Services/RuntimeLinux/Server/RuntimeService.swift @@ -1422,8 +1422,12 @@ extension ContainerResource.Bundle { /// Create the raw block file backing the container's swap area. /// /// It carries no filesystem: the guest agent writes the swap header to the - /// device and enables it. Fully allocated rather than sparse, because the - /// kernel maps a swap area's blocks directly and refuses one with holes. + /// device and enables it. The file is sparse, so it costs the host only the + /// pages the guest has actually swapped out, and gives them back on + /// discard. A swap area held in a file has to be free of holes, since the + /// kernel walks its extents; the guest reaches this one as a block device, + /// which the kernel takes as a single extent without consulting the host's + /// layout. https://github.com/torvalds/linux/blob/master/mm/swapfile.c /// The area holds nothing that outlives the container, so it is made afresh /// with every bootstrap and the host is told not to synchronize it. func createSwapDevice(size: UInt64) throws -> Filesystem { From e8112a55cd0e47468ad425cd22e0ca41c9ff6a13 Mon Sep 17 00:00:00 2001 From: Aaron Paterson Date: Fri, 7 Aug 2026 14:35:08 +0000 Subject: [PATCH 03/29] config: let a swap default sit beside the memory default Swap was the only container resource a user could ask for but not configure a default for: --swap existed with nothing behind it, while --cpus and --memory both fall back to their [container] keys. Add [container] swap, defaulting to zero so a container still gets no area unless one is asked for, and resolve it the way the other two resolve. --- .../ContainerPersistence/ContainerSystemConfig.swift | 8 +++++++- .../Services/ContainerAPIService/Client/Parser.swift | 10 ++++++++-- .../Services/ContainerAPIService/Client/Utility.swift | 3 ++- 3 files changed, 17 insertions(+), 4 deletions(-) diff --git a/Sources/ContainerPersistence/ContainerSystemConfig.swift b/Sources/ContainerPersistence/ContainerSystemConfig.swift index cd3a156f9..d18f26c5b 100644 --- a/Sources/ContainerPersistence/ContainerSystemConfig.swift +++ b/Sources/ContainerPersistence/ContainerSystemConfig.swift @@ -115,19 +115,25 @@ final public class BuildConfig: Codable, Sendable { final public class ContainerConfig: Codable, Sendable { public static let defaultCPUs = 4 public static let defaultMemory = try! MemorySize("1g") + /// No swap area, so a container stays within the memory it is given + /// unless the operator asks for headroom beyond it. + public static let defaultSwap = try! MemorySize("0") public let cpus: Int public let memory: MemorySize + public let swap: MemorySize - public init(cpus: Int = defaultCPUs, memory: MemorySize = defaultMemory) { + public init(cpus: Int = defaultCPUs, memory: MemorySize = defaultMemory, swap: MemorySize = defaultSwap) { self.cpus = cpus self.memory = memory + self.swap = swap } public init(from decoder: any Decoder) throws { let container = try decoder.container(keyedBy: CodingKeys.self) self.cpus = try container.decodeIfPresent(Int.self, forKey: .cpus) ?? Self.defaultCPUs self.memory = try container.decodeIfPresent(MemorySize.self, forKey: .memory) ?? Self.defaultMemory + self.swap = try container.decodeIfPresent(MemorySize.self, forKey: .swap) ?? Self.defaultSwap } } diff --git a/Sources/Services/ContainerAPIService/Client/Parser.swift b/Sources/Services/ContainerAPIService/Client/Parser.swift index d0c900f9c..57b9ce960 100644 --- a/Sources/Services/ContainerAPIService/Client/Parser.swift +++ b/Sources/Services/ContainerAPIService/Client/Parser.swift @@ -108,10 +108,15 @@ public struct Parser { swap: String? = nil, defaultCPUs: Int, defaultMemory: MemorySize, + defaultSwap: MemorySize = ContainerConfig.defaultSwap, ) throws -> ContainerConfiguration.Resources { var resource = ContainerConfiguration.Resources() resource.cpus = defaultCPUs resource.memoryInBytes = Int64(defaultMemory.measurement.converted(to: .mebibytes).value).mib() + let defaultSwapInBytes = Int64(defaultSwap.measurement.converted(to: .mebibytes).value).mib() + if defaultSwapInBytes > 0 { + resource.swapInBytes = defaultSwapInBytes + } if let cpus { resource.cpus = Int(cpus) @@ -121,8 +126,9 @@ public struct Parser { resource.memoryInBytes = try Parser.memoryStringAsMiB(memory).mib() } - // Left unset the container gets no swap area at all, which is what a - // container that is expected to stay within its memory wants. + // Left unset everywhere the container gets no swap area at all, which + // is what a container expected to stay within its memory wants; the + // flag overrides the configured default the way memory does. if let swap { resource.swapInBytes = try Parser.memoryStringAsMiB(swap).mib() } diff --git a/Sources/Services/ContainerAPIService/Client/Utility.swift b/Sources/Services/ContainerAPIService/Client/Utility.swift index ee7d0ff83..0bef87be9 100644 --- a/Sources/Services/ContainerAPIService/Client/Utility.swift +++ b/Sources/Services/ContainerAPIService/Client/Utility.swift @@ -158,7 +158,8 @@ public struct Utility { memory: resource.memory, swap: resource.swap, defaultCPUs: containerSystemConfig.container.cpus, - defaultMemory: containerSystemConfig.container.memory + defaultMemory: containerSystemConfig.container.memory, + defaultSwap: containerSystemConfig.container.swap ) let tmpfs = try Parser.tmpfsMounts(management.tmpFs) From bd860489be74c5f758bc84422a8d0ac56217d6c1 Mon Sep 17 00:00:00 2001 From: Aaron Paterson Date: Sun, 9 Aug 2026 20:45:49 +0000 Subject: [PATCH 04/29] Describe a pod, and what a sandbox is to the runtime A container runs in a pod, sharing that pod's machine with whatever else is in it. The pod holds what the machine is: its processors, memory and swap, the networks its containers reach the world through, the hostname they answer to, the resolver and hosts file they read, and the kernel parameters they share. A container holds what is its own: its image, its process, its mounts and its filesystem. That division is the one the container runtime interface draws between a sandbox and the containers in it, and it decides where each setting lives here. A container names the pod it runs in. A caller that names none is asking for a pod of its own and is given a name for it before the container is made, so the rest of the code never has to ask whether a container has one. The runtime reads a bundle to learn what it drives, so a bundle says which of the two it holds, and a pod's carries the machine while the containers placed in it carry their own. https://github.com/kubernetes/cri-api/blob/master/pkg/apis/runtime/v1/api.proto --- Package.resolved | 7 +- .../ContainerResource/Container/Bundle.swift | 22 +++ .../Container/ContainerConfiguration.swift | 16 ++ .../Pod/PodConfiguration.swift | 152 ++++++++++++++++++ .../ContainerResource/Pod/PodSnapshot.swift | 64 ++++++++ .../RuntimeClient/RuntimeConfiguration.swift | 6 + .../RuntimeLinux/Server/Sandbox.swift | 90 +++++++++++ 7 files changed, 353 insertions(+), 4 deletions(-) create mode 100644 Sources/ContainerResource/Pod/PodConfiguration.swift create mode 100644 Sources/ContainerResource/Pod/PodSnapshot.swift create mode 100644 Sources/Services/RuntimeLinux/Server/Sandbox.swift diff --git a/Package.resolved b/Package.resolved index 31cd926e9..8e7e53201 100644 --- a/Package.resolved +++ b/Package.resolved @@ -1,5 +1,5 @@ { - "originHash" : "35d1f07a3595a3ebabac1430f46ebb8b4e54d4531874f4ff7df33b64092df5f3", + "originHash" : "bfd17c6a1af5c6b5efd9e28e4f336c0cce54aa18dec4cad43b9d8a577f32df88", "pins" : [ { "identity" : "async-http-client", @@ -13,10 +13,9 @@ { "identity" : "containerization", "kind" : "remoteSourceControl", - "location" : "https://github.com/apple/containerization.git", + "location" : "https://github.com/MayCXC/containerization.git", "state" : { - "revision" : "5427fd21ded4b84034126caef5b3182900b4776d", - "version" : "0.41.0" + "revision" : "67e97e92d2843eaabb9d4b28d5edb3594a227e40" } }, { diff --git a/Sources/ContainerResource/Container/Bundle.swift b/Sources/ContainerResource/Container/Bundle.swift index 2fd005ca0..36f418a54 100644 --- a/Sources/ContainerResource/Container/Bundle.swift +++ b/Sources/ContainerResource/Container/Bundle.swift @@ -27,6 +27,7 @@ public struct Bundle: Sendable { private static let containerRootFsFilename = "rootfs.json" static let containerConfigFilename = "config.json" + static let podConfigFilename = "pod.json" /// The path to the bundle. public let path: URL @@ -81,6 +82,22 @@ public struct Bundle: Sendable { try load(path: self.path.appendingPathComponent(Self.containerConfigFilename)) } } + + /// The configuration of the pod this bundle holds, when the bundle is a + /// pod's rather than a single container's. + public var podConfiguration: PodConfiguration { + get throws { + try load(path: self.path.appendingPathComponent(Self.podConfigFilename)) + } + } + + /// Whether this bundle holds a pod, whose containers keep bundles of their + /// own, rather than a single container. + public var isPod: Bool { + FileManager.default.fileExists( + atPath: self.path.appendingPathComponent(Self.podConfigFilename).path + ) + } } extension Bundle { @@ -89,6 +106,7 @@ extension Bundle { initialFilesystem: Filesystem, kernel: Kernel, containerConfiguration: ContainerConfiguration? = nil, + podConfiguration: PodConfiguration? = nil, containerRootFilesystem: Filesystem? = nil, options: ContainerCreateOptions? = nil ) throws -> Bundle { @@ -116,6 +134,10 @@ extension Bundle { try bundle.write(filename: Self.containerConfigFilename, value: containerConfiguration) } + if let podConfiguration { + try bundle.write(filename: Self.podConfigFilename, value: podConfiguration) + } + if let rootFsOverride = options?.rootFsOverride { try bundle.setContainerRootFs(fs: rootFsOverride) } else if let containerRootFilesystem { diff --git a/Sources/ContainerResource/Container/ContainerConfiguration.swift b/Sources/ContainerResource/Container/ContainerConfiguration.swift index b8a868b17..8cbdc88fa 100644 --- a/Sources/ContainerResource/Container/ContainerConfiguration.swift +++ b/Sources/ContainerResource/Container/ContainerConfiguration.swift @@ -45,6 +45,15 @@ public struct ContainerConfiguration: Sendable, Codable { /// Resource values for the container. public var resources: Resources = .init() /// Name of the runtime that supports the container. + /// The pod the container runs in. + /// + /// A container is always in one, sharing that pod's machine with whatever + /// else is in it, so a container alone in a pod is the same arrangement + /// with one member. A caller that names no pod is asking for one of its + /// own and is given a name for it before the container is made, which is + /// why this is not a question the rest of the code has to ask. + public var pod: String = PodConfiguration.generateId() + public var runtimeHandler: String = "container-runtime-linux" /// Configure exposing virtualization support in the container. public var virtualization: Bool = false @@ -87,6 +96,7 @@ public struct ContainerConfiguration: Sendable, Codable { case initProcess case platform case resources + case pod case runtimeHandler case virtualization case ssh @@ -125,6 +135,12 @@ public struct ContainerConfiguration: Sendable, Codable { initProcess = try container.decode(ProcessConfiguration.self, forKey: .initProcess) platform = try container.decodeIfPresent(ContainerizationOCI.Platform.self, forKey: .platform) ?? .current resources = try container.decodeIfPresent(Resources.self, forKey: .resources) ?? .init() + // A container written before a container was always in a pod carries + // no pod, and there is no machine for it to be in. It fails to read, + // and a container that fails to read is taken away at boot, which is + // what already happens to any container this version cannot make sense + // of. + pod = try container.decode(String.self, forKey: .pod) runtimeHandler = try container.decodeIfPresent(String.self, forKey: .runtimeHandler) ?? "container-runtime-linux" virtualization = try container.decodeIfPresent(Bool.self, forKey: .virtualization) ?? false ssh = try container.decodeIfPresent(Bool.self, forKey: .ssh) ?? false diff --git a/Sources/ContainerResource/Pod/PodConfiguration.swift b/Sources/ContainerResource/Pod/PodConfiguration.swift new file mode 100644 index 000000000..1314c7713 --- /dev/null +++ b/Sources/ContainerResource/Pod/PodConfiguration.swift @@ -0,0 +1,152 @@ +//===----------------------------------------------------------------------===// +// Copyright © 2026 Apple Inc. and the container project authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//===----------------------------------------------------------------------===// + +import ContainerizationOCI +import Foundation + +/// The configuration of a pod. +/// +/// The shape follows the runtime interface's `PodSandboxConfig`. +/// https://github.com/kubernetes/cri-api/blob/master/pkg/apis/runtime/v1/api.proto +public struct PodConfiguration: Sendable, Codable { + /// Identifier for the pod. + public var id: String + + /// Mint ids the way containers mint theirs, a lowercased UUID. + /// + /// A pod is a lifetime rather than a description of what it holds, so it + /// is told apart by the id it is given. Two pods may hold containers from + /// the same image, and a pod named after a container it holds would claim + /// the runtime service that container claims. + public static func generateId() -> String { UUID().uuidString.lowercased() } + + /// The runtime that runs the pod's machine. + /// + /// A pod is one machine, so its containers run under one runtime, and it + /// is the pod that names it. A container placed in a pod asking for a + /// different one is asking for a machine this pod is not. + public var runtimeHandler: String = "container-runtime-linux" + + /// Resources like cpu, memory and swap. A container may hold its own + /// limit within these; left alone it draws on the whole pool. + public var resources: ContainerConfiguration.Resources = .init() + + /// The hostname for the pod. + public var hostname: String? + + /// The DNS configuration for the pod. + public var dns: ContainerConfiguration.DNSConfiguration? + + /// Kernel parameters for the pod. Its containers share one kernel, so none + /// of them can set one for itself alone. + public var sysctls: [String: String] = [:] + + /// The networks the pod attaches to. + public var networks: [AttachmentConfiguration] = [] + + /// Ports published to the host. + public var publishedPorts: [PublishPort] = [] + + /// Whether the pod's containers see each other's processes. + public var shareProcessNamespace: Bool = false + + /// Enable nested virtualization support. + public var virtualization: Bool = false + + /// Enable Rosetta. + public var rosetta: Bool = false + + /// Key-value properties for the pod. + public var labels: [String: String] = [:] + + /// Configured platform for the pod. + public var platform: ContainerizationOCI.Platform = .current + + /// The time at which the pod was created. + public var creationDate: Date = Date() + + public init(id: String) { + self.id = id + } + + /// The sandbox a container asks for when it names no pod of its own. + /// + /// The runtime interface has the caller create a sandbox and then create + /// containers in it, so a container that came without one has a sandbox + /// made for it first, out of the fields that are the sandbox's to hold. + /// https://github.com/kubernetes/cri-api/blob/master/pkg/apis/runtime/v1/api.proto + public init(sandboxFor container: ContainerConfiguration) { + self.init(id: container.pod) + runtimeHandler = container.runtimeHandler + resources = container.resources + dns = container.dns + sysctls = container.sysctls + networks = container.networks + publishedPorts = container.publishedPorts + virtualization = container.virtualization + rosetta = container.rosetta + platform = container.platform + labels = container.labels + // Nobody named this pod. It carries a name of its own because the + // container needed a machine to run in and was given one, which is what + // a volume mounted without a name is. + labels[Self.anonymousLabel] = "" + } + + /// Read a pod written by any version of this service. + /// + /// A pod outlives the process that wrote it, so a field this type gains is + /// a field the pods already on disk do not carry. The compiler's own + /// decoding asks for every key and fails on the first one missing, which + /// would make every pod written before the field unreadable, and a pod + /// that cannot be read is a pod that is not there: the service skips it, + /// and the container it holds is told its pod does not exist. Each field + /// that has a default is taken as absent-means-default, which is how the + /// container configuration reads its own. + public init(from decoder: Decoder) throws { + let values = try decoder.container(keyedBy: CodingKeys.self) + id = try values.decode(String.self, forKey: .id) + runtimeHandler = try values.decodeIfPresent(String.self, forKey: .runtimeHandler) ?? "container-runtime-linux" + resources = try values.decodeIfPresent(ContainerConfiguration.Resources.self, forKey: .resources) ?? .init() + hostname = try values.decodeIfPresent(String.self, forKey: .hostname) + dns = try values.decodeIfPresent(ContainerConfiguration.DNSConfiguration.self, forKey: .dns) + sysctls = try values.decodeIfPresent([String: String].self, forKey: .sysctls) ?? [:] + networks = try values.decodeIfPresent([AttachmentConfiguration].self, forKey: .networks) ?? [] + publishedPorts = try values.decodeIfPresent([PublishPort].self, forKey: .publishedPorts) ?? [] + shareProcessNamespace = try values.decodeIfPresent(Bool.self, forKey: .shareProcessNamespace) ?? false + virtualization = try values.decodeIfPresent(Bool.self, forKey: .virtualization) ?? false + rosetta = try values.decodeIfPresent(Bool.self, forKey: .rosetta) ?? false + labels = try values.decodeIfPresent([String: String].self, forKey: .labels) ?? [:] + platform = try values.decodeIfPresent(ContainerizationOCI.Platform.self, forKey: .platform) ?? .current + creationDate = try values.decodeIfPresent(Date.self, forKey: .creationDate) ?? Date() + } +} + +/// The runtime state of a pod. +public enum PodState: String, Sendable, Codable { + case ready + case notReady +} + +extension PodConfiguration { + /// Reserved label key for marking anonymous pods + public static let anonymousLabel = "com.apple.container.resource.anonymous" + + /// Whether this is an anonymous pod (detected via label) + public var isAnonymous: Bool { + labels[Self.anonymousLabel] != nil + } +} diff --git a/Sources/ContainerResource/Pod/PodSnapshot.swift b/Sources/ContainerResource/Pod/PodSnapshot.swift new file mode 100644 index 000000000..8f77f0556 --- /dev/null +++ b/Sources/ContainerResource/Pod/PodSnapshot.swift @@ -0,0 +1,64 @@ +//===----------------------------------------------------------------------===// +// Copyright © 2026 Apple Inc. and the container project authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//===----------------------------------------------------------------------===// + +import ContainerizationOCI +import Foundation + +/// A snapshot of a pod along with its configuration +/// and any runtime state information. +/// +/// The shape follows the runtime interface's `PodSandboxStatus`. +/// https://github.com/kubernetes/cri-api/blob/master/pkg/apis/runtime/v1/api.proto +public struct PodSnapshot: Codable, Sendable { + /// The configuration of the pod. + public var configuration: PodConfiguration + + /// Identifier of the pod. + public var id: String { + configuration.id + } + + /// Configured platform for the pod. + public var platform: ContainerizationOCI.Platform { + configuration.platform + } + + /// The runtime state of the pod. + public var state: PodState + + /// Network interfaces attached to the pod. + public var networks: [Attachment] + + /// Identifiers of the containers in the pod. + public var containers: [String] + + /// When the pod was started. + public var startedDate: Date? + + public init( + configuration: PodConfiguration, + state: PodState, + networks: [Attachment], + containers: [String] = [], + startedDate: Date? = nil + ) { + self.configuration = configuration + self.state = state + self.networks = networks + self.containers = containers + self.startedDate = startedDate + } +} diff --git a/Sources/Services/Runtime/RuntimeClient/RuntimeConfiguration.swift b/Sources/Services/Runtime/RuntimeClient/RuntimeConfiguration.swift index 29507bb70..3caa0d025 100644 --- a/Sources/Services/Runtime/RuntimeClient/RuntimeConfiguration.swift +++ b/Sources/Services/Runtime/RuntimeClient/RuntimeConfiguration.swift @@ -28,6 +28,10 @@ public struct RuntimeConfiguration: Codable, Sendable { public let initialFilesystem: Filesystem public let kernel: Kernel public let containerConfiguration: ContainerConfiguration? + /// The pod this instance drives, when it drives a pod rather than a single + /// container. The containers that share the pod's machine keep runtime + /// configurations of their own. + public let podConfiguration: PodConfiguration? public let containerRootFilesystem: Filesystem? public let options: ContainerCreateOptions? public let runtimeData: Data? @@ -37,6 +41,7 @@ public struct RuntimeConfiguration: Codable, Sendable { initialFilesystem: Filesystem, kernel: Kernel, containerConfiguration: ContainerConfiguration? = nil, + podConfiguration: PodConfiguration? = nil, containerRootFilesystem: Filesystem? = nil, options: ContainerCreateOptions? = nil, runtimeData: Data? = nil @@ -45,6 +50,7 @@ public struct RuntimeConfiguration: Codable, Sendable { self.initialFilesystem = initialFilesystem self.kernel = kernel self.containerConfiguration = containerConfiguration + self.podConfiguration = podConfiguration self.containerRootFilesystem = containerRootFilesystem self.options = options self.runtimeData = runtimeData diff --git a/Sources/Services/RuntimeLinux/Server/Sandbox.swift b/Sources/Services/RuntimeLinux/Server/Sandbox.swift new file mode 100644 index 000000000..5266e7386 --- /dev/null +++ b/Sources/Services/RuntimeLinux/Server/Sandbox.swift @@ -0,0 +1,90 @@ +//===----------------------------------------------------------------------===// +// Copyright © 2026 Apple Inc. and the container project authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//===----------------------------------------------------------------------===// + +import Containerization +import ContainerizationError +import ContainerizationOS +import Foundation + +import struct ContainerizationOS.Terminal + +/// The machine the runtime service drives, and the containers running in it. +/// +/// A pod holds several containers and a standalone container holds one. The +/// service speaks to both the same way, naming the container it means, which +/// is how the runtime interface addresses a container in a sandbox. +/// https://github.com/kubernetes/cri-api/blob/master/pkg/apis/runtime/v1/api.proto +protocol Sandbox: Sendable { + /// Boot the machine and set up the containers registered in it. + func create() async throws + + /// Start a container's init process. + func startContainer(_ id: String) async throws + + /// Stop a container, leaving the machine running for the others. + func stopContainer(_ id: String) async throws + + /// Signal a container's init process. + func killContainer(_ id: String, signal: Signal) async throws + + /// Wait for a container's init process to exit. + func waitContainer(_ id: String, timeoutInSeconds: Int64?) async throws -> ExitStatus + + /// Resize the terminal of a container's init process. + func resizeContainer(_ id: String, to: Terminal.Size) async throws + + /// Run an additional process in a container. + func execInContainer( + _ id: String, + processID: String, + configuration: @Sendable @escaping (inout LinuxProcessConfiguration) throws -> Void + ) async throws -> LinuxProcess + + /// Resource usage, for the named containers or for all of them. + func statistics(containerIDs: [String]?, categories: StatCategory) async throws -> [ContainerStatistics] + + /// Act on a path in a container's filesystem. + func filesystemOperation(_ id: String, operation: FilesystemOperation, path: String) async throws + + /// Copy a file or directory from the host into a container. + func copyIn(_ id: String, from source: URL, to destination: URL, mode: UInt32, createParents: Bool) async throws + + /// Copy a file or directory out of a container to the host. + func copyOut(_ id: String, from source: URL, to destination: URL, createParents: Bool) async throws + + /// Open a vsock connection to a port in the machine. + func dialVsock(port: UInt32) async throws -> FileHandle + + /// Stop every container and power the machine off. + func stop() async throws + + /// Ask the guest to hold itself to a memory size, which the machine's + /// containers share. + func setTargetMemorySize(_ bytes: UInt64) async throws +} + +/// A pod already addresses its containers by name, so it is a sandbox as it +/// stands, save for the copies, which take a transfer size the runtime leaves +/// at its default. +extension LinuxPod: Sandbox { + func copyIn(_ id: String, from source: URL, to destination: URL, mode: UInt32, createParents: Bool) async throws { + try await self.copyIn(id, from: source, to: destination, mode: mode, createParents: createParents, chunkSize: Self.defaultCopyChunkSize) + } + + func copyOut(_ id: String, from source: URL, to destination: URL, createParents: Bool) async throws { + try await self.copyOut(id, from: source, to: destination, createParents: createParents, chunkSize: Self.defaultCopyChunkSize) + } +} From 1178136089c9a39191cd359adb535894ac6ac195 Mon Sep 17 00:00:00 2001 From: Aaron Paterson Date: Sun, 9 Aug 2026 20:46:40 +0000 Subject: [PATCH 05/29] Run every container in a pod A container ran in a machine of its own, which nothing else could be in. It runs in a pod, and the pod holds that machine. What the machine held for one container a pod holds for the containers in it, so the settings that describe a machine belong to the pod and the operations on a container stay with the container. The pod holds the hostname, the resolver, the hosts file and the kernel parameters. The runtime interface carries all four on the sandbox, and the last of them cannot be a container's: the containers share one kernel, so none of them can set a parameter for itself alone. A container asks for none of them, and a pod given none derives its resolver, its name and its hosts file from the network it attaches to, the way a machine derived them from its own attachments. The pod is written down where the container is, from the same configuration, and the kernel and the init filesystem the machine boots are resolved there already, so the pod is written from those rather than from a second look at the image. A pod named on the command line has to be there before a container names it, since naming one joins it, and joining a pod is joining its network, so the options that describe a network are refused to a container that joins one. The machine comes into being when a container in it starts, which is when a container brought up the machine that was its own. It is asked to run holding the containers the pod holds, one request whether it is coming up around them or is already up and taking in one that is new to it, and the runtime knows which of the two it is from the state it keeps. Stopping a container stops that container. The machine goes down once the last container in it has stopped, which is one rule for any number of them: a machine given a single container goes down when that container stops, and a machine holding several stays up for the rest. Nothing asks how many there are. Stopping one container in a machine is its own call, since the call that stops a machine stops everything in it first. A request names the container it means, so a machine holding one is answered the same way as a machine holding several and no request means "the only one here". https://github.com/kubernetes/cri-api/blob/master/pkg/apis/runtime/v1/api.proto --- Sources/APIServer/APIServer+Start.swift | 38 + .../Container/ContainerCreate.swift | 1 + .../RuntimeLinuxHelper+Start.swift | 2 + .../Client/ClientPod.swift | 117 ++ .../ContainerAPIService/Client/Flags.swift | 11 + .../ContainerAPIService/Client/Utility.swift | 73 +- .../ContainerAPIService/Client/XPC+.swift | 18 + .../Server/Containers/ContainersService.swift | 211 ++-- .../Server/Pods/PodsHarness.swift | 106 ++ .../Server/Pods/PodsService.swift | 569 +++++++++ .../Runtime/RuntimeClient/RuntimeClient.swift | 140 ++- .../Runtime/RuntimeClient/RuntimeKeys.swift | 12 + .../Runtime/RuntimeClient/RuntimeRoutes.swift | 10 +- .../RuntimeLinux/Server/RuntimeService.swift | 1015 +++++++++++------ Tests/K8sPluginTests/K8sListTests.swift | 1 + 15 files changed, 1817 insertions(+), 507 deletions(-) create mode 100644 Sources/Services/ContainerAPIService/Client/ClientPod.swift create mode 100644 Sources/Services/ContainerAPIService/Server/Pods/PodsHarness.swift create mode 100644 Sources/Services/ContainerAPIService/Server/Pods/PodsService.swift diff --git a/Sources/APIServer/APIServer+Start.swift b/Sources/APIServer/APIServer+Start.swift index 936abd913..f54c0517b 100644 --- a/Sources/APIServer/APIServer+Start.swift +++ b/Sources/APIServer/APIServer+Start.swift @@ -78,6 +78,15 @@ extension APIServer { routes: &routes ) await containersService.setNetworksService(networkService) + let podsService = try initializePodsService( + pluginLoader: pluginLoader, + containerSystemConfig: containerSystemConfig, + log: log, + routes: &routes + ) + await podsService.setContainersService(containersService) + await podsService.setNetworksService(networkService) + await containersService.setPodsService(podsService) initializeHealthCheckService(log: log, routes: &routes) try initializeKernelService(log: log, routes: &routes) let volumesService = try await initializeVolumeService(containersService: containersService, log: log, routes: &routes) @@ -271,6 +280,35 @@ extension APIServer { routes[XPCRoute.getDefaultKernel] = XPCServer.route(harness.getDefaultKernel) } + private func initializePodsService( + pluginLoader: PluginLoader, + containerSystemConfig: ContainerSystemConfig, + log: Logger, + routes: inout [XPCRoute: XPCServer.RouteHandler] + ) throws -> PodsService { + log.info("initializing pods service") + + let appRootURL = URL(fileURLWithPath: appRoot.string) + let service = try PodsService( + appRoot: appRootURL, + pluginLoader: pluginLoader, + containerSystemConfig: containerSystemConfig, + debugHelpers: debug, + log: log + ) + let harness = PodsHarness(service: service, log: log) + + routes[XPCRoute.podCreate] = XPCServer.route(harness.create) + routes[XPCRoute.podStart] = XPCServer.route(harness.start) + routes[XPCRoute.podStop] = XPCServer.route(harness.stop) + routes[XPCRoute.podDelete] = XPCServer.route(harness.delete) + routes[XPCRoute.podInspect] = XPCServer.route(harness.inspect) + routes[XPCRoute.podList] = XPCServer.route(harness.list) + routes[XPCRoute.podUpdate] = XPCServer.route(harness.update) + + return service + } + private func initializeContainersService( pluginLoader: PluginLoader, containerSystemConfig: ContainerSystemConfig, diff --git a/Sources/ContainerCommands/Container/ContainerCreate.swift b/Sources/ContainerCommands/Container/ContainerCreate.swift index 97febb060..4a4279f29 100644 --- a/Sources/ContainerCommands/Container/ContainerCreate.swift +++ b/Sources/ContainerCommands/Container/ContainerCreate.swift @@ -91,6 +91,7 @@ extension Application { let options = ContainerCreateOptions(autoRemove: managementFlags.remove) let client = ContainerClient() + try await client.create(configuration: ck.0, options: options, kernel: ck.1, initImage: ck.2) if !self.managementFlags.cidfile.isEmpty { diff --git a/Sources/Plugins/RuntimeLinux/RuntimeLinuxHelper+Start.swift b/Sources/Plugins/RuntimeLinux/RuntimeLinuxHelper+Start.swift index d4c049b4b..c84753bb1 100644 --- a/Sources/Plugins/RuntimeLinux/RuntimeLinuxHelper+Start.swift +++ b/Sources/Plugins/RuntimeLinux/RuntimeLinuxHelper+Start.swift @@ -94,9 +94,11 @@ extension RuntimeLinuxHelper { connection: anonymousConnection, routes: [ RuntimeRoutes.bootstrap.rawValue: XPCServer.route(server.bootstrap), + RuntimeRoutes.updateResources.rawValue: XPCServer.route(server.updateResources), RuntimeRoutes.createProcess.rawValue: XPCServer.route(server.createProcess), RuntimeRoutes.state.rawValue: XPCServer.route(server.state), RuntimeRoutes.stop.rawValue: XPCServer.route(server.stop), + RuntimeRoutes.stopContainer.rawValue: XPCServer.route(server.stopContainer), RuntimeRoutes.kill.rawValue: XPCServer.route(server.kill), RuntimeRoutes.resize.rawValue: XPCServer.route(server.resize), RuntimeRoutes.wait.rawValue: XPCServer.route(server.wait), diff --git a/Sources/Services/ContainerAPIService/Client/ClientPod.swift b/Sources/Services/ContainerAPIService/Client/ClientPod.swift new file mode 100644 index 000000000..6a609aff9 --- /dev/null +++ b/Sources/Services/ContainerAPIService/Client/ClientPod.swift @@ -0,0 +1,117 @@ +//===----------------------------------------------------------------------===// +// Copyright © 2026 Apple Inc. and the container project authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//===----------------------------------------------------------------------===// + +import ContainerResource +import ContainerXPC +import Containerization +import ContainerizationError +import Foundation + +/// Pods: machines that several containers run inside and share. +public struct ClientPod { + static let serviceIdentifier = "com.apple.container.apiserver" + + /// Write down a pod, so containers can be placed in it before it boots. + public static func create( + configuration: PodConfiguration, + kernel: Kernel, + initImage: String? = nil + ) async throws { + let client = XPCClient(service: serviceIdentifier) + let message = XPCMessage(route: .podCreate) + message.set(key: .podConfig, value: try JSONEncoder().encode(configuration)) + message.set(key: .kernel, value: try JSONEncoder().encode(kernel)) + if let initImage { + message.set(key: .initImage, value: initImage) + } + _ = try await client.send(message) + } + + /// Make the sandbox a container is created in. + /// + /// The runtime interface has a sandbox created and left ready before + /// containers are created in it. Here a container joins its pod's machine + /// before that machine boots, and booting it is what starts the containers + /// placed in it, so the sandbox is written down here and comes up with its + /// container rather than ahead of it. + /// https://github.com/kubernetes/cri-api/blob/master/pkg/apis/runtime/v1/api.proto + public static func run( + configuration: PodConfiguration, + kernel: Kernel, + initImage: String? = nil + ) async throws { + try await Self.create(configuration: configuration, kernel: kernel, initImage: initImage) + } + + /// Boot a pod's machine, with the containers that belong to it inside. + public static func start(_ id: String) async throws { + let client = XPCClient(service: serviceIdentifier) + let message = XPCMessage(route: .podStart) + message.set(key: .podId, value: id) + _ = try await client.send(message) + } + + /// Stop a pod's machine, and with it every container inside. + public static func stop(_ id: String) async throws { + let client = XPCClient(service: serviceIdentifier) + let message = XPCMessage(route: .podStop) + message.set(key: .podId, value: id) + _ = try await client.send(message) + } + + /// Take a pod away, along with its containers when forced. + public static func delete(_ id: String, force: Bool = false) async throws { + let client = XPCClient(service: serviceIdentifier) + let message = XPCMessage(route: .podDelete) + message.set(key: .podId, value: id) + message.set(key: .forceDelete, value: force) + _ = try await client.send(message) + } + + /// Everything known about a pod, including the containers in it. + public static func inspect(_ id: String) async throws -> PodSnapshot { + let client = XPCClient(service: serviceIdentifier) + let message = XPCMessage(route: .podInspect) + message.set(key: .podId, value: id) + let reply = try await client.send(message) + + guard let data = reply.dataNoCopy(key: .podSnapshot) else { + throw ContainerizationError(.notFound, message: "pod not found: \(id)") + } + return try JSONDecoder().decode(PodSnapshot.self, from: data) + } + + /// Every pod. + public static func list() async throws -> [PodSnapshot] { + let client = XPCClient(service: serviceIdentifier) + let message = XPCMessage(route: .podList) + let reply = try await client.send(message) + + guard let data = reply.dataNoCopy(key: .podSnapshots) else { + return [] + } + return try JSONDecoder().decode([PodSnapshot].self, from: data) + } + + /// Hold a running pod to a memory size, which its containers share. + public static func update(_ id: String, memoryInBytes: UInt64) async throws { + let client = XPCClient(service: serviceIdentifier) + let message = XPCMessage(route: .podUpdate) + message.set(key: .podId, value: id) + message.set(key: .memoryInBytes, value: memoryInBytes) + _ = try await client.send(message) + } +} diff --git a/Sources/Services/ContainerAPIService/Client/Flags.swift b/Sources/Services/ContainerAPIService/Client/Flags.swift index 19947f555..6e840e98d 100644 --- a/Sources/Services/ContainerAPIService/Client/Flags.swift +++ b/Sources/Services/ContainerAPIService/Client/Flags.swift @@ -195,6 +195,7 @@ public struct Flags { networks: [String], os: String, platform: String?, + pod: String? = nil, publishPorts: [String], publishSockets: [String], readOnly: Bool, @@ -227,6 +228,7 @@ public struct Flags { self.networks = networks self.os = os self.platform = platform + self.pod = pod self.publishPorts = publishPorts self.publishSockets = publishSockets self.readOnly = readOnly @@ -322,6 +324,15 @@ public struct Flags { @Option(name: .long, help: "Use the specified name as the container ID") public var name: String? + @Option( + name: .long, + help: """ + Run the container in a pod, whose machine it shares with the pod's other \ + containers. Without this the container is given a machine of its own. + """ + ) + public var pod: String? + @Option(name: [.customLong("network")], help: "Attach the container to a network (format: [,mac=XX:XX:XX:XX:XX:XX][,mtu=VALUE])") public var networks: [String] = [] diff --git a/Sources/Services/ContainerAPIService/Client/Utility.swift b/Sources/Services/ContainerAPIService/Client/Utility.swift index 0bef87be9..edee04a9f 100644 --- a/Sources/Services/ContainerAPIService/Client/Utility.swift +++ b/Sources/Services/ContainerAPIService/Client/Utility.swift @@ -153,14 +153,32 @@ public struct Utility { var config = ContainerConfiguration(id: id, image: description, process: pc) config.platform = requestedPlatform - config.resources = try Parser.resources( - cpus: resource.cpus, - memory: resource.memory, - swap: resource.swap, - defaultCPUs: containerSystemConfig.container.cpus, - defaultMemory: containerSystemConfig.container.memory, - defaultSwap: containerSystemConfig.container.swap - ) + if let pod = management.pod { + // A container in a pod falls back to what the pod's machine holds + // rather than to what a machine of its own would be given, so a + // container that named no limit draws on the whole of the pod's, + // and one that named a limit still holds to it. + var podResources = try await ClientPod.inspect(pod).configuration.resources + if let cpus = resource.cpus { + podResources.cpus = Int(cpus) + } + if let memory = resource.memory { + podResources.memoryInBytes = try Parser.memoryStringAsMiB(memory).mib() + } + if let swap = resource.swap { + podResources.swapInBytes = try Parser.memoryStringAsMiB(swap).mib() + } + config.resources = podResources + } else { + config.resources = try Parser.resources( + cpus: resource.cpus, + memory: resource.memory, + swap: resource.swap, + defaultCPUs: containerSystemConfig.container.cpus, + defaultMemory: containerSystemConfig.container.memory, + defaultSwap: containerSystemConfig.container.swap + ) + } let tmpfs = try Parser.tmpfsMounts(management.tmpFs) let volumesOrFs = try Parser.volumes(management.volumes) @@ -237,6 +255,41 @@ public struct Utility { } config.labels = try Parser.labels(management.labels) + // A container runs in a pod. Naming none asks for one of its own, so + // one is named here rather than left for something later to notice was + // missing, which is how a sandbox is made for a container that came + // without one. + // https://github.com/kubernetes/cri-api/blob/master/pkg/apis/runtime/v1/api.proto + config.pod = management.pod ?? PodConfiguration.generateId() + + // Naming a pod joins one that is already there, so it has to be there, + // and joining it is joining its network: how that network is reached is + // the pod's to have been given, and asking for it here is asking the pod + // to be something it already is. Both are refused rather than accepted + // and ignored, which is how nerdctl answers a container joining + // another's network. The name a container answers to travels inside its + // attachment, so --hostname is covered by --network. + // https://github.com/containerd/nerdctl/blob/main/pkg/containerutil/container_network_manager.go + if let named = management.pod { + guard (try? await ClientPod.inspect(named)) != nil else { + throw ContainerizationError(.notFound, message: "pod \(named) does not exist") + } + var held: [String] = [] + if !management.publishPorts.isEmpty { held.append("-p/--publish") } + if !management.dns.nameservers.isEmpty || management.dns.domain != nil + || !management.dns.searchDomains.isEmpty || !management.dns.options.isEmpty + { + held.append("--dns") + } + if !management.networks.isEmpty { held.append("--network") } + guard held.isEmpty else { + throw ContainerizationError( + .invalidArgument, + message: + "these belong to the pod whose network the container joins, so they are not the container's to ask for: \(held.joined(separator: ", "))" + ) + } + } config.publishedPorts = try Parser.publishPorts(management.publishPorts) guard config.publishedPorts.count <= publishedPortCountLimit else { @@ -268,7 +321,9 @@ public struct Utility { return (config, kernel, management.initImage) } - static func getAttachmentConfigurations( + /// The networks a container or a pod attaches to, resolved from what the + /// caller named and the built-in network when it named none. + public static func getAttachmentConfigurations( containerId: String, builtinNetworkId: String?, networks: [Parser.ParsedNetwork], diff --git a/Sources/Services/ContainerAPIService/Client/XPC+.swift b/Sources/Services/ContainerAPIService/Client/XPC+.swift index a4d5aebd3..e564b2e15 100644 --- a/Sources/Services/ContainerAPIService/Client/XPC+.swift +++ b/Sources/Services/ContainerAPIService/Client/XPC+.swift @@ -117,6 +117,13 @@ public enum XPCKeys: String { /// Init image reference case initImage + /// Pod + case podId + case podConfig + case podSnapshot + case podSnapshots + case memoryInBytes + /// Volume case volume case volumes @@ -181,6 +188,17 @@ public enum XPCRoute: String { case volumeList case volumeInspect + // The sandbox verbs the runtime interface specifies, named for what they + // do to a pod rather than for the containers inside it. + // https://github.com/kubernetes/cri-api/blob/master/pkg/apis/runtime/v1/api.proto + case podCreate + case podStart + case podStop + case podDelete + case podInspect + case podList + case podUpdate + case volumeDiskUsage case systemDiskUsage diff --git a/Sources/Services/ContainerAPIService/Server/Containers/ContainersService.swift b/Sources/Services/ContainerAPIService/Server/Containers/ContainersService.swift index 81612495f..259d63645 100644 --- a/Sources/Services/ContainerAPIService/Server/Containers/ContainersService.swift +++ b/Sources/Services/ContainerAPIService/Server/Containers/ContainersService.swift @@ -64,6 +64,7 @@ public actor ContainersService { // FIXME: Find a better mechanism for services running on the APIServer to work with each other private weak var networksService: NetworksService? + private weak var podsService: PodsService? public init( appRoot: URL, @@ -85,6 +86,16 @@ public actor ContainersService { self.containers = try Self.loadAtBoot(root: containerRoot, loader: pluginLoader, log: log) } + /// Where a container keeps what it is made of, which the pod it belongs + /// to hands the runtime when placing it in the machine. + public func path(for id: String) -> URL { + self.containerRoot.appendingPathComponent(id) + } + + public func setPodsService(_ service: PodsService) async { + self.podsService = service + } + public func setNetworksService(_ service: NetworksService) async { self.networksService = service } @@ -345,6 +356,22 @@ public actor ContainersService { ) let initFilesystem = try await self.getInitBlock(for: systemPlatform.ociPlatform(), imageRef: initImage) + guard let podsService = await self.podsService else { + throw ContainerizationError(.internalError, message: "no pod service to make pod \(configuration.pod)") + } + do { + try await podsService.create( + configuration: PodConfiguration(sandboxFor: configuration), + kernel: kernel, + initialFilesystem: initFilesystem + ) + } catch let error as ContainerizationError { + guard error.code == .exists else { + throw error + } + // The pod is already there, and the container joins it. + } + do { self.log.debug( "create snapshot", @@ -382,6 +409,8 @@ public actor ContainersService { ) await self.setContainerState(configuration.id, ContainerState(snapshot: snapshot), context: context) } catch { + // The pod goes with the container it was made for. + await podsService.removeIfAnonymous(id: configuration.pod) throw error } } @@ -420,47 +449,29 @@ public actor ContainersService { let path = self.containerRoot.appendingPathComponent(id) let (config, _) = try Self.getContainerConfiguration(at: path) - var networkBootstrapInfos = [NetworkBootstrapInfo]() - for n in config.networks { - guard let plugin = try await self.networksService?.plugin(for: n.network) else { - throw ContainerizationError(.internalError, message: "failed to get plugin for network \(n.network)") - } - networkBootstrapInfos.append(NetworkBootstrapInfo(plugin: plugin)) + let pod = config.pod + guard let podsService = await self.podsService else { + throw ContainerizationError(.internalError, message: "no pod service to reach pod \(pod)") } - do { - try Self.registerService( - plugin: self.runtimePlugins.first { $0.name == config.runtimeHandler }!, - loader: self.pluginLoader, - configuration: config, - path: path, - debug: self.debugHelpers - ) - - let runtime = state.snapshot.configuration.runtimeHandler - let runtimeClient = try await RuntimeClient.create( - id: id, - runtime: runtime - ) - try await runtimeClient.bootstrap(stdio: stdio, networkBootstrapInfos: networkBootstrapInfos, dynamicEnv: dynamicEnv) - - try await self.exitMonitor.registerProcess( - id: id, - onExit: self.handleContainerExit - ) - - state.client = runtimeClient - await self.setContainerState(id, state, context: context) - } catch { - let label = Self.fullLaunchdServiceLabel( - runtimeName: config.runtimeHandler, - instanceId: id - ) - - await self.exitMonitor.stopTracking(id: id) - try? ServiceManager.deregister(fullServiceLabel: label) - throw error - } + // A container is in the pod's machine, so it has no machine of its + // own to register and reaches the one it shares through the pod. + // + // It asks for the pod to run with it in it, which is one call for + // the container that brings the machine up and the container that + // finds it up already. + try await podsService.start( + id: pod, + container: id, + startup: PodsService.ContainerStartup(stdio: stdio, dynamicEnv: dynamicEnv) + ) + let podClient = try await podsService.client(for: pod).addressing(id) + try await self.exitMonitor.registerProcess( + id: id, + onExit: self.handleContainerExit + ) + state.client = podClient + await self.setContainerState(id, state, context: context) } } @@ -623,9 +634,8 @@ public actor ContainersService { let state = try self._getContainerState(id: id) // Stop should be idempotent. - let client: RuntimeClient do { - client = try state.getClient() + _ = try state.getClient() } catch { return } @@ -636,7 +646,12 @@ public actor ContainersService { } do { - try await client.stop(options: resolvedOptions) + // Stopping a container stops that container. The machine it runs in + // is stopped by its own call, whether it holds one container or + // several, so nothing here decides the machine's fate on a + // container's behalf. + // https://github.com/kubernetes/cri-api/blob/master/pkg/apis/runtime/v1/api.proto + try await state.client?.stopContainer(options: resolvedOptions) } catch let err as ContainerizationError { if err.code != .interrupted { throw err @@ -742,17 +757,25 @@ public actor ContainersService { ) } - // Logs doesn't care if the container is running or not, just that - // the bundle is there, and that the files actually exist. We do - // first try and get the container state so we get a nicer error message + // Logs doesn't care if the container is running or not, just that the + // bundles are there and the files exist. What the container itself + // wrote is its own bundle's; the boot it came up on belongs to the + // machine the pod runs, so the pod is asked for that one. We do first + // try and get the container state so we get a nicer error message // (container foo not found) however. do { - _ = try _getContainerState(id: id) + let state = try _getContainerState(id: id) let path = self.containerRoot.appendingPathComponent(id) let bundle = ContainerResource.Bundle(path: path) + guard let podsService = self.podsService else { + throw ContainerizationError( + .internalError, + message: "no pod service to reach the machine running \(id)" + ) + } return [ try FileHandle(forReadingFrom: bundle.containerLog), - try FileHandle(forReadingFrom: bundle.bootlog), + try await podsService.bootLog(for: state.snapshot.configuration.pod), ] } catch { throw ContainerizationError( @@ -844,7 +867,11 @@ public actor ContainersService { signal: "SIGKILL" ) let client = try state.getClient() - try await client.stop(options: opts) + // Removing a container removes that container; the machine it + // shares is not this call's to stop, and goes down on its own + // once the last container in it has stopped. + // https://github.com/kubernetes/cri-api/blob/master/pkg/apis/runtime/v1/api.proto + try await client.stopContainer(options: opts) try await self.lock.withLock(logMetadata: ["acquirer": "\(#function)", "id": "\(id)"]) { context in self.log.info( "ContainersService: attempt cleanup", @@ -948,47 +975,10 @@ public actor ContainersService { await self.exitMonitor.stopTracking(id: id) - // Shutdown and deregister the runtime service - self.log.info("shutting down runtime service", metadata: ["id": "\(id)"]) - - let path = self.containerRoot.appendingPathComponent(id) - let bundle = ContainerResource.Bundle(path: path) - let config = try bundle.configuration - let label = Self.fullLaunchdServiceLabel( - runtimeName: config.runtimeHandler, - instanceId: id - ) - - // Try to shutdown the client gracefully, but if the runtime service - // is already dead (e.g., killed externally), we should still continue - // with state cleanup. - if let client = state.client { - do { - try await client.shutdown() - } catch { - self.log.error( - "failed to shutdown runtime service", - metadata: [ - "id": "\(id)", - "error": "\(error)", - ]) - } - } - - // Deregister the service, launchd will terminate the process. - // This may also fail if the service was already deregistered or - // the process was killed externally. - do { - try ServiceManager.deregister(fullServiceLabel: label) - self.log.info("deregistered runtime service", metadata: ["id": "\(id)"]) - } catch { - self.log.error( - "failed to deregister runtime service", - metadata: [ - "id": "\(id)", - "error": "\(error)", - ]) - } + // A container's exit is the container's alone. The machine it shared + // stops itself once the last container in it is gone, and its service + // is the pod's, deregistered when the pod is deleted; a machine other + // containers still run in is not touched at all. state.snapshot.status = .stopped state.snapshot.networks = [] @@ -1034,31 +1024,13 @@ public actor ContainersService { await self.exitMonitor.stopTracking(id: id) let path = self.containerRoot.appendingPathComponent(id) - // Try to get config for service deregistration - // Don't fail if bundle is incomplete - var config: ContainerConfiguration? let bundle = ContainerResource.Bundle(path: path) - do { - config = try bundle.configuration - } catch { - self.log.warning( - "failed to read bundle configuration during cleanup for container", - metadata: [ - "id": "\(id)", - "error": "\(error)", - ]) - } - - // Only try to deregister service if we have a valid config - // TODO: Change this so we don't have to reread the config - // possibly store the container ID to service label mapping - if let config = config { - let label = Self.fullLaunchdServiceLabel( - runtimeName: config.runtimeHandler, - instanceId: id - ) - try? ServiceManager.deregister(fullServiceLabel: label) - } + // Which machine this container runs in, read the way a container's + // configuration is read anywhere it may not have started yet: from its + // bundle, and from the runtime configuration when the bundle holds + // nothing, since a container that was made and never started has its + // configuration only there. + let pod = try? Self.getContainerConfiguration(at: path).0.pod // Always try to delete the bundle directory, even if it's incomplete do { @@ -1073,6 +1045,19 @@ public actor ContainersService { } self.containers.removeValue(forKey: id) + + // A container took its machine down as it was removed, when the machine + // was its own. A pod nobody named holds the machine in its place, so it + // goes here too; a pod someone named was not given to this container + // and stays. + // + // The container is out of the pod above before the pod is asked to go, + // because a pod removes the containers it still holds as it goes, and + // asked while this one was still in it the pod would ask for it to be + // removed, which asks for the lock this cleanup already holds. + if let pod { + await self.podsService?.removeIfAnonymous(id: pod) + } } private func cleanUp(id: String, context: AsyncLock.Context) async throws { diff --git a/Sources/Services/ContainerAPIService/Server/Pods/PodsHarness.swift b/Sources/Services/ContainerAPIService/Server/Pods/PodsHarness.swift new file mode 100644 index 000000000..db52c3897 --- /dev/null +++ b/Sources/Services/ContainerAPIService/Server/Pods/PodsHarness.swift @@ -0,0 +1,106 @@ +//===----------------------------------------------------------------------===// +// Copyright © 2026 Apple Inc. and the container project authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//===----------------------------------------------------------------------===// + +import ContainerAPIClient +import ContainerResource +import ContainerXPC +import Containerization +import ContainerizationError +import Foundation +import Logging + +public struct PodsHarness: Sendable { + let log: Logging.Logger + let service: PodsService + + public init(service: PodsService, log: Logging.Logger) { + self.log = log + self.service = service + } + + @Sendable + public func create(_ message: XPCMessage) async throws -> XPCMessage { + guard let data = message.dataNoCopy(key: .podConfig) else { + throw ContainerizationError(.invalidArgument, message: "a pod configuration is required") + } + let configuration = try JSONDecoder().decode(PodConfiguration.self, from: data) + + guard let kernelData = message.dataNoCopy(key: .kernel) else { + throw ContainerizationError(.invalidArgument, message: "a kernel is required") + } + let kernel = try JSONDecoder().decode(Kernel.self, from: kernelData) + + try await service.create( + configuration: configuration, + kernel: kernel, + initImage: message.string(key: .initImage) + ) + return message.reply() + } + + @Sendable + public func start(_ message: XPCMessage) async throws -> XPCMessage { + try await service.start(id: try message.podId()) + return message.reply() + } + + @Sendable + public func stop(_ message: XPCMessage) async throws -> XPCMessage { + try await service.stop(id: try message.podId()) + return message.reply() + } + + @Sendable + public func delete(_ message: XPCMessage) async throws -> XPCMessage { + try await service.delete(id: try message.podId(), force: message.bool(key: .forceDelete)) + return message.reply() + } + + @Sendable + public func inspect(_ message: XPCMessage) async throws -> XPCMessage { + let snapshot = try await service.inspect(id: try message.podId()) + let reply = message.reply() + reply.set(key: .podSnapshot, value: try JSONEncoder().encode(snapshot)) + return reply + } + + @Sendable + public func list(_ message: XPCMessage) async throws -> XPCMessage { + let snapshots = await service.list() + let reply = message.reply() + reply.set(key: .podSnapshots, value: try JSONEncoder().encode(snapshots)) + return reply + } + + @Sendable + public func update(_ message: XPCMessage) async throws -> XPCMessage { + let bytes = message.uint64(key: .memoryInBytes) + guard bytes > 0 else { + throw ContainerizationError(.invalidArgument, message: "a memory size is required") + } + try await service.update(id: try message.podId(), memoryInBytes: bytes) + return message.reply() + } +} + +extension XPCMessage { + fileprivate func podId() throws -> String { + guard let id = self.string(key: .podId), !id.isEmpty else { + throw ContainerizationError(.invalidArgument, message: "a pod name is required") + } + return id + } +} diff --git a/Sources/Services/ContainerAPIService/Server/Pods/PodsService.swift b/Sources/Services/ContainerAPIService/Server/Pods/PodsService.swift new file mode 100644 index 000000000..465976870 --- /dev/null +++ b/Sources/Services/ContainerAPIService/Server/Pods/PodsService.swift @@ -0,0 +1,569 @@ +//===----------------------------------------------------------------------===// +// Copyright © 2026 Apple Inc. and the container project authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//===----------------------------------------------------------------------===// + +import ContainerAPIClient +import ContainerPersistence +import ContainerPlugin +import ContainerResource +import ContainerRuntimeClient +import Containerization +import ContainerizationError +import ContainerizationExtras +import ContainerizationOCI +import Foundation +import Logging + +/// Manages pods: machines that several containers run inside and share. +/// +/// The verbs follow the runtime interface's sandbox lifecycle, which every +/// runtime serving Kubernetes implements, so what a pod does here is what a pod +/// does elsewhere. +/// https://github.com/kubernetes/cri-api/blob/master/pkg/apis/runtime/v1/api.proto +public actor PodsService { + struct PodState { + var configuration: PodConfiguration + var state: PodState.Lifecycle = .notReady + var client: RuntimeClient? = nil + var startedDate: Date? = nil + + enum Lifecycle { + case notReady + case ready + } + + func getClient() throws -> RuntimeClient { + guard let client else { + throw ContainerizationError( + .invalidState, + message: "pod \(configuration.id) is not running" + ) + } + return client + } + } + + /// The runtime plugin a pod's machine is driven by. + static let runtimeHandler = "container-runtime-linux" + + private static let machServicePrefix = "com.apple.container" + + /// The launchd domain, asked for once while the service is being built. + /// + /// Answering it runs `launchctl` and waits for it, which is a thread + /// blocked until that process is done. Held in a stored property it is a + /// wait the service makes as it is constructed; held in a static it would + /// be made the first time a label is wanted, and the first time is inside + /// a task, on a thread the concurrency pool needs back. That wait never + /// ends, and because a static is initialized once every later caller waits + /// on the same unfinished initialization, with nothing thrown and nothing + /// logged to say so. + private let launchdDomainString: String + + private static func fullLaunchdServiceLabel(domain: String, runtimeName: String, instanceId: String) -> String { + "\(domain)/\(Self.machServicePrefix).\(runtimeName).\(instanceId)" + } + + private let log: Logger + private let debugHelpers: Bool + private let podRoot: URL + private let pluginLoader: PluginLoader + private let runtimePlugins: [Plugin] + private let containerSystemConfig: ContainerSystemConfig + + private let lock: AsyncLock + private var pods: [String: PodState] + + // The containers a pod holds are the containers service's to know, so it + // is asked rather than tracked twice. + private weak var containersService: ContainersService? + private weak var networksService: NetworksService? + + public init( + appRoot: URL, + pluginLoader: PluginLoader, + containerSystemConfig: ContainerSystemConfig, + debugHelpers: Bool = false, + log: Logger + ) throws { + self.log = log + self.debugHelpers = debugHelpers + self.podRoot = appRoot.appendingPathComponent("pods") + self.pluginLoader = pluginLoader + self.runtimePlugins = pluginLoader.findPlugins().filter { $0.hasType(.runtime) } + self.containerSystemConfig = containerSystemConfig + self.lock = AsyncLock() + self.launchdDomainString = try ServiceManager.getDomainString() + self.pods = try Self.loadAtBoot(root: self.podRoot, log: log) + } + + public func setContainersService(_ service: ContainersService) async { + self.containersService = service + } + + public func setNetworksService(_ service: NetworksService) async { + self.networksService = service + } + + /// The pods on disk, which outlive the process that made them. + /// + /// A pod's bundle is materialized by its machine's first boot, so a pod + /// created and not yet booted has only the runtime configuration its + /// create wrote; the pod's own configuration is read from whichever of + /// the two holds it. + static func loadAtBoot(root: URL, log: Logger) throws -> [String: PodState] { + try FileManager.default.createDirectory(at: root, withIntermediateDirectories: true) + var pods: [String: PodState] = [:] + let entries = try FileManager.default.contentsOfDirectory(at: root, includingPropertiesForKeys: nil) + for entry in entries { + do { + let bundle = ContainerResource.Bundle(path: entry) + let configuration: PodConfiguration + if bundle.isPod { + configuration = try bundle.podConfiguration + } else if let embedded = try RuntimeConfiguration.readRuntimeConfiguration(from: entry).podConfiguration { + configuration = embedded + } else { + log.warning("skipping a bundle that is not a pod's", metadata: ["path": "\(entry.path)"]) + continue + } + pods[configuration.id] = PodState(configuration: configuration) + } catch { + log.warning("skipping unreadable pod", metadata: ["path": "\(entry.path)", "error": "\(error)"]) + } + } + return pods + } + + /// Where a pod keeps what it is made of. + public func path(for id: String) -> URL { + self.podRoot.appendingPathComponent(id) + } + + /// The boot log of the machine a pod runs. + /// + /// The machine is the pod's rather than any one container's: every + /// container the pod holds boots on it, and one log records that boot, so + /// the pod is what answers for it. + public func bootLog(for id: String) throws -> FileHandle { + let bundle = ContainerResource.Bundle(path: self.path(for: id)) + return try FileHandle(forReadingFrom: bundle.bootlog) + } + + /// The client for the machine a pod runs, which its containers are + /// reached through. + public func client(for id: String) throws -> RuntimeClient { + try self._getPodState(id: id).getClient() + } + + /// The initial filesystem a pod's machine boots, which holds the agent. + private func getInitBlock(for platform: Platform, imageRef: String? = nil) async throws -> Filesystem { + let ref = imageRef ?? containerSystemConfig.vminit.image + let initImage = try await ClientImage.fetch(reference: ref, platform: platform, containerSystemConfig: containerSystemConfig) + var fs = try await initImage.getCreateSnapshot(platform: platform) + fs.options = ["ro"] + return fs + } + + /// Write down a pod that boots the init image named, or the default one. + public func create(configuration: PodConfiguration, kernel: Kernel, initImage: String? = nil) async throws { + try await self.create( + configuration: configuration, + kernel: kernel, + initialFilesystem: try await self.getInitBlock(for: kernel.platform.ociPlatform(), imageRef: initImage) + ) + } + + /// Write down a pod, so containers can be placed in it before it boots. + /// + /// A caller holding the init filesystem already, such as one making the pod + /// for a container whose own init image was resolved when it was created, + /// passes it here rather than naming an image to resolve again. + public func create(configuration: PodConfiguration, kernel: Kernel, initialFilesystem: Filesystem) async throws { + log.debug( + "PodsService: enter", + metadata: [ + "func": "\(#function)", + "id": "\(configuration.id)", + ] + ) + defer { + log.debug( + "PodsService: exit", + metadata: [ + "func": "\(#function)", + "id": "\(configuration.id)", + ] + ) + } + + try await self.lock.withLock(logMetadata: ["acquirer": "\(#function)", "id": "\(configuration.id)"]) { context in + guard await self.pods[configuration.id] == nil else { + throw ContainerizationError(.exists, message: "pod already exists: \(configuration.id)") + } + + // A name on a network answers for one thing on it. The names live + // on the attachments a pod holds, since the pod holds the network + // its containers share, so it is here that two of them are kept + // from answering to the same one, which is what a container with a + // machine of its own was held to when it held its own attachments. + var taken = Set() + for pod in await self.pods.values { + for attachment in pod.configuration.networks { + taken.insert(attachment.options.hostname) + } + } + let clashing = configuration.networks.map { $0.options.hostname }.filter { taken.contains($0) } + guard clashing.isEmpty else { + throw ContainerizationError(.exists, message: "hostname(s) already exist: \(clashing)") + } + + guard self.runtimePlugins.first(where: { $0.name == configuration.runtimeHandler }) != nil else { + throw ContainerizationError( + .notFound, + message: "unable to locate runtime plugin \(configuration.runtimeHandler)" + ) + } + + // The floor a machine needs to boot at all, the same one a + // container with a machine of its own is held to. + let minimumMemory: UInt64 = 200.mib() + guard configuration.resources.memoryInBytes >= minimumMemory else { + throw ContainerizationError( + .invalidArgument, + message: "minimum memory amount allowed is 200 MiB (got \(configuration.resources.memoryInBytes) bytes)" + ) + } + + let path = await self.path(for: configuration.id) + let runtimeConfig = RuntimeConfiguration( + path: path, + initialFilesystem: initialFilesystem, + kernel: kernel, + podConfiguration: configuration + ) + try runtimeConfig.writeRuntimeConfiguration() + + await self.setPodState(configuration.id, PodState(configuration: configuration), context: context) + } + } + + /// What a caller holds on behalf of a container it is starting a pod for. + /// + /// A container placed in a pod is given these as it goes in, the way a + /// container with a machine of its own is given them as the machine is + /// bootstrapped. They belong to the container rather than to the pod, so + /// they travel under its id. + public struct ContainerStartup: Sendable { + /// The streams the caller opened for the container. + public var stdio: [FileHandle?] + /// The environment the caller was asked to add to the container's. + public var dynamicEnv: [String: String] + + public init(stdio: [FileHandle?], dynamicEnv: [String: String] = [:]) { + self.stdio = stdio + self.dynamicEnv = dynamicEnv + } + } + + /// Boot a pod's machine and place the containers that belong to it inside. + /// + /// The containers go in before the machine starts, which is what a machine + /// with no way to attach storage while running requires. + /// + /// A container the caller is starting the pod for brings what the caller + /// holds for it. The machine itself is nobody's to read and has no + /// environment of its own, so it is bootstrapped with neither. + /// The bundles of the containers a pod holds, which its machine runs. + private func bundlePaths(of id: String) async throws -> [String] { + var paths = [String]() + for member in await self.containers(of: id) { + guard let path = await self.containersService?.path(for: member.id) else { + throw ContainerizationError(.internalError, message: "no container service to place \(member.id)") + } + paths.append(path.path) + } + return paths + } + + public func start(id: String, container: String? = nil, startup: ContainerStartup? = nil) async throws { + log.debug("PodsService: enter", metadata: ["func": "\(#function)", "id": "\(id)"]) + defer { log.debug("PodsService: exit", metadata: ["func": "\(#function)", "id": "\(id)"]) } + + try await self.lock.withLock(logMetadata: ["acquirer": "\(#function)", "id": "\(id)"]) { context in + var state = try await self._getPodState(id: id) + var running = state.client + + // A machine can be gone while the service that ran it lingers: + // stopped out of band, crashed, or torn down without the pod + // hearing of it. A pod that offered that machine would be + // offering one that is not there, so the client is believed only + // while its machine answers running; otherwise the service is + // taken down and the pod boots a fresh machine the way it booted + // the first. + if let held = running, (try? await held.state())?.status != .running { + await self.deregister(id: id) + state.client = nil + state.state = .notReady + state.startedDate = nil + await self.setPodState(id, state, context: context) + running = nil + } + + do { + let client: RuntimeClient + var networkBootstrapInfos = [NetworkBootstrapInfo]() + if let running { + client = running + } else { + let path = await self.path(for: id) + let runtime = state.configuration.runtimeHandler + guard let plugin = self.runtimePlugins.first(where: { $0.name == runtime }) else { + throw ContainerizationError(.notFound, message: "unable to locate runtime plugin \(runtime)") + } + try Self.registerService( + plugin: plugin, + loader: self.pluginLoader, + id: id, + path: path, + debug: self.debugHelpers, + domain: self.launchdDomainString + ) + + // The pod claims its addresses, which every container placed + // in it shares, having no network namespace of its own. + for n in state.configuration.networks { + guard let plugin = try await self.networksService?.plugin(for: n.network) else { + throw ContainerizationError(.internalError, message: "failed to get plugin for network \(n.network)") + } + networkBootstrapInfos.append(NetworkBootstrapInfo(plugin: plugin)) + } + client = try await RuntimeClient.create(id: id, runtime: runtime) + } + + // The pod is asked to run holding its containers, which is one + // request whether it is coming up around them or already up and + // taking in the one that is new to it. + try await client.bootstrap( + bundlePaths: try await self.bundlePaths(of: id), + stdioFor: container, + stdio: startup?.stdio ?? [nil, nil, nil], + networkBootstrapInfos: networkBootstrapInfos, + dynamicEnv: startup?.dynamicEnv ?? [:] + ) + + if running == nil { + state.client = client + state.state = .ready + state.startedDate = Date() + await self.setPodState(id, state, context: context) + } + } catch { + if running == nil { + await self.deregister(id: id) + } + throw error + } + } + } + + /// Stop a pod's machine, and with it every container inside. + public func stop(id: String, options: ContainerStopOptions = ContainerStopOptions(timeoutInSeconds: 5, signal: "SIGTERM")) async throws { + log.debug("PodsService: enter", metadata: ["func": "\(#function)", "id": "\(id)"]) + defer { log.debug("PodsService: exit", metadata: ["func": "\(#function)", "id": "\(id)"]) } + + try await self.lock.withLock(logMetadata: ["acquirer": "\(#function)", "id": "\(id)"]) { context in + var state = try await self._getPodState(id: id) + guard let client = state.client else { + return + } + + try? await client.stop(options: options) + try? await client.shutdown() + await self.deregister(id: id) + + state.client = nil + state.state = .notReady + state.startedDate = nil + await self.setPodState(id, state, context: context) + } + } + + /// Take a pod away. A pod still holding containers is kept unless the + /// caller insists, in which case the containers go with it. + public func delete(id: String, force: Bool) async throws { + log.debug("PodsService: enter", metadata: ["func": "\(#function)", "id": "\(id)"]) + defer { log.debug("PodsService: exit", metadata: ["func": "\(#function)", "id": "\(id)"]) } + + let members = await self.containers(of: id) + if !members.isEmpty { + guard force else { + throw ContainerizationError( + .invalidState, + message: "pod \(id) holds \(members.count) container(s); stop and remove them, or force" + ) + } + for member in members { + try await self.containersService?.delete(id: member.id, force: true) + } + } + + // Taking the last container out of a pod that was made for one is what + // takes that pod away, so the members above may have carried this pod + // off as they went. A pod already gone is what the caller asked for. + guard (try? self._getPodState(id: id)) != nil else { + return + } + + try await self.stop(id: id) + await self.lock.withLock(logMetadata: ["acquirer": "\(#function)", "id": "\(id)"]) { context in + try? FileManager.default.removeItem(at: await self.path(for: id)) + await self.removePodState(id, context: context) + } + } + + /// Everything known about a pod, including the containers in it. + public func inspect(id: String) async throws -> PodSnapshot { + let state = try self._getPodState(id: id) + return await self.snapshot(state) + } + + /// Every pod, in a settled order. + public func list() async -> [PodSnapshot] { + var snapshots: [PodSnapshot] = [] + for state in self.pods.values.sorted(by: { $0.configuration.id < $1.configuration.id }) { + snapshots.append(await self.snapshot(state)) + } + return snapshots + } + + /// Hold a running pod to a memory size, which its containers share. + /// + /// This is the runtime interface's `UpdatePodSandboxResources` for the one + /// resource a running machine can be held to after it has booted. + public func update(id: String, memoryInBytes: UInt64) async throws { + let state = try self._getPodState(id: id) + let client = try state.getClient() + try await client.setTargetMemorySize(memoryInBytes) + } + + /// Take away a pod a container was given for itself. + /// + /// A container took its machine down as it was removed, by deregistering the + /// service that ran it, because the machine was the container's and nothing + /// else could be in it. A pod nobody named holds the machine in its place + /// and exists because the container needed one, so it goes the same way. + /// + /// A pod someone named was not given to this container. It was there before + /// the container and is there after, so a container leaving says nothing + /// about it. Neither does a container leaving one that others are still in: + /// a container may be placed in a pod it was given the name of, anonymous + /// or not, and holds it up the way any other in it would. + /// + /// What a pod holds is what the containers say they are in, so this is + /// asked with the container lock held by a caller that has already taken + /// its own container out of it. Held that way the answer cannot change + /// between the asking and the removal, since a container joins a pod only + /// by being created. + public func removeIfAnonymous(id: String) async { + guard let state = try? self._getPodState(id: id), state.configuration.isAnonymous else { + return + } + guard await self.containers(of: id).isEmpty else { + return + } + do { + try await self.delete(id: id, force: true) + } catch { + self.log.error( + "failed to remove the machine a container had to itself", + metadata: ["pod": "\(id)", "error": "\(error)"]) + } + } + + private func snapshot(_ state: PodState) async -> PodSnapshot { + let members = await self.containers(of: state.configuration.id) + var networks: [Attachment] = [] + // A pod is ready when its machine is running, which is what the machine + // says rather than what it was last told to do: the machine goes down on + // its own once the last container in it has stopped, and a pod that + // answered ready after that would be offering a machine that is not + // there. + var running = false + if let client = state.client, let sandbox = try? await client.state() { + networks = sandbox.networks + running = sandbox.status == .running + } + return PodSnapshot( + configuration: state.configuration, + state: state.state == .ready && running ? .ready : .notReady, + networks: networks, + containers: members.map { $0.id }.sorted(), + startedDate: state.startedDate + ) + } + + private func containers(of id: String) async -> [ContainerSnapshot] { + guard let containersService = self.containersService else { + return [] + } + let all = (try? await containersService.list()) ?? [] + return all.filter { $0.configuration.pod == id } + } + + private static func registerService( + plugin: Plugin, + loader: PluginLoader, + id: String, + path: URL, + debug: Bool, + domain: String + ) throws { + let args = [ + "start", + "--root", path.path, + "--uuid", id, + debug ? "--debug" : nil, + ].compactMap { $0 } + try loader.registerWithLaunchd( + plugin: plugin, + pluginStateRoot: path, + args: args, + instanceId: id + ) + } + + private func deregister(id: String) async { + let runtime = (try? self._getPodState(id: id).configuration.runtimeHandler) ?? Self.runtimeHandler + let label = Self.fullLaunchdServiceLabel( + domain: self.launchdDomainString, runtimeName: runtime, instanceId: id) + try? ServiceManager.deregister(fullServiceLabel: label) + } + + private func setPodState(_ id: String, _ state: PodState, context: AsyncLock.Context) async { + self.pods[id] = state + } + + private func removePodState(_ id: String, context: AsyncLock.Context) async { + self.pods.removeValue(forKey: id) + } + + private func _getPodState(id: String) throws -> PodState { + guard let state = self.pods[id] else { + throw ContainerizationError(.notFound, message: "pod with ID \(id) not found") + } + return state + } +} diff --git a/Sources/Services/Runtime/RuntimeClient/RuntimeClient.swift b/Sources/Services/Runtime/RuntimeClient/RuntimeClient.swift index 32a4db062..abffb83af 100644 --- a/Sources/Services/Runtime/RuntimeClient/RuntimeClient.swift +++ b/Sources/Services/Runtime/RuntimeClient/RuntimeClient.swift @@ -35,16 +35,41 @@ public struct RuntimeClient: Sendable { Self.machServiceLabel(runtime: runtime, id: id) } + /// The sandbox this client talks to, which is the container itself when + /// the container has a machine of its own, and the pod when it shares one. let id: String + /// The container in that sandbox the client addresses. + /// The container a request is addressed to, when it is addressed to one. + /// + /// A sandbox's own calls name no container. A container's calls name it, + /// however many containers the sandbox holds, so that no call has to be + /// read as meaning the only one there. + let containerId: String? let runtime: String let client: XPCClient - init(id: String, runtime: String, client: XPCClient) { + init(id: String, containerId: String? = nil, runtime: String, client: XPCClient) { self.id = id + self.containerId = containerId self.runtime = runtime self.client = client } + /// The same client, addressing a different container in the same sandbox. + public func addressing(_ containerId: String) -> RuntimeClient { + RuntimeClient(id: self.id, containerId: containerId, runtime: self.runtime, client: self.client) + } + + /// A request to the sandbox, naming the container it is addressed to. A + /// sandbox holding one container still hears which container is meant. + func request(_ route: String) -> XPCMessage { + let message = XPCMessage(route: route) + if let containerId = self.containerId { + message.set(key: RuntimeKeys.containerId.rawValue, value: containerId) + } + return message + } + /// Create a RuntimeClient by ID and runtime string. The returned client is ready to be used /// without additional steps. public static func create(id: String, runtime: String, timeout: Duration = XPCClient.xpcRegistrationTimeout) async throws -> RuntimeClient { @@ -77,33 +102,32 @@ public struct RuntimeClient: Sendable { // Runtime Methods extension RuntimeClient { + /// Run the sandbox with the containers it holds in it. + /// + /// The sandbox is brought up with every container named here in it, and + /// asking again for one already up puts in whichever of them it does not + /// hold yet. The standard streams belong to the container named by + /// `stdioFor`, the one whose start this is; the rest are placed with none. public func bootstrap( + bundlePaths: [String], + stdioFor: String? = nil, stdio: [FileHandle?], networkBootstrapInfos: [NetworkBootstrapInfo], dynamicEnv: [String: String] = [:] ) async throws { - let request = XPCMessage(route: RuntimeRoutes.bootstrap.rawValue) - - for (i, h) in stdio.enumerated() { - let key: RuntimeKeys = try { - switch i { - case 0: .stdin - case 1: .stdout - case 2: .stderr - default: - throw ContainerizationError(.invalidArgument, message: "invalid fd \(i)") - } - }() - - if let h { - request.set(key: key.rawValue, value: h) - } - } + let request = self.request(RuntimeRoutes.bootstrap.rawValue) + try request.setStdio(stdio) do { let dynamicEnv = try JSONEncoder().encode(dynamicEnv) request.set(key: RuntimeKeys.dynamicEnv.rawValue, value: dynamicEnv) + let pathsData = try JSONEncoder().encode(bundlePaths) + request.set(key: RuntimeKeys.bundlePaths.rawValue, value: pathsData) + if let stdioFor { + request.set(key: RuntimeKeys.containerId.rawValue, value: stdioFor) + } + let infosData = try JSONEncoder().encode(networkBootstrapInfos) request.set(key: RuntimeKeys.networkBootstrapInfos.rawValue, value: infosData) try await self.client.send(request) @@ -116,8 +140,23 @@ extension RuntimeClient { } } + /// Hold the running sandbox to a memory size, which its containers share. + public func setTargetMemorySize(_ bytes: UInt64) async throws { + let request = self.request(RuntimeRoutes.updateResources.rawValue) + request.set(key: RuntimeKeys.memoryInBytes.rawValue, value: bytes) + do { + try await self.client.send(request) + } catch { + throw ContainerizationError( + .internalError, + message: "failed to set the memory size of sandbox \(self.id)", + cause: error + ) + } + } + public func state() async throws -> SandboxSnapshot { - let request = XPCMessage(route: RuntimeRoutes.state.rawValue) + let request = self.request(RuntimeRoutes.state.rawValue) let response: XPCMessage do { response = try await self.client.send(request) @@ -132,7 +171,7 @@ extension RuntimeClient { } public func createProcess(_ id: String, config: ProcessConfiguration, stdio: [FileHandle?]) async throws { - let request = XPCMessage(route: RuntimeRoutes.createProcess.rawValue) + let request = self.request(RuntimeRoutes.createProcess.rawValue) request.set(key: RuntimeKeys.id.rawValue, value: id) let data = try JSONEncoder().encode(config) request.set(key: RuntimeKeys.processConfig.rawValue, value: data) @@ -165,7 +204,7 @@ extension RuntimeClient { } public func startProcess(_ id: String) async throws { - let request = XPCMessage(route: RuntimeRoutes.start.rawValue) + let request = self.request(RuntimeRoutes.start.rawValue) request.set(key: RuntimeKeys.id.rawValue, value: id) do { try await self.client.send(request) @@ -178,8 +217,27 @@ extension RuntimeClient { } } + /// Stop the container this client addresses, leaving the machine it shares + /// and the containers beside it running. + public func stopContainer(options: ContainerStopOptions) async throws { + let request = self.request(RuntimeRoutes.stopContainer.rawValue) + + let data = try JSONEncoder().encode(options) + request.set(key: RuntimeKeys.stopOptions.rawValue, value: data) + + do { + try await self.client.send(request) + } catch { + throw ContainerizationError( + .internalError, + message: "failed to stop container \(self.id)", + cause: error + ) + } + } + public func stop(options: ContainerStopOptions) async throws { - let request = XPCMessage(route: RuntimeRoutes.stop.rawValue) + let request = self.request(RuntimeRoutes.stop.rawValue) let data = try JSONEncoder().encode(options) request.set(key: RuntimeKeys.stopOptions.rawValue, value: data) @@ -196,7 +254,7 @@ extension RuntimeClient { } public func kill(_ id: String, signal: String) async throws { - let request = XPCMessage(route: RuntimeRoutes.kill.rawValue) + let request = self.request(RuntimeRoutes.kill.rawValue) request.set(key: RuntimeKeys.id.rawValue, value: id) request.set(key: RuntimeKeys.signal.rawValue, value: signal) @@ -212,7 +270,7 @@ extension RuntimeClient { } public func resize(_ id: String, size: Terminal.Size) async throws { - let request = XPCMessage(route: RuntimeRoutes.resize.rawValue) + let request = self.request(RuntimeRoutes.resize.rawValue) request.set(key: RuntimeKeys.id.rawValue, value: id) request.set(key: RuntimeKeys.width.rawValue, value: UInt64(size.width)) request.set(key: RuntimeKeys.height.rawValue, value: UInt64(size.height)) @@ -229,7 +287,7 @@ extension RuntimeClient { } public func wait(_ id: String) async throws -> ExitStatus { - let request = XPCMessage(route: RuntimeRoutes.wait.rawValue) + let request = self.request(RuntimeRoutes.wait.rawValue) request.set(key: RuntimeKeys.id.rawValue, value: id) let response: XPCMessage @@ -248,7 +306,7 @@ extension RuntimeClient { } public func dial(_ port: UInt32) async throws -> FileHandle { - let request = XPCMessage(route: RuntimeRoutes.dial.rawValue) + let request = self.request(RuntimeRoutes.dial.rawValue) request.set(key: RuntimeKeys.port.rawValue, value: UInt64(port)) let response: XPCMessage @@ -271,7 +329,7 @@ extension RuntimeClient { } public func shutdown() async throws { - let request = XPCMessage(route: RuntimeRoutes.shutdown.rawValue) + let request = self.request(RuntimeRoutes.shutdown.rawValue) do { _ = try await self.client.send(request) @@ -285,7 +343,7 @@ extension RuntimeClient { } public func copyIn(source: String, destination: String, mode: UInt32, createParents: Bool = true) async throws { - let request = XPCMessage(route: RuntimeRoutes.copyIn.rawValue) + let request = self.request(RuntimeRoutes.copyIn.rawValue) request.set(key: RuntimeKeys.sourcePath.rawValue, value: source) request.set(key: RuntimeKeys.destinationPath.rawValue, value: destination) request.set(key: RuntimeKeys.fileMode.rawValue, value: UInt64(mode)) @@ -303,7 +361,7 @@ extension RuntimeClient { } public func copyOut(source: String, destination: String, createParents: Bool = true) async throws { - let request = XPCMessage(route: RuntimeRoutes.copyOut.rawValue) + let request = self.request(RuntimeRoutes.copyOut.rawValue) request.set(key: RuntimeKeys.sourcePath.rawValue, value: source) request.set(key: RuntimeKeys.destinationPath.rawValue, value: destination) request.set(key: RuntimeKeys.createParents.rawValue, value: createParents) @@ -320,7 +378,7 @@ extension RuntimeClient { } public func snapshotDisk(imagePath: String, destinationPath: String) async throws { - let request = XPCMessage(route: RuntimeRoutes.snapshotDisk.rawValue) + let request = self.request(RuntimeRoutes.snapshotDisk.rawValue) request.set(key: RuntimeKeys.imagePath.rawValue, value: imagePath) request.set(key: RuntimeKeys.destinationPath.rawValue, value: destinationPath) @@ -336,7 +394,7 @@ extension RuntimeClient { } public func statistics() async throws -> ContainerStats { - let request = XPCMessage(route: RuntimeRoutes.statistics.rawValue) + let request = self.request(RuntimeRoutes.statistics.rawValue) let response: XPCMessage do { @@ -389,4 +447,24 @@ extension XPCMessage { } return try JSONDecoder().decode([NetworkBootstrapInfo].self, from: data) } + + /// Carry the standard streams of a container, in the order the guest + /// numbers them. + func setStdio(_ stdio: [FileHandle?]) throws { + for (i, h) in stdio.enumerated() { + let key: RuntimeKeys = try { + switch i { + case 0: .stdin + case 1: .stdout + case 2: .stderr + default: + throw ContainerizationError(.invalidArgument, message: "invalid fd \(i)") + } + }() + + if let h { + self.set(key: key.rawValue, value: h) + } + } + } } diff --git a/Sources/Services/Runtime/RuntimeClient/RuntimeKeys.swift b/Sources/Services/Runtime/RuntimeClient/RuntimeKeys.swift index 1d3548cfe..6f2da4342 100644 --- a/Sources/Services/Runtime/RuntimeClient/RuntimeKeys.swift +++ b/Sources/Services/Runtime/RuntimeClient/RuntimeKeys.swift @@ -17,6 +17,18 @@ public enum RuntimeKeys: String { /// ID key. case id + /// The container in the sandbox a request is addressed to. A sandbox + /// holding one container still names it, so every request says which + /// container it means. + case containerId + /// The configuration of a container being placed in a sandbox. + case containerConfig + /// The path to the bundle of a container being placed in a sandbox. + case bundlePath + /// The paths to the bundles of every container the sandbox holds. + case bundlePaths + /// A memory size in bytes the sandbox is to be held to. + case memoryInBytes /// Vsock port number key. case port /// Exit code for a process diff --git a/Sources/Services/Runtime/RuntimeClient/RuntimeRoutes.swift b/Sources/Services/Runtime/RuntimeClient/RuntimeRoutes.swift index bbe1485f4..98516f918 100644 --- a/Sources/Services/Runtime/RuntimeClient/RuntimeRoutes.swift +++ b/Sources/Services/Runtime/RuntimeClient/RuntimeRoutes.swift @@ -25,10 +25,18 @@ public enum RuntimeRoutes: String { // MARK: - Sandbox lifecycle - /// Bootstrap the sandbox: create the VM, configure networks, and boot the guest. + /// Bootstrap the sandbox: create the VM, configure networks, put the + /// containers it holds in it, and boot the guest. case bootstrap = "com.apple.container.runtime/bootstrap" + /// Hold the running sandbox to a memory size, which its containers share. + case updateResources = "com.apple.container.runtime/updateResources" /// Stop the sandbox and all processes running inside it. case stop = "com.apple.container.runtime/stop" + /// Stop one container in the sandbox, leaving the sandbox and the other + /// containers in it running. The runtime interface stops a container and + /// stops a sandbox with separate calls for this reason. + /// https://github.com/kubernetes/cri-api/blob/master/pkg/apis/runtime/v1/api.proto + case stopContainer = "com.apple.container.runtime/stopContainer" /// Return the current state of the sandbox. case state = "com.apple.container.runtime/state" /// Get resource usage statistics for the sandbox. diff --git a/Sources/Services/RuntimeLinux/Server/RuntimeService.swift b/Sources/Services/RuntimeLinux/Server/RuntimeService.swift index 57f756b89..e059e07ac 100644 --- a/Sources/Services/RuntimeLinux/Server/RuntimeService.swift +++ b/Sources/Services/RuntimeLinux/Server/RuntimeService.swift @@ -36,12 +36,22 @@ import SystemPackage import struct ContainerizationOCI.Mount import struct ContainerizationOCI.Process -/// An XPC service that manages the lifecycle of a single VM-backed container. +/// An XPC service that manages the lifecycle of a VM-backed sandbox and the +/// containers running in it. public actor RuntimeService { private let connection: xpc_connection_t private let root: URL private let interfaceStrategies: [NetworkInterfaceKey: InterfaceStrategy] - private var container: ContainerInfo? + /// The machine this service drives, once it has been bootstrapped. + private var sandbox: (any Sandbox)? + /// The containers in that machine, by identifier. A pod holds several and + /// a standalone container holds one. + private var containers: [String: ContainerInfo] = [:] + /// The addresses a pod claimed, which every container placed in it shares. + private var podAttachments: [Attachment] = [] + /// The ports published on those addresses, which are the pod's because the + /// addresses are, and are forwarded once for all of its containers. + private var podPublishedPorts: [PublishPort] = [] private let monitor: ExitMonitor private let eventLoopGroup: any EventLoopGroup private var waiters: [String: ExitWaiter] = [:] @@ -145,179 +155,27 @@ public actor RuntimeService { } return try await self.lock.withLock { _ in + // A machine that is not waiting to be brought up is running, and a + // request for one that is running is a container joining it: what + // the machine does not hold yet goes in, and the machine stays as + // it is. guard await self.state == .created else { - throw ContainerizationError( - .invalidState, - message: "container expected to be in created state, got: \(await self.state)" - ) + try await self.placeContainers(message) + return message.reply() } - let dynamicEnv = try message.dynamicEnv() - let bundle = ContainerResource.Bundle(path: self.root) try bundle.createLogFile() - var config = try bundle.configuration - - var kernel = try bundle.kernel - // Built-in defaults keyed by arg name. Each is applied only if the user did not already - // supply the same key via --kernel-arg, letting custom kernels override them (e.g. lsm=...,bpf). - let defaultKernelArgs: KeyValuePairs = [ - "oops": "panic", - "lsm": "lockdown,capability,landlock,yama,apparmor", - ] - for (key, value) in defaultKernelArgs { - guard !kernel.commandLine.kernelArgs.contains(where: { $0.hasPrefix("\(key)=") }) else { - continue - } - kernel.commandLine.kernelArgs.append("\(key)=\(value)") - } - let vmm = VZVirtualMachineManager( - kernel: kernel, - initialFilesystem: bundle.initialFilesystem.asMount, - rosetta: config.rosetta, - logger: self.log - ) - - let networkBootstrapInfos = try message.networkBootstrapInfos() - - var sessions: [XPCClientSession] = [] - var attachments: [Attachment] = [] - var interfaces: [Interface] = [] - do { - for (index, info) in networkBootstrapInfos.enumerated() { - let attachmentConfig = config.networks[index] - let client = ContainerNetworkClient.NetworkClient(id: attachmentConfig.network, plugin: info.plugin) - let session = client.connect() - sessions.append(session) - var (attachment, additionalData) = try await client.allocate( - hostname: attachmentConfig.options.hostname, - macAddress: attachmentConfig.options.macAddress, - on: session - ) - if let mtu = attachmentConfig.options.mtu { - attachment = Attachment( - network: attachment.network, - hostname: attachment.hostname, - ipv4Address: attachment.ipv4Address, - ipv4Gateway: attachment.ipv4Gateway, - ipv6Address: attachment.ipv6Address, - macAddress: attachment.macAddress, - mtu: mtu, - variant: attachment.variant - ) - } - guard let iStrategy = self.interfaceStrategies[NetworkInterfaceKey(plugin: info.plugin, variant: attachment.variant)] else { - throw ContainerizationError( - .internalError, - message: "no available interface strategy for network \(attachment.network), plugin=\(info.plugin) variant=\(attachment.variant ?? "nil")") - } - let interface = try iStrategy.toInterface( - attachment: attachment, - interfaceIndex: index, - additionalData: additionalData - ) - attachments.append(attachment) - interfaces.append(interface) - } - } catch { - for session in sessions { session.close() } - throw error - } - - // Dynamically configure the DNS nameserver from a network if no explicit configuration - if let dns = config.dns, dns.nameservers.isEmpty { - let defaultNameservers = self.getDefaultNameservers(from: attachments) - if !defaultNameservers.isEmpty { - config.dns = ContainerConfiguration.DNSConfiguration( - nameservers: defaultNameservers, - domain: dns.domain, - searchDomains: dns.searchDomains, - options: dns.options - ) - } - } - - let stdio = message.stdio() - let containerLog = try FileHandle(forWritingTo: bundle.containerLog) - let stdout = { - if let h = stdio[1] { - return MultiWriter(handles: [h, containerLog]) - } - return MultiWriter(handles: [containerLog]) - }() - - let stderr: MultiWriter? = { - if !config.initProcess.terminal { - if let h = stdio[2] { - return MultiWriter(handles: [h, containerLog]) - } - return MultiWriter(handles: [containerLog]) - } - return nil - }() - - let stdin = { - stdio[0] ?? nil - }() - - let id = config.id - let rootfs = try bundle.containerRootfs.asMount - // A container is only given a swap area when it asked for one. - let swapLayer = try config.resources.swapInBytes.map { - try bundle.createSwapDevice(size: $0).asMount - } - let container = try LinuxContainer(id, rootfs: rootfs, vmm: vmm, logger: self.log) { czConfig in - try Self.configureContainer(czConfig: &czConfig, config: config, dynamicEnv: dynamicEnv, log: self.log) - czConfig.swapLayer = swapLayer - czConfig.interfaces = interfaces - czConfig.process.stdout = stdout - czConfig.process.stderr = stderr - czConfig.process.stdin = stdin - // NOTE: We can support a user providing new entries eventually, but for now craft - // a default /etc/hosts. - var hostsEntries = [Hosts.Entry.localHostIPV4()] - if !interfaces.isEmpty { - let primaryIfaceAddr = interfaces[0].ipv4Address - hostsEntries.append( - Hosts.Entry( - ipAddress: primaryIfaceAddr.address.description, - hostnames: [czConfig.hostname ?? id], - )) - } - czConfig.hosts = Hosts(entries: hostsEntries) - czConfig.bootLog = BootLog.file(path: bundle.bootlog, append: true) - } - - let ctrInfo = ContainerInfo( - container: container, - config: config, - attachments: attachments, - bundle: bundle, - io: (in: stdin, out: stdout, err: stderr) - ) - await self.setContainer(ctrInfo) - await self.setNetworkSessions(sessions) - - do { - try await container.create() - - try await self.initializeWaiters(for: id) - try await self.monitor.registerProcess(id: config.id, onExit: self.onContainerExit) - if !container.interfaces.isEmpty { - try await self.startSocketForwarders(attachment: attachments[0], publishedPorts: config.publishedPorts) - } - await self.setState(.booted) - } catch { - do { - try await self.cleanUpContainer(containerInfo: ctrInfo) - await self.setState(.stopped) - } catch { - self.log.error("failed to clean up container", metadata: ["error": "\(error)"]) - } - throw error + // Every container runs in a pod, so every machine this service + // drives is a pod's. + guard bundle.isPod else { + throw ContainerizationError( + .invalidState, + message: "a sandbox is bootstrapped from a pod, and \(self.root.path) holds none" + ) } - return message.reply() + return try await self.bootstrapPod(message, bundle: bundle) } } @@ -336,10 +194,10 @@ public actor RuntimeService { return try await self.lock.withLock { lock in let id = try message.id() - let containerInfo = try await self.getContainer() - let containerId = containerInfo.container.id + let containerInfo = try await self.addressedContainer(message) + let containerId = containerInfo.id if id == containerId { - try await self.startInitProcess(lock: lock) + try await self.startInitProcess(containerId, lock: lock) await self.setState(.running) } else { try await self.startExecProcess(processId: id, lock: lock) @@ -363,8 +221,14 @@ public actor RuntimeService { defer { self.log.debug("exit", metadata: ["func": "\(#function)"]) } return try await self.lock.withLock { lock in - let containerInfo = try await self.getContainer() - let stats = try await containerInfo.container.statistics() + let containerInfo = try await self.addressedContainer(message) + let sandbox = try await self.getSandbox() + guard let stats = try await sandbox.statistics(containerIDs: [containerInfo.id], categories: .all).first else { + throw ContainerizationError( + .notFound, + message: "no statistics for container \(containerInfo.id)" + ) + } let containerStats = ContainerStats( id: stats.id, @@ -435,8 +299,9 @@ public actor RuntimeService { let id = try message.id() let config = try message.processConfig() let stdio = message.stdio() + let container = try await self.addressedContainer(message) - try await self.addNewProcess(id, config, stdio) + try await self.addNewProcess(id, in: container.id, config, stdio) try await self.initializeWaiters(for: id) do { @@ -485,7 +350,7 @@ public actor RuntimeService { var status: RuntimeStatus = .unknown var networks: [Attachment] = [] - var cs: ContainerSnapshot? + var snapshots: [ContainerSnapshot] = [] switch state { case .created, .stopped, .booted, .shuttingDown: @@ -493,15 +358,19 @@ public actor RuntimeService { case .stopping: status = .stopping case .running: - let ctr = try getContainer() - status = .running - networks = ctr.attachments - cs = ContainerSnapshot( - configuration: ctr.config, - status: RuntimeStatus.running, - networks: networks - ) + // The attachments belong to the machine, so any container in it + // reports the same ones. + networks = self.containers.values.first?.attachments ?? [] + snapshots = self.containers.values + .sorted { $0.id < $1.id } + .map { + ContainerSnapshot( + configuration: $0.config, + status: RuntimeStatus.running, + networks: $0.attachments + ) + } } let reply = message.reply() @@ -509,7 +378,7 @@ public actor RuntimeService { .init( status: status, networks: networks, - containers: cs != nil ? [cs!] : [] + containers: snapshots ) ) return reply @@ -538,18 +407,27 @@ public actor RuntimeService { case .running, .booted: await self.setState(.stopping) - let ctr = try await self.getContainer() - let exitStatus = try await self.gracefulStopContainer( - ctr.container, - signal: signal, - timeout: timeout - ) + let sandbox = try await self.getSandbox() + // Every container in the machine is stopped before the machine + // itself goes, so each is given its chance to end on its own. + var exitStatuses: [String: ExitStatus] = [:] + for ctr in await self.sortedContainers() { + exitStatuses[ctr.id] = try await self.gracefulStopContainer( + sandbox, + id: ctr.id, + signal: signal, + timeout: timeout + ) + } + try await sandbox.stop() do { if case .stopped = await self.state { return message.reply() } - try await self.cleanUpContainer(containerInfo: ctr, exitStatus: exitStatus) + for ctr in await self.sortedContainers() { + try await self.cleanUpContainer(containerInfo: ctr, exitStatus: exitStatuses[ctr.id]) + } } catch { self.log.error("failed to clean up container", metadata: ["error": "\(error)"]) } @@ -570,6 +448,34 @@ public actor RuntimeService { /// /// - Returns: An XPC message with no parameters. @Sendable + /// Stop the container a message is addressed to. + /// + /// The machine holds it and whatever else was put in it, and runs while any + /// of them runs, so it goes down here only once the last one has stopped. + /// A machine given a single container therefore goes down with it, which is + /// what it did when a container had a machine to itself, and a machine + /// holding several stays up for the rest. + public func stopContainer(_ message: XPCMessage) async throws -> XPCMessage { + self.log.debug("enter", metadata: ["func": "\(#function)"]) + defer { self.log.debug("exit", metadata: ["func": "\(#function)"]) } + + let container = try self.addressedContainer(message) + let stopOptions = try message.stopOptions() + let signal = try Signal(stopOptions.signal ?? "SIGTERM") + let timeout: Duration = .seconds(stopOptions.timeoutInSeconds) + + return try await self.lock.withLock { _ in + let sandbox = try await self.getSandbox() + _ = try await self.gracefulStopContainer( + sandbox, + id: container.config.id, + signal: signal, + timeout: timeout + ) + return message.reply() + } + } + public func kill(_ message: XPCMessage) async throws -> XPCMessage { self.log.debug("enter", metadata: ["func": "\(#function)"]) defer { self.log.debug("exit", metadata: ["func": "\(#function)"]) } @@ -580,8 +486,9 @@ public actor RuntimeService { try await self.lock.withLock { [self] _ in switch await self.state { case .running: - let ctr = try await getContainer() - if id != ctr.container.id { + // A process named for a container in the machine is that + // container's init; anything else was started by an exec. + guard await self.isContainer(id) else { guard let processInfo = await self.processes[id] else { throw ContainerizationError(.invalidState, message: "process \(id) does not exist") } @@ -593,7 +500,7 @@ public actor RuntimeService { return } - try await ctr.container.kill(signal) + try await self.getSandbox().killContainer(id, signal: signal) default: throw ContainerizationError( .invalidState, @@ -630,11 +537,13 @@ public actor RuntimeService { switch self.state { case .running: let id = try message.id() - let ctr = try getContainer() let width = message.uint64(key: RuntimeKeys.width.rawValue) let height = message.uint64(key: RuntimeKeys.height.rawValue) + let size = Terminal.Size(width: UInt16(width), height: UInt16(height)) - if id != ctr.container.id { + if self.isContainer(id) { + try await self.getSandbox().resizeContainer(id, to: size) + } else { guard let processInfo = self.processes[id] else { throw ContainerizationError( .invalidState, @@ -649,17 +558,7 @@ public actor RuntimeService { ) } - try await proc.resize( - to: .init( - width: UInt16(width), - height: UInt16(height)) - ) - } else { - try await ctr.container.resize( - to: .init( - width: UInt16(width), - height: UInt16(height)) - ) + try await proc.resize(to: size) } return message.reply() @@ -726,8 +625,9 @@ public actor RuntimeService { let mode = UInt32(message.uint64(key: RuntimeKeys.fileMode.rawValue)) let createParents = message.bool(key: RuntimeKeys.createParents.rawValue) - let ctr = try getContainer() - try await ctr.container.copyIn( + let ctr = try addressedContainer(message) + try await self.getSandbox().copyIn( + ctr.id, from: URL(fileURLWithPath: source), to: URL(fileURLWithPath: destination), mode: mode, @@ -771,8 +671,9 @@ public actor RuntimeService { let createParents = message.bool(key: RuntimeKeys.createParents.rawValue) - let ctr = try getContainer() - try await ctr.container.copyOut( + let ctr = try addressedContainer(message) + try await self.getSandbox().copyOut( + ctr.id, from: URL(fileURLWithPath: source), to: URL(fileURLWithPath: destination), createParents: createParents @@ -816,11 +717,12 @@ public actor RuntimeService { ) } - let ctr = try getContainer() + let ctr = try addressedContainer(message) + let sandbox = try getSandbox() let shouldFreeze = self.state == .running if shouldFreeze { - try await ctr.container.filesystemOperation(operation: .freeze, path: "/") + try await sandbox.filesystemOperation(ctr.id, operation: .freeze, path: "/") } do { @@ -828,7 +730,7 @@ public actor RuntimeService { } catch { if shouldFreeze { do { - try await ctr.container.filesystemOperation(operation: .thaw, path: "/") + try await sandbox.filesystemOperation(ctr.id, operation: .thaw, path: "/") } catch { self.log.error( "failed to thaw filesystem after snapshotDisk error", @@ -841,7 +743,7 @@ public actor RuntimeService { } if shouldFreeze { - try await ctr.container.filesystemOperation(operation: .thaw, path: "/") + try await sandbox.filesystemOperation(ctr.id, operation: .thaw, path: "/") } return message.reply() @@ -876,8 +778,7 @@ public actor RuntimeService { ) } - let ctr = try getContainer() - let fh = try await ctr.container.dialVsock(port: UInt32(port)) + let fh = try await getSandbox().dialVsock(port: UInt32(port)) let reply = message.reply() reply.set(key: RuntimeKeys.fd.rawValue, value: fh) @@ -890,12 +791,11 @@ public actor RuntimeService { } } - private func startInitProcess(lock: AsyncLock.Context) async throws { - let info = try self.getContainer() - let container = info.container - let id = container.id + private func startInitProcess(_ id: String, lock: AsyncLock.Context) async throws { + let info = try self.getContainer(id) + let sandbox = try self.getSandbox() - guard self.state == .booted else { + guard self.state == .booted || self.state == .running else { throw ContainerizationError( .invalidState, message: "container expected to be in booted state, got: \(self.state)" @@ -905,9 +805,9 @@ public actor RuntimeService { do { let io = info.io - try await container.start() + try await sandbox.startContainer(id) let waitFunc: ExitMonitor.WaitHandler = { - let code = try await container.wait() + let code = try await sandbox.waitContainer(id, timeoutInSeconds: nil) if let out = io.out { try out.close() } @@ -925,19 +825,21 @@ public actor RuntimeService { } private func startExecProcess(processId id: String, lock: AsyncLock.Context) async throws { - let container = try self.getContainer().container + let sandbox = try self.getSandbox() guard let processInfo = self.processes[id] else { throw ContainerizationError(.notFound, message: "process with id \(id)") } - let containerInfo = try self.getContainer() + let containerInfo = try self.getContainer(processInfo.containerId) let czConfig = try self.configureProcessConfig( config: processInfo.config, stdio: processInfo.io, containerConfig: containerInfo.config, ) - let process = try await container.exec(id, configuration: czConfig) + let process = try await sandbox.execInContainer(containerInfo.id, processID: id) { config in + config = czConfig + } try self.setUnderlyingProcess(id, process) try await process.start() @@ -1038,10 +940,10 @@ public actor RuntimeService { } private func onContainerExit(id: String, exitStatus: ExitStatus) async throws { - self.log.info("init process exited", metadata: ["status": "\(exitStatus)"]) + self.log.info("init process exited", metadata: ["id": "\(id)", "status": "\(exitStatus)"]) try await self.lock.withLock { [self] _ in - let ctrInfo = try await getContainer() + let ctrInfo = try await getContainer(id) switch await self.state { case .stopped, .stopping: @@ -1055,101 +957,23 @@ public actor RuntimeService { } catch { self.log.error("failed to clean up container", metadata: ["error": "\(error)"]) } - await setState(.stopped) - } - } - private static func configureContainer( - czConfig: inout LinuxContainer.Configuration, - config: ContainerConfiguration, - dynamicEnv: [String: String] = [:], - log: Logger? = nil, - ) throws { - czConfig.cpus = config.resources.cpus - czConfig.cpuOverhead = config.resources.cpuOverhead - czConfig.memoryInBytes = config.resources.memoryInBytes - // Overcommit memory and allow more memory mappings than the kernel default - // so workloads inside swap-less guest VMs hit limits less easily. - var sysctls = config.sysctls - sysctls["vm.overcommit_memory"] = "1" - sysctls["vm.max_map_count"] = "262144" - czConfig.sysctl = sysctls - // If the host doesn't support this, we'll throw on container creation. - czConfig.virtualization = config.virtualization - czConfig.useInit = config.useInit - - // nil leaves LinuxContainer's own default set in place. - if let maskedPaths = config.maskedPaths { - czConfig.maskedPaths = maskedPaths - } - if let readonlyPaths = config.readonlyPaths { - czConfig.readonlyPaths = readonlyPaths - } - - if let shmSize = config.shmSize { - for i in czConfig.mounts.indices { - if czConfig.mounts[i].destination == "/dev/shm" { - czConfig.mounts[i].options.removeAll { $0.hasPrefix("size=") } - czConfig.mounts[i].options.append("size=\(shmSize)") - } + // A pod's machine is its sandbox, which outlives the containers + // that come and go in it: it holds the addresses and namespaces + // they share and is taken down when the pod is, not when a + // container in it leaves. A single container's machine is the + // container's own, so it stops with the last thing in it. + // https://github.com/kubernetes/cri-api/blob/master/pkg/apis/runtime/v1/api.proto + let sandbox = try await self.getSandbox() + guard !(sandbox is LinuxPod) else { + return } - } - - for mount in config.mounts { - if try mount.isSocket() { - let attrs = try? FileManager.default.attributesOfItem(atPath: mount.source) - let permissions = (attrs?[.posixPermissions] as? NSNumber) - .map { FilePermissions(rawValue: mode_t($0.intValue)) } - let socket = UnixSocketConfiguration( - source: URL(filePath: mount.source), - destination: URL(filePath: mount.destination), - permissions: permissions, - direction: .into, - ) - czConfig.sockets.append(socket) - } else { - czConfig.mounts.append(mount.asMount) + guard await self.containers.isEmpty else { + return } + try? await sandbox.stop() + await setState(.stopped) } - - for publishedSocket in config.publishedSockets { - // UnixSocketConfiguration (Containerization) takes URL; convert from FilePath at the boundary. - let socketConfig = UnixSocketConfiguration( - source: URL(filePath: publishedSocket.containerPath.string), - destination: URL(filePath: publishedSocket.hostPath.string), - permissions: publishedSocket.permissions, - direction: .outOf - ) - czConfig.sockets.append(socketConfig) - } - - if let socketUrl = Self.sshAuthSocketHostUrl(config: config, dynamicEnv: dynamicEnv, log: log) { - let socketPath = socketUrl.path(percentEncoded: false) - let attrs = try? FileManager.default.attributesOfItem(atPath: socketPath) - let permissions = (attrs?[.posixPermissions] as? NSNumber) - .map { FilePermissions(rawValue: mode_t($0.intValue)) } - let socketConfig = UnixSocketConfiguration( - source: socketUrl, - destination: URL(fileURLWithPath: Self.sshAuthSocketGuestPath), - permissions: permissions, - direction: .into, - ) - czConfig.sockets.append(socketConfig) - } - - let hostnameSource = config.networks.first?.options.hostname ?? config.id - czConfig.hostname = - hostnameSource.split(separator: ".", maxSplits: 1, omittingEmptySubsequences: true) - .first - .map { String($0) } ?? config.id - - if let dns = config.dns { - czConfig.dns = DNS( - nameservers: dns.nameservers, domain: dns.domain, - searchDomains: dns.searchDomains, options: dns.options) - } - - try Self.configureInitialProcess(czConfig: &czConfig, config: config) } private nonisolated func getDefaultNameservers(from attachments: [Attachment]) -> [String] { @@ -1159,37 +983,39 @@ public actor RuntimeService { return [] } + /// The init process a container starts with, which is the same whether the + /// container has a machine to itself or shares one with a pod's others. private static func configureInitialProcess( - czConfig: inout LinuxContainer.Configuration, + process czProcess: inout LinuxProcessConfiguration, config: ContainerConfiguration, ) throws { let process = config.initProcess - czConfig.process.arguments = [process.executable] + process.arguments - czConfig.process.environmentVariables = process.environment + czProcess.arguments = [process.executable] + process.arguments + czProcess.environmentVariables = process.environment if config.ssh { - if !czConfig.process.environmentVariables.contains(where: { $0.starts(with: "\(Self.sshAuthSocketEnvVar)=") }) { - czConfig.process.environmentVariables.append("\(Self.sshAuthSocketEnvVar)=\(Self.sshAuthSocketGuestPath)") + if !czProcess.environmentVariables.contains(where: { $0.starts(with: "\(Self.sshAuthSocketEnvVar)=") }) { + czProcess.environmentVariables.append("\(Self.sshAuthSocketEnvVar)=\(Self.sshAuthSocketGuestPath)") } } - czConfig.process.terminal = process.terminal - czConfig.process.workingDirectory = process.workingDirectory - try czConfig.process.rlimits = process.rlimits.map { + czProcess.terminal = process.terminal + czProcess.workingDirectory = process.workingDirectory + try czProcess.rlimits = process.rlimits.map { LinuxRLimit( kind: try LinuxRLimit.Kind($0.limit), hard: $0.hard, soft: $0.soft ) } - czConfig.process.capabilities = try Self.effectiveCapabilities( + czProcess.capabilities = try Self.effectiveCapabilities( capAdd: config.capAdd, capDrop: config.capDrop ) switch process.user { case .raw(let name): - czConfig.process.user = .init( + czProcess.user = .init( uid: 0, gid: 0, umask: nil, @@ -1197,7 +1023,7 @@ public actor RuntimeService { username: name ) case .id(let uid, let gid): - czConfig.process.user = .init( + czProcess.user = .init( uid: uid, gid: gid, umask: nil, @@ -1207,6 +1033,123 @@ public actor RuntimeService { } } + /// The sockets a container asks for: those it is given, those it publishes, + /// and the host's ssh agent when it asked for one. + private static func sockets( + config: ContainerConfiguration, + dynamicEnv: [String: String], + log: Logger? + ) throws -> (sockets: [UnixSocketConfiguration], mounts: [Filesystem]) { + var sockets: [UnixSocketConfiguration] = [] + var mounts: [Filesystem] = [] + + for mount in config.mounts { + if try mount.isSocket() { + let attrs = try? FileManager.default.attributesOfItem(atPath: mount.source) + let permissions = (attrs?[.posixPermissions] as? NSNumber) + .map { FilePermissions(rawValue: mode_t($0.intValue)) } + sockets.append( + UnixSocketConfiguration( + source: URL(filePath: mount.source), + destination: URL(filePath: mount.destination), + permissions: permissions, + direction: .into, + )) + } else { + mounts.append(mount) + } + } + + for publishedSocket in config.publishedSockets { + // UnixSocketConfiguration (Containerization) takes URL; convert from FilePath at the boundary. + sockets.append( + UnixSocketConfiguration( + source: URL(filePath: publishedSocket.containerPath.string), + destination: URL(filePath: publishedSocket.hostPath.string), + permissions: publishedSocket.permissions, + direction: .outOf + )) + } + + if let socketUrl = Self.sshAuthSocketHostUrl(config: config, dynamicEnv: dynamicEnv, log: log) { + let socketPath = socketUrl.path(percentEncoded: false) + let attrs = try? FileManager.default.attributesOfItem(atPath: socketPath) + let permissions = (attrs?[.posixPermissions] as? NSNumber) + .map { FilePermissions(rawValue: mode_t($0.intValue)) } + sockets.append( + UnixSocketConfiguration( + source: socketUrl, + destination: URL(fileURLWithPath: Self.sshAuthSocketGuestPath), + permissions: permissions, + direction: .into, + )) + } + + return (sockets, mounts) + } + + /// The hostname a container reports, taken from the name its network knows + /// it by, up to the first dot. + /// The name the sandbox answers to, taken from the first network it + /// attaches to and falling back to its own id. + private static func hostname(networks: [AttachmentConfiguration], id: String) -> String { + let hostnameSource = networks.first?.options.hostname ?? id + return + hostnameSource.split(separator: ".", maxSplits: 1, omittingEmptySubsequences: true) + .first + .map { String($0) } ?? id + } + + /// Configure a container that shares a pod's machine. + /// + /// The machine's processors, memory, swap, addresses and boot log are the + /// pod's, so what is set here is the container's alone: what it runs, what + /// it can see, and the limits it holds itself to within the pod's. + private static func configurePodContainer( + czConfig: inout LinuxPod.ContainerConfiguration, + config: ContainerConfiguration, + dynamicEnv: [String: String] = [:], + log: Logger? = nil, + ) throws { + // A container in a pod draws on the machine's processors and memory + // unless it was given a limit of its own. + czConfig.cpus = config.resources.cpus + czConfig.memoryInBytes = config.resources.memoryInBytes + czConfig.swapInBytes = config.resources.swapInBytes + + czConfig.useInit = config.useInit + + // nil leaves the library's own default set in place. + if let maskedPaths = config.maskedPaths { + czConfig.maskedPaths = maskedPaths + } + if let readonlyPaths = config.readonlyPaths { + czConfig.readonlyPaths = readonlyPaths + } + + if let shmSize = config.shmSize { + for i in czConfig.mounts.indices { + if czConfig.mounts[i].destination == "/dev/shm" { + czConfig.mounts[i].options.removeAll { $0.hasPrefix("size=") } + czConfig.mounts[i].options.append("size=\(shmSize)") + } + } + } + + let (sockets, mounts) = try Self.sockets(config: config, dynamicEnv: dynamicEnv, log: log) + czConfig.sockets.append(contentsOf: sockets) + czConfig.mounts.append(contentsOf: mounts.map { $0.asMount }) + + // The hostname, the resolver and the hosts file are the sandbox's: the + // runtime interface carries all three on the pod and gives a container + // none of its own, so the pod holds them and its containers inherit + // them. The library lets a container override each one; a container + // here asks for none, so the pod's stand. + // https://github.com/kubernetes/cri-api/blob/master/pkg/apis/runtime/v1/api.proto + + try Self.configureInitialProcess(process: &czConfig.process, config: config) + } + private nonisolated func configureProcessConfig(config: ProcessConfiguration, stdio: [FileHandle?], containerConfig: ContainerConfiguration) throws -> LinuxProcessConfiguration { @@ -1299,29 +1242,375 @@ public actor RuntimeService { } } - private func getContainer() throws -> ContainerInfo { - guard let container else { + /// Run the machine a pod's containers share, with those containers in it. + /// + /// The pod's own bundle carries the machine: its kernel, its initial + /// filesystem, and the size, networks and name resolution its containers + /// draw on. The containers keep bundles of their own, named in the request, + /// and go in on the way to a machine that runs holding them. + /// + private func bootstrapPod(_ message: XPCMessage, bundle: ContainerResource.Bundle) async throws -> XPCMessage { + let config = try bundle.podConfiguration + let kernel = try self.kernelWithDefaultArgs(bundle.kernel) + let vmm = VZVirtualMachineManager( + kernel: kernel, + initialFilesystem: bundle.initialFilesystem.asMount, + rosetta: config.rosetta, + logger: self.log + ) + + let (sessions, attachments, interfaces) = try await self.allocateNetworks( + config.networks, + infos: try message.networkBootstrapInfos() + ) + + // Dynamically configure the DNS nameserver from a network if no explicit + // configuration. The network belongs to the pod, so the resolver derived + // from it does too, the way it was derived from the machine's attachments + // when the machine held the network for one container. + var nameservers = config.dns?.nameservers ?? [] + if nameservers.isEmpty { + nameservers = self.getDefaultNameservers(from: attachments) + } + + // One swap area serves the whole pod, which is what makes the pool its + // containers reclaim to a shared one. + let swapLayer = try config.resources.swapInBytes.map { + try bundle.createSwapDevice(size: $0).asMount + } + + let pod = try LinuxPod(config.id, vmm: vmm, logger: self.log) { podConfig in + podConfig.cpus = config.resources.cpus + podConfig.memoryInBytes = config.resources.memoryInBytes + // The machine is built larger than the pod by what the guest agent + // takes, so what the pod was given is what its containers have. A + // caller sizing the machine itself asks for none of that overhead + // and gets the size it named. + podConfig.cpuOverhead = config.resources.cpuOverhead + podConfig.swapLayer = swapLayer + podConfig.interfaces = interfaces + podConfig.virtualization = config.virtualization + podConfig.shareProcessNamespace = config.shareProcessNamespace + podConfig.hostname = config.hostname ?? Self.hostname(networks: config.networks, id: config.id) + // The hosts file names the pod at its own address so its containers + // reach the name they answer to, and it is written once for the pod + // the way the resolver and the hostname are. + var hostsEntries = [Hosts.Entry.localHostIPV4()] + if let primary = attachments.first { + hostsEntries.append( + Hosts.Entry( + ipAddress: primary.ipv4Address.address.description, + hostnames: [podConfig.hostname ?? config.id], + )) + } + podConfig.hosts = Hosts(entries: hostsEntries) + // The runtime asks for these two of every machine it boots; they + // stand alongside whatever the pod was given. + var sysctls = config.sysctls + sysctls["vm.overcommit_memory"] = "1" + sysctls["vm.max_map_count"] = "262144" + podConfig.sysctl = sysctls + if !nameservers.isEmpty { + podConfig.dns = DNS( + nameservers: nameservers, + domain: config.dns?.domain, + searchDomains: config.dns?.searchDomains ?? [], + options: config.dns?.options ?? [] + ) + } + podConfig.bootLog = BootLog.file(path: bundle.bootlog, append: true) + } + + self.setSandbox(pod) + self.setNetworkSessions(sessions) + self.podAttachments = attachments + self.podPublishedPorts = config.publishedPorts + + try await self.placeContainers(message) + + try await pod.create() + // The pod holds one address for every container in it, so the ports + // published on it are the pod's and are forwarded once. Forwarding each + // container's separately would let two of them claim one host port, + // which the overlap check cannot see when it is asked about one + // container at a time. + if let primary = attachments.first { + try await self.startSocketForwarders(attachment: primary, publishedPorts: config.publishedPorts) + } + self.setState(.booted) + + return message.reply() + } + + /// Put the containers a request names in the sandbox. + /// + /// Each brings its own bundle, holding its configuration and its root + /// filesystem, and takes the machine's processors, memory, swap and + /// addresses as they are. One already in the machine stays as it is, so a + /// request naming every container the pod holds puts in what is missing and + /// leaves the rest alone. + /// + /// The standard streams belong to the one container whose start the request + /// is; the others are placed with none and are given theirs when they are + /// started in turn. + private func placeContainers(_ message: XPCMessage) async throws { + self.log.debug("enter", metadata: ["func": "\(#function)"]) + defer { self.log.debug("exit", metadata: ["func": "\(#function)"]) } + + guard let pathsData = message.dataNoCopy(key: RuntimeKeys.bundlePaths.rawValue) else { + return + } + let paths = try JSONDecoder().decode([String].self, from: pathsData) + let stdioFor = message.string(key: RuntimeKeys.containerId.rawValue) + + for path in paths { + try await self.placeContainer( + at: path, + stdio: URL(filePath: path).lastPathComponent == stdioFor ? message.stdio() : [nil, nil, nil], + dynamicEnv: try message.dynamicEnv() + ) + } + } + + private func placeContainer(at path: String, stdio: [FileHandle?], dynamicEnv: [String: String]) async throws { + let sandbox = try self.getSandbox() + guard let pod = sandbox as? LinuxPod else { throw ContainerizationError( .invalidState, - message: "no container found" + message: "the sandbox holds a single container and takes no others" + ) + } + + // A container the machine already holds is one this request has nothing + // to do for. + guard !self.isContainer(URL(filePath: path).lastPathComponent) else { + return + } + + do { + // A container placed in a pod has no machine of its own to build + // its bundle, so the pod's machine builds it on the way in. + let root = URL(filePath: path) + if !self.bundleExists(at: root) { + try self.createBundle(at: root) + } + + let bundle = ContainerResource.Bundle(path: root) + try bundle.createLogFile() + let config = try bundle.configuration + let containerLog = try FileHandle(forWritingTo: bundle.containerLog) + let stdout = { + if let h = stdio[1] { + return MultiWriter(handles: [h, containerLog]) + } + return MultiWriter(handles: [containerLog]) + }() + let stderr: MultiWriter? = { + if !config.initProcess.terminal { + if let h = stdio[2] { + return MultiWriter(handles: [h, containerLog]) + } + return MultiWriter(handles: [containerLog]) + } + return nil + }() + let stdin = stdio[0] ?? nil + + let rootfs = try bundle.containerRootfs.asMount + let attachments = self.podAttachments + + try await pod.addContainer(config.id, rootfs: rootfs) { czConfig in + try Self.configurePodContainer( + czConfig: &czConfig, + config: config, + dynamicEnv: dynamicEnv, + log: self.log + ) + czConfig.process.stdout = stdout + czConfig.process.stderr = stderr + czConfig.process.stdin = stdin + } + + self.setContainer( + ContainerInfo( + config: config, + attachments: attachments, + bundle: bundle, + io: (in: stdin, out: stdout, err: stderr) + ) + ) + + // What waits on the container waits from the moment it is in the + // machine, so a container that boots with the machine and one that + // joins a machine already running are both waited on the same way. + try self.initializeWaiters(for: config.id) + try await self.monitor.registerProcess(id: config.id, onExit: self.onContainerExit) + } + } + + /// Hold the running machine to a memory size, which its containers share. + /// + /// - Parameters: + /// - message: An XPC message with the following parameters: + /// - memoryInBytes: The size to hold the machine to. + /// + /// - Returns: An XPC message with no parameters. + @Sendable + public func updateResources(_ message: XPCMessage) async throws -> XPCMessage { + self.log.debug("enter", metadata: ["func": "\(#function)"]) + defer { self.log.debug("exit", metadata: ["func": "\(#function)"]) } + + let bytes = message.uint64(key: RuntimeKeys.memoryInBytes.rawValue) + guard bytes > 0 else { + throw ContainerizationError(.invalidArgument, message: "a memory size is required") + } + try await self.getSandbox().setTargetMemorySize(bytes) + return message.reply() + } + + /// Kernel arguments applied unless the caller already supplied the same + /// key, so a custom kernel can override them (e.g. lsm=...,bpf). + private func kernelWithDefaultArgs(_ kernel: Kernel) -> Kernel { + var kernel = kernel + let defaultKernelArgs: KeyValuePairs = [ + "oops": "panic", + "lsm": "lockdown,capability,landlock,yama,apparmor", + ] + for (key, value) in defaultKernelArgs { + guard !kernel.commandLine.kernelArgs.contains(where: { $0.hasPrefix("\(key)=") }) else { + continue + } + kernel.commandLine.kernelArgs.append("\(key)=\(value)") + } + return kernel + } + + /// Claim an address on each of the sandbox's networks. + /// + /// The attachments belong to the machine, which every container in it + /// shares, since a container in a sandbox is given no network namespace of + /// its own. + private func allocateNetworks( + _ networks: [AttachmentConfiguration], + infos: [NetworkBootstrapInfo] + ) async throws -> (sessions: [XPCClientSession], attachments: [Attachment], interfaces: [Interface]) { + var sessions: [XPCClientSession] = [] + var attachments: [Attachment] = [] + var interfaces: [Interface] = [] + do { + for (index, info) in infos.enumerated() { + let attachmentConfig = networks[index] + let client = ContainerNetworkClient.NetworkClient(id: attachmentConfig.network, plugin: info.plugin) + let session = client.connect() + sessions.append(session) + var (attachment, additionalData) = try await client.allocate( + hostname: attachmentConfig.options.hostname, + macAddress: attachmentConfig.options.macAddress, + on: session + ) + if let mtu = attachmentConfig.options.mtu { + attachment = Attachment( + network: attachment.network, + hostname: attachment.hostname, + ipv4Address: attachment.ipv4Address, + ipv4Gateway: attachment.ipv4Gateway, + ipv6Address: attachment.ipv6Address, + macAddress: attachment.macAddress, + mtu: mtu, + variant: attachment.variant + ) + } + guard let iStrategy = self.interfaceStrategies[NetworkInterfaceKey(plugin: info.plugin, variant: attachment.variant)] else { + throw ContainerizationError( + .internalError, + message: "no available interface strategy for network \(attachment.network), plugin=\(info.plugin) variant=\(attachment.variant ?? "nil")") + } + let interface = try iStrategy.toInterface( + attachment: attachment, + interfaceIndex: index, + additionalData: additionalData + ) + attachments.append(attachment) + interfaces.append(interface) + } + } catch { + for session in sessions { session.close() } + throw error + } + return (sessions, attachments, interfaces) + } + + /// The machine this service drives. + private func getSandbox() throws -> any Sandbox { + guard let sandbox else { + throw ContainerizationError( + .invalidState, + message: "no sandbox found" + ) + } + return sandbox + } + + /// The machine's containers in a settled order, so that what is done to + /// each of them in turn happens the same way every time. + private func sortedContainers() -> [ContainerInfo] { + self.containers.values.sorted { $0.id < $1.id } + } + + /// Whether a name is one of the machine's containers, which is what tells + /// a container's init process apart from a process an exec started. + private func isContainer(_ id: String) -> Bool { + self.containers[id] != nil + } + + /// A container in the machine, by name. + private func getContainer(_ id: String) throws -> ContainerInfo { + guard let container = self.containers[id] else { + throw ContainerizationError( + .notFound, + message: "container \(id) not found in sandbox" ) } return container } - private func gracefulStopContainer(_ lc: LinuxContainer, signal: Signal, timeout: Duration) async throws -> ExitStatus { + /// The container a message is addressed to. + /// + /// A request for a container names it, whatever else the machine holds, so + /// that a machine holding one is answered the same way as a machine holding + /// several and no request means "the only one here". + private func addressedContainer(_ message: XPCMessage) throws -> ContainerInfo { + guard let id = message.string(key: RuntimeKeys.containerId.rawValue), !id.isEmpty else { + throw ContainerizationError( + .invalidArgument, + message: "the request names no container to act on" + ) + } + return try getContainer(id) + } + + /// Stop one container in the sandbox and wait for it, then leave. + /// + /// The machine stays up, since the sandbox's other containers are still in + /// it. Powering it off is the sandbox's own stop. + private func gracefulStopContainer( + _ sandbox: any Sandbox, + id: String, + signal: Signal, + timeout: Duration + ) async throws -> ExitStatus { // Try and gracefully shut down the process. Even if this succeeds we need to power off // the vm, but we should try this first always. var code = ExitStatus(exitCode: 255) do { code = try await withThrowingTaskGroup(of: ExitStatus.self) { group in group.addTask { - try await lc.wait() + try await sandbox.waitContainer(id, timeoutInSeconds: nil) } group.addTask { - try await lc.kill(signal) + try await sandbox.killContainer(id, signal: signal) try await Task.sleep(for: timeout) - try await lc.kill(.kill) + try await sandbox.killContainer(id, signal: .kill) return ExitStatus(exitCode: 137) } @@ -1339,29 +1628,38 @@ public actor RuntimeService { self.log.error("graceful stop failed; forcing vm shutdown", metadata: ["error": "\(error)"]) } - // Now actually bring down the vm. - try await lc.stop() - return code } private func cleanUpContainer(containerInfo: ContainerInfo, exitStatus: ExitStatus? = nil) async throws { - let container = containerInfo.container - let id = container.id + let id = containerInfo.id do { - try await container.stop() + try await self.getSandbox().stopContainer(id) } catch { self.log.error("failed to stop container during cleanup", metadata: ["error": "\(error)"]) } - await self.stopSocketForwarders() + self.containers.removeValue(forKey: id) + self.processes.removeValue(forKey: id) + await self.monitor.stopTracking(id: id) - for session in networkSessions { session.close() } - networkSessions = [] + // The forwarders and the network sessions are the machine's, which the + // sandbox's containers share, so they are given up once the last of + // them is gone. + if self.containers.isEmpty { + await self.stopSocketForwarders() + + for session in networkSessions { session.close() } + networkSessions = [] + } let status = exitStatus ?? ExitStatus(exitCode: 255) self.releaseWaiters(for: id, status: status) + // The waiter's name is given back with the container's: whoever was + // waiting has been answered, and the next container under this name + // registers a waiter of its own. + self.waiters.removeValue(forKey: id) } } @@ -1614,21 +1912,27 @@ extension RuntimeService { } private func setContainer(_ info: ContainerInfo) { - self.container = info + self.containers[info.id] = info + } + + private func setSandbox(_ sandbox: any Sandbox) { + self.sandbox = sandbox } private func setNetworkSessions(_ sessions: [XPCClientSession]) { self.networkSessions = sessions } - private func addNewProcess(_ id: String, _ config: ProcessConfiguration, _ io: [FileHandle?]) throws { + private func addNewProcess(_ id: String, in containerId: String, _ config: ProcessConfiguration, _ io: [FileHandle?]) throws { guard self.processes[id] == nil else { throw ContainerizationError(.invalidArgument, message: "process \(id) already exists") } - self.processes[id] = ProcessInfo(config: config, process: nil, state: .created, io: io) + self.processes[id] = ProcessInfo(containerId: containerId, config: config, process: nil, state: .created, io: io) } private struct ProcessInfo { + /// The container in the sandbox the process runs in. + let containerId: String let config: ProcessConfiguration var process: LinuxProcess? var state: State @@ -1636,11 +1940,12 @@ extension RuntimeService { } private struct ContainerInfo { - let container: LinuxContainer let config: ContainerConfiguration let attachments: [Attachment] let bundle: ContainerResource.Bundle let io: (in: FileHandle?, out: MultiWriter?, err: MultiWriter?) + + var id: String { config.id } } /// States the underlying sandbox can be in. @@ -1671,6 +1976,9 @@ extension RuntimeService { } let bundle = ContainerResource.Bundle(path: path) + if bundle.isPod { + return true + } do { _ = try bundle.configuration return true @@ -1680,14 +1988,15 @@ extension RuntimeService { } /// Create bundle from RuntimeConfiguration - private func createBundle() throws { + private func createBundle(at root: URL? = nil) throws { do { - let runtimeConfig = try RuntimeConfiguration.readRuntimeConfiguration(from: self.root) + let runtimeConfig = try RuntimeConfiguration.readRuntimeConfiguration(from: root ?? self.root) _ = try ContainerResource.Bundle.create( path: runtimeConfig.path, initialFilesystem: runtimeConfig.initialFilesystem, kernel: runtimeConfig.kernel, containerConfiguration: runtimeConfig.containerConfiguration, + podConfiguration: runtimeConfig.podConfiguration, containerRootFilesystem: runtimeConfig.containerRootFilesystem, options: runtimeConfig.options ) diff --git a/Tests/K8sPluginTests/K8sListTests.swift b/Tests/K8sPluginTests/K8sListTests.swift index f8d04b422..326e379a7 100644 --- a/Tests/K8sPluginTests/K8sListTests.swift +++ b/Tests/K8sPluginTests/K8sListTests.swift @@ -48,6 +48,7 @@ private func makeSnapshot( { "configuration": { "id": "\(id)", + "pod": "\(id)-pod", "image": { "reference": "docker.io/kindest/node:v1.35.5", "descriptor": {"mediaType":"","digest":"sha256:abc","size":0} From 0a49daafb4a191d02c6fe05fc584cc5ef9c0bf61 Mon Sep 17 00:00:00 2001 From: Aaron Paterson Date: Sun, 9 Aug 2026 20:46:53 +0000 Subject: [PATCH 06/29] Give the command line a pod A pod is a thing to make, look at, list, start, stop and remove, so it has the commands the other resources have, and a container is placed in one by name. A pod nobody named is anonymous, the way a volume mounted without a name is, and prune takes those away: a pod someone named is left alone unless all of them are asked for, which is what prune does for volumes. --- Sources/ContainerCommands/Application.swift | 6 + .../ContainerCommands/Pod/PodCommand.swift | 43 +++++ Sources/ContainerCommands/Pod/PodCreate.swift | 114 ++++++++++++ .../ContainerCommands/Pod/PodLifecycle.swift | 172 ++++++++++++++++++ Sources/ContainerCommands/Pod/PodPrune.swift | 83 +++++++++ .../Pod/PodSnapshot+ListDisplayable.swift | 42 +++++ 6 files changed, 460 insertions(+) create mode 100644 Sources/ContainerCommands/Pod/PodCommand.swift create mode 100644 Sources/ContainerCommands/Pod/PodCreate.swift create mode 100644 Sources/ContainerCommands/Pod/PodLifecycle.swift create mode 100644 Sources/ContainerCommands/Pod/PodPrune.swift create mode 100644 Sources/ContainerCommands/Pod/PodSnapshot+ListDisplayable.swift diff --git a/Sources/ContainerCommands/Application.swift b/Sources/ContainerCommands/Application.swift index 6845bb15c..57a3cb741 100644 --- a/Sources/ContainerCommands/Application.swift +++ b/Sources/ContainerCommands/Application.swift @@ -90,6 +90,12 @@ public struct Application: AsyncLoggableCommand { VolumeCommand.self ] ), + CommandGroup( + name: "Pod", + subcommands: [ + PodCommand.self + ] + ), CommandGroup( name: "Other", subcommands: Self.otherCommands() diff --git a/Sources/ContainerCommands/Pod/PodCommand.swift b/Sources/ContainerCommands/Pod/PodCommand.swift new file mode 100644 index 000000000..2a88fcbcd --- /dev/null +++ b/Sources/ContainerCommands/Pod/PodCommand.swift @@ -0,0 +1,43 @@ +//===----------------------------------------------------------------------===// +// Copyright © 2026 Apple Inc. and the container project authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//===----------------------------------------------------------------------===// + +import ArgumentParser +import ContainerAPIClient + +extension Application { + public struct PodCommand: AsyncLoggableCommand { + public static let configuration = CommandConfiguration( + commandName: "pod", + abstract: "Manage pods, machines that several containers share", + subcommands: [ + PodCreate.self, + PodStart.self, + PodStop.self, + PodDelete.self, + PodList.self, + PodInspect.self, + PodPrune.self, + PodUpdate.self, + ], + aliases: ["p"] + ) + + public init() {} + + @OptionGroup + public var logOptions: Flags.Logging + } +} diff --git a/Sources/ContainerCommands/Pod/PodCreate.swift b/Sources/ContainerCommands/Pod/PodCreate.swift new file mode 100644 index 000000000..ba437aae2 --- /dev/null +++ b/Sources/ContainerCommands/Pod/PodCreate.swift @@ -0,0 +1,114 @@ +//===----------------------------------------------------------------------===// +// Copyright © 2026 Apple Inc. and the container project authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//===----------------------------------------------------------------------===// + +import ArgumentParser +import ContainerAPIClient +import ContainerPersistence +import ContainerResource +import ContainerizationError +import Foundation + +extension Application.PodCommand { + public struct PodCreate: AsyncLoggableCommand { + public static let configuration = CommandConfiguration( + commandName: "create", + abstract: "Create a pod for containers to be placed in" + ) + + @OptionGroup + public var logOptions: Flags.Logging + + @OptionGroup + public var resource: Flags.Resource + + @OptionGroup + public var dns: Flags.DNS + + @Option(name: .long, help: "Hostname the pod's machine reports, which its containers share") + var hostname: String? + + @Option(name: .long, help: "Network to attach the pod to, which its containers share") + var network: [String] = [] + + @Flag(name: .long, help: "Let the pod's containers see each other's processes") + var sharePidNamespace: Bool = false + + @Flag(name: .long, help: "Expose nested virtualization to the pod's containers") + var virtualization: Bool = false + + @Flag(name: .long, help: "Enable Rosetta in the pod's containers") + var rosetta: Bool = false + + @Option(name: .long, help: "Key=value metadata for the pod") + var label: [String] = [] + + @Argument(help: "Name for the pod") + var name: String + + public init() {} + + public func run() async throws { + let containerSystemConfig: ContainerSystemConfig = try await Application.loadContainerSystemConfig() + + guard ManagedContainer.nameValid(name) else { + throw ContainerizationError(.invalidArgument, message: "pod name \(name) is not a valid name") + } + + var configuration = PodConfiguration(id: name) + configuration.resources = try Parser.resources( + cpus: resource.cpus, + memory: resource.memory, + swap: resource.swap, + defaultCPUs: containerSystemConfig.container.cpus, + defaultMemory: containerSystemConfig.container.memory, + defaultSwap: containerSystemConfig.container.swap + ) + configuration.hostname = hostname + configuration.shareProcessNamespace = sharePidNamespace + configuration.virtualization = virtualization + configuration.rosetta = rosetta + configuration.labels = try Parser.labels(label) + + if !dns.nameservers.isEmpty || dns.domain != nil || !dns.searchDomains.isEmpty || !dns.options.isEmpty { + configuration.dns = ContainerConfiguration.DNSConfiguration( + nameservers: dns.nameservers.isEmpty + ? ContainerConfiguration.DNSConfiguration.defaultNameservers + : dns.nameservers, + domain: dns.domain, + searchDomains: dns.searchDomains, + options: dns.options + ) + } + + let parsedNetworks = try network.map { try Parser.network($0) } + let networkClient = NetworkClient() + let builtinNetworkId = try await networkClient.builtin?.id + configuration.networks = try Utility.getAttachmentConfigurations( + containerId: name, + builtinNetworkId: builtinNetworkId, + networks: parsedNetworks, + dnsDomain: containerSystemConfig.dns.domain, + ) + for attachmentConfiguration in configuration.networks { + _ = try await networkClient.get(id: attachmentConfiguration.network) + } + + let kernel = try await ClientKernel.getDefaultKernel(for: .current) + try await ClientPod.create(configuration: configuration, kernel: kernel) + print(name) + } + } +} diff --git a/Sources/ContainerCommands/Pod/PodLifecycle.swift b/Sources/ContainerCommands/Pod/PodLifecycle.swift new file mode 100644 index 000000000..b97c025aa --- /dev/null +++ b/Sources/ContainerCommands/Pod/PodLifecycle.swift @@ -0,0 +1,172 @@ +//===----------------------------------------------------------------------===// +// Copyright © 2026 Apple Inc. and the container project authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//===----------------------------------------------------------------------===// + +import ArgumentParser +import ContainerAPIClient +import ContainerResource +import ContainerizationError +import ContainerizationExtras +import Foundation + +extension Application.PodCommand { + public struct PodStart: AsyncLoggableCommand { + public static let configuration = CommandConfiguration( + commandName: "start", + abstract: "Boot a pod's machine, with the containers in it" + ) + + @OptionGroup + public var logOptions: Flags.Logging + + @Argument(help: "Pods to start") + var names: [String] + + public init() {} + + public func run() async throws { + for name in names { + try await ClientPod.start(name) + print(name) + } + } + } + + public struct PodStop: AsyncLoggableCommand { + public static let configuration = CommandConfiguration( + commandName: "stop", + abstract: "Stop a pod's machine, and with it every container inside" + ) + + @OptionGroup + public var logOptions: Flags.Logging + + @Argument(help: "Pods to stop") + var names: [String] + + public init() {} + + public func run() async throws { + for name in names { + try await ClientPod.stop(name) + print(name) + } + } + } + + public struct PodDelete: AsyncLoggableCommand { + public static let configuration = CommandConfiguration( + commandName: "delete", + abstract: "Delete one or more pods", + aliases: ["rm"] + ) + + @OptionGroup + public var logOptions: Flags.Logging + + @Flag(name: .shortAndLong, help: "Delete the pod's containers along with it") + var force: Bool = false + + @Argument(help: "Pods to delete") + var names: [String] + + public init() {} + + public func run() async throws { + for name in names { + try await ClientPod.delete(name, force: force) + print(name) + } + } + } + + public struct PodUpdate: AsyncLoggableCommand { + public static let configuration = CommandConfiguration( + commandName: "update", + abstract: "Hold a running pod to a memory size, which its containers share" + ) + + @OptionGroup + public var logOptions: Flags.Logging + + @Option( + name: .shortAndLong, + help: """ + Memory the pod's machine is to hold (1MiByte granularity), with optional K, M, G, \ + T, or P suffix. The guest gives back the difference, and takes it again when the \ + size is raised. + """ + ) + var memory: String + + @Argument(help: "Pod to hold") + var name: String + + public init() {} + + public func run() async throws { + let bytes = try Parser.memoryStringAsMiB(memory).mib() + try await ClientPod.update(name, memoryInBytes: bytes) + print(name) + } + } + + public struct PodInspect: AsyncLoggableCommand { + public static let configuration = CommandConfiguration( + commandName: "inspect", + abstract: "Display information about one or more pods" + ) + + @OptionGroup + public var logOptions: Flags.Logging + + @Argument(help: "Pods to inspect") + var names: [String] + + public init() {} + + public func run() async throws { + var snapshots: [PodSnapshot] = [] + for name in Set(names).sorted() { + snapshots.append(try await ClientPod.inspect(name)) + } + try Output.emit(Output.renderJSON(snapshots, options: .pretty)) + } + } + + public struct PodList: AsyncLoggableCommand { + public static let configuration = CommandConfiguration( + commandName: "list", + abstract: "List pods", + aliases: ["ls"] + ) + + @OptionGroup + public var logOptions: Flags.Logging + + @Flag(name: .shortAndLong, help: "Only output the pod names") + var quiet: Bool = false + + @Option(name: .long, help: "Format of the output") + var format: ListFormat = .table + + public init() {} + + public func run() async throws { + let pods = try await ClientPod.list() + try Output.render(payload: pods, display: pods, format: format, quiet: quiet) + } + } +} diff --git a/Sources/ContainerCommands/Pod/PodPrune.swift b/Sources/ContainerCommands/Pod/PodPrune.swift new file mode 100644 index 000000000..00debd81f --- /dev/null +++ b/Sources/ContainerCommands/Pod/PodPrune.swift @@ -0,0 +1,83 @@ +//===----------------------------------------------------------------------===// +// Copyright © 2026 Apple Inc. and the container project authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//===----------------------------------------------------------------------===// + +import ArgumentParser +import ContainerAPIClient +import ContainerResource +import Foundation + +extension Application.PodCommand { + public struct PodPrune: AsyncLoggableCommand { + public init() {} + public static let configuration = CommandConfiguration( + commandName: "prune", + abstract: "Remove anonymous pods with no containers in them") + + @Flag(name: .shortAndLong, help: "Remove pods that were named too, not only anonymous ones") + var all = false + + @OptionGroup + public var logOptions: Flags.Logging + + public func run() async throws { + let allPods = try await ClientPod.list() + + // Find all pods that hold no container + let client = ContainerClient() + let containers = try await client.list() + var podsInUse = Set() + for container in containers { + podsInUse.insert(container.configuration.pod) + } + + // A pod someone named is theirs, and an empty one is still theirs to + // put something in, so a prune leaves it alone unless asked for all + // of them. A pod nobody named was made because a container needed a + // machine, and is of no use to anyone once no container is in it. + // https://github.com/containerd/nerdctl/blob/main/pkg/cmd/volume/prune.go + let podsToPrune = allPods.filter { pod in + guard !podsInUse.contains(pod.configuration.id) else { + return false + } + return all || pod.configuration.isAnonymous + } + + var prunedPods = [String]() + + for pod in podsToPrune { + do { + try await ClientPod.delete(pod.configuration.id, force: true) + prunedPods.append(pod.configuration.id) + } catch { + log.error( + "failed to prune pod", + metadata: [ + "id": "\(pod.configuration.id)", + "error": "\(error)", + ] + ) + } + } + + if !prunedPods.isEmpty { + print("Deleted Pods:") + for pod in prunedPods { + print(pod) + } + } + } + } +} diff --git a/Sources/ContainerCommands/Pod/PodSnapshot+ListDisplayable.swift b/Sources/ContainerCommands/Pod/PodSnapshot+ListDisplayable.swift new file mode 100644 index 000000000..09dae9c97 --- /dev/null +++ b/Sources/ContainerCommands/Pod/PodSnapshot+ListDisplayable.swift @@ -0,0 +1,42 @@ +//===----------------------------------------------------------------------===// +// Copyright © 2026 Apple Inc. and the container project authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//===----------------------------------------------------------------------===// + +import ContainerResource +import ContainerizationExtras +import Foundation + +extension PodSnapshot: ListDisplayable { + public static var tableHeader: [String] { + ["ID", "STATE", "CPUS", "MEMORY", "CONTAINERS", "ADDRESS"] + } + + public var tableRow: [String] { + let formatter = ByteCountFormatter() + formatter.countStyle = .memory + return [ + id, + state == .ready ? "ready" : "not ready", + "\(configuration.resources.cpus)", + formatter.string(fromByteCount: Int64(configuration.resources.memoryInBytes)), + containers.isEmpty ? "" : containers.joined(separator: ","), + networks.first?.ipv4Address.description ?? "", + ] + } + + public var quietValue: String { + id + } +} From 2e4acaa4f97b85fd99d541b0b396a01d2eab05d7 Mon Sep 17 00:00:00 2001 From: Aaron Paterson Date: Sun, 9 Aug 2026 20:47:04 +0000 Subject: [PATCH 07/29] Take away the anonymous resources a container was given A volume mounted without a name is made for the container that mounted it and is the container's to keep, so removing the container takes it away. A pod nobody named is the same kind of thing: it holds the machine the container would otherwise have had to itself, and goes with it. A resource someone named is not the container's. It was there before the container and is there after, so a container leaving says nothing about it, and neither does a container leaving a pod that others are still in. https://github.com/containerd/nerdctl/blob/main/pkg/cmd/container/remove.go --- .../Container/AnonymousResources.swift | 81 +++++++++++++++++++ .../Container/ContainerDelete.swift | 11 +++ 2 files changed, 92 insertions(+) create mode 100644 Sources/ContainerCommands/Container/AnonymousResources.swift diff --git a/Sources/ContainerCommands/Container/AnonymousResources.swift b/Sources/ContainerCommands/Container/AnonymousResources.swift new file mode 100644 index 000000000..e464e83ec --- /dev/null +++ b/Sources/ContainerCommands/Container/AnonymousResources.swift @@ -0,0 +1,81 @@ +//===----------------------------------------------------------------------===// +// Copyright © 2026 Apple Inc. and the container project authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//===----------------------------------------------------------------------===// + +import ContainerAPIClient +import ContainerResource +import Logging + +/// The resources a container was given rather than asked for by name. +/// +/// A volume mounted without a name, and the pod a container that named none was +/// given, are both made because the container needed one and are of no use to +/// anyone once it is gone. They are read off the container before it is removed, +/// since the container is what records them, and taken away after. +/// +/// nerdctl removes these with the container when the container is removed with +/// its volumes, when it was run to be removed on exit, and when containers are +/// pruned, and leaves them alone on a plain removal. +/// https://github.com/containerd/nerdctl/blob/main/pkg/cmd/container/remove.go +public struct AnonymousResources: Sendable { + let volumes: [String] + let pod: String? + + /// Read what a container was given, before it is removed. + public static func given(to container: ContainerSnapshot) async -> AnonymousResources { + var volumes: [String] = [] + for mount in container.configuration.mounts { + guard mount.isVolume, let name = mount.volumeName else { + continue + } + guard let volume = try? await ClientVolume.inspect(name), volume.isAnonymous else { + continue + } + volumes.append(name) + } + + let pod = try? await ClientPod.inspect(container.configuration.pod) + return AnonymousResources( + volumes: volumes, + pod: (pod?.configuration.isAnonymous ?? false) ? pod?.configuration.id : nil + ) + } + + /// Take them away, now that the container that was given them is gone. + /// + /// A resource that will not go is reported and passed over: the container it + /// belonged to is already gone, so failing here would fail a removal that + /// has already happened. + public func remove(log: Logger) async { + for volume in volumes { + do { + try await ClientVolume.delete(name: volume) + } catch { + log.warning( + "failed to remove an anonymous volume", + metadata: ["volume": "\(volume)", "error": "\(error)"]) + } + } + if let pod { + do { + try await ClientPod.delete(pod, force: true) + } catch { + log.warning( + "failed to remove an anonymous pod", + metadata: ["pod": "\(pod)", "error": "\(error)"]) + } + } + } +} diff --git a/Sources/ContainerCommands/Container/ContainerDelete.swift b/Sources/ContainerCommands/Container/ContainerDelete.swift index 1eddc6b85..25892764c 100644 --- a/Sources/ContainerCommands/Container/ContainerDelete.swift +++ b/Sources/ContainerCommands/Container/ContainerDelete.swift @@ -35,6 +35,9 @@ extension Application { @Flag(name: .shortAndLong, help: "Delete containers even if they are running") var force = false + @Flag(name: .shortAndLong, help: "Remove the anonymous volumes and pod the container was given") + var volumes = false + @OptionGroup public var logOptions: Flags.Logging @@ -56,6 +59,8 @@ extension Application { public mutating func run() async throws { let client = ContainerClient() let force = self.force + let removeAnonymous = self.volumes + let log = self.log let containers: [String] if all { @@ -76,7 +81,13 @@ extension Application { for container in containers { group.addTask { do { + // What the container was given is recorded on the + // container, so it is read before the container goes. + let given = + removeAnonymous + ? await AnonymousResources.given(to: try await client.get(id: container)) : nil try await client.delete(id: container, force: force) + await given?.remove(log: log) print(container) return nil } catch { From f1af4c84c3c7cf0a070d8451eaa8c81ed76008ad Mon Sep 17 00:00:00 2001 From: Aaron Paterson Date: Mon, 10 Aug 2026 05:45:45 +0000 Subject: [PATCH 08/29] Adopt the machines a restarted control plane finds running A machine is a launchd service, so it outlives the process that registered it. A control plane that starts over a running machine held every pod as not ready and every container as stopped: stops returned through their idempotence guards, deletes removed records while processes ran on, and the next placement under a freed name was refused against the place its stopped predecessor never gave back. The kubelet reconciles the same gap by listing what its runtime holds when it starts, and containerd by re-dialing the shims it finds alive. Before serving, each pod whose service still answers launchd is dialed and believed: the machine's own snapshot says what runs. Containers take their machine's word and are adopted running, tracked by the exit monitor the way bootstrap tracked them first. A stopped container's place is given back to the machine when it is cleaned up, and a pod whose machine still answers refuses a new record under its name. https://github.com/kubernetes/cri-api/blob/master/pkg/apis/runtime/v1/api.proto --- Sources/APIServer/APIServer+Start.swift | 6 ++ .../Server/Containers/ContainersService.swift | 76 +++++++++++++++++-- .../Server/Pods/PodsService.swift | 54 +++++++++++++ .../RuntimeLinux/Server/RuntimeService.swift | 10 +++ .../RuntimeLinux/Server/Sandbox.swift | 4 + 5 files changed, 145 insertions(+), 5 deletions(-) diff --git a/Sources/APIServer/APIServer+Start.swift b/Sources/APIServer/APIServer+Start.swift index f54c0517b..fd7c2ae9a 100644 --- a/Sources/APIServer/APIServer+Start.swift +++ b/Sources/APIServer/APIServer+Start.swift @@ -87,6 +87,12 @@ extension APIServer { await podsService.setContainersService(containersService) await podsService.setNetworksService(networkService) await containersService.setPodsService(podsService) + + // Machines outlive the process that made them, so before + // serving, adopt the ones still running: pods dial their + // launchd services, containers take their machine's word. + await podsService.reconnect() + await containersService.reconnect() initializeHealthCheckService(log: log, routes: &routes) try initializeKernelService(log: log, routes: &routes) let volumesService = try await initializeVolumeService(containersService: containersService, log: log, routes: &routes) diff --git a/Sources/Services/ContainerAPIService/Server/Containers/ContainersService.swift b/Sources/Services/ContainerAPIService/Server/Containers/ContainersService.swift index 259d63645..ba6f28110 100644 --- a/Sources/Services/ContainerAPIService/Server/Containers/ContainersService.swift +++ b/Sources/Services/ContainerAPIService/Server/Containers/ContainersService.swift @@ -152,11 +152,17 @@ public actor ContainersService { ), ) results[config.id] = state - guard runtimePlugins.first(where: { $0.name == config.runtimeHandler }) != nil else { - throw ContainerizationError( - .internalError, - message: "failed to find runtime plugin \(config.runtimeHandler)" - ) + // A missing plugin says nothing about the container: the + // loader's answer varies with how this process was spawned, + // and a bundle outlives any one spawn. Removal is reserved + // for a bundle that cannot be read at all. + if runtimePlugins.first(where: { $0.name == config.runtimeHandler }) == nil { + log.warning( + "no runtime plugin for container", + metadata: [ + "id": "\(config.id)", + "runtime": "\(config.runtimeHandler)", + ]) } } catch { try? FileManager.default.removeItem(at: dir) @@ -577,6 +583,66 @@ public actor ContainersService { } } + /// Adopt the containers the reconnected machines report. + /// + /// The machines a restarted control plane finds alive were dialed by the + /// pods service; each reports the containers it holds and their state. + /// A container the machine says is running is adopted as running: its + /// client is the pod's, addressed to it, and the exit monitor tracks it + /// again the way bootstrap tracked it first, so its exit is handled by + /// whoever is serving when it comes. + public func reconnect() async { + guard let podsService = self.podsService else { + return + } + await self.lock.withLock(logMetadata: ["acquirer": "\(#function)"]) { context in + for (id, var state) in await self.containers { + guard state.client == nil else { + continue + } + let pod = state.snapshot.configuration.pod + guard let podClient = try? await podsService.client(for: pod) else { + continue + } + let client = podClient.addressing(id) + guard let sandbox = try? await client.state(), + let reported = sandbox.containers.first(where: { $0.id == id }), + reported.status == .running + else { + continue + } + do { + let log = self.log + try await self.exitMonitor.registerProcess( + id: id, + onExit: self.handleContainerExit + ) + let waitFunc: ExitMonitor.WaitHandler = { + let code = try await client.wait(id) + log.info( + "container finished in exit monitor", + metadata: [ + "id": "\(id)", + "rc": "\(code)", + ]) + return code + } + try await self.exitMonitor.track(id: id, waitingOn: waitFunc) + state.client = client + state.snapshot.status = .running + state.snapshot.networks = sandbox.networks + state.snapshot.startedDate = reported.startedDate + await self.setContainerState(id, state, context: context) + self.log.info("adopted a running container", metadata: ["id": "\(id)", "pod": "\(pod)"]) + } catch { + self.log.warning( + "failed to adopt a running container", + metadata: ["id": "\(id)", "error": "\(error)"]) + } + } + } + } + /// Send a signal to the container. public func kill(id: String, processID: String, signal: String) async throws { log.debug( diff --git a/Sources/Services/ContainerAPIService/Server/Pods/PodsService.swift b/Sources/Services/ContainerAPIService/Server/Pods/PodsService.swift index 465976870..6af23cf6a 100644 --- a/Sources/Services/ContainerAPIService/Server/Pods/PodsService.swift +++ b/Sources/Services/ContainerAPIService/Server/Pods/PodsService.swift @@ -147,6 +147,60 @@ public actor PodsService { return pods } + /// Adopt the machines that outlived the process that made them. + /// + /// A pod's machine is a launchd service, so it survives the control + /// plane that registered it. A restarted control plane would otherwise + /// hold every pod as not ready and answer stops and deletes against + /// machines it cannot reach; the kubelet reconciles the same gap by + /// listing what its runtime actually holds when it starts, and + /// containerd by re-dialing the shims it finds alive. + /// https://github.com/kubernetes/cri-api/blob/master/pkg/apis/runtime/v1/api.proto + /// + /// Each pod whose service still answers is dialed and believed: the + /// machine's own snapshot says whether it runs. A pod whose service is + /// gone stays written down and not ready, which is what it was before + /// its machine first booted. + public func reconnect() async { + await self.lock.withLock(logMetadata: ["acquirer": "\(#function)"]) { context in + let pods = await self.pods + self.log.info("looking for machines to adopt", metadata: ["pods": "\(pods.count)"]) + for (id, var state) in pods { + guard state.client == nil else { + continue + } + let runtime = state.configuration.runtimeHandler + // launchctl answers for the service's bare label: not the + // domain-prefixed form bootout takes, and not the mach name + // the client dials, which carries the runtime prefix. + let label = "\(Self.machServicePrefix).\(runtime).\(id)" + guard (try? ServiceManager.isRegistered(fullServiceLabel: label)) == true else { + self.log.info("no service answers for the pod", metadata: ["pod": "\(id)"]) + continue + } + do { + let client = try await RuntimeClient.create(id: id, runtime: runtime) + let sandbox = try await client.state() + guard sandbox.status == .running else { + self.log.info( + "a pod's machine answered but is not running", + metadata: ["pod": "\(id)", "status": "\(sandbox.status)"]) + continue + } + state.client = client + state.state = .ready + state.startedDate = sandbox.containers.compactMap { $0.startedDate }.min() + await self.setPodState(id, state, context: context) + self.log.info("adopted a running machine", metadata: ["pod": "\(id)"]) + } catch { + self.log.warning( + "a pod's service answered launchd but not the runtime", + metadata: ["pod": "\(id)", "error": "\(error)"]) + } + } + } + } + /// Where a pod keeps what it is made of. public func path(for id: String) -> URL { self.podRoot.appendingPathComponent(id) diff --git a/Sources/Services/RuntimeLinux/Server/RuntimeService.swift b/Sources/Services/RuntimeLinux/Server/RuntimeService.swift index e059e07ac..9d9e9cdf7 100644 --- a/Sources/Services/RuntimeLinux/Server/RuntimeService.swift +++ b/Sources/Services/RuntimeLinux/Server/RuntimeService.swift @@ -1640,6 +1640,16 @@ public actor RuntimeService { self.log.error("failed to stop container during cleanup", metadata: ["error": "\(error)"]) } + // The machine keeps a stopped container's place until it is given + // back. The registry below forgets the name, so the machine must give + // it up too, or the next placement under it is refused against a + // place nothing holds. + do { + try await self.getSandbox().removeContainer(id) + } catch { + self.log.error("failed to remove container during cleanup", metadata: ["error": "\(error)"]) + } + self.containers.removeValue(forKey: id) self.processes.removeValue(forKey: id) await self.monitor.stopTracking(id: id) diff --git a/Sources/Services/RuntimeLinux/Server/Sandbox.swift b/Sources/Services/RuntimeLinux/Server/Sandbox.swift index 5266e7386..6b03283d7 100644 --- a/Sources/Services/RuntimeLinux/Server/Sandbox.swift +++ b/Sources/Services/RuntimeLinux/Server/Sandbox.swift @@ -37,6 +37,10 @@ protocol Sandbox: Sendable { /// Stop a container, leaving the machine running for the others. func stopContainer(_ id: String) async throws + /// Take a stopped container out of the machine, so its name is free to + /// place again. + func removeContainer(_ id: String) async throws + /// Signal a container's init process. func killContainer(_ id: String, signal: Signal) async throws From fc3fa75291dcf503d97af8b6f4bd0ad6b6b3ef65 Mon Sep 17 00:00:00 2001 From: Aaron Paterson Date: Mon, 10 Aug 2026 03:25:25 +0000 Subject: [PATCH 09/29] Ask the containers service, not its lock, for who references what A volume or network deciding a delete ran its check inside the containers lock through withContainerList, coupling the services at the lock. The decision each caller needs is a named question: the volumes containers mount, the containers holding a volume, the containers attached to a network. Each is answered inside the containers lock and returned, so the answer is as strong as the closure was for reading. What moves out of the lock is the act that followed the answer: a delete now runs after the query, accepting the window a container create can race into, which is the window image delete already accepts; the create that loses names the missing volume or network in its error. --- .../Server/Containers/ContainersService.swift | 38 ++++++++++ .../Server/Networks/NetworksService.swift | 75 ++++++++----------- .../Server/Volumes/VolumesService.swift | 57 ++++++-------- 3 files changed, 92 insertions(+), 78 deletions(-) diff --git a/Sources/Services/ContainerAPIService/Server/Containers/ContainersService.swift b/Sources/Services/ContainerAPIService/Server/Containers/ContainersService.swift index ba6f28110..e615c067d 100644 --- a/Sources/Services/ContainerAPIService/Server/Containers/ContainersService.swift +++ b/Sources/Services/ContainerAPIService/Server/Containers/ContainersService.swift @@ -243,6 +243,44 @@ public actor ContainersService { } } + /// The name of every volume a container mounts, gathered inside the + /// containers lock so no container is created into the answer. + public func volumeNamesInUse() async throws -> Set { + try await withContainerList(logMetadata: ["acquirer": "\(#function)"]) { containers in + var names = Set() + for container in containers { + for mount in container.configuration.mounts { + if mount.isVolume, let volumeName = mount.volumeName { + names.insert(volumeName) + } + } + } + return names + } + } + + /// The containers that mount the named volume, gathered inside the + /// containers lock. An empty answer says the volume was free when asked, + /// which is the strongest claim one resource can make about another from + /// outside the other's lock. + public func containersReferencingVolume(_ name: String) async throws -> [String] { + try await withContainerList(logMetadata: ["acquirer": "\(#function)", "name": "\(name)"]) { containers in + containers.filter { container in + container.configuration.mounts.contains { $0.isVolume && $0.volumeName == name } + }.map { $0.configuration.id } + } + } + + /// The containers attached to the named network, gathered inside the + /// containers lock. + public func containersAttachedToNetwork(_ id: String) async throws -> [String] { + try await withContainerList(logMetadata: ["acquirer": "\(#function)", "id": "\(id)"]) { containers in + containers.filter { container in + container.configuration.networks.contains { $0.network == id } + }.map { $0.configuration.id } + } + } + /// Calculate disk usage for containers /// - Returns: Tuple of (total count, active count, total size, reclaimable size) public func calculateDiskUsage() async -> (Int, Int, UInt64, UInt64) { diff --git a/Sources/Services/ContainerAPIService/Server/Networks/NetworksService.swift b/Sources/Services/ContainerAPIService/Server/Networks/NetworksService.swift index 7fe35fae8..ecf0e308d 100644 --- a/Sources/Services/ContainerAPIService/Server/Networks/NetworksService.swift +++ b/Sources/Services/ContainerAPIService/Server/Networks/NetworksService.swift @@ -248,50 +248,41 @@ public actor NetworksService { throw ContainerizationError(.invalidArgument, message: "cannot delete builtin network: \(id)") } - // prevent container operations while we atomically check and delete - try await self.containersService.withContainerList(logMetadata: ["acquirer": "\(#function)", "id": "\(id)"]) { containers in - // find all containers that refer to the network - var referringContainers = Set() - for container in containers { - for attachmentConfiguration in container.configuration.networks { - if attachmentConfiguration.network == id { - referringContainers.insert(container.configuration.id) - break - } - } - } - - // bail if any referring containers - guard referringContainers.isEmpty else { - throw ContainerizationError( - .invalidState, - message: "cannot delete subnet \(id) with referring containers: \(referringContainers.joined(separator: ", "))" - ) - } + // A container created after this answer can attach to the network + // while it is deleted, the same window image delete accepts against + // container create; the attach then fails naming the missing network. + let referringContainers = try await self.containersService.containersAttachedToNetwork(id) + + // bail if any referring containers + guard referringContainers.isEmpty else { + throw ContainerizationError( + .invalidState, + message: "cannot delete subnet \(id) with referring containers: \(referringContainers.joined(separator: ", "))" + ) + } - // start network deletion, this is the last place we'll want to throw - do { - try await self.deregisterService(configuration: serviceState.configuration) - } catch { - self.log.error( - "failed to deregister network service", - metadata: [ - "id": "\(id)", - "error": "\(error.localizedDescription)", - ]) - } + // start network deletion, this is the last place we'll want to throw + do { + try await self.deregisterService(configuration: serviceState.configuration) + } catch { + self.log.error( + "failed to deregister network service", + metadata: [ + "id": "\(id)", + "error": "\(error.localizedDescription)", + ]) + } - // deletion is underway, do not throw anything now - do { - try await self.store.delete(id) - } catch { - self.log.error( - "failed to delete network from configuration store", - metadata: [ - "id": "\(id)", - "error": "\(error.localizedDescription)", - ]) - } + // deletion is underway, do not throw anything now + do { + try await self.store.delete(id) + } catch { + self.log.error( + "failed to delete network from configuration store", + metadata: [ + "id": "\(id)", + "error": "\(error.localizedDescription)", + ]) } // having deleted successfully, remove the runtime state diff --git a/Sources/Services/ContainerAPIService/Server/Volumes/VolumesService.swift b/Sources/Services/ContainerAPIService/Server/Volumes/VolumesService.swift index fe607e762..297122f87 100644 --- a/Sources/Services/ContainerAPIService/Server/Volumes/VolumesService.swift +++ b/Sources/Services/ContainerAPIService/Server/Volumes/VolumesService.swift @@ -198,35 +198,23 @@ public actor VolumesService { return try await lock.withLock { _ in let allVolumes = try await self.store.list() - // Atomically get active volumes with container list - return try await self.containersService.withContainerList(logMetadata: ["acquirer": "\(#function)"]) { containers in - var inUseSet = Set() - - // Find all mounted volumes - for container in containers { - for mount in container.configuration.mounts { - if mount.isVolume, let volumeName = mount.volumeName { - inUseSet.insert(volumeName) - } - } - } + let inUseSet = try await self.containersService.volumeNamesInUse() - var totalSize: UInt64 = 0 - var reclaimableSize: UInt64 = 0 + var totalSize: UInt64 = 0 + var reclaimableSize: UInt64 = 0 - // Calculate sizes - for volume in allVolumes { - let volumePath = self.volumePath(for: volume.name) - let volumeSize = FileManager.default.allocatedSize(of: URL(fileURLWithPath: volumePath)) - totalSize += volumeSize + // Calculate sizes + for volume in allVolumes { + let volumePath = self.volumePath(for: volume.name) + let volumeSize = FileManager.default.allocatedSize(of: URL(fileURLWithPath: volumePath)) + totalSize += volumeSize - if !inUseSet.contains(volume.name) { - reclaimableSize += volumeSize - } + if !inUseSet.contains(volume.name) { + reclaimableSize += volumeSize } - - return (allVolumes.count, inUseSet.count, totalSize, reclaimableSize) } + + return (allVolumes.count, inUseSet.count, totalSize, reclaimableSize) } } @@ -372,20 +360,17 @@ public actor VolumesService { throw VolumeError.volumeNotFound(name) } - // Check if volume is in use by any container atomically - try await containersService.withContainerList(logMetadata: ["acquirer": "\(#function)", "name": "\(name)"]) { containers in - for container in containers { - for mount in container.configuration.mounts { - if mount.isVolume && mount.volumeName == name { - throw VolumeError.volumeInUse(name) - } - } - } - - try await self.store.delete(name) - try self.removeVolumeDirectory(for: name) + // A container created after this answer can name the volume and lose + // it, the same window image delete accepts against container create; + // the create then fails naming the missing volume. + let referencing = try await containersService.containersReferencingVolume(name) + guard referencing.isEmpty else { + throw VolumeError.volumeInUse(name) } + try await self.store.delete(name) + try self.removeVolumeDirectory(for: name) + log.info("deleted volume", metadata: ["name": "\(name)"]) } From c137838464bb420e45ca506d1cbf6e9c30115b1a Mon Sep 17 00:00:00 2001 From: Aaron Paterson Date: Mon, 10 Aug 2026 04:03:03 +0000 Subject: [PATCH 10/29] Give containers, pods, and networks to a core plugin The sandbox domain is one consistency domain: a container bootstraps by asking its pod to run holding it, a pod forced away takes its containers with it, and both claim addresses from the networks they attach. The container-core-containers plugin holds the three services in one process, where those calls keep their lock conventions, the shape CoreImages already gave one plugin holding images and content. The API server keeps what faces the host: health, kernels, volumes, disk usage aggregation, and the DNS server, which resolves container hostnames through the same networkLookup route any client may use. Volumes and disk usage reach containers through named atomic queries over XPC, each decided inside the containers lock and returned, the arrangement every service already used to reach images. Clients of the moved resources dial the plugin's mach service. The routes and their harnesses move unchanged; the plugin boots them the way the API server did, from the same configuration, behind one XPCServer. --- Makefile | 6 + Package.swift | 18 ++ Sources/APIServer/APIServer+Start.swift | 155 +-------- Sources/APIServer/ContainerDNSHandler.swift | 11 +- .../CoreContainers/ContainersHelper.swift | 296 ++++++++++++++++++ Sources/Plugins/CoreContainers/config.toml | 12 + .../Client/ClientPod.swift | 2 +- .../Client/ClientProcess.swift | 2 +- .../Client/ContainerClient.swift | 104 +++++- .../Client/NetworkClient.swift | 29 +- .../ContainerAPIService/Client/XPC+.swift | 14 + .../Server/Containers/ContainersHarness.swift | 56 ++++ .../Server/Containers/ContainersService.swift | 10 +- .../Server/DiskUsage/DiskUsageService.swift | 26 +- .../Server/Networks/NetworksHarness.swift | 12 + .../Server/Networks/NetworksService.swift | 6 +- .../Server/Volumes/VolumesService.swift | 19 +- .../DiskUsagePathTests.swift | 1 - 18 files changed, 582 insertions(+), 197 deletions(-) create mode 100644 Sources/Plugins/CoreContainers/ContainersHelper.swift create mode 100644 Sources/Plugins/CoreContainers/config.toml diff --git a/Makefile b/Makefile index 1855256d2..5c2aff350 100644 --- a/Makefile +++ b/Makefile @@ -131,6 +131,7 @@ $(STAGING_DIR): @mkdir -p "$(join $(STAGING_DIR), libexec/container/plugins/container-runtime-linux/bin)" @mkdir -p "$(join $(STAGING_DIR), libexec/container/plugins/container-network-vmnet/bin)" @mkdir -p "$(join $(STAGING_DIR), libexec/container/plugins/container-core-images/bin)" + @mkdir -p "$(join $(STAGING_DIR), libexec/container/plugins/container-core-containers/bin)" @mkdir -p "$(join $(STAGING_DIR), libexec/container/plugins/machine-apiserver/bin)" @mkdir -p "$(join $(STAGING_DIR), libexec/container/plugins/machine-apiserver/resources)" @mkdir -p "$(join $(STAGING_DIR), libexec/container/plugins/k8s/bin)" @@ -144,6 +145,8 @@ $(STAGING_DIR): @install Sources/Plugins/NetworkVmnet/config.toml "$(join $(STAGING_DIR), libexec/container/plugins/container-network-vmnet/config.toml)" @install "$(BUILD_BIN_DIR)/container-core-images" "$(join $(STAGING_DIR), libexec/container/plugins/container-core-images/bin/container-core-images)" @install Sources/Plugins/CoreImages/config.toml "$(join $(STAGING_DIR), libexec/container/plugins/container-core-images/config.toml)" + @install "$(BUILD_BIN_DIR)/container-core-containers" "$(join $(STAGING_DIR), libexec/container/plugins/container-core-containers/bin/container-core-containers)" + @install Sources/Plugins/CoreContainers/config.toml "$(join $(STAGING_DIR), libexec/container/plugins/container-core-containers/config.toml)" @install "$(BUILD_BIN_DIR)/machine-apiserver" "$(join $(STAGING_DIR), libexec/container/plugins/machine-apiserver/bin/machine-apiserver)" @install Sources/Plugins/MachineAPIServer/config.toml "$(join $(STAGING_DIR), libexec/container/plugins/machine-apiserver/config.toml)" @install Sources/Plugins/MachineAPIServer/Resources/init "$(join $(STAGING_DIR), libexec/container/plugins/machine-apiserver/resources/init)" @@ -163,6 +166,7 @@ installer-pkg: $(STAGING_DIR) @codesign $(CODESIGN_OPTS) --identifier com.apple.container.cli "$(join $(STAGING_DIR), bin/container)" @codesign $(CODESIGN_OPTS) --identifier com.apple.container.apiserver "$(join $(STAGING_DIR), bin/container-apiserver)" @codesign $(CODESIGN_OPTS) --prefix=com.apple.container. "$(join $(STAGING_DIR), libexec/container/plugins/container-core-images/bin/container-core-images)" + @codesign $(CODESIGN_OPTS) --prefix=com.apple.container. "$(join $(STAGING_DIR), libexec/container/plugins/container-core-containers/bin/container-core-containers)" @codesign $(CODESIGN_OPTS) --prefix=com.apple.container. --entitlements=signing/container-runtime-linux.entitlements "$(join $(STAGING_DIR), libexec/container/plugins/container-runtime-linux/bin/container-runtime-linux)" @codesign $(CODESIGN_OPTS) --prefix=com.apple.container. --entitlements=signing/container-network-vmnet.entitlements "$(join $(STAGING_DIR), libexec/container/plugins/container-network-vmnet/bin/container-network-vmnet)" @codesign $(CODESIGN_OPTS) --prefix=com.apple.container. "$(join $(STAGING_DIR), libexec/container/plugins/machine-apiserver/bin/machine-apiserver)" @@ -180,6 +184,7 @@ dsym: @cp -a "$(BUILD_BIN_DIR)/container-runtime-linux.dSYM" "$(DSYM_DIR)" @cp -a "$(BUILD_BIN_DIR)/container-network-vmnet.dSYM" "$(DSYM_DIR)" @cp -a "$(BUILD_BIN_DIR)/container-core-images.dSYM" "$(DSYM_DIR)" + @cp -a "$(BUILD_BIN_DIR)/container-core-containers.dSYM" "$(DSYM_DIR)" @cp -a "$(BUILD_BIN_DIR)/container-apiserver.dSYM" "$(DSYM_DIR)" @cp -a "$(BUILD_BIN_DIR)/container.dSYM" "$(DSYM_DIR)" @@ -212,6 +217,7 @@ COV_BINARIES := \ $(BUILD_BIN_DIR)/container-runtime-linux \ $(BUILD_BIN_DIR)/container-network-vmnet \ $(BUILD_BIN_DIR)/container-core-images \ + $(BUILD_BIN_DIR)/container-core-containers \ $(BUILD_BIN_DIR)/machine-apiserver COV_OBJECT_FLAGS := $(patsubst %,-object %,$(COV_BINARIES)) # Set of files we do not want to get caught in the coverage generation diff --git a/Package.swift b/Package.swift index d53fd9c0e..543664878 100644 --- a/Package.swift +++ b/Package.swift @@ -312,6 +312,24 @@ let package = Package( path: "Sources/Plugins/CoreImages", exclude: ["config.toml"] ), + .executableTarget( + name: "container-core-containers", + dependencies: [ + .product(name: "ArgumentParser", package: "swift-argument-parser"), + .product(name: "Logging", package: "swift-log"), + .product(name: "Containerization", package: "containerization"), + .product(name: "SystemPackage", package: "swift-system"), + "ContainerAPIClient", + "ContainerAPIService", + "ContainerLog", + "ContainerPersistence", + "ContainerPlugin", + "ContainerVersion", + "ContainerXPC", + ], + path: "Sources/Plugins/CoreContainers", + exclude: ["config.toml"] + ), .target( name: "ContainerImagesService", dependencies: [ diff --git a/Sources/APIServer/APIServer+Start.swift b/Sources/APIServer/APIServer+Start.swift index fd7c2ae9a..04aa23e8c 100644 --- a/Sources/APIServer/APIServer+Start.swift +++ b/Sources/APIServer/APIServer+Start.swift @@ -49,7 +49,6 @@ extension APIServer { var logRoot = LogRoot.path func run() async throws { - let containerSystemConfig: ContainerSystemConfig = try await ConfigurationLoader.load() let commandName = APIServer._commandName let logPath = logRoot.map { $0.appending(FilePath.Component("\(commandName).log") ?? "unknown") } let log = ServiceLogger.bootstrap(category: "APIServer", debug: debug, logPath: logPath) @@ -64,40 +63,10 @@ extension APIServer { let pluginLoader = try initializePluginLoader(log: log) try await initializePlugins(pluginLoader: pluginLoader, log: log, routes: &routes, debug: debug) - let containersService = try initializeContainersService( - pluginLoader: pluginLoader, - containerSystemConfig: containerSystemConfig, - log: log, - routes: &routes - ) - let networkService = try await initializeNetworksService( - pluginLoader: pluginLoader, - containersService: containersService, - containerSystemConfig: containerSystemConfig, - log: log, - routes: &routes - ) - await containersService.setNetworksService(networkService) - let podsService = try initializePodsService( - pluginLoader: pluginLoader, - containerSystemConfig: containerSystemConfig, - log: log, - routes: &routes - ) - await podsService.setContainersService(containersService) - await podsService.setNetworksService(networkService) - await containersService.setPodsService(podsService) - - // Machines outlive the process that made them, so before - // serving, adopt the ones still running: pods dial their - // launchd services, containers take their machine's word. - await podsService.reconnect() - await containersService.reconnect() initializeHealthCheckService(log: log, routes: &routes) try initializeKernelService(log: log, routes: &routes) - let volumesService = try await initializeVolumeService(containersService: containersService, log: log, routes: &routes) + let volumesService = try await initializeVolumeService(log: log, routes: &routes) try initializeDiskUsageService( - containersService: containersService, volumesService: volumesService, log: log, routes: &routes @@ -124,7 +93,7 @@ extension APIServer { // start up host table DNS group.addTask { - let hostsResolver = ContainerDNSHandler(networkService: networkService) + let hostsResolver = ContainerDNSHandler(networks: NetworkClient()) let nxDomainResolver = NxDomainResolver() let compositeResolver = CompositeResolver(handlers: [hostsResolver, nxDomainResolver]) let hostsQueryValidator = StandardQueryValidator(handler: compositeResolver) @@ -286,130 +255,14 @@ extension APIServer { routes[XPCRoute.getDefaultKernel] = XPCServer.route(harness.getDefaultKernel) } - private func initializePodsService( - pluginLoader: PluginLoader, - containerSystemConfig: ContainerSystemConfig, - log: Logger, - routes: inout [XPCRoute: XPCServer.RouteHandler] - ) throws -> PodsService { - log.info("initializing pods service") - - let appRootURL = URL(fileURLWithPath: appRoot.string) - let service = try PodsService( - appRoot: appRootURL, - pluginLoader: pluginLoader, - containerSystemConfig: containerSystemConfig, - debugHelpers: debug, - log: log - ) - let harness = PodsHarness(service: service, log: log) - - routes[XPCRoute.podCreate] = XPCServer.route(harness.create) - routes[XPCRoute.podStart] = XPCServer.route(harness.start) - routes[XPCRoute.podStop] = XPCServer.route(harness.stop) - routes[XPCRoute.podDelete] = XPCServer.route(harness.delete) - routes[XPCRoute.podInspect] = XPCServer.route(harness.inspect) - routes[XPCRoute.podList] = XPCServer.route(harness.list) - routes[XPCRoute.podUpdate] = XPCServer.route(harness.update) - - return service - } - - private func initializeContainersService( - pluginLoader: PluginLoader, - containerSystemConfig: ContainerSystemConfig, - log: Logger, - routes: inout [XPCRoute: XPCServer.RouteHandler] - ) throws -> ContainersService { - log.info("initializing containers service") - - // TODO: Remove when we convert ContainersService to FilePath - let appRootURL = URL(fileURLWithPath: appRoot.string) - let service = try ContainersService( - appRoot: appRootURL, - pluginLoader: pluginLoader, - containerSystemConfig: containerSystemConfig, - log: log, - debugHelpers: debug - ) - let harness = ContainersHarness(service: service, log: log) - - routes[XPCRoute.containerList] = XPCServer.route(harness.list) - routes[XPCRoute.containerCreate] = XPCServer.route(harness.create) - routes[XPCRoute.containerDelete] = XPCServer.route(harness.delete) - routes[XPCRoute.containerLogs] = XPCServer.route(harness.logs) - routes[XPCRoute.containerBootstrap] = XPCServer.route(harness.bootstrap) - routes[XPCRoute.containerDial] = XPCServer.route(harness.dial) - routes[XPCRoute.containerStop] = XPCServer.route(harness.stop) - routes[XPCRoute.containerStartProcess] = XPCServer.route(harness.startProcess) - routes[XPCRoute.containerCreateProcess] = XPCServer.route(harness.createProcess) - routes[XPCRoute.containerResize] = XPCServer.route(harness.resize) - routes[XPCRoute.containerWait] = XPCServer.route(harness.wait) - routes[XPCRoute.containerKill] = XPCServer.route(harness.kill) - routes[XPCRoute.containerStats] = XPCServer.route(harness.stats) - routes[XPCRoute.containerDiskUsage] = XPCServer.route(harness.diskUsage) - routes[XPCRoute.containerCopyIn] = XPCServer.route(harness.copyIn) - routes[XPCRoute.containerCopyOut] = XPCServer.route(harness.copyOut) - routes[XPCRoute.containerExport] = XPCServer.route(harness.export) - - return service - } - - private func initializeNetworksService( - pluginLoader: PluginLoader, - containersService: ContainersService, - containerSystemConfig: ContainerSystemConfig, - log: Logger, - routes: inout [XPCRoute: XPCServer.RouteHandler] - ) async throws -> NetworksService { - log.info("initializing networks service") - - let resourceRoot = appRoot.appending(FilePath.Component("networks")) - let defaultNetworkConfig = try NetworkConfiguration( - name: NetworkClient.defaultNetworkName, - mode: .nat, - ipv4Subnet: containerSystemConfig.network.subnet, - ipv6Subnet: containerSystemConfig.network.subnetv6, - labels: try .init([ResourceLabelKeys.role: ResourceRoleValues.builtin]), - plugin: "container-network-vmnet" - ) - let service = try await NetworksService( - pluginLoader: pluginLoader, - resourceRoot: resourceRoot, - containersService: containersService, - defaultNetworkConfiguration: defaultNetworkConfig, - log: log, - debugHelpers: debug - ) - - let defaultNetwork = try await service.list() - .filter { $0.isBuiltin } - .first - if defaultNetwork == nil { - // FIXME: default network should be configurable elsewhere - _ = try await service.create(configuration: defaultNetworkConfig) - } - - let harness = NetworksHarness(service: service, log: log) - - if #available(macOS 26, *) { - routes[XPCRoute.networkCreate] = XPCServer.route(harness.create) - } - routes[XPCRoute.networkList] = XPCServer.route(harness.list) - routes[XPCRoute.networkDelete] = XPCServer.route(harness.delete) - - return service - } - private func initializeVolumeService( - containersService: ContainersService, log: Logger, routes: inout [XPCRoute: XPCServer.RouteHandler] ) async throws -> VolumesService { log.info("initializing volume service") let resourceRoot = appRoot.appending(FilePath.Component("volumes")) - let service = try await VolumesService(resourceRoot: resourceRoot, containersService: containersService, log: log) + let service = try await VolumesService(resourceRoot: resourceRoot, log: log) let harness = VolumesHarness(service: service, log: log) routes[XPCRoute.volumeCreate] = XPCServer.route(harness.create) @@ -422,7 +275,6 @@ extension APIServer { } private func initializeDiskUsageService( - containersService: ContainersService, volumesService: VolumesService, log: Logger, routes: inout [XPCRoute: XPCServer.RouteHandler] @@ -430,7 +282,6 @@ extension APIServer { log.info("initializing disk usage service") let service = DiskUsageService( - containersService: containersService, volumesService: volumesService, log: log ) diff --git a/Sources/APIServer/ContainerDNSHandler.swift b/Sources/APIServer/ContainerDNSHandler.swift index 78a207467..a7143e8a6 100644 --- a/Sources/APIServer/ContainerDNSHandler.swift +++ b/Sources/APIServer/ContainerDNSHandler.swift @@ -14,17 +14,18 @@ // limitations under the License. //===----------------------------------------------------------------------===// +import ContainerAPIClient import ContainerAPIService import ContainerizationExtras import DNSServer /// Handler that uses table lookup to resolve hostnames. struct ContainerDNSHandler: DNSHandler { - private let networkService: NetworksService + private let networks: NetworkClient private let ttl: UInt32 - public init(networkService: NetworksService, ttl: UInt32 = 5) { - self.networkService = networkService + public init(networks: NetworkClient, ttl: UInt32 = 5) { + self.networks = networks self.ttl = ttl } @@ -76,7 +77,7 @@ struct ContainerDNSHandler: DNSHandler { } private func answerHost(question: Question) async throws -> ResourceRecord? { - guard let ipAllocation = try await networkService.lookup(hostname: question.name) else { + guard let ipAllocation = try await networks.lookup(hostname: question.name) else { return nil } let ipv4 = ipAllocation.ipv4Address.address.description @@ -88,7 +89,7 @@ struct ContainerDNSHandler: DNSHandler { } private func answerHost6(question: Question) async throws -> (record: ResourceRecord?, hostnameExists: Bool) { - guard let ipAllocation = try await networkService.lookup(hostname: question.name) else { + guard let ipAllocation = try await networks.lookup(hostname: question.name) else { return (nil, false) } guard let ipv6Address = ipAllocation.ipv6Address else { diff --git a/Sources/Plugins/CoreContainers/ContainersHelper.swift b/Sources/Plugins/CoreContainers/ContainersHelper.swift new file mode 100644 index 000000000..39aea483c --- /dev/null +++ b/Sources/Plugins/CoreContainers/ContainersHelper.swift @@ -0,0 +1,296 @@ +//===----------------------------------------------------------------------===// +// Copyright © 2026 Apple Inc. and the container project authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//===----------------------------------------------------------------------===// + +import ArgumentParser +import ContainerAPIClient +import ContainerAPIService +import ContainerLog +import ContainerPersistence +import ContainerPlugin +import ContainerResource +import ContainerVersion +import ContainerXPC +import Foundation +import Logging +import SystemPackage + +@main +struct ContainersHelper: AsyncParsableCommand { + static let configuration = CommandConfiguration( + commandName: "container-core-containers", + abstract: "XPC service for managing containers and pods", + version: ReleaseVersion.singleLine(appName: "container-core-containers"), + subcommands: [ + Start.self + ] + ) +} + +extension ContainersHelper { + struct Start: AsyncParsableCommand { + static let configuration = CommandConfiguration( + commandName: "start", + abstract: "Starts the container and pod plugin" + ) + + @Flag(name: .long, help: "Enable debug logging") + var debug = false + + @Option(name: .long, help: "XPC service prefix") + var serviceIdentifier: String = "com.apple.container.core.container-core-containers" + + var appRoot = ApplicationRoot.path + + var installRoot = InstallRoot.path + + var logRoot = LogRoot.path + + func run() async throws { + let containerSystemConfig: ContainerSystemConfig = try await ConfigurationLoader.load() + let commandName = ContainersHelper._commandName + let logPath = logRoot.map { $0.appending("\(commandName).log") } + let log = ServiceLogger.bootstrap(category: "ContainersHelper", debug: debug, logPath: logPath) + log.info("starting helper", metadata: ["name": "\(commandName)"]) + defer { + log.info("stopping helper", metadata: ["name": "\(commandName)"]) + } + + do { + log.info("configuring XPC server") + let pluginLoader = try initializePluginLoader(log: log) + var routes = [String: XPCServer.RouteHandler]() + let containersService = try initializeContainersService( + pluginLoader: pluginLoader, + containerSystemConfig: containerSystemConfig, + log: log, + routes: &routes + ) + let podsService = try initializePodsService( + pluginLoader: pluginLoader, + containerSystemConfig: containerSystemConfig, + log: log, + routes: &routes + ) + + let networksService = try await initializeNetworksService( + pluginLoader: pluginLoader, + containerSystemConfig: containerSystemConfig, + log: log, + routes: &routes + ) + + // The three services address each other directly: a container + // bootstraps by asking its pod to run holding it, a pod forced + // away takes its containers with it, and both claim addresses + // from the networks they attach. They share one process so + // those calls keep their lock conventions. + await podsService.setContainersService(containersService) + await containersService.setPodsService(podsService) + await containersService.setNetworksService(networksService) + await podsService.setNetworksService(networksService) + + // Machines outlive the process that made them, so before + // serving, adopt the ones still running: pods dial their + // launchd services, containers take their machine's word. + await podsService.reconnect() + await containersService.reconnect() + + let xpc = XPCServer( + identifier: serviceIdentifier, + routes: routes, + log: log + ) + log.info("starting XPC server") + try await xpc.listen() + } catch { + log.error( + "helper failed", + metadata: [ + "name": "\(commandName)", + "error": "\(error)", + ]) + ContainersHelper.exit(withError: error) + } + } + + private func initializePluginLoader(log: Logger) throws -> PluginLoader { + log.info( + "initializing plugin loader", + metadata: [ + "installRoot": "\(installRoot.string)" + ]) + + // TODO: Remove when we convert PluginLoader to FilePath + let installRootURL = URL(fileURLWithPath: installRoot.string) + let pluginsURL = PluginLoader.userPluginsDir(installRoot: installRootURL) + log.info("detecting user plugins directory", metadata: ["path": "\(pluginsURL.path(percentEncoded: false))"]) + var directoryExists: ObjCBool = false + _ = FileManager.default.fileExists(atPath: pluginsURL.path, isDirectory: &directoryExists) + let userPluginsURL = directoryExists.boolValue ? pluginsURL : nil + + // plugins built into the application installed as a Unix-like application + let installRootPluginsPath = + installRoot + .appending(FilePath.Component("libexec")) + .appending(FilePath.Component("container")) + .appending(FilePath.Component("plugins")) + let installRootPluginsURL = URL(fileURLWithPath: installRootPluginsPath.string) + + let pluginDirectories = [ + userPluginsURL, + installRootPluginsURL, + ].compactMap { $0 } + + let pluginFactories: [PluginFactory] = [ + DefaultPluginFactory(logger: log), + AppBundlePluginFactory(logger: log), + ] + + for pluginDirectory in pluginDirectories { + log.info("discovered plugin directory", metadata: ["path": "\(pluginDirectory.path(percentEncoded: false))"]) + } + + let appRootURL = URL(fileURLWithPath: appRoot.string) + return try PluginLoader( + appRoot: appRootURL, + installRoot: installRootURL, + logRoot: logRoot, + pluginDirectories: pluginDirectories, + pluginFactories: pluginFactories, + log: log + ) + } + + private func initializeContainersService( + pluginLoader: PluginLoader, + containerSystemConfig: ContainerSystemConfig, + log: Logger, + routes: inout [String: XPCServer.RouteHandler] + ) throws -> ContainersService { + log.info("initializing containers service") + + // TODO: Remove when we convert ContainersService to FilePath + let appRootURL = URL(fileURLWithPath: appRoot.string) + let service = try ContainersService( + appRoot: appRootURL, + pluginLoader: pluginLoader, + containerSystemConfig: containerSystemConfig, + log: log, + debugHelpers: debug + ) + let harness = ContainersHarness(service: service, log: log) + + routes[XPCRoute.containerList.rawValue] = XPCServer.route(harness.list) + routes[XPCRoute.containerCreate.rawValue] = XPCServer.route(harness.create) + routes[XPCRoute.containerDelete.rawValue] = XPCServer.route(harness.delete) + routes[XPCRoute.containerLogs.rawValue] = XPCServer.route(harness.logs) + routes[XPCRoute.containerBootstrap.rawValue] = XPCServer.route(harness.bootstrap) + routes[XPCRoute.containerDial.rawValue] = XPCServer.route(harness.dial) + routes[XPCRoute.containerStop.rawValue] = XPCServer.route(harness.stop) + routes[XPCRoute.containerStartProcess.rawValue] = XPCServer.route(harness.startProcess) + routes[XPCRoute.containerCreateProcess.rawValue] = XPCServer.route(harness.createProcess) + routes[XPCRoute.containerResize.rawValue] = XPCServer.route(harness.resize) + routes[XPCRoute.containerWait.rawValue] = XPCServer.route(harness.wait) + routes[XPCRoute.containerKill.rawValue] = XPCServer.route(harness.kill) + routes[XPCRoute.containerStats.rawValue] = XPCServer.route(harness.stats) + routes[XPCRoute.containerDiskUsage.rawValue] = XPCServer.route(harness.diskUsage) + routes[XPCRoute.containerCopyIn.rawValue] = XPCServer.route(harness.copyIn) + routes[XPCRoute.containerCopyOut.rawValue] = XPCServer.route(harness.copyOut) + routes[XPCRoute.containerExport.rawValue] = XPCServer.route(harness.export) + routes[XPCRoute.containerVolumesInUse.rawValue] = XPCServer.route(harness.volumeNamesInUse) + routes[XPCRoute.containerVolumeReferences.rawValue] = XPCServer.route(harness.volumeReferences) + routes[XPCRoute.containerNetworkReferences.rawValue] = XPCServer.route(harness.networkReferences) + routes[XPCRoute.containerImageReferences.rawValue] = XPCServer.route(harness.imageReferences) + routes[XPCRoute.containerUsageTotals.rawValue] = XPCServer.route(harness.usageTotals) + + return service + } + + private func initializePodsService( + pluginLoader: PluginLoader, + containerSystemConfig: ContainerSystemConfig, + log: Logger, + routes: inout [String: XPCServer.RouteHandler] + ) throws -> PodsService { + log.info("initializing pods service") + + let appRootURL = URL(fileURLWithPath: appRoot.string) + let service = try PodsService( + appRoot: appRootURL, + pluginLoader: pluginLoader, + containerSystemConfig: containerSystemConfig, + debugHelpers: debug, + log: log + ) + let harness = PodsHarness(service: service, log: log) + + routes[XPCRoute.podCreate.rawValue] = XPCServer.route(harness.create) + routes[XPCRoute.podStart.rawValue] = XPCServer.route(harness.start) + routes[XPCRoute.podStop.rawValue] = XPCServer.route(harness.stop) + routes[XPCRoute.podDelete.rawValue] = XPCServer.route(harness.delete) + routes[XPCRoute.podInspect.rawValue] = XPCServer.route(harness.inspect) + routes[XPCRoute.podList.rawValue] = XPCServer.route(harness.list) + routes[XPCRoute.podUpdate.rawValue] = XPCServer.route(harness.update) + + return service + } + + private func initializeNetworksService( + pluginLoader: PluginLoader, + containerSystemConfig: ContainerSystemConfig, + log: Logger, + routes: inout [String: XPCServer.RouteHandler] + ) async throws -> NetworksService { + log.info("initializing networks service") + + let resourceRoot = appRoot.appending(FilePath.Component("networks")) + let defaultNetworkConfig = try NetworkConfiguration( + name: NetworkClient.defaultNetworkName, + mode: .nat, + ipv4Subnet: containerSystemConfig.network.subnet, + ipv6Subnet: containerSystemConfig.network.subnetv6, + labels: try .init([ResourceLabelKeys.role: ResourceRoleValues.builtin]), + plugin: "container-network-vmnet" + ) + let service = try await NetworksService( + pluginLoader: pluginLoader, + resourceRoot: resourceRoot, + defaultNetworkConfiguration: defaultNetworkConfig, + log: log, + debugHelpers: debug + ) + + let defaultNetwork = try await service.list() + .filter { $0.isBuiltin } + .first + if defaultNetwork == nil { + // FIXME: default network should be configurable elsewhere + _ = try await service.create(configuration: defaultNetworkConfig) + } + + let harness = NetworksHarness(service: service, log: log) + + if #available(macOS 26, *) { + routes[XPCRoute.networkCreate.rawValue] = XPCServer.route(harness.create) + } + routes[XPCRoute.networkList.rawValue] = XPCServer.route(harness.list) + routes[XPCRoute.networkDelete.rawValue] = XPCServer.route(harness.delete) + routes[XPCRoute.networkLookup.rawValue] = XPCServer.route(harness.lookup) + + return service + } + } +} diff --git a/Sources/Plugins/CoreContainers/config.toml b/Sources/Plugins/CoreContainers/config.toml new file mode 100644 index 000000000..1c2a6b3ea --- /dev/null +++ b/Sources/Plugins/CoreContainers/config.toml @@ -0,0 +1,12 @@ +abstract = "Core container and pod management plugin" +author = "Apple" +version = 0.1 + +[servicesConfig] +loadAtBoot = true +runAtLoad = false +defaultArguments = [] + +[[servicesConfig.services]] +type = "core" +description = "Provide an XPC interface to manage containers and the pods that machine them." diff --git a/Sources/Services/ContainerAPIService/Client/ClientPod.swift b/Sources/Services/ContainerAPIService/Client/ClientPod.swift index 6a609aff9..f9d4b9901 100644 --- a/Sources/Services/ContainerAPIService/Client/ClientPod.swift +++ b/Sources/Services/ContainerAPIService/Client/ClientPod.swift @@ -22,7 +22,7 @@ import Foundation /// Pods: machines that several containers run inside and share. public struct ClientPod { - static let serviceIdentifier = "com.apple.container.apiserver" + static let serviceIdentifier = "com.apple.container.core.container-core-containers" /// Write down a pod, so containers can be placed in it before it boots. public static func create( diff --git a/Sources/Services/ContainerAPIService/Client/ClientProcess.swift b/Sources/Services/ContainerAPIService/Client/ClientProcess.swift index 5a5f8543e..07d133625 100644 --- a/Sources/Services/ContainerAPIService/Client/ClientProcess.swift +++ b/Sources/Services/ContainerAPIService/Client/ClientProcess.swift @@ -43,7 +43,7 @@ public protocol ClientProcess: Sendable { } struct ClientProcessImpl: ClientProcess, Sendable { - static let serviceIdentifier = "com.apple.container.apiserver" + static let serviceIdentifier = "com.apple.container.core.container-core-containers" /// ID of the process. public var id: String { diff --git a/Sources/Services/ContainerAPIService/Client/ContainerClient.swift b/Sources/Services/ContainerAPIService/Client/ContainerClient.swift index 5a2b6d0d3..c12fdf261 100644 --- a/Sources/Services/ContainerAPIService/Client/ContainerClient.swift +++ b/Sources/Services/ContainerAPIService/Client/ContainerClient.swift @@ -27,7 +27,7 @@ import Foundation /// container lifecycle operations. All methods that operate on a specific /// container take an `id` parameter. public struct ContainerClient: Sendable { - private static let serviceIdentifier = "com.apple.container.apiserver" + private static let serviceIdentifier = "com.apple.container.core.container-core-containers" private let xpcClient: XPCClient @@ -82,6 +82,108 @@ public struct ContainerClient: Sendable { } } + /// The name of every volume a container mounts. + public func volumeNamesInUse() async throws -> Set { + do { + let request = XPCMessage(route: .containerVolumesInUse) + let response = try await xpcSend(message: request) + guard let data = response.dataNoCopy(key: .references) else { + return [] + } + return try JSONDecoder().decode(Set.self, from: data) + } catch let error as ContainerizationError { + throw error + } catch { + throw ContainerizationError( + .internalError, + message: "failed to list volumes in use", + cause: error + ) + } + } + + /// The containers that mount the named volume. + public func containersReferencingVolume(_ name: String) async throws -> [String] { + do { + let request = XPCMessage(route: .containerVolumeReferences) + request.set(key: .volumeName, value: name) + let response = try await xpcSend(message: request) + guard let data = response.dataNoCopy(key: .references) else { + return [] + } + return try JSONDecoder().decode([String].self, from: data) + } catch let error as ContainerizationError { + throw error + } catch { + throw ContainerizationError( + .internalError, + message: "failed to list containers referencing volume \(name)", + cause: error + ) + } + } + + /// The containers attached to the named network. + public func containersAttachedToNetwork(_ id: String) async throws -> [String] { + do { + let request = XPCMessage(route: .containerNetworkReferences) + request.set(key: .id, value: id) + let response = try await xpcSend(message: request) + guard let data = response.dataNoCopy(key: .references) else { + return [] + } + return try JSONDecoder().decode([String].self, from: data) + } catch let error as ContainerizationError { + throw error + } catch { + throw ContainerizationError( + .internalError, + message: "failed to list containers attached to network \(id)", + cause: error + ) + } + } + + /// The image references containers hold. + public func activeImageReferences() async throws -> Set { + do { + let request = XPCMessage(route: .containerImageReferences) + let response = try await xpcSend(message: request) + guard let data = response.dataNoCopy(key: .references) else { + return [] + } + return try JSONDecoder().decode(Set.self, from: data) + } catch let error as ContainerizationError { + throw error + } catch { + throw ContainerizationError( + .internalError, + message: "failed to list active image references", + cause: error + ) + } + } + + /// Disk usage totals for containers. + public func calculateDiskUsage() async throws -> ResourceUsage { + do { + let request = XPCMessage(route: .containerUsageTotals) + let response = try await xpcSend(message: request) + guard let data = response.dataNoCopy(key: .usageTotals) else { + throw ContainerizationError(.internalError, message: "usage totals missing from reply") + } + return try JSONDecoder().decode(ResourceUsage.self, from: data) + } catch let error as ContainerizationError { + throw error + } catch { + throw ContainerizationError( + .internalError, + message: "failed to calculate container disk usage", + cause: error + ) + } + } + /// List containers matching the given filters. public func list(filters: ContainerListFilters = .all) async throws -> [ContainerSnapshot] { do { diff --git a/Sources/Services/ContainerAPIService/Client/NetworkClient.swift b/Sources/Services/ContainerAPIService/Client/NetworkClient.swift index b7ea51f31..5af61aeea 100644 --- a/Sources/Services/ContainerAPIService/Client/NetworkClient.swift +++ b/Sources/Services/ContainerAPIService/Client/NetworkClient.swift @@ -38,7 +38,7 @@ public struct NetworkClient: Sendable { /// /// Pass a different value to ``init(serviceIdentifier:)`` to connect to an /// alternative service endpoint, for example during testing. - public static let defaultServiceIdentifier = "com.apple.container.apiserver" + public static let defaultServiceIdentifier = "com.apple.container.core.container-core-containers" /// The name of the default network created automatically on first use. public static let defaultNetworkName = "default" @@ -88,7 +88,7 @@ public struct NetworkClient: Sendable { return try JSONDecoder().decode(NetworkResource.self, from: resourceData) } - /// Returns the current state of all networks known to the API server. + /// Returns the current state of all networks the core plugin holds. /// /// - Returns: An array of ``NetworkResource`` values, or an empty array if no /// networks exist or the server returns no data. @@ -96,7 +96,11 @@ public struct NetworkClient: Sendable { public func list() async throws -> [NetworkResource] { let request = XPCMessage(route: .networkList) - let response = try await xpcSend(message: request, timeout: .seconds(1)) + // The route is served by a plugin, so the wait is the one a request to + // a service that may still have to launch is given. Launching one takes + // seconds, and a budget shorter than that fails a caller for the state + // of the machine rather than for anything it asked. + let response = try await xpcSend(message: request) guard let resourceData = response.dataNoCopy(key: .networkResources) else { return [] @@ -104,6 +108,25 @@ public struct NetworkClient: Sendable { return try JSONDecoder().decode([NetworkResource].self, from: resourceData) } + /// Resolve a container hostname to its network attachment. + /// + /// - Parameter hostname: A canonical DNS hostname with a trailing dot. + /// - Returns: The attachment whose hostname matches, or nil when no + /// network knows the name. + public func lookup(hostname: String) async throws -> Attachment? { + let request = XPCMessage(route: .networkLookup) + request.set(key: .hostname, value: hostname) + + // Served by the same plugin as the listing, and given the same wait for + // the same reason. + let response = try await xpcSend(message: request) + + guard let data = response.dataNoCopy(key: .attachment) else { + return nil + } + return try JSONDecoder().decode(Attachment.self, from: data) + } + /// Returns the network with the given identifier. /// /// - Parameter id: The identifier of the network to look up. diff --git a/Sources/Services/ContainerAPIService/Client/XPC+.swift b/Sources/Services/ContainerAPIService/Client/XPC+.swift index e564b2e15..1ab481331 100644 --- a/Sources/Services/ContainerAPIService/Client/XPC+.swift +++ b/Sources/Services/ContainerAPIService/Client/XPC+.swift @@ -128,6 +128,14 @@ public enum XPCKeys: String { case volume case volumes case volumeName + /// JSON array of container, volume, or image identifiers a query returns. + case references + /// JSON UsageTotals a resource kind reports. + case usageTotals + /// A canonical DNS hostname, with its trailing dot. + case hostname + /// JSON Attachment a network lookup returns. + case attachment case volumeSize case volumeDriver case volumeDriverOpts @@ -172,6 +180,11 @@ public enum XPCRoute: String { case containerCopyIn case containerCopyOut case containerExport + case containerVolumesInUse + case containerVolumeReferences + case containerNetworkReferences + case containerImageReferences + case containerUsageTotals case pluginLoad case pluginGet @@ -182,6 +195,7 @@ public enum XPCRoute: String { case networkCreate case networkDelete case networkList + case networkLookup case volumeCreate case volumeDelete diff --git a/Sources/Services/ContainerAPIService/Server/Containers/ContainersHarness.swift b/Sources/Services/ContainerAPIService/Server/Containers/ContainersHarness.swift index 1871cd149..0f96696b7 100644 --- a/Sources/Services/ContainerAPIService/Server/Containers/ContainersHarness.swift +++ b/Sources/Services/ContainerAPIService/Server/Containers/ContainersHarness.swift @@ -45,6 +45,62 @@ public struct ContainersHarness: Sendable { return reply } + @Sendable + public func volumeNamesInUse(_ message: XPCMessage) async throws -> XPCMessage { + let names = try await service.volumeNamesInUse() + let data = try JSONEncoder().encode(names) + + let reply = message.reply() + reply.set(key: .references, value: data) + return reply + } + + @Sendable + public func volumeReferences(_ message: XPCMessage) async throws -> XPCMessage { + guard let name = message.string(key: .volumeName) else { + throw ContainerizationError(.invalidArgument, message: "volume name cannot be empty") + } + let ids = try await service.containersReferencingVolume(name) + let data = try JSONEncoder().encode(ids) + + let reply = message.reply() + reply.set(key: .references, value: data) + return reply + } + + @Sendable + public func networkReferences(_ message: XPCMessage) async throws -> XPCMessage { + guard let id = message.string(key: .id) else { + throw ContainerizationError(.invalidArgument, message: "network id cannot be empty") + } + let ids = try await service.containersAttachedToNetwork(id) + let data = try JSONEncoder().encode(ids) + + let reply = message.reply() + reply.set(key: .references, value: data) + return reply + } + + @Sendable + public func imageReferences(_ message: XPCMessage) async throws -> XPCMessage { + let references = await service.getActiveImageReferences() + let data = try JSONEncoder().encode(references) + + let reply = message.reply() + reply.set(key: .references, value: data) + return reply + } + + @Sendable + public func usageTotals(_ message: XPCMessage) async throws -> XPCMessage { + let totals = await service.calculateDiskUsage() + let data = try JSONEncoder().encode(totals) + + let reply = message.reply() + reply.set(key: .usageTotals, value: data) + return reply + } + @Sendable public func bootstrap(_ message: XPCMessage) async throws -> XPCMessage { let id = message.string(key: .id) diff --git a/Sources/Services/ContainerAPIService/Server/Containers/ContainersService.swift b/Sources/Services/ContainerAPIService/Server/Containers/ContainersService.swift index e615c067d..1c7934cb0 100644 --- a/Sources/Services/ContainerAPIService/Server/Containers/ContainersService.swift +++ b/Sources/Services/ContainerAPIService/Server/Containers/ContainersService.swift @@ -282,8 +282,7 @@ public actor ContainersService { } /// Calculate disk usage for containers - /// - Returns: Tuple of (total count, active count, total size, reclaimable size) - public func calculateDiskUsage() async -> (Int, Int, UInt64, UInt64) { + public func calculateDiskUsage() async -> ResourceUsage { await lock.withLock(logMetadata: ["acquirer": "\(#function)"]) { _ in var totalSize: UInt64 = 0 var reclaimableSize: UInt64 = 0 @@ -302,7 +301,12 @@ public actor ContainersService { } } - return (await self.containers.count, activeCount, totalSize, reclaimableSize) + return ResourceUsage( + total: await self.containers.count, + active: activeCount, + sizeInBytes: totalSize, + reclaimable: reclaimableSize + ) } } diff --git a/Sources/Services/ContainerAPIService/Server/DiskUsage/DiskUsageService.swift b/Sources/Services/ContainerAPIService/Server/DiskUsage/DiskUsageService.swift index 39d43b8d0..b2d5a59fa 100644 --- a/Sources/Services/ContainerAPIService/Server/DiskUsage/DiskUsageService.swift +++ b/Sources/Services/ContainerAPIService/Server/DiskUsage/DiskUsageService.swift @@ -19,16 +19,14 @@ import Logging /// Service for calculating disk usage across all resource types public actor DiskUsageService { - private let containersService: ContainersService + private let containers = ContainerClient() private let volumesService: VolumesService private let log: Logger public init( - containersService: ContainersService, volumesService: VolumesService, log: Logger ) { - self.containersService = containersService self.volumesService = volumesService self.log = log } @@ -38,11 +36,11 @@ public actor DiskUsageService { log.debug("calculating disk usage for all resources") // Get active image references first (needed for image calculation) - let activeImageRefs = await containersService.getActiveImageReferences() + let activeImageRefs = try await containers.activeImageReferences() // Query all services concurrently async let imageStats = ClientImage.calculateDiskUsage(activeReferences: activeImageRefs) - async let containerStats = containersService.calculateDiskUsage() + async let containerStats = containers.calculateDiskUsage() async let volumeStats = volumesService.calculateDiskUsage() let (imageData, containerData, volumeData) = try await (imageStats, containerStats, volumeStats) @@ -54,26 +52,16 @@ public actor DiskUsageService { sizeInBytes: imageData.totalSize, reclaimable: imageData.reclaimableSize ), - containers: ResourceUsage( - total: containerData.0, - active: containerData.1, - sizeInBytes: containerData.2, - reclaimable: containerData.3 - ), - volumes: ResourceUsage( - total: volumeData.0, - active: volumeData.1, - sizeInBytes: volumeData.2, - reclaimable: volumeData.3 - ) + containers: containerData, + volumes: volumeData ) log.debug( "disk usage calculation complete", metadata: [ "images_total": "\(imageData.totalCount)", - "containers_total": "\(containerData.0)", - "volumes_total": "\(volumeData.0)", + "containers_total": "\(containerData.total)", + "volumes_total": "\(volumeData.total)", ]) return stats diff --git a/Sources/Services/ContainerAPIService/Server/Networks/NetworksHarness.swift b/Sources/Services/ContainerAPIService/Server/Networks/NetworksHarness.swift index ec525be83..2a836ba66 100644 --- a/Sources/Services/ContainerAPIService/Server/Networks/NetworksHarness.swift +++ b/Sources/Services/ContainerAPIService/Server/Networks/NetworksHarness.swift @@ -40,6 +40,18 @@ public struct NetworksHarness: Sendable { return reply } + @Sendable + public func lookup(_ message: XPCMessage) async throws -> XPCMessage { + guard let hostname = message.string(key: .hostname) else { + throw ContainerizationError(.invalidArgument, message: "hostname cannot be empty") + } + let reply = message.reply() + if let attachment = try await service.lookup(hostname: hostname) { + reply.set(key: .attachment, value: try JSONEncoder().encode(attachment)) + } + return reply + } + @Sendable public func create(_ message: XPCMessage) async throws -> XPCMessage { let data = message.dataNoCopy(key: .networkConfig) diff --git a/Sources/Services/ContainerAPIService/Server/Networks/NetworksService.swift b/Sources/Services/ContainerAPIService/Server/Networks/NetworksService.swift index ecf0e308d..00163434e 100644 --- a/Sources/Services/ContainerAPIService/Server/Networks/NetworksService.swift +++ b/Sources/Services/ContainerAPIService/Server/Networks/NetworksService.swift @@ -36,7 +36,7 @@ public actor NetworksService { private let pluginLoader: PluginLoader private let resourceRoot: FilePath - private let containersService: ContainersService + private let containers = ContainerClient() private let log: Logger private let debugHelpers: Bool @@ -50,14 +50,12 @@ public actor NetworksService { public init( pluginLoader: PluginLoader, resourceRoot: FilePath, - containersService: ContainersService, defaultNetworkConfiguration: NetworkConfiguration, log: Logger, debugHelpers: Bool = false, ) async throws { self.pluginLoader = pluginLoader self.resourceRoot = resourceRoot - self.containersService = containersService self.log = log self.debugHelpers = debugHelpers @@ -251,7 +249,7 @@ public actor NetworksService { // A container created after this answer can attach to the network // while it is deleted, the same window image delete accepts against // container create; the attach then fails naming the missing network. - let referringContainers = try await self.containersService.containersAttachedToNetwork(id) + let referringContainers = try await self.containers.containersAttachedToNetwork(id) // bail if any referring containers guard referringContainers.isEmpty else { diff --git a/Sources/Services/ContainerAPIService/Server/Volumes/VolumesService.swift b/Sources/Services/ContainerAPIService/Server/Volumes/VolumesService.swift index 297122f87..dc76076a6 100644 --- a/Sources/Services/ContainerAPIService/Server/Volumes/VolumesService.swift +++ b/Sources/Services/ContainerAPIService/Server/Volumes/VolumesService.swift @@ -14,6 +14,7 @@ // limitations under the License. //===----------------------------------------------------------------------===// +import ContainerAPIClient import ContainerPersistence import ContainerResource import Containerization @@ -31,17 +32,16 @@ public actor VolumesService { private let store: ContainerPersistence.FilesystemEntityStore private let log: Logger private let lock = AsyncLock() - private let containersService: ContainersService + private let containers = ContainerClient() // Storage constants private static let entityFile = "entity.json" private static let blockFile = "volume.img" - public init(resourceRoot: FilePath, containersService: ContainersService, log: Logger) async throws { + public init(resourceRoot: FilePath, log: Logger) async throws { try FileManager.default.createDirectory(atPath: resourceRoot.string, withIntermediateDirectories: true) self.resourceRoot = resourceRoot self.store = try FilesystemEntityStore(path: resourceRoot, type: "volumes", log: log) - self.containersService = containersService self.log = log // Migrate configs stored with the old `createdAt` key to `creationDate`. @@ -179,7 +179,7 @@ public actor VolumesService { /// Calculate disk usage for volumes /// - Returns: Tuple of (total count, active count, total size, reclaimable size) - public func calculateDiskUsage() async throws -> (Int, Int, UInt64, UInt64) { + public func calculateDiskUsage() async throws -> ResourceUsage { log.debug( "VolumesService: enter", metadata: [ @@ -198,7 +198,7 @@ public actor VolumesService { return try await lock.withLock { _ in let allVolumes = try await self.store.list() - let inUseSet = try await self.containersService.volumeNamesInUse() + let inUseSet = try await self.containers.volumeNamesInUse() var totalSize: UInt64 = 0 var reclaimableSize: UInt64 = 0 @@ -214,7 +214,12 @@ public actor VolumesService { } } - return (allVolumes.count, inUseSet.count, totalSize, reclaimableSize) + return ResourceUsage( + total: allVolumes.count, + active: inUseSet.count, + sizeInBytes: totalSize, + reclaimable: reclaimableSize + ) } } @@ -363,7 +368,7 @@ public actor VolumesService { // A container created after this answer can name the volume and lose // it, the same window image delete accepts against container create; // the create then fails naming the missing volume. - let referencing = try await containersService.containersReferencingVolume(name) + let referencing = try await containers.containersReferencingVolume(name) guard referencing.isEmpty else { throw VolumeError.volumeInUse(name) } diff --git a/Tests/ContainerAPIServiceTests/DiskUsagePathTests.swift b/Tests/ContainerAPIServiceTests/DiskUsagePathTests.swift index 947d680e4..d15878cb4 100644 --- a/Tests/ContainerAPIServiceTests/DiskUsagePathTests.swift +++ b/Tests/ContainerAPIServiceTests/DiskUsagePathTests.swift @@ -50,7 +50,6 @@ struct DiskUsagePathTests { private func makeVolumesService(appRoot: FilePath) async throws -> VolumesService { try await VolumesService( resourceRoot: appRoot.appending("volumes"), - containersService: makeContainersService(appRoot: appRoot), log: log ) } From 73bf8e85ab3f406ff8a3497e8abc64475774cce2 Mon Sep 17 00:00:00 2001 From: Aaron Paterson Date: Mon, 10 Aug 2026 05:18:17 +0000 Subject: [PATCH 11/29] Give the command line's pod and run to CLI plugins An unrecognized subcommand already resolves to a container- plugin binary and execs it, which is how k8s ships. pod and run are the same kind of surface: each plugin re-exposes commands the ContainerCommands library already makes public, so the binaries are declarations with no logic of their own. The pod command's single-letter alias does not survive the move: the compiled-in command list carried it, and plugin resolution goes by the one name the binary is installed under. --- Makefile | 8 +++++ Package.swift | 18 +++++++++++ Sources/ContainerCommands/Application.swift | 7 ---- Sources/Plugins/PodCLI/PodPlugin.swift | 36 +++++++++++++++++++++ Sources/Plugins/PodCLI/config.toml | 3 ++ Sources/Plugins/RunCLI/RunPlugin.swift | 25 ++++++++++++++ Sources/Plugins/RunCLI/config.toml | 3 ++ 7 files changed, 93 insertions(+), 7 deletions(-) create mode 100644 Sources/Plugins/PodCLI/PodPlugin.swift create mode 100644 Sources/Plugins/PodCLI/config.toml create mode 100644 Sources/Plugins/RunCLI/RunPlugin.swift create mode 100644 Sources/Plugins/RunCLI/config.toml diff --git a/Makefile b/Makefile index 5c2aff350..6d18ac41d 100644 --- a/Makefile +++ b/Makefile @@ -136,6 +136,8 @@ $(STAGING_DIR): @mkdir -p "$(join $(STAGING_DIR), libexec/container/plugins/machine-apiserver/resources)" @mkdir -p "$(join $(STAGING_DIR), libexec/container/plugins/k8s/bin)" @mkdir -p "$(join $(STAGING_DIR), libexec/container/plugins/k8s/resources)" + @mkdir -p "$(join $(STAGING_DIR), libexec/container/plugins/pod/bin)" + @mkdir -p "$(join $(STAGING_DIR), libexec/container/plugins/run/bin)" @install "$(BUILD_BIN_DIR)/container" "$(join $(STAGING_DIR), bin/container)" @install "$(BUILD_BIN_DIR)/container-apiserver" "$(join $(STAGING_DIR), bin/container-apiserver)" @@ -154,6 +156,10 @@ $(STAGING_DIR): @install "$(BUILD_BIN_DIR)/k8s" "$(join $(STAGING_DIR), libexec/container/plugins/k8s/bin/k8s)" @install Sources/Plugins/K8s/config.toml "$(join $(STAGING_DIR), libexec/container/plugins/k8s/config.toml)" @install Sources/Plugins/K8s/Resources/kindnet.yaml "$(join $(STAGING_DIR), libexec/container/plugins/k8s/resources/kindnet.yaml)" + @install "$(BUILD_BIN_DIR)/pod" "$(join $(STAGING_DIR), libexec/container/plugins/pod/bin/pod)" + @install Sources/Plugins/PodCLI/config.toml "$(join $(STAGING_DIR), libexec/container/plugins/pod/config.toml)" + @install "$(BUILD_BIN_DIR)/run" "$(join $(STAGING_DIR), libexec/container/plugins/run/bin/run)" + @install Sources/Plugins/RunCLI/config.toml "$(join $(STAGING_DIR), libexec/container/plugins/run/config.toml)" @echo Install update script @install scripts/update-container.sh "$(join $(STAGING_DIR), bin/update-container.sh)" @@ -171,6 +177,8 @@ installer-pkg: $(STAGING_DIR) @codesign $(CODESIGN_OPTS) --prefix=com.apple.container. --entitlements=signing/container-network-vmnet.entitlements "$(join $(STAGING_DIR), libexec/container/plugins/container-network-vmnet/bin/container-network-vmnet)" @codesign $(CODESIGN_OPTS) --prefix=com.apple.container. "$(join $(STAGING_DIR), libexec/container/plugins/machine-apiserver/bin/machine-apiserver)" @codesign $(CODESIGN_OPTS) --prefix=com.apple.container. "$(join $(STAGING_DIR), libexec/container/plugins/k8s/bin/k8s)" + @codesign $(CODESIGN_OPTS) --prefix=com.apple.container. "$(join $(STAGING_DIR), libexec/container/plugins/pod/bin/pod)" + @codesign $(CODESIGN_OPTS) --prefix=com.apple.container. "$(join $(STAGING_DIR), libexec/container/plugins/run/bin/run)" @echo Creating application installer @pkgbuild --root "$(STAGING_DIR)" --identifier com.apple.container-installer --install-location /usr/local --version ${RELEASE_VERSION} $(PKG_PATH) diff --git a/Package.swift b/Package.swift index 543664878..4c36336c7 100644 --- a/Package.swift +++ b/Package.swift @@ -330,6 +330,24 @@ let package = Package( path: "Sources/Plugins/CoreContainers", exclude: ["config.toml"] ), + .executableTarget( + name: "pod", + dependencies: [ + .product(name: "ArgumentParser", package: "swift-argument-parser"), + "ContainerCommands", + ], + path: "Sources/Plugins/PodCLI", + exclude: ["config.toml"] + ), + .executableTarget( + name: "run", + dependencies: [ + .product(name: "ArgumentParser", package: "swift-argument-parser"), + "ContainerCommands", + ], + path: "Sources/Plugins/RunCLI", + exclude: ["config.toml"] + ), .target( name: "ContainerImagesService", dependencies: [ diff --git a/Sources/ContainerCommands/Application.swift b/Sources/ContainerCommands/Application.swift index 57a3cb741..35a989357 100644 --- a/Sources/ContainerCommands/Application.swift +++ b/Sources/ContainerCommands/Application.swift @@ -63,7 +63,6 @@ public struct Application: AsyncLoggableCommand { ContainerKill.self, ContainerList.self, ContainerLogs.self, - ContainerRun.self, ContainerStart.self, ContainerStats.self, ContainerStop.self, @@ -90,12 +89,6 @@ public struct Application: AsyncLoggableCommand { VolumeCommand.self ] ), - CommandGroup( - name: "Pod", - subcommands: [ - PodCommand.self - ] - ), CommandGroup( name: "Other", subcommands: Self.otherCommands() diff --git a/Sources/Plugins/PodCLI/PodPlugin.swift b/Sources/Plugins/PodCLI/PodPlugin.swift new file mode 100644 index 000000000..6f1a9a54f --- /dev/null +++ b/Sources/Plugins/PodCLI/PodPlugin.swift @@ -0,0 +1,36 @@ +//===----------------------------------------------------------------------===// +// Copyright © 2026 Apple Inc. and the container project authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//===----------------------------------------------------------------------===// + +import ArgumentParser +import ContainerCommands + +@main +struct PodPlugin: AsyncParsableCommand { + static let configuration = CommandConfiguration( + commandName: "pod", + abstract: "Manage pods, machines that several containers share", + subcommands: [ + Application.PodCommand.PodCreate.self, + Application.PodCommand.PodStart.self, + Application.PodCommand.PodStop.self, + Application.PodCommand.PodDelete.self, + Application.PodCommand.PodList.self, + Application.PodCommand.PodInspect.self, + Application.PodCommand.PodPrune.self, + Application.PodCommand.PodUpdate.self, + ] + ) +} diff --git a/Sources/Plugins/PodCLI/config.toml b/Sources/Plugins/PodCLI/config.toml new file mode 100644 index 000000000..9255fd9ca --- /dev/null +++ b/Sources/Plugins/PodCLI/config.toml @@ -0,0 +1,3 @@ +abstract = "Manage pods, machines that several containers share" +author = "Apple" +version = 0.1 diff --git a/Sources/Plugins/RunCLI/RunPlugin.swift b/Sources/Plugins/RunCLI/RunPlugin.swift new file mode 100644 index 000000000..ceb883e18 --- /dev/null +++ b/Sources/Plugins/RunCLI/RunPlugin.swift @@ -0,0 +1,25 @@ +//===----------------------------------------------------------------------===// +// Copyright © 2026 Apple Inc. and the container project authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//===----------------------------------------------------------------------===// + +import ArgumentParser +import ContainerCommands + +@main +struct RunPlugin { + static func main() async { + await Application.ContainerRun.main() + } +} diff --git a/Sources/Plugins/RunCLI/config.toml b/Sources/Plugins/RunCLI/config.toml new file mode 100644 index 000000000..22a169ef5 --- /dev/null +++ b/Sources/Plugins/RunCLI/config.toml @@ -0,0 +1,3 @@ +abstract = "Run a container from an image" +author = "Apple" +version = 0.1 From 69034eb8ee653de504c8649d4f47274e8160937d Mon Sep 17 00:00:00 2001 From: Aaron Paterson Date: Fri, 14 Aug 2026 00:51:03 +0000 Subject: [PATCH 12/29] Carry the caller's agent through pod start to every member A pod's machine boot starts the containers inside it, and each with --ssh forwards the agent named by the donation the boot delivers. The start paths a single container takes collect SSH_AUTH_SOCK into the boot's dynamic environment, and pod start now does the same: the CLI reads the caller's socket, the client sends it with the start message, and the harness hands it to the service, whose machine bootstrap already places every bundle with the boot's environment. A pod booted by pod start behaves like one booted through a member's own start. --- Sources/ContainerCommands/Pod/PodLifecycle.swift | 8 +++++++- .../Services/ContainerAPIService/Client/ClientPod.swift | 9 ++++++++- .../ContainerAPIService/Server/Pods/PodsHarness.swift | 6 +++++- 3 files changed, 20 insertions(+), 3 deletions(-) diff --git a/Sources/ContainerCommands/Pod/PodLifecycle.swift b/Sources/ContainerCommands/Pod/PodLifecycle.swift index b97c025aa..3f3df786d 100644 --- a/Sources/ContainerCommands/Pod/PodLifecycle.swift +++ b/Sources/ContainerCommands/Pod/PodLifecycle.swift @@ -37,8 +37,14 @@ extension Application.PodCommand { public init() {} public func run() async throws { + // The caller's agent rides into every member the machine boots, + // the donation each sibling boot path carries. + var dynamicEnv: [String: String] = [:] + if let agent = ProcessInfo.processInfo.environment["SSH_AUTH_SOCK"] { + dynamicEnv["SSH_AUTH_SOCK"] = agent + } for name in names { - try await ClientPod.start(name) + try await ClientPod.start(name, dynamicEnv: dynamicEnv) print(name) } } diff --git a/Sources/Services/ContainerAPIService/Client/ClientPod.swift b/Sources/Services/ContainerAPIService/Client/ClientPod.swift index f9d4b9901..8c3f0163a 100644 --- a/Sources/Services/ContainerAPIService/Client/ClientPod.swift +++ b/Sources/Services/ContainerAPIService/Client/ClientPod.swift @@ -57,10 +57,17 @@ public struct ClientPod { } /// Boot a pod's machine, with the containers that belong to it inside. - public static func start(_ id: String) async throws { + /// + /// `dynamicEnv` carries per-boot environment such as the caller's + /// SSH_AUTH_SOCK to every container the machine starts, the same + /// donation a container's own start delivers. + public static func start(_ id: String, dynamicEnv: [String: String] = [:]) async throws { let client = XPCClient(service: serviceIdentifier) let message = XPCMessage(route: .podStart) message.set(key: .podId, value: id) + if !dynamicEnv.isEmpty { + message.set(key: .dynamicEnv, value: try JSONEncoder().encode(dynamicEnv)) + } _ = try await client.send(message) } diff --git a/Sources/Services/ContainerAPIService/Server/Pods/PodsHarness.swift b/Sources/Services/ContainerAPIService/Server/Pods/PodsHarness.swift index db52c3897..c9123b9dd 100644 --- a/Sources/Services/ContainerAPIService/Server/Pods/PodsHarness.swift +++ b/Sources/Services/ContainerAPIService/Server/Pods/PodsHarness.swift @@ -53,7 +53,11 @@ public struct PodsHarness: Sendable { @Sendable public func start(_ message: XPCMessage) async throws -> XPCMessage { - try await service.start(id: try message.podId()) + let data = message.dataNoCopy(key: .dynamicEnv) + let dynamicEnv = try data.map { try JSONDecoder().decode([String: String].self, from: $0) } ?? [:] + let startup: PodsService.ContainerStartup? = + dynamicEnv.isEmpty ? nil : .init(stdio: [nil, nil, nil], dynamicEnv: dynamicEnv) + try await service.start(id: try message.podId(), startup: startup) return message.reply() } From 8599e6f9ba7cdca901cbcfed04d353be6439196a Mon Sep 17 00:00:00 2001 From: Aaron Paterson Date: Fri, 14 Aug 2026 02:44:50 +0000 Subject: [PATCH 13/29] Stop a container's dedicated machine when its last container stops A machine nobody named exists because its one container needed it, and it held the container's devices, its named volumes among them, from boot. A machine held past its container's exit keeps those claims: the virtual machine's helper process stays alive with the volume images open, so the next container attaching the same named volume is refused with an invalid storage device attachment. The boot request now says which kind of machine it is booting: one made for its container stops with the last thing in it, releasing what it held, while a machine someone named outlives its members the way a pod's machine does. https://github.com/kubernetes/cri-api/blob/master/pkg/apis/runtime/v1/api.proto --- .../Server/Pods/PodsService.swift | 3 ++- .../Runtime/RuntimeClient/RuntimeClient.swift | 4 +++- .../Runtime/RuntimeClient/RuntimeKeys.swift | 4 ++++ .../RuntimeLinux/Server/RuntimeService.swift | 15 ++++++++++++--- 4 files changed, 21 insertions(+), 5 deletions(-) diff --git a/Sources/Services/ContainerAPIService/Server/Pods/PodsService.swift b/Sources/Services/ContainerAPIService/Server/Pods/PodsService.swift index 6af23cf6a..846f417d1 100644 --- a/Sources/Services/ContainerAPIService/Server/Pods/PodsService.swift +++ b/Sources/Services/ContainerAPIService/Server/Pods/PodsService.swift @@ -415,7 +415,8 @@ public actor PodsService { stdioFor: container, stdio: startup?.stdio ?? [nil, nil, nil], networkBootstrapInfos: networkBootstrapInfos, - dynamicEnv: startup?.dynamicEnv ?? [:] + dynamicEnv: startup?.dynamicEnv ?? [:], + stopsWithContainers: state.configuration.isAnonymous ) if running == nil { diff --git a/Sources/Services/Runtime/RuntimeClient/RuntimeClient.swift b/Sources/Services/Runtime/RuntimeClient/RuntimeClient.swift index abffb83af..24cf27ce2 100644 --- a/Sources/Services/Runtime/RuntimeClient/RuntimeClient.swift +++ b/Sources/Services/Runtime/RuntimeClient/RuntimeClient.swift @@ -113,7 +113,8 @@ extension RuntimeClient { stdioFor: String? = nil, stdio: [FileHandle?], networkBootstrapInfos: [NetworkBootstrapInfo], - dynamicEnv: [String: String] = [:] + dynamicEnv: [String: String] = [:], + stopsWithContainers: Bool = false ) async throws { let request = self.request(RuntimeRoutes.bootstrap.rawValue) try request.setStdio(stdio) @@ -121,6 +122,7 @@ extension RuntimeClient { do { let dynamicEnv = try JSONEncoder().encode(dynamicEnv) request.set(key: RuntimeKeys.dynamicEnv.rawValue, value: dynamicEnv) + request.set(key: RuntimeKeys.sandboxStopsWithContainers.rawValue, value: stopsWithContainers) let pathsData = try JSONEncoder().encode(bundlePaths) request.set(key: RuntimeKeys.bundlePaths.rawValue, value: pathsData) diff --git a/Sources/Services/Runtime/RuntimeClient/RuntimeKeys.swift b/Sources/Services/Runtime/RuntimeClient/RuntimeKeys.swift index 6f2da4342..8b2303440 100644 --- a/Sources/Services/Runtime/RuntimeClient/RuntimeKeys.swift +++ b/Sources/Services/Runtime/RuntimeClient/RuntimeKeys.swift @@ -27,6 +27,10 @@ public enum RuntimeKeys: String { case bundlePath /// The paths to the bundles of every container the sandbox holds. case bundlePaths + /// Whether the sandbox stops once its last container has stopped, the + /// way a machine made for one container does; absent means the sandbox + /// outlives its containers, the way a named pod's machine does. + case sandboxStopsWithContainers /// A memory size in bytes the sandbox is to be held to. case memoryInBytes /// Vsock port number key. diff --git a/Sources/Services/RuntimeLinux/Server/RuntimeService.swift b/Sources/Services/RuntimeLinux/Server/RuntimeService.swift index 9d9e9cdf7..14156da7d 100644 --- a/Sources/Services/RuntimeLinux/Server/RuntimeService.swift +++ b/Sources/Services/RuntimeLinux/Server/RuntimeService.swift @@ -47,6 +47,11 @@ public actor RuntimeService { /// The containers in that machine, by identifier. A pod holds several and /// a standalone container holds one. private var containers: [String: ContainerInfo] = [:] + + /// Whether the machine stops once its last container has stopped. The + /// boot request says, since only the control plane knows whether this + /// sandbox was named by someone or exists for its one container. + private var sandboxStopsWithContainers = false /// The addresses a pod claimed, which every container placed in it shares. private var podAttachments: [Attachment] = [] /// The ports published on those addresses, which are the pod's because the @@ -149,6 +154,8 @@ public actor RuntimeService { self.log.debug("enter", metadata: ["func": "\(#function)"]) defer { self.log.debug("exit", metadata: ["func": "\(#function)"]) } + self.sandboxStopsWithContainers = message.bool(key: RuntimeKeys.sandboxStopsWithContainers.rawValue) + // Create the bundle if it doesn't exist yet if !self.bundleExists(at: self.root) { try self.createBundle() @@ -961,11 +968,13 @@ public actor RuntimeService { // A pod's machine is its sandbox, which outlives the containers // that come and go in it: it holds the addresses and namespaces // they share and is taken down when the pod is, not when a - // container in it leaves. A single container's machine is the - // container's own, so it stops with the last thing in it. + // container in it leaves. A machine nobody named exists for its + // one container, so it stops with the last thing in it and + // releases the devices it held; the boot request says which + // kind this machine is. // https://github.com/kubernetes/cri-api/blob/master/pkg/apis/runtime/v1/api.proto let sandbox = try await self.getSandbox() - guard !(sandbox is LinuxPod) else { + if sandbox is LinuxPod, !self.sandboxStopsWithContainers { return } guard await self.containers.isEmpty else { From 78d4db14d760bcd63cdd5d5991fe41010cfa618a Mon Sep 17 00:00:00 2001 From: Aaron Paterson Date: Fri, 14 Aug 2026 02:48:47 +0000 Subject: [PATCH 14/29] Await the machine's own held flag from the exit closure The exit handler runs its body in a sendable closure, which reads the actor's properties the way any outside caller would. --- Sources/Services/RuntimeLinux/Server/RuntimeService.swift | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Sources/Services/RuntimeLinux/Server/RuntimeService.swift b/Sources/Services/RuntimeLinux/Server/RuntimeService.swift index 14156da7d..a45006a5e 100644 --- a/Sources/Services/RuntimeLinux/Server/RuntimeService.swift +++ b/Sources/Services/RuntimeLinux/Server/RuntimeService.swift @@ -974,7 +974,7 @@ public actor RuntimeService { // kind this machine is. // https://github.com/kubernetes/cri-api/blob/master/pkg/apis/runtime/v1/api.proto let sandbox = try await self.getSandbox() - if sandbox is LinuxPod, !self.sandboxStopsWithContainers { + if sandbox is LinuxPod, !(await self.sandboxStopsWithContainers) { return } guard await self.containers.isEmpty else { From 00eddfd8058bc12fce250c20ce92b80db97baaa5 Mon Sep 17 00:00:00 2001 From: Aaron Paterson Date: Fri, 14 Aug 2026 02:53:47 +0000 Subject: [PATCH 15/29] Leave the guest without a resolver when DNS is declined A container created with no DNS flags carries a configuration naming no resolver, and the network fills it out: the gateway resolves for the machine's containers. A container created with --no-dns carries no configuration at all, and the machine boots without one, so the guest gets no resolv.conf; filling that case from the network turned the explicit refusal into the default it refused. --- .../RuntimeLinux/Server/RuntimeService.swift | 25 +++++++++++++------ 1 file changed, 17 insertions(+), 8 deletions(-) diff --git a/Sources/Services/RuntimeLinux/Server/RuntimeService.swift b/Sources/Services/RuntimeLinux/Server/RuntimeService.swift index a45006a5e..93cb01d49 100644 --- a/Sources/Services/RuntimeLinux/Server/RuntimeService.swift +++ b/Sources/Services/RuntimeLinux/Server/RuntimeService.swift @@ -1277,9 +1277,18 @@ public actor RuntimeService { // configuration. The network belongs to the pod, so the resolver derived // from it does too, the way it was derived from the machine's attachments // when the machine held the network for one container. - var nameservers = config.dns?.nameservers ?? [] - if nameservers.isEmpty { - nameservers = self.getDefaultNameservers(from: attachments) + // DNS the way the record says: a configuration given but naming no + // resolver is filled out from the network, the gateway resolving for + // the machine's containers; no configuration at all is the --no-dns + // request, and leaves the guest without a resolv.conf. + let dns: ContainerConfiguration.DNSConfiguration? = config.dns.map { configured in + guard configured.nameservers.isEmpty else { return configured } + return ContainerConfiguration.DNSConfiguration( + nameservers: self.getDefaultNameservers(from: attachments), + domain: configured.domain, + searchDomains: configured.searchDomains, + options: configured.options + ) } // One swap area serves the whole pod, which is what makes the pool its @@ -1319,12 +1328,12 @@ public actor RuntimeService { sysctls["vm.overcommit_memory"] = "1" sysctls["vm.max_map_count"] = "262144" podConfig.sysctl = sysctls - if !nameservers.isEmpty { + if let dns { podConfig.dns = DNS( - nameservers: nameservers, - domain: config.dns?.domain, - searchDomains: config.dns?.searchDomains ?? [], - options: config.dns?.options ?? [] + nameservers: dns.nameservers, + domain: dns.domain, + searchDomains: dns.searchDomains, + options: dns.options ) } podConfig.bootLog = BootLog.file(path: bundle.bootlog, append: true) From 09e7747976ecacf8efa6275520eec2fb4b5aa92e Mon Sep 17 00:00:00 2001 From: Aaron Paterson Date: Fri, 14 Aug 2026 04:46:00 +0000 Subject: [PATCH 16/29] Fail a pod start whose machine does not answer, keeping the client The machine is believed only while it answers running, and an answer of anything else takes the service down so a fresh machine can boot. A query that fails carries no such answer: treating it as the machine gone deregistered a live client and booted a second machine against devices the first still held, which failed at its attachments and left the pod unmanageable. The start now fails on the unanswered query and the held client stands, the way kubelet recreates a pod sandbox only on a positive verdict (absent, duplicated, or a status of not ready) and returns a failed status query as the operation's error. https://github.com/kubernetes/kubernetes/blob/master/pkg/kubelet/kuberuntime/util/util.go --- .../Server/Pods/PodsService.swift | 31 +++++++++++++------ 1 file changed, 21 insertions(+), 10 deletions(-) diff --git a/Sources/Services/ContainerAPIService/Server/Pods/PodsService.swift b/Sources/Services/ContainerAPIService/Server/Pods/PodsService.swift index 846f417d1..de5ab0e0c 100644 --- a/Sources/Services/ContainerAPIService/Server/Pods/PodsService.swift +++ b/Sources/Services/ContainerAPIService/Server/Pods/PodsService.swift @@ -364,16 +364,27 @@ public actor PodsService { // stopped out of band, crashed, or torn down without the pod // hearing of it. A pod that offered that machine would be // offering one that is not there, so the client is believed only - // while its machine answers running; otherwise the service is - // taken down and the pod boots a fresh machine the way it booted - // the first. - if let held = running, (try? await held.state())?.status != .running { - await self.deregister(id: id) - state.client = nil - state.state = .notReady - state.startedDate = nil - await self.setPodState(id, state, context: context) - running = nil + // while its machine answers running: an answer of anything else + // means the machine is gone, and the service is taken down so + // the pod boots a fresh machine the way it booted the first. No + // answer at all is the query failing, not the machine standing + // down; a fresh machine booted against devices a live one still + // holds fails at its attachments, so the start fails on the + // query instead and the held client stands. + if let held = running { + guard let observed = try? await held.state() else { + throw ContainerizationError( + .internalError, + message: "the machine of pod \(id) did not answer a state query") + } + if observed.status != .running { + await self.deregister(id: id) + state.client = nil + state.state = .notReady + state.startedDate = nil + await self.setPodState(id, state, context: context) + running = nil + } } do { From 42e10990b01ac394c9b2d4d5df06a8357eb8cc3f Mon Sep 17 00:00:00 2001 From: Aaron Paterson Date: Fri, 14 Aug 2026 17:52:20 +0000 Subject: [PATCH 17/29] Keep the default network on the address range it was given The network's stored configuration was replaced at every start with a computed one naming no address range, so the range was assigned afresh each time and every guest that outlived the restart held an address, a route, and a resolver belonging to a range that no longer existed. The range now rides through the refresh, and a network that named none is written down with the one it was given, so it asks for the same range from then on. --- .../Server/Networks/NetworksService.swift | 33 ++++++++++++++++++- 1 file changed, 32 insertions(+), 1 deletion(-) diff --git a/Sources/Services/ContainerAPIService/Server/Networks/NetworksService.swift b/Sources/Services/ContainerAPIService/Server/Networks/NetworksService.swift index 00163434e..3827a32c0 100644 --- a/Sources/Services/ContainerAPIService/Server/Networks/NetworksService.swift +++ b/Sources/Services/ContainerAPIService/Server/Networks/NetworksService.swift @@ -83,7 +83,20 @@ public actor NetworksService { // computed default network configuration from the apiserver to ensure we // have the correct default values configured. if effectiveConfiguration.id == NetworkClient.defaultNetworkName { - effectiveConfiguration = defaultNetworkConfiguration + // The address range the network came up on is its own, so it + // rides through the refresh and is asked for again: a network + // that returns on a different range leaves every guest that + // outlived the restart holding an address, a route, and a + // resolver belonging to a range that is gone. + effectiveConfiguration = try NetworkConfiguration( + name: defaultNetworkConfiguration.name, + mode: defaultNetworkConfiguration.mode, + ipv4Subnet: defaultNetworkConfiguration.ipv4Subnet ?? configuration.ipv4Subnet, + ipv6Subnet: defaultNetworkConfiguration.ipv6Subnet ?? configuration.ipv6Subnet, + labels: defaultNetworkConfiguration.labels, + plugin: defaultNetworkConfiguration.plugin, + options: defaultNetworkConfiguration.options + ) try await store.update(effectiveConfiguration) } @@ -96,6 +109,24 @@ public actor NetworksService { try await registerService(configuration: effectiveConfiguration) let client = try Self.getClient(configuration: effectiveConfiguration) let networkStatus = try await client.status() + + // A network that named no range is given one as it comes up, + // and that range is written down as the range it asks for from + // then on, so the guests it addresses keep answering to the + // same addresses across a restart. + if effectiveConfiguration.ipv4Subnet == nil { + effectiveConfiguration = try NetworkConfiguration( + name: effectiveConfiguration.name, + mode: effectiveConfiguration.mode, + ipv4Subnet: networkStatus.ipv4Subnet, + ipv6Subnet: effectiveConfiguration.ipv6Subnet, + labels: effectiveConfiguration.labels, + plugin: effectiveConfiguration.plugin, + options: effectiveConfiguration.options + ) + try await store.update(effectiveConfiguration) + } + serviceStates[effectiveConfiguration.id] = NetworkEntry( configuration: effectiveConfiguration, status: networkStatus, From 031efa66fc89b24db55d7585bf51cd9cdd20077c Mon Sep 17 00:00:00 2001 From: Aaron Paterson Date: Tue, 18 Aug 2026 19:54:47 +0000 Subject: [PATCH 18/29] Give a machine someone named the network's resolver Nothing recorded is a refusal only where a refusal can be made: a container declines DNS with --no-dns and the machine made for it carries that refusal, while a pod is never asked and so records nothing when it is created without DNS flags. Reading that silence as a refusal left every guest in a named pod without a resolver, holding whatever its root filesystem was last written with, so a machine that came back on a new address range kept answering to the old one. A named pod is given the network's resolver, a pod that names one keeps it, and a container's own machine still declines when the container did. A pod created now records what it was told either way. --- Sources/ContainerCommands/Pod/PodCreate.swift | 19 +++++----- .../RuntimeLinux/Server/RuntimeService.swift | 36 ++++++++++--------- 2 files changed, 28 insertions(+), 27 deletions(-) diff --git a/Sources/ContainerCommands/Pod/PodCreate.swift b/Sources/ContainerCommands/Pod/PodCreate.swift index ba437aae2..62132f3a1 100644 --- a/Sources/ContainerCommands/Pod/PodCreate.swift +++ b/Sources/ContainerCommands/Pod/PodCreate.swift @@ -82,16 +82,15 @@ extension Application.PodCommand { configuration.rosetta = rosetta configuration.labels = try Parser.labels(label) - if !dns.nameservers.isEmpty || dns.domain != nil || !dns.searchDomains.isEmpty || !dns.options.isEmpty { - configuration.dns = ContainerConfiguration.DNSConfiguration( - nameservers: dns.nameservers.isEmpty - ? ContainerConfiguration.DNSConfiguration.defaultNameservers - : dns.nameservers, - domain: dns.domain, - searchDomains: dns.searchDomains, - options: dns.options - ) - } + // A pod records a DNS configuration whatever it was told, since + // what it was not told is filled from the network its machine + // comes up on: a record naming no resolver is what asks for that. + configuration.dns = ContainerConfiguration.DNSConfiguration( + nameservers: dns.nameservers, + domain: dns.domain, + searchDomains: dns.searchDomains, + options: dns.options + ) let parsedNetworks = try network.map { try Parser.network($0) } let networkClient = NetworkClient() diff --git a/Sources/Services/RuntimeLinux/Server/RuntimeService.swift b/Sources/Services/RuntimeLinux/Server/RuntimeService.swift index 93cb01d49..88e3e283e 100644 --- a/Sources/Services/RuntimeLinux/Server/RuntimeService.swift +++ b/Sources/Services/RuntimeLinux/Server/RuntimeService.swift @@ -1273,23 +1273,25 @@ public actor RuntimeService { infos: try message.networkBootstrapInfos() ) - // Dynamically configure the DNS nameserver from a network if no explicit - // configuration. The network belongs to the pod, so the resolver derived - // from it does too, the way it was derived from the machine's attachments - // when the machine held the network for one container. - // DNS the way the record says: a configuration given but naming no - // resolver is filled out from the network, the gateway resolving for - // the machine's containers; no configuration at all is the --no-dns - // request, and leaves the guest without a resolv.conf. - let dns: ContainerConfiguration.DNSConfiguration? = config.dns.map { configured in - guard configured.nameservers.isEmpty else { return configured } - return ContainerConfiguration.DNSConfiguration( - nameservers: self.getDefaultNameservers(from: attachments), - domain: configured.domain, - searchDomains: configured.searchDomains, - options: configured.options - ) - } + // DNS the way the record says. A configuration naming no resolver is + // filled out from the network, the gateway resolving for the machine's + // containers. Nothing recorded is a refusal only where a refusal can + // be made: a container declines with --no-dns and the machine made for + // it carries that refusal, while a machine someone named holds no such + // request and is given the network's resolver, the way it was before + // it could be asked. + let derived = ContainerConfiguration.DNSConfiguration( + nameservers: self.getDefaultNameservers(from: attachments), + domain: config.dns?.domain, + searchDomains: config.dns?.searchDomains ?? [], + options: config.dns?.options ?? [] + ) + let dns: ContainerConfiguration.DNSConfiguration? = { + guard let configured = config.dns else { + return config.isAnonymous ? nil : derived + } + return configured.nameservers.isEmpty ? derived : configured + }() // One swap area serves the whole pod, which is what makes the pool its // containers reclaim to a shared one. From 9599c63e327b00c42e2c4bb54a29925e3a4cc25e Mon Sep 17 00:00:00 2001 From: Aaron Paterson Date: Tue, 18 Aug 2026 21:29:25 +0000 Subject: [PATCH 19/29] Give back what a network holds, and come up without it if it is gone A vmnet network's address range is reserved for as long as the network object lives, so a helper that goes away still holding it leaves the range spoken for by a network nobody holds: no interface, no route, no process to point at, and every later attempt on that range refused. The helper releases the network when it is asked to stop, which is where a range is given back for the asking. A range that cannot be taken is then survivable rather than fatal. The range the default network asks for is the one it was given last time, a preference and not a demand, so a default network that cannot take it comes up on whatever is free and says which range it lost; a network someone asked for by name still fails, since that range is the request. Calls to a network helper wait a bounded time, so a helper that never answers is reported instead of leaving every command that needs a network waiting forever with nothing to show. --- .../NetworkVmnetHelper+Start.swift | 15 ++++++++++ .../Server/Networks/NetworksService.swift | 28 ++++++++++++++++++- .../Network/Client/NetworkClient.swift | 14 ++++++++-- Sources/Services/Network/Server/Network.swift | 6 ++++ .../Server/AllocationOnlyVmnetNetwork.swift | 6 ++++ .../Server/ReservedVmnetNetwork.swift | 21 ++++++++++++++ 6 files changed, 86 insertions(+), 4 deletions(-) diff --git a/Sources/Plugins/NetworkVmnet/NetworkVmnetHelper+Start.swift b/Sources/Plugins/NetworkVmnet/NetworkVmnetHelper+Start.swift index 7d67f1f32..a2458100f 100644 --- a/Sources/Plugins/NetworkVmnet/NetworkVmnetHelper+Start.swift +++ b/Sources/Plugins/NetworkVmnet/NetworkVmnetHelper+Start.swift @@ -24,6 +24,8 @@ import ContainerResource import ContainerXPC import ContainerizationError import ContainerizationExtras +import ContainerizationOS +import Darwin import Foundation import Logging @@ -109,6 +111,19 @@ extension NetworkVmnetHelper { log: log ) + // What the network holds is given back when this helper is + // asked to go away, so the addresses it was given can be + // handed out again; a helper that exits still holding them + // leaves the range spoken for by nobody. + let signals = AsyncSignalHandler.create(notify: [SIGINT, SIGTERM]) + Task { + for await _ in signals.signals { + log.info("releasing the network before exit") + await network.stop() + Darwin.exit(0) + } + } + log.info("starting XPC server") try await xpc.listen() } catch { diff --git a/Sources/Services/ContainerAPIService/Server/Networks/NetworksService.swift b/Sources/Services/ContainerAPIService/Server/Networks/NetworksService.swift index 3827a32c0..22eff35e2 100644 --- a/Sources/Services/ContainerAPIService/Server/Networks/NetworksService.swift +++ b/Sources/Services/ContainerAPIService/Server/Networks/NetworksService.swift @@ -106,7 +106,33 @@ public actor NetworksService { // 5 seconds or considerably more from the registration of this first // network service to its execution. do { - try await registerService(configuration: effectiveConfiguration) + do { + try await registerService(configuration: effectiveConfiguration) + } catch where effectiveConfiguration.id == NetworkClient.defaultNetworkName && effectiveConfiguration.ipv4Subnet != nil { + // The range the default network asks for is the one it was + // given last time, which is a preference and not a demand: + // a range held by something else, or reserved to a network + // nobody holds any more, would otherwise leave the system + // with no network at all and every call that needs one + // waiting forever. The network comes up on whatever range + // is free and says which range it lost. + log.error( + "the default network could not take the address range it was given; taking another", + metadata: [ + "range": "\(effectiveConfiguration.ipv4Subnet?.description ?? "")", + "error": "\(error)", + ]) + effectiveConfiguration = try NetworkConfiguration( + name: effectiveConfiguration.name, + mode: effectiveConfiguration.mode, + ipv4Subnet: nil, + ipv6Subnet: effectiveConfiguration.ipv6Subnet, + labels: effectiveConfiguration.labels, + plugin: effectiveConfiguration.plugin, + options: effectiveConfiguration.options + ) + try await registerService(configuration: effectiveConfiguration) + } let client = try Self.getClient(configuration: effectiveConfiguration) let networkStatus = try await client.status() diff --git a/Sources/Services/Network/Client/NetworkClient.swift b/Sources/Services/Network/Client/NetworkClient.swift index 97598a786..37fe8d01b 100644 --- a/Sources/Services/Network/Client/NetworkClient.swift +++ b/Sources/Services/Network/Client/NetworkClient.swift @@ -53,11 +53,19 @@ extension NetworkClient { createClient().openSession() } + /// How long a call waits on a network helper before it is answered. + /// + /// A helper that never comes up answers nothing, and a call with no bound + /// waits on it forever: every command that needs a network hangs with no + /// error to show for it. The wait is generous enough for a helper still + /// starting and short enough that its absence is reported. + static let callTimeout: Duration = .seconds(30) + public func status() async throws -> NetworkStatus { let request = XPCMessage(route: NetworkRoutes.status.rawValue) let client = createClient() - let response = try await client.send(request) + let response = try await client.send(request, responseTimeout: Self.callTimeout) let status = try response.status() return status } @@ -77,7 +85,7 @@ extension NetworkClient { if let macAddress = macAddress { request.set(key: NetworkKeys.macAddress.rawValue, value: macAddress.description) } - let response = try await session.send(request) + let response = try await session.send(request, responseTimeout: Self.callTimeout) let attachment = try response.attachment() let additionalData = response.additionalData() return (attachment, additionalData) @@ -89,7 +97,7 @@ extension NetworkClient { let client = createClient() - let response = try await client.send(request) + let response = try await client.send(request, responseTimeout: Self.callTimeout) return try response.dataNoCopy(key: NetworkKeys.attachment.rawValue).map { try JSONDecoder().decode(Attachment.self, from: $0) } diff --git a/Sources/Services/Network/Server/Network.swift b/Sources/Services/Network/Server/Network.swift index 6f1ff18e1..1e5db1bd1 100644 --- a/Sources/Services/Network/Server/Network.swift +++ b/Sources/Services/Network/Server/Network.swift @@ -36,4 +36,10 @@ public protocol Network: Sendable { /// Start the network. func start() async throws + + /// Give up whatever the network holds, so the addresses it was given can + /// be handed out again. A network that goes away without this leaves its + /// range spoken for by nobody, and the next network asking for that range + /// is refused. + func stop() async } diff --git a/Sources/Services/NetworkVmnet/Server/AllocationOnlyVmnetNetwork.swift b/Sources/Services/NetworkVmnet/Server/AllocationOnlyVmnetNetwork.swift index 131e8af28..91361505c 100644 --- a/Sources/Services/NetworkVmnet/Server/AllocationOnlyVmnetNetwork.swift +++ b/Sources/Services/NetworkVmnet/Server/AllocationOnlyVmnetNetwork.swift @@ -58,6 +58,12 @@ public actor AllocationOnlyVmnetNetwork: Network { try handler(nil) } + /// The addresses this network hands out are its own bookkeeping, held + /// nowhere outside this process, so giving them up is forgetting them. + public func stop() async { + self._status = nil + } + public func start() async throws { guard _status == nil else { throw ContainerizationError(.invalidState, message: "cannot start network \(configuration.id): already started") diff --git a/Sources/Services/NetworkVmnet/Server/ReservedVmnetNetwork.swift b/Sources/Services/NetworkVmnet/Server/ReservedVmnetNetwork.swift index 5b0fee6ad..5d861c6ce 100644 --- a/Sources/Services/NetworkVmnet/Server/ReservedVmnetNetwork.swift +++ b/Sources/Services/NetworkVmnet/Server/ReservedVmnetNetwork.swift @@ -92,6 +92,27 @@ public final class ReservedVmnetNetwork: ContainerNetworkServer.Network { } } + /// The reservation lives as long as the network object the framework + /// hands back, and the object is returned retained, so releasing it is + /// what gives the address range back. A helper that goes away without + /// releasing leaves the range reserved to a network nobody holds, and + /// every later attempt on that range is refused with no interface, route, + /// or process to point at. + /// vmnet.h, vmnet_network_create: "The lifetime of such reservation is + /// the same as that of `vmnet_network_ref`. Use `CFRelease()` to release + /// the network object." + public func stop() async { + stateMutex.withLock { state in + guard let network = state.network else { + return + } + CFRelease(unsafeBitCast(network, to: CFTypeRef.self)) + state.network = nil + state.status = nil + log.info("released vmnet network", metadata: ["id": "\(configuration.id)"]) + } + } + private static func serialize_network_ref(ref: vmnet_network_ref) throws -> XPCMessage { var status: vmnet_return_t = .VMNET_SUCCESS guard let refObject = vmnet_network_copy_serialization(ref, &status) else { From fa6e404a042176fde3f663e1da4332f8e2a99816 Mon Sep 17 00:00:00 2001 From: Aaron Paterson Date: Tue, 18 Aug 2026 21:31:54 +0000 Subject: [PATCH 20/29] Balance the retain the framework hands back The network arrives as a plain pointer rather than a managed object, so the retain that comes with it is this side's to balance when the reservation is given up. --- .../Services/NetworkVmnet/Server/ReservedVmnetNetwork.swift | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/Sources/Services/NetworkVmnet/Server/ReservedVmnetNetwork.swift b/Sources/Services/NetworkVmnet/Server/ReservedVmnetNetwork.swift index 5d861c6ce..4bf18a8d9 100644 --- a/Sources/Services/NetworkVmnet/Server/ReservedVmnetNetwork.swift +++ b/Sources/Services/NetworkVmnet/Server/ReservedVmnetNetwork.swift @@ -106,7 +106,10 @@ public final class ReservedVmnetNetwork: ContainerNetworkServer.Network { guard let network = state.network else { return } - CFRelease(unsafeBitCast(network, to: CFTypeRef.self)) + // The framework hands the network back retained, and it arrives as + // a plain pointer rather than a managed object, so the retain is + // this side's to balance. + Unmanaged.fromOpaque(UnsafeRawPointer(network)).release() state.network = nil state.status = nil log.info("released vmnet network", metadata: ["id": "\(configuration.id)"]) From 00f10d7359753cd0f7023372fdd034566d8541d2 Mon Sep 17 00:00:00 2001 From: Aaron Paterson Date: Tue, 18 Aug 2026 21:38:23 +0000 Subject: [PATCH 21/29] Hold the machine's capabilities to the pod that runs it Running a foreign architecture and exposing nested virtualization are the machine's to do, and the machine is the pod's, booted with or without them before a container joins. Asking for either as a container joining a pod was accepted and ignored, so a container that needed one started in a machine that could not give it. Both are refused the way the network options already are, and an image of a foreign architecture joining a pod whose machine cannot translate says so instead of failing where the reason is no longer visible. https://github.com/kubernetes/cri-api/blob/master/pkg/apis/runtime/v1/api.proto Calls to a network helper take the timeout the calls to the api server take, so a caller that wants to wait as long as it takes still can. --- .../ContainerAPIService/Client/Utility.swift | 17 +++++++++++-- .../Network/Client/NetworkClient.swift | 25 ++++++++----------- 2 files changed, 26 insertions(+), 16 deletions(-) diff --git a/Sources/Services/ContainerAPIService/Client/Utility.swift b/Sources/Services/ContainerAPIService/Client/Utility.swift index edee04a9f..4c4a6af76 100644 --- a/Sources/Services/ContainerAPIService/Client/Utility.swift +++ b/Sources/Services/ContainerAPIService/Client/Utility.swift @@ -271,10 +271,12 @@ public struct Utility { // attachment, so --hostname is covered by --network. // https://github.com/containerd/nerdctl/blob/main/pkg/containerutil/container_network_manager.go if let named = management.pod { - guard (try? await ClientPod.inspect(named)) != nil else { + guard let pod = try? await ClientPod.inspect(named) else { throw ContainerizationError(.notFound, message: "pod \(named) does not exist") } var held: [String] = [] + if management.rosetta { held.append("--rosetta") } + if management.virtualization { held.append("--virtualization") } if !management.publishPorts.isEmpty { held.append("-p/--publish") } if !management.dns.nameservers.isEmpty || management.dns.domain != nil || !management.dns.searchDomains.isEmpty || !management.dns.options.isEmpty @@ -286,7 +288,18 @@ public struct Utility { throw ContainerizationError( .invalidArgument, message: - "these belong to the pod whose network the container joins, so they are not the container's to ask for: \(held.joined(separator: ", "))" + "these belong to the pod whose machine the container runs in, so they are not the container's to ask for: \(held.joined(separator: ", "))" + ) + } + + // Running a foreign architecture is the machine's to do, and the + // machine was booted with or without it before this container + // existed, so a container that needs it says so rather than + // starting in a machine that cannot run it. + if Platform.current.architecture == "arm64", requestedPlatform.architecture == "amd64", !pod.configuration.rosetta { + throw ContainerizationError( + .invalidArgument, + message: "pod \(named) runs a machine without Rosetta, which \(requestedPlatform.description) needs; create the pod with --rosetta" ) } } diff --git a/Sources/Services/Network/Client/NetworkClient.swift b/Sources/Services/Network/Client/NetworkClient.swift index 37fe8d01b..0a81f5b19 100644 --- a/Sources/Services/Network/Client/NetworkClient.swift +++ b/Sources/Services/Network/Client/NetworkClient.swift @@ -53,19 +53,15 @@ extension NetworkClient { createClient().openSession() } - /// How long a call waits on a network helper before it is answered. - /// - /// A helper that never comes up answers nothing, and a call with no bound - /// waits on it forever: every command that needs a network hangs with no - /// error to show for it. The wait is generous enough for a helper still - /// starting and short enough that its absence is reported. - static let callTimeout: Duration = .seconds(30) - - public func status() async throws -> NetworkStatus { + /// A helper that never comes up answers nothing, and a call waiting on it + /// with no bound takes every command that needs a network down with it, so + /// the wait is bounded the way the calls to the api server are. Passing no + /// timeout waits as long as it takes. + public func status(timeout: Duration? = XPCClient.xpcRegistrationTimeout) async throws -> NetworkStatus { let request = XPCMessage(route: NetworkRoutes.status.rawValue) let client = createClient() - let response = try await client.send(request, responseTimeout: Self.callTimeout) + let response = try await client.send(request, responseTimeout: timeout) let status = try response.status() return status } @@ -78,26 +74,27 @@ extension NetworkClient { public func allocate( hostname: String, macAddress: MACAddress? = nil, - on session: XPCClientSession + on session: XPCClientSession, + timeout: Duration? = XPCClient.xpcRegistrationTimeout ) async throws -> (attachment: Attachment, additionalData: XPCMessage?) { let request = XPCMessage(route: NetworkRoutes.allocate.rawValue) request.set(key: NetworkKeys.hostname.rawValue, value: hostname) if let macAddress = macAddress { request.set(key: NetworkKeys.macAddress.rawValue, value: macAddress.description) } - let response = try await session.send(request, responseTimeout: Self.callTimeout) + let response = try await session.send(request, responseTimeout: timeout) let attachment = try response.attachment() let additionalData = response.additionalData() return (attachment, additionalData) } - public func lookup(hostname: String) async throws -> Attachment? { + public func lookup(hostname: String, timeout: Duration? = XPCClient.xpcRegistrationTimeout) async throws -> Attachment? { let request = XPCMessage(route: NetworkRoutes.lookup.rawValue) request.set(key: NetworkKeys.hostname.rawValue, value: hostname) let client = createClient() - let response = try await client.send(request, responseTimeout: Self.callTimeout) + let response = try await client.send(request, responseTimeout: timeout) return try response.dataNoCopy(key: NetworkKeys.attachment.rawValue).map { try JSONDecoder().decode(Attachment.self, from: $0) } From 5a451e4d8109ebd1b5e7ab74d4cbf5ef6d31c6c6 Mon Sep 17 00:00:00 2001 From: Aaron Paterson Date: Tue, 18 Aug 2026 21:51:30 +0000 Subject: [PATCH 22/29] Leave kernel parameters where the kernel is Containers in a pod share one kernel, so a kernel parameter is the machine's and the runtime interface carries it on the sandbox alone. A container carried a field for them that nothing wrote and nothing read, copied into the sandbox made for a container that named no pod, which described a setting a container could never have. https://github.com/kubernetes/cri-api/blob/master/pkg/apis/runtime/v1/api.proto --- .../ContainerResource/Container/ContainerConfiguration.swift | 4 ---- Sources/ContainerResource/Pod/PodConfiguration.swift | 1 - 2 files changed, 5 deletions(-) diff --git a/Sources/ContainerResource/Container/ContainerConfiguration.swift b/Sources/ContainerResource/Container/ContainerConfiguration.swift index 8cbdc88fa..09975c55f 100644 --- a/Sources/ContainerResource/Container/ContainerConfiguration.swift +++ b/Sources/ContainerResource/Container/ContainerConfiguration.swift @@ -30,8 +30,6 @@ public struct ContainerConfiguration: Sendable, Codable { public var publishedSockets: [PublishSocket] = [] /// Key/Value labels for the container. public var labels: [String: String] = [:] - /// System controls for the container. - public var sysctls: [String: String] = [:] /// The networks the container will be added to. public var networks: [AttachmentConfiguration] = [] /// The DNS configuration for the container. @@ -89,7 +87,6 @@ public struct ContainerConfiguration: Sendable, Codable { case publishedPorts case publishedSockets case labels - case sysctls case networks case dns case rosetta @@ -122,7 +119,6 @@ public struct ContainerConfiguration: Sendable, Codable { publishedPorts = try container.decodeIfPresent([PublishPort].self, forKey: .publishedPorts) ?? [] publishedSockets = try container.decodeIfPresent([PublishSocket].self, forKey: .publishedSockets) ?? [] labels = try container.decodeIfPresent([String: String].self, forKey: .labels) ?? [:] - sysctls = try container.decodeIfPresent([String: String].self, forKey: .sysctls) ?? [:] if container.contains(.networks) { networks = try container.decode([AttachmentConfiguration].self, forKey: .networks) diff --git a/Sources/ContainerResource/Pod/PodConfiguration.swift b/Sources/ContainerResource/Pod/PodConfiguration.swift index 1314c7713..8c735d6b8 100644 --- a/Sources/ContainerResource/Pod/PodConfiguration.swift +++ b/Sources/ContainerResource/Pod/PodConfiguration.swift @@ -93,7 +93,6 @@ public struct PodConfiguration: Sendable, Codable { runtimeHandler = container.runtimeHandler resources = container.resources dns = container.dns - sysctls = container.sysctls networks = container.networks publishedPorts = container.publishedPorts virtualization = container.virtualization From 37d7783a50c5dc6bed3770eb968e71f0832e0245 Mon Sep 17 00:00:00 2001 From: Aaron Paterson Date: Tue, 18 Aug 2026 22:52:41 +0000 Subject: [PATCH 23/29] Give a host back the address and hardware address it had An address is what a host is known by on its network: resolver entries, hosts files, and the caches on the other side of it all name it, and an IPv6 address is derived from the hardware address, so a host that comes back with new ones comes back as somebody else. Both were minted afresh every time, the addresses out of a rotating allocator whose record of who had what lived only as long as the process, so a restart shuffled them among the hosts that came back. What a host was given is written down beside the network that gave it, and a host attaching again is given it back when it is still free, falling to the next free address when it is not. A host-local allocator keeps its allocations the same way, under a directory of its own. https://cni.dev/plugins/current/ipam/host-local/ --- .../NetworkVmnetHelper+Start.swift | 13 ++- .../Network/Server/AttachmentAllocator.swift | 87 ++++++++++++++++--- .../Server/DefaultNetworkService.swift | 19 +++- .../AttachmentAllocatorTest.swift | 75 +++++++++++----- 4 files changed, 153 insertions(+), 41 deletions(-) diff --git a/Sources/Plugins/NetworkVmnet/NetworkVmnetHelper+Start.swift b/Sources/Plugins/NetworkVmnet/NetworkVmnetHelper+Start.swift index a2458100f..4c81a49d4 100644 --- a/Sources/Plugins/NetworkVmnet/NetworkVmnetHelper+Start.swift +++ b/Sources/Plugins/NetworkVmnet/NetworkVmnetHelper+Start.swift @@ -19,6 +19,7 @@ import ContainerLog import ContainerNetworkClient import ContainerNetworkServer import ContainerNetworkVmnetServer +import ContainerPersistence import ContainerPlugin import ContainerResource import ContainerXPC @@ -99,7 +100,17 @@ extension NetworkVmnetHelper { log: log ) try await network.start() - let service = try await DefaultNetworkService(network: network, log: log) + // The addresses this network hands out are written down beside + // the network they belong to, the way a host-local allocator + // keeps its allocations under a directory of its own. + // https://cni.dev/plugins/current/ipam/host-local/ + let leases = URL( + filePath: PathUtils.BaseConfigPath.appRoot.basePath() + .appending("networks") + .appending(id) + .appending("leases.json") + .string) + let service = try await DefaultNetworkService(network: network, leases: leases, log: log) let harness = NetworkHarness(service: service) let xpc = XPCServer( identifier: serviceIdentifier, diff --git a/Sources/Services/Network/Server/AttachmentAllocator.swift b/Sources/Services/Network/Server/AttachmentAllocator.swift index b7d3aeebb..b903b3971 100644 --- a/Sources/Services/Network/Server/AttachmentAllocator.swift +++ b/Sources/Services/Network/Server/AttachmentAllocator.swift @@ -16,44 +16,103 @@ import ContainerizationError import ContainerizationExtras +import Foundation +import Logging actor AttachmentAllocator { + /// What a host was given the last time it asked, kept so that asking + /// again gets the same answer. + struct Lease: Codable, Sendable { + let index: UInt32 + let macAddress: MACAddress + } + private let allocator: any AddressAllocator - private var hostnames: [String: UInt32] = [:] + private var allocated: [String: Lease] = [:] + private var leases: [String: Lease] = [:] + private let store: URL? + private let log: Logger? - init(lower: UInt32, size: Int) throws { + /// - Parameters: + /// - store: where the leases are written, so a host keeps its address + /// and hardware address across restarts of this service. Nothing is + /// remembered without one. + init(lower: UInt32, size: Int, store: URL? = nil, log: Logger? = nil) throws { allocator = try UInt32.rotatingAllocator( lower: lower, size: UInt32(size) ) + self.store = store + self.log = log + self.leases = Self.read(store: store, log: log) } /// Allocate a network address for a host. - func allocate(hostname: String) async throws -> UInt32 { - // Client is responsible for ensuring two containers don't use same hostname, so provide existing IP if hostname exists - if let index = hostnames[hostname] { - return index + /// + /// A host that is already attached keeps what it has. One that attached + /// before is given what it had, when that address is still free: an + /// address a host answers to outlives the attachment, since the resolver + /// entries, the hosts files, and the caches on the other side of the + /// network all name it. Otherwise the next free address is taken, and + /// what the host was given is written down. + /// https://cni.dev/plugins/current/ipam/host-local/ + func allocate(hostname: String, macAddress: MACAddress) async throws -> Lease { + if let held = allocated[hostname] { + return held } - let index = try allocator.allocate() - hostnames[hostname] = index + if let remembered = leases[hostname], (try? allocator.reserve(remembered.index)) != nil { + allocated[hostname] = remembered + return remembered + } - return index + let lease = Lease(index: try allocator.allocate(), macAddress: macAddress) + allocated[hostname] = lease + leases[hostname] = lease + write() + return lease } - /// Free an allocated network address by hostname. + /// Free an allocated network address by hostname. What the host was given + /// is still remembered, so the same host attaching again is given it back. @discardableResult func deallocate(hostname: String) async throws -> UInt32? { - guard let index = hostnames.removeValue(forKey: hostname) else { + guard let lease = allocated.removeValue(forKey: hostname) else { return nil } - try allocator.release(index) - return index + try allocator.release(lease.index) + return lease.index } /// Retrieve the allocator index for a hostname. func lookup(hostname: String) async throws -> UInt32? { - hostnames[hostname] + allocated[hostname]?.index + } + + private static func read(store: URL?, log: Logger?) -> [String: Lease] { + guard let store, let data = FileManager.default.contents(atPath: store.path) else { + return [:] + } + do { + return try JSONDecoder().decode([String: Lease].self, from: data) + } catch { + // What was written down is a convenience, so a file that cannot be + // read costs the addresses their stability and nothing else. + log?.warning("cannot read the addresses given out before", metadata: ["path": "\(store.path)", "error": "\(error)"]) + return [:] + } + } + + private func write() { + guard let store else { + return + } + do { + try FileManager.default.createDirectory(at: store.deletingLastPathComponent(), withIntermediateDirectories: true) + try JSONEncoder().encode(leases).write(to: store, options: .atomic) + } catch { + log?.warning("cannot write down the addresses given out", metadata: ["path": "\(store.path)", "error": "\(error)"]) + } } } diff --git a/Sources/Services/Network/Server/DefaultNetworkService.swift b/Sources/Services/Network/Server/DefaultNetworkService.swift index 70d17d396..de5f4f93f 100644 --- a/Sources/Services/Network/Server/DefaultNetworkService.swift +++ b/Sources/Services/Network/Server/DefaultNetworkService.swift @@ -18,6 +18,7 @@ import ContainerResource import ContainerXPC import ContainerizationError import ContainerizationExtras +import Foundation import Logging public actor DefaultNetworkService: NetworkService { @@ -28,8 +29,13 @@ public actor DefaultNetworkService: NetworkService { private var allocationsBySession: [XPCServerSession: [(hostname: String, index: UInt32)]] /// Set up a network service for the specified network. + /// - Parameters: + /// - leases: where the addresses handed out are written down, so a host + /// attaching again is given what it had. Nothing is remembered without + /// one, which is what a caller wanting a network that forgets passes. public init( network: any Network, + leases: URL? = nil, log: Logger ) async throws { guard let status = await network.status else { @@ -40,7 +46,7 @@ public actor DefaultNetworkService: NetworkService { let size = Int(subnet.upper.value - subnet.lower.value - 3) self.network = network self.log = log - self.allocator = try AttachmentAllocator(lower: subnet.lower.value + 2, size: size) + self.allocator = try AttachmentAllocator(lower: subnet.lower.value + 2, size: size, store: leases, log: log) self.macAddresses = [:] self.allocationsBySession = [:] } @@ -66,8 +72,15 @@ public actor DefaultNetworkService: NetworkService { throw ContainerizationError(.invalidState, message: "network \(network.id) must be running") } - let macAddress = macAddress ?? MACAddress((UInt64.random(in: 0...UInt64.max) & 0x0cff_ffff_ffff) | 0xf200_0000_0000) - let index = try await allocator.allocate(hostname: hostname) + // A hardware address is minted for a host that names none, and the one + // a host was given before is handed back when it attaches again: the + // address is what the network knows a host by, and an IPv6 address is + // derived from it, so a host that comes back with a new one comes back + // as somebody else. + let requested = macAddress ?? MACAddress((UInt64.random(in: 0...UInt64.max) & 0x0cff_ffff_ffff) | 0xf200_0000_0000) + let lease = try await allocator.allocate(hostname: hostname, macAddress: requested) + let macAddress = macAddress ?? lease.macAddress + let index = lease.index let ipv6Address = try status.ipv6Subnet .map { try CIDRv6(macAddress.ipv6Address(network: $0.lower), prefix: $0.prefix) } let ip = IPv4Address(index) diff --git a/Tests/ContainerNetworkServerTests/AttachmentAllocatorTest.swift b/Tests/ContainerNetworkServerTests/AttachmentAllocatorTest.swift index 86ea3eff0..eefb13b9f 100644 --- a/Tests/ContainerNetworkServerTests/AttachmentAllocatorTest.swift +++ b/Tests/ContainerNetworkServerTests/AttachmentAllocatorTest.swift @@ -14,15 +14,27 @@ // limitations under the License. //===----------------------------------------------------------------------===// +import ContainerizationExtras +import Foundation import Testing @testable import ContainerNetworkServer struct AttachmentAllocatorTest { + /// A hardware address to attach with, different for each host so that a + /// remembered one is recognizable. + private func mac(_ last: UInt8) -> MACAddress { + MACAddress(0xf200_0000_0000 | UInt64(last)) + } + + private func allocate(_ allocator: AttachmentAllocator, _ hostname: String, _ last: UInt8 = 1) async throws -> UInt32 { + try await allocator.allocate(hostname: hostname, macAddress: mac(last)).index + } + @Test func testAllocateSingleHostname() async throws { let allocator = try AttachmentAllocator(lower: 100, size: 10) - let address = try await allocator.allocate(hostname: "test-host") + let address = try await allocate(allocator, "test-host") #expect(address >= 100) #expect(address < 110) @@ -31,8 +43,8 @@ struct AttachmentAllocatorTest { @Test func testAllocateSameHostnameTwice() async throws { let allocator = try AttachmentAllocator(lower: 100, size: 10) - let address1 = try await allocator.allocate(hostname: "test-host") - let address2 = try await allocator.allocate(hostname: "test-host") + let address1 = try await allocate(allocator, "test-host") + let address2 = try await allocate(allocator, "test-host") #expect(address1 == address2) } @@ -40,9 +52,9 @@ struct AttachmentAllocatorTest { @Test func testAllocateMultipleHostnames() async throws { let allocator = try AttachmentAllocator(lower: 100, size: 10) - let address1 = try await allocator.allocate(hostname: "host1") - let address2 = try await allocator.allocate(hostname: "host2") - let address3 = try await allocator.allocate(hostname: "host3") + let address1 = try await allocate(allocator, "host1") + let address2 = try await allocate(allocator, "host2") + let address3 = try await allocate(allocator, "host3") #expect(address1 != address2) #expect(address2 != address3) @@ -52,7 +64,7 @@ struct AttachmentAllocatorTest { @Test func testLookupAllocatedHostname() async throws { let allocator = try AttachmentAllocator(lower: 100, size: 10) - let allocatedAddress = try await allocator.allocate(hostname: "test-host") + let allocatedAddress = try await allocate(allocator, "test-host") let lookedUpAddress = try await allocator.lookup(hostname: "test-host") #expect(lookedUpAddress == allocatedAddress) @@ -69,7 +81,7 @@ struct AttachmentAllocatorTest { @Test func testDeallocateAllocatedHostname() async throws { let allocator = try AttachmentAllocator(lower: 100, size: 10) - let allocatedAddress = try await allocator.allocate(hostname: "test-host") + let allocatedAddress = try await allocate(allocator, "test-host") let deallocatedAddress = try await allocator.deallocate(hostname: "test-host") #expect(deallocatedAddress == allocatedAddress) @@ -87,17 +99,34 @@ struct AttachmentAllocatorTest { #expect(deallocatedAddress == nil) } - @Test func testReallocateAfterDeallocation() async throws { + @Test func testHostAttachingAgainIsGivenWhatItHad() async throws { let allocator = try AttachmentAllocator(lower: 100, size: 10) - let address1 = try await allocator.allocate(hostname: "test-host") - let released1 = try await allocator.deallocate(hostname: "test-host") - #expect(address1 == released1) - let address2 = try await allocator.allocate(hostname: "test-host") + let first = try await allocator.allocate(hostname: "test-host", macAddress: mac(7)) + _ = try await allocator.deallocate(hostname: "test-host") + // Another host takes an address in between, so the answer is the one + // remembered rather than the one next in line. + _ = try await allocate(allocator, "other-host", 8) + let again = try await allocator.allocate(hostname: "test-host", macAddress: mac(9)) + + #expect(again.index == first.index) + #expect(again.macAddress == first.macAddress) + } + + @Test func testWhatAHostWasGivenOutlivesTheAllocator() async throws { + let store = FileManager.default.temporaryDirectory + .appendingPathComponent(UUID().uuidString) + .appendingPathComponent("leases.json") + defer { try? FileManager.default.removeItem(at: store.deletingLastPathComponent()) } + + let first = try await AttachmentAllocator(lower: 100, size: 10, store: store) + .allocate(hostname: "test-host", macAddress: mac(3)) + + let second = try await AttachmentAllocator(lower: 100, size: 10, store: store) + .allocate(hostname: "test-host", macAddress: mac(4)) - // After deallocation, allocating the same hostname should give a new address - #expect(address2 >= 100) - #expect(address2 < 110) + #expect(second.index == first.index) + #expect(second.macAddress == first.macAddress) } @Test func testAllocateUntilFull() async throws { @@ -106,12 +135,12 @@ struct AttachmentAllocatorTest { // Allocate up to the limit for i in 0..= 100) #expect(newAddress < 103) @@ -146,7 +175,7 @@ struct AttachmentAllocatorTest { @Test func testMultipleDeallocationsOfSameHostname() async throws { let allocator = try AttachmentAllocator(lower: 100, size: 10) - let address = try await allocator.allocate(hostname: "test-host") + let address = try await allocate(allocator, "test-host") let firstDeallocate = try await allocator.deallocate(hostname: "test-host") #expect(firstDeallocate == address) From 7ea79bc236af123f62213b51a84599f56352d7c5 Mon Sep 17 00:00:00 2001 From: Aaron Paterson Date: Tue, 18 Aug 2026 23:06:24 +0000 Subject: [PATCH 24/29] Keep what a host was given where the plugin keeps its state A network hands out addresses; where what it handed out is written down is the plugin's to say, since it knows where its own state belongs. The allocator asks a store for what was given before and tells it what it gives, and the vmnet plugin keeps those under the network's own directory in the entity store the rest of the plugin state uses, one lease to an entry, the way a host-local allocator keeps its allocations. https://cni.dev/plugins/current/ipam/host-local/ --- .../FilesystemAttachmentLeaseStore.swift | 45 +++++++++++++ .../NetworkVmnetHelper+Start.swift | 17 +++-- .../Network/Server/AttachmentAllocator.swift | 63 ++++++++----------- .../Network/Server/AttachmentLeaseStore.swift | 29 +++++++++ .../Server/DefaultNetworkService.swift | 10 +-- .../AttachmentAllocatorTest.swift | 41 +++++++----- 6 files changed, 141 insertions(+), 64 deletions(-) create mode 100644 Sources/Plugins/NetworkVmnet/FilesystemAttachmentLeaseStore.swift create mode 100644 Sources/Services/Network/Server/AttachmentLeaseStore.swift diff --git a/Sources/Plugins/NetworkVmnet/FilesystemAttachmentLeaseStore.swift b/Sources/Plugins/NetworkVmnet/FilesystemAttachmentLeaseStore.swift new file mode 100644 index 000000000..619d489c1 --- /dev/null +++ b/Sources/Plugins/NetworkVmnet/FilesystemAttachmentLeaseStore.swift @@ -0,0 +1,45 @@ +//===----------------------------------------------------------------------===// +// Copyright © 2026 Apple Inc. and the container project authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//===----------------------------------------------------------------------===// + +import ContainerNetworkServer +import ContainerPersistence +import Logging + +/// Keeps what each host was given on disk, one lease to an entry, under the +/// directory belonging to the network that gave it. +struct FilesystemAttachmentLeaseStore: AttachmentLeaseStore { + let store: FilesystemEntityStore + let log: Logger + + func load() async -> [AttachmentAllocator.Lease] { + do { + return try await store.list() + } catch { + log.warning("cannot read what hosts were given", metadata: ["error": "\(error)"]) + return [] + } + } + + func save(_ lease: AttachmentAllocator.Lease) async { + do { + try await store.upsert(lease) + } catch { + log.warning( + "cannot write down what a host was given", + metadata: ["hostname": "\(lease.id)", "error": "\(error)"]) + } + } +} diff --git a/Sources/Plugins/NetworkVmnet/NetworkVmnetHelper+Start.swift b/Sources/Plugins/NetworkVmnet/NetworkVmnetHelper+Start.swift index 4c81a49d4..74f0a02da 100644 --- a/Sources/Plugins/NetworkVmnet/NetworkVmnetHelper+Start.swift +++ b/Sources/Plugins/NetworkVmnet/NetworkVmnetHelper+Start.swift @@ -104,12 +104,17 @@ extension NetworkVmnetHelper { // the network they belong to, the way a host-local allocator // keeps its allocations under a directory of its own. // https://cni.dev/plugins/current/ipam/host-local/ - let leases = URL( - filePath: PathUtils.BaseConfigPath.appRoot.basePath() - .appending("networks") - .appending(id) - .appending("leases.json") - .string) + let leases = FilesystemAttachmentLeaseStore( + store: try FilesystemEntityStore( + path: PathUtils.BaseConfigPath.appRoot.basePath() + .appending("networks") + .appending(id) + .appending("leases"), + type: "lease", + log: log + ), + log: log + ) let service = try await DefaultNetworkService(network: network, leases: leases, log: log) let harness = NetworkHarness(service: service) let xpc = XPCServer( diff --git a/Sources/Services/Network/Server/AttachmentAllocator.swift b/Sources/Services/Network/Server/AttachmentAllocator.swift index b903b3971..9fd7aa351 100644 --- a/Sources/Services/Network/Server/AttachmentAllocator.swift +++ b/Sources/Services/Network/Server/AttachmentAllocator.swift @@ -19,32 +19,45 @@ import ContainerizationExtras import Foundation import Logging -actor AttachmentAllocator { +public actor AttachmentAllocator { /// What a host was given the last time it asked, kept so that asking - /// again gets the same answer. - struct Lease: Codable, Sendable { - let index: UInt32 - let macAddress: MACAddress + /// again gets the same answer. The host's name identifies it, the way an + /// allocation is kept under the address it holds. + public struct Lease: Codable, Sendable, Identifiable { + public let id: String + public let index: UInt32 + public let macAddress: MACAddress + + public init(id: String, index: UInt32, macAddress: MACAddress) { + self.id = id + self.index = index + self.macAddress = macAddress + } } private let allocator: any AddressAllocator private var allocated: [String: Lease] = [:] private var leases: [String: Lease] = [:] - private let store: URL? + private let store: (any AttachmentLeaseStore)? private let log: Logger? /// - Parameters: - /// - store: where the leases are written, so a host keeps its address - /// and hardware address across restarts of this service. Nothing is + /// - store: keeps what each host was given, so a host attaching again is + /// given it back across restarts of this service. Nothing is /// remembered without one. - init(lower: UInt32, size: Int, store: URL? = nil, log: Logger? = nil) throws { + init(lower: UInt32, size: Int, store: (any AttachmentLeaseStore)? = nil, log: Logger? = nil) async throws { allocator = try UInt32.rotatingAllocator( lower: lower, size: UInt32(size) ) self.store = store self.log = log - self.leases = Self.read(store: store, log: log) + if let store { + // What was written down is a convenience, so leases that cannot be + // read cost the addresses their stability and nothing else. + let written = await store.load() + self.leases = Dictionary(uniqueKeysWithValues: written.map { ($0.id, $0) }) + } } /// Allocate a network address for a host. @@ -66,10 +79,10 @@ actor AttachmentAllocator { return remembered } - let lease = Lease(index: try allocator.allocate(), macAddress: macAddress) + let lease = Lease(id: hostname, index: try allocator.allocate(), macAddress: macAddress) allocated[hostname] = lease leases[hostname] = lease - write() + await store?.save(lease) return lease } @@ -89,30 +102,4 @@ actor AttachmentAllocator { func lookup(hostname: String) async throws -> UInt32? { allocated[hostname]?.index } - - private static func read(store: URL?, log: Logger?) -> [String: Lease] { - guard let store, let data = FileManager.default.contents(atPath: store.path) else { - return [:] - } - do { - return try JSONDecoder().decode([String: Lease].self, from: data) - } catch { - // What was written down is a convenience, so a file that cannot be - // read costs the addresses their stability and nothing else. - log?.warning("cannot read the addresses given out before", metadata: ["path": "\(store.path)", "error": "\(error)"]) - return [:] - } - } - - private func write() { - guard let store else { - return - } - do { - try FileManager.default.createDirectory(at: store.deletingLastPathComponent(), withIntermediateDirectories: true) - try JSONEncoder().encode(leases).write(to: store, options: .atomic) - } catch { - log?.warning("cannot write down the addresses given out", metadata: ["path": "\(store.path)", "error": "\(error)"]) - } - } } diff --git a/Sources/Services/Network/Server/AttachmentLeaseStore.swift b/Sources/Services/Network/Server/AttachmentLeaseStore.swift new file mode 100644 index 000000000..d564b5f26 --- /dev/null +++ b/Sources/Services/Network/Server/AttachmentLeaseStore.swift @@ -0,0 +1,29 @@ +//===----------------------------------------------------------------------===// +// Copyright © 2026 Apple Inc. and the container project authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//===----------------------------------------------------------------------===// + +/// Keeps what each host was given, so a host attaching again is given it back +/// after this service has been restarted. +/// +/// What a lease is written to is the caller's, since a network plugin knows +/// where its own state belongs and this module holds none. +public protocol AttachmentLeaseStore: Sendable { + /// Everything written down so far. A store that cannot be read answers + /// with nothing, which costs the addresses their stability and no more. + func load() async -> [AttachmentAllocator.Lease] + + /// Write down what a host was given. + func save(_ lease: AttachmentAllocator.Lease) async +} diff --git a/Sources/Services/Network/Server/DefaultNetworkService.swift b/Sources/Services/Network/Server/DefaultNetworkService.swift index de5f4f93f..f799d7247 100644 --- a/Sources/Services/Network/Server/DefaultNetworkService.swift +++ b/Sources/Services/Network/Server/DefaultNetworkService.swift @@ -30,12 +30,12 @@ public actor DefaultNetworkService: NetworkService { /// Set up a network service for the specified network. /// - Parameters: - /// - leases: where the addresses handed out are written down, so a host - /// attaching again is given what it had. Nothing is remembered without - /// one, which is what a caller wanting a network that forgets passes. + /// - leases: keeps what each host was given, so a host attaching again is + /// given what it had. Nothing is remembered without one, which is what + /// a caller wanting a network that forgets passes. public init( network: any Network, - leases: URL? = nil, + leases: (any AttachmentLeaseStore)? = nil, log: Logger ) async throws { guard let status = await network.status else { @@ -46,7 +46,7 @@ public actor DefaultNetworkService: NetworkService { let size = Int(subnet.upper.value - subnet.lower.value - 3) self.network = network self.log = log - self.allocator = try AttachmentAllocator(lower: subnet.lower.value + 2, size: size, store: leases, log: log) + self.allocator = try await AttachmentAllocator(lower: subnet.lower.value + 2, size: size, store: leases, log: log) self.macAddresses = [:] self.allocationsBySession = [:] } diff --git a/Tests/ContainerNetworkServerTests/AttachmentAllocatorTest.swift b/Tests/ContainerNetworkServerTests/AttachmentAllocatorTest.swift index eefb13b9f..a3788f1e3 100644 --- a/Tests/ContainerNetworkServerTests/AttachmentAllocatorTest.swift +++ b/Tests/ContainerNetworkServerTests/AttachmentAllocatorTest.swift @@ -20,6 +20,20 @@ import Testing @testable import ContainerNetworkServer +/// A store that keeps leases the way a file would, so what is written down +/// can be read back by an allocator that was not there when it was written. +private actor RememberingStore: AttachmentLeaseStore { + private var leases: [String: AttachmentAllocator.Lease] = [:] + + func load() async -> [AttachmentAllocator.Lease] { + Array(leases.values) + } + + func save(_ lease: AttachmentAllocator.Lease) async { + leases[lease.id] = lease + } +} + struct AttachmentAllocatorTest { /// A hardware address to attach with, different for each host so that a /// remembered one is recognizable. @@ -32,7 +46,7 @@ struct AttachmentAllocatorTest { } @Test func testAllocateSingleHostname() async throws { - let allocator = try AttachmentAllocator(lower: 100, size: 10) + let allocator = try await AttachmentAllocator(lower: 100, size: 10) let address = try await allocate(allocator, "test-host") @@ -41,7 +55,7 @@ struct AttachmentAllocatorTest { } @Test func testAllocateSameHostnameTwice() async throws { - let allocator = try AttachmentAllocator(lower: 100, size: 10) + let allocator = try await AttachmentAllocator(lower: 100, size: 10) let address1 = try await allocate(allocator, "test-host") let address2 = try await allocate(allocator, "test-host") @@ -50,7 +64,7 @@ struct AttachmentAllocatorTest { } @Test func testAllocateMultipleHostnames() async throws { - let allocator = try AttachmentAllocator(lower: 100, size: 10) + let allocator = try await AttachmentAllocator(lower: 100, size: 10) let address1 = try await allocate(allocator, "host1") let address2 = try await allocate(allocator, "host2") @@ -62,7 +76,7 @@ struct AttachmentAllocatorTest { } @Test func testLookupAllocatedHostname() async throws { - let allocator = try AttachmentAllocator(lower: 100, size: 10) + let allocator = try await AttachmentAllocator(lower: 100, size: 10) let allocatedAddress = try await allocate(allocator, "test-host") let lookedUpAddress = try await allocator.lookup(hostname: "test-host") @@ -71,7 +85,7 @@ struct AttachmentAllocatorTest { } @Test func testLookupNonExistentHostname() async throws { - let allocator = try AttachmentAllocator(lower: 100, size: 10) + let allocator = try await AttachmentAllocator(lower: 100, size: 10) let address = try await allocator.lookup(hostname: "non-existent") @@ -79,7 +93,7 @@ struct AttachmentAllocatorTest { } @Test func testDeallocateAllocatedHostname() async throws { - let allocator = try AttachmentAllocator(lower: 100, size: 10) + let allocator = try await AttachmentAllocator(lower: 100, size: 10) let allocatedAddress = try await allocate(allocator, "test-host") let deallocatedAddress = try await allocator.deallocate(hostname: "test-host") @@ -92,7 +106,7 @@ struct AttachmentAllocatorTest { } @Test func testDeallocateNonExistentHostname() async throws { - let allocator = try AttachmentAllocator(lower: 100, size: 10) + let allocator = try await AttachmentAllocator(lower: 100, size: 10) let deallocatedAddress = try await allocator.deallocate(hostname: "non-existent") @@ -100,7 +114,7 @@ struct AttachmentAllocatorTest { } @Test func testHostAttachingAgainIsGivenWhatItHad() async throws { - let allocator = try AttachmentAllocator(lower: 100, size: 10) + let allocator = try await AttachmentAllocator(lower: 100, size: 10) let first = try await allocator.allocate(hostname: "test-host", macAddress: mac(7)) _ = try await allocator.deallocate(hostname: "test-host") @@ -114,10 +128,7 @@ struct AttachmentAllocatorTest { } @Test func testWhatAHostWasGivenOutlivesTheAllocator() async throws { - let store = FileManager.default.temporaryDirectory - .appendingPathComponent(UUID().uuidString) - .appendingPathComponent("leases.json") - defer { try? FileManager.default.removeItem(at: store.deletingLastPathComponent()) } + let store = RememberingStore() let first = try await AttachmentAllocator(lower: 100, size: 10, store: store) .allocate(hostname: "test-host", macAddress: mac(3)) @@ -131,7 +142,7 @@ struct AttachmentAllocatorTest { @Test func testAllocateUntilFull() async throws { let size = 5 - let allocator = try AttachmentAllocator(lower: 100, size: size) + let allocator = try await AttachmentAllocator(lower: 100, size: size) // Allocate up to the limit for i in 0.. Date: Wed, 19 Aug 2026 15:29:20 +0000 Subject: [PATCH 25/29] Give the addresses back however the helper leaves Leaving the wait is leaving the network, whether the wait ended by being asked to stop, by an error, or by returning. Releasing on the asking alone left the other ways out holding the range, so a helper that failed or finished left it held by nobody and the next network asking for that range was refused, with no interface, route, or process to point at. --- .../NetworkVmnet/NetworkVmnetHelper+Start.swift | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/Sources/Plugins/NetworkVmnet/NetworkVmnetHelper+Start.swift b/Sources/Plugins/NetworkVmnet/NetworkVmnetHelper+Start.swift index 74f0a02da..aefc762eb 100644 --- a/Sources/Plugins/NetworkVmnet/NetworkVmnetHelper+Start.swift +++ b/Sources/Plugins/NetworkVmnet/NetworkVmnetHelper+Start.swift @@ -141,7 +141,17 @@ extension NetworkVmnetHelper { } log.info("starting XPC server") - try await xpc.listen() + do { + try await xpc.listen() + } catch { + // Whatever ends the wait, the addresses go back: a helper + // that leaves holding them leaves them held by nobody, and + // the next network asking for that range is refused with + // no interface, route, or process to point at. + await network.stop() + throw error + } + await network.stop() } catch { log.error( "helper failed", From ef3201d27dea04391cedaa3cba9cadae14641694 Mon Sep 17 00:00:00 2001 From: Aaron Paterson Date: Thu, 20 Aug 2026 15:29:23 +0000 Subject: [PATCH 26/29] Have a store make somewhere to keep what it holds A store reads what it holds as it opens, and reading a directory that is not there fails, so every caller had to make one first and a caller that forgot took down whatever depended on it: the network helper came up, announced its network, and died on a missing directory, leaving every call that needed a network waiting on a helper that kept restarting. The directory belongs to the store, which makes it as it opens, and the callers that were making it no longer do. --- .../Container/ContainerStart.swift | 11 +++++++++++ Sources/ContainerPersistence/EntityStore.swift | 5 +++++ .../Server/Networks/NetworksService.swift | 3 +-- .../Server/Volumes/VolumesService.swift | 1 - .../RuntimeLinux/Server/RuntimeService.swift | 11 +++++++++++ .../FilesystemEntityStoreTests.swift | 13 +++++++++++++ 6 files changed, 41 insertions(+), 3 deletions(-) diff --git a/Sources/ContainerCommands/Container/ContainerStart.swift b/Sources/ContainerCommands/Container/ContainerStart.swift index 00bc47a1c..ce2876ffa 100644 --- a/Sources/ContainerCommands/Container/ContainerStart.swift +++ b/Sources/ContainerCommands/Container/ContainerStart.swift @@ -98,6 +98,17 @@ extension Application { if detach { try await process.start() try io.closeAfterStart() + // What this command says it did is that the container is + // running, so it says so only once the container answers + // that it is: a caller starting a container and then using + // it has nothing else to go on. + let started = try await client.get(id: container.id) + guard started.status == .running else { + throw ContainerizationError( + .invalidState, + message: "container \(container.id) did not start; it is \(started.status)" + ) + } print(self.containerId) return } diff --git a/Sources/ContainerPersistence/EntityStore.swift b/Sources/ContainerPersistence/EntityStore.swift index 60b080582..b3c0a0fa9 100644 --- a/Sources/ContainerPersistence/EntityStore.swift +++ b/Sources/ContainerPersistence/EntityStore.swift @@ -45,6 +45,11 @@ public actor FilesystemEntityStore: EntityStore where T: Codable & Identifiab self.path = path self.type = type self.log = log + // The store keeps what it holds under this directory, so it is the + // store's to make: a caller that has never written an entity has no + // reason to have made somewhere to put them, and a store that reads + // before anything is written finds nothing rather than failing. + try FileManager.default.createDirectory(atPath: path.string, withIntermediateDirectories: true) self.index = try Self.load(path: path, log: log) } diff --git a/Sources/Services/ContainerAPIService/Server/Networks/NetworksService.swift b/Sources/Services/ContainerAPIService/Server/Networks/NetworksService.swift index 22eff35e2..a6472251a 100644 --- a/Sources/Services/ContainerAPIService/Server/Networks/NetworksService.swift +++ b/Sources/Services/ContainerAPIService/Server/Networks/NetworksService.swift @@ -59,7 +59,6 @@ public actor NetworksService { self.log = log self.debugHelpers = debugHelpers - try FileManager.default.createDirectory(atPath: resourceRoot.string, withIntermediateDirectories: true) self.store = try FilesystemEntityStore( path: resourceRoot, type: "network", @@ -108,7 +107,7 @@ public actor NetworksService { do { do { try await registerService(configuration: effectiveConfiguration) - } catch where effectiveConfiguration.id == NetworkClient.defaultNetworkName && effectiveConfiguration.ipv4Subnet != nil { + } catch where effectiveConfiguration.id == NetworkClient.defaultNetworkName && effectiveConfiguration.ipv4Subnet != nil { // The range the default network asks for is the one it was // given last time, which is a preference and not a demand: // a range held by something else, or reserved to a network diff --git a/Sources/Services/ContainerAPIService/Server/Volumes/VolumesService.swift b/Sources/Services/ContainerAPIService/Server/Volumes/VolumesService.swift index dc76076a6..14413b8f6 100644 --- a/Sources/Services/ContainerAPIService/Server/Volumes/VolumesService.swift +++ b/Sources/Services/ContainerAPIService/Server/Volumes/VolumesService.swift @@ -39,7 +39,6 @@ public actor VolumesService { private static let blockFile = "volume.img" public init(resourceRoot: FilePath, log: Logger) async throws { - try FileManager.default.createDirectory(atPath: resourceRoot.string, withIntermediateDirectories: true) self.resourceRoot = resourceRoot self.store = try FilesystemEntityStore(path: resourceRoot, type: "volumes", log: log) self.log = log diff --git a/Sources/Services/RuntimeLinux/Server/RuntimeService.swift b/Sources/Services/RuntimeLinux/Server/RuntimeService.swift index 88e3e283e..f17b95aa0 100644 --- a/Sources/Services/RuntimeLinux/Server/RuntimeService.swift +++ b/Sources/Services/RuntimeLinux/Server/RuntimeService.swift @@ -167,6 +167,17 @@ public actor RuntimeService { // the machine does not hold yet goes in, and the machine stays as // it is. guard await self.state == .created else { + // A machine on its way down takes no more containers: placing + // one in it leaves a start that reported success and a + // container that never ran, which is what the caller is told + // instead. + let held = await self.state + guard held != .stopping, held != .stopped, held != .shuttingDown else { + throw ContainerizationError( + .invalidState, + message: "the machine is \(held) and takes no containers; wait for it to stop and start it again" + ) + } try await self.placeContainers(message) return message.reply() } diff --git a/Tests/ContainerPersistenceTests/FilesystemEntityStoreTests.swift b/Tests/ContainerPersistenceTests/FilesystemEntityStoreTests.swift index 03c136d1e..3c64ffa39 100644 --- a/Tests/ContainerPersistenceTests/FilesystemEntityStoreTests.swift +++ b/Tests/ContainerPersistenceTests/FilesystemEntityStoreTests.swift @@ -171,6 +171,19 @@ struct FilesystemEntityStoreTests { } } + @Test func testStoreMakesSomewhereToKeepWhatItHolds() async throws { + try await TemporaryStorage.withTempDir { path in + // Nothing has been written yet, so the directory the store keeps + // its entities under is not there to be read. + let fresh = path.appending("never-written") + let store = try Self.makeStore(at: fresh) + + #expect(try await store.list().isEmpty) + try await store.create(Item(id: "foo", value: "hello")) + #expect(try await store.retrieve("foo")?.value == "hello") + } + } + private static func makeStore(at path: FilePath) throws -> FilesystemEntityStore { try FilesystemEntityStore(path: path, type: "item", log: Logger(label: "test")) } From 3598a35414b971f072247e1899c4b355f2aa989e Mon Sep 17 00:00:00 2001 From: Aaron Paterson Date: Thu, 20 Aug 2026 17:42:15 +0000 Subject: [PATCH 27/29] Record the init image a pod's machine boots A pod clones the init image's filesystem when it is made and boots that clone for as long as it lives, so the image it came from is the only account of which agent its containers talk to. A container records the image it was made from and a caller compares that against the store to decide the container is stale; a pod held nothing to compare, so a machine could outlive any number of guest rebuilds while still booting the generation it was made from. --- Sources/ContainerResource/Pod/PodConfiguration.swift | 9 +++++++++ .../Server/Containers/ContainersService.swift | 10 ++++++---- .../Server/Pods/PodsService.swift | 12 ++++++++---- 3 files changed, 23 insertions(+), 8 deletions(-) diff --git a/Sources/ContainerResource/Pod/PodConfiguration.swift b/Sources/ContainerResource/Pod/PodConfiguration.swift index 8c735d6b8..d250758ff 100644 --- a/Sources/ContainerResource/Pod/PodConfiguration.swift +++ b/Sources/ContainerResource/Pod/PodConfiguration.swift @@ -75,6 +75,15 @@ public struct PodConfiguration: Sendable, Codable { /// Configured platform for the pod. public var platform: ContainerizationOCI.Platform = .current + /// The init image the pod's machine boots. + /// + /// A pod clones the image's filesystem when it is made and boots that + /// clone for as long as it lives, so the image it was made from is the + /// only account of which agent its containers talk to. A caller comparing + /// this against the init image the runtime is configured with is asking + /// whether the machine still matches the plane driving it. + public var initImage: ImageDescription? + /// The time at which the pod was created. public var creationDate: Date = Date() diff --git a/Sources/Services/ContainerAPIService/Server/Containers/ContainersService.swift b/Sources/Services/ContainerAPIService/Server/Containers/ContainersService.swift index 1c7934cb0..c5851da07 100644 --- a/Sources/Services/ContainerAPIService/Server/Containers/ContainersService.swift +++ b/Sources/Services/ContainerAPIService/Server/Containers/ContainersService.swift @@ -402,14 +402,16 @@ public actor ContainersService { "id": "\(configuration.id)" ] ) - let initFilesystem = try await self.getInitBlock(for: systemPlatform.ociPlatform(), imageRef: initImage) + let (initFilesystem, initDescription) = try await self.getInitBlock(for: systemPlatform.ociPlatform(), imageRef: initImage) guard let podsService = await self.podsService else { throw ContainerizationError(.internalError, message: "no pod service to make pod \(configuration.pod)") } do { + var podConfiguration = PodConfiguration(sandboxFor: configuration) + podConfiguration.initImage = initDescription try await podsService.create( - configuration: PodConfiguration(sandboxFor: configuration), + configuration: podConfiguration, kernel: kernel, initialFilesystem: initFilesystem ) @@ -1179,12 +1181,12 @@ public actor ContainersService { return options } - private func getInitBlock(for platform: Platform, imageRef: String? = nil) async throws -> Filesystem { + private func getInitBlock(for platform: Platform, imageRef: String? = nil) async throws -> (Filesystem, ImageDescription) { let ref = imageRef ?? containerSystemConfig.vminit.image let initImage = try await ClientImage.fetch(reference: ref, platform: platform, containerSystemConfig: containerSystemConfig) var fs = try await initImage.getCreateSnapshot(platform: platform) fs.options = ["ro"] - return fs + return (fs, initImage.description) } private static func registerService( diff --git a/Sources/Services/ContainerAPIService/Server/Pods/PodsService.swift b/Sources/Services/ContainerAPIService/Server/Pods/PodsService.swift index de5ab0e0c..b4bfc5906 100644 --- a/Sources/Services/ContainerAPIService/Server/Pods/PodsService.swift +++ b/Sources/Services/ContainerAPIService/Server/Pods/PodsService.swift @@ -222,21 +222,25 @@ public actor PodsService { try self._getPodState(id: id).getClient() } - /// The initial filesystem a pod's machine boots, which holds the agent. - private func getInitBlock(for platform: Platform, imageRef: String? = nil) async throws -> Filesystem { + /// The initial filesystem a pod's machine boots, which holds the agent, + /// and the image it was taken from. + private func getInitBlock(for platform: Platform, imageRef: String? = nil) async throws -> (Filesystem, ImageDescription) { let ref = imageRef ?? containerSystemConfig.vminit.image let initImage = try await ClientImage.fetch(reference: ref, platform: platform, containerSystemConfig: containerSystemConfig) var fs = try await initImage.getCreateSnapshot(platform: platform) fs.options = ["ro"] - return fs + return (fs, initImage.description) } /// Write down a pod that boots the init image named, or the default one. public func create(configuration: PodConfiguration, kernel: Kernel, initImage: String? = nil) async throws { + let (initialFilesystem, description) = try await self.getInitBlock(for: kernel.platform.ociPlatform(), imageRef: initImage) + var configuration = configuration + configuration.initImage = description try await self.create( configuration: configuration, kernel: kernel, - initialFilesystem: try await self.getInitBlock(for: kernel.platform.ociPlatform(), imageRef: initImage) + initialFilesystem: initialFilesystem ) } From 65ba57cb4658cbbcbb889ff8e09bb6df2329f01b Mon Sep 17 00:00:00 2001 From: Aaron Paterson Date: Mon, 10 Aug 2026 20:09:29 +0000 Subject: [PATCH 28/29] Name the host host.containers.internal from a pod's containers A pod's containers reach the host they run on by the gateway of the network the pod is attached to. That address is written into the pod's hosts file as host.containers.internal, the cross-runtime name Podman established, and host.docker.internal, the name Docker's tools look for, so a container that expects either name finds the host without being told its address. --- .../RuntimeLinux/Server/RuntimeService.swift | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/Sources/Services/RuntimeLinux/Server/RuntimeService.swift b/Sources/Services/RuntimeLinux/Server/RuntimeService.swift index f17b95aa0..44cd95cad 100644 --- a/Sources/Services/RuntimeLinux/Server/RuntimeService.swift +++ b/Sources/Services/RuntimeLinux/Server/RuntimeService.swift @@ -1324,8 +1324,12 @@ public actor RuntimeService { podConfig.shareProcessNamespace = config.shareProcessNamespace podConfig.hostname = config.hostname ?? Self.hostname(networks: config.networks, id: config.id) // The hosts file names the pod at its own address so its containers - // reach the name they answer to, and it is written once for the pod - // the way the resolver and the hostname are. + // reach the name they answer to, and names the network's gateway so + // they reach the host they run on: `host.containers.internal` is the + // cross-runtime name for it, which Podman established, and + // `host.docker.internal` the one Docker's tools look for, so both are + // given as Podman gives them. It is written once for the pod the way + // the resolver and the hostname are. var hostsEntries = [Hosts.Entry.localHostIPV4()] if let primary = attachments.first { hostsEntries.append( @@ -1333,6 +1337,11 @@ public actor RuntimeService { ipAddress: primary.ipv4Address.address.description, hostnames: [podConfig.hostname ?? config.id], )) + hostsEntries.append( + Hosts.Entry( + ipAddress: primary.ipv4Gateway.description, + hostnames: ["host.containers.internal", "host.docker.internal"], + )) } podConfig.hosts = Hosts(entries: hostsEntries) // The runtime asks for these two of every machine it boots; they From 8513803f97c44c8cfe23ee73ca57bbe9c4bd6cd3 Mon Sep 17 00:00:00 2001 From: Aaron Paterson Date: Mon, 10 Aug 2026 20:47:52 +0000 Subject: [PATCH 29/29] Carry the network's IPv6 gateway to a pod's containers The attachment a pod bootstraps from carried the interface's IPv6 address but not the network's IPv6 gateway, so a container on a dual-stack network had no name for the host over IPv6. Attachment gains an ipv6Gateway, derived where the attachment is built from the IPv6 subnet the way the IPv4 gateway is, carried through the runtime, and named host.containers.internal and host.docker.internal beside the IPv4 gateway when the network has one. A network with no IPv6 subnet is unchanged. --- Sources/ContainerResource/Network/Attachment.swift | 7 +++++++ .../Network/Server/DefaultNetworkService.swift | 2 ++ .../RuntimeLinux/Server/RuntimeService.swift | 13 +++++++++++-- 3 files changed, 20 insertions(+), 2 deletions(-) diff --git a/Sources/ContainerResource/Network/Attachment.swift b/Sources/ContainerResource/Network/Attachment.swift index a6351ab3b..285d7c1de 100644 --- a/Sources/ContainerResource/Network/Attachment.swift +++ b/Sources/ContainerResource/Network/Attachment.swift @@ -29,6 +29,8 @@ public struct Attachment: Codable, Sendable { /// The CIDR address describing the interface IPv6 address, with the prefix length of the subnet. /// The address is nil if the IPv6 subnet could not be determined at network creation time. public let ipv6Address: CIDRv6? + /// The IPv6 gateway address, nil when the network carries no IPv6 subnet. + public let ipv6Gateway: IPv6Address? /// The MAC address associated with the attachment (optional). public let macAddress: MACAddress? /// The MTU for the network interface. @@ -42,6 +44,7 @@ public struct Attachment: Codable, Sendable { ipv4Address: CIDRv4, ipv4Gateway: IPv4Address, ipv6Address: CIDRv6?, + ipv6Gateway: IPv6Address? = nil, macAddress: MACAddress?, mtu: UInt32? = nil, variant: String? = nil @@ -51,6 +54,7 @@ public struct Attachment: Codable, Sendable { self.ipv4Address = ipv4Address self.ipv4Gateway = ipv4Gateway self.ipv6Address = ipv6Address + self.ipv6Gateway = ipv6Gateway self.macAddress = macAddress self.mtu = mtu self.variant = variant @@ -62,6 +66,7 @@ public struct Attachment: Codable, Sendable { case ipv4Address case ipv4Gateway case ipv6Address + case ipv6Gateway case macAddress case mtu case variant @@ -88,6 +93,7 @@ public struct Attachment: Codable, Sendable { ipv4Gateway = try container.decode(IPv4Address.self, forKey: .gateway) } ipv6Address = try container.decodeIfPresent(CIDRv6.self, forKey: .ipv6Address) + ipv6Gateway = try container.decodeIfPresent(IPv6Address.self, forKey: .ipv6Gateway) macAddress = try container.decodeIfPresent(MACAddress.self, forKey: .macAddress) mtu = try container.decodeIfPresent(UInt32.self, forKey: .mtu) variant = try container.decodeIfPresent(String.self, forKey: .variant) @@ -102,6 +108,7 @@ public struct Attachment: Codable, Sendable { try container.encode(ipv4Address, forKey: .ipv4Address) try container.encode(ipv4Gateway, forKey: .ipv4Gateway) try container.encodeIfPresent(ipv6Address, forKey: .ipv6Address) + try container.encodeIfPresent(ipv6Gateway, forKey: .ipv6Gateway) try container.encodeIfPresent(macAddress, forKey: .macAddress) try container.encodeIfPresent(mtu, forKey: .mtu) try container.encodeIfPresent(variant, forKey: .variant) diff --git a/Sources/Services/Network/Server/DefaultNetworkService.swift b/Sources/Services/Network/Server/DefaultNetworkService.swift index f799d7247..e2a8191ea 100644 --- a/Sources/Services/Network/Server/DefaultNetworkService.swift +++ b/Sources/Services/Network/Server/DefaultNetworkService.swift @@ -90,6 +90,7 @@ public actor DefaultNetworkService: NetworkService { ipv4Address: try CIDRv4(ip, prefix: status.ipv4Subnet.prefix), ipv4Gateway: status.ipv4Gateway, ipv6Address: ipv6Address, + ipv6Gateway: status.ipv6Subnet?.gateway, macAddress: macAddress, variant: network.variant ) @@ -160,6 +161,7 @@ public actor DefaultNetworkService: NetworkService { ipv4Address: ipv4Address, ipv4Gateway: status.ipv4Gateway, ipv6Address: ipv6Address, + ipv6Gateway: status.ipv6Subnet?.gateway, macAddress: macAddress, variant: network.variant ) diff --git a/Sources/Services/RuntimeLinux/Server/RuntimeService.swift b/Sources/Services/RuntimeLinux/Server/RuntimeService.swift index 44cd95cad..647025a7f 100644 --- a/Sources/Services/RuntimeLinux/Server/RuntimeService.swift +++ b/Sources/Services/RuntimeLinux/Server/RuntimeService.swift @@ -1328,8 +1328,9 @@ public actor RuntimeService { // they reach the host they run on: `host.containers.internal` is the // cross-runtime name for it, which Podman established, and // `host.docker.internal` the one Docker's tools look for, so both are - // given as Podman gives them. It is written once for the pod the way - // the resolver and the hostname are. + // given as Podman gives them, at the IPv4 gateway and, on a dual-stack + // network, the IPv6 gateway too. It is written once for the pod the + // way the resolver and the hostname are. var hostsEntries = [Hosts.Entry.localHostIPV4()] if let primary = attachments.first { hostsEntries.append( @@ -1342,6 +1343,13 @@ public actor RuntimeService { ipAddress: primary.ipv4Gateway.description, hostnames: ["host.containers.internal", "host.docker.internal"], )) + if let ipv6Gateway = primary.ipv6Gateway { + hostsEntries.append( + Hosts.Entry( + ipAddress: ipv6Gateway.description, + hostnames: ["host.containers.internal", "host.docker.internal"], + )) + } } podConfig.hosts = Hosts(entries: hostsEntries) // The runtime asks for these two of every machine it boots; they @@ -1555,6 +1563,7 @@ public actor RuntimeService { ipv4Address: attachment.ipv4Address, ipv4Gateway: attachment.ipv4Gateway, ipv6Address: attachment.ipv6Address, + ipv6Gateway: attachment.ipv6Gateway, macAddress: attachment.macAddress, mtu: mtu, variant: attachment.variant