diff --git a/Makefile b/Makefile index 1855256d2..6d18ac41d 100644 --- a/Makefile +++ b/Makefile @@ -131,10 +131,13 @@ $(STAGING_DIR): @mkdir -p "$(join $(STAGING_DIR), libexec/container/plugins/container-runtime-linux/bin)" @mkdir -p "$(join $(STAGING_DIR), libexec/container/plugins/container-network-vmnet/bin)" @mkdir -p "$(join $(STAGING_DIR), libexec/container/plugins/container-core-images/bin)" + @mkdir -p "$(join $(STAGING_DIR), libexec/container/plugins/container-core-containers/bin)" @mkdir -p "$(join $(STAGING_DIR), libexec/container/plugins/machine-apiserver/bin)" @mkdir -p "$(join $(STAGING_DIR), libexec/container/plugins/machine-apiserver/resources)" @mkdir -p "$(join $(STAGING_DIR), libexec/container/plugins/k8s/bin)" @mkdir -p "$(join $(STAGING_DIR), libexec/container/plugins/k8s/resources)" + @mkdir -p "$(join $(STAGING_DIR), libexec/container/plugins/pod/bin)" + @mkdir -p "$(join $(STAGING_DIR), libexec/container/plugins/run/bin)" @install "$(BUILD_BIN_DIR)/container" "$(join $(STAGING_DIR), bin/container)" @install "$(BUILD_BIN_DIR)/container-apiserver" "$(join $(STAGING_DIR), bin/container-apiserver)" @@ -144,6 +147,8 @@ $(STAGING_DIR): @install Sources/Plugins/NetworkVmnet/config.toml "$(join $(STAGING_DIR), libexec/container/plugins/container-network-vmnet/config.toml)" @install "$(BUILD_BIN_DIR)/container-core-images" "$(join $(STAGING_DIR), libexec/container/plugins/container-core-images/bin/container-core-images)" @install Sources/Plugins/CoreImages/config.toml "$(join $(STAGING_DIR), libexec/container/plugins/container-core-images/config.toml)" + @install "$(BUILD_BIN_DIR)/container-core-containers" "$(join $(STAGING_DIR), libexec/container/plugins/container-core-containers/bin/container-core-containers)" + @install Sources/Plugins/CoreContainers/config.toml "$(join $(STAGING_DIR), libexec/container/plugins/container-core-containers/config.toml)" @install "$(BUILD_BIN_DIR)/machine-apiserver" "$(join $(STAGING_DIR), libexec/container/plugins/machine-apiserver/bin/machine-apiserver)" @install Sources/Plugins/MachineAPIServer/config.toml "$(join $(STAGING_DIR), libexec/container/plugins/machine-apiserver/config.toml)" @install Sources/Plugins/MachineAPIServer/Resources/init "$(join $(STAGING_DIR), libexec/container/plugins/machine-apiserver/resources/init)" @@ -151,6 +156,10 @@ $(STAGING_DIR): @install "$(BUILD_BIN_DIR)/k8s" "$(join $(STAGING_DIR), libexec/container/plugins/k8s/bin/k8s)" @install Sources/Plugins/K8s/config.toml "$(join $(STAGING_DIR), libexec/container/plugins/k8s/config.toml)" @install Sources/Plugins/K8s/Resources/kindnet.yaml "$(join $(STAGING_DIR), libexec/container/plugins/k8s/resources/kindnet.yaml)" + @install "$(BUILD_BIN_DIR)/pod" "$(join $(STAGING_DIR), libexec/container/plugins/pod/bin/pod)" + @install Sources/Plugins/PodCLI/config.toml "$(join $(STAGING_DIR), libexec/container/plugins/pod/config.toml)" + @install "$(BUILD_BIN_DIR)/run" "$(join $(STAGING_DIR), libexec/container/plugins/run/bin/run)" + @install Sources/Plugins/RunCLI/config.toml "$(join $(STAGING_DIR), libexec/container/plugins/run/config.toml)" @echo Install update script @install scripts/update-container.sh "$(join $(STAGING_DIR), bin/update-container.sh)" @@ -163,10 +172,13 @@ installer-pkg: $(STAGING_DIR) @codesign $(CODESIGN_OPTS) --identifier com.apple.container.cli "$(join $(STAGING_DIR), bin/container)" @codesign $(CODESIGN_OPTS) --identifier com.apple.container.apiserver "$(join $(STAGING_DIR), bin/container-apiserver)" @codesign $(CODESIGN_OPTS) --prefix=com.apple.container. "$(join $(STAGING_DIR), libexec/container/plugins/container-core-images/bin/container-core-images)" + @codesign $(CODESIGN_OPTS) --prefix=com.apple.container. "$(join $(STAGING_DIR), libexec/container/plugins/container-core-containers/bin/container-core-containers)" @codesign $(CODESIGN_OPTS) --prefix=com.apple.container. --entitlements=signing/container-runtime-linux.entitlements "$(join $(STAGING_DIR), libexec/container/plugins/container-runtime-linux/bin/container-runtime-linux)" @codesign $(CODESIGN_OPTS) --prefix=com.apple.container. --entitlements=signing/container-network-vmnet.entitlements "$(join $(STAGING_DIR), libexec/container/plugins/container-network-vmnet/bin/container-network-vmnet)" @codesign $(CODESIGN_OPTS) --prefix=com.apple.container. "$(join $(STAGING_DIR), libexec/container/plugins/machine-apiserver/bin/machine-apiserver)" @codesign $(CODESIGN_OPTS) --prefix=com.apple.container. "$(join $(STAGING_DIR), libexec/container/plugins/k8s/bin/k8s)" + @codesign $(CODESIGN_OPTS) --prefix=com.apple.container. "$(join $(STAGING_DIR), libexec/container/plugins/pod/bin/pod)" + @codesign $(CODESIGN_OPTS) --prefix=com.apple.container. "$(join $(STAGING_DIR), libexec/container/plugins/run/bin/run)" @echo Creating application installer @pkgbuild --root "$(STAGING_DIR)" --identifier com.apple.container-installer --install-location /usr/local --version ${RELEASE_VERSION} $(PKG_PATH) @@ -180,6 +192,7 @@ dsym: @cp -a "$(BUILD_BIN_DIR)/container-runtime-linux.dSYM" "$(DSYM_DIR)" @cp -a "$(BUILD_BIN_DIR)/container-network-vmnet.dSYM" "$(DSYM_DIR)" @cp -a "$(BUILD_BIN_DIR)/container-core-images.dSYM" "$(DSYM_DIR)" + @cp -a "$(BUILD_BIN_DIR)/container-core-containers.dSYM" "$(DSYM_DIR)" @cp -a "$(BUILD_BIN_DIR)/container-apiserver.dSYM" "$(DSYM_DIR)" @cp -a "$(BUILD_BIN_DIR)/container.dSYM" "$(DSYM_DIR)" @@ -212,6 +225,7 @@ COV_BINARIES := \ $(BUILD_BIN_DIR)/container-runtime-linux \ $(BUILD_BIN_DIR)/container-network-vmnet \ $(BUILD_BIN_DIR)/container-core-images \ + $(BUILD_BIN_DIR)/container-core-containers \ $(BUILD_BIN_DIR)/machine-apiserver COV_OBJECT_FLAGS := $(patsubst %,-object %,$(COV_BINARIES)) # Set of files we do not want to get caught in the coverage generation diff --git a/Package.resolved b/Package.resolved index 31cd926e9..8e7e53201 100644 --- a/Package.resolved +++ b/Package.resolved @@ -1,5 +1,5 @@ { - "originHash" : "35d1f07a3595a3ebabac1430f46ebb8b4e54d4531874f4ff7df33b64092df5f3", + "originHash" : "bfd17c6a1af5c6b5efd9e28e4f336c0cce54aa18dec4cad43b9d8a577f32df88", "pins" : [ { "identity" : "async-http-client", @@ -13,10 +13,9 @@ { "identity" : "containerization", "kind" : "remoteSourceControl", - "location" : "https://github.com/apple/containerization.git", + "location" : "https://github.com/MayCXC/containerization.git", "state" : { - "revision" : "5427fd21ded4b84034126caef5b3182900b4776d", - "version" : "0.41.0" + "revision" : "67e97e92d2843eaabb9d4b28d5edb3594a227e40" } }, { diff --git a/Package.swift b/Package.swift index d53fd9c0e..4c36336c7 100644 --- a/Package.swift +++ b/Package.swift @@ -312,6 +312,42 @@ let package = Package( path: "Sources/Plugins/CoreImages", exclude: ["config.toml"] ), + .executableTarget( + name: "container-core-containers", + dependencies: [ + .product(name: "ArgumentParser", package: "swift-argument-parser"), + .product(name: "Logging", package: "swift-log"), + .product(name: "Containerization", package: "containerization"), + .product(name: "SystemPackage", package: "swift-system"), + "ContainerAPIClient", + "ContainerAPIService", + "ContainerLog", + "ContainerPersistence", + "ContainerPlugin", + "ContainerVersion", + "ContainerXPC", + ], + path: "Sources/Plugins/CoreContainers", + exclude: ["config.toml"] + ), + .executableTarget( + name: "pod", + dependencies: [ + .product(name: "ArgumentParser", package: "swift-argument-parser"), + "ContainerCommands", + ], + path: "Sources/Plugins/PodCLI", + exclude: ["config.toml"] + ), + .executableTarget( + name: "run", + dependencies: [ + .product(name: "ArgumentParser", package: "swift-argument-parser"), + "ContainerCommands", + ], + path: "Sources/Plugins/RunCLI", + exclude: ["config.toml"] + ), .target( name: "ContainerImagesService", dependencies: [ diff --git a/Sources/APIServer/APIServer+Start.swift b/Sources/APIServer/APIServer+Start.swift index 936abd913..04aa23e8c 100644 --- a/Sources/APIServer/APIServer+Start.swift +++ b/Sources/APIServer/APIServer+Start.swift @@ -49,7 +49,6 @@ extension APIServer { var logRoot = LogRoot.path func run() async throws { - let containerSystemConfig: ContainerSystemConfig = try await ConfigurationLoader.load() let commandName = APIServer._commandName let logPath = logRoot.map { $0.appending(FilePath.Component("\(commandName).log") ?? "unknown") } let log = ServiceLogger.bootstrap(category: "APIServer", debug: debug, logPath: logPath) @@ -64,25 +63,10 @@ extension APIServer { let pluginLoader = try initializePluginLoader(log: log) try await initializePlugins(pluginLoader: pluginLoader, log: log, routes: &routes, debug: debug) - let containersService = try initializeContainersService( - pluginLoader: pluginLoader, - containerSystemConfig: containerSystemConfig, - log: log, - routes: &routes - ) - let networkService = try await initializeNetworksService( - pluginLoader: pluginLoader, - containersService: containersService, - containerSystemConfig: containerSystemConfig, - log: log, - routes: &routes - ) - await containersService.setNetworksService(networkService) initializeHealthCheckService(log: log, routes: &routes) try initializeKernelService(log: log, routes: &routes) - let volumesService = try await initializeVolumeService(containersService: containersService, log: log, routes: &routes) + let volumesService = try await initializeVolumeService(log: log, routes: &routes) try initializeDiskUsageService( - containersService: containersService, volumesService: volumesService, log: log, routes: &routes @@ -109,7 +93,7 @@ extension APIServer { // start up host table DNS group.addTask { - let hostsResolver = ContainerDNSHandler(networkService: networkService) + let hostsResolver = ContainerDNSHandler(networks: NetworkClient()) let nxDomainResolver = NxDomainResolver() let compositeResolver = CompositeResolver(handlers: [hostsResolver, nxDomainResolver]) let hostsQueryValidator = StandardQueryValidator(handler: compositeResolver) @@ -271,101 +255,14 @@ extension APIServer { routes[XPCRoute.getDefaultKernel] = XPCServer.route(harness.getDefaultKernel) } - private func initializeContainersService( - pluginLoader: PluginLoader, - containerSystemConfig: ContainerSystemConfig, - log: Logger, - routes: inout [XPCRoute: XPCServer.RouteHandler] - ) throws -> ContainersService { - log.info("initializing containers service") - - // TODO: Remove when we convert ContainersService to FilePath - let appRootURL = URL(fileURLWithPath: appRoot.string) - let service = try ContainersService( - appRoot: appRootURL, - pluginLoader: pluginLoader, - containerSystemConfig: containerSystemConfig, - log: log, - debugHelpers: debug - ) - let harness = ContainersHarness(service: service, log: log) - - routes[XPCRoute.containerList] = XPCServer.route(harness.list) - routes[XPCRoute.containerCreate] = XPCServer.route(harness.create) - routes[XPCRoute.containerDelete] = XPCServer.route(harness.delete) - routes[XPCRoute.containerLogs] = XPCServer.route(harness.logs) - routes[XPCRoute.containerBootstrap] = XPCServer.route(harness.bootstrap) - routes[XPCRoute.containerDial] = XPCServer.route(harness.dial) - routes[XPCRoute.containerStop] = XPCServer.route(harness.stop) - routes[XPCRoute.containerStartProcess] = XPCServer.route(harness.startProcess) - routes[XPCRoute.containerCreateProcess] = XPCServer.route(harness.createProcess) - routes[XPCRoute.containerResize] = XPCServer.route(harness.resize) - routes[XPCRoute.containerWait] = XPCServer.route(harness.wait) - routes[XPCRoute.containerKill] = XPCServer.route(harness.kill) - routes[XPCRoute.containerStats] = XPCServer.route(harness.stats) - routes[XPCRoute.containerDiskUsage] = XPCServer.route(harness.diskUsage) - routes[XPCRoute.containerCopyIn] = XPCServer.route(harness.copyIn) - routes[XPCRoute.containerCopyOut] = XPCServer.route(harness.copyOut) - routes[XPCRoute.containerExport] = XPCServer.route(harness.export) - - return service - } - - private func initializeNetworksService( - pluginLoader: PluginLoader, - containersService: ContainersService, - containerSystemConfig: ContainerSystemConfig, - log: Logger, - routes: inout [XPCRoute: XPCServer.RouteHandler] - ) async throws -> NetworksService { - log.info("initializing networks service") - - let resourceRoot = appRoot.appending(FilePath.Component("networks")) - let defaultNetworkConfig = try NetworkConfiguration( - name: NetworkClient.defaultNetworkName, - mode: .nat, - ipv4Subnet: containerSystemConfig.network.subnet, - ipv6Subnet: containerSystemConfig.network.subnetv6, - labels: try .init([ResourceLabelKeys.role: ResourceRoleValues.builtin]), - plugin: "container-network-vmnet" - ) - let service = try await NetworksService( - pluginLoader: pluginLoader, - resourceRoot: resourceRoot, - containersService: containersService, - defaultNetworkConfiguration: defaultNetworkConfig, - log: log, - debugHelpers: debug - ) - - let defaultNetwork = try await service.list() - .filter { $0.isBuiltin } - .first - if defaultNetwork == nil { - // FIXME: default network should be configurable elsewhere - _ = try await service.create(configuration: defaultNetworkConfig) - } - - let harness = NetworksHarness(service: service, log: log) - - if #available(macOS 26, *) { - routes[XPCRoute.networkCreate] = XPCServer.route(harness.create) - } - routes[XPCRoute.networkList] = XPCServer.route(harness.list) - routes[XPCRoute.networkDelete] = XPCServer.route(harness.delete) - - return service - } - private func initializeVolumeService( - containersService: ContainersService, log: Logger, routes: inout [XPCRoute: XPCServer.RouteHandler] ) async throws -> VolumesService { log.info("initializing volume service") let resourceRoot = appRoot.appending(FilePath.Component("volumes")) - let service = try await VolumesService(resourceRoot: resourceRoot, containersService: containersService, log: log) + let service = try await VolumesService(resourceRoot: resourceRoot, log: log) let harness = VolumesHarness(service: service, log: log) routes[XPCRoute.volumeCreate] = XPCServer.route(harness.create) @@ -378,7 +275,6 @@ extension APIServer { } private func initializeDiskUsageService( - containersService: ContainersService, volumesService: VolumesService, log: Logger, routes: inout [XPCRoute: XPCServer.RouteHandler] @@ -386,7 +282,6 @@ extension APIServer { log.info("initializing disk usage service") let service = DiskUsageService( - containersService: containersService, volumesService: volumesService, log: log ) diff --git a/Sources/APIServer/ContainerDNSHandler.swift b/Sources/APIServer/ContainerDNSHandler.swift index 78a207467..a7143e8a6 100644 --- a/Sources/APIServer/ContainerDNSHandler.swift +++ b/Sources/APIServer/ContainerDNSHandler.swift @@ -14,17 +14,18 @@ // limitations under the License. //===----------------------------------------------------------------------===// +import ContainerAPIClient import ContainerAPIService import ContainerizationExtras import DNSServer /// Handler that uses table lookup to resolve hostnames. struct ContainerDNSHandler: DNSHandler { - private let networkService: NetworksService + private let networks: NetworkClient private let ttl: UInt32 - public init(networkService: NetworksService, ttl: UInt32 = 5) { - self.networkService = networkService + public init(networks: NetworkClient, ttl: UInt32 = 5) { + self.networks = networks self.ttl = ttl } @@ -76,7 +77,7 @@ struct ContainerDNSHandler: DNSHandler { } private func answerHost(question: Question) async throws -> ResourceRecord? { - guard let ipAllocation = try await networkService.lookup(hostname: question.name) else { + guard let ipAllocation = try await networks.lookup(hostname: question.name) else { return nil } let ipv4 = ipAllocation.ipv4Address.address.description @@ -88,7 +89,7 @@ struct ContainerDNSHandler: DNSHandler { } private func answerHost6(question: Question) async throws -> (record: ResourceRecord?, hostnameExists: Bool) { - guard let ipAllocation = try await networkService.lookup(hostname: question.name) else { + guard let ipAllocation = try await networks.lookup(hostname: question.name) else { return (nil, false) } guard let ipv6Address = ipAllocation.ipv6Address else { diff --git a/Sources/ContainerCommands/Application.swift b/Sources/ContainerCommands/Application.swift index 6845bb15c..35a989357 100644 --- a/Sources/ContainerCommands/Application.swift +++ b/Sources/ContainerCommands/Application.swift @@ -63,7 +63,6 @@ public struct Application: AsyncLoggableCommand { ContainerKill.self, ContainerList.self, ContainerLogs.self, - ContainerRun.self, ContainerStart.self, ContainerStats.self, ContainerStop.self, diff --git a/Sources/ContainerCommands/Container/AnonymousResources.swift b/Sources/ContainerCommands/Container/AnonymousResources.swift new file mode 100644 index 000000000..e464e83ec --- /dev/null +++ b/Sources/ContainerCommands/Container/AnonymousResources.swift @@ -0,0 +1,81 @@ +//===----------------------------------------------------------------------===// +// Copyright © 2026 Apple Inc. and the container project authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//===----------------------------------------------------------------------===// + +import ContainerAPIClient +import ContainerResource +import Logging + +/// The resources a container was given rather than asked for by name. +/// +/// A volume mounted without a name, and the pod a container that named none was +/// given, are both made because the container needed one and are of no use to +/// anyone once it is gone. They are read off the container before it is removed, +/// since the container is what records them, and taken away after. +/// +/// nerdctl removes these with the container when the container is removed with +/// its volumes, when it was run to be removed on exit, and when containers are +/// pruned, and leaves them alone on a plain removal. +/// https://github.com/containerd/nerdctl/blob/main/pkg/cmd/container/remove.go +public struct AnonymousResources: Sendable { + let volumes: [String] + let pod: String? + + /// Read what a container was given, before it is removed. + public static func given(to container: ContainerSnapshot) async -> AnonymousResources { + var volumes: [String] = [] + for mount in container.configuration.mounts { + guard mount.isVolume, let name = mount.volumeName else { + continue + } + guard let volume = try? await ClientVolume.inspect(name), volume.isAnonymous else { + continue + } + volumes.append(name) + } + + let pod = try? await ClientPod.inspect(container.configuration.pod) + return AnonymousResources( + volumes: volumes, + pod: (pod?.configuration.isAnonymous ?? false) ? pod?.configuration.id : nil + ) + } + + /// Take them away, now that the container that was given them is gone. + /// + /// A resource that will not go is reported and passed over: the container it + /// belonged to is already gone, so failing here would fail a removal that + /// has already happened. + public func remove(log: Logger) async { + for volume in volumes { + do { + try await ClientVolume.delete(name: volume) + } catch { + log.warning( + "failed to remove an anonymous volume", + metadata: ["volume": "\(volume)", "error": "\(error)"]) + } + } + if let pod { + do { + try await ClientPod.delete(pod, force: true) + } catch { + log.warning( + "failed to remove an anonymous pod", + metadata: ["pod": "\(pod)", "error": "\(error)"]) + } + } + } +} diff --git a/Sources/ContainerCommands/Container/ContainerCreate.swift b/Sources/ContainerCommands/Container/ContainerCreate.swift index 97febb060..4a4279f29 100644 --- a/Sources/ContainerCommands/Container/ContainerCreate.swift +++ b/Sources/ContainerCommands/Container/ContainerCreate.swift @@ -91,6 +91,7 @@ extension Application { let options = ContainerCreateOptions(autoRemove: managementFlags.remove) let client = ContainerClient() + try await client.create(configuration: ck.0, options: options, kernel: ck.1, initImage: ck.2) if !self.managementFlags.cidfile.isEmpty { diff --git a/Sources/ContainerCommands/Container/ContainerDelete.swift b/Sources/ContainerCommands/Container/ContainerDelete.swift index 1eddc6b85..25892764c 100644 --- a/Sources/ContainerCommands/Container/ContainerDelete.swift +++ b/Sources/ContainerCommands/Container/ContainerDelete.swift @@ -35,6 +35,9 @@ extension Application { @Flag(name: .shortAndLong, help: "Delete containers even if they are running") var force = false + @Flag(name: .shortAndLong, help: "Remove the anonymous volumes and pod the container was given") + var volumes = false + @OptionGroup public var logOptions: Flags.Logging @@ -56,6 +59,8 @@ extension Application { public mutating func run() async throws { let client = ContainerClient() let force = self.force + let removeAnonymous = self.volumes + let log = self.log let containers: [String] if all { @@ -76,7 +81,13 @@ extension Application { for container in containers { group.addTask { do { + // What the container was given is recorded on the + // container, so it is read before the container goes. + let given = + removeAnonymous + ? await AnonymousResources.given(to: try await client.get(id: container)) : nil try await client.delete(id: container, force: force) + await given?.remove(log: log) print(container) return nil } catch { diff --git a/Sources/ContainerCommands/Container/ContainerStart.swift b/Sources/ContainerCommands/Container/ContainerStart.swift index 00bc47a1c..ce2876ffa 100644 --- a/Sources/ContainerCommands/Container/ContainerStart.swift +++ b/Sources/ContainerCommands/Container/ContainerStart.swift @@ -98,6 +98,17 @@ extension Application { if detach { try await process.start() try io.closeAfterStart() + // What this command says it did is that the container is + // running, so it says so only once the container answers + // that it is: a caller starting a container and then using + // it has nothing else to go on. + let started = try await client.get(id: container.id) + guard started.status == .running else { + throw ContainerizationError( + .invalidState, + message: "container \(container.id) did not start; it is \(started.status)" + ) + } print(self.containerId) return } diff --git a/Sources/ContainerCommands/Pod/PodCommand.swift b/Sources/ContainerCommands/Pod/PodCommand.swift new file mode 100644 index 000000000..2a88fcbcd --- /dev/null +++ b/Sources/ContainerCommands/Pod/PodCommand.swift @@ -0,0 +1,43 @@ +//===----------------------------------------------------------------------===// +// Copyright © 2026 Apple Inc. and the container project authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//===----------------------------------------------------------------------===// + +import ArgumentParser +import ContainerAPIClient + +extension Application { + public struct PodCommand: AsyncLoggableCommand { + public static let configuration = CommandConfiguration( + commandName: "pod", + abstract: "Manage pods, machines that several containers share", + subcommands: [ + PodCreate.self, + PodStart.self, + PodStop.self, + PodDelete.self, + PodList.self, + PodInspect.self, + PodPrune.self, + PodUpdate.self, + ], + aliases: ["p"] + ) + + public init() {} + + @OptionGroup + public var logOptions: Flags.Logging + } +} diff --git a/Sources/ContainerCommands/Pod/PodCreate.swift b/Sources/ContainerCommands/Pod/PodCreate.swift new file mode 100644 index 000000000..62132f3a1 --- /dev/null +++ b/Sources/ContainerCommands/Pod/PodCreate.swift @@ -0,0 +1,113 @@ +//===----------------------------------------------------------------------===// +// Copyright © 2026 Apple Inc. and the container project authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//===----------------------------------------------------------------------===// + +import ArgumentParser +import ContainerAPIClient +import ContainerPersistence +import ContainerResource +import ContainerizationError +import Foundation + +extension Application.PodCommand { + public struct PodCreate: AsyncLoggableCommand { + public static let configuration = CommandConfiguration( + commandName: "create", + abstract: "Create a pod for containers to be placed in" + ) + + @OptionGroup + public var logOptions: Flags.Logging + + @OptionGroup + public var resource: Flags.Resource + + @OptionGroup + public var dns: Flags.DNS + + @Option(name: .long, help: "Hostname the pod's machine reports, which its containers share") + var hostname: String? + + @Option(name: .long, help: "Network to attach the pod to, which its containers share") + var network: [String] = [] + + @Flag(name: .long, help: "Let the pod's containers see each other's processes") + var sharePidNamespace: Bool = false + + @Flag(name: .long, help: "Expose nested virtualization to the pod's containers") + var virtualization: Bool = false + + @Flag(name: .long, help: "Enable Rosetta in the pod's containers") + var rosetta: Bool = false + + @Option(name: .long, help: "Key=value metadata for the pod") + var label: [String] = [] + + @Argument(help: "Name for the pod") + var name: String + + public init() {} + + public func run() async throws { + let containerSystemConfig: ContainerSystemConfig = try await Application.loadContainerSystemConfig() + + guard ManagedContainer.nameValid(name) else { + throw ContainerizationError(.invalidArgument, message: "pod name \(name) is not a valid name") + } + + var configuration = PodConfiguration(id: name) + configuration.resources = try Parser.resources( + cpus: resource.cpus, + memory: resource.memory, + swap: resource.swap, + defaultCPUs: containerSystemConfig.container.cpus, + defaultMemory: containerSystemConfig.container.memory, + defaultSwap: containerSystemConfig.container.swap + ) + configuration.hostname = hostname + configuration.shareProcessNamespace = sharePidNamespace + configuration.virtualization = virtualization + configuration.rosetta = rosetta + configuration.labels = try Parser.labels(label) + + // A pod records a DNS configuration whatever it was told, since + // what it was not told is filled from the network its machine + // comes up on: a record naming no resolver is what asks for that. + configuration.dns = ContainerConfiguration.DNSConfiguration( + nameservers: dns.nameservers, + domain: dns.domain, + searchDomains: dns.searchDomains, + options: dns.options + ) + + let parsedNetworks = try network.map { try Parser.network($0) } + let networkClient = NetworkClient() + let builtinNetworkId = try await networkClient.builtin?.id + configuration.networks = try Utility.getAttachmentConfigurations( + containerId: name, + builtinNetworkId: builtinNetworkId, + networks: parsedNetworks, + dnsDomain: containerSystemConfig.dns.domain, + ) + for attachmentConfiguration in configuration.networks { + _ = try await networkClient.get(id: attachmentConfiguration.network) + } + + let kernel = try await ClientKernel.getDefaultKernel(for: .current) + try await ClientPod.create(configuration: configuration, kernel: kernel) + print(name) + } + } +} diff --git a/Sources/ContainerCommands/Pod/PodLifecycle.swift b/Sources/ContainerCommands/Pod/PodLifecycle.swift new file mode 100644 index 000000000..3f3df786d --- /dev/null +++ b/Sources/ContainerCommands/Pod/PodLifecycle.swift @@ -0,0 +1,178 @@ +//===----------------------------------------------------------------------===// +// Copyright © 2026 Apple Inc. and the container project authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//===----------------------------------------------------------------------===// + +import ArgumentParser +import ContainerAPIClient +import ContainerResource +import ContainerizationError +import ContainerizationExtras +import Foundation + +extension Application.PodCommand { + public struct PodStart: AsyncLoggableCommand { + public static let configuration = CommandConfiguration( + commandName: "start", + abstract: "Boot a pod's machine, with the containers in it" + ) + + @OptionGroup + public var logOptions: Flags.Logging + + @Argument(help: "Pods to start") + var names: [String] + + public init() {} + + public func run() async throws { + // The caller's agent rides into every member the machine boots, + // the donation each sibling boot path carries. + var dynamicEnv: [String: String] = [:] + if let agent = ProcessInfo.processInfo.environment["SSH_AUTH_SOCK"] { + dynamicEnv["SSH_AUTH_SOCK"] = agent + } + for name in names { + try await ClientPod.start(name, dynamicEnv: dynamicEnv) + print(name) + } + } + } + + public struct PodStop: AsyncLoggableCommand { + public static let configuration = CommandConfiguration( + commandName: "stop", + abstract: "Stop a pod's machine, and with it every container inside" + ) + + @OptionGroup + public var logOptions: Flags.Logging + + @Argument(help: "Pods to stop") + var names: [String] + + public init() {} + + public func run() async throws { + for name in names { + try await ClientPod.stop(name) + print(name) + } + } + } + + public struct PodDelete: AsyncLoggableCommand { + public static let configuration = CommandConfiguration( + commandName: "delete", + abstract: "Delete one or more pods", + aliases: ["rm"] + ) + + @OptionGroup + public var logOptions: Flags.Logging + + @Flag(name: .shortAndLong, help: "Delete the pod's containers along with it") + var force: Bool = false + + @Argument(help: "Pods to delete") + var names: [String] + + public init() {} + + public func run() async throws { + for name in names { + try await ClientPod.delete(name, force: force) + print(name) + } + } + } + + public struct PodUpdate: AsyncLoggableCommand { + public static let configuration = CommandConfiguration( + commandName: "update", + abstract: "Hold a running pod to a memory size, which its containers share" + ) + + @OptionGroup + public var logOptions: Flags.Logging + + @Option( + name: .shortAndLong, + help: """ + Memory the pod's machine is to hold (1MiByte granularity), with optional K, M, G, \ + T, or P suffix. The guest gives back the difference, and takes it again when the \ + size is raised. + """ + ) + var memory: String + + @Argument(help: "Pod to hold") + var name: String + + public init() {} + + public func run() async throws { + let bytes = try Parser.memoryStringAsMiB(memory).mib() + try await ClientPod.update(name, memoryInBytes: bytes) + print(name) + } + } + + public struct PodInspect: AsyncLoggableCommand { + public static let configuration = CommandConfiguration( + commandName: "inspect", + abstract: "Display information about one or more pods" + ) + + @OptionGroup + public var logOptions: Flags.Logging + + @Argument(help: "Pods to inspect") + var names: [String] + + public init() {} + + public func run() async throws { + var snapshots: [PodSnapshot] = [] + for name in Set(names).sorted() { + snapshots.append(try await ClientPod.inspect(name)) + } + try Output.emit(Output.renderJSON(snapshots, options: .pretty)) + } + } + + public struct PodList: AsyncLoggableCommand { + public static let configuration = CommandConfiguration( + commandName: "list", + abstract: "List pods", + aliases: ["ls"] + ) + + @OptionGroup + public var logOptions: Flags.Logging + + @Flag(name: .shortAndLong, help: "Only output the pod names") + var quiet: Bool = false + + @Option(name: .long, help: "Format of the output") + var format: ListFormat = .table + + public init() {} + + public func run() async throws { + let pods = try await ClientPod.list() + try Output.render(payload: pods, display: pods, format: format, quiet: quiet) + } + } +} diff --git a/Sources/ContainerCommands/Pod/PodPrune.swift b/Sources/ContainerCommands/Pod/PodPrune.swift new file mode 100644 index 000000000..00debd81f --- /dev/null +++ b/Sources/ContainerCommands/Pod/PodPrune.swift @@ -0,0 +1,83 @@ +//===----------------------------------------------------------------------===// +// Copyright © 2026 Apple Inc. and the container project authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//===----------------------------------------------------------------------===// + +import ArgumentParser +import ContainerAPIClient +import ContainerResource +import Foundation + +extension Application.PodCommand { + public struct PodPrune: AsyncLoggableCommand { + public init() {} + public static let configuration = CommandConfiguration( + commandName: "prune", + abstract: "Remove anonymous pods with no containers in them") + + @Flag(name: .shortAndLong, help: "Remove pods that were named too, not only anonymous ones") + var all = false + + @OptionGroup + public var logOptions: Flags.Logging + + public func run() async throws { + let allPods = try await ClientPod.list() + + // Find all pods that hold no container + let client = ContainerClient() + let containers = try await client.list() + var podsInUse = Set() + for container in containers { + podsInUse.insert(container.configuration.pod) + } + + // A pod someone named is theirs, and an empty one is still theirs to + // put something in, so a prune leaves it alone unless asked for all + // of them. A pod nobody named was made because a container needed a + // machine, and is of no use to anyone once no container is in it. + // https://github.com/containerd/nerdctl/blob/main/pkg/cmd/volume/prune.go + let podsToPrune = allPods.filter { pod in + guard !podsInUse.contains(pod.configuration.id) else { + return false + } + return all || pod.configuration.isAnonymous + } + + var prunedPods = [String]() + + for pod in podsToPrune { + do { + try await ClientPod.delete(pod.configuration.id, force: true) + prunedPods.append(pod.configuration.id) + } catch { + log.error( + "failed to prune pod", + metadata: [ + "id": "\(pod.configuration.id)", + "error": "\(error)", + ] + ) + } + } + + if !prunedPods.isEmpty { + print("Deleted Pods:") + for pod in prunedPods { + print(pod) + } + } + } + } +} diff --git a/Sources/ContainerCommands/Pod/PodSnapshot+ListDisplayable.swift b/Sources/ContainerCommands/Pod/PodSnapshot+ListDisplayable.swift new file mode 100644 index 000000000..09dae9c97 --- /dev/null +++ b/Sources/ContainerCommands/Pod/PodSnapshot+ListDisplayable.swift @@ -0,0 +1,42 @@ +//===----------------------------------------------------------------------===// +// Copyright © 2026 Apple Inc. and the container project authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//===----------------------------------------------------------------------===// + +import ContainerResource +import ContainerizationExtras +import Foundation + +extension PodSnapshot: ListDisplayable { + public static var tableHeader: [String] { + ["ID", "STATE", "CPUS", "MEMORY", "CONTAINERS", "ADDRESS"] + } + + public var tableRow: [String] { + let formatter = ByteCountFormatter() + formatter.countStyle = .memory + return [ + id, + state == .ready ? "ready" : "not ready", + "\(configuration.resources.cpus)", + formatter.string(fromByteCount: Int64(configuration.resources.memoryInBytes)), + containers.isEmpty ? "" : containers.joined(separator: ","), + networks.first?.ipv4Address.description ?? "", + ] + } + + public var quietValue: String { + id + } +} diff --git a/Sources/ContainerPersistence/ContainerSystemConfig.swift b/Sources/ContainerPersistence/ContainerSystemConfig.swift index cd3a156f9..d18f26c5b 100644 --- a/Sources/ContainerPersistence/ContainerSystemConfig.swift +++ b/Sources/ContainerPersistence/ContainerSystemConfig.swift @@ -115,19 +115,25 @@ final public class BuildConfig: Codable, Sendable { final public class ContainerConfig: Codable, Sendable { public static let defaultCPUs = 4 public static let defaultMemory = try! MemorySize("1g") + /// No swap area, so a container stays within the memory it is given + /// unless the operator asks for headroom beyond it. + public static let defaultSwap = try! MemorySize("0") public let cpus: Int public let memory: MemorySize + public let swap: MemorySize - public init(cpus: Int = defaultCPUs, memory: MemorySize = defaultMemory) { + public init(cpus: Int = defaultCPUs, memory: MemorySize = defaultMemory, swap: MemorySize = defaultSwap) { self.cpus = cpus self.memory = memory + self.swap = swap } public init(from decoder: any Decoder) throws { let container = try decoder.container(keyedBy: CodingKeys.self) self.cpus = try container.decodeIfPresent(Int.self, forKey: .cpus) ?? Self.defaultCPUs self.memory = try container.decodeIfPresent(MemorySize.self, forKey: .memory) ?? Self.defaultMemory + self.swap = try container.decodeIfPresent(MemorySize.self, forKey: .swap) ?? Self.defaultSwap } } diff --git a/Sources/ContainerPersistence/EntityStore.swift b/Sources/ContainerPersistence/EntityStore.swift index 60b080582..b3c0a0fa9 100644 --- a/Sources/ContainerPersistence/EntityStore.swift +++ b/Sources/ContainerPersistence/EntityStore.swift @@ -45,6 +45,11 @@ public actor FilesystemEntityStore: EntityStore where T: Codable & Identifiab self.path = path self.type = type self.log = log + // The store keeps what it holds under this directory, so it is the + // store's to make: a caller that has never written an entity has no + // reason to have made somewhere to put them, and a store that reads + // before anything is written finds nothing rather than failing. + try FileManager.default.createDirectory(atPath: path.string, withIntermediateDirectories: true) self.index = try Self.load(path: path, log: log) } diff --git a/Sources/ContainerResource/Container/Bundle.swift b/Sources/ContainerResource/Container/Bundle.swift index 217531b8c..36f418a54 100644 --- a/Sources/ContainerResource/Container/Bundle.swift +++ b/Sources/ContainerResource/Container/Bundle.swift @@ -23,9 +23,11 @@ public struct Bundle: Sendable { private static let kernelFilename = "kernel.json" private static let kernelBinaryFilename = "kernel.bin" private static let containerRootFsBlockFilename = "rootfs.ext4" + private static let containerSwapBlockFilename = "swap.raw" private static let containerRootFsFilename = "rootfs.json" static let containerConfigFilename = "config.json" + static let podConfigFilename = "pod.json" /// The path to the bundle. public let path: URL @@ -42,6 +44,11 @@ public struct Bundle: Sendable { self.path.appendingPathComponent(Self.containerRootFsBlockFilename) } + /// The raw block file backing the container's swap area, when it has one. + public var containerSwapBlock: URL { + self.path.appendingPathComponent(Self.containerSwapBlockFilename) + } + private var containerRootfsConfig: URL { self.path.appendingPathComponent(Self.containerRootFsFilename) } @@ -75,6 +82,22 @@ public struct Bundle: Sendable { try load(path: self.path.appendingPathComponent(Self.containerConfigFilename)) } } + + /// The configuration of the pod this bundle holds, when the bundle is a + /// pod's rather than a single container's. + public var podConfiguration: PodConfiguration { + get throws { + try load(path: self.path.appendingPathComponent(Self.podConfigFilename)) + } + } + + /// Whether this bundle holds a pod, whose containers keep bundles of their + /// own, rather than a single container. + public var isPod: Bool { + FileManager.default.fileExists( + atPath: self.path.appendingPathComponent(Self.podConfigFilename).path + ) + } } extension Bundle { @@ -83,6 +106,7 @@ extension Bundle { initialFilesystem: Filesystem, kernel: Kernel, containerConfiguration: ContainerConfiguration? = nil, + podConfiguration: PodConfiguration? = nil, containerRootFilesystem: Filesystem? = nil, options: ContainerCreateOptions? = nil ) throws -> Bundle { @@ -110,6 +134,10 @@ extension Bundle { try bundle.write(filename: Self.containerConfigFilename, value: containerConfiguration) } + if let podConfiguration { + try bundle.write(filename: Self.podConfigFilename, value: podConfiguration) + } + if let rootFsOverride = options?.rootFsOverride { try bundle.setContainerRootFs(fs: rootFsOverride) } else if let containerRootFilesystem { diff --git a/Sources/ContainerResource/Container/ContainerConfiguration.swift b/Sources/ContainerResource/Container/ContainerConfiguration.swift index 87e0f9049..09975c55f 100644 --- a/Sources/ContainerResource/Container/ContainerConfiguration.swift +++ b/Sources/ContainerResource/Container/ContainerConfiguration.swift @@ -30,8 +30,6 @@ public struct ContainerConfiguration: Sendable, Codable { public var publishedSockets: [PublishSocket] = [] /// Key/Value labels for the container. public var labels: [String: String] = [:] - /// System controls for the container. - public var sysctls: [String: String] = [:] /// The networks the container will be added to. public var networks: [AttachmentConfiguration] = [] /// The DNS configuration for the container. @@ -45,6 +43,15 @@ public struct ContainerConfiguration: Sendable, Codable { /// Resource values for the container. public var resources: Resources = .init() /// Name of the runtime that supports the container. + /// The pod the container runs in. + /// + /// A container is always in one, sharing that pod's machine with whatever + /// else is in it, so a container alone in a pod is the same arrangement + /// with one member. A caller that names no pod is asking for one of its + /// own and is given a name for it before the container is made, which is + /// why this is not a question the rest of the code has to ask. + public var pod: String = PodConfiguration.generateId() + public var runtimeHandler: String = "container-runtime-linux" /// Configure exposing virtualization support in the container. public var virtualization: Bool = false @@ -80,13 +87,13 @@ public struct ContainerConfiguration: Sendable, Codable { case publishedPorts case publishedSockets case labels - case sysctls case networks case dns case rosetta case initProcess case platform case resources + case pod case runtimeHandler case virtualization case ssh @@ -112,7 +119,6 @@ public struct ContainerConfiguration: Sendable, Codable { publishedPorts = try container.decodeIfPresent([PublishPort].self, forKey: .publishedPorts) ?? [] publishedSockets = try container.decodeIfPresent([PublishSocket].self, forKey: .publishedSockets) ?? [] labels = try container.decodeIfPresent([String: String].self, forKey: .labels) ?? [:] - sysctls = try container.decodeIfPresent([String: String].self, forKey: .sysctls) ?? [:] if container.contains(.networks) { networks = try container.decode([AttachmentConfiguration].self, forKey: .networks) @@ -125,6 +131,12 @@ public struct ContainerConfiguration: Sendable, Codable { initProcess = try container.decode(ProcessConfiguration.self, forKey: .initProcess) platform = try container.decodeIfPresent(ContainerizationOCI.Platform.self, forKey: .platform) ?? .current resources = try container.decodeIfPresent(Resources.self, forKey: .resources) ?? .init() + // A container written before a container was always in a pod carries + // no pod, and there is no machine for it to be in. It fails to read, + // and a container that fails to read is taken away at boot, which is + // what already happens to any container this version cannot make sense + // of. + pod = try container.decode(String.self, forKey: .pod) runtimeHandler = try container.decodeIfPresent(String.self, forKey: .runtimeHandler) ?? "container-runtime-linux" virtualization = try container.decodeIfPresent(Bool.self, forKey: .virtualization) ?? false ssh = try container.decodeIfPresent(Bool.self, forKey: .ssh) ?? false @@ -166,6 +178,11 @@ public struct ContainerConfiguration: Sendable, Codable { public var cpus: Int = 4 /// Memory in bytes allocated. public var memoryInBytes: UInt64 = 1024.mib() + /// Swap in bytes allocated. When set, a raw block device of this size + /// backs the container's swap area, which the guest enables so that a + /// workload exceeding `memoryInBytes` reclaims to it. Counts swap + /// alone, not the memory and swap total the runtime spec carries. + public var swapInBytes: UInt64? /// Storage quota/size in bytes. public var storage: UInt64? /// Additional CPU cores allocated for VM overhead (guest agent, etc). @@ -177,6 +194,7 @@ public struct ContainerConfiguration: Sendable, Codable { let c = try decoder.container(keyedBy: CodingKeys.self) self.cpus = try c.decodeIfPresent(Int.self, forKey: .cpus) ?? 4 self.memoryInBytes = try c.decodeIfPresent(UInt64.self, forKey: .memoryInBytes) ?? 1024.mib() + self.swapInBytes = try c.decodeIfPresent(UInt64.self, forKey: .swapInBytes) self.storage = try c.decodeIfPresent(UInt64.self, forKey: .storage) self.cpuOverhead = try c.decodeIfPresent(Int.self, forKey: .cpuOverhead) ?? 1 } diff --git a/Sources/ContainerResource/Network/Attachment.swift b/Sources/ContainerResource/Network/Attachment.swift index a6351ab3b..285d7c1de 100644 --- a/Sources/ContainerResource/Network/Attachment.swift +++ b/Sources/ContainerResource/Network/Attachment.swift @@ -29,6 +29,8 @@ public struct Attachment: Codable, Sendable { /// The CIDR address describing the interface IPv6 address, with the prefix length of the subnet. /// The address is nil if the IPv6 subnet could not be determined at network creation time. public let ipv6Address: CIDRv6? + /// The IPv6 gateway address, nil when the network carries no IPv6 subnet. + public let ipv6Gateway: IPv6Address? /// The MAC address associated with the attachment (optional). public let macAddress: MACAddress? /// The MTU for the network interface. @@ -42,6 +44,7 @@ public struct Attachment: Codable, Sendable { ipv4Address: CIDRv4, ipv4Gateway: IPv4Address, ipv6Address: CIDRv6?, + ipv6Gateway: IPv6Address? = nil, macAddress: MACAddress?, mtu: UInt32? = nil, variant: String? = nil @@ -51,6 +54,7 @@ public struct Attachment: Codable, Sendable { self.ipv4Address = ipv4Address self.ipv4Gateway = ipv4Gateway self.ipv6Address = ipv6Address + self.ipv6Gateway = ipv6Gateway self.macAddress = macAddress self.mtu = mtu self.variant = variant @@ -62,6 +66,7 @@ public struct Attachment: Codable, Sendable { case ipv4Address case ipv4Gateway case ipv6Address + case ipv6Gateway case macAddress case mtu case variant @@ -88,6 +93,7 @@ public struct Attachment: Codable, Sendable { ipv4Gateway = try container.decode(IPv4Address.self, forKey: .gateway) } ipv6Address = try container.decodeIfPresent(CIDRv6.self, forKey: .ipv6Address) + ipv6Gateway = try container.decodeIfPresent(IPv6Address.self, forKey: .ipv6Gateway) macAddress = try container.decodeIfPresent(MACAddress.self, forKey: .macAddress) mtu = try container.decodeIfPresent(UInt32.self, forKey: .mtu) variant = try container.decodeIfPresent(String.self, forKey: .variant) @@ -102,6 +108,7 @@ public struct Attachment: Codable, Sendable { try container.encode(ipv4Address, forKey: .ipv4Address) try container.encode(ipv4Gateway, forKey: .ipv4Gateway) try container.encodeIfPresent(ipv6Address, forKey: .ipv6Address) + try container.encodeIfPresent(ipv6Gateway, forKey: .ipv6Gateway) try container.encodeIfPresent(macAddress, forKey: .macAddress) try container.encodeIfPresent(mtu, forKey: .mtu) try container.encodeIfPresent(variant, forKey: .variant) diff --git a/Sources/ContainerResource/Pod/PodConfiguration.swift b/Sources/ContainerResource/Pod/PodConfiguration.swift new file mode 100644 index 000000000..d250758ff --- /dev/null +++ b/Sources/ContainerResource/Pod/PodConfiguration.swift @@ -0,0 +1,160 @@ +//===----------------------------------------------------------------------===// +// Copyright © 2026 Apple Inc. and the container project authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//===----------------------------------------------------------------------===// + +import ContainerizationOCI +import Foundation + +/// The configuration of a pod. +/// +/// The shape follows the runtime interface's `PodSandboxConfig`. +/// https://github.com/kubernetes/cri-api/blob/master/pkg/apis/runtime/v1/api.proto +public struct PodConfiguration: Sendable, Codable { + /// Identifier for the pod. + public var id: String + + /// Mint ids the way containers mint theirs, a lowercased UUID. + /// + /// A pod is a lifetime rather than a description of what it holds, so it + /// is told apart by the id it is given. Two pods may hold containers from + /// the same image, and a pod named after a container it holds would claim + /// the runtime service that container claims. + public static func generateId() -> String { UUID().uuidString.lowercased() } + + /// The runtime that runs the pod's machine. + /// + /// A pod is one machine, so its containers run under one runtime, and it + /// is the pod that names it. A container placed in a pod asking for a + /// different one is asking for a machine this pod is not. + public var runtimeHandler: String = "container-runtime-linux" + + /// Resources like cpu, memory and swap. A container may hold its own + /// limit within these; left alone it draws on the whole pool. + public var resources: ContainerConfiguration.Resources = .init() + + /// The hostname for the pod. + public var hostname: String? + + /// The DNS configuration for the pod. + public var dns: ContainerConfiguration.DNSConfiguration? + + /// Kernel parameters for the pod. Its containers share one kernel, so none + /// of them can set one for itself alone. + public var sysctls: [String: String] = [:] + + /// The networks the pod attaches to. + public var networks: [AttachmentConfiguration] = [] + + /// Ports published to the host. + public var publishedPorts: [PublishPort] = [] + + /// Whether the pod's containers see each other's processes. + public var shareProcessNamespace: Bool = false + + /// Enable nested virtualization support. + public var virtualization: Bool = false + + /// Enable Rosetta. + public var rosetta: Bool = false + + /// Key-value properties for the pod. + public var labels: [String: String] = [:] + + /// Configured platform for the pod. + public var platform: ContainerizationOCI.Platform = .current + + /// The init image the pod's machine boots. + /// + /// A pod clones the image's filesystem when it is made and boots that + /// clone for as long as it lives, so the image it was made from is the + /// only account of which agent its containers talk to. A caller comparing + /// this against the init image the runtime is configured with is asking + /// whether the machine still matches the plane driving it. + public var initImage: ImageDescription? + + /// The time at which the pod was created. + public var creationDate: Date = Date() + + public init(id: String) { + self.id = id + } + + /// The sandbox a container asks for when it names no pod of its own. + /// + /// The runtime interface has the caller create a sandbox and then create + /// containers in it, so a container that came without one has a sandbox + /// made for it first, out of the fields that are the sandbox's to hold. + /// https://github.com/kubernetes/cri-api/blob/master/pkg/apis/runtime/v1/api.proto + public init(sandboxFor container: ContainerConfiguration) { + self.init(id: container.pod) + runtimeHandler = container.runtimeHandler + resources = container.resources + dns = container.dns + networks = container.networks + publishedPorts = container.publishedPorts + virtualization = container.virtualization + rosetta = container.rosetta + platform = container.platform + labels = container.labels + // Nobody named this pod. It carries a name of its own because the + // container needed a machine to run in and was given one, which is what + // a volume mounted without a name is. + labels[Self.anonymousLabel] = "" + } + + /// Read a pod written by any version of this service. + /// + /// A pod outlives the process that wrote it, so a field this type gains is + /// a field the pods already on disk do not carry. The compiler's own + /// decoding asks for every key and fails on the first one missing, which + /// would make every pod written before the field unreadable, and a pod + /// that cannot be read is a pod that is not there: the service skips it, + /// and the container it holds is told its pod does not exist. Each field + /// that has a default is taken as absent-means-default, which is how the + /// container configuration reads its own. + public init(from decoder: Decoder) throws { + let values = try decoder.container(keyedBy: CodingKeys.self) + id = try values.decode(String.self, forKey: .id) + runtimeHandler = try values.decodeIfPresent(String.self, forKey: .runtimeHandler) ?? "container-runtime-linux" + resources = try values.decodeIfPresent(ContainerConfiguration.Resources.self, forKey: .resources) ?? .init() + hostname = try values.decodeIfPresent(String.self, forKey: .hostname) + dns = try values.decodeIfPresent(ContainerConfiguration.DNSConfiguration.self, forKey: .dns) + sysctls = try values.decodeIfPresent([String: String].self, forKey: .sysctls) ?? [:] + networks = try values.decodeIfPresent([AttachmentConfiguration].self, forKey: .networks) ?? [] + publishedPorts = try values.decodeIfPresent([PublishPort].self, forKey: .publishedPorts) ?? [] + shareProcessNamespace = try values.decodeIfPresent(Bool.self, forKey: .shareProcessNamespace) ?? false + virtualization = try values.decodeIfPresent(Bool.self, forKey: .virtualization) ?? false + rosetta = try values.decodeIfPresent(Bool.self, forKey: .rosetta) ?? false + labels = try values.decodeIfPresent([String: String].self, forKey: .labels) ?? [:] + platform = try values.decodeIfPresent(ContainerizationOCI.Platform.self, forKey: .platform) ?? .current + creationDate = try values.decodeIfPresent(Date.self, forKey: .creationDate) ?? Date() + } +} + +/// The runtime state of a pod. +public enum PodState: String, Sendable, Codable { + case ready + case notReady +} + +extension PodConfiguration { + /// Reserved label key for marking anonymous pods + public static let anonymousLabel = "com.apple.container.resource.anonymous" + + /// Whether this is an anonymous pod (detected via label) + public var isAnonymous: Bool { + labels[Self.anonymousLabel] != nil + } +} diff --git a/Sources/ContainerResource/Pod/PodSnapshot.swift b/Sources/ContainerResource/Pod/PodSnapshot.swift new file mode 100644 index 000000000..8f77f0556 --- /dev/null +++ b/Sources/ContainerResource/Pod/PodSnapshot.swift @@ -0,0 +1,64 @@ +//===----------------------------------------------------------------------===// +// Copyright © 2026 Apple Inc. and the container project authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//===----------------------------------------------------------------------===// + +import ContainerizationOCI +import Foundation + +/// A snapshot of a pod along with its configuration +/// and any runtime state information. +/// +/// The shape follows the runtime interface's `PodSandboxStatus`. +/// https://github.com/kubernetes/cri-api/blob/master/pkg/apis/runtime/v1/api.proto +public struct PodSnapshot: Codable, Sendable { + /// The configuration of the pod. + public var configuration: PodConfiguration + + /// Identifier of the pod. + public var id: String { + configuration.id + } + + /// Configured platform for the pod. + public var platform: ContainerizationOCI.Platform { + configuration.platform + } + + /// The runtime state of the pod. + public var state: PodState + + /// Network interfaces attached to the pod. + public var networks: [Attachment] + + /// Identifiers of the containers in the pod. + public var containers: [String] + + /// When the pod was started. + public var startedDate: Date? + + public init( + configuration: PodConfiguration, + state: PodState, + networks: [Attachment], + containers: [String] = [], + startedDate: Date? = nil + ) { + self.configuration = configuration + self.state = state + self.networks = networks + self.containers = containers + self.startedDate = startedDate + } +} diff --git a/Sources/Plugins/CoreContainers/ContainersHelper.swift b/Sources/Plugins/CoreContainers/ContainersHelper.swift new file mode 100644 index 000000000..39aea483c --- /dev/null +++ b/Sources/Plugins/CoreContainers/ContainersHelper.swift @@ -0,0 +1,296 @@ +//===----------------------------------------------------------------------===// +// Copyright © 2026 Apple Inc. and the container project authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//===----------------------------------------------------------------------===// + +import ArgumentParser +import ContainerAPIClient +import ContainerAPIService +import ContainerLog +import ContainerPersistence +import ContainerPlugin +import ContainerResource +import ContainerVersion +import ContainerXPC +import Foundation +import Logging +import SystemPackage + +@main +struct ContainersHelper: AsyncParsableCommand { + static let configuration = CommandConfiguration( + commandName: "container-core-containers", + abstract: "XPC service for managing containers and pods", + version: ReleaseVersion.singleLine(appName: "container-core-containers"), + subcommands: [ + Start.self + ] + ) +} + +extension ContainersHelper { + struct Start: AsyncParsableCommand { + static let configuration = CommandConfiguration( + commandName: "start", + abstract: "Starts the container and pod plugin" + ) + + @Flag(name: .long, help: "Enable debug logging") + var debug = false + + @Option(name: .long, help: "XPC service prefix") + var serviceIdentifier: String = "com.apple.container.core.container-core-containers" + + var appRoot = ApplicationRoot.path + + var installRoot = InstallRoot.path + + var logRoot = LogRoot.path + + func run() async throws { + let containerSystemConfig: ContainerSystemConfig = try await ConfigurationLoader.load() + let commandName = ContainersHelper._commandName + let logPath = logRoot.map { $0.appending("\(commandName).log") } + let log = ServiceLogger.bootstrap(category: "ContainersHelper", debug: debug, logPath: logPath) + log.info("starting helper", metadata: ["name": "\(commandName)"]) + defer { + log.info("stopping helper", metadata: ["name": "\(commandName)"]) + } + + do { + log.info("configuring XPC server") + let pluginLoader = try initializePluginLoader(log: log) + var routes = [String: XPCServer.RouteHandler]() + let containersService = try initializeContainersService( + pluginLoader: pluginLoader, + containerSystemConfig: containerSystemConfig, + log: log, + routes: &routes + ) + let podsService = try initializePodsService( + pluginLoader: pluginLoader, + containerSystemConfig: containerSystemConfig, + log: log, + routes: &routes + ) + + let networksService = try await initializeNetworksService( + pluginLoader: pluginLoader, + containerSystemConfig: containerSystemConfig, + log: log, + routes: &routes + ) + + // The three services address each other directly: a container + // bootstraps by asking its pod to run holding it, a pod forced + // away takes its containers with it, and both claim addresses + // from the networks they attach. They share one process so + // those calls keep their lock conventions. + await podsService.setContainersService(containersService) + await containersService.setPodsService(podsService) + await containersService.setNetworksService(networksService) + await podsService.setNetworksService(networksService) + + // Machines outlive the process that made them, so before + // serving, adopt the ones still running: pods dial their + // launchd services, containers take their machine's word. + await podsService.reconnect() + await containersService.reconnect() + + let xpc = XPCServer( + identifier: serviceIdentifier, + routes: routes, + log: log + ) + log.info("starting XPC server") + try await xpc.listen() + } catch { + log.error( + "helper failed", + metadata: [ + "name": "\(commandName)", + "error": "\(error)", + ]) + ContainersHelper.exit(withError: error) + } + } + + private func initializePluginLoader(log: Logger) throws -> PluginLoader { + log.info( + "initializing plugin loader", + metadata: [ + "installRoot": "\(installRoot.string)" + ]) + + // TODO: Remove when we convert PluginLoader to FilePath + let installRootURL = URL(fileURLWithPath: installRoot.string) + let pluginsURL = PluginLoader.userPluginsDir(installRoot: installRootURL) + log.info("detecting user plugins directory", metadata: ["path": "\(pluginsURL.path(percentEncoded: false))"]) + var directoryExists: ObjCBool = false + _ = FileManager.default.fileExists(atPath: pluginsURL.path, isDirectory: &directoryExists) + let userPluginsURL = directoryExists.boolValue ? pluginsURL : nil + + // plugins built into the application installed as a Unix-like application + let installRootPluginsPath = + installRoot + .appending(FilePath.Component("libexec")) + .appending(FilePath.Component("container")) + .appending(FilePath.Component("plugins")) + let installRootPluginsURL = URL(fileURLWithPath: installRootPluginsPath.string) + + let pluginDirectories = [ + userPluginsURL, + installRootPluginsURL, + ].compactMap { $0 } + + let pluginFactories: [PluginFactory] = [ + DefaultPluginFactory(logger: log), + AppBundlePluginFactory(logger: log), + ] + + for pluginDirectory in pluginDirectories { + log.info("discovered plugin directory", metadata: ["path": "\(pluginDirectory.path(percentEncoded: false))"]) + } + + let appRootURL = URL(fileURLWithPath: appRoot.string) + return try PluginLoader( + appRoot: appRootURL, + installRoot: installRootURL, + logRoot: logRoot, + pluginDirectories: pluginDirectories, + pluginFactories: pluginFactories, + log: log + ) + } + + private func initializeContainersService( + pluginLoader: PluginLoader, + containerSystemConfig: ContainerSystemConfig, + log: Logger, + routes: inout [String: XPCServer.RouteHandler] + ) throws -> ContainersService { + log.info("initializing containers service") + + // TODO: Remove when we convert ContainersService to FilePath + let appRootURL = URL(fileURLWithPath: appRoot.string) + let service = try ContainersService( + appRoot: appRootURL, + pluginLoader: pluginLoader, + containerSystemConfig: containerSystemConfig, + log: log, + debugHelpers: debug + ) + let harness = ContainersHarness(service: service, log: log) + + routes[XPCRoute.containerList.rawValue] = XPCServer.route(harness.list) + routes[XPCRoute.containerCreate.rawValue] = XPCServer.route(harness.create) + routes[XPCRoute.containerDelete.rawValue] = XPCServer.route(harness.delete) + routes[XPCRoute.containerLogs.rawValue] = XPCServer.route(harness.logs) + routes[XPCRoute.containerBootstrap.rawValue] = XPCServer.route(harness.bootstrap) + routes[XPCRoute.containerDial.rawValue] = XPCServer.route(harness.dial) + routes[XPCRoute.containerStop.rawValue] = XPCServer.route(harness.stop) + routes[XPCRoute.containerStartProcess.rawValue] = XPCServer.route(harness.startProcess) + routes[XPCRoute.containerCreateProcess.rawValue] = XPCServer.route(harness.createProcess) + routes[XPCRoute.containerResize.rawValue] = XPCServer.route(harness.resize) + routes[XPCRoute.containerWait.rawValue] = XPCServer.route(harness.wait) + routes[XPCRoute.containerKill.rawValue] = XPCServer.route(harness.kill) + routes[XPCRoute.containerStats.rawValue] = XPCServer.route(harness.stats) + routes[XPCRoute.containerDiskUsage.rawValue] = XPCServer.route(harness.diskUsage) + routes[XPCRoute.containerCopyIn.rawValue] = XPCServer.route(harness.copyIn) + routes[XPCRoute.containerCopyOut.rawValue] = XPCServer.route(harness.copyOut) + routes[XPCRoute.containerExport.rawValue] = XPCServer.route(harness.export) + routes[XPCRoute.containerVolumesInUse.rawValue] = XPCServer.route(harness.volumeNamesInUse) + routes[XPCRoute.containerVolumeReferences.rawValue] = XPCServer.route(harness.volumeReferences) + routes[XPCRoute.containerNetworkReferences.rawValue] = XPCServer.route(harness.networkReferences) + routes[XPCRoute.containerImageReferences.rawValue] = XPCServer.route(harness.imageReferences) + routes[XPCRoute.containerUsageTotals.rawValue] = XPCServer.route(harness.usageTotals) + + return service + } + + private func initializePodsService( + pluginLoader: PluginLoader, + containerSystemConfig: ContainerSystemConfig, + log: Logger, + routes: inout [String: XPCServer.RouteHandler] + ) throws -> PodsService { + log.info("initializing pods service") + + let appRootURL = URL(fileURLWithPath: appRoot.string) + let service = try PodsService( + appRoot: appRootURL, + pluginLoader: pluginLoader, + containerSystemConfig: containerSystemConfig, + debugHelpers: debug, + log: log + ) + let harness = PodsHarness(service: service, log: log) + + routes[XPCRoute.podCreate.rawValue] = XPCServer.route(harness.create) + routes[XPCRoute.podStart.rawValue] = XPCServer.route(harness.start) + routes[XPCRoute.podStop.rawValue] = XPCServer.route(harness.stop) + routes[XPCRoute.podDelete.rawValue] = XPCServer.route(harness.delete) + routes[XPCRoute.podInspect.rawValue] = XPCServer.route(harness.inspect) + routes[XPCRoute.podList.rawValue] = XPCServer.route(harness.list) + routes[XPCRoute.podUpdate.rawValue] = XPCServer.route(harness.update) + + return service + } + + private func initializeNetworksService( + pluginLoader: PluginLoader, + containerSystemConfig: ContainerSystemConfig, + log: Logger, + routes: inout [String: XPCServer.RouteHandler] + ) async throws -> NetworksService { + log.info("initializing networks service") + + let resourceRoot = appRoot.appending(FilePath.Component("networks")) + let defaultNetworkConfig = try NetworkConfiguration( + name: NetworkClient.defaultNetworkName, + mode: .nat, + ipv4Subnet: containerSystemConfig.network.subnet, + ipv6Subnet: containerSystemConfig.network.subnetv6, + labels: try .init([ResourceLabelKeys.role: ResourceRoleValues.builtin]), + plugin: "container-network-vmnet" + ) + let service = try await NetworksService( + pluginLoader: pluginLoader, + resourceRoot: resourceRoot, + defaultNetworkConfiguration: defaultNetworkConfig, + log: log, + debugHelpers: debug + ) + + let defaultNetwork = try await service.list() + .filter { $0.isBuiltin } + .first + if defaultNetwork == nil { + // FIXME: default network should be configurable elsewhere + _ = try await service.create(configuration: defaultNetworkConfig) + } + + let harness = NetworksHarness(service: service, log: log) + + if #available(macOS 26, *) { + routes[XPCRoute.networkCreate.rawValue] = XPCServer.route(harness.create) + } + routes[XPCRoute.networkList.rawValue] = XPCServer.route(harness.list) + routes[XPCRoute.networkDelete.rawValue] = XPCServer.route(harness.delete) + routes[XPCRoute.networkLookup.rawValue] = XPCServer.route(harness.lookup) + + return service + } + } +} diff --git a/Sources/Plugins/CoreContainers/config.toml b/Sources/Plugins/CoreContainers/config.toml new file mode 100644 index 000000000..1c2a6b3ea --- /dev/null +++ b/Sources/Plugins/CoreContainers/config.toml @@ -0,0 +1,12 @@ +abstract = "Core container and pod management plugin" +author = "Apple" +version = 0.1 + +[servicesConfig] +loadAtBoot = true +runAtLoad = false +defaultArguments = [] + +[[servicesConfig.services]] +type = "core" +description = "Provide an XPC interface to manage containers and the pods that machine them." diff --git a/Sources/Plugins/NetworkVmnet/FilesystemAttachmentLeaseStore.swift b/Sources/Plugins/NetworkVmnet/FilesystemAttachmentLeaseStore.swift new file mode 100644 index 000000000..619d489c1 --- /dev/null +++ b/Sources/Plugins/NetworkVmnet/FilesystemAttachmentLeaseStore.swift @@ -0,0 +1,45 @@ +//===----------------------------------------------------------------------===// +// Copyright © 2026 Apple Inc. and the container project authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//===----------------------------------------------------------------------===// + +import ContainerNetworkServer +import ContainerPersistence +import Logging + +/// Keeps what each host was given on disk, one lease to an entry, under the +/// directory belonging to the network that gave it. +struct FilesystemAttachmentLeaseStore: AttachmentLeaseStore { + let store: FilesystemEntityStore + let log: Logger + + func load() async -> [AttachmentAllocator.Lease] { + do { + return try await store.list() + } catch { + log.warning("cannot read what hosts were given", metadata: ["error": "\(error)"]) + return [] + } + } + + func save(_ lease: AttachmentAllocator.Lease) async { + do { + try await store.upsert(lease) + } catch { + log.warning( + "cannot write down what a host was given", + metadata: ["hostname": "\(lease.id)", "error": "\(error)"]) + } + } +} diff --git a/Sources/Plugins/NetworkVmnet/NetworkVmnetHelper+Start.swift b/Sources/Plugins/NetworkVmnet/NetworkVmnetHelper+Start.swift index 7d67f1f32..aefc762eb 100644 --- a/Sources/Plugins/NetworkVmnet/NetworkVmnetHelper+Start.swift +++ b/Sources/Plugins/NetworkVmnet/NetworkVmnetHelper+Start.swift @@ -19,11 +19,14 @@ import ContainerLog import ContainerNetworkClient import ContainerNetworkServer import ContainerNetworkVmnetServer +import ContainerPersistence import ContainerPlugin import ContainerResource import ContainerXPC import ContainerizationError import ContainerizationExtras +import ContainerizationOS +import Darwin import Foundation import Logging @@ -97,7 +100,22 @@ extension NetworkVmnetHelper { log: log ) try await network.start() - let service = try await DefaultNetworkService(network: network, log: log) + // The addresses this network hands out are written down beside + // the network they belong to, the way a host-local allocator + // keeps its allocations under a directory of its own. + // https://cni.dev/plugins/current/ipam/host-local/ + let leases = FilesystemAttachmentLeaseStore( + store: try FilesystemEntityStore( + path: PathUtils.BaseConfigPath.appRoot.basePath() + .appending("networks") + .appending(id) + .appending("leases"), + type: "lease", + log: log + ), + log: log + ) + let service = try await DefaultNetworkService(network: network, leases: leases, log: log) let harness = NetworkHarness(service: service) let xpc = XPCServer( identifier: serviceIdentifier, @@ -109,8 +127,31 @@ extension NetworkVmnetHelper { log: log ) + // What the network holds is given back when this helper is + // asked to go away, so the addresses it was given can be + // handed out again; a helper that exits still holding them + // leaves the range spoken for by nobody. + let signals = AsyncSignalHandler.create(notify: [SIGINT, SIGTERM]) + Task { + for await _ in signals.signals { + log.info("releasing the network before exit") + await network.stop() + Darwin.exit(0) + } + } + log.info("starting XPC server") - try await xpc.listen() + do { + try await xpc.listen() + } catch { + // Whatever ends the wait, the addresses go back: a helper + // that leaves holding them leaves them held by nobody, and + // the next network asking for that range is refused with + // no interface, route, or process to point at. + await network.stop() + throw error + } + await network.stop() } catch { log.error( "helper failed", diff --git a/Sources/Plugins/PodCLI/PodPlugin.swift b/Sources/Plugins/PodCLI/PodPlugin.swift new file mode 100644 index 000000000..6f1a9a54f --- /dev/null +++ b/Sources/Plugins/PodCLI/PodPlugin.swift @@ -0,0 +1,36 @@ +//===----------------------------------------------------------------------===// +// Copyright © 2026 Apple Inc. and the container project authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//===----------------------------------------------------------------------===// + +import ArgumentParser +import ContainerCommands + +@main +struct PodPlugin: AsyncParsableCommand { + static let configuration = CommandConfiguration( + commandName: "pod", + abstract: "Manage pods, machines that several containers share", + subcommands: [ + Application.PodCommand.PodCreate.self, + Application.PodCommand.PodStart.self, + Application.PodCommand.PodStop.self, + Application.PodCommand.PodDelete.self, + Application.PodCommand.PodList.self, + Application.PodCommand.PodInspect.self, + Application.PodCommand.PodPrune.self, + Application.PodCommand.PodUpdate.self, + ] + ) +} diff --git a/Sources/Plugins/PodCLI/config.toml b/Sources/Plugins/PodCLI/config.toml new file mode 100644 index 000000000..9255fd9ca --- /dev/null +++ b/Sources/Plugins/PodCLI/config.toml @@ -0,0 +1,3 @@ +abstract = "Manage pods, machines that several containers share" +author = "Apple" +version = 0.1 diff --git a/Sources/Plugins/RunCLI/RunPlugin.swift b/Sources/Plugins/RunCLI/RunPlugin.swift new file mode 100644 index 000000000..ceb883e18 --- /dev/null +++ b/Sources/Plugins/RunCLI/RunPlugin.swift @@ -0,0 +1,25 @@ +//===----------------------------------------------------------------------===// +// Copyright © 2026 Apple Inc. and the container project authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//===----------------------------------------------------------------------===// + +import ArgumentParser +import ContainerCommands + +@main +struct RunPlugin { + static func main() async { + await Application.ContainerRun.main() + } +} diff --git a/Sources/Plugins/RunCLI/config.toml b/Sources/Plugins/RunCLI/config.toml new file mode 100644 index 000000000..22a169ef5 --- /dev/null +++ b/Sources/Plugins/RunCLI/config.toml @@ -0,0 +1,3 @@ +abstract = "Run a container from an image" +author = "Apple" +version = 0.1 diff --git a/Sources/Plugins/RuntimeLinux/RuntimeLinuxHelper+Start.swift b/Sources/Plugins/RuntimeLinux/RuntimeLinuxHelper+Start.swift index d4c049b4b..c84753bb1 100644 --- a/Sources/Plugins/RuntimeLinux/RuntimeLinuxHelper+Start.swift +++ b/Sources/Plugins/RuntimeLinux/RuntimeLinuxHelper+Start.swift @@ -94,9 +94,11 @@ extension RuntimeLinuxHelper { connection: anonymousConnection, routes: [ RuntimeRoutes.bootstrap.rawValue: XPCServer.route(server.bootstrap), + RuntimeRoutes.updateResources.rawValue: XPCServer.route(server.updateResources), RuntimeRoutes.createProcess.rawValue: XPCServer.route(server.createProcess), RuntimeRoutes.state.rawValue: XPCServer.route(server.state), RuntimeRoutes.stop.rawValue: XPCServer.route(server.stop), + RuntimeRoutes.stopContainer.rawValue: XPCServer.route(server.stopContainer), RuntimeRoutes.kill.rawValue: XPCServer.route(server.kill), RuntimeRoutes.resize.rawValue: XPCServer.route(server.resize), RuntimeRoutes.wait.rawValue: XPCServer.route(server.wait), diff --git a/Sources/Services/ContainerAPIService/Client/ClientPod.swift b/Sources/Services/ContainerAPIService/Client/ClientPod.swift new file mode 100644 index 000000000..8c3f0163a --- /dev/null +++ b/Sources/Services/ContainerAPIService/Client/ClientPod.swift @@ -0,0 +1,124 @@ +//===----------------------------------------------------------------------===// +// Copyright © 2026 Apple Inc. and the container project authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//===----------------------------------------------------------------------===// + +import ContainerResource +import ContainerXPC +import Containerization +import ContainerizationError +import Foundation + +/// Pods: machines that several containers run inside and share. +public struct ClientPod { + static let serviceIdentifier = "com.apple.container.core.container-core-containers" + + /// Write down a pod, so containers can be placed in it before it boots. + public static func create( + configuration: PodConfiguration, + kernel: Kernel, + initImage: String? = nil + ) async throws { + let client = XPCClient(service: serviceIdentifier) + let message = XPCMessage(route: .podCreate) + message.set(key: .podConfig, value: try JSONEncoder().encode(configuration)) + message.set(key: .kernel, value: try JSONEncoder().encode(kernel)) + if let initImage { + message.set(key: .initImage, value: initImage) + } + _ = try await client.send(message) + } + + /// Make the sandbox a container is created in. + /// + /// The runtime interface has a sandbox created and left ready before + /// containers are created in it. Here a container joins its pod's machine + /// before that machine boots, and booting it is what starts the containers + /// placed in it, so the sandbox is written down here and comes up with its + /// container rather than ahead of it. + /// https://github.com/kubernetes/cri-api/blob/master/pkg/apis/runtime/v1/api.proto + public static func run( + configuration: PodConfiguration, + kernel: Kernel, + initImage: String? = nil + ) async throws { + try await Self.create(configuration: configuration, kernel: kernel, initImage: initImage) + } + + /// Boot a pod's machine, with the containers that belong to it inside. + /// + /// `dynamicEnv` carries per-boot environment such as the caller's + /// SSH_AUTH_SOCK to every container the machine starts, the same + /// donation a container's own start delivers. + public static func start(_ id: String, dynamicEnv: [String: String] = [:]) async throws { + let client = XPCClient(service: serviceIdentifier) + let message = XPCMessage(route: .podStart) + message.set(key: .podId, value: id) + if !dynamicEnv.isEmpty { + message.set(key: .dynamicEnv, value: try JSONEncoder().encode(dynamicEnv)) + } + _ = try await client.send(message) + } + + /// Stop a pod's machine, and with it every container inside. + public static func stop(_ id: String) async throws { + let client = XPCClient(service: serviceIdentifier) + let message = XPCMessage(route: .podStop) + message.set(key: .podId, value: id) + _ = try await client.send(message) + } + + /// Take a pod away, along with its containers when forced. + public static func delete(_ id: String, force: Bool = false) async throws { + let client = XPCClient(service: serviceIdentifier) + let message = XPCMessage(route: .podDelete) + message.set(key: .podId, value: id) + message.set(key: .forceDelete, value: force) + _ = try await client.send(message) + } + + /// Everything known about a pod, including the containers in it. + public static func inspect(_ id: String) async throws -> PodSnapshot { + let client = XPCClient(service: serviceIdentifier) + let message = XPCMessage(route: .podInspect) + message.set(key: .podId, value: id) + let reply = try await client.send(message) + + guard let data = reply.dataNoCopy(key: .podSnapshot) else { + throw ContainerizationError(.notFound, message: "pod not found: \(id)") + } + return try JSONDecoder().decode(PodSnapshot.self, from: data) + } + + /// Every pod. + public static func list() async throws -> [PodSnapshot] { + let client = XPCClient(service: serviceIdentifier) + let message = XPCMessage(route: .podList) + let reply = try await client.send(message) + + guard let data = reply.dataNoCopy(key: .podSnapshots) else { + return [] + } + return try JSONDecoder().decode([PodSnapshot].self, from: data) + } + + /// Hold a running pod to a memory size, which its containers share. + public static func update(_ id: String, memoryInBytes: UInt64) async throws { + let client = XPCClient(service: serviceIdentifier) + let message = XPCMessage(route: .podUpdate) + message.set(key: .podId, value: id) + message.set(key: .memoryInBytes, value: memoryInBytes) + _ = try await client.send(message) + } +} diff --git a/Sources/Services/ContainerAPIService/Client/ClientProcess.swift b/Sources/Services/ContainerAPIService/Client/ClientProcess.swift index 5a5f8543e..07d133625 100644 --- a/Sources/Services/ContainerAPIService/Client/ClientProcess.swift +++ b/Sources/Services/ContainerAPIService/Client/ClientProcess.swift @@ -43,7 +43,7 @@ public protocol ClientProcess: Sendable { } struct ClientProcessImpl: ClientProcess, Sendable { - static let serviceIdentifier = "com.apple.container.apiserver" + static let serviceIdentifier = "com.apple.container.core.container-core-containers" /// ID of the process. public var id: String { diff --git a/Sources/Services/ContainerAPIService/Client/ContainerClient.swift b/Sources/Services/ContainerAPIService/Client/ContainerClient.swift index 5a2b6d0d3..c12fdf261 100644 --- a/Sources/Services/ContainerAPIService/Client/ContainerClient.swift +++ b/Sources/Services/ContainerAPIService/Client/ContainerClient.swift @@ -27,7 +27,7 @@ import Foundation /// container lifecycle operations. All methods that operate on a specific /// container take an `id` parameter. public struct ContainerClient: Sendable { - private static let serviceIdentifier = "com.apple.container.apiserver" + private static let serviceIdentifier = "com.apple.container.core.container-core-containers" private let xpcClient: XPCClient @@ -82,6 +82,108 @@ public struct ContainerClient: Sendable { } } + /// The name of every volume a container mounts. + public func volumeNamesInUse() async throws -> Set { + do { + let request = XPCMessage(route: .containerVolumesInUse) + let response = try await xpcSend(message: request) + guard let data = response.dataNoCopy(key: .references) else { + return [] + } + return try JSONDecoder().decode(Set.self, from: data) + } catch let error as ContainerizationError { + throw error + } catch { + throw ContainerizationError( + .internalError, + message: "failed to list volumes in use", + cause: error + ) + } + } + + /// The containers that mount the named volume. + public func containersReferencingVolume(_ name: String) async throws -> [String] { + do { + let request = XPCMessage(route: .containerVolumeReferences) + request.set(key: .volumeName, value: name) + let response = try await xpcSend(message: request) + guard let data = response.dataNoCopy(key: .references) else { + return [] + } + return try JSONDecoder().decode([String].self, from: data) + } catch let error as ContainerizationError { + throw error + } catch { + throw ContainerizationError( + .internalError, + message: "failed to list containers referencing volume \(name)", + cause: error + ) + } + } + + /// The containers attached to the named network. + public func containersAttachedToNetwork(_ id: String) async throws -> [String] { + do { + let request = XPCMessage(route: .containerNetworkReferences) + request.set(key: .id, value: id) + let response = try await xpcSend(message: request) + guard let data = response.dataNoCopy(key: .references) else { + return [] + } + return try JSONDecoder().decode([String].self, from: data) + } catch let error as ContainerizationError { + throw error + } catch { + throw ContainerizationError( + .internalError, + message: "failed to list containers attached to network \(id)", + cause: error + ) + } + } + + /// The image references containers hold. + public func activeImageReferences() async throws -> Set { + do { + let request = XPCMessage(route: .containerImageReferences) + let response = try await xpcSend(message: request) + guard let data = response.dataNoCopy(key: .references) else { + return [] + } + return try JSONDecoder().decode(Set.self, from: data) + } catch let error as ContainerizationError { + throw error + } catch { + throw ContainerizationError( + .internalError, + message: "failed to list active image references", + cause: error + ) + } + } + + /// Disk usage totals for containers. + public func calculateDiskUsage() async throws -> ResourceUsage { + do { + let request = XPCMessage(route: .containerUsageTotals) + let response = try await xpcSend(message: request) + guard let data = response.dataNoCopy(key: .usageTotals) else { + throw ContainerizationError(.internalError, message: "usage totals missing from reply") + } + return try JSONDecoder().decode(ResourceUsage.self, from: data) + } catch let error as ContainerizationError { + throw error + } catch { + throw ContainerizationError( + .internalError, + message: "failed to calculate container disk usage", + cause: error + ) + } + } + /// List containers matching the given filters. public func list(filters: ContainerListFilters = .all) async throws -> [ContainerSnapshot] { do { diff --git a/Sources/Services/ContainerAPIService/Client/Flags.swift b/Sources/Services/ContainerAPIService/Client/Flags.swift index 39962d436..6e840e98d 100644 --- a/Sources/Services/ContainerAPIService/Client/Flags.swift +++ b/Sources/Services/ContainerAPIService/Client/Flags.swift @@ -101,9 +101,10 @@ public struct Flags { public struct Resource: ParsableArguments { public init() {} - public init(cpus: Int64?, memory: String?) { + public init(cpus: Int64?, memory: String?, swap: String? = nil) { self.cpus = cpus self.memory = memory + self.swap = swap } @Option(name: .shortAndLong, help: "Number of CPUs to allocate to the container") @@ -114,6 +115,16 @@ public struct Flags { help: "Amount of memory (1MiByte granularity), with optional K, M, G, T, or P suffix" ) public var memory: String? + + @Option( + name: .customLong("swap"), + help: """ + Amount of swap to give the container (1MiByte granularity), with optional K, M, G, \ + T, or P suffix. A workload whose memory exceeds its limit reclaims to it rather \ + than meeting the out of memory killer. Counts swap alone, not memory plus swap. + """ + ) + public var swap: String? } public struct DNS: ParsableArguments { @@ -184,6 +195,7 @@ public struct Flags { networks: [String], os: String, platform: String?, + pod: String? = nil, publishPorts: [String], publishSockets: [String], readOnly: Bool, @@ -216,6 +228,7 @@ public struct Flags { self.networks = networks self.os = os self.platform = platform + self.pod = pod self.publishPorts = publishPorts self.publishSockets = publishSockets self.readOnly = readOnly @@ -311,6 +324,15 @@ public struct Flags { @Option(name: .long, help: "Use the specified name as the container ID") public var name: String? + @Option( + name: .long, + help: """ + Run the container in a pod, whose machine it shares with the pod's other \ + containers. Without this the container is given a machine of its own. + """ + ) + public var pod: String? + @Option(name: [.customLong("network")], help: "Attach the container to a network (format: [,mac=XX:XX:XX:XX:XX:XX][,mtu=VALUE])") public var networks: [String] = [] diff --git a/Sources/Services/ContainerAPIService/Client/NetworkClient.swift b/Sources/Services/ContainerAPIService/Client/NetworkClient.swift index b7ea51f31..5af61aeea 100644 --- a/Sources/Services/ContainerAPIService/Client/NetworkClient.swift +++ b/Sources/Services/ContainerAPIService/Client/NetworkClient.swift @@ -38,7 +38,7 @@ public struct NetworkClient: Sendable { /// /// Pass a different value to ``init(serviceIdentifier:)`` to connect to an /// alternative service endpoint, for example during testing. - public static let defaultServiceIdentifier = "com.apple.container.apiserver" + public static let defaultServiceIdentifier = "com.apple.container.core.container-core-containers" /// The name of the default network created automatically on first use. public static let defaultNetworkName = "default" @@ -88,7 +88,7 @@ public struct NetworkClient: Sendable { return try JSONDecoder().decode(NetworkResource.self, from: resourceData) } - /// Returns the current state of all networks known to the API server. + /// Returns the current state of all networks the core plugin holds. /// /// - Returns: An array of ``NetworkResource`` values, or an empty array if no /// networks exist or the server returns no data. @@ -96,7 +96,11 @@ public struct NetworkClient: Sendable { public func list() async throws -> [NetworkResource] { let request = XPCMessage(route: .networkList) - let response = try await xpcSend(message: request, timeout: .seconds(1)) + // The route is served by a plugin, so the wait is the one a request to + // a service that may still have to launch is given. Launching one takes + // seconds, and a budget shorter than that fails a caller for the state + // of the machine rather than for anything it asked. + let response = try await xpcSend(message: request) guard let resourceData = response.dataNoCopy(key: .networkResources) else { return [] @@ -104,6 +108,25 @@ public struct NetworkClient: Sendable { return try JSONDecoder().decode([NetworkResource].self, from: resourceData) } + /// Resolve a container hostname to its network attachment. + /// + /// - Parameter hostname: A canonical DNS hostname with a trailing dot. + /// - Returns: The attachment whose hostname matches, or nil when no + /// network knows the name. + public func lookup(hostname: String) async throws -> Attachment? { + let request = XPCMessage(route: .networkLookup) + request.set(key: .hostname, value: hostname) + + // Served by the same plugin as the listing, and given the same wait for + // the same reason. + let response = try await xpcSend(message: request) + + guard let data = response.dataNoCopy(key: .attachment) else { + return nil + } + return try JSONDecoder().decode(Attachment.self, from: data) + } + /// Returns the network with the given identifier. /// /// - Parameter id: The identifier of the network to look up. diff --git a/Sources/Services/ContainerAPIService/Client/Parser.swift b/Sources/Services/ContainerAPIService/Client/Parser.swift index a52c1499e..57b9ce960 100644 --- a/Sources/Services/ContainerAPIService/Client/Parser.swift +++ b/Sources/Services/ContainerAPIService/Client/Parser.swift @@ -105,12 +105,18 @@ public struct Parser { public static func resources( cpus: Int64?, memory: String?, + swap: String? = nil, defaultCPUs: Int, defaultMemory: MemorySize, + defaultSwap: MemorySize = ContainerConfig.defaultSwap, ) throws -> ContainerConfiguration.Resources { var resource = ContainerConfiguration.Resources() resource.cpus = defaultCPUs resource.memoryInBytes = Int64(defaultMemory.measurement.converted(to: .mebibytes).value).mib() + let defaultSwapInBytes = Int64(defaultSwap.measurement.converted(to: .mebibytes).value).mib() + if defaultSwapInBytes > 0 { + resource.swapInBytes = defaultSwapInBytes + } if let cpus { resource.cpus = Int(cpus) @@ -120,6 +126,13 @@ public struct Parser { resource.memoryInBytes = try Parser.memoryStringAsMiB(memory).mib() } + // Left unset everywhere the container gets no swap area at all, which + // is what a container expected to stay within its memory wants; the + // flag overrides the configured default the way memory does. + if let swap { + resource.swapInBytes = try Parser.memoryStringAsMiB(swap).mib() + } + return resource } diff --git a/Sources/Services/ContainerAPIService/Client/Utility.swift b/Sources/Services/ContainerAPIService/Client/Utility.swift index f6329c35a..4c4a6af76 100644 --- a/Sources/Services/ContainerAPIService/Client/Utility.swift +++ b/Sources/Services/ContainerAPIService/Client/Utility.swift @@ -153,12 +153,32 @@ public struct Utility { var config = ContainerConfiguration(id: id, image: description, process: pc) config.platform = requestedPlatform - config.resources = try Parser.resources( - cpus: resource.cpus, - memory: resource.memory, - defaultCPUs: containerSystemConfig.container.cpus, - defaultMemory: containerSystemConfig.container.memory - ) + if let pod = management.pod { + // A container in a pod falls back to what the pod's machine holds + // rather than to what a machine of its own would be given, so a + // container that named no limit draws on the whole of the pod's, + // and one that named a limit still holds to it. + var podResources = try await ClientPod.inspect(pod).configuration.resources + if let cpus = resource.cpus { + podResources.cpus = Int(cpus) + } + if let memory = resource.memory { + podResources.memoryInBytes = try Parser.memoryStringAsMiB(memory).mib() + } + if let swap = resource.swap { + podResources.swapInBytes = try Parser.memoryStringAsMiB(swap).mib() + } + config.resources = podResources + } else { + config.resources = try Parser.resources( + cpus: resource.cpus, + memory: resource.memory, + swap: resource.swap, + defaultCPUs: containerSystemConfig.container.cpus, + defaultMemory: containerSystemConfig.container.memory, + defaultSwap: containerSystemConfig.container.swap + ) + } let tmpfs = try Parser.tmpfsMounts(management.tmpFs) let volumesOrFs = try Parser.volumes(management.volumes) @@ -235,6 +255,54 @@ public struct Utility { } config.labels = try Parser.labels(management.labels) + // A container runs in a pod. Naming none asks for one of its own, so + // one is named here rather than left for something later to notice was + // missing, which is how a sandbox is made for a container that came + // without one. + // https://github.com/kubernetes/cri-api/blob/master/pkg/apis/runtime/v1/api.proto + config.pod = management.pod ?? PodConfiguration.generateId() + + // Naming a pod joins one that is already there, so it has to be there, + // and joining it is joining its network: how that network is reached is + // the pod's to have been given, and asking for it here is asking the pod + // to be something it already is. Both are refused rather than accepted + // and ignored, which is how nerdctl answers a container joining + // another's network. The name a container answers to travels inside its + // attachment, so --hostname is covered by --network. + // https://github.com/containerd/nerdctl/blob/main/pkg/containerutil/container_network_manager.go + if let named = management.pod { + guard let pod = try? await ClientPod.inspect(named) else { + throw ContainerizationError(.notFound, message: "pod \(named) does not exist") + } + var held: [String] = [] + if management.rosetta { held.append("--rosetta") } + if management.virtualization { held.append("--virtualization") } + if !management.publishPorts.isEmpty { held.append("-p/--publish") } + if !management.dns.nameservers.isEmpty || management.dns.domain != nil + || !management.dns.searchDomains.isEmpty || !management.dns.options.isEmpty + { + held.append("--dns") + } + if !management.networks.isEmpty { held.append("--network") } + guard held.isEmpty else { + throw ContainerizationError( + .invalidArgument, + message: + "these belong to the pod whose machine the container runs in, so they are not the container's to ask for: \(held.joined(separator: ", "))" + ) + } + + // Running a foreign architecture is the machine's to do, and the + // machine was booted with or without it before this container + // existed, so a container that needs it says so rather than + // starting in a machine that cannot run it. + if Platform.current.architecture == "arm64", requestedPlatform.architecture == "amd64", !pod.configuration.rosetta { + throw ContainerizationError( + .invalidArgument, + message: "pod \(named) runs a machine without Rosetta, which \(requestedPlatform.description) needs; create the pod with --rosetta" + ) + } + } config.publishedPorts = try Parser.publishPorts(management.publishPorts) guard config.publishedPorts.count <= publishedPortCountLimit else { @@ -266,7 +334,9 @@ public struct Utility { return (config, kernel, management.initImage) } - static func getAttachmentConfigurations( + /// The networks a container or a pod attaches to, resolved from what the + /// caller named and the built-in network when it named none. + public static func getAttachmentConfigurations( containerId: String, builtinNetworkId: String?, networks: [Parser.ParsedNetwork], diff --git a/Sources/Services/ContainerAPIService/Client/XPC+.swift b/Sources/Services/ContainerAPIService/Client/XPC+.swift index a4d5aebd3..1ab481331 100644 --- a/Sources/Services/ContainerAPIService/Client/XPC+.swift +++ b/Sources/Services/ContainerAPIService/Client/XPC+.swift @@ -117,10 +117,25 @@ public enum XPCKeys: String { /// Init image reference case initImage + /// Pod + case podId + case podConfig + case podSnapshot + case podSnapshots + case memoryInBytes + /// Volume case volume case volumes case volumeName + /// JSON array of container, volume, or image identifiers a query returns. + case references + /// JSON UsageTotals a resource kind reports. + case usageTotals + /// A canonical DNS hostname, with its trailing dot. + case hostname + /// JSON Attachment a network lookup returns. + case attachment case volumeSize case volumeDriver case volumeDriverOpts @@ -165,6 +180,11 @@ public enum XPCRoute: String { case containerCopyIn case containerCopyOut case containerExport + case containerVolumesInUse + case containerVolumeReferences + case containerNetworkReferences + case containerImageReferences + case containerUsageTotals case pluginLoad case pluginGet @@ -175,12 +195,24 @@ public enum XPCRoute: String { case networkCreate case networkDelete case networkList + case networkLookup case volumeCreate case volumeDelete case volumeList case volumeInspect + // The sandbox verbs the runtime interface specifies, named for what they + // do to a pod rather than for the containers inside it. + // https://github.com/kubernetes/cri-api/blob/master/pkg/apis/runtime/v1/api.proto + case podCreate + case podStart + case podStop + case podDelete + case podInspect + case podList + case podUpdate + case volumeDiskUsage case systemDiskUsage diff --git a/Sources/Services/ContainerAPIService/Server/Containers/ContainersHarness.swift b/Sources/Services/ContainerAPIService/Server/Containers/ContainersHarness.swift index 1871cd149..0f96696b7 100644 --- a/Sources/Services/ContainerAPIService/Server/Containers/ContainersHarness.swift +++ b/Sources/Services/ContainerAPIService/Server/Containers/ContainersHarness.swift @@ -45,6 +45,62 @@ public struct ContainersHarness: Sendable { return reply } + @Sendable + public func volumeNamesInUse(_ message: XPCMessage) async throws -> XPCMessage { + let names = try await service.volumeNamesInUse() + let data = try JSONEncoder().encode(names) + + let reply = message.reply() + reply.set(key: .references, value: data) + return reply + } + + @Sendable + public func volumeReferences(_ message: XPCMessage) async throws -> XPCMessage { + guard let name = message.string(key: .volumeName) else { + throw ContainerizationError(.invalidArgument, message: "volume name cannot be empty") + } + let ids = try await service.containersReferencingVolume(name) + let data = try JSONEncoder().encode(ids) + + let reply = message.reply() + reply.set(key: .references, value: data) + return reply + } + + @Sendable + public func networkReferences(_ message: XPCMessage) async throws -> XPCMessage { + guard let id = message.string(key: .id) else { + throw ContainerizationError(.invalidArgument, message: "network id cannot be empty") + } + let ids = try await service.containersAttachedToNetwork(id) + let data = try JSONEncoder().encode(ids) + + let reply = message.reply() + reply.set(key: .references, value: data) + return reply + } + + @Sendable + public func imageReferences(_ message: XPCMessage) async throws -> XPCMessage { + let references = await service.getActiveImageReferences() + let data = try JSONEncoder().encode(references) + + let reply = message.reply() + reply.set(key: .references, value: data) + return reply + } + + @Sendable + public func usageTotals(_ message: XPCMessage) async throws -> XPCMessage { + let totals = await service.calculateDiskUsage() + let data = try JSONEncoder().encode(totals) + + let reply = message.reply() + reply.set(key: .usageTotals, value: data) + return reply + } + @Sendable public func bootstrap(_ message: XPCMessage) async throws -> XPCMessage { let id = message.string(key: .id) diff --git a/Sources/Services/ContainerAPIService/Server/Containers/ContainersService.swift b/Sources/Services/ContainerAPIService/Server/Containers/ContainersService.swift index 81612495f..c5851da07 100644 --- a/Sources/Services/ContainerAPIService/Server/Containers/ContainersService.swift +++ b/Sources/Services/ContainerAPIService/Server/Containers/ContainersService.swift @@ -64,6 +64,7 @@ public actor ContainersService { // FIXME: Find a better mechanism for services running on the APIServer to work with each other private weak var networksService: NetworksService? + private weak var podsService: PodsService? public init( appRoot: URL, @@ -85,6 +86,16 @@ public actor ContainersService { self.containers = try Self.loadAtBoot(root: containerRoot, loader: pluginLoader, log: log) } + /// Where a container keeps what it is made of, which the pod it belongs + /// to hands the runtime when placing it in the machine. + public func path(for id: String) -> URL { + self.containerRoot.appendingPathComponent(id) + } + + public func setPodsService(_ service: PodsService) async { + self.podsService = service + } + public func setNetworksService(_ service: NetworksService) async { self.networksService = service } @@ -141,11 +152,17 @@ public actor ContainersService { ), ) results[config.id] = state - guard runtimePlugins.first(where: { $0.name == config.runtimeHandler }) != nil else { - throw ContainerizationError( - .internalError, - message: "failed to find runtime plugin \(config.runtimeHandler)" - ) + // A missing plugin says nothing about the container: the + // loader's answer varies with how this process was spawned, + // and a bundle outlives any one spawn. Removal is reserved + // for a bundle that cannot be read at all. + if runtimePlugins.first(where: { $0.name == config.runtimeHandler }) == nil { + log.warning( + "no runtime plugin for container", + metadata: [ + "id": "\(config.id)", + "runtime": "\(config.runtimeHandler)", + ]) } } catch { try? FileManager.default.removeItem(at: dir) @@ -226,9 +243,46 @@ public actor ContainersService { } } + /// The name of every volume a container mounts, gathered inside the + /// containers lock so no container is created into the answer. + public func volumeNamesInUse() async throws -> Set { + try await withContainerList(logMetadata: ["acquirer": "\(#function)"]) { containers in + var names = Set() + for container in containers { + for mount in container.configuration.mounts { + if mount.isVolume, let volumeName = mount.volumeName { + names.insert(volumeName) + } + } + } + return names + } + } + + /// The containers that mount the named volume, gathered inside the + /// containers lock. An empty answer says the volume was free when asked, + /// which is the strongest claim one resource can make about another from + /// outside the other's lock. + public func containersReferencingVolume(_ name: String) async throws -> [String] { + try await withContainerList(logMetadata: ["acquirer": "\(#function)", "name": "\(name)"]) { containers in + containers.filter { container in + container.configuration.mounts.contains { $0.isVolume && $0.volumeName == name } + }.map { $0.configuration.id } + } + } + + /// The containers attached to the named network, gathered inside the + /// containers lock. + public func containersAttachedToNetwork(_ id: String) async throws -> [String] { + try await withContainerList(logMetadata: ["acquirer": "\(#function)", "id": "\(id)"]) { containers in + containers.filter { container in + container.configuration.networks.contains { $0.network == id } + }.map { $0.configuration.id } + } + } + /// Calculate disk usage for containers - /// - Returns: Tuple of (total count, active count, total size, reclaimable size) - public func calculateDiskUsage() async -> (Int, Int, UInt64, UInt64) { + public func calculateDiskUsage() async -> ResourceUsage { await lock.withLock(logMetadata: ["acquirer": "\(#function)"]) { _ in var totalSize: UInt64 = 0 var reclaimableSize: UInt64 = 0 @@ -247,7 +301,12 @@ public actor ContainersService { } } - return (await self.containers.count, activeCount, totalSize, reclaimableSize) + return ResourceUsage( + total: await self.containers.count, + active: activeCount, + sizeInBytes: totalSize, + reclaimable: reclaimableSize + ) } } @@ -343,7 +402,25 @@ public actor ContainersService { "id": "\(configuration.id)" ] ) - let initFilesystem = try await self.getInitBlock(for: systemPlatform.ociPlatform(), imageRef: initImage) + let (initFilesystem, initDescription) = try await self.getInitBlock(for: systemPlatform.ociPlatform(), imageRef: initImage) + + guard let podsService = await self.podsService else { + throw ContainerizationError(.internalError, message: "no pod service to make pod \(configuration.pod)") + } + do { + var podConfiguration = PodConfiguration(sandboxFor: configuration) + podConfiguration.initImage = initDescription + try await podsService.create( + configuration: podConfiguration, + kernel: kernel, + initialFilesystem: initFilesystem + ) + } catch let error as ContainerizationError { + guard error.code == .exists else { + throw error + } + // The pod is already there, and the container joins it. + } do { self.log.debug( @@ -382,6 +459,8 @@ public actor ContainersService { ) await self.setContainerState(configuration.id, ContainerState(snapshot: snapshot), context: context) } catch { + // The pod goes with the container it was made for. + await podsService.removeIfAnonymous(id: configuration.pod) throw error } } @@ -420,47 +499,29 @@ public actor ContainersService { let path = self.containerRoot.appendingPathComponent(id) let (config, _) = try Self.getContainerConfiguration(at: path) - var networkBootstrapInfos = [NetworkBootstrapInfo]() - for n in config.networks { - guard let plugin = try await self.networksService?.plugin(for: n.network) else { - throw ContainerizationError(.internalError, message: "failed to get plugin for network \(n.network)") - } - networkBootstrapInfos.append(NetworkBootstrapInfo(plugin: plugin)) + let pod = config.pod + guard let podsService = await self.podsService else { + throw ContainerizationError(.internalError, message: "no pod service to reach pod \(pod)") } - do { - try Self.registerService( - plugin: self.runtimePlugins.first { $0.name == config.runtimeHandler }!, - loader: self.pluginLoader, - configuration: config, - path: path, - debug: self.debugHelpers - ) - - let runtime = state.snapshot.configuration.runtimeHandler - let runtimeClient = try await RuntimeClient.create( - id: id, - runtime: runtime - ) - try await runtimeClient.bootstrap(stdio: stdio, networkBootstrapInfos: networkBootstrapInfos, dynamicEnv: dynamicEnv) - - try await self.exitMonitor.registerProcess( - id: id, - onExit: self.handleContainerExit - ) - - state.client = runtimeClient - await self.setContainerState(id, state, context: context) - } catch { - let label = Self.fullLaunchdServiceLabel( - runtimeName: config.runtimeHandler, - instanceId: id - ) - - await self.exitMonitor.stopTracking(id: id) - try? ServiceManager.deregister(fullServiceLabel: label) - throw error - } + // A container is in the pod's machine, so it has no machine of its + // own to register and reaches the one it shares through the pod. + // + // It asks for the pod to run with it in it, which is one call for + // the container that brings the machine up and the container that + // finds it up already. + try await podsService.start( + id: pod, + container: id, + startup: PodsService.ContainerStartup(stdio: stdio, dynamicEnv: dynamicEnv) + ) + let podClient = try await podsService.client(for: pod).addressing(id) + try await self.exitMonitor.registerProcess( + id: id, + onExit: self.handleContainerExit + ) + state.client = podClient + await self.setContainerState(id, state, context: context) } } @@ -566,6 +627,66 @@ public actor ContainersService { } } + /// Adopt the containers the reconnected machines report. + /// + /// The machines a restarted control plane finds alive were dialed by the + /// pods service; each reports the containers it holds and their state. + /// A container the machine says is running is adopted as running: its + /// client is the pod's, addressed to it, and the exit monitor tracks it + /// again the way bootstrap tracked it first, so its exit is handled by + /// whoever is serving when it comes. + public func reconnect() async { + guard let podsService = self.podsService else { + return + } + await self.lock.withLock(logMetadata: ["acquirer": "\(#function)"]) { context in + for (id, var state) in await self.containers { + guard state.client == nil else { + continue + } + let pod = state.snapshot.configuration.pod + guard let podClient = try? await podsService.client(for: pod) else { + continue + } + let client = podClient.addressing(id) + guard let sandbox = try? await client.state(), + let reported = sandbox.containers.first(where: { $0.id == id }), + reported.status == .running + else { + continue + } + do { + let log = self.log + try await self.exitMonitor.registerProcess( + id: id, + onExit: self.handleContainerExit + ) + let waitFunc: ExitMonitor.WaitHandler = { + let code = try await client.wait(id) + log.info( + "container finished in exit monitor", + metadata: [ + "id": "\(id)", + "rc": "\(code)", + ]) + return code + } + try await self.exitMonitor.track(id: id, waitingOn: waitFunc) + state.client = client + state.snapshot.status = .running + state.snapshot.networks = sandbox.networks + state.snapshot.startedDate = reported.startedDate + await self.setContainerState(id, state, context: context) + self.log.info("adopted a running container", metadata: ["id": "\(id)", "pod": "\(pod)"]) + } catch { + self.log.warning( + "failed to adopt a running container", + metadata: ["id": "\(id)", "error": "\(error)"]) + } + } + } + } + /// Send a signal to the container. public func kill(id: String, processID: String, signal: String) async throws { log.debug( @@ -623,9 +744,8 @@ public actor ContainersService { let state = try self._getContainerState(id: id) // Stop should be idempotent. - let client: RuntimeClient do { - client = try state.getClient() + _ = try state.getClient() } catch { return } @@ -636,7 +756,12 @@ public actor ContainersService { } do { - try await client.stop(options: resolvedOptions) + // Stopping a container stops that container. The machine it runs in + // is stopped by its own call, whether it holds one container or + // several, so nothing here decides the machine's fate on a + // container's behalf. + // https://github.com/kubernetes/cri-api/blob/master/pkg/apis/runtime/v1/api.proto + try await state.client?.stopContainer(options: resolvedOptions) } catch let err as ContainerizationError { if err.code != .interrupted { throw err @@ -742,17 +867,25 @@ public actor ContainersService { ) } - // Logs doesn't care if the container is running or not, just that - // the bundle is there, and that the files actually exist. We do - // first try and get the container state so we get a nicer error message + // Logs doesn't care if the container is running or not, just that the + // bundles are there and the files exist. What the container itself + // wrote is its own bundle's; the boot it came up on belongs to the + // machine the pod runs, so the pod is asked for that one. We do first + // try and get the container state so we get a nicer error message // (container foo not found) however. do { - _ = try _getContainerState(id: id) + let state = try _getContainerState(id: id) let path = self.containerRoot.appendingPathComponent(id) let bundle = ContainerResource.Bundle(path: path) + guard let podsService = self.podsService else { + throw ContainerizationError( + .internalError, + message: "no pod service to reach the machine running \(id)" + ) + } return [ try FileHandle(forReadingFrom: bundle.containerLog), - try FileHandle(forReadingFrom: bundle.bootlog), + try await podsService.bootLog(for: state.snapshot.configuration.pod), ] } catch { throw ContainerizationError( @@ -844,7 +977,11 @@ public actor ContainersService { signal: "SIGKILL" ) let client = try state.getClient() - try await client.stop(options: opts) + // Removing a container removes that container; the machine it + // shares is not this call's to stop, and goes down on its own + // once the last container in it has stopped. + // https://github.com/kubernetes/cri-api/blob/master/pkg/apis/runtime/v1/api.proto + try await client.stopContainer(options: opts) try await self.lock.withLock(logMetadata: ["acquirer": "\(#function)", "id": "\(id)"]) { context in self.log.info( "ContainersService: attempt cleanup", @@ -948,47 +1085,10 @@ public actor ContainersService { await self.exitMonitor.stopTracking(id: id) - // Shutdown and deregister the runtime service - self.log.info("shutting down runtime service", metadata: ["id": "\(id)"]) - - let path = self.containerRoot.appendingPathComponent(id) - let bundle = ContainerResource.Bundle(path: path) - let config = try bundle.configuration - let label = Self.fullLaunchdServiceLabel( - runtimeName: config.runtimeHandler, - instanceId: id - ) - - // Try to shutdown the client gracefully, but if the runtime service - // is already dead (e.g., killed externally), we should still continue - // with state cleanup. - if let client = state.client { - do { - try await client.shutdown() - } catch { - self.log.error( - "failed to shutdown runtime service", - metadata: [ - "id": "\(id)", - "error": "\(error)", - ]) - } - } - - // Deregister the service, launchd will terminate the process. - // This may also fail if the service was already deregistered or - // the process was killed externally. - do { - try ServiceManager.deregister(fullServiceLabel: label) - self.log.info("deregistered runtime service", metadata: ["id": "\(id)"]) - } catch { - self.log.error( - "failed to deregister runtime service", - metadata: [ - "id": "\(id)", - "error": "\(error)", - ]) - } + // A container's exit is the container's alone. The machine it shared + // stops itself once the last container in it is gone, and its service + // is the pod's, deregistered when the pod is deleted; a machine other + // containers still run in is not touched at all. state.snapshot.status = .stopped state.snapshot.networks = [] @@ -1034,31 +1134,13 @@ public actor ContainersService { await self.exitMonitor.stopTracking(id: id) let path = self.containerRoot.appendingPathComponent(id) - // Try to get config for service deregistration - // Don't fail if bundle is incomplete - var config: ContainerConfiguration? let bundle = ContainerResource.Bundle(path: path) - do { - config = try bundle.configuration - } catch { - self.log.warning( - "failed to read bundle configuration during cleanup for container", - metadata: [ - "id": "\(id)", - "error": "\(error)", - ]) - } - - // Only try to deregister service if we have a valid config - // TODO: Change this so we don't have to reread the config - // possibly store the container ID to service label mapping - if let config = config { - let label = Self.fullLaunchdServiceLabel( - runtimeName: config.runtimeHandler, - instanceId: id - ) - try? ServiceManager.deregister(fullServiceLabel: label) - } + // Which machine this container runs in, read the way a container's + // configuration is read anywhere it may not have started yet: from its + // bundle, and from the runtime configuration when the bundle holds + // nothing, since a container that was made and never started has its + // configuration only there. + let pod = try? Self.getContainerConfiguration(at: path).0.pod // Always try to delete the bundle directory, even if it's incomplete do { @@ -1073,6 +1155,19 @@ public actor ContainersService { } self.containers.removeValue(forKey: id) + + // A container took its machine down as it was removed, when the machine + // was its own. A pod nobody named holds the machine in its place, so it + // goes here too; a pod someone named was not given to this container + // and stays. + // + // The container is out of the pod above before the pod is asked to go, + // because a pod removes the containers it still holds as it goes, and + // asked while this one was still in it the pod would ask for it to be + // removed, which asks for the lock this cleanup already holds. + if let pod { + await self.podsService?.removeIfAnonymous(id: pod) + } } private func cleanUp(id: String, context: AsyncLock.Context) async throws { @@ -1086,12 +1181,12 @@ public actor ContainersService { return options } - private func getInitBlock(for platform: Platform, imageRef: String? = nil) async throws -> Filesystem { + private func getInitBlock(for platform: Platform, imageRef: String? = nil) async throws -> (Filesystem, ImageDescription) { let ref = imageRef ?? containerSystemConfig.vminit.image let initImage = try await ClientImage.fetch(reference: ref, platform: platform, containerSystemConfig: containerSystemConfig) var fs = try await initImage.getCreateSnapshot(platform: platform) fs.options = ["ro"] - return fs + return (fs, initImage.description) } private static func registerService( diff --git a/Sources/Services/ContainerAPIService/Server/DiskUsage/DiskUsageService.swift b/Sources/Services/ContainerAPIService/Server/DiskUsage/DiskUsageService.swift index 39d43b8d0..b2d5a59fa 100644 --- a/Sources/Services/ContainerAPIService/Server/DiskUsage/DiskUsageService.swift +++ b/Sources/Services/ContainerAPIService/Server/DiskUsage/DiskUsageService.swift @@ -19,16 +19,14 @@ import Logging /// Service for calculating disk usage across all resource types public actor DiskUsageService { - private let containersService: ContainersService + private let containers = ContainerClient() private let volumesService: VolumesService private let log: Logger public init( - containersService: ContainersService, volumesService: VolumesService, log: Logger ) { - self.containersService = containersService self.volumesService = volumesService self.log = log } @@ -38,11 +36,11 @@ public actor DiskUsageService { log.debug("calculating disk usage for all resources") // Get active image references first (needed for image calculation) - let activeImageRefs = await containersService.getActiveImageReferences() + let activeImageRefs = try await containers.activeImageReferences() // Query all services concurrently async let imageStats = ClientImage.calculateDiskUsage(activeReferences: activeImageRefs) - async let containerStats = containersService.calculateDiskUsage() + async let containerStats = containers.calculateDiskUsage() async let volumeStats = volumesService.calculateDiskUsage() let (imageData, containerData, volumeData) = try await (imageStats, containerStats, volumeStats) @@ -54,26 +52,16 @@ public actor DiskUsageService { sizeInBytes: imageData.totalSize, reclaimable: imageData.reclaimableSize ), - containers: ResourceUsage( - total: containerData.0, - active: containerData.1, - sizeInBytes: containerData.2, - reclaimable: containerData.3 - ), - volumes: ResourceUsage( - total: volumeData.0, - active: volumeData.1, - sizeInBytes: volumeData.2, - reclaimable: volumeData.3 - ) + containers: containerData, + volumes: volumeData ) log.debug( "disk usage calculation complete", metadata: [ "images_total": "\(imageData.totalCount)", - "containers_total": "\(containerData.0)", - "volumes_total": "\(volumeData.0)", + "containers_total": "\(containerData.total)", + "volumes_total": "\(volumeData.total)", ]) return stats diff --git a/Sources/Services/ContainerAPIService/Server/Networks/NetworksHarness.swift b/Sources/Services/ContainerAPIService/Server/Networks/NetworksHarness.swift index ec525be83..2a836ba66 100644 --- a/Sources/Services/ContainerAPIService/Server/Networks/NetworksHarness.swift +++ b/Sources/Services/ContainerAPIService/Server/Networks/NetworksHarness.swift @@ -40,6 +40,18 @@ public struct NetworksHarness: Sendable { return reply } + @Sendable + public func lookup(_ message: XPCMessage) async throws -> XPCMessage { + guard let hostname = message.string(key: .hostname) else { + throw ContainerizationError(.invalidArgument, message: "hostname cannot be empty") + } + let reply = message.reply() + if let attachment = try await service.lookup(hostname: hostname) { + reply.set(key: .attachment, value: try JSONEncoder().encode(attachment)) + } + return reply + } + @Sendable public func create(_ message: XPCMessage) async throws -> XPCMessage { let data = message.dataNoCopy(key: .networkConfig) diff --git a/Sources/Services/ContainerAPIService/Server/Networks/NetworksService.swift b/Sources/Services/ContainerAPIService/Server/Networks/NetworksService.swift index 7fe35fae8..a6472251a 100644 --- a/Sources/Services/ContainerAPIService/Server/Networks/NetworksService.swift +++ b/Sources/Services/ContainerAPIService/Server/Networks/NetworksService.swift @@ -36,7 +36,7 @@ public actor NetworksService { private let pluginLoader: PluginLoader private let resourceRoot: FilePath - private let containersService: ContainersService + private let containers = ContainerClient() private let log: Logger private let debugHelpers: Bool @@ -50,18 +50,15 @@ public actor NetworksService { public init( pluginLoader: PluginLoader, resourceRoot: FilePath, - containersService: ContainersService, defaultNetworkConfiguration: NetworkConfiguration, log: Logger, debugHelpers: Bool = false, ) async throws { self.pluginLoader = pluginLoader self.resourceRoot = resourceRoot - self.containersService = containersService self.log = log self.debugHelpers = debugHelpers - try FileManager.default.createDirectory(atPath: resourceRoot.string, withIntermediateDirectories: true) self.store = try FilesystemEntityStore( path: resourceRoot, type: "network", @@ -85,7 +82,20 @@ public actor NetworksService { // computed default network configuration from the apiserver to ensure we // have the correct default values configured. if effectiveConfiguration.id == NetworkClient.defaultNetworkName { - effectiveConfiguration = defaultNetworkConfiguration + // The address range the network came up on is its own, so it + // rides through the refresh and is asked for again: a network + // that returns on a different range leaves every guest that + // outlived the restart holding an address, a route, and a + // resolver belonging to a range that is gone. + effectiveConfiguration = try NetworkConfiguration( + name: defaultNetworkConfiguration.name, + mode: defaultNetworkConfiguration.mode, + ipv4Subnet: defaultNetworkConfiguration.ipv4Subnet ?? configuration.ipv4Subnet, + ipv6Subnet: defaultNetworkConfiguration.ipv6Subnet ?? configuration.ipv6Subnet, + labels: defaultNetworkConfiguration.labels, + plugin: defaultNetworkConfiguration.plugin, + options: defaultNetworkConfiguration.options + ) try await store.update(effectiveConfiguration) } @@ -95,9 +105,53 @@ public actor NetworksService { // 5 seconds or considerably more from the registration of this first // network service to its execution. do { - try await registerService(configuration: effectiveConfiguration) + do { + try await registerService(configuration: effectiveConfiguration) + } catch where effectiveConfiguration.id == NetworkClient.defaultNetworkName && effectiveConfiguration.ipv4Subnet != nil { + // The range the default network asks for is the one it was + // given last time, which is a preference and not a demand: + // a range held by something else, or reserved to a network + // nobody holds any more, would otherwise leave the system + // with no network at all and every call that needs one + // waiting forever. The network comes up on whatever range + // is free and says which range it lost. + log.error( + "the default network could not take the address range it was given; taking another", + metadata: [ + "range": "\(effectiveConfiguration.ipv4Subnet?.description ?? "")", + "error": "\(error)", + ]) + effectiveConfiguration = try NetworkConfiguration( + name: effectiveConfiguration.name, + mode: effectiveConfiguration.mode, + ipv4Subnet: nil, + ipv6Subnet: effectiveConfiguration.ipv6Subnet, + labels: effectiveConfiguration.labels, + plugin: effectiveConfiguration.plugin, + options: effectiveConfiguration.options + ) + try await registerService(configuration: effectiveConfiguration) + } let client = try Self.getClient(configuration: effectiveConfiguration) let networkStatus = try await client.status() + + // A network that named no range is given one as it comes up, + // and that range is written down as the range it asks for from + // then on, so the guests it addresses keep answering to the + // same addresses across a restart. + if effectiveConfiguration.ipv4Subnet == nil { + effectiveConfiguration = try NetworkConfiguration( + name: effectiveConfiguration.name, + mode: effectiveConfiguration.mode, + ipv4Subnet: networkStatus.ipv4Subnet, + ipv6Subnet: effectiveConfiguration.ipv6Subnet, + labels: effectiveConfiguration.labels, + plugin: effectiveConfiguration.plugin, + options: effectiveConfiguration.options + ) + try await store.update(effectiveConfiguration) + } + serviceStates[effectiveConfiguration.id] = NetworkEntry( configuration: effectiveConfiguration, status: networkStatus, @@ -248,50 +302,41 @@ public actor NetworksService { throw ContainerizationError(.invalidArgument, message: "cannot delete builtin network: \(id)") } - // prevent container operations while we atomically check and delete - try await self.containersService.withContainerList(logMetadata: ["acquirer": "\(#function)", "id": "\(id)"]) { containers in - // find all containers that refer to the network - var referringContainers = Set() - for container in containers { - for attachmentConfiguration in container.configuration.networks { - if attachmentConfiguration.network == id { - referringContainers.insert(container.configuration.id) - break - } - } - } + // A container created after this answer can attach to the network + // while it is deleted, the same window image delete accepts against + // container create; the attach then fails naming the missing network. + let referringContainers = try await self.containers.containersAttachedToNetwork(id) - // bail if any referring containers - guard referringContainers.isEmpty else { - throw ContainerizationError( - .invalidState, - message: "cannot delete subnet \(id) with referring containers: \(referringContainers.joined(separator: ", "))" - ) - } + // bail if any referring containers + guard referringContainers.isEmpty else { + throw ContainerizationError( + .invalidState, + message: "cannot delete subnet \(id) with referring containers: \(referringContainers.joined(separator: ", "))" + ) + } - // start network deletion, this is the last place we'll want to throw - do { - try await self.deregisterService(configuration: serviceState.configuration) - } catch { - self.log.error( - "failed to deregister network service", - metadata: [ - "id": "\(id)", - "error": "\(error.localizedDescription)", - ]) - } + // start network deletion, this is the last place we'll want to throw + do { + try await self.deregisterService(configuration: serviceState.configuration) + } catch { + self.log.error( + "failed to deregister network service", + metadata: [ + "id": "\(id)", + "error": "\(error.localizedDescription)", + ]) + } - // deletion is underway, do not throw anything now - do { - try await self.store.delete(id) - } catch { - self.log.error( - "failed to delete network from configuration store", - metadata: [ - "id": "\(id)", - "error": "\(error.localizedDescription)", - ]) - } + // deletion is underway, do not throw anything now + do { + try await self.store.delete(id) + } catch { + self.log.error( + "failed to delete network from configuration store", + metadata: [ + "id": "\(id)", + "error": "\(error.localizedDescription)", + ]) } // having deleted successfully, remove the runtime state diff --git a/Sources/Services/ContainerAPIService/Server/Pods/PodsHarness.swift b/Sources/Services/ContainerAPIService/Server/Pods/PodsHarness.swift new file mode 100644 index 000000000..c9123b9dd --- /dev/null +++ b/Sources/Services/ContainerAPIService/Server/Pods/PodsHarness.swift @@ -0,0 +1,110 @@ +//===----------------------------------------------------------------------===// +// Copyright © 2026 Apple Inc. and the container project authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//===----------------------------------------------------------------------===// + +import ContainerAPIClient +import ContainerResource +import ContainerXPC +import Containerization +import ContainerizationError +import Foundation +import Logging + +public struct PodsHarness: Sendable { + let log: Logging.Logger + let service: PodsService + + public init(service: PodsService, log: Logging.Logger) { + self.log = log + self.service = service + } + + @Sendable + public func create(_ message: XPCMessage) async throws -> XPCMessage { + guard let data = message.dataNoCopy(key: .podConfig) else { + throw ContainerizationError(.invalidArgument, message: "a pod configuration is required") + } + let configuration = try JSONDecoder().decode(PodConfiguration.self, from: data) + + guard let kernelData = message.dataNoCopy(key: .kernel) else { + throw ContainerizationError(.invalidArgument, message: "a kernel is required") + } + let kernel = try JSONDecoder().decode(Kernel.self, from: kernelData) + + try await service.create( + configuration: configuration, + kernel: kernel, + initImage: message.string(key: .initImage) + ) + return message.reply() + } + + @Sendable + public func start(_ message: XPCMessage) async throws -> XPCMessage { + let data = message.dataNoCopy(key: .dynamicEnv) + let dynamicEnv = try data.map { try JSONDecoder().decode([String: String].self, from: $0) } ?? [:] + let startup: PodsService.ContainerStartup? = + dynamicEnv.isEmpty ? nil : .init(stdio: [nil, nil, nil], dynamicEnv: dynamicEnv) + try await service.start(id: try message.podId(), startup: startup) + return message.reply() + } + + @Sendable + public func stop(_ message: XPCMessage) async throws -> XPCMessage { + try await service.stop(id: try message.podId()) + return message.reply() + } + + @Sendable + public func delete(_ message: XPCMessage) async throws -> XPCMessage { + try await service.delete(id: try message.podId(), force: message.bool(key: .forceDelete)) + return message.reply() + } + + @Sendable + public func inspect(_ message: XPCMessage) async throws -> XPCMessage { + let snapshot = try await service.inspect(id: try message.podId()) + let reply = message.reply() + reply.set(key: .podSnapshot, value: try JSONEncoder().encode(snapshot)) + return reply + } + + @Sendable + public func list(_ message: XPCMessage) async throws -> XPCMessage { + let snapshots = await service.list() + let reply = message.reply() + reply.set(key: .podSnapshots, value: try JSONEncoder().encode(snapshots)) + return reply + } + + @Sendable + public func update(_ message: XPCMessage) async throws -> XPCMessage { + let bytes = message.uint64(key: .memoryInBytes) + guard bytes > 0 else { + throw ContainerizationError(.invalidArgument, message: "a memory size is required") + } + try await service.update(id: try message.podId(), memoryInBytes: bytes) + return message.reply() + } +} + +extension XPCMessage { + fileprivate func podId() throws -> String { + guard let id = self.string(key: .podId), !id.isEmpty else { + throw ContainerizationError(.invalidArgument, message: "a pod name is required") + } + return id + } +} diff --git a/Sources/Services/ContainerAPIService/Server/Pods/PodsService.swift b/Sources/Services/ContainerAPIService/Server/Pods/PodsService.swift new file mode 100644 index 000000000..b4bfc5906 --- /dev/null +++ b/Sources/Services/ContainerAPIService/Server/Pods/PodsService.swift @@ -0,0 +1,639 @@ +//===----------------------------------------------------------------------===// +// Copyright © 2026 Apple Inc. and the container project authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//===----------------------------------------------------------------------===// + +import ContainerAPIClient +import ContainerPersistence +import ContainerPlugin +import ContainerResource +import ContainerRuntimeClient +import Containerization +import ContainerizationError +import ContainerizationExtras +import ContainerizationOCI +import Foundation +import Logging + +/// Manages pods: machines that several containers run inside and share. +/// +/// The verbs follow the runtime interface's sandbox lifecycle, which every +/// runtime serving Kubernetes implements, so what a pod does here is what a pod +/// does elsewhere. +/// https://github.com/kubernetes/cri-api/blob/master/pkg/apis/runtime/v1/api.proto +public actor PodsService { + struct PodState { + var configuration: PodConfiguration + var state: PodState.Lifecycle = .notReady + var client: RuntimeClient? = nil + var startedDate: Date? = nil + + enum Lifecycle { + case notReady + case ready + } + + func getClient() throws -> RuntimeClient { + guard let client else { + throw ContainerizationError( + .invalidState, + message: "pod \(configuration.id) is not running" + ) + } + return client + } + } + + /// The runtime plugin a pod's machine is driven by. + static let runtimeHandler = "container-runtime-linux" + + private static let machServicePrefix = "com.apple.container" + + /// The launchd domain, asked for once while the service is being built. + /// + /// Answering it runs `launchctl` and waits for it, which is a thread + /// blocked until that process is done. Held in a stored property it is a + /// wait the service makes as it is constructed; held in a static it would + /// be made the first time a label is wanted, and the first time is inside + /// a task, on a thread the concurrency pool needs back. That wait never + /// ends, and because a static is initialized once every later caller waits + /// on the same unfinished initialization, with nothing thrown and nothing + /// logged to say so. + private let launchdDomainString: String + + private static func fullLaunchdServiceLabel(domain: String, runtimeName: String, instanceId: String) -> String { + "\(domain)/\(Self.machServicePrefix).\(runtimeName).\(instanceId)" + } + + private let log: Logger + private let debugHelpers: Bool + private let podRoot: URL + private let pluginLoader: PluginLoader + private let runtimePlugins: [Plugin] + private let containerSystemConfig: ContainerSystemConfig + + private let lock: AsyncLock + private var pods: [String: PodState] + + // The containers a pod holds are the containers service's to know, so it + // is asked rather than tracked twice. + private weak var containersService: ContainersService? + private weak var networksService: NetworksService? + + public init( + appRoot: URL, + pluginLoader: PluginLoader, + containerSystemConfig: ContainerSystemConfig, + debugHelpers: Bool = false, + log: Logger + ) throws { + self.log = log + self.debugHelpers = debugHelpers + self.podRoot = appRoot.appendingPathComponent("pods") + self.pluginLoader = pluginLoader + self.runtimePlugins = pluginLoader.findPlugins().filter { $0.hasType(.runtime) } + self.containerSystemConfig = containerSystemConfig + self.lock = AsyncLock() + self.launchdDomainString = try ServiceManager.getDomainString() + self.pods = try Self.loadAtBoot(root: self.podRoot, log: log) + } + + public func setContainersService(_ service: ContainersService) async { + self.containersService = service + } + + public func setNetworksService(_ service: NetworksService) async { + self.networksService = service + } + + /// The pods on disk, which outlive the process that made them. + /// + /// A pod's bundle is materialized by its machine's first boot, so a pod + /// created and not yet booted has only the runtime configuration its + /// create wrote; the pod's own configuration is read from whichever of + /// the two holds it. + static func loadAtBoot(root: URL, log: Logger) throws -> [String: PodState] { + try FileManager.default.createDirectory(at: root, withIntermediateDirectories: true) + var pods: [String: PodState] = [:] + let entries = try FileManager.default.contentsOfDirectory(at: root, includingPropertiesForKeys: nil) + for entry in entries { + do { + let bundle = ContainerResource.Bundle(path: entry) + let configuration: PodConfiguration + if bundle.isPod { + configuration = try bundle.podConfiguration + } else if let embedded = try RuntimeConfiguration.readRuntimeConfiguration(from: entry).podConfiguration { + configuration = embedded + } else { + log.warning("skipping a bundle that is not a pod's", metadata: ["path": "\(entry.path)"]) + continue + } + pods[configuration.id] = PodState(configuration: configuration) + } catch { + log.warning("skipping unreadable pod", metadata: ["path": "\(entry.path)", "error": "\(error)"]) + } + } + return pods + } + + /// Adopt the machines that outlived the process that made them. + /// + /// A pod's machine is a launchd service, so it survives the control + /// plane that registered it. A restarted control plane would otherwise + /// hold every pod as not ready and answer stops and deletes against + /// machines it cannot reach; the kubelet reconciles the same gap by + /// listing what its runtime actually holds when it starts, and + /// containerd by re-dialing the shims it finds alive. + /// https://github.com/kubernetes/cri-api/blob/master/pkg/apis/runtime/v1/api.proto + /// + /// Each pod whose service still answers is dialed and believed: the + /// machine's own snapshot says whether it runs. A pod whose service is + /// gone stays written down and not ready, which is what it was before + /// its machine first booted. + public func reconnect() async { + await self.lock.withLock(logMetadata: ["acquirer": "\(#function)"]) { context in + let pods = await self.pods + self.log.info("looking for machines to adopt", metadata: ["pods": "\(pods.count)"]) + for (id, var state) in pods { + guard state.client == nil else { + continue + } + let runtime = state.configuration.runtimeHandler + // launchctl answers for the service's bare label: not the + // domain-prefixed form bootout takes, and not the mach name + // the client dials, which carries the runtime prefix. + let label = "\(Self.machServicePrefix).\(runtime).\(id)" + guard (try? ServiceManager.isRegistered(fullServiceLabel: label)) == true else { + self.log.info("no service answers for the pod", metadata: ["pod": "\(id)"]) + continue + } + do { + let client = try await RuntimeClient.create(id: id, runtime: runtime) + let sandbox = try await client.state() + guard sandbox.status == .running else { + self.log.info( + "a pod's machine answered but is not running", + metadata: ["pod": "\(id)", "status": "\(sandbox.status)"]) + continue + } + state.client = client + state.state = .ready + state.startedDate = sandbox.containers.compactMap { $0.startedDate }.min() + await self.setPodState(id, state, context: context) + self.log.info("adopted a running machine", metadata: ["pod": "\(id)"]) + } catch { + self.log.warning( + "a pod's service answered launchd but not the runtime", + metadata: ["pod": "\(id)", "error": "\(error)"]) + } + } + } + } + + /// Where a pod keeps what it is made of. + public func path(for id: String) -> URL { + self.podRoot.appendingPathComponent(id) + } + + /// The boot log of the machine a pod runs. + /// + /// The machine is the pod's rather than any one container's: every + /// container the pod holds boots on it, and one log records that boot, so + /// the pod is what answers for it. + public func bootLog(for id: String) throws -> FileHandle { + let bundle = ContainerResource.Bundle(path: self.path(for: id)) + return try FileHandle(forReadingFrom: bundle.bootlog) + } + + /// The client for the machine a pod runs, which its containers are + /// reached through. + public func client(for id: String) throws -> RuntimeClient { + try self._getPodState(id: id).getClient() + } + + /// The initial filesystem a pod's machine boots, which holds the agent, + /// and the image it was taken from. + private func getInitBlock(for platform: Platform, imageRef: String? = nil) async throws -> (Filesystem, ImageDescription) { + let ref = imageRef ?? containerSystemConfig.vminit.image + let initImage = try await ClientImage.fetch(reference: ref, platform: platform, containerSystemConfig: containerSystemConfig) + var fs = try await initImage.getCreateSnapshot(platform: platform) + fs.options = ["ro"] + return (fs, initImage.description) + } + + /// Write down a pod that boots the init image named, or the default one. + public func create(configuration: PodConfiguration, kernel: Kernel, initImage: String? = nil) async throws { + let (initialFilesystem, description) = try await self.getInitBlock(for: kernel.platform.ociPlatform(), imageRef: initImage) + var configuration = configuration + configuration.initImage = description + try await self.create( + configuration: configuration, + kernel: kernel, + initialFilesystem: initialFilesystem + ) + } + + /// Write down a pod, so containers can be placed in it before it boots. + /// + /// A caller holding the init filesystem already, such as one making the pod + /// for a container whose own init image was resolved when it was created, + /// passes it here rather than naming an image to resolve again. + public func create(configuration: PodConfiguration, kernel: Kernel, initialFilesystem: Filesystem) async throws { + log.debug( + "PodsService: enter", + metadata: [ + "func": "\(#function)", + "id": "\(configuration.id)", + ] + ) + defer { + log.debug( + "PodsService: exit", + metadata: [ + "func": "\(#function)", + "id": "\(configuration.id)", + ] + ) + } + + try await self.lock.withLock(logMetadata: ["acquirer": "\(#function)", "id": "\(configuration.id)"]) { context in + guard await self.pods[configuration.id] == nil else { + throw ContainerizationError(.exists, message: "pod already exists: \(configuration.id)") + } + + // A name on a network answers for one thing on it. The names live + // on the attachments a pod holds, since the pod holds the network + // its containers share, so it is here that two of them are kept + // from answering to the same one, which is what a container with a + // machine of its own was held to when it held its own attachments. + var taken = Set() + for pod in await self.pods.values { + for attachment in pod.configuration.networks { + taken.insert(attachment.options.hostname) + } + } + let clashing = configuration.networks.map { $0.options.hostname }.filter { taken.contains($0) } + guard clashing.isEmpty else { + throw ContainerizationError(.exists, message: "hostname(s) already exist: \(clashing)") + } + + guard self.runtimePlugins.first(where: { $0.name == configuration.runtimeHandler }) != nil else { + throw ContainerizationError( + .notFound, + message: "unable to locate runtime plugin \(configuration.runtimeHandler)" + ) + } + + // The floor a machine needs to boot at all, the same one a + // container with a machine of its own is held to. + let minimumMemory: UInt64 = 200.mib() + guard configuration.resources.memoryInBytes >= minimumMemory else { + throw ContainerizationError( + .invalidArgument, + message: "minimum memory amount allowed is 200 MiB (got \(configuration.resources.memoryInBytes) bytes)" + ) + } + + let path = await self.path(for: configuration.id) + let runtimeConfig = RuntimeConfiguration( + path: path, + initialFilesystem: initialFilesystem, + kernel: kernel, + podConfiguration: configuration + ) + try runtimeConfig.writeRuntimeConfiguration() + + await self.setPodState(configuration.id, PodState(configuration: configuration), context: context) + } + } + + /// What a caller holds on behalf of a container it is starting a pod for. + /// + /// A container placed in a pod is given these as it goes in, the way a + /// container with a machine of its own is given them as the machine is + /// bootstrapped. They belong to the container rather than to the pod, so + /// they travel under its id. + public struct ContainerStartup: Sendable { + /// The streams the caller opened for the container. + public var stdio: [FileHandle?] + /// The environment the caller was asked to add to the container's. + public var dynamicEnv: [String: String] + + public init(stdio: [FileHandle?], dynamicEnv: [String: String] = [:]) { + self.stdio = stdio + self.dynamicEnv = dynamicEnv + } + } + + /// Boot a pod's machine and place the containers that belong to it inside. + /// + /// The containers go in before the machine starts, which is what a machine + /// with no way to attach storage while running requires. + /// + /// A container the caller is starting the pod for brings what the caller + /// holds for it. The machine itself is nobody's to read and has no + /// environment of its own, so it is bootstrapped with neither. + /// The bundles of the containers a pod holds, which its machine runs. + private func bundlePaths(of id: String) async throws -> [String] { + var paths = [String]() + for member in await self.containers(of: id) { + guard let path = await self.containersService?.path(for: member.id) else { + throw ContainerizationError(.internalError, message: "no container service to place \(member.id)") + } + paths.append(path.path) + } + return paths + } + + public func start(id: String, container: String? = nil, startup: ContainerStartup? = nil) async throws { + log.debug("PodsService: enter", metadata: ["func": "\(#function)", "id": "\(id)"]) + defer { log.debug("PodsService: exit", metadata: ["func": "\(#function)", "id": "\(id)"]) } + + try await self.lock.withLock(logMetadata: ["acquirer": "\(#function)", "id": "\(id)"]) { context in + var state = try await self._getPodState(id: id) + var running = state.client + + // A machine can be gone while the service that ran it lingers: + // stopped out of band, crashed, or torn down without the pod + // hearing of it. A pod that offered that machine would be + // offering one that is not there, so the client is believed only + // while its machine answers running: an answer of anything else + // means the machine is gone, and the service is taken down so + // the pod boots a fresh machine the way it booted the first. No + // answer at all is the query failing, not the machine standing + // down; a fresh machine booted against devices a live one still + // holds fails at its attachments, so the start fails on the + // query instead and the held client stands. + if let held = running { + guard let observed = try? await held.state() else { + throw ContainerizationError( + .internalError, + message: "the machine of pod \(id) did not answer a state query") + } + if observed.status != .running { + await self.deregister(id: id) + state.client = nil + state.state = .notReady + state.startedDate = nil + await self.setPodState(id, state, context: context) + running = nil + } + } + + do { + let client: RuntimeClient + var networkBootstrapInfos = [NetworkBootstrapInfo]() + if let running { + client = running + } else { + let path = await self.path(for: id) + let runtime = state.configuration.runtimeHandler + guard let plugin = self.runtimePlugins.first(where: { $0.name == runtime }) else { + throw ContainerizationError(.notFound, message: "unable to locate runtime plugin \(runtime)") + } + try Self.registerService( + plugin: plugin, + loader: self.pluginLoader, + id: id, + path: path, + debug: self.debugHelpers, + domain: self.launchdDomainString + ) + + // The pod claims its addresses, which every container placed + // in it shares, having no network namespace of its own. + for n in state.configuration.networks { + guard let plugin = try await self.networksService?.plugin(for: n.network) else { + throw ContainerizationError(.internalError, message: "failed to get plugin for network \(n.network)") + } + networkBootstrapInfos.append(NetworkBootstrapInfo(plugin: plugin)) + } + client = try await RuntimeClient.create(id: id, runtime: runtime) + } + + // The pod is asked to run holding its containers, which is one + // request whether it is coming up around them or already up and + // taking in the one that is new to it. + try await client.bootstrap( + bundlePaths: try await self.bundlePaths(of: id), + stdioFor: container, + stdio: startup?.stdio ?? [nil, nil, nil], + networkBootstrapInfos: networkBootstrapInfos, + dynamicEnv: startup?.dynamicEnv ?? [:], + stopsWithContainers: state.configuration.isAnonymous + ) + + if running == nil { + state.client = client + state.state = .ready + state.startedDate = Date() + await self.setPodState(id, state, context: context) + } + } catch { + if running == nil { + await self.deregister(id: id) + } + throw error + } + } + } + + /// Stop a pod's machine, and with it every container inside. + public func stop(id: String, options: ContainerStopOptions = ContainerStopOptions(timeoutInSeconds: 5, signal: "SIGTERM")) async throws { + log.debug("PodsService: enter", metadata: ["func": "\(#function)", "id": "\(id)"]) + defer { log.debug("PodsService: exit", metadata: ["func": "\(#function)", "id": "\(id)"]) } + + try await self.lock.withLock(logMetadata: ["acquirer": "\(#function)", "id": "\(id)"]) { context in + var state = try await self._getPodState(id: id) + guard let client = state.client else { + return + } + + try? await client.stop(options: options) + try? await client.shutdown() + await self.deregister(id: id) + + state.client = nil + state.state = .notReady + state.startedDate = nil + await self.setPodState(id, state, context: context) + } + } + + /// Take a pod away. A pod still holding containers is kept unless the + /// caller insists, in which case the containers go with it. + public func delete(id: String, force: Bool) async throws { + log.debug("PodsService: enter", metadata: ["func": "\(#function)", "id": "\(id)"]) + defer { log.debug("PodsService: exit", metadata: ["func": "\(#function)", "id": "\(id)"]) } + + let members = await self.containers(of: id) + if !members.isEmpty { + guard force else { + throw ContainerizationError( + .invalidState, + message: "pod \(id) holds \(members.count) container(s); stop and remove them, or force" + ) + } + for member in members { + try await self.containersService?.delete(id: member.id, force: true) + } + } + + // Taking the last container out of a pod that was made for one is what + // takes that pod away, so the members above may have carried this pod + // off as they went. A pod already gone is what the caller asked for. + guard (try? self._getPodState(id: id)) != nil else { + return + } + + try await self.stop(id: id) + await self.lock.withLock(logMetadata: ["acquirer": "\(#function)", "id": "\(id)"]) { context in + try? FileManager.default.removeItem(at: await self.path(for: id)) + await self.removePodState(id, context: context) + } + } + + /// Everything known about a pod, including the containers in it. + public func inspect(id: String) async throws -> PodSnapshot { + let state = try self._getPodState(id: id) + return await self.snapshot(state) + } + + /// Every pod, in a settled order. + public func list() async -> [PodSnapshot] { + var snapshots: [PodSnapshot] = [] + for state in self.pods.values.sorted(by: { $0.configuration.id < $1.configuration.id }) { + snapshots.append(await self.snapshot(state)) + } + return snapshots + } + + /// Hold a running pod to a memory size, which its containers share. + /// + /// This is the runtime interface's `UpdatePodSandboxResources` for the one + /// resource a running machine can be held to after it has booted. + public func update(id: String, memoryInBytes: UInt64) async throws { + let state = try self._getPodState(id: id) + let client = try state.getClient() + try await client.setTargetMemorySize(memoryInBytes) + } + + /// Take away a pod a container was given for itself. + /// + /// A container took its machine down as it was removed, by deregistering the + /// service that ran it, because the machine was the container's and nothing + /// else could be in it. A pod nobody named holds the machine in its place + /// and exists because the container needed one, so it goes the same way. + /// + /// A pod someone named was not given to this container. It was there before + /// the container and is there after, so a container leaving says nothing + /// about it. Neither does a container leaving one that others are still in: + /// a container may be placed in a pod it was given the name of, anonymous + /// or not, and holds it up the way any other in it would. + /// + /// What a pod holds is what the containers say they are in, so this is + /// asked with the container lock held by a caller that has already taken + /// its own container out of it. Held that way the answer cannot change + /// between the asking and the removal, since a container joins a pod only + /// by being created. + public func removeIfAnonymous(id: String) async { + guard let state = try? self._getPodState(id: id), state.configuration.isAnonymous else { + return + } + guard await self.containers(of: id).isEmpty else { + return + } + do { + try await self.delete(id: id, force: true) + } catch { + self.log.error( + "failed to remove the machine a container had to itself", + metadata: ["pod": "\(id)", "error": "\(error)"]) + } + } + + private func snapshot(_ state: PodState) async -> PodSnapshot { + let members = await self.containers(of: state.configuration.id) + var networks: [Attachment] = [] + // A pod is ready when its machine is running, which is what the machine + // says rather than what it was last told to do: the machine goes down on + // its own once the last container in it has stopped, and a pod that + // answered ready after that would be offering a machine that is not + // there. + var running = false + if let client = state.client, let sandbox = try? await client.state() { + networks = sandbox.networks + running = sandbox.status == .running + } + return PodSnapshot( + configuration: state.configuration, + state: state.state == .ready && running ? .ready : .notReady, + networks: networks, + containers: members.map { $0.id }.sorted(), + startedDate: state.startedDate + ) + } + + private func containers(of id: String) async -> [ContainerSnapshot] { + guard let containersService = self.containersService else { + return [] + } + let all = (try? await containersService.list()) ?? [] + return all.filter { $0.configuration.pod == id } + } + + private static func registerService( + plugin: Plugin, + loader: PluginLoader, + id: String, + path: URL, + debug: Bool, + domain: String + ) throws { + let args = [ + "start", + "--root", path.path, + "--uuid", id, + debug ? "--debug" : nil, + ].compactMap { $0 } + try loader.registerWithLaunchd( + plugin: plugin, + pluginStateRoot: path, + args: args, + instanceId: id + ) + } + + private func deregister(id: String) async { + let runtime = (try? self._getPodState(id: id).configuration.runtimeHandler) ?? Self.runtimeHandler + let label = Self.fullLaunchdServiceLabel( + domain: self.launchdDomainString, runtimeName: runtime, instanceId: id) + try? ServiceManager.deregister(fullServiceLabel: label) + } + + private func setPodState(_ id: String, _ state: PodState, context: AsyncLock.Context) async { + self.pods[id] = state + } + + private func removePodState(_ id: String, context: AsyncLock.Context) async { + self.pods.removeValue(forKey: id) + } + + private func _getPodState(id: String) throws -> PodState { + guard let state = self.pods[id] else { + throw ContainerizationError(.notFound, message: "pod with ID \(id) not found") + } + return state + } +} diff --git a/Sources/Services/ContainerAPIService/Server/Volumes/VolumesService.swift b/Sources/Services/ContainerAPIService/Server/Volumes/VolumesService.swift index fe607e762..14413b8f6 100644 --- a/Sources/Services/ContainerAPIService/Server/Volumes/VolumesService.swift +++ b/Sources/Services/ContainerAPIService/Server/Volumes/VolumesService.swift @@ -14,6 +14,7 @@ // limitations under the License. //===----------------------------------------------------------------------===// +import ContainerAPIClient import ContainerPersistence import ContainerResource import Containerization @@ -31,17 +32,15 @@ public actor VolumesService { private let store: ContainerPersistence.FilesystemEntityStore private let log: Logger private let lock = AsyncLock() - private let containersService: ContainersService + private let containers = ContainerClient() // Storage constants private static let entityFile = "entity.json" private static let blockFile = "volume.img" - public init(resourceRoot: FilePath, containersService: ContainersService, log: Logger) async throws { - try FileManager.default.createDirectory(atPath: resourceRoot.string, withIntermediateDirectories: true) + public init(resourceRoot: FilePath, log: Logger) async throws { self.resourceRoot = resourceRoot self.store = try FilesystemEntityStore(path: resourceRoot, type: "volumes", log: log) - self.containersService = containersService self.log = log // Migrate configs stored with the old `createdAt` key to `creationDate`. @@ -179,7 +178,7 @@ public actor VolumesService { /// Calculate disk usage for volumes /// - Returns: Tuple of (total count, active count, total size, reclaimable size) - public func calculateDiskUsage() async throws -> (Int, Int, UInt64, UInt64) { + public func calculateDiskUsage() async throws -> ResourceUsage { log.debug( "VolumesService: enter", metadata: [ @@ -198,35 +197,28 @@ public actor VolumesService { return try await lock.withLock { _ in let allVolumes = try await self.store.list() - // Atomically get active volumes with container list - return try await self.containersService.withContainerList(logMetadata: ["acquirer": "\(#function)"]) { containers in - var inUseSet = Set() - - // Find all mounted volumes - for container in containers { - for mount in container.configuration.mounts { - if mount.isVolume, let volumeName = mount.volumeName { - inUseSet.insert(volumeName) - } - } - } + let inUseSet = try await self.containers.volumeNamesInUse() - var totalSize: UInt64 = 0 - var reclaimableSize: UInt64 = 0 + var totalSize: UInt64 = 0 + var reclaimableSize: UInt64 = 0 - // Calculate sizes - for volume in allVolumes { - let volumePath = self.volumePath(for: volume.name) - let volumeSize = FileManager.default.allocatedSize(of: URL(fileURLWithPath: volumePath)) - totalSize += volumeSize + // Calculate sizes + for volume in allVolumes { + let volumePath = self.volumePath(for: volume.name) + let volumeSize = FileManager.default.allocatedSize(of: URL(fileURLWithPath: volumePath)) + totalSize += volumeSize - if !inUseSet.contains(volume.name) { - reclaimableSize += volumeSize - } + if !inUseSet.contains(volume.name) { + reclaimableSize += volumeSize } - - return (allVolumes.count, inUseSet.count, totalSize, reclaimableSize) } + + return ResourceUsage( + total: allVolumes.count, + active: inUseSet.count, + sizeInBytes: totalSize, + reclaimable: reclaimableSize + ) } } @@ -372,20 +364,17 @@ public actor VolumesService { throw VolumeError.volumeNotFound(name) } - // Check if volume is in use by any container atomically - try await containersService.withContainerList(logMetadata: ["acquirer": "\(#function)", "name": "\(name)"]) { containers in - for container in containers { - for mount in container.configuration.mounts { - if mount.isVolume && mount.volumeName == name { - throw VolumeError.volumeInUse(name) - } - } - } - - try await self.store.delete(name) - try self.removeVolumeDirectory(for: name) + // A container created after this answer can name the volume and lose + // it, the same window image delete accepts against container create; + // the create then fails naming the missing volume. + let referencing = try await containers.containersReferencingVolume(name) + guard referencing.isEmpty else { + throw VolumeError.volumeInUse(name) } + try await self.store.delete(name) + try self.removeVolumeDirectory(for: name) + log.info("deleted volume", metadata: ["name": "\(name)"]) } diff --git a/Sources/Services/Network/Client/NetworkClient.swift b/Sources/Services/Network/Client/NetworkClient.swift index 97598a786..0a81f5b19 100644 --- a/Sources/Services/Network/Client/NetworkClient.swift +++ b/Sources/Services/Network/Client/NetworkClient.swift @@ -53,11 +53,15 @@ extension NetworkClient { createClient().openSession() } - public func status() async throws -> NetworkStatus { + /// A helper that never comes up answers nothing, and a call waiting on it + /// with no bound takes every command that needs a network down with it, so + /// the wait is bounded the way the calls to the api server are. Passing no + /// timeout waits as long as it takes. + public func status(timeout: Duration? = XPCClient.xpcRegistrationTimeout) async throws -> NetworkStatus { let request = XPCMessage(route: NetworkRoutes.status.rawValue) let client = createClient() - let response = try await client.send(request) + let response = try await client.send(request, responseTimeout: timeout) let status = try response.status() return status } @@ -70,26 +74,27 @@ extension NetworkClient { public func allocate( hostname: String, macAddress: MACAddress? = nil, - on session: XPCClientSession + on session: XPCClientSession, + timeout: Duration? = XPCClient.xpcRegistrationTimeout ) async throws -> (attachment: Attachment, additionalData: XPCMessage?) { let request = XPCMessage(route: NetworkRoutes.allocate.rawValue) request.set(key: NetworkKeys.hostname.rawValue, value: hostname) if let macAddress = macAddress { request.set(key: NetworkKeys.macAddress.rawValue, value: macAddress.description) } - let response = try await session.send(request) + let response = try await session.send(request, responseTimeout: timeout) let attachment = try response.attachment() let additionalData = response.additionalData() return (attachment, additionalData) } - public func lookup(hostname: String) async throws -> Attachment? { + public func lookup(hostname: String, timeout: Duration? = XPCClient.xpcRegistrationTimeout) async throws -> Attachment? { let request = XPCMessage(route: NetworkRoutes.lookup.rawValue) request.set(key: NetworkKeys.hostname.rawValue, value: hostname) let client = createClient() - let response = try await client.send(request) + let response = try await client.send(request, responseTimeout: timeout) return try response.dataNoCopy(key: NetworkKeys.attachment.rawValue).map { try JSONDecoder().decode(Attachment.self, from: $0) } diff --git a/Sources/Services/Network/Server/AttachmentAllocator.swift b/Sources/Services/Network/Server/AttachmentAllocator.swift index b7d3aeebb..9fd7aa351 100644 --- a/Sources/Services/Network/Server/AttachmentAllocator.swift +++ b/Sources/Services/Network/Server/AttachmentAllocator.swift @@ -16,44 +16,90 @@ import ContainerizationError import ContainerizationExtras +import Foundation +import Logging + +public actor AttachmentAllocator { + /// What a host was given the last time it asked, kept so that asking + /// again gets the same answer. The host's name identifies it, the way an + /// allocation is kept under the address it holds. + public struct Lease: Codable, Sendable, Identifiable { + public let id: String + public let index: UInt32 + public let macAddress: MACAddress + + public init(id: String, index: UInt32, macAddress: MACAddress) { + self.id = id + self.index = index + self.macAddress = macAddress + } + } -actor AttachmentAllocator { private let allocator: any AddressAllocator - private var hostnames: [String: UInt32] = [:] + private var allocated: [String: Lease] = [:] + private var leases: [String: Lease] = [:] + private let store: (any AttachmentLeaseStore)? + private let log: Logger? - init(lower: UInt32, size: Int) throws { + /// - Parameters: + /// - store: keeps what each host was given, so a host attaching again is + /// given it back across restarts of this service. Nothing is + /// remembered without one. + init(lower: UInt32, size: Int, store: (any AttachmentLeaseStore)? = nil, log: Logger? = nil) async throws { allocator = try UInt32.rotatingAllocator( lower: lower, size: UInt32(size) ) + self.store = store + self.log = log + if let store { + // What was written down is a convenience, so leases that cannot be + // read cost the addresses their stability and nothing else. + let written = await store.load() + self.leases = Dictionary(uniqueKeysWithValues: written.map { ($0.id, $0) }) + } } /// Allocate a network address for a host. - func allocate(hostname: String) async throws -> UInt32 { - // Client is responsible for ensuring two containers don't use same hostname, so provide existing IP if hostname exists - if let index = hostnames[hostname] { - return index + /// + /// A host that is already attached keeps what it has. One that attached + /// before is given what it had, when that address is still free: an + /// address a host answers to outlives the attachment, since the resolver + /// entries, the hosts files, and the caches on the other side of the + /// network all name it. Otherwise the next free address is taken, and + /// what the host was given is written down. + /// https://cni.dev/plugins/current/ipam/host-local/ + func allocate(hostname: String, macAddress: MACAddress) async throws -> Lease { + if let held = allocated[hostname] { + return held } - let index = try allocator.allocate() - hostnames[hostname] = index + if let remembered = leases[hostname], (try? allocator.reserve(remembered.index)) != nil { + allocated[hostname] = remembered + return remembered + } - return index + let lease = Lease(id: hostname, index: try allocator.allocate(), macAddress: macAddress) + allocated[hostname] = lease + leases[hostname] = lease + await store?.save(lease) + return lease } - /// Free an allocated network address by hostname. + /// Free an allocated network address by hostname. What the host was given + /// is still remembered, so the same host attaching again is given it back. @discardableResult func deallocate(hostname: String) async throws -> UInt32? { - guard let index = hostnames.removeValue(forKey: hostname) else { + guard let lease = allocated.removeValue(forKey: hostname) else { return nil } - try allocator.release(index) - return index + try allocator.release(lease.index) + return lease.index } /// Retrieve the allocator index for a hostname. func lookup(hostname: String) async throws -> UInt32? { - hostnames[hostname] + allocated[hostname]?.index } } diff --git a/Sources/Services/Network/Server/AttachmentLeaseStore.swift b/Sources/Services/Network/Server/AttachmentLeaseStore.swift new file mode 100644 index 000000000..d564b5f26 --- /dev/null +++ b/Sources/Services/Network/Server/AttachmentLeaseStore.swift @@ -0,0 +1,29 @@ +//===----------------------------------------------------------------------===// +// Copyright © 2026 Apple Inc. and the container project authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//===----------------------------------------------------------------------===// + +/// Keeps what each host was given, so a host attaching again is given it back +/// after this service has been restarted. +/// +/// What a lease is written to is the caller's, since a network plugin knows +/// where its own state belongs and this module holds none. +public protocol AttachmentLeaseStore: Sendable { + /// Everything written down so far. A store that cannot be read answers + /// with nothing, which costs the addresses their stability and no more. + func load() async -> [AttachmentAllocator.Lease] + + /// Write down what a host was given. + func save(_ lease: AttachmentAllocator.Lease) async +} diff --git a/Sources/Services/Network/Server/DefaultNetworkService.swift b/Sources/Services/Network/Server/DefaultNetworkService.swift index 70d17d396..e2a8191ea 100644 --- a/Sources/Services/Network/Server/DefaultNetworkService.swift +++ b/Sources/Services/Network/Server/DefaultNetworkService.swift @@ -18,6 +18,7 @@ import ContainerResource import ContainerXPC import ContainerizationError import ContainerizationExtras +import Foundation import Logging public actor DefaultNetworkService: NetworkService { @@ -28,8 +29,13 @@ public actor DefaultNetworkService: NetworkService { private var allocationsBySession: [XPCServerSession: [(hostname: String, index: UInt32)]] /// Set up a network service for the specified network. + /// - Parameters: + /// - leases: keeps what each host was given, so a host attaching again is + /// given what it had. Nothing is remembered without one, which is what + /// a caller wanting a network that forgets passes. public init( network: any Network, + leases: (any AttachmentLeaseStore)? = nil, log: Logger ) async throws { guard let status = await network.status else { @@ -40,7 +46,7 @@ public actor DefaultNetworkService: NetworkService { let size = Int(subnet.upper.value - subnet.lower.value - 3) self.network = network self.log = log - self.allocator = try AttachmentAllocator(lower: subnet.lower.value + 2, size: size) + self.allocator = try await AttachmentAllocator(lower: subnet.lower.value + 2, size: size, store: leases, log: log) self.macAddresses = [:] self.allocationsBySession = [:] } @@ -66,8 +72,15 @@ public actor DefaultNetworkService: NetworkService { throw ContainerizationError(.invalidState, message: "network \(network.id) must be running") } - let macAddress = macAddress ?? MACAddress((UInt64.random(in: 0...UInt64.max) & 0x0cff_ffff_ffff) | 0xf200_0000_0000) - let index = try await allocator.allocate(hostname: hostname) + // A hardware address is minted for a host that names none, and the one + // a host was given before is handed back when it attaches again: the + // address is what the network knows a host by, and an IPv6 address is + // derived from it, so a host that comes back with a new one comes back + // as somebody else. + let requested = macAddress ?? MACAddress((UInt64.random(in: 0...UInt64.max) & 0x0cff_ffff_ffff) | 0xf200_0000_0000) + let lease = try await allocator.allocate(hostname: hostname, macAddress: requested) + let macAddress = macAddress ?? lease.macAddress + let index = lease.index let ipv6Address = try status.ipv6Subnet .map { try CIDRv6(macAddress.ipv6Address(network: $0.lower), prefix: $0.prefix) } let ip = IPv4Address(index) @@ -77,6 +90,7 @@ public actor DefaultNetworkService: NetworkService { ipv4Address: try CIDRv4(ip, prefix: status.ipv4Subnet.prefix), ipv4Gateway: status.ipv4Gateway, ipv6Address: ipv6Address, + ipv6Gateway: status.ipv6Subnet?.gateway, macAddress: macAddress, variant: network.variant ) @@ -147,6 +161,7 @@ public actor DefaultNetworkService: NetworkService { ipv4Address: ipv4Address, ipv4Gateway: status.ipv4Gateway, ipv6Address: ipv6Address, + ipv6Gateway: status.ipv6Subnet?.gateway, macAddress: macAddress, variant: network.variant ) diff --git a/Sources/Services/Network/Server/Network.swift b/Sources/Services/Network/Server/Network.swift index 6f1ff18e1..1e5db1bd1 100644 --- a/Sources/Services/Network/Server/Network.swift +++ b/Sources/Services/Network/Server/Network.swift @@ -36,4 +36,10 @@ public protocol Network: Sendable { /// Start the network. func start() async throws + + /// Give up whatever the network holds, so the addresses it was given can + /// be handed out again. A network that goes away without this leaves its + /// range spoken for by nobody, and the next network asking for that range + /// is refused. + func stop() async } diff --git a/Sources/Services/NetworkVmnet/Server/AllocationOnlyVmnetNetwork.swift b/Sources/Services/NetworkVmnet/Server/AllocationOnlyVmnetNetwork.swift index 131e8af28..91361505c 100644 --- a/Sources/Services/NetworkVmnet/Server/AllocationOnlyVmnetNetwork.swift +++ b/Sources/Services/NetworkVmnet/Server/AllocationOnlyVmnetNetwork.swift @@ -58,6 +58,12 @@ public actor AllocationOnlyVmnetNetwork: Network { try handler(nil) } + /// The addresses this network hands out are its own bookkeeping, held + /// nowhere outside this process, so giving them up is forgetting them. + public func stop() async { + self._status = nil + } + public func start() async throws { guard _status == nil else { throw ContainerizationError(.invalidState, message: "cannot start network \(configuration.id): already started") diff --git a/Sources/Services/NetworkVmnet/Server/ReservedVmnetNetwork.swift b/Sources/Services/NetworkVmnet/Server/ReservedVmnetNetwork.swift index 5b0fee6ad..4bf18a8d9 100644 --- a/Sources/Services/NetworkVmnet/Server/ReservedVmnetNetwork.swift +++ b/Sources/Services/NetworkVmnet/Server/ReservedVmnetNetwork.swift @@ -92,6 +92,30 @@ public final class ReservedVmnetNetwork: ContainerNetworkServer.Network { } } + /// The reservation lives as long as the network object the framework + /// hands back, and the object is returned retained, so releasing it is + /// what gives the address range back. A helper that goes away without + /// releasing leaves the range reserved to a network nobody holds, and + /// every later attempt on that range is refused with no interface, route, + /// or process to point at. + /// vmnet.h, vmnet_network_create: "The lifetime of such reservation is + /// the same as that of `vmnet_network_ref`. Use `CFRelease()` to release + /// the network object." + public func stop() async { + stateMutex.withLock { state in + guard let network = state.network else { + return + } + // The framework hands the network back retained, and it arrives as + // a plain pointer rather than a managed object, so the retain is + // this side's to balance. + Unmanaged.fromOpaque(UnsafeRawPointer(network)).release() + state.network = nil + state.status = nil + log.info("released vmnet network", metadata: ["id": "\(configuration.id)"]) + } + } + private static func serialize_network_ref(ref: vmnet_network_ref) throws -> XPCMessage { var status: vmnet_return_t = .VMNET_SUCCESS guard let refObject = vmnet_network_copy_serialization(ref, &status) else { diff --git a/Sources/Services/Runtime/RuntimeClient/RuntimeClient.swift b/Sources/Services/Runtime/RuntimeClient/RuntimeClient.swift index 32a4db062..24cf27ce2 100644 --- a/Sources/Services/Runtime/RuntimeClient/RuntimeClient.swift +++ b/Sources/Services/Runtime/RuntimeClient/RuntimeClient.swift @@ -35,16 +35,41 @@ public struct RuntimeClient: Sendable { Self.machServiceLabel(runtime: runtime, id: id) } + /// The sandbox this client talks to, which is the container itself when + /// the container has a machine of its own, and the pod when it shares one. let id: String + /// The container in that sandbox the client addresses. + /// The container a request is addressed to, when it is addressed to one. + /// + /// A sandbox's own calls name no container. A container's calls name it, + /// however many containers the sandbox holds, so that no call has to be + /// read as meaning the only one there. + let containerId: String? let runtime: String let client: XPCClient - init(id: String, runtime: String, client: XPCClient) { + init(id: String, containerId: String? = nil, runtime: String, client: XPCClient) { self.id = id + self.containerId = containerId self.runtime = runtime self.client = client } + /// The same client, addressing a different container in the same sandbox. + public func addressing(_ containerId: String) -> RuntimeClient { + RuntimeClient(id: self.id, containerId: containerId, runtime: self.runtime, client: self.client) + } + + /// A request to the sandbox, naming the container it is addressed to. A + /// sandbox holding one container still hears which container is meant. + func request(_ route: String) -> XPCMessage { + let message = XPCMessage(route: route) + if let containerId = self.containerId { + message.set(key: RuntimeKeys.containerId.rawValue, value: containerId) + } + return message + } + /// Create a RuntimeClient by ID and runtime string. The returned client is ready to be used /// without additional steps. public static func create(id: String, runtime: String, timeout: Duration = XPCClient.xpcRegistrationTimeout) async throws -> RuntimeClient { @@ -77,32 +102,33 @@ public struct RuntimeClient: Sendable { // Runtime Methods extension RuntimeClient { + /// Run the sandbox with the containers it holds in it. + /// + /// The sandbox is brought up with every container named here in it, and + /// asking again for one already up puts in whichever of them it does not + /// hold yet. The standard streams belong to the container named by + /// `stdioFor`, the one whose start this is; the rest are placed with none. public func bootstrap( + bundlePaths: [String], + stdioFor: String? = nil, stdio: [FileHandle?], networkBootstrapInfos: [NetworkBootstrapInfo], - dynamicEnv: [String: String] = [:] + dynamicEnv: [String: String] = [:], + stopsWithContainers: Bool = false ) async throws { - let request = XPCMessage(route: RuntimeRoutes.bootstrap.rawValue) - - for (i, h) in stdio.enumerated() { - let key: RuntimeKeys = try { - switch i { - case 0: .stdin - case 1: .stdout - case 2: .stderr - default: - throw ContainerizationError(.invalidArgument, message: "invalid fd \(i)") - } - }() - - if let h { - request.set(key: key.rawValue, value: h) - } - } + let request = self.request(RuntimeRoutes.bootstrap.rawValue) + try request.setStdio(stdio) do { let dynamicEnv = try JSONEncoder().encode(dynamicEnv) request.set(key: RuntimeKeys.dynamicEnv.rawValue, value: dynamicEnv) + request.set(key: RuntimeKeys.sandboxStopsWithContainers.rawValue, value: stopsWithContainers) + + let pathsData = try JSONEncoder().encode(bundlePaths) + request.set(key: RuntimeKeys.bundlePaths.rawValue, value: pathsData) + if let stdioFor { + request.set(key: RuntimeKeys.containerId.rawValue, value: stdioFor) + } let infosData = try JSONEncoder().encode(networkBootstrapInfos) request.set(key: RuntimeKeys.networkBootstrapInfos.rawValue, value: infosData) @@ -116,8 +142,23 @@ extension RuntimeClient { } } + /// Hold the running sandbox to a memory size, which its containers share. + public func setTargetMemorySize(_ bytes: UInt64) async throws { + let request = self.request(RuntimeRoutes.updateResources.rawValue) + request.set(key: RuntimeKeys.memoryInBytes.rawValue, value: bytes) + do { + try await self.client.send(request) + } catch { + throw ContainerizationError( + .internalError, + message: "failed to set the memory size of sandbox \(self.id)", + cause: error + ) + } + } + public func state() async throws -> SandboxSnapshot { - let request = XPCMessage(route: RuntimeRoutes.state.rawValue) + let request = self.request(RuntimeRoutes.state.rawValue) let response: XPCMessage do { response = try await self.client.send(request) @@ -132,7 +173,7 @@ extension RuntimeClient { } public func createProcess(_ id: String, config: ProcessConfiguration, stdio: [FileHandle?]) async throws { - let request = XPCMessage(route: RuntimeRoutes.createProcess.rawValue) + let request = self.request(RuntimeRoutes.createProcess.rawValue) request.set(key: RuntimeKeys.id.rawValue, value: id) let data = try JSONEncoder().encode(config) request.set(key: RuntimeKeys.processConfig.rawValue, value: data) @@ -165,7 +206,7 @@ extension RuntimeClient { } public func startProcess(_ id: String) async throws { - let request = XPCMessage(route: RuntimeRoutes.start.rawValue) + let request = self.request(RuntimeRoutes.start.rawValue) request.set(key: RuntimeKeys.id.rawValue, value: id) do { try await self.client.send(request) @@ -178,8 +219,27 @@ extension RuntimeClient { } } + /// Stop the container this client addresses, leaving the machine it shares + /// and the containers beside it running. + public func stopContainer(options: ContainerStopOptions) async throws { + let request = self.request(RuntimeRoutes.stopContainer.rawValue) + + let data = try JSONEncoder().encode(options) + request.set(key: RuntimeKeys.stopOptions.rawValue, value: data) + + do { + try await self.client.send(request) + } catch { + throw ContainerizationError( + .internalError, + message: "failed to stop container \(self.id)", + cause: error + ) + } + } + public func stop(options: ContainerStopOptions) async throws { - let request = XPCMessage(route: RuntimeRoutes.stop.rawValue) + let request = self.request(RuntimeRoutes.stop.rawValue) let data = try JSONEncoder().encode(options) request.set(key: RuntimeKeys.stopOptions.rawValue, value: data) @@ -196,7 +256,7 @@ extension RuntimeClient { } public func kill(_ id: String, signal: String) async throws { - let request = XPCMessage(route: RuntimeRoutes.kill.rawValue) + let request = self.request(RuntimeRoutes.kill.rawValue) request.set(key: RuntimeKeys.id.rawValue, value: id) request.set(key: RuntimeKeys.signal.rawValue, value: signal) @@ -212,7 +272,7 @@ extension RuntimeClient { } public func resize(_ id: String, size: Terminal.Size) async throws { - let request = XPCMessage(route: RuntimeRoutes.resize.rawValue) + let request = self.request(RuntimeRoutes.resize.rawValue) request.set(key: RuntimeKeys.id.rawValue, value: id) request.set(key: RuntimeKeys.width.rawValue, value: UInt64(size.width)) request.set(key: RuntimeKeys.height.rawValue, value: UInt64(size.height)) @@ -229,7 +289,7 @@ extension RuntimeClient { } public func wait(_ id: String) async throws -> ExitStatus { - let request = XPCMessage(route: RuntimeRoutes.wait.rawValue) + let request = self.request(RuntimeRoutes.wait.rawValue) request.set(key: RuntimeKeys.id.rawValue, value: id) let response: XPCMessage @@ -248,7 +308,7 @@ extension RuntimeClient { } public func dial(_ port: UInt32) async throws -> FileHandle { - let request = XPCMessage(route: RuntimeRoutes.dial.rawValue) + let request = self.request(RuntimeRoutes.dial.rawValue) request.set(key: RuntimeKeys.port.rawValue, value: UInt64(port)) let response: XPCMessage @@ -271,7 +331,7 @@ extension RuntimeClient { } public func shutdown() async throws { - let request = XPCMessage(route: RuntimeRoutes.shutdown.rawValue) + let request = self.request(RuntimeRoutes.shutdown.rawValue) do { _ = try await self.client.send(request) @@ -285,7 +345,7 @@ extension RuntimeClient { } public func copyIn(source: String, destination: String, mode: UInt32, createParents: Bool = true) async throws { - let request = XPCMessage(route: RuntimeRoutes.copyIn.rawValue) + let request = self.request(RuntimeRoutes.copyIn.rawValue) request.set(key: RuntimeKeys.sourcePath.rawValue, value: source) request.set(key: RuntimeKeys.destinationPath.rawValue, value: destination) request.set(key: RuntimeKeys.fileMode.rawValue, value: UInt64(mode)) @@ -303,7 +363,7 @@ extension RuntimeClient { } public func copyOut(source: String, destination: String, createParents: Bool = true) async throws { - let request = XPCMessage(route: RuntimeRoutes.copyOut.rawValue) + let request = self.request(RuntimeRoutes.copyOut.rawValue) request.set(key: RuntimeKeys.sourcePath.rawValue, value: source) request.set(key: RuntimeKeys.destinationPath.rawValue, value: destination) request.set(key: RuntimeKeys.createParents.rawValue, value: createParents) @@ -320,7 +380,7 @@ extension RuntimeClient { } public func snapshotDisk(imagePath: String, destinationPath: String) async throws { - let request = XPCMessage(route: RuntimeRoutes.snapshotDisk.rawValue) + let request = self.request(RuntimeRoutes.snapshotDisk.rawValue) request.set(key: RuntimeKeys.imagePath.rawValue, value: imagePath) request.set(key: RuntimeKeys.destinationPath.rawValue, value: destinationPath) @@ -336,7 +396,7 @@ extension RuntimeClient { } public func statistics() async throws -> ContainerStats { - let request = XPCMessage(route: RuntimeRoutes.statistics.rawValue) + let request = self.request(RuntimeRoutes.statistics.rawValue) let response: XPCMessage do { @@ -389,4 +449,24 @@ extension XPCMessage { } return try JSONDecoder().decode([NetworkBootstrapInfo].self, from: data) } + + /// Carry the standard streams of a container, in the order the guest + /// numbers them. + func setStdio(_ stdio: [FileHandle?]) throws { + for (i, h) in stdio.enumerated() { + let key: RuntimeKeys = try { + switch i { + case 0: .stdin + case 1: .stdout + case 2: .stderr + default: + throw ContainerizationError(.invalidArgument, message: "invalid fd \(i)") + } + }() + + if let h { + self.set(key: key.rawValue, value: h) + } + } + } } diff --git a/Sources/Services/Runtime/RuntimeClient/RuntimeConfiguration.swift b/Sources/Services/Runtime/RuntimeClient/RuntimeConfiguration.swift index 29507bb70..3caa0d025 100644 --- a/Sources/Services/Runtime/RuntimeClient/RuntimeConfiguration.swift +++ b/Sources/Services/Runtime/RuntimeClient/RuntimeConfiguration.swift @@ -28,6 +28,10 @@ public struct RuntimeConfiguration: Codable, Sendable { public let initialFilesystem: Filesystem public let kernel: Kernel public let containerConfiguration: ContainerConfiguration? + /// The pod this instance drives, when it drives a pod rather than a single + /// container. The containers that share the pod's machine keep runtime + /// configurations of their own. + public let podConfiguration: PodConfiguration? public let containerRootFilesystem: Filesystem? public let options: ContainerCreateOptions? public let runtimeData: Data? @@ -37,6 +41,7 @@ public struct RuntimeConfiguration: Codable, Sendable { initialFilesystem: Filesystem, kernel: Kernel, containerConfiguration: ContainerConfiguration? = nil, + podConfiguration: PodConfiguration? = nil, containerRootFilesystem: Filesystem? = nil, options: ContainerCreateOptions? = nil, runtimeData: Data? = nil @@ -45,6 +50,7 @@ public struct RuntimeConfiguration: Codable, Sendable { self.initialFilesystem = initialFilesystem self.kernel = kernel self.containerConfiguration = containerConfiguration + self.podConfiguration = podConfiguration self.containerRootFilesystem = containerRootFilesystem self.options = options self.runtimeData = runtimeData diff --git a/Sources/Services/Runtime/RuntimeClient/RuntimeKeys.swift b/Sources/Services/Runtime/RuntimeClient/RuntimeKeys.swift index 1d3548cfe..8b2303440 100644 --- a/Sources/Services/Runtime/RuntimeClient/RuntimeKeys.swift +++ b/Sources/Services/Runtime/RuntimeClient/RuntimeKeys.swift @@ -17,6 +17,22 @@ public enum RuntimeKeys: String { /// ID key. case id + /// The container in the sandbox a request is addressed to. A sandbox + /// holding one container still names it, so every request says which + /// container it means. + case containerId + /// The configuration of a container being placed in a sandbox. + case containerConfig + /// The path to the bundle of a container being placed in a sandbox. + case bundlePath + /// The paths to the bundles of every container the sandbox holds. + case bundlePaths + /// Whether the sandbox stops once its last container has stopped, the + /// way a machine made for one container does; absent means the sandbox + /// outlives its containers, the way a named pod's machine does. + case sandboxStopsWithContainers + /// A memory size in bytes the sandbox is to be held to. + case memoryInBytes /// Vsock port number key. case port /// Exit code for a process diff --git a/Sources/Services/Runtime/RuntimeClient/RuntimeRoutes.swift b/Sources/Services/Runtime/RuntimeClient/RuntimeRoutes.swift index bbe1485f4..98516f918 100644 --- a/Sources/Services/Runtime/RuntimeClient/RuntimeRoutes.swift +++ b/Sources/Services/Runtime/RuntimeClient/RuntimeRoutes.swift @@ -25,10 +25,18 @@ public enum RuntimeRoutes: String { // MARK: - Sandbox lifecycle - /// Bootstrap the sandbox: create the VM, configure networks, and boot the guest. + /// Bootstrap the sandbox: create the VM, configure networks, put the + /// containers it holds in it, and boot the guest. case bootstrap = "com.apple.container.runtime/bootstrap" + /// Hold the running sandbox to a memory size, which its containers share. + case updateResources = "com.apple.container.runtime/updateResources" /// Stop the sandbox and all processes running inside it. case stop = "com.apple.container.runtime/stop" + /// Stop one container in the sandbox, leaving the sandbox and the other + /// containers in it running. The runtime interface stops a container and + /// stops a sandbox with separate calls for this reason. + /// https://github.com/kubernetes/cri-api/blob/master/pkg/apis/runtime/v1/api.proto + case stopContainer = "com.apple.container.runtime/stopContainer" /// Return the current state of the sandbox. case state = "com.apple.container.runtime/state" /// Get resource usage statistics for the sandbox. diff --git a/Sources/Services/RuntimeLinux/Server/RuntimeService.swift b/Sources/Services/RuntimeLinux/Server/RuntimeService.swift index 948a65603..647025a7f 100644 --- a/Sources/Services/RuntimeLinux/Server/RuntimeService.swift +++ b/Sources/Services/RuntimeLinux/Server/RuntimeService.swift @@ -36,12 +36,27 @@ import SystemPackage import struct ContainerizationOCI.Mount import struct ContainerizationOCI.Process -/// An XPC service that manages the lifecycle of a single VM-backed container. +/// An XPC service that manages the lifecycle of a VM-backed sandbox and the +/// containers running in it. public actor RuntimeService { private let connection: xpc_connection_t private let root: URL private let interfaceStrategies: [NetworkInterfaceKey: InterfaceStrategy] - private var container: ContainerInfo? + /// The machine this service drives, once it has been bootstrapped. + private var sandbox: (any Sandbox)? + /// The containers in that machine, by identifier. A pod holds several and + /// a standalone container holds one. + private var containers: [String: ContainerInfo] = [:] + + /// Whether the machine stops once its last container has stopped. The + /// boot request says, since only the control plane knows whether this + /// sandbox was named by someone or exists for its one container. + private var sandboxStopsWithContainers = false + /// The addresses a pod claimed, which every container placed in it shares. + private var podAttachments: [Attachment] = [] + /// The ports published on those addresses, which are the pod's because the + /// addresses are, and are forwarded once for all of its containers. + private var podPublishedPorts: [PublishPort] = [] private let monitor: ExitMonitor private let eventLoopGroup: any EventLoopGroup private var waiters: [String: ExitWaiter] = [:] @@ -139,180 +154,46 @@ public actor RuntimeService { self.log.debug("enter", metadata: ["func": "\(#function)"]) defer { self.log.debug("exit", metadata: ["func": "\(#function)"]) } + self.sandboxStopsWithContainers = message.bool(key: RuntimeKeys.sandboxStopsWithContainers.rawValue) + // Create the bundle if it doesn't exist yet if !self.bundleExists(at: self.root) { try self.createBundle() } return try await self.lock.withLock { _ in + // A machine that is not waiting to be brought up is running, and a + // request for one that is running is a container joining it: what + // the machine does not hold yet goes in, and the machine stays as + // it is. guard await self.state == .created else { - throw ContainerizationError( - .invalidState, - message: "container expected to be in created state, got: \(await self.state)" - ) - } - - let dynamicEnv = try message.dynamicEnv() - - let bundle = ContainerResource.Bundle(path: self.root) - try bundle.createLogFile() - - var config = try bundle.configuration - - var kernel = try bundle.kernel - // Built-in defaults keyed by arg name. Each is applied only if the user did not already - // supply the same key via --kernel-arg, letting custom kernels override them (e.g. lsm=...,bpf). - let defaultKernelArgs: KeyValuePairs = [ - "oops": "panic", - "lsm": "lockdown,capability,landlock,yama,apparmor", - ] - for (key, value) in defaultKernelArgs { - guard !kernel.commandLine.kernelArgs.contains(where: { $0.hasPrefix("\(key)=") }) else { - continue - } - kernel.commandLine.kernelArgs.append("\(key)=\(value)") - } - let vmm = VZVirtualMachineManager( - kernel: kernel, - initialFilesystem: bundle.initialFilesystem.asMount, - rosetta: config.rosetta, - logger: self.log - ) - - let networkBootstrapInfos = try message.networkBootstrapInfos() - - var sessions: [XPCClientSession] = [] - var attachments: [Attachment] = [] - var interfaces: [Interface] = [] - do { - for (index, info) in networkBootstrapInfos.enumerated() { - let attachmentConfig = config.networks[index] - let client = ContainerNetworkClient.NetworkClient(id: attachmentConfig.network, plugin: info.plugin) - let session = client.connect() - sessions.append(session) - var (attachment, additionalData) = try await client.allocate( - hostname: attachmentConfig.options.hostname, - macAddress: attachmentConfig.options.macAddress, - on: session - ) - if let mtu = attachmentConfig.options.mtu { - attachment = Attachment( - network: attachment.network, - hostname: attachment.hostname, - ipv4Address: attachment.ipv4Address, - ipv4Gateway: attachment.ipv4Gateway, - ipv6Address: attachment.ipv6Address, - macAddress: attachment.macAddress, - mtu: mtu, - variant: attachment.variant - ) - } - guard let iStrategy = self.interfaceStrategies[NetworkInterfaceKey(plugin: info.plugin, variant: attachment.variant)] else { - throw ContainerizationError( - .internalError, - message: "no available interface strategy for network \(attachment.network), plugin=\(info.plugin) variant=\(attachment.variant ?? "nil")") - } - let interface = try iStrategy.toInterface( - attachment: attachment, - interfaceIndex: index, - additionalData: additionalData - ) - attachments.append(attachment) - interfaces.append(interface) - } - } catch { - for session in sessions { session.close() } - throw error - } - - // Dynamically configure the DNS nameserver from a network if no explicit configuration - if let dns = config.dns, dns.nameservers.isEmpty { - let defaultNameservers = self.getDefaultNameservers(from: attachments) - if !defaultNameservers.isEmpty { - config.dns = ContainerConfiguration.DNSConfiguration( - nameservers: defaultNameservers, - domain: dns.domain, - searchDomains: dns.searchDomains, - options: dns.options + // A machine on its way down takes no more containers: placing + // one in it leaves a start that reported success and a + // container that never ran, which is what the caller is told + // instead. + let held = await self.state + guard held != .stopping, held != .stopped, held != .shuttingDown else { + throw ContainerizationError( + .invalidState, + message: "the machine is \(held) and takes no containers; wait for it to stop and start it again" ) } + try await self.placeContainers(message) + return message.reply() } - let stdio = message.stdio() - let containerLog = try FileHandle(forWritingTo: bundle.containerLog) - let stdout = { - if let h = stdio[1] { - return MultiWriter(handles: [h, containerLog]) - } - return MultiWriter(handles: [containerLog]) - }() - - let stderr: MultiWriter? = { - if !config.initProcess.terminal { - if let h = stdio[2] { - return MultiWriter(handles: [h, containerLog]) - } - return MultiWriter(handles: [containerLog]) - } - return nil - }() - - let stdin = { - stdio[0] ?? nil - }() - - let id = config.id - let rootfs = try bundle.containerRootfs.asMount - let container = try LinuxContainer(id, rootfs: rootfs, vmm: vmm, logger: self.log) { czConfig in - try Self.configureContainer(czConfig: &czConfig, config: config, dynamicEnv: dynamicEnv, log: self.log) - czConfig.interfaces = interfaces - czConfig.process.stdout = stdout - czConfig.process.stderr = stderr - czConfig.process.stdin = stdin - // NOTE: We can support a user providing new entries eventually, but for now craft - // a default /etc/hosts. - var hostsEntries = [Hosts.Entry.localHostIPV4()] - if !interfaces.isEmpty { - let primaryIfaceAddr = interfaces[0].ipv4Address - hostsEntries.append( - Hosts.Entry( - ipAddress: primaryIfaceAddr.address.description, - hostnames: [czConfig.hostname ?? id], - )) - } - czConfig.hosts = Hosts(entries: hostsEntries) - czConfig.bootLog = BootLog.file(path: bundle.bootlog, append: true) - } - - let ctrInfo = ContainerInfo( - container: container, - config: config, - attachments: attachments, - bundle: bundle, - io: (in: stdin, out: stdout, err: stderr) - ) - await self.setContainer(ctrInfo) - await self.setNetworkSessions(sessions) - - do { - try await container.create() + let bundle = ContainerResource.Bundle(path: self.root) + try bundle.createLogFile() - try await self.initializeWaiters(for: id) - try await self.monitor.registerProcess(id: config.id, onExit: self.onContainerExit) - if !container.interfaces.isEmpty { - try await self.startSocketForwarders(attachment: attachments[0], publishedPorts: config.publishedPorts) - } - await self.setState(.booted) - } catch { - do { - try await self.cleanUpContainer(containerInfo: ctrInfo) - await self.setState(.stopped) - } catch { - self.log.error("failed to clean up container", metadata: ["error": "\(error)"]) - } - throw error + // Every container runs in a pod, so every machine this service + // drives is a pod's. + guard bundle.isPod else { + throw ContainerizationError( + .invalidState, + message: "a sandbox is bootstrapped from a pod, and \(self.root.path) holds none" + ) } - return message.reply() + return try await self.bootstrapPod(message, bundle: bundle) } } @@ -331,10 +212,10 @@ public actor RuntimeService { return try await self.lock.withLock { lock in let id = try message.id() - let containerInfo = try await self.getContainer() - let containerId = containerInfo.container.id + let containerInfo = try await self.addressedContainer(message) + let containerId = containerInfo.id if id == containerId { - try await self.startInitProcess(lock: lock) + try await self.startInitProcess(containerId, lock: lock) await self.setState(.running) } else { try await self.startExecProcess(processId: id, lock: lock) @@ -358,8 +239,14 @@ public actor RuntimeService { defer { self.log.debug("exit", metadata: ["func": "\(#function)"]) } return try await self.lock.withLock { lock in - let containerInfo = try await self.getContainer() - let stats = try await containerInfo.container.statistics() + let containerInfo = try await self.addressedContainer(message) + let sandbox = try await self.getSandbox() + guard let stats = try await sandbox.statistics(containerIDs: [containerInfo.id], categories: .all).first else { + throw ContainerizationError( + .notFound, + message: "no statistics for container \(containerInfo.id)" + ) + } let containerStats = ContainerStats( id: stats.id, @@ -430,8 +317,9 @@ public actor RuntimeService { let id = try message.id() let config = try message.processConfig() let stdio = message.stdio() + let container = try await self.addressedContainer(message) - try await self.addNewProcess(id, config, stdio) + try await self.addNewProcess(id, in: container.id, config, stdio) try await self.initializeWaiters(for: id) do { @@ -480,7 +368,7 @@ public actor RuntimeService { var status: RuntimeStatus = .unknown var networks: [Attachment] = [] - var cs: ContainerSnapshot? + var snapshots: [ContainerSnapshot] = [] switch state { case .created, .stopped, .booted, .shuttingDown: @@ -488,15 +376,19 @@ public actor RuntimeService { case .stopping: status = .stopping case .running: - let ctr = try getContainer() - status = .running - networks = ctr.attachments - cs = ContainerSnapshot( - configuration: ctr.config, - status: RuntimeStatus.running, - networks: networks - ) + // The attachments belong to the machine, so any container in it + // reports the same ones. + networks = self.containers.values.first?.attachments ?? [] + snapshots = self.containers.values + .sorted { $0.id < $1.id } + .map { + ContainerSnapshot( + configuration: $0.config, + status: RuntimeStatus.running, + networks: $0.attachments + ) + } } let reply = message.reply() @@ -504,7 +396,7 @@ public actor RuntimeService { .init( status: status, networks: networks, - containers: cs != nil ? [cs!] : [] + containers: snapshots ) ) return reply @@ -533,18 +425,27 @@ public actor RuntimeService { case .running, .booted: await self.setState(.stopping) - let ctr = try await self.getContainer() - let exitStatus = try await self.gracefulStopContainer( - ctr.container, - signal: signal, - timeout: timeout - ) + let sandbox = try await self.getSandbox() + // Every container in the machine is stopped before the machine + // itself goes, so each is given its chance to end on its own. + var exitStatuses: [String: ExitStatus] = [:] + for ctr in await self.sortedContainers() { + exitStatuses[ctr.id] = try await self.gracefulStopContainer( + sandbox, + id: ctr.id, + signal: signal, + timeout: timeout + ) + } + try await sandbox.stop() do { if case .stopped = await self.state { return message.reply() } - try await self.cleanUpContainer(containerInfo: ctr, exitStatus: exitStatus) + for ctr in await self.sortedContainers() { + try await self.cleanUpContainer(containerInfo: ctr, exitStatus: exitStatuses[ctr.id]) + } } catch { self.log.error("failed to clean up container", metadata: ["error": "\(error)"]) } @@ -565,6 +466,34 @@ public actor RuntimeService { /// /// - Returns: An XPC message with no parameters. @Sendable + /// Stop the container a message is addressed to. + /// + /// The machine holds it and whatever else was put in it, and runs while any + /// of them runs, so it goes down here only once the last one has stopped. + /// A machine given a single container therefore goes down with it, which is + /// what it did when a container had a machine to itself, and a machine + /// holding several stays up for the rest. + public func stopContainer(_ message: XPCMessage) async throws -> XPCMessage { + self.log.debug("enter", metadata: ["func": "\(#function)"]) + defer { self.log.debug("exit", metadata: ["func": "\(#function)"]) } + + let container = try self.addressedContainer(message) + let stopOptions = try message.stopOptions() + let signal = try Signal(stopOptions.signal ?? "SIGTERM") + let timeout: Duration = .seconds(stopOptions.timeoutInSeconds) + + return try await self.lock.withLock { _ in + let sandbox = try await self.getSandbox() + _ = try await self.gracefulStopContainer( + sandbox, + id: container.config.id, + signal: signal, + timeout: timeout + ) + return message.reply() + } + } + public func kill(_ message: XPCMessage) async throws -> XPCMessage { self.log.debug("enter", metadata: ["func": "\(#function)"]) defer { self.log.debug("exit", metadata: ["func": "\(#function)"]) } @@ -575,8 +504,9 @@ public actor RuntimeService { try await self.lock.withLock { [self] _ in switch await self.state { case .running: - let ctr = try await getContainer() - if id != ctr.container.id { + // A process named for a container in the machine is that + // container's init; anything else was started by an exec. + guard await self.isContainer(id) else { guard let processInfo = await self.processes[id] else { throw ContainerizationError(.invalidState, message: "process \(id) does not exist") } @@ -588,7 +518,7 @@ public actor RuntimeService { return } - try await ctr.container.kill(signal) + try await self.getSandbox().killContainer(id, signal: signal) default: throw ContainerizationError( .invalidState, @@ -625,11 +555,13 @@ public actor RuntimeService { switch self.state { case .running: let id = try message.id() - let ctr = try getContainer() let width = message.uint64(key: RuntimeKeys.width.rawValue) let height = message.uint64(key: RuntimeKeys.height.rawValue) + let size = Terminal.Size(width: UInt16(width), height: UInt16(height)) - if id != ctr.container.id { + if self.isContainer(id) { + try await self.getSandbox().resizeContainer(id, to: size) + } else { guard let processInfo = self.processes[id] else { throw ContainerizationError( .invalidState, @@ -644,17 +576,7 @@ public actor RuntimeService { ) } - try await proc.resize( - to: .init( - width: UInt16(width), - height: UInt16(height)) - ) - } else { - try await ctr.container.resize( - to: .init( - width: UInt16(width), - height: UInt16(height)) - ) + try await proc.resize(to: size) } return message.reply() @@ -721,8 +643,9 @@ public actor RuntimeService { let mode = UInt32(message.uint64(key: RuntimeKeys.fileMode.rawValue)) let createParents = message.bool(key: RuntimeKeys.createParents.rawValue) - let ctr = try getContainer() - try await ctr.container.copyIn( + let ctr = try addressedContainer(message) + try await self.getSandbox().copyIn( + ctr.id, from: URL(fileURLWithPath: source), to: URL(fileURLWithPath: destination), mode: mode, @@ -766,8 +689,9 @@ public actor RuntimeService { let createParents = message.bool(key: RuntimeKeys.createParents.rawValue) - let ctr = try getContainer() - try await ctr.container.copyOut( + let ctr = try addressedContainer(message) + try await self.getSandbox().copyOut( + ctr.id, from: URL(fileURLWithPath: source), to: URL(fileURLWithPath: destination), createParents: createParents @@ -811,11 +735,12 @@ public actor RuntimeService { ) } - let ctr = try getContainer() + let ctr = try addressedContainer(message) + let sandbox = try getSandbox() let shouldFreeze = self.state == .running if shouldFreeze { - try await ctr.container.filesystemOperation(operation: .freeze, path: "/") + try await sandbox.filesystemOperation(ctr.id, operation: .freeze, path: "/") } do { @@ -823,7 +748,7 @@ public actor RuntimeService { } catch { if shouldFreeze { do { - try await ctr.container.filesystemOperation(operation: .thaw, path: "/") + try await sandbox.filesystemOperation(ctr.id, operation: .thaw, path: "/") } catch { self.log.error( "failed to thaw filesystem after snapshotDisk error", @@ -836,7 +761,7 @@ public actor RuntimeService { } if shouldFreeze { - try await ctr.container.filesystemOperation(operation: .thaw, path: "/") + try await sandbox.filesystemOperation(ctr.id, operation: .thaw, path: "/") } return message.reply() @@ -871,8 +796,7 @@ public actor RuntimeService { ) } - let ctr = try getContainer() - let fh = try await ctr.container.dialVsock(port: UInt32(port)) + let fh = try await getSandbox().dialVsock(port: UInt32(port)) let reply = message.reply() reply.set(key: RuntimeKeys.fd.rawValue, value: fh) @@ -885,12 +809,11 @@ public actor RuntimeService { } } - private func startInitProcess(lock: AsyncLock.Context) async throws { - let info = try self.getContainer() - let container = info.container - let id = container.id + private func startInitProcess(_ id: String, lock: AsyncLock.Context) async throws { + let info = try self.getContainer(id) + let sandbox = try self.getSandbox() - guard self.state == .booted else { + guard self.state == .booted || self.state == .running else { throw ContainerizationError( .invalidState, message: "container expected to be in booted state, got: \(self.state)" @@ -900,9 +823,9 @@ public actor RuntimeService { do { let io = info.io - try await container.start() + try await sandbox.startContainer(id) let waitFunc: ExitMonitor.WaitHandler = { - let code = try await container.wait() + let code = try await sandbox.waitContainer(id, timeoutInSeconds: nil) if let out = io.out { try out.close() } @@ -920,19 +843,21 @@ public actor RuntimeService { } private func startExecProcess(processId id: String, lock: AsyncLock.Context) async throws { - let container = try self.getContainer().container + let sandbox = try self.getSandbox() guard let processInfo = self.processes[id] else { throw ContainerizationError(.notFound, message: "process with id \(id)") } - let containerInfo = try self.getContainer() + let containerInfo = try self.getContainer(processInfo.containerId) let czConfig = try self.configureProcessConfig( config: processInfo.config, stdio: processInfo.io, containerConfig: containerInfo.config, ) - let process = try await container.exec(id, configuration: czConfig) + let process = try await sandbox.execInContainer(containerInfo.id, processID: id) { config in + config = czConfig + } try self.setUnderlyingProcess(id, process) try await process.start() @@ -1033,10 +958,10 @@ public actor RuntimeService { } private func onContainerExit(id: String, exitStatus: ExitStatus) async throws { - self.log.info("init process exited", metadata: ["status": "\(exitStatus)"]) + self.log.info("init process exited", metadata: ["id": "\(id)", "status": "\(exitStatus)"]) try await self.lock.withLock { [self] _ in - let ctrInfo = try await getContainer() + let ctrInfo = try await getContainer(id) switch await self.state { case .stopped, .stopping: @@ -1050,101 +975,25 @@ public actor RuntimeService { } catch { self.log.error("failed to clean up container", metadata: ["error": "\(error)"]) } - await setState(.stopped) - } - } - - private static func configureContainer( - czConfig: inout LinuxContainer.Configuration, - config: ContainerConfiguration, - dynamicEnv: [String: String] = [:], - log: Logger? = nil, - ) throws { - czConfig.cpus = config.resources.cpus - czConfig.cpuOverhead = config.resources.cpuOverhead - czConfig.memoryInBytes = config.resources.memoryInBytes - // Overcommit memory and allow more memory mappings than the kernel default - // so workloads inside swap-less guest VMs hit limits less easily. - var sysctls = config.sysctls - sysctls["vm.overcommit_memory"] = "1" - sysctls["vm.max_map_count"] = "262144" - czConfig.sysctl = sysctls - // If the host doesn't support this, we'll throw on container creation. - czConfig.virtualization = config.virtualization - czConfig.useInit = config.useInit - - // nil leaves LinuxContainer's own default set in place. - if let maskedPaths = config.maskedPaths { - czConfig.maskedPaths = maskedPaths - } - if let readonlyPaths = config.readonlyPaths { - czConfig.readonlyPaths = readonlyPaths - } - if let shmSize = config.shmSize { - for i in czConfig.mounts.indices { - if czConfig.mounts[i].destination == "/dev/shm" { - czConfig.mounts[i].options.removeAll { $0.hasPrefix("size=") } - czConfig.mounts[i].options.append("size=\(shmSize)") - } + // A pod's machine is its sandbox, which outlives the containers + // that come and go in it: it holds the addresses and namespaces + // they share and is taken down when the pod is, not when a + // container in it leaves. A machine nobody named exists for its + // one container, so it stops with the last thing in it and + // releases the devices it held; the boot request says which + // kind this machine is. + // https://github.com/kubernetes/cri-api/blob/master/pkg/apis/runtime/v1/api.proto + let sandbox = try await self.getSandbox() + if sandbox is LinuxPod, !(await self.sandboxStopsWithContainers) { + return } - } - - for mount in config.mounts { - if try mount.isSocket() { - let attrs = try? FileManager.default.attributesOfItem(atPath: mount.source) - let permissions = (attrs?[.posixPermissions] as? NSNumber) - .map { FilePermissions(rawValue: mode_t($0.intValue)) } - let socket = UnixSocketConfiguration( - source: URL(filePath: mount.source), - destination: URL(filePath: mount.destination), - permissions: permissions, - direction: .into, - ) - czConfig.sockets.append(socket) - } else { - czConfig.mounts.append(mount.asMount) + guard await self.containers.isEmpty else { + return } + try? await sandbox.stop() + await setState(.stopped) } - - for publishedSocket in config.publishedSockets { - // UnixSocketConfiguration (Containerization) takes URL; convert from FilePath at the boundary. - let socketConfig = UnixSocketConfiguration( - source: URL(filePath: publishedSocket.containerPath.string), - destination: URL(filePath: publishedSocket.hostPath.string), - permissions: publishedSocket.permissions, - direction: .outOf - ) - czConfig.sockets.append(socketConfig) - } - - if let socketUrl = Self.sshAuthSocketHostUrl(config: config, dynamicEnv: dynamicEnv, log: log) { - let socketPath = socketUrl.path(percentEncoded: false) - let attrs = try? FileManager.default.attributesOfItem(atPath: socketPath) - let permissions = (attrs?[.posixPermissions] as? NSNumber) - .map { FilePermissions(rawValue: mode_t($0.intValue)) } - let socketConfig = UnixSocketConfiguration( - source: socketUrl, - destination: URL(fileURLWithPath: Self.sshAuthSocketGuestPath), - permissions: permissions, - direction: .into, - ) - czConfig.sockets.append(socketConfig) - } - - let hostnameSource = config.networks.first?.options.hostname ?? config.id - czConfig.hostname = - hostnameSource.split(separator: ".", maxSplits: 1, omittingEmptySubsequences: true) - .first - .map { String($0) } ?? config.id - - if let dns = config.dns { - czConfig.dns = DNS( - nameservers: dns.nameservers, domain: dns.domain, - searchDomains: dns.searchDomains, options: dns.options) - } - - try Self.configureInitialProcess(czConfig: &czConfig, config: config) } private nonisolated func getDefaultNameservers(from attachments: [Attachment]) -> [String] { @@ -1154,37 +1003,39 @@ public actor RuntimeService { return [] } + /// The init process a container starts with, which is the same whether the + /// container has a machine to itself or shares one with a pod's others. private static func configureInitialProcess( - czConfig: inout LinuxContainer.Configuration, + process czProcess: inout LinuxProcessConfiguration, config: ContainerConfiguration, ) throws { let process = config.initProcess - czConfig.process.arguments = [process.executable] + process.arguments - czConfig.process.environmentVariables = process.environment + czProcess.arguments = [process.executable] + process.arguments + czProcess.environmentVariables = process.environment if config.ssh { - if !czConfig.process.environmentVariables.contains(where: { $0.starts(with: "\(Self.sshAuthSocketEnvVar)=") }) { - czConfig.process.environmentVariables.append("\(Self.sshAuthSocketEnvVar)=\(Self.sshAuthSocketGuestPath)") + if !czProcess.environmentVariables.contains(where: { $0.starts(with: "\(Self.sshAuthSocketEnvVar)=") }) { + czProcess.environmentVariables.append("\(Self.sshAuthSocketEnvVar)=\(Self.sshAuthSocketGuestPath)") } } - czConfig.process.terminal = process.terminal - czConfig.process.workingDirectory = process.workingDirectory - try czConfig.process.rlimits = process.rlimits.map { + czProcess.terminal = process.terminal + czProcess.workingDirectory = process.workingDirectory + try czProcess.rlimits = process.rlimits.map { LinuxRLimit( kind: try LinuxRLimit.Kind($0.limit), hard: $0.hard, soft: $0.soft ) } - czConfig.process.capabilities = try Self.effectiveCapabilities( + czProcess.capabilities = try Self.effectiveCapabilities( capAdd: config.capAdd, capDrop: config.capDrop ) switch process.user { case .raw(let name): - czConfig.process.user = .init( + czProcess.user = .init( uid: 0, gid: 0, umask: nil, @@ -1192,7 +1043,7 @@ public actor RuntimeService { username: name ) case .id(let uid, let gid): - czConfig.process.user = .init( + czProcess.user = .init( uid: uid, gid: gid, umask: nil, @@ -1202,6 +1053,123 @@ public actor RuntimeService { } } + /// The sockets a container asks for: those it is given, those it publishes, + /// and the host's ssh agent when it asked for one. + private static func sockets( + config: ContainerConfiguration, + dynamicEnv: [String: String], + log: Logger? + ) throws -> (sockets: [UnixSocketConfiguration], mounts: [Filesystem]) { + var sockets: [UnixSocketConfiguration] = [] + var mounts: [Filesystem] = [] + + for mount in config.mounts { + if try mount.isSocket() { + let attrs = try? FileManager.default.attributesOfItem(atPath: mount.source) + let permissions = (attrs?[.posixPermissions] as? NSNumber) + .map { FilePermissions(rawValue: mode_t($0.intValue)) } + sockets.append( + UnixSocketConfiguration( + source: URL(filePath: mount.source), + destination: URL(filePath: mount.destination), + permissions: permissions, + direction: .into, + )) + } else { + mounts.append(mount) + } + } + + for publishedSocket in config.publishedSockets { + // UnixSocketConfiguration (Containerization) takes URL; convert from FilePath at the boundary. + sockets.append( + UnixSocketConfiguration( + source: URL(filePath: publishedSocket.containerPath.string), + destination: URL(filePath: publishedSocket.hostPath.string), + permissions: publishedSocket.permissions, + direction: .outOf + )) + } + + if let socketUrl = Self.sshAuthSocketHostUrl(config: config, dynamicEnv: dynamicEnv, log: log) { + let socketPath = socketUrl.path(percentEncoded: false) + let attrs = try? FileManager.default.attributesOfItem(atPath: socketPath) + let permissions = (attrs?[.posixPermissions] as? NSNumber) + .map { FilePermissions(rawValue: mode_t($0.intValue)) } + sockets.append( + UnixSocketConfiguration( + source: socketUrl, + destination: URL(fileURLWithPath: Self.sshAuthSocketGuestPath), + permissions: permissions, + direction: .into, + )) + } + + return (sockets, mounts) + } + + /// The hostname a container reports, taken from the name its network knows + /// it by, up to the first dot. + /// The name the sandbox answers to, taken from the first network it + /// attaches to and falling back to its own id. + private static func hostname(networks: [AttachmentConfiguration], id: String) -> String { + let hostnameSource = networks.first?.options.hostname ?? id + return + hostnameSource.split(separator: ".", maxSplits: 1, omittingEmptySubsequences: true) + .first + .map { String($0) } ?? id + } + + /// Configure a container that shares a pod's machine. + /// + /// The machine's processors, memory, swap, addresses and boot log are the + /// pod's, so what is set here is the container's alone: what it runs, what + /// it can see, and the limits it holds itself to within the pod's. + private static func configurePodContainer( + czConfig: inout LinuxPod.ContainerConfiguration, + config: ContainerConfiguration, + dynamicEnv: [String: String] = [:], + log: Logger? = nil, + ) throws { + // A container in a pod draws on the machine's processors and memory + // unless it was given a limit of its own. + czConfig.cpus = config.resources.cpus + czConfig.memoryInBytes = config.resources.memoryInBytes + czConfig.swapInBytes = config.resources.swapInBytes + + czConfig.useInit = config.useInit + + // nil leaves the library's own default set in place. + if let maskedPaths = config.maskedPaths { + czConfig.maskedPaths = maskedPaths + } + if let readonlyPaths = config.readonlyPaths { + czConfig.readonlyPaths = readonlyPaths + } + + if let shmSize = config.shmSize { + for i in czConfig.mounts.indices { + if czConfig.mounts[i].destination == "/dev/shm" { + czConfig.mounts[i].options.removeAll { $0.hasPrefix("size=") } + czConfig.mounts[i].options.append("size=\(shmSize)") + } + } + } + + let (sockets, mounts) = try Self.sockets(config: config, dynamicEnv: dynamicEnv, log: log) + czConfig.sockets.append(contentsOf: sockets) + czConfig.mounts.append(contentsOf: mounts.map { $0.asMount }) + + // The hostname, the resolver and the hosts file are the sandbox's: the + // runtime interface carries all three on the pod and gives a container + // none of its own, so the pod holds them and its containers inherit + // them. The library lets a container override each one; a container + // here asks for none, so the pod's stand. + // https://github.com/kubernetes/cri-api/blob/master/pkg/apis/runtime/v1/api.proto + + try Self.configureInitialProcess(process: &czConfig.process, config: config) + } + private nonisolated func configureProcessConfig(config: ProcessConfiguration, stdio: [FileHandle?], containerConfig: ContainerConfiguration) throws -> LinuxProcessConfiguration { @@ -1294,29 +1262,404 @@ public actor RuntimeService { } } - private func getContainer() throws -> ContainerInfo { - guard let container else { + /// Run the machine a pod's containers share, with those containers in it. + /// + /// The pod's own bundle carries the machine: its kernel, its initial + /// filesystem, and the size, networks and name resolution its containers + /// draw on. The containers keep bundles of their own, named in the request, + /// and go in on the way to a machine that runs holding them. + /// + private func bootstrapPod(_ message: XPCMessage, bundle: ContainerResource.Bundle) async throws -> XPCMessage { + let config = try bundle.podConfiguration + let kernel = try self.kernelWithDefaultArgs(bundle.kernel) + let vmm = VZVirtualMachineManager( + kernel: kernel, + initialFilesystem: bundle.initialFilesystem.asMount, + rosetta: config.rosetta, + logger: self.log + ) + + let (sessions, attachments, interfaces) = try await self.allocateNetworks( + config.networks, + infos: try message.networkBootstrapInfos() + ) + + // DNS the way the record says. A configuration naming no resolver is + // filled out from the network, the gateway resolving for the machine's + // containers. Nothing recorded is a refusal only where a refusal can + // be made: a container declines with --no-dns and the machine made for + // it carries that refusal, while a machine someone named holds no such + // request and is given the network's resolver, the way it was before + // it could be asked. + let derived = ContainerConfiguration.DNSConfiguration( + nameservers: self.getDefaultNameservers(from: attachments), + domain: config.dns?.domain, + searchDomains: config.dns?.searchDomains ?? [], + options: config.dns?.options ?? [] + ) + let dns: ContainerConfiguration.DNSConfiguration? = { + guard let configured = config.dns else { + return config.isAnonymous ? nil : derived + } + return configured.nameservers.isEmpty ? derived : configured + }() + + // One swap area serves the whole pod, which is what makes the pool its + // containers reclaim to a shared one. + let swapLayer = try config.resources.swapInBytes.map { + try bundle.createSwapDevice(size: $0).asMount + } + + let pod = try LinuxPod(config.id, vmm: vmm, logger: self.log) { podConfig in + podConfig.cpus = config.resources.cpus + podConfig.memoryInBytes = config.resources.memoryInBytes + // The machine is built larger than the pod by what the guest agent + // takes, so what the pod was given is what its containers have. A + // caller sizing the machine itself asks for none of that overhead + // and gets the size it named. + podConfig.cpuOverhead = config.resources.cpuOverhead + podConfig.swapLayer = swapLayer + podConfig.interfaces = interfaces + podConfig.virtualization = config.virtualization + podConfig.shareProcessNamespace = config.shareProcessNamespace + podConfig.hostname = config.hostname ?? Self.hostname(networks: config.networks, id: config.id) + // The hosts file names the pod at its own address so its containers + // reach the name they answer to, and names the network's gateway so + // they reach the host they run on: `host.containers.internal` is the + // cross-runtime name for it, which Podman established, and + // `host.docker.internal` the one Docker's tools look for, so both are + // given as Podman gives them, at the IPv4 gateway and, on a dual-stack + // network, the IPv6 gateway too. It is written once for the pod the + // way the resolver and the hostname are. + var hostsEntries = [Hosts.Entry.localHostIPV4()] + if let primary = attachments.first { + hostsEntries.append( + Hosts.Entry( + ipAddress: primary.ipv4Address.address.description, + hostnames: [podConfig.hostname ?? config.id], + )) + hostsEntries.append( + Hosts.Entry( + ipAddress: primary.ipv4Gateway.description, + hostnames: ["host.containers.internal", "host.docker.internal"], + )) + if let ipv6Gateway = primary.ipv6Gateway { + hostsEntries.append( + Hosts.Entry( + ipAddress: ipv6Gateway.description, + hostnames: ["host.containers.internal", "host.docker.internal"], + )) + } + } + podConfig.hosts = Hosts(entries: hostsEntries) + // The runtime asks for these two of every machine it boots; they + // stand alongside whatever the pod was given. + var sysctls = config.sysctls + sysctls["vm.overcommit_memory"] = "1" + sysctls["vm.max_map_count"] = "262144" + podConfig.sysctl = sysctls + if let dns { + podConfig.dns = DNS( + nameservers: dns.nameservers, + domain: dns.domain, + searchDomains: dns.searchDomains, + options: dns.options + ) + } + podConfig.bootLog = BootLog.file(path: bundle.bootlog, append: true) + } + + self.setSandbox(pod) + self.setNetworkSessions(sessions) + self.podAttachments = attachments + self.podPublishedPorts = config.publishedPorts + + try await self.placeContainers(message) + + try await pod.create() + // The pod holds one address for every container in it, so the ports + // published on it are the pod's and are forwarded once. Forwarding each + // container's separately would let two of them claim one host port, + // which the overlap check cannot see when it is asked about one + // container at a time. + if let primary = attachments.first { + try await self.startSocketForwarders(attachment: primary, publishedPorts: config.publishedPorts) + } + self.setState(.booted) + + return message.reply() + } + + /// Put the containers a request names in the sandbox. + /// + /// Each brings its own bundle, holding its configuration and its root + /// filesystem, and takes the machine's processors, memory, swap and + /// addresses as they are. One already in the machine stays as it is, so a + /// request naming every container the pod holds puts in what is missing and + /// leaves the rest alone. + /// + /// The standard streams belong to the one container whose start the request + /// is; the others are placed with none and are given theirs when they are + /// started in turn. + private func placeContainers(_ message: XPCMessage) async throws { + self.log.debug("enter", metadata: ["func": "\(#function)"]) + defer { self.log.debug("exit", metadata: ["func": "\(#function)"]) } + + guard let pathsData = message.dataNoCopy(key: RuntimeKeys.bundlePaths.rawValue) else { + return + } + let paths = try JSONDecoder().decode([String].self, from: pathsData) + let stdioFor = message.string(key: RuntimeKeys.containerId.rawValue) + + for path in paths { + try await self.placeContainer( + at: path, + stdio: URL(filePath: path).lastPathComponent == stdioFor ? message.stdio() : [nil, nil, nil], + dynamicEnv: try message.dynamicEnv() + ) + } + } + + private func placeContainer(at path: String, stdio: [FileHandle?], dynamicEnv: [String: String]) async throws { + let sandbox = try self.getSandbox() + guard let pod = sandbox as? LinuxPod else { + throw ContainerizationError( + .invalidState, + message: "the sandbox holds a single container and takes no others" + ) + } + + // A container the machine already holds is one this request has nothing + // to do for. + guard !self.isContainer(URL(filePath: path).lastPathComponent) else { + return + } + + do { + // A container placed in a pod has no machine of its own to build + // its bundle, so the pod's machine builds it on the way in. + let root = URL(filePath: path) + if !self.bundleExists(at: root) { + try self.createBundle(at: root) + } + + let bundle = ContainerResource.Bundle(path: root) + try bundle.createLogFile() + let config = try bundle.configuration + let containerLog = try FileHandle(forWritingTo: bundle.containerLog) + let stdout = { + if let h = stdio[1] { + return MultiWriter(handles: [h, containerLog]) + } + return MultiWriter(handles: [containerLog]) + }() + let stderr: MultiWriter? = { + if !config.initProcess.terminal { + if let h = stdio[2] { + return MultiWriter(handles: [h, containerLog]) + } + return MultiWriter(handles: [containerLog]) + } + return nil + }() + let stdin = stdio[0] ?? nil + + let rootfs = try bundle.containerRootfs.asMount + let attachments = self.podAttachments + + try await pod.addContainer(config.id, rootfs: rootfs) { czConfig in + try Self.configurePodContainer( + czConfig: &czConfig, + config: config, + dynamicEnv: dynamicEnv, + log: self.log + ) + czConfig.process.stdout = stdout + czConfig.process.stderr = stderr + czConfig.process.stdin = stdin + } + + self.setContainer( + ContainerInfo( + config: config, + attachments: attachments, + bundle: bundle, + io: (in: stdin, out: stdout, err: stderr) + ) + ) + + // What waits on the container waits from the moment it is in the + // machine, so a container that boots with the machine and one that + // joins a machine already running are both waited on the same way. + try self.initializeWaiters(for: config.id) + try await self.monitor.registerProcess(id: config.id, onExit: self.onContainerExit) + } + } + + /// Hold the running machine to a memory size, which its containers share. + /// + /// - Parameters: + /// - message: An XPC message with the following parameters: + /// - memoryInBytes: The size to hold the machine to. + /// + /// - Returns: An XPC message with no parameters. + @Sendable + public func updateResources(_ message: XPCMessage) async throws -> XPCMessage { + self.log.debug("enter", metadata: ["func": "\(#function)"]) + defer { self.log.debug("exit", metadata: ["func": "\(#function)"]) } + + let bytes = message.uint64(key: RuntimeKeys.memoryInBytes.rawValue) + guard bytes > 0 else { + throw ContainerizationError(.invalidArgument, message: "a memory size is required") + } + try await self.getSandbox().setTargetMemorySize(bytes) + return message.reply() + } + + /// Kernel arguments applied unless the caller already supplied the same + /// key, so a custom kernel can override them (e.g. lsm=...,bpf). + private func kernelWithDefaultArgs(_ kernel: Kernel) -> Kernel { + var kernel = kernel + let defaultKernelArgs: KeyValuePairs = [ + "oops": "panic", + "lsm": "lockdown,capability,landlock,yama,apparmor", + ] + for (key, value) in defaultKernelArgs { + guard !kernel.commandLine.kernelArgs.contains(where: { $0.hasPrefix("\(key)=") }) else { + continue + } + kernel.commandLine.kernelArgs.append("\(key)=\(value)") + } + return kernel + } + + /// Claim an address on each of the sandbox's networks. + /// + /// The attachments belong to the machine, which every container in it + /// shares, since a container in a sandbox is given no network namespace of + /// its own. + private func allocateNetworks( + _ networks: [AttachmentConfiguration], + infos: [NetworkBootstrapInfo] + ) async throws -> (sessions: [XPCClientSession], attachments: [Attachment], interfaces: [Interface]) { + var sessions: [XPCClientSession] = [] + var attachments: [Attachment] = [] + var interfaces: [Interface] = [] + do { + for (index, info) in infos.enumerated() { + let attachmentConfig = networks[index] + let client = ContainerNetworkClient.NetworkClient(id: attachmentConfig.network, plugin: info.plugin) + let session = client.connect() + sessions.append(session) + var (attachment, additionalData) = try await client.allocate( + hostname: attachmentConfig.options.hostname, + macAddress: attachmentConfig.options.macAddress, + on: session + ) + if let mtu = attachmentConfig.options.mtu { + attachment = Attachment( + network: attachment.network, + hostname: attachment.hostname, + ipv4Address: attachment.ipv4Address, + ipv4Gateway: attachment.ipv4Gateway, + ipv6Address: attachment.ipv6Address, + ipv6Gateway: attachment.ipv6Gateway, + macAddress: attachment.macAddress, + mtu: mtu, + variant: attachment.variant + ) + } + guard let iStrategy = self.interfaceStrategies[NetworkInterfaceKey(plugin: info.plugin, variant: attachment.variant)] else { + throw ContainerizationError( + .internalError, + message: "no available interface strategy for network \(attachment.network), plugin=\(info.plugin) variant=\(attachment.variant ?? "nil")") + } + let interface = try iStrategy.toInterface( + attachment: attachment, + interfaceIndex: index, + additionalData: additionalData + ) + attachments.append(attachment) + interfaces.append(interface) + } + } catch { + for session in sessions { session.close() } + throw error + } + return (sessions, attachments, interfaces) + } + + /// The machine this service drives. + private func getSandbox() throws -> any Sandbox { + guard let sandbox else { throw ContainerizationError( .invalidState, - message: "no container found" + message: "no sandbox found" + ) + } + return sandbox + } + + /// The machine's containers in a settled order, so that what is done to + /// each of them in turn happens the same way every time. + private func sortedContainers() -> [ContainerInfo] { + self.containers.values.sorted { $0.id < $1.id } + } + + /// Whether a name is one of the machine's containers, which is what tells + /// a container's init process apart from a process an exec started. + private func isContainer(_ id: String) -> Bool { + self.containers[id] != nil + } + + /// A container in the machine, by name. + private func getContainer(_ id: String) throws -> ContainerInfo { + guard let container = self.containers[id] else { + throw ContainerizationError( + .notFound, + message: "container \(id) not found in sandbox" ) } return container } - private func gracefulStopContainer(_ lc: LinuxContainer, signal: Signal, timeout: Duration) async throws -> ExitStatus { + /// The container a message is addressed to. + /// + /// A request for a container names it, whatever else the machine holds, so + /// that a machine holding one is answered the same way as a machine holding + /// several and no request means "the only one here". + private func addressedContainer(_ message: XPCMessage) throws -> ContainerInfo { + guard let id = message.string(key: RuntimeKeys.containerId.rawValue), !id.isEmpty else { + throw ContainerizationError( + .invalidArgument, + message: "the request names no container to act on" + ) + } + return try getContainer(id) + } + + /// Stop one container in the sandbox and wait for it, then leave. + /// + /// The machine stays up, since the sandbox's other containers are still in + /// it. Powering it off is the sandbox's own stop. + private func gracefulStopContainer( + _ sandbox: any Sandbox, + id: String, + signal: Signal, + timeout: Duration + ) async throws -> ExitStatus { // Try and gracefully shut down the process. Even if this succeeds we need to power off // the vm, but we should try this first always. var code = ExitStatus(exitCode: 255) do { code = try await withThrowingTaskGroup(of: ExitStatus.self) { group in group.addTask { - try await lc.wait() + try await sandbox.waitContainer(id, timeoutInSeconds: nil) } group.addTask { - try await lc.kill(signal) + try await sandbox.killContainer(id, signal: signal) try await Task.sleep(for: timeout) - try await lc.kill(.kill) + try await sandbox.killContainer(id, signal: .kill) return ExitStatus(exitCode: 137) } @@ -1334,29 +1677,48 @@ public actor RuntimeService { self.log.error("graceful stop failed; forcing vm shutdown", metadata: ["error": "\(error)"]) } - // Now actually bring down the vm. - try await lc.stop() - return code } private func cleanUpContainer(containerInfo: ContainerInfo, exitStatus: ExitStatus? = nil) async throws { - let container = containerInfo.container - let id = container.id + let id = containerInfo.id do { - try await container.stop() + try await self.getSandbox().stopContainer(id) } catch { self.log.error("failed to stop container during cleanup", metadata: ["error": "\(error)"]) } - await self.stopSocketForwarders() + // The machine keeps a stopped container's place until it is given + // back. The registry below forgets the name, so the machine must give + // it up too, or the next placement under it is refused against a + // place nothing holds. + do { + try await self.getSandbox().removeContainer(id) + } catch { + self.log.error("failed to remove container during cleanup", metadata: ["error": "\(error)"]) + } + + self.containers.removeValue(forKey: id) + self.processes.removeValue(forKey: id) + await self.monitor.stopTracking(id: id) + + // The forwarders and the network sessions are the machine's, which the + // sandbox's containers share, so they are given up once the last of + // them is gone. + if self.containers.isEmpty { + await self.stopSocketForwarders() - for session in networkSessions { session.close() } - networkSessions = [] + for session in networkSessions { session.close() } + networkSessions = [] + } let status = exitStatus ?? ExitStatus(exitCode: 255) self.releaseWaiters(for: id, status: status) + // The waiter's name is given back with the container's: whoever was + // waiting has been answered, and the next container under this name + // registers a waiter of its own. + self.waiters.removeValue(forKey: id) } } @@ -1414,6 +1776,35 @@ extension XPCMessage { } extension ContainerResource.Bundle { + /// Create the raw block file backing the container's swap area. + /// + /// It carries no filesystem: the guest agent writes the swap header to the + /// device and enables it. The file is sparse, so it costs the host only the + /// pages the guest has actually swapped out, and gives them back on + /// discard. A swap area held in a file has to be free of holes, since the + /// kernel walks its extents; the guest reaches this one as a block device, + /// which the kernel takes as a single extent without consulting the host's + /// layout. https://github.com/torvalds/linux/blob/master/mm/swapfile.c + /// The area holds nothing that outlives the container, so it is made afresh + /// with every bootstrap and the host is told not to synchronize it. + func createSwapDevice(size: UInt64) throws -> Filesystem { + let path = self.containerSwapBlock + guard FileManager.default.createFile(atPath: path.path, contents: nil) else { + throw ContainerizationError( + .internalError, message: "failed to create swap device at \(path.path)") + } + let handle = try FileHandle(forWritingTo: path) + defer { try? handle.close() } + try handle.truncate(atOffset: size) + return .block( + format: Swap.mountType, + source: path.path, + destination: "", + options: [], + sync: .nosync + ) + } + func createLogFile() throws { // Create the log file we'll write stdio to. // O_TRUNC resolves a log delay issue on restarted containers by force-updating internal state @@ -1580,21 +1971,27 @@ extension RuntimeService { } private func setContainer(_ info: ContainerInfo) { - self.container = info + self.containers[info.id] = info + } + + private func setSandbox(_ sandbox: any Sandbox) { + self.sandbox = sandbox } private func setNetworkSessions(_ sessions: [XPCClientSession]) { self.networkSessions = sessions } - private func addNewProcess(_ id: String, _ config: ProcessConfiguration, _ io: [FileHandle?]) throws { + private func addNewProcess(_ id: String, in containerId: String, _ config: ProcessConfiguration, _ io: [FileHandle?]) throws { guard self.processes[id] == nil else { throw ContainerizationError(.invalidArgument, message: "process \(id) already exists") } - self.processes[id] = ProcessInfo(config: config, process: nil, state: .created, io: io) + self.processes[id] = ProcessInfo(containerId: containerId, config: config, process: nil, state: .created, io: io) } private struct ProcessInfo { + /// The container in the sandbox the process runs in. + let containerId: String let config: ProcessConfiguration var process: LinuxProcess? var state: State @@ -1602,11 +1999,12 @@ extension RuntimeService { } private struct ContainerInfo { - let container: LinuxContainer let config: ContainerConfiguration let attachments: [Attachment] let bundle: ContainerResource.Bundle let io: (in: FileHandle?, out: MultiWriter?, err: MultiWriter?) + + var id: String { config.id } } /// States the underlying sandbox can be in. @@ -1637,6 +2035,9 @@ extension RuntimeService { } let bundle = ContainerResource.Bundle(path: path) + if bundle.isPod { + return true + } do { _ = try bundle.configuration return true @@ -1646,14 +2047,15 @@ extension RuntimeService { } /// Create bundle from RuntimeConfiguration - private func createBundle() throws { + private func createBundle(at root: URL? = nil) throws { do { - let runtimeConfig = try RuntimeConfiguration.readRuntimeConfiguration(from: self.root) + let runtimeConfig = try RuntimeConfiguration.readRuntimeConfiguration(from: root ?? self.root) _ = try ContainerResource.Bundle.create( path: runtimeConfig.path, initialFilesystem: runtimeConfig.initialFilesystem, kernel: runtimeConfig.kernel, containerConfiguration: runtimeConfig.containerConfiguration, + podConfiguration: runtimeConfig.podConfiguration, containerRootFilesystem: runtimeConfig.containerRootFilesystem, options: runtimeConfig.options ) diff --git a/Sources/Services/RuntimeLinux/Server/Sandbox.swift b/Sources/Services/RuntimeLinux/Server/Sandbox.swift new file mode 100644 index 000000000..6b03283d7 --- /dev/null +++ b/Sources/Services/RuntimeLinux/Server/Sandbox.swift @@ -0,0 +1,94 @@ +//===----------------------------------------------------------------------===// +// Copyright © 2026 Apple Inc. and the container project authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//===----------------------------------------------------------------------===// + +import Containerization +import ContainerizationError +import ContainerizationOS +import Foundation + +import struct ContainerizationOS.Terminal + +/// The machine the runtime service drives, and the containers running in it. +/// +/// A pod holds several containers and a standalone container holds one. The +/// service speaks to both the same way, naming the container it means, which +/// is how the runtime interface addresses a container in a sandbox. +/// https://github.com/kubernetes/cri-api/blob/master/pkg/apis/runtime/v1/api.proto +protocol Sandbox: Sendable { + /// Boot the machine and set up the containers registered in it. + func create() async throws + + /// Start a container's init process. + func startContainer(_ id: String) async throws + + /// Stop a container, leaving the machine running for the others. + func stopContainer(_ id: String) async throws + + /// Take a stopped container out of the machine, so its name is free to + /// place again. + func removeContainer(_ id: String) async throws + + /// Signal a container's init process. + func killContainer(_ id: String, signal: Signal) async throws + + /// Wait for a container's init process to exit. + func waitContainer(_ id: String, timeoutInSeconds: Int64?) async throws -> ExitStatus + + /// Resize the terminal of a container's init process. + func resizeContainer(_ id: String, to: Terminal.Size) async throws + + /// Run an additional process in a container. + func execInContainer( + _ id: String, + processID: String, + configuration: @Sendable @escaping (inout LinuxProcessConfiguration) throws -> Void + ) async throws -> LinuxProcess + + /// Resource usage, for the named containers or for all of them. + func statistics(containerIDs: [String]?, categories: StatCategory) async throws -> [ContainerStatistics] + + /// Act on a path in a container's filesystem. + func filesystemOperation(_ id: String, operation: FilesystemOperation, path: String) async throws + + /// Copy a file or directory from the host into a container. + func copyIn(_ id: String, from source: URL, to destination: URL, mode: UInt32, createParents: Bool) async throws + + /// Copy a file or directory out of a container to the host. + func copyOut(_ id: String, from source: URL, to destination: URL, createParents: Bool) async throws + + /// Open a vsock connection to a port in the machine. + func dialVsock(port: UInt32) async throws -> FileHandle + + /// Stop every container and power the machine off. + func stop() async throws + + /// Ask the guest to hold itself to a memory size, which the machine's + /// containers share. + func setTargetMemorySize(_ bytes: UInt64) async throws +} + +/// A pod already addresses its containers by name, so it is a sandbox as it +/// stands, save for the copies, which take a transfer size the runtime leaves +/// at its default. +extension LinuxPod: Sandbox { + func copyIn(_ id: String, from source: URL, to destination: URL, mode: UInt32, createParents: Bool) async throws { + try await self.copyIn(id, from: source, to: destination, mode: mode, createParents: createParents, chunkSize: Self.defaultCopyChunkSize) + } + + func copyOut(_ id: String, from source: URL, to destination: URL, createParents: Bool) async throws { + try await self.copyOut(id, from: source, to: destination, createParents: createParents, chunkSize: Self.defaultCopyChunkSize) + } +} diff --git a/Tests/ContainerAPIServiceTests/DiskUsagePathTests.swift b/Tests/ContainerAPIServiceTests/DiskUsagePathTests.swift index 947d680e4..d15878cb4 100644 --- a/Tests/ContainerAPIServiceTests/DiskUsagePathTests.swift +++ b/Tests/ContainerAPIServiceTests/DiskUsagePathTests.swift @@ -50,7 +50,6 @@ struct DiskUsagePathTests { private func makeVolumesService(appRoot: FilePath) async throws -> VolumesService { try await VolumesService( resourceRoot: appRoot.appending("volumes"), - containersService: makeContainersService(appRoot: appRoot), log: log ) } diff --git a/Tests/ContainerNetworkServerTests/AttachmentAllocatorTest.swift b/Tests/ContainerNetworkServerTests/AttachmentAllocatorTest.swift index 86ea3eff0..a3788f1e3 100644 --- a/Tests/ContainerNetworkServerTests/AttachmentAllocatorTest.swift +++ b/Tests/ContainerNetworkServerTests/AttachmentAllocatorTest.swift @@ -14,35 +14,61 @@ // limitations under the License. //===----------------------------------------------------------------------===// +import ContainerizationExtras +import Foundation import Testing @testable import ContainerNetworkServer +/// A store that keeps leases the way a file would, so what is written down +/// can be read back by an allocator that was not there when it was written. +private actor RememberingStore: AttachmentLeaseStore { + private var leases: [String: AttachmentAllocator.Lease] = [:] + + func load() async -> [AttachmentAllocator.Lease] { + Array(leases.values) + } + + func save(_ lease: AttachmentAllocator.Lease) async { + leases[lease.id] = lease + } +} + struct AttachmentAllocatorTest { + /// A hardware address to attach with, different for each host so that a + /// remembered one is recognizable. + private func mac(_ last: UInt8) -> MACAddress { + MACAddress(0xf200_0000_0000 | UInt64(last)) + } + + private func allocate(_ allocator: AttachmentAllocator, _ hostname: String, _ last: UInt8 = 1) async throws -> UInt32 { + try await allocator.allocate(hostname: hostname, macAddress: mac(last)).index + } + @Test func testAllocateSingleHostname() async throws { - let allocator = try AttachmentAllocator(lower: 100, size: 10) + let allocator = try await AttachmentAllocator(lower: 100, size: 10) - let address = try await allocator.allocate(hostname: "test-host") + let address = try await allocate(allocator, "test-host") #expect(address >= 100) #expect(address < 110) } @Test func testAllocateSameHostnameTwice() async throws { - let allocator = try AttachmentAllocator(lower: 100, size: 10) + let allocator = try await AttachmentAllocator(lower: 100, size: 10) - let address1 = try await allocator.allocate(hostname: "test-host") - let address2 = try await allocator.allocate(hostname: "test-host") + let address1 = try await allocate(allocator, "test-host") + let address2 = try await allocate(allocator, "test-host") #expect(address1 == address2) } @Test func testAllocateMultipleHostnames() async throws { - let allocator = try AttachmentAllocator(lower: 100, size: 10) + let allocator = try await AttachmentAllocator(lower: 100, size: 10) - let address1 = try await allocator.allocate(hostname: "host1") - let address2 = try await allocator.allocate(hostname: "host2") - let address3 = try await allocator.allocate(hostname: "host3") + let address1 = try await allocate(allocator, "host1") + let address2 = try await allocate(allocator, "host2") + let address3 = try await allocate(allocator, "host3") #expect(address1 != address2) #expect(address2 != address3) @@ -50,16 +76,16 @@ struct AttachmentAllocatorTest { } @Test func testLookupAllocatedHostname() async throws { - let allocator = try AttachmentAllocator(lower: 100, size: 10) + let allocator = try await AttachmentAllocator(lower: 100, size: 10) - let allocatedAddress = try await allocator.allocate(hostname: "test-host") + let allocatedAddress = try await allocate(allocator, "test-host") let lookedUpAddress = try await allocator.lookup(hostname: "test-host") #expect(lookedUpAddress == allocatedAddress) } @Test func testLookupNonExistentHostname() async throws { - let allocator = try AttachmentAllocator(lower: 100, size: 10) + let allocator = try await AttachmentAllocator(lower: 100, size: 10) let address = try await allocator.lookup(hostname: "non-existent") @@ -67,9 +93,9 @@ struct AttachmentAllocatorTest { } @Test func testDeallocateAllocatedHostname() async throws { - let allocator = try AttachmentAllocator(lower: 100, size: 10) + let allocator = try await AttachmentAllocator(lower: 100, size: 10) - let allocatedAddress = try await allocator.allocate(hostname: "test-host") + let allocatedAddress = try await allocate(allocator, "test-host") let deallocatedAddress = try await allocator.deallocate(hostname: "test-host") #expect(deallocatedAddress == allocatedAddress) @@ -80,56 +106,70 @@ struct AttachmentAllocatorTest { } @Test func testDeallocateNonExistentHostname() async throws { - let allocator = try AttachmentAllocator(lower: 100, size: 10) + let allocator = try await AttachmentAllocator(lower: 100, size: 10) let deallocatedAddress = try await allocator.deallocate(hostname: "non-existent") #expect(deallocatedAddress == nil) } - @Test func testReallocateAfterDeallocation() async throws { - let allocator = try AttachmentAllocator(lower: 100, size: 10) + @Test func testHostAttachingAgainIsGivenWhatItHad() async throws { + let allocator = try await AttachmentAllocator(lower: 100, size: 10) + + let first = try await allocator.allocate(hostname: "test-host", macAddress: mac(7)) + _ = try await allocator.deallocate(hostname: "test-host") + // Another host takes an address in between, so the answer is the one + // remembered rather than the one next in line. + _ = try await allocate(allocator, "other-host", 8) + let again = try await allocator.allocate(hostname: "test-host", macAddress: mac(9)) + + #expect(again.index == first.index) + #expect(again.macAddress == first.macAddress) + } + + @Test func testWhatAHostWasGivenOutlivesTheAllocator() async throws { + let store = RememberingStore() + + let first = try await AttachmentAllocator(lower: 100, size: 10, store: store) + .allocate(hostname: "test-host", macAddress: mac(3)) - let address1 = try await allocator.allocate(hostname: "test-host") - let released1 = try await allocator.deallocate(hostname: "test-host") - #expect(address1 == released1) - let address2 = try await allocator.allocate(hostname: "test-host") + let second = try await AttachmentAllocator(lower: 100, size: 10, store: store) + .allocate(hostname: "test-host", macAddress: mac(4)) - // After deallocation, allocating the same hostname should give a new address - #expect(address2 >= 100) - #expect(address2 < 110) + #expect(second.index == first.index) + #expect(second.macAddress == first.macAddress) } @Test func testAllocateUntilFull() async throws { let size = 5 - let allocator = try AttachmentAllocator(lower: 100, size: size) + let allocator = try await AttachmentAllocator(lower: 100, size: size) // Allocate up to the limit for i in 0..= 100) #expect(newAddress < 103) @@ -144,9 +184,9 @@ struct AttachmentAllocatorTest { } @Test func testMultipleDeallocationsOfSameHostname() async throws { - let allocator = try AttachmentAllocator(lower: 100, size: 10) + let allocator = try await AttachmentAllocator(lower: 100, size: 10) - let address = try await allocator.allocate(hostname: "test-host") + let address = try await allocate(allocator, "test-host") let firstDeallocate = try await allocator.deallocate(hostname: "test-host") #expect(firstDeallocate == address) diff --git a/Tests/ContainerPersistenceTests/FilesystemEntityStoreTests.swift b/Tests/ContainerPersistenceTests/FilesystemEntityStoreTests.swift index 03c136d1e..3c64ffa39 100644 --- a/Tests/ContainerPersistenceTests/FilesystemEntityStoreTests.swift +++ b/Tests/ContainerPersistenceTests/FilesystemEntityStoreTests.swift @@ -171,6 +171,19 @@ struct FilesystemEntityStoreTests { } } + @Test func testStoreMakesSomewhereToKeepWhatItHolds() async throws { + try await TemporaryStorage.withTempDir { path in + // Nothing has been written yet, so the directory the store keeps + // its entities under is not there to be read. + let fresh = path.appending("never-written") + let store = try Self.makeStore(at: fresh) + + #expect(try await store.list().isEmpty) + try await store.create(Item(id: "foo", value: "hello")) + #expect(try await store.retrieve("foo")?.value == "hello") + } + } + private static func makeStore(at path: FilePath) throws -> FilesystemEntityStore { try FilesystemEntityStore(path: path, type: "item", log: Logger(label: "test")) } diff --git a/Tests/ContainerResourceTests/ContainerConfigurationTests.swift b/Tests/ContainerResourceTests/ContainerConfigurationTests.swift index b1aafa0b0..eb28564f9 100644 --- a/Tests/ContainerResourceTests/ContainerConfigurationTests.swift +++ b/Tests/ContainerResourceTests/ContainerConfigurationTests.swift @@ -71,6 +71,29 @@ struct ContainerConfigurationResourcesTests { let decoded = try JSONDecoder().decode(ContainerConfiguration.self, from: stripped) #expect(decoded.resources.cpuOverhead == 1) } + + /// The size asked for has to reach the runtime, which reads the container's + /// configuration back rather than being handed the flags. + @Test func roundTripsSwap() throws { + var config = makeTestConfiguration() + config.resources.swapInBytes = 512.mib() + let data = try JSONEncoder().encode(config) + let decoded = try JSONDecoder().decode(ContainerConfiguration.self, from: data) + #expect(decoded.resources.swapInBytes == 512.mib()) + } + + /// A container that asked for no swap gets none, rather than a default size. + @Test func decodesMissingSwapAsNone() throws { + let config = makeTestConfiguration() + let data = try JSONEncoder().encode(config) + var obj = try #require(try JSONSerialization.jsonObject(with: data) as? [String: Any]) + var resources = try #require(obj["resources"] as? [String: Any]) + resources.removeValue(forKey: "swapInBytes") + obj["resources"] = resources + let stripped = try JSONSerialization.data(withJSONObject: obj) + let decoded = try JSONDecoder().decode(ContainerConfiguration.self, from: stripped) + #expect(decoded.resources.swapInBytes == nil) + } } struct ContainerConfigurationCreationDateTests { diff --git a/Tests/K8sPluginTests/K8sListTests.swift b/Tests/K8sPluginTests/K8sListTests.swift index f8d04b422..326e379a7 100644 --- a/Tests/K8sPluginTests/K8sListTests.swift +++ b/Tests/K8sPluginTests/K8sListTests.swift @@ -48,6 +48,7 @@ private func makeSnapshot( { "configuration": { "id": "\(id)", + "pod": "\(id)-pod", "image": { "reference": "docker.io/kindest/node:v1.35.5", "descriptor": {"mediaType":"","digest":"sha256:abc","size":0}